From 63a0d393c24ef2a2d5e3cd18156e1f70c1ce5ce3 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Fri, 1 Oct 2021 12:04:52 -0400 Subject: [PATCH 1/7] first commit (CLI function) --- Algorithmia/CLI.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 02d27f8..17c9cef 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -245,6 +245,12 @@ def cat(self, path, client): return result + def compileAlgo(self, client, manifest_path="model_manifest.json"): + if os.path.exists(manifest_path): + + else: + + # algo cp def cp(self, src, dest, client): From 3ad7619f863bb8d8a4730d6179fb36a2609cfa89 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 5 Oct 2021 11:31:25 -0400 Subject: [PATCH 2/7] wip --- Algorithmia/CLI.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 17c9cef..bd22ab9 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -5,7 +5,7 @@ import json, re, requests, six import toml import shutil - +import hashlib class CLI: def __init__(self): @@ -247,9 +247,11 @@ def cat(self, path, client): def compileAlgo(self, client, manifest_path="model_manifest.json"): if os.path.exists(manifest_path): - + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + for else: - + print("Expected to find a model_manifest.json file, none was discovered in working directory") # algo cp def cp(self, src, dest, client): From 04fb64e6ae0f5e701471365f20a4b9664ff0230d Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 5 Oct 2021 11:52:21 -0400 Subject: [PATCH 3/7] initial functional commit with Compilation --- Algorithmia/CLI.py | 21 +++++++++++++++++++-- Algorithmia/util.py | 13 +++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index bd22ab9..162fe55 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -2,10 +2,11 @@ import os from Algorithmia.errors import DataApiError from Algorithmia.algo_response import AlgoResponse +from Algorithmia.util import md5_for_file, md5_for_str import json, re, requests, six import toml import shutil -import hashlib +from time import time class CLI: def __init__(self): @@ -249,7 +250,23 @@ def compileAlgo(self, client, manifest_path="model_manifest.json"): if os.path.exists(manifest_path): with open(manifest_path, 'r') as f: manifest_file = json.load(f) - for + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = client.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = client.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open('model_manifest.json.lock', 'w') as f: + json.dump(manifest_file, f) else: print("Expected to find a model_manifest.json file, none was discovered in working directory") diff --git a/Algorithmia/util.py b/Algorithmia/util.py index 382b586..aa2ffbb 100644 --- a/Algorithmia/util.py +++ b/Algorithmia/util.py @@ -1,4 +1,5 @@ import re +import hashlib from Algorithmia.errors import DataApiError FNAME_MATCH = re.compile(r'/([^/]+)$') # From the last slash to the end of the string @@ -30,3 +31,15 @@ def pathJoin(parent, base): if parent.endswith('/'): return parent + base return parent + '/' + base + +def md5_for_file(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return str(hash_md5.hexdigest()) + +def md5_for_str(content): + hash_md5 = hashlib.md5() + hash_md5.update(content.encode()) + return str(hash_md5.hexdigest()) \ No newline at end of file From e0b69424bf2b448bedd845cb1f1372920e4c26cd Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 5 Oct 2021 12:05:48 -0400 Subject: [PATCH 4/7] added CLI endpoint to CLI tool --- Algorithmia/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index c45e5fb..eb30822 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -132,6 +132,10 @@ def main(): args = parser.parse_args() + #sub parser for compile + subparsers.add_parser('compile', help="compile's a model_manifest.json file into a model_manifest.json.lock") + + #run auth before trying to create a client if args.cmd == 'auth': @@ -215,6 +219,9 @@ def main(): elif args.cmd == 'builds': print(CLI().getBuildLogs(args.user, args.algo, client)) + elif args.cmd == "compile": + print(CLI().compileAlgo(client)) + else: parser.parse_args(['-h']) From 02bc4864521c303e4c93d594e554b6152b168f4c Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 5 Oct 2021 12:25:18 -0400 Subject: [PATCH 5/7] moved subparser before parser arg processing --- Algorithmia/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index eb30822..03d70a9 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -108,7 +108,7 @@ def main(): parser_cat.add_argument('--profile', action = 'store', type = str, default = 'default') #sub parser for getting environment template - parser_template = subparsers.add_parser('template',help='template downloads an environment template to the destination') + parser_template = subparsers.add_parser('template', help='template downloads an environment template to the destination') parser_template.add_argument('envid',help='environment specification id') parser_template.add_argument('dest',help='destination for template download') @@ -130,10 +130,10 @@ def main(): subparsers.add_parser('help') parser.add_argument('--profile', action = 'store', type = str, default = 'default') - args = parser.parse_args() - #sub parser for compile subparsers.add_parser('compile', help="compile's a model_manifest.json file into a model_manifest.json.lock") + + args = parser.parse_args() #run auth before trying to create a client From cf703962daddb9b5b3e964f0aa15fc45346ff5f3 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:36:28 -0400 Subject: [PATCH 6/7] adjusted client_test to not point to test cluster, and instead use Quality user --- Algorithmia/__main__.py | 2 +- Test/client_test.py | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index 03d70a9..bc04aa4 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -132,7 +132,7 @@ def main(): #sub parser for compile subparsers.add_parser('compile', help="compile's a model_manifest.json file into a model_manifest.json.lock") - + args = parser.parse_args() diff --git a/Test/client_test.py b/Test/client_test.py index edfda54..d484610 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -23,12 +23,12 @@ class ClientTest(unittest.TestCase): environment_name = "Python 3.9" def setUp(self): - self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) - self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) + self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY_1')) + self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY_1')) self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", + self.admin_client = Algorithmia.client(api_address="https://api.algorithmia.com", api_key=self.admin_api_key) self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', api_key=self.regular_api_key) @@ -70,9 +70,7 @@ def test_get_environment(self): self.assertTrue(response is not None and u'environments' in response) def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) + algo_path = self.regular_client.username() + "/hello" result = self.regular_client.algo(algo_path).build_logs() if u'error' in result: @@ -83,9 +81,7 @@ def test_get_build_logs(self): def test_get_build_logs_no_ssl(self): client = Algorithmia.client(api_address='https://api.algorithmia.com', api_key=self.regular_api_key, ca_cert=False) - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = u'Echo' - result = client.algo(user + '/' + algo).build_logs() + result = client.algo(client.username() + '/' + "hello").build_logs() if u'error' in result: print(result) self.assertTrue("error" not in result) From 1b69e2a5b414af4945ffbd7cd2ca60bf3ecaba1b Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Thu, 7 Oct 2021 10:01:32 -0400 Subject: [PATCH 7/7] renamed compile to lock, fixed grammer as per lemonez --- Algorithmia/CLI.py | 2 +- Algorithmia/__main__.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 162fe55..5703f49 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -246,7 +246,7 @@ def cat(self, path, client): return result - def compileAlgo(self, client, manifest_path="model_manifest.json"): + def lockAlgo(self, client, manifest_path="model_manifest.json"): if os.path.exists(manifest_path): with open(manifest_path, 'r') as f: manifest_file = json.load(f) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index bc04aa4..1befbcb 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -130,8 +130,8 @@ def main(): subparsers.add_parser('help') parser.add_argument('--profile', action = 'store', type = str, default = 'default') - #sub parser for compile - subparsers.add_parser('compile', help="compile's a model_manifest.json file into a model_manifest.json.lock") + #sub parser for lock + subparsers.add_parser('lock', help="locks a model_manifest.json file into a model_manifest.json.lock") args = parser.parse_args() @@ -219,8 +219,8 @@ def main(): elif args.cmd == 'builds': print(CLI().getBuildLogs(args.user, args.algo, client)) - elif args.cmd == "compile": - print(CLI().compileAlgo(client)) + elif args.cmd == "lock": + print(CLI().lockAlgo(client)) else: parser.parse_args(['-h'])