diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ccbf5f4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/Users/locona/.local/share/virtualenvs/livy-tyJWaNGI/bin/python" +} \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..25fff04 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.DEFAULT_GOAL := pyspark + +pyspark: + @python pyspark.py diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..6873ef8 --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +requests = "*" +flake8 = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..9006c0a --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,93 @@ +{ + "_meta": { + "hash": { + "sha256": "ccab0b3f8e17eeca326c200be2739f33df9b8f4076e3653830ef0e792186d69a" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3", + "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f" + ], + "version": "==2019.11.28" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, + "flake8": { + "hashes": [ + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" + ], + "index": "pypi", + "version": "==3.7.9" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" + ], + "version": "==2.5.0" + }, + "pyflakes": { + "hashes": [ + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" + ], + "version": "==2.1.1" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, + "urllib3": { + "hashes": [ + "sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", + "sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" + ], + "version": "==1.25.8" + } + }, + "develop": {} +} diff --git a/delete.py b/delete.py new file mode 100644 index 0000000..6fc4763 --- /dev/null +++ b/delete.py @@ -0,0 +1,19 @@ +import requests +from pprint import pprint + +host = 'http://localhost:8998' +headers = {'Content-Type': 'application/json'} + + +def delete_all(): + r = requests.get(host + '/sessions', {}, headers=headers) + sessions = r.json()['sessions'] + for sess in sessions: + session_id = sess["id"] + session_url = f"http://localhost:8998/sessions/${session_id}" + print(session_url) + r = requests.delete(session_url, headers=headers) + pprint(r.json()) + + +delete_all() diff --git a/livy.py b/livy.py new file mode 100644 index 0000000..99db0f7 --- /dev/null +++ b/livy.py @@ -0,0 +1,39 @@ +from pprint import pprint +import json, pprint, requests, textwrap + +host = 'htp://localhost:8998' +data = {'kind': 'spark'} +headers = {'Content-Type': 'application/json'} + +r = requests.post(host + '/sessions', data=json.dumps(data), headers=headers) + +session_url = host + r.headers['location'] +r = requests.get(session_url, headers=headers) +pprint(r.json()) +pprint(r.headers) + +pprint(r.json()) + +statements_url = session_url + '/statements' +data = {'code': '1 + 1'} +r = requests.post(statements_url, data=json.dumps(data), headers=headers) +pprint(r.json()) + +data = { + 'code': textwrap.dedent(""" + val NUM_SAMPLES = 100000; + val count = sc.parallelize(1 to NUM_SAMPLES).map { i => + val x = Math.random(); + val y = Math.random(); + if (x*x + y*y < 1) 1 else 0 + }.reduce(_ + _); + println(\"Pi is roughly \" + 4.0 * count / NUM_SAMPLES) + """) +} + +r = requests.post(statements_url, data=json.dumps(data), headers=headers) +pprint(r.json()) + +statement_url = host + r.headers['location'] +r = requests.get(statement_url, headers=headers) +pprint.pprint(r.json()) \ No newline at end of file diff --git a/pyspark.py b/pyspark.py new file mode 100644 index 0000000..66e02d8 --- /dev/null +++ b/pyspark.py @@ -0,0 +1,56 @@ +import json +import requests +import textwrap +import sys + +from pprint import pprint + +host = 'http://localhost:8998' +headers = {'Content-Type': 'application/json'} + + +def session(): + data = {'kind': 'pyspark'} + r = requests.post(host + '/sessions', data=json.dumps(data), headers=headers) + pprint(r.json()) + + session_url = host + r.headers['location'] + r = requests.get(session_url, headers=headers) + pprint(r.json()) + print(f"session_url: ${session_url}") + + +def run(session_url): + statements_url = session_url + '/statements' + data = {'code': '1 + 1'} + r = requests.post(statements_url, data=json.dumps(data), headers=headers) + print(f"statements_url: ${statements_url}") + r.json() + + data = { + "code": textwrap.dedent(""" + import random + NUM_SAMPLES = 100000 + def sample(p): + x, y = random.random(), random.random() + return 1 if x*x + y*y < 1 else 0 + + count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample).reduce(lambda a, b: a + b) + print "Pi is roughly %f" % (4.0 * count / NUM_SAMPLES) + """) + } + + print(f"statements_url: ${statements_url}") + r = requests.post(statements_url, data = json.dumps(data), headers = headers) + + print() + pprint(r.json()) + + +if __name__ == '__main__': + args = sys.argv + print(args) + if len(args) == 1: + session() + else: + run(args[1]) diff --git a/spark.py b/spark.py new file mode 100644 index 0000000..fbe8388 --- /dev/null +++ b/spark.py @@ -0,0 +1,57 @@ +import json +import requests +import textwrap +import sys + +from pprint import pprint + +host = 'http://localhost:8998' +headers = {'Content-Type': 'application/json'} + + +def session(): + data = {'kind': 'spark'} + r = requests.post(host + '/sessions', data=json.dumps(data), headers=headers) + pprint(r.json()) + + session_url = host + r.headers['location'] + r = requests.get(session_url, headers=headers) + pprint(r.json()) + print(f"session_url: ${session_url}") + + +def run(session_url): + statements_url = session_url + '/statements' + data = {'code': '1 + 1'} + r = requests.post(statements_url, data=json.dumps(data), headers=headers) + print(f"statements_url: ${statements_url}") + r.json() + + data = { + 'code': textwrap.dedent(""" + val NUM_SAMPLES = 100000; + val count = sc.parallelize(1 to NUM_SAMPLES).map { i => + val x = Math.random(); + val y = Math.random(); + if (x*x + y*y < 1) 1 else 0 + }.reduce(_ + _); + println(\"Pi is roughly \" + 4.0 * count / NUM_SAMPLES) + """) + } + + print(f"statements_url: ${statements_url}") + r = requests.post(statements_url, data=json.dumps(data), headers=headers) + + statement_url = host + r.headers['location'] + print(statement_url) + r = requests.get(statement_url, headers=headers) + pprint(r.json()) + + +if __name__ == '__main__': + args = sys.argv + print(args) + if len(args) == 1: + session() + else: + run(args[1]) diff --git a/statement.py b/statement.py new file mode 100644 index 0000000..d453eaf --- /dev/null +++ b/statement.py @@ -0,0 +1,13 @@ +import requests +from pprint import pprint + +host = 'http://localhost:8998' +headers = {'Content-Type': 'application/json'} + + +def delete_all(): + statement_url = "http://localhost:8998/sessions/0/statements" + r = requests.get(statement_url, headers=headers) + pprint(r.json()) + +delete_all()