Skip to content
This repository has been archived by the owner on Aug 1, 2023. It is now read-only.

Commit

Permalink
Merge pull request #126 from datawire/develop
Browse files Browse the repository at this point in the history
Python 3, fix NPM package.
  • Loading branch information
itamarst authored Sep 12, 2016
2 parents a68c78a + dde2a77 commit 36dd248
Show file tree
Hide file tree
Showing 40 changed files with 176 additions and 105 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.0.11
current_version = 2.0.12
commit = False
tag = False
tag_name = "v{new_version}"
Expand Down
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ build
*.pyc

node_modules
tests/source/*_java/target
functionaltests/source/*_java/target

quark/output
output/
QUARK_VERSION-dirty.txt

venv
virtualenv
virtualenv3
build-tools

# KDE
Expand All @@ -31,4 +33,4 @@ build-tools

.cache
npm-debug.log
dist/
dist/
12 changes: 0 additions & 12 deletions .travis-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,5 @@ gem --version
# Prepare virtualenv:
make setup

# Enter the virtualenv:
source virtualenv/bin/activate

echo "Git commit is:" $TRAVIS_COMMIT
for CODE_LANG in --python --ruby --java --javascript ; do
echo "Language is:" $CODE_LANG
bash install.sh $CODE_LANG $TRAVIS_COMMIT
done

# Make sure we can build packages, if not there's no point in even continuing:
make packages

# Run the tests:
make test
55 changes: 46 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,29 +9,66 @@ default:
echo "* 'make release-patch' to do a patch release (2.0.x)"
echo "* 'make release-minor' to do a minor release (2.x.0)"
echo "* 'make upload-packages' to upload packages to native repos (e.g. .whl to PyPI, .gem to RubyGems.org, etc.)"
echo "* 'make clean' to undo setup and packages"

.PHONY: clean
clean:
rm -fr virtualenv
rm -fr virtualenv3
rm -fr output
rm -fr dist
rm -f quark/*.qc
rm -fr ~/.m2/repository/datawire_mdk
rm -fr ~/.m2/repository/io/datawire/mdk

virtualenv:
virtualenv virtualenv
virtualenv -p python2 virtualenv

.PHONY: python-dependencies
python-dependencies: virtualenv
virtualenv/bin/pip install -r dev-requirements.txt

virtualenv3:
virtualenv -p python3 virtualenv3

.PHONY: python3-dependencies
python3-dependencies: virtualenv3
virtualenv3/bin/pip install -r dev-requirements.txt

.PHONY: setup
setup: python-dependencies
setup: python-dependencies python3-dependencies install-quark

.PHONY: install-quark
install-quark:
which quark || \
curl -# -L https://raw.githubusercontent.com/datawire/quark/master/install.sh | \
bash -s -- -q `cat QUARK_VERSION.txt`

.PHONY: install-mdk
install-mdk: packages
virtualenv/bin/pip install --upgrade dist/datawire_mdk-*-py2*-none-any.whl
virtualenv3/bin/pip install --upgrade dist/datawire_mdk-*-*py3-none-any.whl
gem install --no-doc dist/datawire_mdk-*.gem
npm install output/js/mdk-2.0
cd output/java/mdk-2.0 && mvn install

.PHONY: test
test:
# For now we rely on either .travis-test.sh or the user to install the
# MDK. This means tests will fail if you are not on Travis
# or have not installed the MDK.
source virtualenv/bin/activate && py.test -n 4 -v tests
test: install-mdk test-python test-python3

.PHONY: test-python
test-python:
virtualenv/bin/py.test -n 4 -v unittests functionaltests

.PHONY: test-python3
test-python3:
# Functional tests don't benefit from being run in another language:
virtualenv3/bin/py.test -n 4 -v unittests

release-minor:
source virtualenv/bin/activate; python scripts/release.py minor
virtualenv/bin/python scripts/release.py minor

release-patch:
source virtualenv/bin/activate; python scripts/release.py patch
virtualenv/bin/python scripts/release.py patch

# Packaging commands:
output: $(wildcard quark/*.q) dist
Expand Down
2 changes: 1 addition & 1 deletion QUARK_VERSION.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v1.0.406
v1.0.443
1 change: 1 addition & 0 deletions dev-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ travispy==0.3.5
twine==1.8.1
pytest==3.0.1
pytest-xdist==1.15.0
future==0.15.2
40 changes: 40 additions & 0 deletions functionaltests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from __future__ import print_function

import os
from subprocess import check_call
import pytest

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
official = open(os.path.join(ROOT_DIR, "QUARK_VERSION.txt")).read().strip()
try:
dirty = open(os.path.join(ROOT_DIR, "QUARK_VERSION-dirty.txt")).read().strip()
QUARK_VERSION = dirty
QUARK_VERSION_STATUS = "Overriding the quark version %s with the local quark version %s" % (official, dirty)
except IOError:
QUARK_VERSION = official
QUARK_VERSION_STATUS = "Testing with quark version %s, QUARK_VERSION-dirty.txt not found" % (official)
# Git tag starts with v, but the Docker tag does not:
if QUARK_VERSION.startswith("v"):
QUARK_VERSION = QUARK_VERSION[1:]


def pytest_report_header(config):
return QUARK_VERSION_STATUS

def run(filepath, language):
"""Install and run a Quark test file."""
docker_path = os.path.join("/code",
filepath[len(ROOT_DIR) + 1:])
print("Installing and running {} in {}...".format(filepath, language))
check_call(["sudo", "docker", "run",
# Mount volume into container so Docker can access quark files:
"-v", ROOT_DIR + ":/code",] +
["datawire/quark-run:" + QUARK_VERSION, "--" + language, '--verbose', docker_path])

@pytest.fixture
def quark_run(request):
return run

@pytest.fixture(params= ["python3", "python", "java", "ruby", "javascript"])
def quark_language(request):
return request.param
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Write some logs."""
from __future__ import print_function

import sys
import time
Expand All @@ -11,7 +12,7 @@
def got_logs(log_events, context):
results = set()
for event in log_events.result:
print event.context, event.category, event.text
print(event.context, event.category, event.text)
if event.context.traceId in context:
results.add((event.category, event.text))
expected = set([("process1", "hello"), ("process2", "world")])
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Register a service given on command-line args."""
from __future__ import print_function

import time
import sys
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
<version>3.4.0-RC1</version>
</dependency>
<dependency>
<groupId>datawire_mdk</groupId>
<groupId>io.datawire.mdk</groupId>
<artifactId>datawire_mdk</artifactId>
<version>[2.0.0,2.1.0)</version>
</dependency>
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
"""Write some logs."""
from __future__ import print_function

import sys
import time
import os
import logging
logging.basicConfig(file=sys.stderr, level=logging.DEBUG)
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)

from mdk import start
mdk = start()
Expand All @@ -24,7 +25,7 @@ def got_message(event):
print("Took more than 60 seconds, giving up.")
os._exit(1)

print(event.category, event.text)
print((event.category, event.text))
if event.category == category:
results.add((event.text, event.level))
if expected == results:
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
<version>3.4.0-RC1</version>
</dependency>
<dependency>
<groupId>datawire_mdk</groupId>
<groupId>io.datawire.mdk</groupId>
<artifactId>datawire_mdk</artifactId>
<version>[2.0.0,)</version>
</dependency>
Expand Down
18 changes: 11 additions & 7 deletions tests/test_endtoend.py → functionaltests/test_endtoend.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""End-to-end tests for the MDK API."""
from builtins import str

import os
import sys
import time
from random import random
from subprocess import Popen, check_output, check_call
Expand All @@ -13,16 +15,18 @@
def random_string():
return "random_" + str(random())[2:]

def decoded_check_output(*args, **kwargs):
return check_output(*args, **kwargs).decode('utf-8')

def run_python(command, extra_args=(), output=False):
"""
Run a Python program.
Returns output if output=True, in which case stderr will cause error.
"""
args = ["python", os.path.join(CODE_PATH, command)] + list(extra_args)
args = [sys.executable, os.path.join(CODE_PATH, command)] + list(extra_args)
if output:
command = check_output
command = decoded_check_output
else:
command = check_call
return command(args)
Expand All @@ -40,8 +44,8 @@ def assertRegisteryDiscoverable(test, discover):
"""
service = random_string()
address = random_string()
p = Popen(["python", os.path.join(CODE_PATH, "register.py"), service, address])
test.addCleanup(lambda: p.terminate())
p = Popen([sys.executable, os.path.join(CODE_PATH, "register.py"), service, address])
test.addCleanup(lambda: p.kill())
resolved_address = discover(service)
test.assertIn(address, resolved_address)
return p, service
Expand Down Expand Up @@ -94,7 +98,7 @@ def test_discovery(self):
"""Minimal discovery end-to-end test with a Javascript client."""
assertRegisteryDiscoverable(
self,
lambda service: check_output(
lambda service: decoded_check_output(
["node", os.path.join(CODE_PATH, "resolve.js"), service]))


Expand All @@ -110,7 +114,7 @@ def test_discovery(self):
"""Minimal discovery end-to-end test with a Javascript client."""
assertRegisteryDiscoverable(
self,
lambda service: check_output(
lambda service: decoded_check_output(
["ruby", os.path.join(CODE_PATH, "resolve.rb"), service]))


Expand All @@ -132,7 +136,7 @@ def test_discovery(self):
"package"])
assertRegisteryDiscoverable(
self,
lambda service: check_output(
lambda service: decoded_check_output(
["java", "-jar", os.path.join(
CODE_PATH,"resolve_java/target/resolve-0.0.1.jar"),
service]))
32 changes: 32 additions & 0 deletions functionaltests/test_quark.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""
Run tests written in Quark.
"""
from __future__ import print_function

import os
from glob import glob
import pytest

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
QUARK_TESTS_DIR = os.path.join(ROOT_DIR, "quark/tests")

# Tests that don't need to run on all languages:
@pytest.fixture(params=[filename for filename in glob(QUARK_TESTS_DIR + "/*.q")
if not filename.endswith("runtime_test.q")])
def filepath(request):
return request.param


def test_run_python_only(quark_run, filepath):
"""Run Quark tests that don't need to run in multiple languages."""
quark_run(filepath, "python")

def test_run_python3_only(quark_run, filepath):
"""Run Quark tests that don't need to run in multiple languages."""
quark_run(filepath, "python3")


def test_run_all_languages(quark_run, quark_language):
"""Run tests that have to be run in all languages."""
quark_run(os.path.join(QUARK_TESTS_DIR, "runtime_test.q"), quark_language)

2 changes: 1 addition & 1 deletion quark/discovery-3.0.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk_discovery 2.0.11;
package datawire_mdk_discovery 2.0.12;

include discovery-protocol-3.0.q;
include synapse.q;
Expand Down
2 changes: 1 addition & 1 deletion quark/introspection-1.0.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk_introspection 2.0.11;
package datawire_mdk_introspection 2.0.12;

/*
* Copyright 2016 Datawire. All rights reserved.
Expand Down
2 changes: 1 addition & 1 deletion quark/mdk-2.0.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk 2.0.11;
package datawire_mdk 2.0.12;

// DATAWIRE MDK

Expand Down
2 changes: 1 addition & 1 deletion quark/mdk_runtime.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk_runtime 2.0.11;
package datawire_mdk_runtime 2.0.12;

include actors_core.q;
include actors_promise.q;
Expand Down
4 changes: 2 additions & 2 deletions quark/mdk_runtime_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ def _mdk_mktempdir():
def _mdk_writefile(path, contents):
"""Write a file to disk."""
with open(path, "wb") as f:
f.write(contents)
f.write(contents.encode("utf-8"))

def _mdk_readfile(path):
"""Read a file's contents."""
with open(path, "rb") as f:
return f.read()
return f.read().decode("utf-8")

def _mdk_deletefile(path):
"""Delete a file."""
Expand Down
2 changes: 1 addition & 1 deletion quark/protocol-1.0.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk_protocol 2.0.11;
package datawire_mdk_protocol 2.0.12;

import quark.concurrent;
import quark.reflect;
Expand Down
2 changes: 1 addition & 1 deletion quark/tracing-2.0.q
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
quark 1.0;

package datawire_mdk_tracing 2.0.11;
package datawire_mdk_tracing 2.0.12;

include protocol-1.0.q;
include introspection-1.0.q;
Expand Down
Loading

0 comments on commit 36dd248

Please sign in to comment.