From 5adaedc7c76d5f37f7d90112a3173959fa1466d2 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Thu, 13 Feb 2020 20:16:19 -0500
Subject: [PATCH 01/15] .
.
.
.
.
.
.
.
.
.
.
.
.
seems to be working
.
.
.
.
.
updates
more tests
adding tests
starting work on ins and outs, async generator support
adding file input
updates, tests
update azurepipelines
Add input folder
add mac and windows platforms
remove py36
remove py36
updates
updates
---
.appveyor.yml | 22 --
.gitignore | 6 +-
.travis.yml | 39 ---
Makefile | 2 +-
README.md | 2 -
azure-pipelines.yml | 153 ++++++++---
setup.py | 1 -
tributary/__init__.py | 1 -
tributary/base.py | 39 ++-
tributary/functional/__init__.py | 2 +-
tributary/lazy/__init__.py | 4 +-
tributary/lazy/base.py | 38 +--
tributary/lazy/calculations/ops.py | 135 +++++-----
.../reactive => lazy/input}/__init__.py | 0
tributary/lazy/node.py | 173 +++---------
tributary/lazy/output/__init__.py | 85 ++++++
tributary/reactive/__init__.py | 14 +-
tributary/reactive/base.py | 102 +++----
tributary/reactive/calculations/ops.py | 56 +---
tributary/reactive/input/socketio.py | 1 -
tributary/reactive/input/ws.py | 1 -
tributary/reactive/output/__init__.py | 17 +-
tributary/reactive/output/ws.py | 1 -
tributary/reactive/utils.py | 4 +-
tributary/streaming/__init__.py | 29 ++
tributary/streaming/base.py | 125 +++++++++
tributary/streaming/calculations/__init__.py | 2 +
tributary/streaming/calculations/ops.py | 160 +++++++++++
.../calculations/rolling.py} | 0
tributary/streaming/input/__init__.py | 6 +
tributary/streaming/input/file.py | 22 ++
tributary/streaming/input/http.py | 50 ++++
tributary/streaming/input/input.py | 77 ++++++
tributary/streaming/input/kafka.py | 57 ++++
tributary/streaming/input/socketio.py | 46 ++++
tributary/streaming/input/ws.py | 35 +++
tributary/streaming/output/__init__.py | 5 +
tributary/streaming/output/file.py | 25 ++
tributary/streaming/output/http.py | 48 ++++
tributary/streaming/output/kafka.py | 41 +++
tributary/streaming/output/output.py | 79 ++++++
.../output/socketio.py} | 0
tributary/streaming/output/ws.py | 50 ++++
.../utils.py} | 0
tributary/symbolic/__init__.py | 10 +-
tributary/tests/functional/test_functional.py | 8 +-
tributary/tests/helpers/dummy_ws.py | 1 +
.../tests/lazy/calculations/test_ops_lazy.py | 12 +-
tributary/tests/lazy/test_lazy.py | 18 +-
.../calculations/test_ops_reactive.py | 189 -------------
.../calculations/test_rolling_reactive.py | 27 --
.../input/test_file_reactive_input.py | 12 -
.../input/test_http_reactive_input.py | 7 -
.../input/test_input_reactive_input.py | 7 -
.../output/test_file_reactive_output.py | 9 -
.../output/test_http_reactive_output.py | 9 -
.../output/test_output_reactive_output.py | 17 --
tributary/tests/reactive/test_reactive.py | 71 -----
.../tests/reactive/test_utils_reactive.py | 255 ------------------
.../__init__.py} | 0
.../calculations/__init__.py} | 0
.../test_calculations_streaming.py} | 0
.../calculations/test_ops_streaming.py | 214 +++++++++++++++
.../input/__init__.py} | 0
.../tests/streaming/input/test_file_data.csv | 4 +
.../streaming/input/test_file_streaming.py | 11 +
.../streaming/input/test_http_streaming.py | 11 +
.../streaming/input/test_input_streaming.py | 118 ++++++++
.../output/__init__.py} | 0
.../streaming/output/test_file_streaming.py | 17 ++
.../streaming/output/test_output_streaming.py | 23 ++
tributary/tests/streaming/test_streaming.py | 36 +++
tributary/tests/symbolic/test_symbolic.py | 24 +-
tributary/tests/test_base.py | 2 +-
74 files changed, 1764 insertions(+), 1103 deletions(-)
delete mode 100644 .appveyor.yml
delete mode 100644 .travis.yml
rename tributary/{tests/reactive => lazy/input}/__init__.py (100%)
create mode 100644 tributary/lazy/output/__init__.py
create mode 100644 tributary/streaming/__init__.py
create mode 100644 tributary/streaming/base.py
create mode 100644 tributary/streaming/calculations/__init__.py
create mode 100644 tributary/streaming/calculations/ops.py
rename tributary/{tests/reactive/calculations/test_calculations_reactive.py => streaming/calculations/rolling.py} (100%)
create mode 100644 tributary/streaming/input/__init__.py
create mode 100644 tributary/streaming/input/file.py
create mode 100644 tributary/streaming/input/http.py
create mode 100644 tributary/streaming/input/input.py
create mode 100644 tributary/streaming/input/kafka.py
create mode 100644 tributary/streaming/input/socketio.py
create mode 100644 tributary/streaming/input/ws.py
create mode 100644 tributary/streaming/output/__init__.py
create mode 100644 tributary/streaming/output/file.py
create mode 100644 tributary/streaming/output/http.py
create mode 100644 tributary/streaming/output/kafka.py
create mode 100644 tributary/streaming/output/output.py
rename tributary/{tests/reactive/input/test_kafka_reactive_input.py => streaming/output/socketio.py} (100%)
create mode 100644 tributary/streaming/output/ws.py
rename tributary/{tests/reactive/input/test_socketio_reactive_input.py => streaming/utils.py} (100%)
delete mode 100644 tributary/tests/reactive/calculations/test_ops_reactive.py
delete mode 100644 tributary/tests/reactive/calculations/test_rolling_reactive.py
delete mode 100644 tributary/tests/reactive/input/test_file_reactive_input.py
delete mode 100644 tributary/tests/reactive/input/test_http_reactive_input.py
delete mode 100644 tributary/tests/reactive/input/test_input_reactive_input.py
delete mode 100644 tributary/tests/reactive/output/test_file_reactive_output.py
delete mode 100644 tributary/tests/reactive/output/test_http_reactive_output.py
delete mode 100644 tributary/tests/reactive/output/test_output_reactive_output.py
delete mode 100644 tributary/tests/reactive/test_reactive.py
delete mode 100644 tributary/tests/reactive/test_utils_reactive.py
rename tributary/tests/{reactive/input/test_ws_reactive_input.py => streaming/__init__.py} (100%)
rename tributary/tests/{reactive/output/test_kafka_reactive_output.py => streaming/calculations/__init__.py} (100%)
rename tributary/tests/{reactive/output/test_socketio_reactive_output.py => streaming/calculations/test_calculations_streaming.py} (100%)
create mode 100644 tributary/tests/streaming/calculations/test_ops_streaming.py
rename tributary/tests/{reactive/output/test_ws_reactive_output.py => streaming/input/__init__.py} (100%)
create mode 100644 tributary/tests/streaming/input/test_file_data.csv
create mode 100644 tributary/tests/streaming/input/test_file_streaming.py
create mode 100644 tributary/tests/streaming/input/test_http_streaming.py
create mode 100644 tributary/tests/streaming/input/test_input_streaming.py
rename tributary/tests/{reactive/test_base_reactive.py => streaming/output/__init__.py} (100%)
create mode 100644 tributary/tests/streaming/output/test_file_streaming.py
create mode 100644 tributary/tests/streaming/output/test_output_streaming.py
create mode 100644 tributary/tests/streaming/test_streaming.py
diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index 623991b..0000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-branches:
- only:
- - master
-
-skip_tags: true
-max_jobs: 1
-
-image:
- - Visual Studio 2017
-
-install:
- - C:\Python37-x64\python -m pip install -r requirements.txt
- - C:\Python37-x64\python -m pip install -e .[dev]
-
-cache:
- - '%LOCALAPPDATA%\pip\Cache'
-
-build_script:
- - C:\Python37-x64\python setup.py build
-
-test_script:
- - C:\Python37-x64\python -m pytest -v tests --cov=tributary
diff --git a/.gitignore b/.gitignore
index 34e5321..259a527 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,4 +169,8 @@ package-lock.json
tmp.py
tmp2.py
-docs/api/
\ No newline at end of file
+docs/api/
+tributary/tests/streaming/output/test_file_data.csv
+python_junit.xml
+tmps
+
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 59dcb60..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-dist: xenial
-language: python
-cache: pip
-
-matrix:
- include:
- - python: "3.7"
- env: PYTHONVER=3
-
-addons:
- apt:
- update: true
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - graphviz
- homebrew:
- update: true
- packages:
- - python
- - graphviz
- - python2
-
-install:
- - python3 -m pip install -e .[dev]
-
-script:
- - make lint
- - make tests
-
-after_success:
- - python3 -m codecov --token 2b1a0cc9-52e4-4942-8816-8554a1f5d0f1
-
-branches:
- only:
- - master
-
-notifications:
- email: false
diff --git a/Makefile b/Makefile
index bb0c9d8..ef7d191 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ buildpy2:
python2 setup.py build
tests: ## Clean and Make unit tests
- python3 -m pytest -v tributary/tests --cov=tributary
+ python3 -m pytest -v tributary/tests --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
notebooks: ## test execute the notebooks
./scripts/test_notebooks.sh
diff --git a/README.md b/README.md
index 25ff763..67e28a2 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,6 @@ Python Data Streams
-
# Installation
Install from pip:
@@ -22,7 +21,6 @@ or from source
`python setup.py install`
-
# Stream Types
Tributary offers several kinds of streams:
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index df46425..920f23f 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -1,33 +1,126 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
trigger:
- master
-pool:
- vmImage: 'ubuntu-latest'
-
-strategy:
- matrix:
- Python36:
- python.version: '3.6'
- Python37:
- python.version: '3.7'
-
-steps:
-- task: UsePythonVersion@0
- inputs:
- versionSpec: '$(python.version)'
- displayName: 'Use Python $(python.version)'
-
-- script: |
- python -m pip install --upgrade pip
- pip install -e .[dev]
- displayName: 'Install dependencies'
-
-- script: |
- make lint
- make tests
- displayName: 'pytest'
+jobs:
+- job: 'Linux'
+ pool:
+ vmImage: 'ubuntu-latest'
+
+ strategy:
+ matrix:
+ Python37:
+ python.version: '3.7'
+ Python38:
+ python.version: '3.8'
+
+ steps:
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
+
+ - script: |
+ make lint
+ displayName: 'Lint'
+
+ - script:
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
+
+- job: 'Mac'
+ pool:
+ vmImage: 'macos-10.14'
+
+ strategy:
+ matrix:
+ Python37:
+ python.version: '3.7'
+ Python38:
+ python.version: '3.8'
+
+ steps:
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
+
+ - script: |
+ make lint
+ displayName: 'Lint'
+
+ - script: |
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
+
+- job: 'Windows'
+ pool:
+ vmImage: 'vs2017-win2016'
+
+ strategy:
+ matrix:
+ Python37:
+ python.version: '3.7'
+ Python38:
+ python.version: '3.8'
+
+ steps:
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
+
+ - script: |
+ make lint
+ displayName: 'Lint'
+
+ - script: |
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
diff --git a/setup.py b/setup.py
index dc16822..c4daf68 100644
--- a/setup.py
+++ b/setup.py
@@ -69,7 +69,6 @@ def get_version(file, name='__version__'):
classifiers=[
'Development Status :: 3 - Alpha',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
],
diff --git a/tributary/__init__.py b/tributary/__init__.py
index 92b08f6..274608d 100644
--- a/tributary/__init__.py
+++ b/tributary/__init__.py
@@ -8,4 +8,3 @@
from ._version import __version__ # noqa: F401
from .functional import pipeline, stop # noqa: F401
-from .reactive import * # noqa: F401, F403
diff --git a/tributary/base.py b/tributary/base.py
index 88aa77e..152709d 100644
--- a/tributary/base.py
+++ b/tributary/base.py
@@ -6,5 +6,42 @@ class StreamEnd:
class StreamNone:
'''indicates that a stream does not have a value'''
- def __init__(self, last):
+
+ def __init__(self, last=None):
self.value = last
+
+ def all_bin_ops(self, other):
+ return self
+
+ def all_un_ops(self):
+ return self
+
+ __add__ = all_bin_ops
+ __radd__ = all_bin_ops
+ __sub__ = all_bin_ops
+ __rsub__ = all_bin_ops
+ __mul__ = all_bin_ops
+ __rmul__ = all_bin_ops
+ __div__ = all_bin_ops
+ __rdiv__ = all_bin_ops
+ __truediv__ = all_bin_ops
+ __rtruediv__ = all_bin_ops
+ __pow__ = all_bin_ops
+ __rpow__ = all_bin_ops
+ __mod__ = all_bin_ops
+ __rmod__ = all_bin_ops
+ __and__ = all_bin_ops
+ __or__ = all_bin_ops
+ __invert__ = all_un_ops
+ def __bool__(self): return False
+ def int(self): return 0
+ def float(self): return 0
+ __lt__ = all_bin_ops
+ __le__ = all_bin_ops
+ __gt__ = all_bin_ops
+ __ge__ = all_bin_ops
+ __eq__ = all_bin_ops
+ __ne__ = all_bin_ops
+ __neg__ = all_un_ops
+ __nonzero__ = all_un_ops
+ __len__ = all_un_ops
diff --git a/tributary/functional/__init__.py b/tributary/functional/__init__.py
index 09a6a31..740daa8 100644
--- a/tributary/functional/__init__.py
+++ b/tributary/functional/__init__.py
@@ -63,7 +63,7 @@ def pipeline(foos, foo_callbacks, foo_kwargs=None, on_data=print, on_data_kwargs
Args:
foos (list of callables): list of functions to pipeline
foo_callbacks (List[str]): list of strings indicating the callback names (kwargs of the foos)
- foo_kwargs (List[dict]):
+ foo_kwargs (List[dict]):
on_data (callable): callable to call at the end of the pipeline
on_data_kwargs (dict): kwargs to pass to the on_data function>?
'''
diff --git a/tributary/lazy/__init__.py b/tributary/lazy/__init__.py
index 33522c1..479c881 100644
--- a/tributary/lazy/__init__.py
+++ b/tributary/lazy/__init__.py
@@ -1,2 +1,4 @@
-from .base import BaseGraph, BaseNode, node # noqa: F401
+from .base import LazyGraph, Node, node # noqa: F401
+from .input import * # noqa: F401, F403
+from .output import * # noqa: F401, F403
from .calculations import * # noqa: F401, F403
diff --git a/tributary/lazy/base.py b/tributary/lazy/base.py
index da238a2..dcbe6a7 100644
--- a/tributary/lazy/base.py
+++ b/tributary/lazy/base.py
@@ -1,7 +1,7 @@
-from .node import BaseNode, node # noqa: F401
+from .node import Node, node # noqa: F401
-class BaseGraph(object):
+class LazyGraph(object):
'''Wrapper class around a collection of lazy nodes.'''
def __init__(self, *args, **kwargs):
pass
@@ -18,39 +18,39 @@ def node(self, name, readonly=False, nullable=True, value=None, trace=False): #
Returns:
BaseNode: the newly constructed lazy node
'''
- if not hasattr(self, '_BaseGraph__nodes'):
+ if not hasattr(self, '_LazyGraph__nodes'):
self.__nodes = {}
if name not in self.__nodes:
- self.__nodes[name] = BaseNode(name=name,
- derived=False,
- readonly=readonly,
- nullable=nullable,
- value=value,
- trace=trace)
+ self.__nodes[name] = Node(name=name,
+ derived=False,
+ readonly=readonly,
+ nullable=nullable,
+ value=value,
+ trace=trace)
setattr(self, name, self.__nodes[name])
return self.__nodes[name]
def __getattribute__(self, name):
- if name == '_BaseGraph__nodes' or name == '__nodes':
- return super(BaseGraph, self).__getattribute__(name)
- elif hasattr(self, '_BaseGraph__nodes') and name in super(BaseGraph, self).__getattribute__('_BaseGraph__nodes'):
- return super(BaseGraph, self).__getattribute__('_BaseGraph__nodes')[name]
+ if name == '_LazyGraph__nodes' or name == '__nodes':
+ return super(LazyGraph, self).__getattribute__(name)
+ elif hasattr(self, '_LazyGraph__nodes') and name in super(LazyGraph, self).__getattribute__('_LazyGraph__nodes'):
+ return super(LazyGraph, self).__getattribute__('_LazyGraph__nodes')[name]
else:
- return super(BaseGraph, self).__getattribute__(name)
+ return super(LazyGraph, self).__getattribute__(name)
def __setattr__(self, name, value):
- if hasattr(self, '_BaseGraph__nodes') and name in super(BaseGraph, self).__getattribute__('_BaseGraph__nodes'):
- node = super(BaseGraph, self).__getattribute__('_BaseGraph__nodes')[name]
- if isinstance(value, BaseNode) and node == value:
+ if hasattr(self, '_LazyGraph__nodes') and name in super(LazyGraph, self).__getattribute__('_LazyGraph__nodes'):
+ node = super(LazyGraph, self).__getattribute__('_LazyGraph__nodes')[name]
+ if isinstance(value, Node) and node == value:
return
- elif isinstance(value, BaseNode):
+ elif isinstance(value, Node):
raise Exception('Cannot set to node')
else:
node._dirty = (node._value != value) or (node._value is not None and abs(node._value - value) > 10**-5)
node._value = value
else:
- super(BaseGraph, self).__setattr__(name, value)
+ super(LazyGraph, self).__setattr__(name, value)
def construct(dag):
diff --git a/tributary/lazy/calculations/ops.py b/tributary/lazy/calculations/ops.py
index 6296f3c..4bee5b1 100644
--- a/tributary/lazy/calculations/ops.py
+++ b/tributary/lazy/calculations/ops.py
@@ -1,7 +1,7 @@
import math
import numpy as np
import scipy as sp
-from ..base import BaseNode
+from ..base import Node
########################
@@ -9,35 +9,35 @@
########################
def Add(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '+' + other._name, (lambda x, y: x.value() + y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '+' + other._name, (lambda x, y: x.value() + y.value()), [self, other], self._trace or other._trace)
def Sub(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '-' + other._name, (lambda x, y: x.value() - y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '-' + other._name, (lambda x, y: x.value() - y.value()), [self, other], self._trace or other._trace)
def Mult(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '*' + other._name, (lambda x, y: x.value() * y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '*' + other._name, (lambda x, y: x.value() * y.value()), [self, other], self._trace or other._trace)
def Div(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '/' + other._name, (lambda x, y: x.value() / y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '/' + other._name, (lambda x, y: x.value() / y.value()), [self, other], self._trace or other._trace)
def Pow(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '^' + other._name, (lambda x, y: x.value() ** y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '^' + other._name, (lambda x, y: x.value() ** y.value()), [self, other], self._trace or other._trace)
@@ -51,14 +51,14 @@ def Negate(self):
#####################
def Or(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '||' + other._name, (lambda x, y: x.value() or y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '||' + other._name, (lambda x, y: x.value() or y.value()), [self, other], self._trace or other._trace)
def And(self, other):
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '&&' + other._name, (lambda x, y: x.value() or y.value()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '&&' + other._name, (lambda x, y: x.value() and y.value()), [self, other], self._trace or other._trace)
@@ -94,6 +94,10 @@ def Arctan(self):
return self._gennode('arctan(' + self._name + ')', (lambda x: math.arctan(self.value())), [self], self._trace)
+def Abs(self):
+ return self._gennode('||' + self._name + '||', (lambda x: abs(self.value())), [self], self._trace)
+
+
def Sqrt(self):
return self._gennode('sqrt(' + self._name + ')', (lambda x: math.sqrt(self.value())), [self], self._trace)
@@ -139,22 +143,22 @@ def Len(self):
###################
def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
if ufunc == np.add:
- if isinstance(inputs[0], BaseNode):
+ if isinstance(inputs[0], Node):
return inputs[0].__add__(inputs[1])
else:
return inputs[1].__add__(inputs[0])
elif ufunc == np.subtract:
- if isinstance(inputs[0], BaseNode):
+ if isinstance(inputs[0], Node):
return inputs[0].__sub__(inputs[1])
else:
return inputs[1].__sub__(inputs[0])
elif ufunc == np.multiply:
- if isinstance(inputs[0], BaseNode):
+ if isinstance(inputs[0], Node):
return inputs[0].__mul__(inputs[1])
else:
return inputs[1].__mul__(inputs[0])
elif ufunc == np.divide:
- if isinstance(inputs[0], BaseNode):
+ if isinstance(inputs[0], Node):
return inputs[0].__truedivide__(inputs[1])
else:
return inputs[1].__truedivide__(inputs[0])
@@ -182,60 +186,60 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
# Comparators #
###############
def Equal(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return True
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '==' + other._name, (lambda x, y: x() == y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '==' + other._name, (lambda x, y: x() == y()), [self, other], self._trace or other._trace)
def NotEqual(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return False
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '!=' + other._name, (lambda x, y: x() != y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '!=' + other._name, (lambda x, y: x() != y()), [self, other], self._trace or other._trace)
def Ge(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return True
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '>=' + other._name, (lambda x, y: x() >= y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '>=' + other._name, (lambda x, y: x() >= y()), [self, other], self._trace or other._trace)
def Gt(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return False
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '>' + other._name, (lambda x, y: x() > y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '>' + other._name, (lambda x, y: x() > y()), [self, other], self._trace or other._trace)
def Le(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return True
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '<=' + other._name, (lambda x, y: x() <= y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '<=' + other._name, (lambda x, y: x() <= y()), [self, other], self._trace or other._trace)
def Lt(self, other):
- if isinstance(other, BaseNode) and super(BaseNode, self).__eq__(other):
+ if isinstance(other, Node) and super(Node, self).__eq__(other):
return False
other = self._tonode(other)
- if isinstance(self._self_reference, BaseNode):
+ if isinstance(self._self_reference, Node):
return self._gennode(self._name + '<' + other._name, (lambda x, y: x() < y()), [self._self_reference, other], self._trace or other._trace)
return self._gennode(self._name + '<' + other._name, (lambda x, y: x() < y()), [self, other], self._trace or other._trace)
@@ -243,68 +247,69 @@ def Lt(self, other):
########################
# Arithmetic Operators #
########################
-BaseNode.__add__ = Add
-BaseNode.__radd__ = Add
-BaseNode.__sub__ = Sub
-BaseNode.__rsub__ = Sub
-BaseNode.__mul__ = Mult
-BaseNode.__rmul__ = Mult
-BaseNode.__div__ = Div
-BaseNode.__rdiv__ = Div
-BaseNode.__truediv__ = Div
-BaseNode.__rtruediv__ = Div
-
-BaseNode.__pow__ = Pow
-BaseNode.__rpow__ = Pow
-# BaseNode.__mod__ = Mod
-# BaseNode.__rmod__ = Mod
+Node.__add__ = Add
+Node.__radd__ = Add
+Node.__sub__ = Sub
+Node.__rsub__ = Sub
+Node.__mul__ = Mult
+Node.__rmul__ = Mult
+Node.__div__ = Div
+Node.__rdiv__ = Div
+Node.__truediv__ = Div
+Node.__rtruediv__ = Div
+
+Node.__pow__ = Pow
+Node.__rpow__ = Pow
+# Node.__mod__ = Mod
+# Node.__rmod__ = Mod
#####################
# Logical Operators #
#####################
-BaseNode.__and__ = And
-BaseNode.__or__ = Or
-BaseNode.__invert__ = Not
+Node.__and__ = And
+Node.__or__ = Or
+Node.__invert__ = Not
##############
# Converters #
##############
-BaseNode.int = Int
-BaseNode.float = Float
-BaseNode.__bool__ = Bool
+Node.int = Int
+Node.float = Float
+Node.__bool__ = Bool
###############
# Comparators #
###############
-BaseNode.__lt__ = Lt
-BaseNode.__le__ = Le
-BaseNode.__gt__ = Gt
-BaseNode.__ge__ = Ge
-BaseNode.__eq__ = Equal
-BaseNode.__ne__ = NotEqual
-BaseNode.__neg__ = Negate
-BaseNode.__nonzero__ = Bool # Py2 compat
+Node.__lt__ = Lt
+Node.__le__ = Le
+Node.__gt__ = Gt
+Node.__ge__ = Ge
+Node.__eq__ = Equal
+Node.__ne__ = NotEqual
+Node.__neg__ = Negate
+Node.__nonzero__ = Bool # Py2 compat
###################
# Python Builtins #
###################
-BaseNode.__len__ = Len
+Node.__len__ = Len
###################
# Numpy Functions #
###################
-BaseNode.__array_ufunc__ = __array_ufunc__
+Node.__array_ufunc__ = __array_ufunc__
##########################
# Mathematical Functions #
##########################
-BaseNode.log = Log
-BaseNode.sin = Sin
-BaseNode.cos = Cos
-BaseNode.tan = Tan
-BaseNode.arcsin = Arcsin
-BaseNode.arccos = Arccos
-BaseNode.arctan = Arctan
-BaseNode.sqrt = Sqrt
-BaseNode.exp = Exp
-BaseNode.erf = Erf
+Node.log = Log
+Node.sin = Sin
+Node.cos = Cos
+Node.tan = Tan
+Node.arcsin = Arcsin
+Node.arccos = Arccos
+Node.arctan = Arctan
+Node.abs = Abs
+Node.sqrt = Sqrt
+Node.exp = Exp
+Node.erf = Erf
diff --git a/tributary/tests/reactive/__init__.py b/tributary/lazy/input/__init__.py
similarity index 100%
rename from tributary/tests/reactive/__init__.py
rename to tributary/lazy/input/__init__.py
diff --git a/tributary/lazy/node.py b/tributary/lazy/node.py
index 9a21147..4c3d432 100644
--- a/tributary/lazy/node.py
+++ b/tributary/lazy/node.py
@@ -1,10 +1,9 @@
-import gc
import six
import inspect
from ..utils import _either_type
-class BaseNode(object):
+class Node(object):
'''Class to represent an operation that is lazy'''
def __init__(self,
name="?",
@@ -98,7 +97,7 @@ def _compute_from_dependencies(self):
else:
new_value = k(*self._dependencies[k][0], **self._dependencies[k][1])
- if isinstance(new_value, BaseNode):
+ if isinstance(new_value, Node):
k._node_wrapper = new_value
new_value = new_value() # get value
@@ -149,22 +148,22 @@ def _recompute(self):
def _gennode(self, name, foo, foo_args, trace=False):
if name not in self._node_op_cache:
self._node_op_cache[name] = \
- BaseNode(name=name,
- derived=True,
- callable=foo,
- callable_args=foo_args,
- trace=trace)
+ Node(name=name,
+ derived=True,
+ callable=foo,
+ callable_args=foo_args,
+ trace=trace)
return self._node_op_cache[name]
def _tonode(self, other, trace=False):
- if isinstance(other, BaseNode):
+ if isinstance(other, Node):
return other
if str(other) not in self._node_op_cache:
self._node_op_cache[str(other)] = \
- BaseNode(name='var(' + str(other)[:5] + ')',
- derived=True,
- value=other,
- trace=trace)
+ Node(name='var(' + str(other)[:5] + ')',
+ derived=True,
+ value=other,
+ trace=trace)
return self._node_op_cache[str(other)]
def setValue(self, value):
@@ -198,114 +197,6 @@ def set(self, *args, **kwargs):
def value(self):
return self._value
- def _print(self, counter=0, cache=None):
- if cache is None:
- cache = {}
-
- key = cache.get(id(self), str(self) + ' (#' + str(counter) + ')')
- cache[id(self)] = key
-
- if self._dirty or self._subtree_dirty() or self._always_dirty:
- key += '(dirty)'
-
- ret = {key: []}
- counter += 1
-
- if self._dependencies:
- for call, deps in six.iteritems(self._dependencies):
- # callable node
- if hasattr(call, '_node_wrapper') and \
- call._node_wrapper is not None:
- val, counter = call._node_wrapper._print(counter, cache)
- ret[key].append(val)
-
- # args
- for arg in deps[0]:
- val, counter = arg._print(counter, cache)
- ret[key].append(val)
-
- # kwargs
- for kwarg in six.itervalues(deps[1]):
- val, counter = kwarg._print(counter, cache)
- ret[key].append(val)
-
- return ret, counter
-
- def print(self):
- return self._print(0, {})[0]
-
- def graph(self):
- return self.print()
-
- def graphviz(self):
- d = self.graph()
-
- from graphviz import Digraph
- dot = Digraph(self._name, strict=True)
- dot.format = 'png'
-
- def rec(nodes, parent):
- for d in nodes:
- if not isinstance(d, dict):
- if '(dirty)' in d:
- dot.node(d.replace('(dirty)', ''), color='red')
- dot.edge(d.replace('(dirty)', ''), parent.replace('(dirty)', ''), color='red')
- else:
- dot.node(d)
- dot.edge(d, parent.replace('(dirty)', ''))
- else:
- for k in d:
- if '(dirty)' in k:
- dot.node(k.replace('(dirty)', ''), color='red')
- rec(d[k], k)
- dot.edge(k.replace('(dirty)', ''), parent.replace('(dirty)', ''), color='red')
- else:
- dot.node(k)
- rec(d[k], k)
- dot.edge(k, parent.replace('(dirty)', ''))
-
- for k in d:
- if '(dirty)' in k:
- dot.node(k.replace('(dirty)', ''), color='red')
- else:
- dot.node(k)
- rec(d[k], k)
- return dot
-
- def networkx(self):
- d = self.graph()
- # FIXME deduplicate
- from pygraphviz import AGraph
- import networkx as nx
- dot = AGraph(strict=True, directed=True)
-
- def rec(nodes, parent):
- for d in nodes:
- if not isinstance(d, dict):
- if '(dirty)' in d:
- d = d.replace('(dirty)', '')
- dot.add_node(d, label=d, color='red')
- dot.add_edge(d, parent, color='red')
- else:
- dot.add_node(d, label=d)
- dot.add_edge(d, parent)
- else:
- for k in d:
- if '(dirty)' in k:
- k = k.replace('(dirty)', '')
- dot.add_node(k, label=k, color='red')
- rec(d[k], k)
- dot.add_edge(k, parent, color='red')
- else:
- dot.add_node(k, label=k)
- rec(d[k], k)
- dot.add_edge(k, parent)
-
- for k in d:
- dot.add_node(k, label=k)
- rec(d[k], k)
- return nx.nx_agraph.from_agraph(dot)
-
def __call__(self):
self._recompute()
return self.value()
@@ -347,20 +238,20 @@ def node(meth, memoize=True, trace=False):
value = None
nullable = False
- node_args.append(BaseNode(name=arg,
- derived=True,
- readonly=False,
- nullable=nullable,
- value=value,
- trace=trace))
+ node_args.append(Node(name=arg,
+ derived=True,
+ readonly=False,
+ nullable=nullable,
+ value=value,
+ trace=trace))
for k, v in six.iteritems(argspec.kwonlydefaults or {}):
- node_kwargs[k] = BaseNode(name=k,
- derived=True,
- readonly=False,
- nullable=True,
- value=v,
- trace=trace)
+ node_kwargs[k] = Node(name=k,
+ derived=True,
+ readonly=False,
+ nullable=True,
+ value=v,
+ trace=trace)
def meth_wrapper(self, *args, **kwargs):
if len(args) > len(node_args):
@@ -375,14 +266,14 @@ def meth_wrapper(self, *args, **kwargs):
val = meth(*(arg.value() for arg in args), **kwargs)
return val
- new_node = BaseNode(name=meth.__name__,
- derived=True,
- callable=meth_wrapper,
- callable_args=node_args,
- callable_kwargs=node_kwargs,
- callable_is_method=is_method,
- always_dirty=not memoize,
- trace=trace)
+ new_node = Node(name=meth.__name__,
+ derived=True,
+ callable=meth_wrapper,
+ callable_args=node_args,
+ callable_kwargs=node_kwargs,
+ callable_is_method=is_method,
+ always_dirty=not memoize,
+ trace=trace)
if is_method:
ret = lambda self, *args, **kwargs: new_node._with_self(self) # noqa: E731
diff --git a/tributary/lazy/output/__init__.py b/tributary/lazy/output/__init__.py
new file mode 100644
index 0000000..a970658
--- /dev/null
+++ b/tributary/lazy/output/__init__.py
@@ -0,0 +1,85 @@
+from ..node import Node
+
+
+def _print(node, counter=0, cache=None):
+ if cache is None:
+ cache = {}
+
+ key = cache.get(id(node), str(node) + ' (#' + str(counter) + ')')
+ cache[id(node)] = key
+
+ if node._dirty or node._subtree_dirty() or node._always_dirty:
+ key += '(dirty)'
+
+ ret = {key: []}
+ counter += 1
+
+ if node._dependencies:
+ for call, deps in node._dependencies.items():
+ # callable node
+ if hasattr(call, '_node_wrapper') and \
+ call._node_wrapper is not None:
+ val, counter = call._node_wrapper._print(counter, cache)
+ ret[key].append(val)
+
+ # args
+ for arg in deps[0]:
+ val, counter = arg._print(counter, cache)
+ ret[key].append(val)
+
+ # kwargs
+ for kwarg in deps[1].values():
+ val, counter = kwarg._print(counter, cache)
+ ret[key].append(val)
+
+ return ret, counter
+
+
+def Print(node):
+ return node._print(0, {})[0]
+
+
+def Graph(node):
+ return node.print()
+
+
+def GraphViz(node):
+ d = node.graph()
+
+ from graphviz import Digraph
+ dot = Digraph(node._name, strict=True)
+ dot.format = 'png'
+
+ def rec(nodes, parent):
+ for d in nodes:
+ if not isinstance(d, dict):
+ if '(dirty)' in d:
+ dot.node(d.replace('(dirty)', ''), color='red')
+ dot.edge(d.replace('(dirty)', ''), parent.replace('(dirty)', ''), color='red')
+ else:
+ dot.node(d)
+ dot.edge(d, parent.replace('(dirty)', ''))
+ else:
+ for k in d:
+ if '(dirty)' in k:
+ dot.node(k.replace('(dirty)', ''), color='red')
+ rec(d[k], k)
+ dot.edge(k.replace('(dirty)', ''), parent.replace('(dirty)', ''), color='red')
+ else:
+ dot.node(k)
+ rec(d[k], k)
+ dot.edge(k, parent.replace('(dirty)', ''))
+
+ for k in d:
+ if '(dirty)' in k:
+ dot.node(k.replace('(dirty)', ''), color='red')
+ else:
+ dot.node(k)
+ rec(d[k], k)
+ return dot
+
+
+Node._print = _print
+Node.print = Print
+Node.graph = Graph
+Node.graphviz = GraphViz
diff --git a/tributary/reactive/__init__.py b/tributary/reactive/__init__.py
index acb740b..a40a5a9 100644
--- a/tributary/reactive/__init__.py
+++ b/tributary/reactive/__init__.py
@@ -1,6 +1,5 @@
import asyncio
-import types
-from .base import _wrap, Foo, Const, Share # noqa: F401
+from .base import _wrap, Foo, Const, Share, FunctionWrapper # noqa: F401
from .calculations import * # noqa: F401, F403
from .input import * # noqa: F401, F403
from .output import * # noqa: F401, F403
@@ -13,14 +12,7 @@ async def _run(foo, **kwargs):
ret = []
try:
async for item in foo():
- if isinstance(item, types.AsyncGeneratorType):
- async for i in item:
- ret.append(i)
- elif isinstance(item, types.CoroutineType):
- ret.append(await item)
- else:
- ret.append(item)
-
+ ret.append(item)
except KeyboardInterrupt:
print('Terminating...')
return ret
@@ -37,7 +29,7 @@ def run(foo, **kwargs):
return x
-class BaseGraph(object):
+class StreamingGraph(object):
def __init__(self, run=None, *args, **kwargs):
self._run = run
diff --git a/tributary/reactive/base.py b/tributary/reactive/base.py
index 1db333c..1de91ac 100644
--- a/tributary/reactive/base.py
+++ b/tributary/reactive/base.py
@@ -31,19 +31,6 @@ def _wrap(foo, foo_kwargs, name='', wraps=(), share=None, state=None):
return ret
-def _call_if_function(f):
- '''call f if it is a function
- Args:
- f (any): a function or value
- Returns:
- any: return either f() or f
- '''
- if isinstance(f, types.FunctionType):
- return f()
- else:
- return f
-
-
def _inc_ref(f_wrapped, f_wrapping):
'''Increment reference count for wrapped f
@@ -54,11 +41,10 @@ def _inc_ref(f_wrapped, f_wrapping):
if f_wrapped._id == f_wrapping._id:
raise Exception('Internal Error')
- if f_wrapped._using is None or f_wrapped._using == id(f_wrapping):
+ if f_wrapped._using is None:
f_wrapped._using = id(f_wrapping)
return
Share(f_wrapped)
- f_wrapped._using = id(f_wrapping)
def Const(val):
@@ -69,10 +55,7 @@ def Const(val):
Returns:
FunctionWrapper: a streaming wrapper
'''
- async def _always(val):
- yield val
-
- return _wrap(_always, dict(val=val), name='Const', wraps=(val,))
+ return _wrap(val, dict(), name='Const', wraps=(val,))
def Foo(foo, foo_kwargs=None):
@@ -125,7 +108,10 @@ def __init__(self, foo, foo_kwargs, name='', wraps=(), share=None, state=None):
if not (isinstance(foo, types.FunctionType) or isinstance(foo, types.CoroutineType)):
# bind to f so foo isnt referenced
- foo = lambda f=foo: f # noqa: E731
+ def _always(val=foo):
+ while True:
+ yield val
+ foo = _always
if len(foo.__code__.co_varnames) > 0 and \
foo.__code__.co_varnames[0] == 'state':
@@ -202,56 +188,36 @@ async def __call__(self, *args, **kwargs):
if DEBUG:
print("calling: {}".format(self._foo))
- while(self._refs == self._refs_orig):
- kwargs.update(self._foo_kwargs)
- ret = self._foo(*args, **kwargs)
-
- if isinstance(ret, types.AsyncGeneratorType):
- async for r in ret:
- tmp = _call_if_function(r)
- if isinstance(tmp, types.CoroutineType):
- tmp = await tmp
-
- if isinstance(tmp, types.AsyncGeneratorType):
- async for rr in tmp:
- self.last = rr
- yield self.last
-
- else:
- self.last = tmp
- yield self.last
- elif isinstance(ret, types.GeneratorType):
- for r in ret:
- tmp = _call_if_function(r)
- if isinstance(tmp, types.CoroutineType):
- tmp = await tmp
-
- if isinstance(tmp, types.GeneratorType):
- for rr in tmp:
- self.last = rr
- yield self.last
-
- else:
- self.last = tmp
- yield self.last
- else:
- tmp = _call_if_function(ret)
- if isinstance(tmp, types.CoroutineType):
- tmp = await tmp
-
- if isinstance(tmp, types.AsyncGeneratorType):
- async for rr in tmp:
- self.last = rr
- yield self.last
- else:
- self.last = tmp
- yield self.last
+ kwargs.update(self._foo_kwargs)
- while(0 < self._refs and self._refs < self._refs_orig):
+ async for item in _extract(self._foo, *args, **kwargs):
+ self.last = item
+ # while 0 < self._refs <= self._refs_orig:
yield self.last
- # reset state to be called again
- self._refs = self._refs_orig
-
def __iter__(self):
yield from self.__call__()
+
+
+async def _extract(item, *args, **kwargs):
+ while isinstance(item, FunctionWrapper) or isinstance(item, types.FunctionType) or isinstance(item, types.CoroutineType):
+ if isinstance(item, FunctionWrapper):
+ item = item()
+
+ if isinstance(item, types.FunctionType):
+ item = item(*args, **kwargs)
+
+ if isinstance(item, types.CoroutineType):
+ item = await item
+
+ if isinstance(item, types.AsyncGeneratorType):
+ async for subitem in item:
+ async for extracted in _extract(subitem):
+ yield extracted
+
+ elif isinstance(item, types.GeneratorType):
+ for subitem in item:
+ async for extracted in _extract(subitem):
+ yield extracted
+ else:
+ yield item
diff --git a/tributary/reactive/calculations/ops.py b/tributary/reactive/calculations/ops.py
index aff79e2..d3ff32e 100644
--- a/tributary/reactive/calculations/ops.py
+++ b/tributary/reactive/calculations/ops.py
@@ -1,64 +1,32 @@
-import types
import math
from aiostream.stream import zip
+
from ..base import _wrap, FunctionWrapper
def unary(lam, foo, foo_kwargs=None, _name=''):
- foo_kwargs = None or {}
+ foo_kwargs = foo_kwargs or {}
foo = _wrap(foo, foo_kwargs)
async def _unary(foo):
- async for gen in foo():
- if isinstance(gen, types.AsyncGeneratorType):
- async for f in gen:
- if isinstance(f, types.CoroutineType):
- yield lam(await f)
- else:
- yield lam(f)
- elif isinstance(gen, types.CoroutineType):
- yield lam(await gen)
- else:
- yield lam(gen)
-
+ async for val in foo():
+ yield lam(val)
return _wrap(_unary, dict(foo=foo), name=_name or 'Unary', wraps=(foo,), share=None)
def bin(lam, foo1, foo2, foo1_kwargs=None, foo2_kwargs=None, _name=''):
- foo1_kwargs = None or {}
- foo2_kwargs = None or {}
+ foo1_kwargs = foo1_kwargs or {}
+ foo2_kwargs = foo2_kwargs or {}
foo1 = _wrap(foo1, foo1_kwargs)
foo2 = _wrap(foo2, foo2_kwargs)
async def _bin(foo1, foo2):
# TODO replace with merge
- async for gen1, gen2 in zip(foo1(), foo2()):
- if isinstance(gen1, types.AsyncGeneratorType) and isinstance(gen2, types.AsyncGeneratorType):
- async for f1, f2 in zip(gen1, gen2):
- if isinstance(f1, types.CoroutineType):
- f1 = await f1
- if isinstance(f2, types.CoroutineType):
- f2 = await f2
- yield lam(f1, f2)
- elif isinstance(gen1, types.AsyncGeneratorType):
- async for f1 in gen1:
- if isinstance(f1, types.CoroutineType):
- f1 = await f1
- if isinstance(gen2, types.CoroutineType):
- gen2 = await gen2
- yield lam(f1, gen2)
- elif isinstance(gen2, types.AsyncGeneratorType):
- async for f2 in gen2:
- if isinstance(gen1, types.CoroutineType):
- gen1 = await gen1
- if isinstance(f2, types.CoroutineType):
- f2 = await f2
- yield lam(gen1, f2)
- else:
- if isinstance(gen1, types.CoroutineType):
- gen1 = await gen1
- if isinstance(gen2, types.CoroutineType):
- gen2 = await gen2
+ if foo1 == foo2:
+ async for gen in foo1():
+ yield lam(gen, gen)
+ else:
+ async for gen1, gen2 in zip(foo1(), foo2()):
yield lam(gen1, gen2)
return _wrap(_bin, dict(foo1=foo1, foo2=foo2), name=_name or 'Binary', wraps=(foo1, foo2), share=None)
@@ -78,6 +46,7 @@ def Negate(foo, foo_kwargs=None):
def Invert(foo, foo_kwargs=None):
return unary(lambda x: 1 / x, foo, foo_kwargs, _name='Invert')
+
def Add(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
return bin(lambda x, y: x + y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Add')
@@ -184,6 +153,7 @@ def Bool(foo, foo_kwargs=None):
def Len(foo, foo_kwargs=None):
return unary(lambda x: len(x), foo, foo_kwargs=foo_kwargs, _name="Len")
+
########################
# Arithmetic Operators #
########################
diff --git a/tributary/reactive/input/socketio.py b/tributary/reactive/input/socketio.py
index a5596ad..3e9d704 100644
--- a/tributary/reactive/input/socketio.py
+++ b/tributary/reactive/input/socketio.py
@@ -7,7 +7,6 @@
from ..base import _wrap
-
def AsyncSocketIO(url, channel='', field='', sendinit=None, json=False, wrap=False, interval=1):
'''Connect to socketIO server and yield back results
diff --git a/tributary/reactive/input/ws.py b/tributary/reactive/input/ws.py
index 454591c..00c7472 100644
--- a/tributary/reactive/input/ws.py
+++ b/tributary/reactive/input/ws.py
@@ -5,7 +5,6 @@
from ...base import StreamNone, StreamEnd
-
def AsyncWebSocket(url, json=False, wrap=False):
'''Connect to websocket and yield back results
diff --git a/tributary/reactive/output/__init__.py b/tributary/reactive/output/__init__.py
index 05e4996..7ba122d 100644
--- a/tributary/reactive/output/__init__.py
+++ b/tributary/reactive/output/__init__.py
@@ -1,28 +1,25 @@
import asyncio
-import types
from pprint import pprint
from IPython.display import display
-from ..base import _wrap, FunctionWrapper
+from ..base import _wrap, _extract, FunctionWrapper
from .file import File as FileSink # noqa: F401
from .http import HTTP as HTTPSink # noqa: F401
from .kafka import Kafka as KafkaSink # noqa: F401
from .ws import WebSocket as WebSocketSink # noqa: F401
-def Print(foo, foo_kwargs=None):
+def Print(foo, foo_kwargs=None, text=''):
foo_kwargs = foo_kwargs or {}
foo = _wrap(foo, foo_kwargs)
async def _print(foo):
- async for r in foo():
- if isinstance(r, types.AsyncGeneratorType):
- async for x in r:
- yield x
- elif isinstance(r, types.CoroutineType):
- yield await r
+ async for r in _extract(foo):
+ if text:
+ print(text, r)
else:
- yield r
+ print(r)
+ yield r
return _wrap(_print, dict(foo=foo), name='Print', wraps=(foo,), share=foo)
diff --git a/tributary/reactive/output/ws.py b/tributary/reactive/output/ws.py
index 63f8fba..9e99571 100644
--- a/tributary/reactive/output/ws.py
+++ b/tributary/reactive/output/ws.py
@@ -5,7 +5,6 @@
from ...base import StreamNone, StreamEnd
-
def AsyncWebSocket(foo, foo_kwargs=None, url='', json=False, wrap=False, field=None, response=False):
'''Connect to websocket and send data
diff --git a/tributary/reactive/utils.py b/tributary/reactive/utils.py
index 8b92bac..a00c74e 100644
--- a/tributary/reactive/utils.py
+++ b/tributary/reactive/utils.py
@@ -25,10 +25,8 @@ def Timer(foo_or_val, kwargs=None, interval=1, repeat=0):
async def _repeater(foo, repeat, interval):
while repeat > 0:
t1 = time.time()
- f = foo()
- yield f
+ yield foo()
t2 = time.time()
-
if interval > 0:
# sleep for rest of time that _p didnt take
await asyncio.sleep(max(0, interval - (t2 - t1)))
diff --git a/tributary/streaming/__init__.py b/tributary/streaming/__init__.py
new file mode 100644
index 0000000..5e56a54
--- /dev/null
+++ b/tributary/streaming/__init__.py
@@ -0,0 +1,29 @@
+import asyncio
+from .base import Node # noqa: F401
+from .calculations import * # noqa: F401, F403
+from .input import * # noqa: F401, F403
+from .output import * # noqa: F401, F403
+from .utils import * # noqa: F401, F403
+from ..base import StreamEnd, StreamNone
+
+
+async def _run(node):
+ ret = []
+ nodes = node._deep_bfs()
+ while True:
+ for level in nodes:
+ await asyncio.gather(*(asyncio.create_task(n()) for n in level))
+ if not isinstance(node.value(), (StreamEnd, StreamNone)):
+ ret.append(node.value())
+ elif isinstance(node.value(), StreamEnd):
+ break
+ return ret
+
+
+def run(node):
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ # return future
+ return asyncio.create_task(_run(node))
+ # block until done
+ return loop.run_until_complete(_run(node))
diff --git a/tributary/streaming/base.py b/tributary/streaming/base.py
new file mode 100644
index 0000000..ba34b88
--- /dev/null
+++ b/tributary/streaming/base.py
@@ -0,0 +1,125 @@
+import asyncio
+import types
+from aiostream.aiter_utils import anext
+from asyncio import Queue, QueueEmpty as Empty
+from ..base import StreamEnd, StreamNone
+
+
+def _gen_to_foo(generator):
+ try:
+ return next(generator)
+ except StopIteration:
+ return StreamEnd()
+
+
+async def _agen_to_foo(generator):
+ try:
+ return await anext(generator)
+ except StopAsyncIteration:
+ return StreamEnd()
+
+
+class Node(object):
+ _id_ref = 0
+
+ def __init__(self, foo, foo_kwargs=None, name=None, inputs=1):
+ self._id = Node._id_ref
+ Node._id_ref += 1
+
+ self._name = '{}#{}'.format(name or self.__class__.__name__, self._id)
+ self._input = [Queue() for _ in range(inputs)]
+ self._active = [StreamNone() for _ in range(inputs)]
+ self._downstream = []
+ self._upstream = []
+
+ self._foo = foo
+ self._foo_kwargs = foo_kwargs or {}
+ self._last = StreamNone()
+ self._finished = False
+
+ def __repr__(self):
+ return '{}'.format(self._name)
+
+ async def _push(self, inp, index):
+ await self._input[index].put(inp)
+
+ async def _execute(self):
+ if asyncio.iscoroutine(self._foo):
+ _last = await self._foo(*self._active, **self._foo_kwargs)
+ elif isinstance(self._foo, types.FunctionType):
+ _last = self._foo(*self._active, **self._foo_kwargs)
+ else:
+ raise Exception('Cannot use type:{}'.format(type(self._foo)))
+
+ if isinstance(_last, types.AsyncGeneratorType):
+ async def _foo(g=_last):
+ return await _agen_to_foo(g)
+ self._foo = _foo
+ _last = await self._foo()
+
+ elif isinstance(_last, types.GeneratorType):
+ self._foo = lambda g=_last: _gen_to_foo(g)
+ _last = self._foo()
+ elif asyncio.iscoroutine(_last):
+ _last = await _last
+
+ self._last = _last
+ await self._output(self._last)
+ for i in range(len(self._active)):
+ self._active[i] = StreamNone()
+
+ async def _finish(self):
+ self._finished = True
+ self._last = StreamEnd()
+ await self._output(self._last)
+
+ async def __call__(self):
+ if self._finished:
+ return await self._finish()
+
+ ready = True
+ # iterate through inputs
+ for i, inp in enumerate(self._input):
+ # if input hasn't received value
+ if isinstance(self._active[i], StreamNone):
+ try:
+ # get from input queue
+ val = inp.get_nowait()
+
+ if isinstance(val, StreamEnd):
+ return await self._finish()
+
+ # set as active
+ self._active[i] = val
+ except Empty:
+ # wait for value
+ ready = False
+
+ if ready:
+ # execute function
+ return await self._execute()
+
+ async def _output(self, ret):
+ # if downstreams, output
+ for down, i in self._downstream:
+ await down._push(ret, i)
+ return ret
+
+ def _deep_bfs(self, reverse=True):
+ nodes = []
+ nodes.append([self])
+
+ upstreams = self._upstream.copy()
+ while upstreams:
+ nodes.append(upstreams)
+ upstreams = []
+ for n in nodes[-1]:
+ upstreams.extend(n._upstream)
+
+ if reverse:
+ nodes.reverse()
+
+ return nodes
+
+ def value(self):
+ return self._last
diff --git a/tributary/streaming/calculations/__init__.py b/tributary/streaming/calculations/__init__.py
new file mode 100644
index 0000000..c89de68
--- /dev/null
+++ b/tributary/streaming/calculations/__init__.py
@@ -0,0 +1,2 @@
+from .ops import * # noqa: F401, F403
+from .rolling import * # noqa: F401, F403
diff --git a/tributary/streaming/calculations/ops.py b/tributary/streaming/calculations/ops.py
new file mode 100644
index 0000000..67e9f91
--- /dev/null
+++ b/tributary/streaming/calculations/ops.py
@@ -0,0 +1,160 @@
+import math
+from ..base import Node
+
+
+def unary(foo, name):
+ def _foo(self):
+ downstream = Node(foo, {}, name=name, inputs=1)
+ self._downstream.append((downstream, 0))
+ downstream._upstream.append(self)
+ return downstream
+ return _foo
+
+
+def binary(foo, name):
+ def _foo(self, other):
+ downstream = Node(foo, {}, name=name, inputs=2)
+ self._downstream.append((downstream, 0))
+ other._downstream.append((downstream, 1))
+ downstream._upstream.extend([self, other])
+ return downstream
+ return _foo
+
+
+########################
+# Arithmetic Operators #
+########################
+Noop = unary(lambda x: x, name='Noop')
+Negate = unary(lambda x: -1 * x, name='Negate')
+Invert = unary(lambda x: 1 / x, name='Invert')
+Add = binary(lambda x, y: x + y, name='Add')
+Sub = binary(lambda x, y: x - y, name='Sub')
+Mult = binary(lambda x, y: x * y, name='Mult')
+Div = binary(lambda x, y: x / y, name='Div')
+RDiv = binary(lambda x, y: y / x, name='RDiv')
+Mod = binary(lambda x, y: x % y, name='Mod')
+Pow = binary(lambda x, y: x ** y, name='Pow')
+
+
+#####################
+# Logical Operators #
+#####################
+Not = unary(lambda x: not x, name='Not')
+And = binary(lambda x, y: x and y, name='And')
+Or = binary(lambda x, y: x or y, name='Or')
+
+
+###############
+# Comparators #
+###############
+Equal = binary(lambda x, y: x == y, name='Equal')
+NotEqual = binary(lambda x, y: x != y, name='NotEqual')
+Lt = binary(lambda x, y: x < y, name='Less')
+Le = binary(lambda x, y: x <= y, name='LessOrEqual')
+Gt = binary(lambda x, y: x > y, name='Greater')
+Ge = binary(lambda x, y: x >= y, name='GreaterOrEqual')
+
+
+##########################
+# Mathematical Functions #
+##########################
+Log = unary(lambda x: math.log(x), name='Log')
+Sin = unary(lambda x: math.sin(x), name='Sin')
+Cos = unary(lambda x: math.cos(x), name='Cos')
+Tan = unary(lambda x: math.tan(x), name='Tan')
+Arcsin = unary(lambda x: math.asin(x), name='Arcsin')
+Arccos = unary(lambda x: math.acos(x), name='Arccos')
+Arctan = unary(lambda x: math.atan(x), name='Arctan')
+Sqrt = unary(lambda x: math.sqrt(x), name='Sqrt')
+Abs = unary(lambda x: abs(x), name='Abs')
+Exp = unary(lambda x: math.exp(x), name='Exp')
+Erf = unary(lambda x: math.erf(x), name='Erf')
+
+
+##############
+# Converters #
+##############
+Int = unary(lambda x: int(x), name='Int')
+Float = unary(lambda x: float(x), name='Float')
+Bool = unary(lambda x: bool(x), name='Bool')
+Str = unary(lambda x: str(x), name='Str')
+
+
+###################
+# Python Builtins #
+###################
+Len = unary(lambda x: len(x), name='Len')
+
+
+########################
+# Arithmetic Operators #
+########################
+Node.__add__ = Add
+Node.__radd__ = Add
+Node.__sub__ = Sub
+Node.__rsub__ = Sub
+Node.__mul__ = Mult
+Node.__rmul__ = Mult
+Node.__div__ = Div
+Node.__rdiv__ = RDiv
+Node.__truediv__ = Div
+Node.__rtruediv__ = RDiv
+
+Node.__pow__ = Pow
+Node.__rpow__ = Pow
+Node.__mod__ = Mod
+Node.__rmod__ = Mod
+
+#####################
+# Logical Operators #
+#####################
+Node.__and__ = And
+Node.__or__ = Or
+Node.__invert__ = Not
+Node.__bool__ = Bool
+# TODO use __bool__ operator
+
+##############
+# Converters #
+##############
+Node.int = Int
+Node.float = Float
+Node.__str__ = Str
+
+###############
+# Comparators #
+###############
+Node.__lt__ = Lt
+Node.__le__ = Le
+Node.__gt__ = Gt
+Node.__ge__ = Ge
+Node.__eq__ = Equal
+Node.__ne__ = NotEqual
+Node.__neg__ = Negate
+# Node.__nonzero__ = Bool # Py2 compat
+
+###################
+# Python Builtins #
+###################
+Node.__len__ = Len
+
+###################
+# Numpy Functions #
+###################
+# Node.__array_ufunc__ = __array_ufunc__
+
+
+##########################
+# Mathematical Functions #
+##########################
+Node.log = Log
+Node.sin = Sin
+Node.cos = Cos
+Node.tan = Tan
+Node.asin = Arcsin
+Node.acos = Arccos
+Node.atan = Arctan
+Node.abs = Abs
+Node.sqrt = Sqrt
+Node.exp = Exp
+Node.erf = Erf
diff --git a/tributary/tests/reactive/calculations/test_calculations_reactive.py b/tributary/streaming/calculations/rolling.py
similarity index 100%
rename from tributary/tests/reactive/calculations/test_calculations_reactive.py
rename to tributary/streaming/calculations/rolling.py
diff --git a/tributary/streaming/input/__init__.py b/tributary/streaming/input/__init__.py
new file mode 100644
index 0000000..1c24f53
--- /dev/null
+++ b/tributary/streaming/input/__init__.py
@@ -0,0 +1,6 @@
+from .input import * # noqa: F401, F403
+from .file import File, File as FileSource # noqa: F401
+from .http import HTTP, HTTP as HTTPSource # noqa: F401
+from .kafka import Kafka, Kafka as KafkaSource # noqa: F401
+from .socketio import SocketIO, SocketIO as SocketIOSource # noqa: F401
+from .ws import WebSocket, WebSocket as WebSocketSource # noqa: F401
diff --git a/tributary/streaming/input/file.py b/tributary/streaming/input/file.py
new file mode 100644
index 0000000..23514e6
--- /dev/null
+++ b/tributary/streaming/input/file.py
@@ -0,0 +1,22 @@
+import aiofiles
+import json as JSON
+from .input import Foo
+
+
+class File(Foo):
+ '''Open up a file and yield back lines in the file
+
+ Args:
+ filename (str): filename to read
+ json (bool): load file line as json
+ '''
+ def __init__(self, filename, json=True):
+ async def _file(filename=filename, json=json):
+ async with aiofiles.open(filename) as f:
+ async for line in f:
+ if json:
+ yield JSON.loads(line)
+ else:
+ yield line
+ super().__init__(foo=_file)
+ self._name = 'File'
diff --git a/tributary/streaming/input/http.py b/tributary/streaming/input/http.py
new file mode 100644
index 0000000..4b7e115
--- /dev/null
+++ b/tributary/streaming/input/http.py
@@ -0,0 +1,50 @@
+import aiohttp
+import json as JSON
+import time
+from .input import Foo
+
+
+class HTTP(Foo):
+ '''Connect to url and yield results
+
+ Args:
+ url (str): url to connect to
+ interval (int): interval to re-query
+ repeat (int): number of times to request
+ json (bool): load http content data as json
+ wrap (bool): wrap result in a list
+ field (str): field to index result by
+ proxies (list): list of URL proxies to pass to requests.get
+ cookies (list): list of cookies to pass to requests.get
+ '''
+
+ def __init__(self, url, interval=1, repeat=1, json=False, wrap=False, field=None, proxies=None, cookies=None):
+ async def _req(url=url, interval=interval, repeat=repeat, json=json, wrap=wrap, field=field, proxies=proxies, cookies=cookies):
+ count = 0
+ while count < repeat:
+ async with aiohttp.ClientSession() as session:
+ async with session.get(url, cookies=cookies, proxy=proxies) as response:
+ msg = await response.text()
+
+ if msg is None or response.status != 200:
+ break
+
+ if json:
+ msg = JSON.loads(msg)
+
+ if field:
+ msg = msg[field]
+
+ if wrap:
+ msg = [msg]
+
+ yield msg
+
+ if interval:
+ time.sleep(interval)
+ if repeat >= 0:
+ count += 1
+
+ super().__init__(foo=_req)
+ self._name = 'Http'
+
diff --git a/tributary/streaming/input/input.py b/tributary/streaming/input/input.py
new file mode 100644
index 0000000..436aa62
--- /dev/null
+++ b/tributary/streaming/input/input.py
@@ -0,0 +1,77 @@
+import asyncio
+import math
+import numpy as np
+
+from ..base import Node
+from ...base import StreamEnd
+
+
+def _gen():
+ S = 100
+ T = 252
+ mu = 0.25
+ vol = 0.5
+
+ returns = np.random.normal(mu / T, vol / math.sqrt(T), T) + 1
+ _list = returns.cumprod() * S
+ return _list
+
+
+class Timer(Node):
+ def __init__(self, foo, foo_kwargs=None, count=1, interval=0):
+ self._count = count
+ self._executed = 0
+ self._interval = interval
+
+ super().__init__(foo=foo, foo_kwargs=foo_kwargs, name='Timer[{}]'.format(foo.__name__), inputs=0)
+
+ async def _execute(self):
+ self._executed += 1
+ await super()._execute()
+
+ async def __call__(self):
+ # sleep if needed
+ if self._interval:
+ await asyncio.sleep(self._interval)
+
+ if self._count > 0 and self._executed >= self._count:
+ self._foo = lambda: StreamEnd()
+
+ return await self._execute()
+
+
+class Const(Timer):
+ def __init__(self, value, count=0):
+ super().__init__(foo=lambda: value, count=count, interval=0)
+ self._name = 'Const[{}]'.format(value)
+
+
+class Foo(Timer):
+ def __init__(self, foo, foo_kwargs=None, count=0, interval=0):
+ super().__init__(foo=foo, foo_kwargs=foo_kwargs, count=count, interval=interval)
+ self._name = 'Foo[{}]'.format(foo.__name__)
+
+
+class Random(Foo):
+ '''Yield a random dictionary of data
+
+ Args:
+ count (int): number of elements to yield
+ interval (float): interval to wait between yields
+ '''
+
+ def __init__(self, count=10, interval=0.1):
+ def _random(count=count, interval=interval):
+ step = 0
+ while step < count:
+ x = {y: _gen() for y in ('A', 'B', 'C', 'D')}
+ for i in range(len(x['A'])):
+ if step >= count:
+ break
+ yield {'A': x['A'][i],
+ 'B': x['B'][i],
+ 'C': x['C'][i],
+ 'D': x['D'][i]}
+ step += 1
+ super().__init__(foo=_random, count=count, interval=interval)
+ self._name = 'Random'
diff --git a/tributary/streaming/input/kafka.py b/tributary/streaming/input/kafka.py
new file mode 100644
index 0000000..6514dfd
--- /dev/null
+++ b/tributary/streaming/input/kafka.py
@@ -0,0 +1,57 @@
+import json as JSON
+import time
+from confluent_kafka import Consumer, KafkaError
+from .input import Foo
+
+
+class Kafka(Foo):
+ '''Connect to kafka server and yield back results
+
+ Args:
+ servers (list): kafka bootstrap servers
+ group (str): kafka group id
+ topics (list): list of kafka topics to connect to
+ json (bool): load input data as json
+ wrap (bool): wrap result in a list
+ interval (int): kafka poll interval
+ '''
+
+ def __init__(self, servers, group, topics, json=False, wrap=False, interval=1):
+ c = Consumer({
+ 'bootstrap.servers': servers,
+ 'group.id': group,
+ 'default.topic.config': {
+ 'auto.offset.reset': 'smallest'
+ }
+ })
+
+ if not isinstance(topics, list):
+ topics = [topics]
+ c.subscribe(topics)
+
+ async def _listen(consumer=c, json=json, wrap=wrap, interval=interval):
+ while True:
+ msg = consumer.poll(interval)
+
+ if msg is None:
+ continue
+ if msg.error():
+ if msg.error().code() == KafkaError._PARTITION_EOF:
+ continue
+ else:
+ print(msg.error())
+ break
+
+ msg = msg.value().decode('utf-8')
+
+ if not msg:
+ break
+ if json:
+ msg = JSON.loads(msg)
+ if wrap:
+ msg = [msg]
+ yield msg
+
+ super().__init__(foo=_listen)
+ self._name = 'Kafka'
+
diff --git a/tributary/streaming/input/socketio.py b/tributary/streaming/input/socketio.py
new file mode 100644
index 0000000..70d6333
--- /dev/null
+++ b/tributary/streaming/input/socketio.py
@@ -0,0 +1,46 @@
+import json as JSON
+import time
+from socketIO_client_nexus import SocketIO as SIO
+from urllib.parse import urlparse
+
+from .input import Foo
+
+
+class SocketIO(Foo):
+ '''Connect to socketIO server and yield back results
+
+ Args:
+ url (str): url to connect to
+ channel (str): socketio channel to connect through
+ field (str): field to index result by
+ sendinit (list): data to send on socketio connection open
+ json (bool): load websocket data as json
+ wrap (bool): wrap result in a list
+ interval (int): socketio wai interval
+ '''
+
+ def __init__(self, url, channel='', field='', sendinit=None, json=False, wrap=False, interval=1):
+ o = urlparse(url)
+ socketIO = SIO(o.scheme + '://' + o.netloc, o.port)
+ if sendinit:
+ socketIO.emit(sendinit)
+
+ async def _sio(url=url, channel=channel, field=field, json=json, wrap=wrap, interval=interval):
+ while True:
+ _data = []
+ socketIO.on(channel, lambda data: _data.append(data))
+ socketIO.wait(seconds=interval)
+ for msg in _data:
+ # FIXME clear _data
+ if json:
+ msg = json.loads(msg)
+
+ if field:
+ msg = msg[field]
+
+ if wrap:
+ msg = [msg]
+
+ yield msg
+ super().__init__(foo=_sio)
+ self._name = 'Kafka'
diff --git a/tributary/streaming/input/ws.py b/tributary/streaming/input/ws.py
new file mode 100644
index 0000000..d197abd
--- /dev/null
+++ b/tributary/streaming/input/ws.py
@@ -0,0 +1,35 @@
+import json as JSON
+import websockets
+import time
+
+from .input import Foo
+from ...base import StreamNone, StreamEnd
+
+
+class WebSocket(Foo):
+ '''Connect to websocket and yield back results
+
+ Args:
+ url (str): websocket url to connect to
+ json (bool): load websocket data as json
+ wrap (bool): wrap result in a list
+ '''
+
+ def __init__(self, url, json=False, wrap=False):
+ async def _listen(url=url, json=json, wrap=wrap):
+ async with websockets.connect(url) as websocket:
+ async for x in websocket:
+ if isinstance(x, StreamNone):
+ continue
+ elif not x or isinstance(x, StreamEnd):
+ break
+
+ if json:
+ x = JSON.loads(x)
+ if wrap:
+ x = [x]
+ yield x
+
+ super().__init__(foo=_listen)
+ self._name = 'WebSocket'
+
diff --git a/tributary/streaming/output/__init__.py b/tributary/streaming/output/__init__.py
new file mode 100644
index 0000000..1582faf
--- /dev/null
+++ b/tributary/streaming/output/__init__.py
@@ -0,0 +1,5 @@
+from .output import * # noqa: F401, F403
+from .file import File as FileSink # noqa: F401
+from .http import HTTP as HTTPSink # noqa: F401
+from .kafka import Kafka as KafkaSink # noqa: F401
+from .ws import WebSocket as WebSocketSink # noqa: F401
diff --git a/tributary/streaming/output/file.py b/tributary/streaming/output/file.py
new file mode 100644
index 0000000..ac039ae
--- /dev/null
+++ b/tributary/streaming/output/file.py
@@ -0,0 +1,25 @@
+import aiofiles
+import json as JSON
+from ..base import Node
+
+
+class File(Node):
+ '''Open up a file and write lines to the file
+
+ Args:
+ node (Node): input stream
+ filename (str): filename to write
+ json (bool): load file line as json
+ '''
+ def __init__(self, node, filename='', json=True):
+ async def _file(data):
+ async with aiofiles.open(filename, mode='a') as fp:
+ if json:
+ fp.write(JSON.dumps(data))
+ else:
+ fp.write(data)
+ return data
+
+ super().__init__(foo=_file, name='File', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
diff --git a/tributary/streaming/output/http.py b/tributary/streaming/output/http.py
new file mode 100644
index 0000000..a19b4b8
--- /dev/null
+++ b/tributary/streaming/output/http.py
@@ -0,0 +1,48 @@
+import requests
+import json as JSON
+from ..base import Node
+from ...base import StreamEnd
+
+
+class HTTP(Node):
+ '''Connect to url and post results to it
+
+ Args:
+ foo (callable): input stream
+ foo_kwargs (dict): kwargs for the input stream
+ url (str): url to post to
+ json (bool): dump data as json
+ wrap (bool): wrap input in a list
+ field (str): field to index result by
+ proxies (list): list of URL proxies to pass to requests.post
+ cookies (list): list of cookies to pass to requests.post
+ '''
+ def __init__(self, node, url='', json=False, wrap=False, field=None, proxies=None, cookies=None):
+ def _send(data, url=url, json=json, wrap=wrap, field=field, proxies=proxies, cookies=cookies):
+ if wrap:
+ data = [data]
+ if json:
+ data = JSON.dumps(data)
+
+ msg = requests.post(url, data=data, cookies=cookies, proxies=proxies)
+
+ if msg is None:
+ return StreamEnd()
+
+ if msg.status_code != 200:
+ return msg
+
+ if json:
+ msg = msg.json()
+
+ if field:
+ msg = msg[field]
+
+ if wrap:
+ msg = [msg]
+
+ return msg
+
+ super().__init__(foo=_send, name='Http', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
diff --git a/tributary/streaming/output/kafka.py b/tributary/streaming/output/kafka.py
new file mode 100644
index 0000000..07f081b
--- /dev/null
+++ b/tributary/streaming/output/kafka.py
@@ -0,0 +1,41 @@
+import json as JSON
+from confluent_kafka import Producer
+from ..base import Node
+
+
+class Kafka(Node):
+ '''Connect to kafka server and send data
+
+ Args:
+ foo (callable): input stream
+ foo_kwargs (dict): kwargs for the input stream
+ servers (list): kafka bootstrap servers
+ group (str): kafka group id
+ topics (list): list of kafka topics to connect to
+ json (bool): load input data as json
+ wrap (bool): wrap result in a list
+ interval (int): kafka poll interval
+ '''
+ def __init__(self, node, servers='', topic='', json=False, wrap=False):
+ p = Producer({'bootstrap.servers': servers})
+
+ async def _send(data, producer=p, topic=topic, json=json, wrap=wrap):
+ ret = []
+ # Trigger any available delivery report callbacks from previous produce() calls
+ producer.poll(0)
+
+ if wrap:
+ data = [data]
+
+ if json:
+ data = JSON.dumps(data)
+
+ producer.produce(topic, data.encode('utf-8'), callback=lambda *args: ret.append(args))
+
+ for data in ret:
+ yield data
+ ret = []
+
+ super().__init__(foo=_send, name='Kafka', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
diff --git a/tributary/streaming/output/output.py b/tributary/streaming/output/output.py
new file mode 100644
index 0000000..0a338b1
--- /dev/null
+++ b/tributary/streaming/output/output.py
@@ -0,0 +1,79 @@
+from IPython.display import display
+from ..base import Node
+
+
+class Print(Node):
+ def __init__(self, node, text=''):
+ def foo(val):
+ print(text + str(val))
+ return val
+ super().__init__(foo=foo, foo_kwargs=None, name='Print', inputs=1)
+
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+
+def Graph(node):
+ if not node._upstream:
+ # leaf node
+ return {node: []}
+ return {node: [_.graph() for _ in node._upstream]}
+
+
+def PPrint(node, level=0):
+ ret = ' ' * (level - 1) if level else ''
+
+ if not node._upstream:
+ # leaf node
+ return ret + ' \\ ' + repr(node)
+ return ' ' * level + repr(node) + '\n' + '\n'.join(_.pprint(level + 1) for _ in node._upstream)
+
+
+def GraphViz(node):
+ d = Graph(node)
+
+ from graphviz import Digraph
+ dot = Digraph("Graph", strict=False)
+ dot.format = 'png'
+
+ def rec(nodes, parent):
+ for d in nodes:
+ if not isinstance(d, dict):
+ dot.node(d)
+ dot.edge(d, parent)
+
+ else:
+ for k in d:
+ dot.node(k._name)
+ rec(d[k], k)
+ dot.edge(k._name, parent._name)
+
+ for k in d:
+ dot.node(k._name)
+ rec(d[k], k)
+
+ return dot
+
+
+class Perspective(Node):
+ def __init__(self, node, text='', psp_kwargs=None):
+ psp_kwargs = psp_kwargs or {}
+ from perspective import PerspectiveWidget
+ p = PerspectiveWidget(psp_kwargs.pop('schema', []), **psp_kwargs)
+
+ def foo(val):
+ p.update(val)
+ return val
+ super().__init__(foo=foo, foo_kwargs=None, name='Print', inputs=1)
+
+ display(p)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+ self._name = "Perspective"
+
+
+Node.graph = Graph
+Node.pprint = PPrint
+Node.graphviz = GraphViz
+Node.print = Print
+Node.perspective = Perspective
diff --git a/tributary/tests/reactive/input/test_kafka_reactive_input.py b/tributary/streaming/output/socketio.py
similarity index 100%
rename from tributary/tests/reactive/input/test_kafka_reactive_input.py
rename to tributary/streaming/output/socketio.py
diff --git a/tributary/streaming/output/ws.py b/tributary/streaming/output/ws.py
new file mode 100644
index 0000000..cfa6485
--- /dev/null
+++ b/tributary/streaming/output/ws.py
@@ -0,0 +1,50 @@
+import json as JSON
+import websockets
+from ..base import Node
+from ...base import StreamNone, StreamEnd
+
+
+class WebSocket(Node):
+ '''Connect to websocket and send data
+
+ Args:
+ foo (callable): input stream
+ foo_kwargs (dict): kwargs for the input stream
+ url (str): websocket url to connect to
+ json (bool): dump data as json
+ wrap (bool): wrap result in a list
+ '''
+ def __init__(self, node, url='', json=False, wrap=False, field=None, response=False):
+ self._websocket = websockets.connect(url)
+
+ async def _send(data, url=url, json=json, wrap=wrap, field=field, response=response):
+ if isinstance(data, (StreamNone, StreamEnd)):
+ return data
+
+ if wrap:
+ data = [data]
+ if json:
+ data = JSON.dumps(data)
+
+ await self._websocket.send(data)
+
+ if response:
+ msg = await self._websocket.recv()
+
+ else:
+ msg = '{}'
+
+ if json:
+ msg = JSON.loads(msg)
+
+ if field:
+ msg = msg[field]
+
+ if wrap:
+ msg = [msg]
+
+ return msg
+
+ super().__init__(foo=_send, name='WebSocket', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
diff --git a/tributary/tests/reactive/input/test_socketio_reactive_input.py b/tributary/streaming/utils.py
similarity index 100%
rename from tributary/tests/reactive/input/test_socketio_reactive_input.py
rename to tributary/streaming/utils.py
diff --git a/tributary/symbolic/__init__.py b/tributary/symbolic/__init__.py
index 8d791c0..4dbe5e6 100644
--- a/tributary/symbolic/__init__.py
+++ b/tributary/symbolic/__init__.py
@@ -51,13 +51,13 @@ def construct_lazy(expr, modules=None):
expr (sympy expression): A Sympy expression
modules (list): a list of modules to use for sympy's lambdify function
Returns:
- tributary.lazy.BaseGraph
+ tributary.lazy.LazyGraph
'''
syms = list(symbols(expr))
names = [s.name for s in syms]
modules = modules or ["scipy", "numpy"]
- class Lazy(tl.BaseGraph):
+ class Lazy(tl.LazyGraph):
def __init__(self, **kwargs):
for n in names:
setattr(self, n, self.node(name=n, value=kwargs.get(n, None)))
@@ -86,7 +86,7 @@ def construct_streaming(expr, modules=None):
names = [s.name for s in syms]
modules = modules or ["scipy", "numpy"]
- class Streaming(tr.BaseGraph):
+ class Streaming(tr.StreamingGraph):
def __init__(self, **kwargs):
self._kwargs = {}
for n in names:
@@ -96,9 +96,9 @@ def __init__(self, **kwargs):
self._kwargs[n] = kwargs.get(n)
self._nodes = [getattr(self, n) for n in names]
- self._function = tr.Foo(lambdify(syms, expr, modules=modules)(**self._kwargs))
+# self._function = tr.Foo(lambdify(syms, expr, modules=modules)(**self._kwargs))
self._expr = expr
- super(Streaming, self).__init__(self._function)
+ super(Streaming, self).__init__(self._foo)
return Streaming
diff --git a/tributary/tests/functional/test_functional.py b/tributary/tests/functional/test_functional.py
index 3e1a858..dfc17d5 100644
--- a/tributary/tests/functional/test_functional.py
+++ b/tributary/tests/functional/test_functional.py
@@ -12,14 +12,14 @@ def test_general(self):
def foo1(on_data):
x = 0
- while x < 100:
+ while x < 5:
on_data({'a': random.random(), 'b': random.randint(0, 1000), 'x': x})
- time.sleep(.1)
- x = x+1
+ time.sleep(.01)
+ x = x + 1
def foo2(data, callback):
callback([{'a': data['a'] * 1000, 'b': data['b'], 'c': 'AAPL', 'x': data['x']}])
t.pipeline([foo1, foo2], ['on_data', 'callback'], on_data=lambda x: None)
- time.sleep(10)
+ time.sleep(1)
t.stop()
diff --git a/tributary/tests/helpers/dummy_ws.py b/tributary/tests/helpers/dummy_ws.py
index 7810d3a..5bf7e27 100644
--- a/tributary/tests/helpers/dummy_ws.py
+++ b/tributary/tests/helpers/dummy_ws.py
@@ -35,5 +35,6 @@ def main():
print('listening on %d' % 8899)
tornado.ioloop.IOLoop.current().start()
+
if __name__ == '__main__':
main()
diff --git a/tributary/tests/lazy/calculations/test_ops_lazy.py b/tributary/tests/lazy/calculations/test_ops_lazy.py
index 74a5ceb..e810dee 100644
--- a/tributary/tests/lazy/calculations/test_ops_lazy.py
+++ b/tributary/tests/lazy/calculations/test_ops_lazy.py
@@ -1,11 +1,12 @@
import tributary.lazy as t
-class Foo1(t.BaseGraph):
+
+class Foo1(t.LazyGraph):
def __init__(self, *args, **kwargs):
self.x = self.node('x', readonly=False, value=1, trace=True)
-class Foo2(t.BaseGraph):
+class Foo2(t.LazyGraph):
def __init__(self, *args, **kwargs):
self.y = self.node('y', readonly=False, value=2, trace=True)
@@ -14,7 +15,7 @@ def __init__(self, *args, **kwargs):
self.x = self.node('x', readonly=False, value=2, trace=True)
-class Foo3(t.BaseGraph):
+class Foo3(t.LazyGraph):
@t.node()
def z(self):
return self.x | self.y()
@@ -56,15 +57,14 @@ def test_multi(self):
def test_or(self):
f = Foo3()
assert f.z()() == 10
- assert f.x() == None
+ assert f.x() is None
f.x = 5
assert f.x() == 5
assert f.z()() == 5
-
f.reset()
- assert f.x() == None
+ assert f.x() is None
assert f.z()() == 10
diff --git a/tributary/tests/lazy/test_lazy.py b/tributary/tests/lazy/test_lazy.py
index 1b6cdcd..d184272 100644
--- a/tributary/tests/lazy/test_lazy.py
+++ b/tributary/tests/lazy/test_lazy.py
@@ -2,12 +2,12 @@
import random
-class Foo1(t.BaseGraph):
+class Foo1(t.LazyGraph):
def __init__(self, *args, **kwargs):
self.x = self.node('x', readonly=False, value=1, trace=True)
-class Foo2(t.BaseGraph):
+class Foo2(t.LazyGraph):
def __init__(self, *args, **kwargs):
self.y = self.node('y', readonly=False, value=2, trace=True)
@@ -16,7 +16,7 @@ def __init__(self, *args, **kwargs):
self.x = self.node('x', readonly=False, value=2, trace=True)
-class Foo3(t.BaseGraph):
+class Foo3(t.LazyGraph):
@t.node(trace=True)
def foo1(self):
return self.random() # test self access
@@ -28,7 +28,8 @@ def random(self):
def foo3(self, x=4):
return 3 + x
-class Foo4(t.BaseGraph):
+
+class Foo4(t.LazyGraph):
@t.node(trace=True)
def foo1(self):
return self.foo2() + 1
@@ -38,7 +39,7 @@ def foo2(self):
return random.random()
-class Foo5(t.BaseGraph):
+class Foo5(t.LazyGraph):
@t.node()
def z(self):
return self.x | self.y()
@@ -111,14 +112,13 @@ def test_misc(self):
assert z.print()
assert z.graph()
assert z.graphviz()
- assert z.networkx()
class TestDirtyPropogation:
def test_or_dirtypropogation(self):
f = Foo5()
assert f.z()() == 10
- assert f.x() == None
+ assert f.x() is None
f.x = 5
@@ -127,13 +127,13 @@ def test_or_dirtypropogation(self):
f.reset()
- assert f.x() == None
+ assert f.x() is None
assert f.z()() == 10
class TestDeclarative:
def test_simple_declarative(self):
- n = t.BaseNode(value=1)
+ n = t.Node(value=1)
z = n + 5
assert z() == 6
n.setValue(2)
diff --git a/tributary/tests/reactive/calculations/test_ops_reactive.py b/tributary/tests/reactive/calculations/test_ops_reactive.py
deleted file mode 100644
index 9da0a9a..0000000
--- a/tributary/tests/reactive/calculations/test_ops_reactive.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import tributary as t
-
-
-class TestOps:
- def test_unary(self):
- def foo():
- return True
-
- unary = t.unary(lambda x: not x, foo)
- out = t.run(unary)
- assert len(out) == 1
- assert out[-1] == False
-
- def test_bin(self):
- def foo1():
- return 1
-
- def foo2():
- return 1
-
- bin = t.bin(lambda x, y: x + y, foo1, foo2)
- out = t.run(bin)
- assert len(out) == 1
- assert out[-1] == 2
-
- def test_noop(self):
- def foo():
- return 5
-
- noop = t.Noop(foo)
- out = t.run(noop)
- assert len(out) == 1
- assert out[-1] == 5
-
- def test_negate(self):
- def foo():
- return 5
-
- neg = t.Negate(foo)
- out = t.run(neg)
- assert len(out) == 1
- assert out[-1] == -5
-
- def test_invert(self):
- def foo():
- return 5
-
- inv = t.Invert(foo)
- out = t.run(inv)
- assert len(out) == 1
- assert out[-1] == 1/5
-
- def test_not(self):
- def foo():
- return True
-
- inv = not t.Foo(foo)
- out = t.run(inv)
- assert len(out) == 1
- assert out[-1] == False
-
- def test_add(self):
- def foo1():
- return 1
-
- def foo2():
- return 1
-
- add = t.Add(foo1, t.Foo(foo2))
- out = t.run(add)
- assert len(out) == 1
- assert out[-1] == 2
-
- def test_sub(self):
- def foo1():
- return 1
-
- def foo2():
- return 1
-
- sub = t.Sub(foo1, foo2)
- out = t.run(sub)
- assert len(out) == 1
- assert out[-1] == 0
-
- def test_mult(self):
- def foo1():
- return 2
-
- def foo2():
- return 3
-
- mult = t.Foo(foo1) * foo2
- out = t.run(mult)
- assert len(out) == 1
- assert out[-1] == 6
-
- def test_div(self):
- def foo1():
- return 3
-
- def foo2():
- return 2
-
- div = foo1 / t.Foo(foo2)
- out = t.run(div)
- assert len(out) == 1
- assert out[-1] == 1.5
-
- def test_mod(self):
- def foo1():
- return 3
-
- def foo2():
- return 2
-
- mod = t.Mod(foo1, foo2)
- out = t.run(mod)
- assert len(out) == 1
- assert out[-1] == 1
-
- def test_pow(self):
- def foo1():
- return 3
-
- def foo2():
- return 2
-
- pow = t.Pow(foo1, t.Foo(foo2))
- out = t.run(pow)
- assert len(out) == 1
- assert out[-1] == 9
-
- def test_and(self):
- def foo1():
- return True
-
- def foo2():
- return False
-
- out = t.run(t.Foo(foo1) and t.Foo(foo2))
- assert len(out) == 1
- assert out[-1] == False
-
- def test_or(self):
- def foo1():
- return True
-
- def foo2():
- return False
-
- out = t.run(foo1 or t.Foo(foo2))
- assert len(out) == 1
- assert out[-1] == True
-
- def test_equal(self):
- def foo1():
- return True
-
- def foo2():
- return True
-
- out = t.run(t.Foo(foo1) == foo2)
- assert len(out) == 1
- print(out)
- assert out[-1] == True
-
- def test_less(self):
- def foo1():
- return 2
-
- def foo2():
- return 3
-
- f = t.Foo(foo2)
- out = t.run(foo1 < f)
- assert len(out) == 1
- assert out[-1] == True
-
- def test_more(self):
- def foo1():
- return 2
-
- def foo2():
- return 3
-
- out = t.run(foo1 > t.Foo(foo2))
- assert len(out) == 1
- assert out[-1] == False
diff --git a/tributary/tests/reactive/calculations/test_rolling_reactive.py b/tributary/tests/reactive/calculations/test_rolling_reactive.py
deleted file mode 100644
index f449dfd..0000000
--- a/tributary/tests/reactive/calculations/test_rolling_reactive.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import tributary as t
-
-
-class TestRolling:
- def test_count(self):
- def foo():
- yield 1
- yield 2
- yield 3
-
- out = t.run(t.Count(foo))
- assert len(out) == 3
- assert out[-1] == 3
- assert out[-2] == 2
- assert out[-3] == 1
-
- def test_sub(self):
- def foo():
- yield 1
- yield 2
- yield 3
-
- out = t.run(t.Sum(foo))
- assert len(out) == 3
- assert out[-1] == 6
- assert out[-2] == 3
- assert out[-3] == 1
diff --git a/tributary/tests/reactive/input/test_file_reactive_input.py b/tributary/tests/reactive/input/test_file_reactive_input.py
deleted file mode 100644
index 620e6b0..0000000
--- a/tributary/tests/reactive/input/test_file_reactive_input.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import tributary as t
-import os
-import os.path
-
-
-class TestFile:
- def test_file(self):
- file = os.path.join(os.path.dirname(__file__),
- '..', '..', 'test_data',
- 'ohlc.csv')
- out = t.run(t.File(file, json=False))
- assert len(out) == 50
diff --git a/tributary/tests/reactive/input/test_http_reactive_input.py b/tributary/tests/reactive/input/test_http_reactive_input.py
deleted file mode 100644
index da572f0..0000000
--- a/tributary/tests/reactive/input/test_http_reactive_input.py
+++ /dev/null
@@ -1,7 +0,0 @@
-
-
-class TestFile:
- def test_file(self):
- import tributary as t
- out = t.run(t.HTTP('http://tim.paine.nyc', json=False))
- assert len(out) > 0
diff --git a/tributary/tests/reactive/input/test_input_reactive_input.py b/tributary/tests/reactive/input/test_input_reactive_input.py
deleted file mode 100644
index a24112a..0000000
--- a/tributary/tests/reactive/input/test_input_reactive_input.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import tributary as t
-
-
-class TestInput:
- def test_file(self):
- out = t.run(t.Random(5))
- assert len(out) == 5
diff --git a/tributary/tests/reactive/output/test_file_reactive_output.py b/tributary/tests/reactive/output/test_file_reactive_output.py
deleted file mode 100644
index 4bbb5d3..0000000
--- a/tributary/tests/reactive/output/test_file_reactive_output.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import tempfile
-import tributary as t
-
-
-class TestFile:
- def test_file(self):
- with tempfile.NamedTemporaryFile() as f:
- out = t.run(t.FileSink(t.Const('test'), filename=f.name, json=False))
- assert len(out) == 1
diff --git a/tributary/tests/reactive/output/test_http_reactive_output.py b/tributary/tests/reactive/output/test_http_reactive_output.py
deleted file mode 100644
index ca06c4c..0000000
--- a/tributary/tests/reactive/output/test_http_reactive_output.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import tributary as t
-
-
-class TestFile:
- def test_file(self):
- def foo():
- return 'test'
- out = t.run(t.HTTPSink(foo, url='http://tim.paine.nyc', json=True))
- assert len(out) > 0
diff --git a/tributary/tests/reactive/output/test_output_reactive_output.py b/tributary/tests/reactive/output/test_output_reactive_output.py
deleted file mode 100644
index 1d6341a..0000000
--- a/tributary/tests/reactive/output/test_output_reactive_output.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import tributary as t
-
-
-class TestOutput:
- def test_print(self):
- out = t.run(t.Print(t.Random(5)))
- assert len(out) == 5
-
- def test_pprint(self):
- t.PPrint(t.Random(5))
-
- def test_perspective(self):
- try:
- out = t.run(t.Perspective(t.Random(5)))
- assert len(out) == 5
- except:
- pass
diff --git a/tributary/tests/reactive/test_reactive.py b/tributary/tests/reactive/test_reactive.py
deleted file mode 100644
index 8bd169e..0000000
--- a/tributary/tests/reactive/test_reactive.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import tributary as t
-import random
-import time
-
-
-class TestReactive:
- def test_1(self):
- def foo():
- return random.random()
-
- print('''
- ******************************
- * test *
- ******************************
- ''')
- test = t.Timer(foo, {}, 0, 5)
- test2 = t.Negate(test)
- res2 = test + test2
- p2 = t.Print(res2)
- t.run(p2)
-
- def test_2(self):
- print('''
- ******************************
- * test2 *
- ******************************
- ''')
-
- def foo():
- return random.random()
-
- def long():
- print('long called')
- time.sleep(.1)
- return 5
-
- rand = t.Timer(foo, interval=0, repeat=5)
- five = t.Timer(long, interval=0, repeat=5)
- one = t.Timer(1, interval=0, repeat=5)
- five2 = t.Timer(5, interval=0, repeat=5)
-
- neg_rand = t.Negate(rand)
-
- x1 = rand + five # 5 + rand
- x2 = x1 - five2 # rand
- x3 = x2 + neg_rand # 0
- res2 = x3 + one # 1
- p2 = t.Print(res2) # 1
-
- t.PPrint(p2)
- t.run(p2)
-
- def test_3(self):
- print('''
- ******************************
- * test3 *
- ******************************
- ''')
-
- def stream():
- for i in range(10):
- yield i
-
- f = t.Foo(stream)
- sum = t.Sum(f)
- count = t.Count(f)
- f3 = sum / count
- p3 = t.Print(f3)
-
- t.PPrint(p3)
- t.run(p3)
diff --git a/tributary/tests/reactive/test_utils_reactive.py b/tributary/tests/reactive/test_utils_reactive.py
deleted file mode 100644
index 4d96389..0000000
--- a/tributary/tests/reactive/test_utils_reactive.py
+++ /dev/null
@@ -1,255 +0,0 @@
-import tributary as t
-from datetime import datetime, timedelta
-
-
-class TestReactiveUtils:
- def test_timer(self):
- def foo():
- return 1
-
- timer = t.Timer(foo, {}, 1, 2)
- first = datetime.now()
- out = t.run(timer)
- last = datetime.now()
- print(last-first)
- assert last - first < timedelta(seconds=3)
- assert last - first > timedelta(seconds=2)
- assert len(out) == 2
-
- def test_delay(self):
- def foo():
- return 1
-
- delay = t.Delay(foo, delay=1)
- first = datetime.now()
- out = t.run(delay)
- last = datetime.now()
- assert last - first < timedelta(seconds=2)
- assert last - first > timedelta(seconds=1)
- assert len(out) == 1
-
- def test_state(self):
- def stream(state):
- for i in range(10):
- yield i + state.val
-
- f = t.Foo(t.State(stream, val=5))
- out = t.run(f)
- assert len(out) == 10
- assert out[-1] == 14
- assert out[-2] == 13
- assert out[0] == 5
-
- def test_apply(self):
- def myfoo(state, data):
- state.count = state.count + 1
- return data + state.count
-
- f = t.Apply(t.State(myfoo, count=0), t.Const(1))
- out = t.run(f)
- assert len(out) == 1
-
- def test_window(self):
- def ran():
- for i in range(10):
- yield i
-
- w = t.Window(ran, size=3, full_only=True)
- out = t.run(w)
- assert len(out) == 8
- assert out[-1] == [7, 8, 9]
-
- def test_unroll(self):
- def ran():
- return [1, 2, 3]
-
- w = t.Unroll(ran)
- out = t.run(w)
- assert len(out) == 3
- assert out[-1] == 3
-
- def test_unrollDF(self):
- import pandas as pd
- df = pd.DataFrame(pd.util.testing.makeTimeSeries())
-
- def ran():
- return df
-
- w = t.UnrollDataFrame(ran)
- out = t.run(w)
- assert len(out) == 30
-
- def test_merge1(self):
- def foo1():
- return 1
-
- def foo2():
- return 2
-
- m = t.Merge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_merge2(self):
- def foo1():
- yield 1
-
- def foo2():
- return 2
-
- m = t.Merge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_merge3(self):
- def foo1():
- return 1
-
- def foo2():
- yield 2
-
- m = t.Merge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_merge4(self):
- def foo1():
- yield 1
-
- def foo2():
- yield 2
-
- m = t.Merge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_list_merge1(self):
- def foo1():
- return [1]
-
- def foo2():
- return [2]
-
- m = t.ListMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_list_merge2(self):
- def foo1():
- yield [1]
-
- def foo2():
- return [2]
-
- m = t.ListMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_list_merge3(self):
- def foo1():
- return [1]
-
- def foo2():
- yield [2]
-
- m = t.ListMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_list_merge4(self):
- def foo1():
- yield [1]
-
- def foo2():
- yield [2]
-
- m = t.ListMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == [1, 2]
-
- def test_dict_merge1(self):
- def foo1():
- return {'a': 1}
-
- def foo2():
- return {'b': 2}
-
- m = t.DictMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == {'a': 1, 'b': 2}
-
- def test_dict_merge2(self):
- def foo1():
- yield {'a': 1}
-
- def foo2():
- return {'b': 2}
-
- m = t.DictMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == {'a': 1, 'b': 2}
-
- def test_dict_merge3(self):
- def foo1():
- return {'a': 1}
-
- def foo2():
- yield {'b': 2}
-
- m = t.DictMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == {'a': 1, 'b': 2}
-
- def test_dict_merge4(self):
- def foo1():
- yield {'a': 1}
-
- def foo2():
- yield {'b': 2}
-
- m = t.DictMerge(foo1, foo2)
- out = t.run(m)
- assert len(out) == 1
- assert out[-1] == {'a': 1, 'b': 2}
-
- def test_reduce(self):
- def foo1():
- return {'b': 2}
-
- def foo2():
- yield {'c': 3}
-
- out = t.run(t.Reduce({'a': 1}, foo1, foo2))
- assert len(out) == 1
- assert out[-1] == [{'a': 1}, {'b': 2}, {'c': 3}]
-
- def test_ensure_count(self):
- import tributary.reactive as tr
-
- def myfoo(state, data):
- state.count = state.count + 1
- return data + state.count
-
- def myfoo2(data):
- return data + 1
-
- out1 = tr.run(tr.Const(1))
- out2 = tr.run(tr.Apply(myfoo2, tr.Const(1)))
- out3 = tr.run(tr.Apply(tr.State(myfoo, count=0), tr.Const(1)))
- print(out1)
- print(out2)
- print(out3)
- assert len(out1) == 1
- assert len(out2) == 1
- assert len(out3) == 1
diff --git a/tributary/tests/reactive/input/test_ws_reactive_input.py b/tributary/tests/streaming/__init__.py
similarity index 100%
rename from tributary/tests/reactive/input/test_ws_reactive_input.py
rename to tributary/tests/streaming/__init__.py
diff --git a/tributary/tests/reactive/output/test_kafka_reactive_output.py b/tributary/tests/streaming/calculations/__init__.py
similarity index 100%
rename from tributary/tests/reactive/output/test_kafka_reactive_output.py
rename to tributary/tests/streaming/calculations/__init__.py
diff --git a/tributary/tests/reactive/output/test_socketio_reactive_output.py b/tributary/tests/streaming/calculations/test_calculations_streaming.py
similarity index 100%
rename from tributary/tests/reactive/output/test_socketio_reactive_output.py
rename to tributary/tests/streaming/calculations/test_calculations_streaming.py
diff --git a/tributary/tests/streaming/calculations/test_ops_streaming.py b/tributary/tests/streaming/calculations/test_ops_streaming.py
new file mode 100644
index 0000000..30bfd7c
--- /dev/null
+++ b/tributary/tests/streaming/calculations/test_ops_streaming.py
@@ -0,0 +1,214 @@
+import math
+import tributary.streaming as ts
+
+
+def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+ yield 5
+
+
+def foo2():
+ yield 1
+ yield 4
+ yield 9
+
+
+def foo3():
+ yield -1
+ yield -2
+
+
+def foo4():
+ yield [1]
+ yield [1, 2]
+ yield [1, 2, 3]
+
+
+class TestOps:
+ def test_Noop(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Noop(t)
+ assert ts.run(out) == [1, 2]
+
+ def test_Negate(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Negate(t)
+ assert ts.run(out) == [-1, -2]
+
+ def test_Invert(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Invert(t)
+ assert ts.run(out) == [1 / 1, 1 / 2]
+
+ def test_Add(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Add(t, t)
+ assert ts.run(out) == [2, 4]
+
+ def test_Sub(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Sub(t, t)
+ assert ts.run(out) == [0, 0]
+
+ def test_Mult(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Mult(t, t)
+ assert ts.run(out) == [1, 4]
+
+ def test_Div(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Div(t, t)
+ assert ts.run(out) == [1, 1]
+
+ def test_RDiv(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.RDiv(t, t)
+ assert ts.run(out) == [1, 1]
+
+ def test_Mod(self):
+ t = ts.Timer(foo, count=5)
+ c = ts.Const(3)
+ out = ts.Mod(t, c)
+ assert ts.run(out) == [1, 2, 0, 1, 2]
+
+ def test_Pow(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(2)
+ out = ts.Pow(t, c)
+ assert ts.run(out) == [1, 4]
+
+ def test_Not(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Not(t)
+ assert ts.run(out) == [False, False]
+
+ def test_And(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.And(t, t)
+ assert ts.run(out) == [1, 2]
+
+ def test_Or(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Or(t, t)
+ assert ts.run(out) == [1, 2]
+
+ def test_Equal(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(1)
+ out = ts.Equal(t, c)
+ assert ts.run(out) == [True, False]
+
+ def test_NotEqual(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(1)
+ out = ts.NotEqual(t, c)
+ assert ts.run(out) == [False, True]
+
+ def test_Lt(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(1)
+ out = ts.Lt(c, t)
+ assert ts.run(out) == [False, True]
+
+ def test_Le(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(2)
+ out = ts.Le(c, t)
+ assert ts.run(out) == [False, True]
+
+ def test_Gt(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(1)
+ out = ts.Gt(t, c)
+ assert ts.run(out) == [False, True]
+
+ def test_Ge(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(2)
+ out = ts.Ge(t, c)
+ assert ts.run(out) == [False, True]
+
+ def test_Log(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Log(t)
+ assert ts.run(out) == [math.log(1), math.log(2)]
+
+ def test_Sin(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Sin(t)
+ assert ts.run(out) == [math.sin(1), math.sin(2)]
+
+ def test_Cos(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Cos(t)
+ assert ts.run(out) == [math.cos(1), math.cos(2)]
+
+ def test_Tan(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Tan(t)
+ assert ts.run(out) == [math.tan(1), math.tan(2)]
+
+ def test_Arcsin(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(value=3)
+ out = ts.Arcsin(ts.Div(t, c))
+ assert ts.run(out) == [math.asin(1 / 3), math.asin(2 / 3)]
+
+ def test_Arccos(self):
+ t = ts.Timer(foo, count=2)
+ c = ts.Const(value=3)
+ out = ts.Arccos(ts.Div(t, c))
+ assert ts.run(out) == [math.acos(1 / 3), math.acos(2 / 3)]
+
+ def test_Arctan(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Arctan(t)
+ assert ts.run(out) == [math.atan(1), math.atan(2)]
+
+ def test_Sqrt(self):
+ t = ts.Timer(foo2, count=3)
+ out = ts.Sqrt(t)
+ assert ts.run(out) == [1.0, 2.0, 3.0]
+
+ def test_Abs(self):
+ t = ts.Timer(foo3, count=2)
+ out = ts.Abs(t)
+ assert ts.run(out) == [1, 2]
+
+ def test_Exp(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Exp(t)
+ assert ts.run(out) == [math.exp(1), math.exp(2)]
+
+ def test_Erf(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Erf(t)
+ assert ts.run(out) == [math.erf(1), math.erf(2)]
+
+ def test_Int(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Int(t)
+ assert ts.run(out) == [1, 2]
+
+ def test_Float(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Float(t)
+ assert ts.run(out) == [1.0, 2.0]
+
+ def test_Bool(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Bool(t)
+ assert ts.run(out) == [True, True]
+
+ def test_Str(self):
+ t = ts.Timer(foo, count=2)
+ out = ts.Str(t)
+ assert ts.run(out) == ['1', '2']
+
+ def test_Len(self):
+ t = ts.Timer(foo4, count=3)
+ out = ts.Len(t)
+ assert ts.run(out) == [1, 2, 3]
diff --git a/tributary/tests/reactive/output/test_ws_reactive_output.py b/tributary/tests/streaming/input/__init__.py
similarity index 100%
rename from tributary/tests/reactive/output/test_ws_reactive_output.py
rename to tributary/tests/streaming/input/__init__.py
diff --git a/tributary/tests/streaming/input/test_file_data.csv b/tributary/tests/streaming/input/test_file_data.csv
new file mode 100644
index 0000000..b178657
--- /dev/null
+++ b/tributary/tests/streaming/input/test_file_data.csv
@@ -0,0 +1,4 @@
+1
+2
+3
+4
\ No newline at end of file
diff --git a/tributary/tests/streaming/input/test_file_streaming.py b/tributary/tests/streaming/input/test_file_streaming.py
new file mode 100644
index 0000000..b20a462
--- /dev/null
+++ b/tributary/tests/streaming/input/test_file_streaming.py
@@ -0,0 +1,11 @@
+import os
+import os.path
+import tributary.streaming as ts
+
+
+class TestFile:
+ def test_file(self):
+ file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'test_file_data.csv'))
+
+ out = ts.Print(ts.File(filename=file))
+ assert ts.run(out) == [1, 2, 3, 4]
diff --git a/tributary/tests/streaming/input/test_http_streaming.py b/tributary/tests/streaming/input/test_http_streaming.py
new file mode 100644
index 0000000..82a067d
--- /dev/null
+++ b/tributary/tests/streaming/input/test_http_streaming.py
@@ -0,0 +1,11 @@
+import os
+import os.path
+import tributary.streaming as ts
+
+
+class TestHttp:
+ def test_http(self):
+ out = ts.Print(ts.HTTP(url='https://google.com'))
+ ret = ts.run(out)
+ print(ret)
+ assert len(ret) == 1
diff --git a/tributary/tests/streaming/input/test_input_streaming.py b/tributary/tests/streaming/input/test_input_streaming.py
new file mode 100644
index 0000000..6251349
--- /dev/null
+++ b/tributary/tests/streaming/input/test_input_streaming.py
@@ -0,0 +1,118 @@
+import tributary.streaming as ts
+
+
+class TestConst:
+ def test_const_1(self):
+ t = ts.Const(value=1, count=1)
+ assert ts.run(t) == [1]
+
+ def test_const_2(self):
+ t = ts.Const(value=1, count=5)
+ assert ts.run(t) == [1, 1, 1, 1, 1]
+
+
+class TestTimer:
+ def test_timer(self):
+ val = 0
+
+ def foo():
+ nonlocal val
+ val += 1
+ return val
+
+ t = ts.Timer(foo, count=5)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=5)
+
+ def test_timer_delay(self):
+ val = 0
+
+ def foo():
+ nonlocal val
+ val += 1
+ return val
+
+ t = ts.Timer(foo, count=5, interval=0.1)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=5)
+
+ def test_timer_generator(self):
+ def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+ yield 5
+
+ t = ts.Timer(foo)
+ assert ts.run(t) == [1]
+
+ t = ts.Timer(foo, count=3)
+ assert ts.run(t) == [1, 2, 3]
+
+ t = ts.Timer(foo, count=5)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=6)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ def test_timer_generator_delay(self):
+ def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+ yield 5
+
+ t = ts.Timer(foo, interval=0.1)
+ assert ts.run(t) == [1]
+
+ t = ts.Timer(foo, count=3, interval=0.1)
+ assert ts.run(t) == [1, 2, 3]
+
+ t = ts.Timer(foo, count=5, interval=0.1)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=6, interval=0.1)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+
+class TestFoo:
+ def test_timer(self):
+ val = 0
+
+ def foo():
+ nonlocal val
+ val += 1
+ return val
+
+ t = ts.Timer(foo, count=5)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=5)
+
+ def test_timer_delay(self):
+ val = 0
+
+ def foo():
+ nonlocal val
+ val += 1
+ return val
+
+ t = ts.Timer(foo, count=5, interval=0.1)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
+
+ t = ts.Timer(foo, count=5)
+
+ def test_foo_generator(self):
+ def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+ yield 5
+
+ t = ts.Foo(foo)
+ assert ts.run(t) == [1, 2, 3, 4, 5]
diff --git a/tributary/tests/reactive/test_base_reactive.py b/tributary/tests/streaming/output/__init__.py
similarity index 100%
rename from tributary/tests/reactive/test_base_reactive.py
rename to tributary/tests/streaming/output/__init__.py
diff --git a/tributary/tests/streaming/output/test_file_streaming.py b/tributary/tests/streaming/output/test_file_streaming.py
new file mode 100644
index 0000000..ce21a20
--- /dev/null
+++ b/tributary/tests/streaming/output/test_file_streaming.py
@@ -0,0 +1,17 @@
+import os
+import os.path
+import tributary.streaming as ts
+
+
+class TestFile:
+ def test_file(self):
+ file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'test_file_data.csv'))
+
+ def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+
+ out = ts.FileSink(ts.Foo(foo), filename=file)
+ assert ts.run(out) == [1, 2, 3, 4]
diff --git a/tributary/tests/streaming/output/test_output_streaming.py b/tributary/tests/streaming/output/test_output_streaming.py
new file mode 100644
index 0000000..e476c83
--- /dev/null
+++ b/tributary/tests/streaming/output/test_output_streaming.py
@@ -0,0 +1,23 @@
+import tributary.streaming as ts
+
+
+class TestOutput:
+ def test_print(self):
+ val = 0
+
+ def foo():
+ nonlocal val
+ val += 1
+ return val
+
+ assert ts.run(ts.Print(ts.Timer(foo, count=2))) == [1, 2]
+
+ def test_print_generator(self):
+ def foo():
+ yield 1
+ yield 2
+ yield 3
+ yield 4
+ yield 5
+
+ assert ts.run(ts.Print(ts.Timer(foo, count=2))) == [1, 2]
diff --git a/tributary/tests/streaming/test_streaming.py b/tributary/tests/streaming/test_streaming.py
new file mode 100644
index 0000000..3c023ef
--- /dev/null
+++ b/tributary/tests/streaming/test_streaming.py
@@ -0,0 +1,36 @@
+import asyncio
+import tributary.streaming as ts
+
+
+class TestStreaming:
+ def test_run_simple(self):
+ t = ts.Const(value=1, count=1)
+ assert ts.run(t) == [1]
+
+ def test_run_foo(self):
+ def foo():
+ return 5
+ t = ts.Foo(foo, count=1)
+ assert ts.run(t) == [5]
+
+ def test_run_generator(self):
+ def foo():
+ yield 1
+ yield 2
+ t = ts.Foo(foo)
+ assert ts.run(t) == [1, 2]
+
+ def test_run_async_foo(self):
+ async def foo():
+ await asyncio.sleep(.1)
+ return 5
+
+ t = ts.Foo(foo, count=1)
+ assert ts.run(t) == [5]
+
+ def test_run_async_generator(self):
+ async def foo():
+ yield 1
+ yield 2
+ t = ts.Foo(foo)
+ assert ts.run(t) == [1, 2]
diff --git a/tributary/tests/symbolic/test_symbolic.py b/tributary/tests/symbolic/test_symbolic.py
index 1084ca3..3f704a6 100644
--- a/tributary/tests/symbolic/test_symbolic.py
+++ b/tributary/tests/symbolic/test_symbolic.py
@@ -49,18 +49,18 @@ def test_parse(self):
from sympy.parsing.sympy_parser import parse_expr
assert parse_expr('x^2') == ts.parse_expression('x^2')
- def test_construct_streaming(self):
- import tributary.symbolic as ts
- import tributary.reactive as tr
+ # def test_construct_streaming(self):
+ # import tributary.symbolic as ts
+ # import tributary.reactive as tr
- expr = ts.parse_expression("10sin**2 x**2 + 3xyz + tan theta")
- clz = ts.construct_streaming(expr)
+ # expr = ts.parse_expression("10sin**2 x**2 + 3xyz + tan theta")
+ # clz = ts.construct_streaming(expr)
- def foo(*args):
- for i in range(10):
- yield i
+ # def foo(*args):
+ # for i in range(10):
+ # yield i
- x = clz(x=tr.Const(1), y=tr.Foo(foo), z=tr.Timer(1, repeat=1), theta=tr.Const(4))
- x.pprint()
- out = x.run()
- assert len(out) == 1
+ # x = clz(x=tr.Const(1), y=tr.Foo(foo), z=tr.Timer(1, repeat=1), theta=tr.Const(4))
+ # x.pprint()
+ # out = x.run()
+ # assert len(out) == 1
diff --git a/tributary/tests/test_base.py b/tributary/tests/test_base.py
index afcbaba..7f2064f 100644
--- a/tributary/tests/test_base.py
+++ b/tributary/tests/test_base.py
@@ -6,4 +6,4 @@ def setup(self):
def test_1(self):
from tributary.base import StreamNone
- assert StreamNone('test')
+ assert not StreamNone('test')
From 59747c3355b42ea8bafed066802ae330c02f94e6 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 21:02:59 -0500
Subject: [PATCH 02/15] fix lint
---
tributary/streaming/input/http.py | 1 -
tributary/streaming/input/kafka.py | 2 --
tributary/streaming/input/socketio.py | 3 +--
tributary/streaming/input/ws.py | 2 --
4 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/tributary/streaming/input/http.py b/tributary/streaming/input/http.py
index 4b7e115..1c0324f 100644
--- a/tributary/streaming/input/http.py
+++ b/tributary/streaming/input/http.py
@@ -47,4 +47,3 @@ async def _req(url=url, interval=interval, repeat=repeat, json=json, wrap=wrap,
super().__init__(foo=_req)
self._name = 'Http'
-
diff --git a/tributary/streaming/input/kafka.py b/tributary/streaming/input/kafka.py
index 6514dfd..8e7194e 100644
--- a/tributary/streaming/input/kafka.py
+++ b/tributary/streaming/input/kafka.py
@@ -1,5 +1,4 @@
import json as JSON
-import time
from confluent_kafka import Consumer, KafkaError
from .input import Foo
@@ -54,4 +53,3 @@ async def _listen(consumer=c, json=json, wrap=wrap, interval=interval):
super().__init__(foo=_listen)
self._name = 'Kafka'
-
diff --git a/tributary/streaming/input/socketio.py b/tributary/streaming/input/socketio.py
index 70d6333..be2cd1e 100644
--- a/tributary/streaming/input/socketio.py
+++ b/tributary/streaming/input/socketio.py
@@ -1,5 +1,4 @@
import json as JSON
-import time
from socketIO_client_nexus import SocketIO as SIO
from urllib.parse import urlparse
@@ -33,7 +32,7 @@ async def _sio(url=url, channel=channel, field=field, json=json, wrap=wrap, inte
for msg in _data:
# FIXME clear _data
if json:
- msg = json.loads(msg)
+ msg = JSON.loads(msg)
if field:
msg = msg[field]
diff --git a/tributary/streaming/input/ws.py b/tributary/streaming/input/ws.py
index d197abd..6086798 100644
--- a/tributary/streaming/input/ws.py
+++ b/tributary/streaming/input/ws.py
@@ -1,6 +1,5 @@
import json as JSON
import websockets
-import time
from .input import Foo
from ...base import StreamNone, StreamEnd
@@ -32,4 +31,3 @@ async def _listen(url=url, json=json, wrap=wrap):
super().__init__(foo=_listen)
self._name = 'WebSocket'
-
From 243add00152c45d097f6b3b7c9a5dd161827a38c Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 21:10:26 -0500
Subject: [PATCH 03/15] updates
---
Makefile | 2 +-
azure-pipelines.yml | 6 ------
setup.py | 1 -
3 files changed, 1 insertion(+), 8 deletions(-)
diff --git a/Makefile b/Makefile
index ef7d191..a3b6761 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ buildpy2:
python2 setup.py build
tests: ## Clean and Make unit tests
- python3 -m pytest -v tributary/tests --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
+ python3 -m pytest -v tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
notebooks: ## test execute the notebooks
./scripts/test_notebooks.sh
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 920f23f..6e687ca 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -10,8 +10,6 @@ jobs:
matrix:
Python37:
python.version: '3.7'
- Python38:
- python.version: '3.8'
steps:
- task: UsePythonVersion@0
@@ -51,8 +49,6 @@ jobs:
matrix:
Python37:
python.version: '3.7'
- Python38:
- python.version: '3.8'
steps:
- task: UsePythonVersion@0
@@ -92,8 +88,6 @@ jobs:
matrix:
Python37:
python.version: '3.7'
- Python38:
- python.version: '3.8'
steps:
- task: UsePythonVersion@0
diff --git a/setup.py b/setup.py
index c4daf68..8b24b47 100644
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,6 @@ def get_version(file, name='__version__'):
'socketIO-client-nexus>=0.7.6',
'sympy>=1.3',
'tornado>=5.1.1',
- 'ujson>=1.35',
'websockets>=8.0',
]
From 476f0f11ad24a3c025b2b5e20be3ec7ea584b3fb Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:00:01 -0500
Subject: [PATCH 04/15] starting work on back pressure
---
tributary/base.py | 6 +
tributary/functional/input.py | 2 +-
tributary/reactive/__init__.py | 45 ---
tributary/reactive/base.py | 223 -------------
tributary/reactive/calculations/__init__.py | 2 -
tributary/reactive/calculations/ops.py | 226 -------------
tributary/reactive/calculations/rolling.py | 47 ---
tributary/reactive/input/__init__.py | 45 ---
tributary/reactive/input/file.py | 21 --
tributary/reactive/input/http.py | 52 ---
tributary/reactive/input/kafka.py | 58 ----
tributary/reactive/input/socketio.py | 50 ---
tributary/reactive/input/ws.py | 35 --
tributary/reactive/output/__init__.py | 80 -----
tributary/reactive/output/file.py | 26 --
tributary/reactive/output/http.py | 55 ----
tributary/reactive/output/kafka.py | 47 ---
tributary/reactive/output/socketio.py | 0
tributary/reactive/output/ws.py | 58 ----
tributary/reactive/utils.py | 308 ------------------
tributary/streaming/__init__.py | 10 +-
tributary/streaming/base.py | 32 +-
tributary/streaming/calculations/rolling.py | 29 ++
tributary/streaming/output/output.py | 1 -
tributary/streaming/utils.py | 290 +++++++++++++++++
tributary/symbolic/__init__.py | 47 ++-
.../calculations/test_rolling_streaming.py | 16 +
tributary/tests/streaming/test_utils.py | 40 +++
28 files changed, 440 insertions(+), 1411 deletions(-)
delete mode 100644 tributary/reactive/__init__.py
delete mode 100644 tributary/reactive/base.py
delete mode 100644 tributary/reactive/calculations/__init__.py
delete mode 100644 tributary/reactive/calculations/ops.py
delete mode 100644 tributary/reactive/calculations/rolling.py
delete mode 100644 tributary/reactive/input/__init__.py
delete mode 100644 tributary/reactive/input/file.py
delete mode 100644 tributary/reactive/input/http.py
delete mode 100644 tributary/reactive/input/kafka.py
delete mode 100644 tributary/reactive/input/socketio.py
delete mode 100644 tributary/reactive/input/ws.py
delete mode 100644 tributary/reactive/output/__init__.py
delete mode 100644 tributary/reactive/output/file.py
delete mode 100644 tributary/reactive/output/http.py
delete mode 100644 tributary/reactive/output/kafka.py
delete mode 100644 tributary/reactive/output/socketio.py
delete mode 100644 tributary/reactive/output/ws.py
delete mode 100644 tributary/reactive/utils.py
create mode 100644 tributary/tests/streaming/calculations/test_rolling_streaming.py
create mode 100644 tributary/tests/streaming/test_utils.py
diff --git a/tributary/base.py b/tributary/base.py
index 152709d..4ada850 100644
--- a/tributary/base.py
+++ b/tributary/base.py
@@ -4,6 +4,12 @@ class StreamEnd:
pass
+class StreamRepeat:
+ '''Indicates that a stream has a gap, this object should be ignored
+ and the previous action repeated'''
+ pass
+
+
class StreamNone:
'''indicates that a stream does not have a value'''
diff --git a/tributary/functional/input.py b/tributary/functional/input.py
index 416f0d5..dd09bf2 100644
--- a/tributary/functional/input.py
+++ b/tributary/functional/input.py
@@ -1,7 +1,7 @@
import requests
import time
from confluent_kafka import Consumer, KafkaError
-from ujson import loads as load_json
+from json import loads as load_json
from websocket import create_connection
from socketIO_client_nexus import SocketIO as SIO
try:
diff --git a/tributary/reactive/__init__.py b/tributary/reactive/__init__.py
deleted file mode 100644
index a40a5a9..0000000
--- a/tributary/reactive/__init__.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import asyncio
-from .base import _wrap, Foo, Const, Share, FunctionWrapper # noqa: F401
-from .calculations import * # noqa: F401, F403
-from .input import * # noqa: F401, F403
-from .output import * # noqa: F401, F403
-from .output import PPrint, GraphViz
-from .utils import * # noqa: F401, F403
-
-
-async def _run(foo, **kwargs):
- foo = _wrap(foo, kwargs)
- ret = []
- try:
- async for item in foo():
- ret.append(item)
- except KeyboardInterrupt:
- print('Terminating...')
- return ret
-
-
-def run(foo, **kwargs):
- loop = asyncio.get_event_loop()
- if loop.is_running():
- # return future
- return asyncio.create_task(_run(foo, **kwargs))
- else:
- # block until done
- x = loop.run_until_complete(_run(foo, **kwargs))
- return x
-
-
-class StreamingGraph(object):
- def __init__(self, run=None, *args, **kwargs):
- self._run = run
-
- def run(self, **kwargs):
- if not hasattr(self, "_run") or not self._run:
- raise Exception("Reactive class improperly constructed, did you forget a super()?")
- return run(self._run, **kwargs)
-
- def pprint(self):
- return PPrint(self._run)
-
- def graphviz(self):
- return GraphViz(self._run)
diff --git a/tributary/reactive/base.py b/tributary/reactive/base.py
deleted file mode 100644
index 1de91ac..0000000
--- a/tributary/reactive/base.py
+++ /dev/null
@@ -1,223 +0,0 @@
-import types
-import sys
-from six import iteritems
-DEBUG = False
-
-
-def _wrap(foo, foo_kwargs, name='', wraps=(), share=None, state=None):
- '''wrap a function in a streaming variant
-
- Args:
- foo (callable): function to wrap
- foo_kwargs (dict): kwargs of function
- name (str): name of function to call
- wraps (tuple): functions or FunctionWrappers that this is wrapping
- share:
- state: state context
-
- Returns:
- FunctionWrapper: wrapped function
- '''
- if isinstance(foo, FunctionWrapper):
- ret = foo
- else:
- ret = FunctionWrapper(foo, foo_kwargs, name, wraps, share, state)
-
- for wrap in wraps:
- if isinstance(wrap, FunctionWrapper):
- if wrap._foo == ret._foo:
- continue
- _inc_ref(wrap, ret)
- return ret
-
-
-def _inc_ref(f_wrapped, f_wrapping):
- '''Increment reference count for wrapped f
-
- Args:
- f_wrapped (FunctionWrapper): function that is wrapped
- f_wrapping (FunctionWrapper): function that wants to use f_wrapped
- '''
- if f_wrapped._id == f_wrapping._id:
- raise Exception('Internal Error')
-
- if f_wrapped._using is None:
- f_wrapped._using = id(f_wrapping)
- return
- Share(f_wrapped)
-
-
-def Const(val):
- '''Streaming wrapper around scalar val
-
- Arguments:
- val (any): a scalar
- Returns:
- FunctionWrapper: a streaming wrapper
- '''
- return _wrap(val, dict(), name='Const', wraps=(val,))
-
-
-def Foo(foo, foo_kwargs=None):
- '''Streaming wrapper around function call
-
- Arguments:
- foo (callable): a function or callable
- foo_kwargs (dict): kwargs for the function or callable foo
- Returns:
- FunctionWrapper: a streaming wrapper around foo
- '''
- return _wrap(foo, foo_kwargs or {}, name='Foo', wraps=(foo,))
-
-
-def Share(f_wrap):
- '''Function to increment dataflow node reference count
-
- Arguments:
- f_wrap (FunctionWrapper): a streaming function
- Returns:
- FunctionWrapper: the same
- '''
- if not isinstance(f_wrap, FunctionWrapper):
- raise Exception('Share expects a tributary')
- f_wrap.inc()
- return f_wrap
-
-
-class FunctionWrapper(object):
- '''Generic streaming wrapper for a function'''
- _id_ref = 0
-
- def __init__(self, foo, foo_kwargs, name='', wraps=(), share=None, state=None):
- '''
- Args:
- foo (callable): function to wrap
- foo_kwargs (dict): kwargs of function
- name (str): name of function to call
- wraps (tuple): functions or FunctionWrappers that this is wrapping
- share:
- state: state context
-
- Returns:
- FunctionWrapper: wrapped function
-
- '''
- self._id = FunctionWrapper._id_ref
- FunctionWrapper._id_ref += 1
- state = state or {}
-
- if not (isinstance(foo, types.FunctionType) or isinstance(foo, types.CoroutineType)):
- # bind to f so foo isnt referenced
- def _always(val=foo):
- while True:
- yield val
- foo = _always
-
- if len(foo.__code__.co_varnames) > 0 and \
- foo.__code__.co_varnames[0] == 'state':
- self._foo = foo.__get__(self, FunctionWrapper) # TODO: remember what this line does
- for k, v in iteritems(state):
- if k not in ('_foo', '_foo_kwargs', '_refs_orig', '_name', '_wraps', '_share'):
- setattr(self, k, v)
- else:
- raise Exception('Reserved Word - %s' % k)
- else:
- self._foo = foo
-
- self._foo_kwargs = foo_kwargs
- self._refs_orig, self._refs = 1, 1
-
- self._name = name
- self._wraps = wraps
- self._using = None
- self._share = share if share else self
-
- def get_last(self):
- '''Get last call value'''
- if not hasattr(self, '_last'):
- raise Exception('Never called!!')
-
- if self._refs < 0:
- raise Exception('Ref mismatch in %s' % str(self._foo))
-
- self._refs -= 1
- return self._last
-
- def set_last(self, val):
- '''Set last call value'''
- self._refs = self._refs_orig
- self._last = val
-
- last = property(get_last, set_last)
-
- def inc(self):
- '''incremenet reference count'''
- self._refs_orig += 1
- self._refs += 1
-
- def view(self, _id=0, _idmap=None):
- '''Return tree representation of data stream'''
- _idmap = _idmap or {}
- ret = {}
-
- # check if i exist already in graph
- if id(self) in _idmap:
- key = _idmap[id(self)]
- else:
- key = self._name + str(_id)
- # _id += 1
- _idmap[id(self)] = key
- _id += 1
-
- ret[key] = []
- for f in self._wraps:
- if isinstance(f, FunctionWrapper):
- r, m = f.view(_id, _idmap)
- ret[key].append(r)
- _id = m
- else:
- if 'pandas' in sys.modules:
- import pandas as pd
- if isinstance(f, pd.DataFrame) or isinstance(f, pd.Series):
- # pprint
- f = 'DataFrame'
- ret[key].append(str(f))
- return ret, _id
-
- async def __call__(self, *args, **kwargs):
- if DEBUG:
- print("calling: {}".format(self._foo))
-
- kwargs.update(self._foo_kwargs)
-
- async for item in _extract(self._foo, *args, **kwargs):
- self.last = item
- # while 0 < self._refs <= self._refs_orig:
- yield self.last
-
- def __iter__(self):
- yield from self.__call__()
-
-
-async def _extract(item, *args, **kwargs):
- while isinstance(item, FunctionWrapper) or isinstance(item, types.FunctionType) or isinstance(item, types.CoroutineType):
- if isinstance(item, FunctionWrapper):
- item = item()
-
- if isinstance(item, types.FunctionType):
- item = item(*args, **kwargs)
-
- if isinstance(item, types.CoroutineType):
- item = await item
-
- if isinstance(item, types.AsyncGeneratorType):
- async for subitem in item:
- async for extracted in _extract(subitem):
- yield extracted
-
- elif isinstance(item, types.GeneratorType):
- for subitem in item:
- async for extracted in _extract(subitem):
- yield extracted
- else:
- yield item
diff --git a/tributary/reactive/calculations/__init__.py b/tributary/reactive/calculations/__init__.py
deleted file mode 100644
index c89de68..0000000
--- a/tributary/reactive/calculations/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .ops import * # noqa: F401, F403
-from .rolling import * # noqa: F401, F403
diff --git a/tributary/reactive/calculations/ops.py b/tributary/reactive/calculations/ops.py
deleted file mode 100644
index d3ff32e..0000000
--- a/tributary/reactive/calculations/ops.py
+++ /dev/null
@@ -1,226 +0,0 @@
-import math
-from aiostream.stream import zip
-
-from ..base import _wrap, FunctionWrapper
-
-
-def unary(lam, foo, foo_kwargs=None, _name=''):
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _unary(foo):
- async for val in foo():
- yield lam(val)
- return _wrap(_unary, dict(foo=foo), name=_name or 'Unary', wraps=(foo,), share=None)
-
-
-def bin(lam, foo1, foo2, foo1_kwargs=None, foo2_kwargs=None, _name=''):
- foo1_kwargs = foo1_kwargs or {}
- foo2_kwargs = foo2_kwargs or {}
- foo1 = _wrap(foo1, foo1_kwargs)
- foo2 = _wrap(foo2, foo2_kwargs)
-
- async def _bin(foo1, foo2):
- # TODO replace with merge
- if foo1 == foo2:
- async for gen in foo1():
- yield lam(gen, gen)
- else:
- async for gen1, gen2 in zip(foo1(), foo2()):
- yield lam(gen1, gen2)
-
- return _wrap(_bin, dict(foo1=foo1, foo2=foo2), name=_name or 'Binary', wraps=(foo1, foo2), share=None)
-
-
-########################
-# Arithmetic Operators #
-########################
-def Noop(foo, foo_kwargs=None):
- return unary(lambda x: x, foo, foo_kwargs, _name='Noop')
-
-
-def Negate(foo, foo_kwargs=None):
- return unary(lambda x: -1 * x, foo, foo_kwargs, _name='Negate')
-
-
-def Invert(foo, foo_kwargs=None):
- return unary(lambda x: 1 / x, foo, foo_kwargs, _name='Invert')
-
-
-def Add(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x + y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Add')
-
-
-def Sub(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x - y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Sub')
-
-
-def Mult(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x * y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Mult')
-
-
-def Div(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x / y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Div')
-
-
-def RDiv(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: y / x, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Div')
-
-
-def Mod(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x % y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Mod')
-
-
-def Pow(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x**y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Pow')
-
-
-#####################
-# Logical Operators #
-#####################
-def Not(foo, foo_kwargs=None):
- return unary(lambda x: not x, foo, foo_kwargs, _name='Not')
-
-
-def And(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x and y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='And')
-
-
-def Or(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x or y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Or')
-
-
-###############
-# Comparators #
-###############
-def Equal(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x == y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Equal')
-
-
-def NotEqual(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x == y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='NotEqual')
-
-
-def Lt(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x < y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Less')
-
-
-def Le(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x <= y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Less-than-or-equal')
-
-
-def Gt(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x > y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Greater')
-
-
-def Ge(foo1, foo2, foo1_kwargs=None, foo2_kwargs=None):
- return bin(lambda x, y: x >= y, foo1, foo2, foo1_kwargs, foo2_kwargs, _name='Greater-than-or-equal')
-
-
-##########################
-# Mathematical Functions #
-##########################
-def Sin(foo, foo_kwargs=None):
- return unary(lambda x: math.sin(x), foo, foo_kwargs, _name='Sin')
-
-
-def Cos(foo, foo_kwargs=None):
- return unary(lambda x: math.cos(x), foo, foo_kwargs, _name='Cos')
-
-
-def Tan(foo, foo_kwargs=None):
- return unary(lambda x: math.tan(x), foo, foo_kwargs, _name='Tan')
-
-
-##############
-# Converters #
-##############
-def Int(foo, foo_kwargs=None):
- return unary(lambda x: int(x), foo, foo_kwargs=foo_kwargs, _name="Int")
-
-
-def Float(foo, foo_kwargs=None):
- return unary(lambda x: float(x), foo, foo_kwargs=foo_kwargs, _name="Float")
-
-
-def Bool(foo, foo_kwargs=None):
- return unary(lambda x: bool(x), foo, foo_kwargs=foo_kwargs, _name="Bool")
-
-
-###################
-# Python Builtins #
-###################
-def Len(foo, foo_kwargs=None):
- return unary(lambda x: len(x), foo, foo_kwargs=foo_kwargs, _name="Len")
-
-
-########################
-# Arithmetic Operators #
-########################
-FunctionWrapper.__add__ = Add
-FunctionWrapper.__radd__ = Add
-FunctionWrapper.__sub__ = Sub
-FunctionWrapper.__rsub__ = Sub
-FunctionWrapper.__mul__ = Mult
-FunctionWrapper.__rmul__ = Mult
-FunctionWrapper.__div__ = Div
-FunctionWrapper.__rdiv__ = RDiv
-FunctionWrapper.__truediv__ = Div
-FunctionWrapper.__rtruediv__ = RDiv
-
-FunctionWrapper.__pow__ = Pow
-FunctionWrapper.__rpow__ = Pow
-FunctionWrapper.__mod__ = Mod
-FunctionWrapper.__rmod__ = Mod
-
-#####################
-# Logical Operators #
-#####################
-# FunctionWrapper.__and__ = And
-# FunctionWrapper.__or__ = Or
-# FunctionWrapper.__invert__ = Not
-# FunctionWrapper.__bool__ = Bool
-# TODO use __bool__ operator
-
-##############
-# Converters #
-##############
-# FunctionWrapper.int = Int
-# FunctionWrapper.float = Float
-
-###############
-# Comparators #
-###############
-FunctionWrapper.__lt__ = Lt
-FunctionWrapper.__le__ = Le
-FunctionWrapper.__gt__ = Gt
-FunctionWrapper.__ge__ = Ge
-FunctionWrapper.__eq__ = Equal
-FunctionWrapper.__ne__ = NotEqual
-FunctionWrapper.__neg__ = Negate
-# FunctionWrapper.__nonzero__ = Bool # Py2 compat
-
-###################
-# Python Builtins #
-###################
-# FunctionWrapper.__len__ = Len
-
-###################
-# Numpy Functions #
-###################
-# FunctionWrapper.__array_ufunc__ = __array_ufunc__
-
-
-##########################
-# Mathematical Functions #
-##########################
-# FunctionWrapper.log = Log
-FunctionWrapper.sin = Sin
-FunctionWrapper.cos = Cos
-FunctionWrapper.tan = Tan
-# FunctionWrapper.arcsin = Arcsin
-# FunctionWrapper.arccos = Arccos
-# FunctionWrapper.arctan = Arctan
-# FunctionWrapper.sqrt = Sqrt
-# FunctionWrapper.exp = Exp
-# FunctionWrapper.erf = Erf
diff --git a/tributary/reactive/calculations/rolling.py b/tributary/reactive/calculations/rolling.py
deleted file mode 100644
index 7aa7d16..0000000
--- a/tributary/reactive/calculations/rolling.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import types
-from ..base import _wrap
-
-
-def Count(foo, foo_kwargs=None):
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _count(foo):
- count = 0
- async for gen in foo():
- if isinstance(gen, types.AsyncGeneratorType):
- async for f in gen:
- count += 1
- yield count
-
- elif isinstance(gen, types.GeneratorType):
- for f in gen:
- count += 1
- yield count
- else:
- count += 1
- yield count
-
- return _wrap(_count, dict(foo=foo), name='Count', wraps=(foo,), share=foo)
-
-
-def Sum(foo, foo_kwargs=None):
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _sum(foo):
- sum = 0
- async for gen in foo():
- if isinstance(gen, types.AsyncGeneratorType):
- async for f in gen:
- sum += f
- yield sum
- elif isinstance(gen, types.GeneratorType):
- for f in gen:
- sum += f
- yield sum
- else:
- sum += gen
- yield sum
-
- return _wrap(_sum, dict(foo=foo), name='Sum', wraps=(foo,), share=foo)
diff --git a/tributary/reactive/input/__init__.py b/tributary/reactive/input/__init__.py
deleted file mode 100644
index 2862afb..0000000
--- a/tributary/reactive/input/__init__.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import asyncio
-import math
-import numpy as np
-
-from ..base import _wrap
-from .file import File, File as FileSource # noqa: F401
-from .http import HTTP, HTTP as HTTPSource # noqa: F401
-from .kafka import Kafka, Kafka as KafkaSource # noqa: F401
-from .socketio import SocketIO, SocketIO as SocketIOSource # noqa: F401
-from .ws import WebSocket, WebSocket as WebSocketSource # noqa: F401
-
-
-def _gen():
- S = 100
- T = 252
- mu = 0.25
- vol = 0.5
-
- returns = np.random.normal(mu / T, vol / math.sqrt(T), T) + 1
- _list = returns.cumprod() * S
- return _list
-
-
-def Random(size=10, interval=0.1):
- '''Yield a random dctionary of data
-
- Args:
- size (int): number of elements to yield
- interval (float): interval to wait between yields
- '''
- async def _random(size, interval):
- step = 0
- while step < size:
- x = {y: _gen() for y in ('A', 'B', 'C', 'D')}
- for i in range(len(x['A'])):
- if step >= size:
- break
- yield {'A': x['A'][i],
- 'B': x['B'][i],
- 'C': x['C'][i],
- 'D': x['D'][i]}
- await asyncio.sleep(interval)
- step += 1
-
- return _wrap(_random, dict(size=size, interval=interval), name='Random')
diff --git a/tributary/reactive/input/file.py b/tributary/reactive/input/file.py
deleted file mode 100644
index 5e28b70..0000000
--- a/tributary/reactive/input/file.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import aiofiles
-import json as JSON
-from ..base import _wrap # noqa: F401
-
-
-def File(filename, json=True):
- '''Open up a file and yield back lines in the file
-
- Args:
- filename (str): filename to read
- json (bool): load file line as json
- '''
- async def _file(filename, json):
- async with aiofiles.open(filename) as f:
- async for line in f:
- if json:
- yield JSON.loads(line)
- else:
- yield line
-
- return _wrap(_file, dict(filename=filename, json=json), name='File')
diff --git a/tributary/reactive/input/http.py b/tributary/reactive/input/http.py
deleted file mode 100644
index 5e00422..0000000
--- a/tributary/reactive/input/http.py
+++ /dev/null
@@ -1,52 +0,0 @@
-import aiohttp
-import json as JSON
-import time
-import functools
-from ..base import _wrap
-
-
-def AsyncHTTP(url, interval=1, repeat=1, json=False, wrap=False, field=None, proxies=None, cookies=None):
- '''Connect to url and yield results
-
- Args:
- url (str): url to connect to
- interval (int): interval to re-query
- repeat (int): number of times to request
- json (bool): load http content data as json
- wrap (bool): wrap result in a list
- field (str): field to index result by
- proxies (list): list of URL proxies to pass to requests.get
- cookies (list): list of cookies to pass to requests.get
- '''
- async def _req(url, interval=1, repeat=1, json=False, wrap=False, field=None, proxies=None, cookies=None):
- count = 0
- while count < repeat:
- async with aiohttp.ClientSession() as session:
- async with session.get(url, cookies=cookies, proxy=proxies) as response:
- msg = await response.text()
-
- if msg is None or response.status != 200:
- break
-
- if json:
- msg = JSON.loads(msg)
-
- if field:
- msg = msg[field]
-
- if wrap:
- msg = [msg]
-
- yield msg
-
- if interval:
- time.sleep(interval)
- if repeat >= 0:
- count += 1
-
- return _wrap(_req, dict(url=url, interval=interval, repeat=repeat, json=json, wrap=wrap, field=field, proxies=proxies, cookies=cookies), name='HTTP')
-
-
-@functools.wraps(AsyncHTTP)
-def HTTP(url, *args, **kwargs):
- return AsyncHTTP(url, *args, **kwargs)
diff --git a/tributary/reactive/input/kafka.py b/tributary/reactive/input/kafka.py
deleted file mode 100644
index c60dcef..0000000
--- a/tributary/reactive/input/kafka.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import functools
-import ujson
-from confluent_kafka import Consumer, KafkaError
-from ..base import _wrap
-
-
-def AsyncKafka(servers, group, topics, json=False, wrap=False, interval=1):
- '''Connect to kafka server and yield back results
-
- Args:
- servers (list): kafka bootstrap servers
- group (str): kafka group id
- topics (list): list of kafka topics to connect to
- json (bool): load input data as json
- wrap (bool): wrap result in a list
- interval (int): kafka poll interval
- '''
- c = Consumer({
- 'bootstrap.servers': servers,
- 'group.id': group,
- 'default.topic.config': {
- 'auto.offset.reset': 'smallest'
- }
- })
-
- if not isinstance(topics, list):
- topics = [topics]
- c.subscribe(topics)
-
- async def _listen(consumer, json, wrap, interval):
- while True:
- msg = consumer.poll(interval)
-
- if msg is None:
- continue
- if msg.error():
- if msg.error().code() == KafkaError._PARTITION_EOF:
- continue
- else:
- print(msg.error())
- break
-
- msg = msg.value().decode('utf-8')
-
- if not msg:
- break
- if json:
- msg = ujson.loads(msg)
- if wrap:
- msg = [msg]
- yield msg
-
- return _wrap(_listen, dict(consumer=c, json=json, wrap=wrap, interval=interval), name='Kafka')
-
-
-@functools.wraps(AsyncKafka)
-def Kafka(*args, **kwargs):
- return AsyncKafka(*args, **kwargs)
diff --git a/tributary/reactive/input/socketio.py b/tributary/reactive/input/socketio.py
deleted file mode 100644
index 3e9d704..0000000
--- a/tributary/reactive/input/socketio.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import functools
-from socketIO_client_nexus import SocketIO as SIO
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
-from ..base import _wrap
-
-
-def AsyncSocketIO(url, channel='', field='', sendinit=None, json=False, wrap=False, interval=1):
- '''Connect to socketIO server and yield back results
-
- Args:
- url (str): url to connect to
- channel (str): socketio channel to connect through
- field (str): field to index result by
- sendinit (list): data to send on socketio connection open
- json (bool): load websocket data as json
- wrap (bool): wrap result in a list
- interval (int): socketio wai interval
- '''
- o = urlparse(url)
- socketIO = SIO(o.scheme + '://' + o.netloc, o.port)
- if sendinit:
- socketIO.emit(sendinit)
-
- async def _sio(url, channel, field='', json=False, wrap=False, interval=1):
- while True:
- _data = []
- socketIO.on(channel, lambda data: _data.append(data))
- socketIO.wait(seconds=interval)
- for msg in _data:
- # FIXME clear _data
- if json:
- msg = json.loads(msg)
-
- if field:
- msg = msg[field]
-
- if wrap:
- msg = [msg]
-
- yield msg
-
- return _wrap(_sio, dict(url=url, channel=channel, field=field, json=json, wrap=wrap, interval=interval), name='SocketIO')
-
-
-@functools.wraps(AsyncSocketIO)
-def SocketIO(url, *args, **kwargs):
- return AsyncSocketIO(url, *args, **kwargs)
diff --git a/tributary/reactive/input/ws.py b/tributary/reactive/input/ws.py
deleted file mode 100644
index 00c7472..0000000
--- a/tributary/reactive/input/ws.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import functools
-import websockets
-from ujson import loads as load_json
-from ..base import _wrap
-from ...base import StreamNone, StreamEnd
-
-
-def AsyncWebSocket(url, json=False, wrap=False):
- '''Connect to websocket and yield back results
-
- Args:
- url (str): websocket url to connect to
- json (bool): load websocket data as json
- wrap (bool): wrap result in a list
- '''
- async def _listen(url, json, wrap):
- async with websockets.connect(url) as websocket:
- async for x in websocket:
- if isinstance(x, StreamNone):
- continue
- elif not x or isinstance(x, StreamEnd):
- break
-
- if json:
- x = load_json(x)
- if wrap:
- x = [x]
- yield x
-
- return _wrap(_listen, dict(url=url, json=json, wrap=wrap), name='WebSocket')
-
-
-@functools.wraps(AsyncWebSocket)
-def WebSocket(url, *args, **kwargs):
- return AsyncWebSocket(url, *args, **kwargs)
diff --git a/tributary/reactive/output/__init__.py b/tributary/reactive/output/__init__.py
deleted file mode 100644
index 7ba122d..0000000
--- a/tributary/reactive/output/__init__.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import asyncio
-from pprint import pprint
-from IPython.display import display
-
-from ..base import _wrap, _extract, FunctionWrapper
-from .file import File as FileSink # noqa: F401
-from .http import HTTP as HTTPSink # noqa: F401
-from .kafka import Kafka as KafkaSink # noqa: F401
-from .ws import WebSocket as WebSocketSink # noqa: F401
-
-
-def Print(foo, foo_kwargs=None, text=''):
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _print(foo):
- async for r in _extract(foo):
- if text:
- print(text, r)
- else:
- print(r)
- yield r
-
- return _wrap(_print, dict(foo=foo), name='Print', wraps=(foo,), share=foo)
-
-
-def Graph(f_wrap):
- if not isinstance(f_wrap, FunctionWrapper):
- raise Exception('ViewGraph expects tributary')
- return f_wrap.view(0)[0]
-
-
-def PPrint(f_wrap):
- pprint(Graph(f_wrap))
-
-
-def GraphViz(f_wrap, name='Graph'):
- d = Graph(f_wrap)
-
- from graphviz import Digraph
- dot = Digraph(name, strict=True)
- dot.format = 'png'
-
- def rec(nodes, parent):
- for d in nodes:
- if not isinstance(d, dict):
- dot.node(d)
- dot.edge(d, parent)
-
- else:
- for k in d:
- dot.node(k)
- rec(d[k], k)
- dot.edge(k, parent)
-
- for k in d:
- dot.node(k)
- rec(d[k], k)
-
- return dot
-
-
-def Perspective(foo, foo_kwargs=None, **psp_kwargs):
- foo = _wrap(foo, foo_kwargs or {})
-
- from perspective import PerspectiveWidget
- p = PerspectiveWidget(psp_kwargs.pop('schema', []), **psp_kwargs)
-
- async def _perspective(foo):
- async for r in foo():
- if isinstance(r, dict):
- r = [r]
- p.update(r)
- # let PSP render
- await asyncio.sleep(.1)
- yield r
-
- display(p)
-
- return _wrap(_perspective, dict(foo=foo), name='Perspective', wraps=(foo,), share=foo)
diff --git a/tributary/reactive/output/file.py b/tributary/reactive/output/file.py
deleted file mode 100644
index 63d48b2..0000000
--- a/tributary/reactive/output/file.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import aiofiles
-import json as JSON
-from ..base import _wrap # noqa: F401
-
-
-def File(foo, foo_kwargs=None, filename='', json=True):
- '''Open up a file and write lines to the file
-
- Args:
- foo (callable): input stream
- foo_kwargs (dict): kwargs for the input stream
- filename (str): filename to write
- json (bool): load file line as json
- '''
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _file(foo, filename, json):
- async for data in foo():
- async with aiofiles.open(filename) as fp:
- if json:
- fp.write(JSON.dumps(data))
- else:
- fp.write(data)
-
- return _wrap(_file, dict(foo=foo, filename=filename, json=json), name='File')
diff --git a/tributary/reactive/output/http.py b/tributary/reactive/output/http.py
deleted file mode 100644
index 8cbd41c..0000000
--- a/tributary/reactive/output/http.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import functools
-import requests
-import ujson
-from ..base import _wrap
-
-
-def AsyncHTTP(foo, foo_kwargs=None, url='', json=False, wrap=False, field=None, proxies=None, cookies=None):
- '''Connect to url and post results to it
-
- Args:
- foo (callable): input stream
- foo_kwargs (dict): kwargs for the input stream
- url (str): url to post to
- json (bool): dump data as json
- wrap (bool): wrap input in a list
- field (str): field to index result by
- proxies (list): list of URL proxies to pass to requests.post
- cookies (list): list of cookies to pass to requests.post
- '''
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _send(foo, url, json=False, wrap=False, field=None, proxies=None, cookies=None):
- async for data in foo():
- if wrap:
- data = [data]
- if json:
- data = ujson.dumps(data)
-
- msg = requests.post(url, data=data, cookies=cookies, proxies=proxies)
-
- if msg is None:
- break
-
- if msg.status_code != 200:
- yield msg
- continue
-
- if json:
- msg = msg.json()
-
- if field:
- msg = msg[field]
-
- if wrap:
- msg = [msg]
-
- yield msg
-
- return _wrap(_send, dict(foo=foo, url=url, json=json, wrap=wrap, field=field, proxies=proxies, cookies=cookies), name='HTTP')
-
-
-@functools.wraps(AsyncHTTP)
-def HTTP(foo, foo_kwargs=None, *args, **kwargs):
- return AsyncHTTP(foo, foo_kwargs, *args, **kwargs)
diff --git a/tributary/reactive/output/kafka.py b/tributary/reactive/output/kafka.py
deleted file mode 100644
index 38c8349..0000000
--- a/tributary/reactive/output/kafka.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import functools
-import ujson
-from confluent_kafka import Producer
-from ..base import _wrap
-
-
-def AsyncKafka(foo, foo_kwargs=None, servers='', topic='', json=False, wrap=False):
- '''Connect to kafka server and send data
-
- Args:
- foo (callable): input stream
- foo_kwargs (dict): kwargs for the input stream
- servers (list): kafka bootstrap servers
- group (str): kafka group id
- topics (list): list of kafka topics to connect to
- json (bool): load input data as json
- wrap (bool): wrap result in a list
- interval (int): kafka poll interval
- '''
- foo = _wrap(foo, foo_kwargs or {})
-
- p = Producer({'bootstrap.servers': servers})
-
- async def _send(foo, producer, topic, json, wrap):
- ret = []
- async for data in foo():
- # Trigger any available delivery report callbacks from previous produce() calls
- producer.poll(0)
-
- if wrap:
- data = [data]
-
- if json:
- data = ujson.dumps(data)
-
- producer.produce(topic, data.encode('utf-8'), callback=lambda *args: ret.append(args))
-
- for data in ret:
- yield data
- ret = []
-
- return _wrap(_send, dict(foo=foo, producer=p, topic=topic, json=json, wrap=wrap), name='Kafka')
-
-
-@functools.wraps(AsyncKafka)
-def Kafka(foo, foo_kwargs=None, **kafka_kwargs):
- return AsyncKafka(foo, foo_kwargs, **kafka_kwargs)
diff --git a/tributary/reactive/output/socketio.py b/tributary/reactive/output/socketio.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tributary/reactive/output/ws.py b/tributary/reactive/output/ws.py
deleted file mode 100644
index 9e99571..0000000
--- a/tributary/reactive/output/ws.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import functools
-import ujson
-import websockets
-from ..base import _wrap
-from ...base import StreamNone, StreamEnd
-
-
-def AsyncWebSocket(foo, foo_kwargs=None, url='', json=False, wrap=False, field=None, response=False):
- '''Connect to websocket and send data
-
- Args:
- foo (callable): input stream
- foo_kwargs (dict): kwargs for the input stream
- url (str): websocket url to connect to
- json (bool): dump data as json
- wrap (bool): wrap result in a list
- '''
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs)
-
- async def _send(foo, url, json=False, wrap=False, field=None, response=False):
- async with websockets.connect(url) as websocket:
- async for data in foo():
- if isinstance(data, StreamNone):
- continue
- elif not data or isinstance(data, StreamEnd):
- break
-
- if wrap:
- data = [data]
- if json:
- data = ujson.dumps(data)
-
- await websocket.send(data)
-
- if response:
- msg = await websocket.recv()
-
- else:
- msg = '{}'
-
- if json:
- msg = json.loads(msg)
-
- if field:
- msg = msg[field]
-
- if wrap:
- msg = [msg]
-
- yield msg
-
- return _wrap(_send, dict(foo=foo, url=url, json=json, wrap=wrap, field=field, response=response), name='WebSocket')
-
-
-@functools.wraps(AsyncWebSocket)
-def WebSocket(foo, foo_kwargs=None, *args, **kwargs):
- return AsyncWebSocket(foo, foo_kwargs, *args, **kwargs)
diff --git a/tributary/reactive/utils.py b/tributary/reactive/utils.py
deleted file mode 100644
index a00c74e..0000000
--- a/tributary/reactive/utils.py
+++ /dev/null
@@ -1,308 +0,0 @@
-import asyncio
-import time
-import types
-from aiostream.stream import zip
-from .base import _wrap, FunctionWrapper, Foo, Const
-
-
-def Timer(foo_or_val, kwargs=None, interval=1, repeat=0):
- '''Streaming wrapper to repeat a value or calls to a function
-
- Arguments:
- foo_or_val (any): function to call or value to return
- kwargs (dict): kwargs for foo_or_val if its a function
- interval (int): time between queries
- repeat (int): number of times to repeat
- Returns:
- FunctionWrapper: a streaming wrapper
- '''
- kwargs = kwargs or {}
- if not isinstance(foo_or_val, types.FunctionType):
- foo = Const(foo_or_val)
- else:
- foo = Foo(foo_or_val, kwargs)
-
- async def _repeater(foo, repeat, interval):
- while repeat > 0:
- t1 = time.time()
- yield foo()
- t2 = time.time()
- if interval > 0:
- # sleep for rest of time that _p didnt take
- await asyncio.sleep(max(0, interval - (t2 - t1)))
- repeat -= 1
-
- return _wrap(_repeater, dict(foo=foo, repeat=repeat, interval=interval), name='Timer', wraps=(foo,), share=foo)
-
-
-def Delay(f_wrap, kwargs=None, delay=1):
- '''Streaming wrapper to delay a stream
-
- Arguments:
- f_wrap (callable): input stream
- kwargs (dict): kwargs for input stream
- delay (float): time to delay input stream
- Returns:
- FunctionWrapper: a streaming wrapper
- '''
- if not isinstance(f_wrap, FunctionWrapper):
- f_wrap = Foo(f_wrap, kwargs or {})
-
- async def _delay(f_wrap, delay):
- async for f in f_wrap():
- yield f
- await asyncio.sleep(delay)
-
- return _wrap(_delay, dict(f_wrap=f_wrap, delay=delay), name='Delay', wraps=(f_wrap,), share=f_wrap)
-
-
-def State(foo, foo_kwargs=None, **state):
- '''Streaming wrapper to maintain state
-
- Arguments:
- foo (callable): input stream
- foo_kwargs (dict): kwargs for input stream
- state (dict): state dictionary of values to hold
- Returns:
- FunctionWrapper: a streaming wrapper
- '''
- foo_kwargs = foo_kwargs or {}
- foo = _wrap(foo, foo_kwargs, name=foo.__name__, wraps=(foo,), state=state)
- return foo
-
-
-def Apply(foo, f_wrap, foo_kwargs=None):
- '''Streaming wrapper to apply a function to an input stream
-
- Arguments:
- foo (callable): function to apply
- f_wrap (callable): input stream
- foo_kwargs (dict): kwargs for function
- Returns:
- FunctionWrapper: a streaming wrapper
- '''
- if not isinstance(f_wrap, FunctionWrapper):
- raise Exception('Apply expects a tributary')
- foo_kwargs = foo_kwargs or {}
- foo = Foo(foo, foo_kwargs)
- foo._wraps = foo._wraps + (f_wrap, )
-
- async def _apply(foo):
- async for f in f_wrap():
- item = foo(f)
- if isinstance(item, types.AsyncGeneratorType):
- async for i in item:
- yield i
- elif isinstance(item, types.CoroutineType):
- yield await item
- else:
- yield item
- return _wrap(_apply, dict(foo=foo), name='Apply', wraps=(foo,), share=foo)
-
-
-def Window(foo, foo_kwargs=None, size=-1, full_only=True):
- foo_kwargs = foo_kwargs or {}
- foo = Foo(foo, foo_kwargs)
-
- accum = []
-
- async def _window(foo, size, full_only, accum):
- async for x in foo():
- if size == 0:
- yield x
- else:
- accum.append(x)
-
- if size > 0:
- accum = accum[-size:]
- if full_only:
- if len(accum) == size or size == -1:
- yield accum
- else:
- yield accum
-
- return _wrap(_window, dict(foo=foo, size=size, full_only=full_only, accum=accum), name='Window', wraps=(foo,), share=foo)
-
-
-def Unroll(foo_or_val, kwargs=None):
- if not isinstance(foo_or_val, types.FunctionType):
- foo = Const(foo_or_val)
- else:
- foo = Foo(foo_or_val, kwargs or {})
-
- async def _unroll(foo):
- async for ret in foo():
- if isinstance(ret, list):
- for f in ret:
- yield f
- elif isinstance(ret, types.AsyncGeneratorType):
- async for f in ret:
- yield f
-
- return _wrap(_unroll, dict(foo=foo), name='Unroll', wraps=(foo,), share=foo)
-
-
-def UnrollDataFrame(foo_or_val, kwargs=None, json=True, wrap=False):
- if not isinstance(foo_or_val, types.FunctionType):
- foo = Const(foo_or_val)
- else:
- foo = Foo(foo_or_val, kwargs or {})
-
- async def _unrolldf(foo):
- async for df in foo():
- for i in range(len(df)):
- row = df.iloc[i]
- if json:
- data = row.to_dict()
- data['index'] = row.name
- yield data
- else:
- yield row
-
- return _wrap(_unrolldf, dict(foo=foo), name='UnrollDataFrame', wraps=(foo,), share=foo)
-
-
-def Merge(f_wrap1, f_wrap2):
- if not isinstance(f_wrap1, FunctionWrapper):
- if not isinstance(f_wrap1, types.FunctionType):
- f_wrap1 = Const(f_wrap1)
- else:
- f_wrap1 = Foo(f_wrap1)
-
- if not isinstance(f_wrap2, FunctionWrapper):
- if not isinstance(f_wrap2, types.FunctionType):
- f_wrap2 = Const(f_wrap2)
- else:
- f_wrap2 = Foo(f_wrap2)
-
- async def _merge(foo1, foo2):
- async for gen1, gen2 in zip(foo1(), foo2()):
- if isinstance(gen1, types.AsyncGeneratorType) and \
- isinstance(gen2, types.AsyncGeneratorType):
- async for f1, f2 in zip(gen1, gen2):
- yield [f1, f2]
- elif isinstance(gen1, types.AsyncGeneratorType):
- async for f1 in gen1:
- yield [f1, gen2]
- elif isinstance(gen2, types.AsyncGeneratorType):
- async for f2 in gen2:
- yield [gen1, f2]
- else:
- yield [gen1, gen2]
-
- return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='Merge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-def ListMerge(f_wrap1, f_wrap2):
- if not isinstance(f_wrap1, FunctionWrapper):
- if not isinstance(f_wrap1, types.FunctionType):
- f_wrap1 = Const(f_wrap1)
- else:
- f_wrap1 = Foo(f_wrap1)
-
- if not isinstance(f_wrap2, FunctionWrapper):
- if not isinstance(f_wrap2, types.FunctionType):
- f_wrap2 = Const(f_wrap2)
- else:
- f_wrap2 = Foo(f_wrap2)
-
- async def _merge(foo1, foo2):
- async for gen1, gen2 in zip(foo1(), foo2()):
- if isinstance(gen1, types.AsyncGeneratorType) and \
- isinstance(gen2, types.AsyncGeneratorType):
- async for f1, f2 in zip(gen1, gen2):
- ret = []
- ret.extend(f1)
- ret.extend(f1)
- yield ret
- elif isinstance(gen1, types.AsyncGeneratorType):
- async for f1 in gen1:
- ret = []
- ret.extend(f1)
- ret.extend(gen2)
- yield ret
- elif isinstance(gen2, types.AsyncGeneratorType):
- async for f2 in gen2:
- ret = []
- ret.extend(gen1)
- ret.extend(f2)
- yield ret
- else:
- ret = []
- ret.extend(gen1)
- ret.extend(gen2)
- yield ret
-
- return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='ListMerge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-def DictMerge(f_wrap1, f_wrap2):
- if not isinstance(f_wrap1, FunctionWrapper):
- if not isinstance(f_wrap1, types.FunctionType):
- f_wrap1 = Const(f_wrap1)
- else:
- f_wrap1 = Foo(f_wrap1)
-
- if not isinstance(f_wrap2, FunctionWrapper):
- if not isinstance(f_wrap2, types.FunctionType):
- f_wrap2 = Const(f_wrap2)
- else:
- f_wrap2 = Foo(f_wrap2)
-
- async def _dictmerge(foo1, foo2):
- async for gen1, gen2 in zip(foo1(), foo2()):
- if isinstance(gen1, types.AsyncGeneratorType) and \
- isinstance(gen2, types.AsyncGeneratorType):
- async for f1, f2 in zip(gen1, gen2):
- ret = {}
- ret.update(f1)
- ret.update(f1)
- yield ret
- elif isinstance(gen1, types.AsyncGeneratorType):
- async for f1 in gen1:
- ret = {}
- ret.update(f1)
- ret.update(gen2)
- yield ret
- elif isinstance(gen2, types.AsyncGeneratorType):
- async for f2 in gen2:
- ret = {}
- ret.update(gen1)
- ret.update(f2)
- yield ret
- else:
- ret = {}
- ret.update(gen1)
- ret.update(gen2)
- yield ret
-
- return _wrap(_dictmerge, dict(foo1=f_wrap1, foo2=f_wrap2), name='DictMerge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-def Reduce(*f_wraps):
- f_wraps = list(f_wraps)
- for i, f_wrap in enumerate(f_wraps):
- if not isinstance(f_wrap, types.FunctionType):
- f_wraps[i] = Const(f_wrap)
- else:
- f_wraps[i] = Foo(f_wrap)
-
- async def _reduce(foos):
- async for all_gens in zip(*[foo() for foo in foos]):
- gens = []
- vals = []
- for gen in all_gens:
- if isinstance(gen, types.AsyncGeneratorType):
- gens.append(gen)
- else:
- vals.append(gen)
- if gens:
- for gens in zip(*gens):
- ret = list(vals)
- for gen in gens:
- ret.append(next(gen))
- yield ret
- else:
- yield vals
-
- return _wrap(_reduce, dict(foos=f_wraps), name='Reduce', wraps=tuple(f_wraps), share=None)
diff --git a/tributary/streaming/__init__.py b/tributary/streaming/__init__.py
index 5e56a54..a7d5587 100644
--- a/tributary/streaming/__init__.py
+++ b/tributary/streaming/__init__.py
@@ -1,10 +1,11 @@
import asyncio
+from copy import deepcopy
from .base import Node # noqa: F401
from .calculations import * # noqa: F401, F403
from .input import * # noqa: F401, F403
from .output import * # noqa: F401, F403
from .utils import * # noqa: F401, F403
-from ..base import StreamEnd, StreamNone
+from ..base import StreamEnd, StreamNone, StreamRepeat
async def _run(node):
@@ -13,9 +14,10 @@ async def _run(node):
while True:
for level in nodes:
await asyncio.gather(*(asyncio.create_task(n()) for n in level))
- if not isinstance(node.value(), (StreamEnd, StreamNone)):
- ret.append(node.value())
- elif isinstance(node.value(), StreamEnd):
+ val = deepcopy(node.value())
+ if not isinstance(val, (StreamEnd, StreamNone, StreamRepeat)):
+ ret.append(val)
+ elif isinstance(val, StreamEnd):
break
return ret
diff --git a/tributary/streaming/base.py b/tributary/streaming/base.py
index ba34b88..bda8722 100644
--- a/tributary/streaming/base.py
+++ b/tributary/streaming/base.py
@@ -2,7 +2,8 @@
import types
from aiostream.aiter_utils import anext
from asyncio import Queue, QueueEmpty as Empty
-from ..base import StreamEnd, StreamNone
+from copy import deepcopy
+from ..base import StreamEnd, StreamNone, StreamRepeat
def _gen_to_foo(generator):
@@ -47,7 +48,13 @@ async def _execute(self):
if asyncio.iscoroutine(self._foo):
_last = await self._foo(*self._active, **self._foo_kwargs)
elif isinstance(self._foo, types.FunctionType):
- _last = self._foo(*self._active, **self._foo_kwargs)
+ try:
+ _last = self._foo(*self._active, **self._foo_kwargs)
+ except ValueError:
+ # Swap back to function
+ self._foo = self._old_foo
+ _last = self._foo(*self._active, **self._foo_kwargs)
+
else:
raise Exception('Cannot use type:{}'.format(type(self._foo)))
@@ -56,10 +63,12 @@ async def _foo(g=_last):
return await _agen_to_foo(g)
self._foo = _foo
_last = await self._foo()
-
elif isinstance(_last, types.GeneratorType):
+ # Swap to generator unroller
+ self._old_foo = self._foo
self._foo = lambda g=_last: _gen_to_foo(g)
_last = self._foo()
+
elif asyncio.iscoroutine(_last):
_last = await _last
@@ -73,10 +82,22 @@ async def _finish(self):
self._last = StreamEnd()
await self._output(self._last)
+ def _backpressure(self):
+ '''check if _downstream are all empty'''
+ ret = not all(n._input[i].empty() for n, i in self._downstream)
+ if ret:
+ print('backpressure!')
+ return ret
+
async def __call__(self):
+ if self._backpressure():
+ return StreamNone()
+
if self._finished:
return await self._finish()
+ print(self._input)
+
ready = True
# iterate through inputs
for i, inp in enumerate(self._input):
@@ -85,12 +106,17 @@ async def __call__(self):
try:
# get from input queue
val = inp.get_nowait()
+ while isinstance(val, StreamRepeat):
+ # Skip entry
+ val = inp.get_nowait()
if isinstance(val, StreamEnd):
+ print('here')
return await self._finish()
# set as active
self._active[i] = val
+
except Empty:
# wait for value
ready = False
diff --git a/tributary/streaming/calculations/rolling.py b/tributary/streaming/calculations/rolling.py
index e69de29..00d2d83 100644
--- a/tributary/streaming/calculations/rolling.py
+++ b/tributary/streaming/calculations/rolling.py
@@ -0,0 +1,29 @@
+from ..base import Node
+
+class Count(Node):
+ '''Node to count inputs'''
+ def __init__(self, node, text=''):
+ self._count = 0
+
+ def foo(val):
+ self._count += 1
+ return self._count
+
+ super().__init__(foo=foo, foo_kwargs=None, name='Count', inputs=1)
+
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+class Sum(Node):
+ '''Node to sum inputs'''
+ def __init__(self, node, text=''):
+ self._sum = 0
+
+ def foo(val):
+ self._sum += val
+ return self._sum
+
+ super().__init__(foo=foo, foo_kwargs=None, name='Sum', inputs=1)
+
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
diff --git a/tributary/streaming/output/output.py b/tributary/streaming/output/output.py
index 0a338b1..8be4331 100644
--- a/tributary/streaming/output/output.py
+++ b/tributary/streaming/output/output.py
@@ -5,7 +5,6 @@
class Print(Node):
def __init__(self, node, text=''):
def foo(val):
- print(text + str(val))
return val
super().__init__(foo=foo, foo_kwargs=None, name='Print', inputs=1)
diff --git a/tributary/streaming/utils.py b/tributary/streaming/utils.py
index e69de29..600bb4f 100644
--- a/tributary/streaming/utils.py
+++ b/tributary/streaming/utils.py
@@ -0,0 +1,290 @@
+import asyncio
+from .base import Node
+from ..base import StreamNone, StreamRepeat
+
+
+class Delay(Node):
+ '''Streaming wrapper to delay a stream
+
+ Arguments:
+ node (node): input stream
+ delay (float): time to delay input stream
+ '''
+
+ def __init__(self, node, delay=1):
+ async def foo(val):
+ await asyncio.sleep(delay)
+ return val
+
+ super().__init__(foo=foo, name='Delay', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+
+# class State(Node):
+# '''Streaming wrapper to delay a stream
+
+# Arguments:
+# node (node): input stream
+# state (dict): state dictionary of values to hold
+# '''
+
+# def __init__(self, node, delay=1):
+# async def foo(val):
+# await asyncio.sleep(delay)
+# return val
+
+# super().__init__(foo=foo, foo_kwargs=None, name='Delay', inputs=1)
+# node._downstream.append((self, 0))
+# self._upstream.append(node)
+
+
+class Apply(Node):
+ '''Streaming wrapper to apply a function to an input stream
+
+ Arguments:
+ node (node): input stream
+ foo (callable): function to apply
+ foo_kwargs (dict): kwargs for function
+ '''
+ def __init__(self, node, foo, foo_kwargs=None):
+ self._apply = foo
+ self._apply_kwargs = foo_kwargs or {}
+
+ async def foo(val):
+ return self._apply(val, **self._apply_kwargs)
+
+ super().__init__(foo=foo, name='Apply', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+
+class Window(Node):
+ '''Streaming wrapper to collect a window of values
+
+ Arguments:
+ node (node): input stream
+ size (int): size of windows to use
+ full_only (bool): only return if list is full
+ '''
+ def __init__(self, node, size=-1, full_only=False):
+ self._accum = []
+
+ def foo(val, size=size, full_only=full_only):
+ if size == 0:
+ return val
+ else:
+ self._accum.append(val)
+
+ if size > 0:
+ self._accum = self._accum[-size:]
+
+ if full_only and len(self._accum) == size:
+ return self._accum
+ elif full_only:
+ return StreamNone()
+ else:
+ return self._accum
+
+ super().__init__(foo=foo, name='Window', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+
+class Unroll(Node):
+ '''Streaming wrapper to unroll an iterable stream
+
+ Arguments:
+ node (node): input stream
+ '''
+ def __init__(self, node):
+ self._count = 0
+
+ async def foo(value):
+ print('foo got:', value)
+ # unrolled
+ if self._count > 0:
+ self._count -= 1
+ return value
+
+ # unrolling
+ try:
+ for v in value:
+ self._count += 1
+ print('pushing:', v)
+ await self._push(v, 0)
+ except TypeError:
+ return value
+ else:
+ return StreamRepeat()
+
+ super().__init__(foo=foo, name='Unroll', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+# class UnrollDataFrame(Node):
+# '''Streaming wrapper to unroll an iterable stream
+
+# Arguments:
+# node (node): input stream
+# '''
+# def __init__(self, node, json=False, wrap=False):
+# def foo(value, json=json, wrap=wrap):
+# for i in range(len(value)):
+# row = df.iloc[i]
+# if json:
+# data = row.to_dict()
+# data['index'] = row.name
+# yield data
+# else:
+# yield row
+
+# super().__init__(foo=foo, name='Unroll', inputs=1)
+# node._downstream.append((self, 0))
+# self._upstream.append(node)
+
+
+# def Merge(f_wrap1, f_wrap2):
+# if not isinstance(f_wrap1, FunctionWrapper):
+# if not isinstance(f_wrap1, types.FunctionType):
+# f_wrap1 = Const(f_wrap1)
+# else:
+# f_wrap1 = Foo(f_wrap1)
+
+# if not isinstance(f_wrap2, FunctionWrapper):
+# if not isinstance(f_wrap2, types.FunctionType):
+# f_wrap2 = Const(f_wrap2)
+# else:
+# f_wrap2 = Foo(f_wrap2)
+
+# async def _merge(foo1, foo2):
+# async for gen1, gen2 in zip(foo1(), foo2()):
+# if isinstance(gen1, types.AsyncGeneratorType) and \
+# isinstance(gen2, types.AsyncGeneratorType):
+# async for f1, f2 in zip(gen1, gen2):
+# yield [f1, f2]
+# elif isinstance(gen1, types.AsyncGeneratorType):
+# async for f1 in gen1:
+# yield [f1, gen2]
+# elif isinstance(gen2, types.AsyncGeneratorType):
+# async for f2 in gen2:
+# yield [gen1, f2]
+# else:
+# yield [gen1, gen2]
+
+# return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='Merge', wraps=(f_wrap1, f_wrap2), share=None)
+
+
+# def ListMerge(f_wrap1, f_wrap2):
+# if not isinstance(f_wrap1, FunctionWrapper):
+# if not isinstance(f_wrap1, types.FunctionType):
+# f_wrap1 = Const(f_wrap1)
+# else:
+# f_wrap1 = Foo(f_wrap1)
+
+# if not isinstance(f_wrap2, FunctionWrapper):
+# if not isinstance(f_wrap2, types.FunctionType):
+# f_wrap2 = Const(f_wrap2)
+# else:
+# f_wrap2 = Foo(f_wrap2)
+
+# async def _merge(foo1, foo2):
+# async for gen1, gen2 in zip(foo1(), foo2()):
+# if isinstance(gen1, types.AsyncGeneratorType) and \
+# isinstance(gen2, types.AsyncGeneratorType):
+# async for f1, f2 in zip(gen1, gen2):
+# ret = []
+# ret.extend(f1)
+# ret.extend(f1)
+# yield ret
+# elif isinstance(gen1, types.AsyncGeneratorType):
+# async for f1 in gen1:
+# ret = []
+# ret.extend(f1)
+# ret.extend(gen2)
+# yield ret
+# elif isinstance(gen2, types.AsyncGeneratorType):
+# async for f2 in gen2:
+# ret = []
+# ret.extend(gen1)
+# ret.extend(f2)
+# yield ret
+# else:
+# ret = []
+# ret.extend(gen1)
+# ret.extend(gen2)
+# yield ret
+
+# return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='ListMerge', wraps=(f_wrap1, f_wrap2), share=None)
+
+
+# def DictMerge(f_wrap1, f_wrap2):
+# if not isinstance(f_wrap1, FunctionWrapper):
+# if not isinstance(f_wrap1, types.FunctionType):
+# f_wrap1 = Const(f_wrap1)
+# else:
+# f_wrap1 = Foo(f_wrap1)
+
+# if not isinstance(f_wrap2, FunctionWrapper):
+# if not isinstance(f_wrap2, types.FunctionType):
+# f_wrap2 = Const(f_wrap2)
+# else:
+# f_wrap2 = Foo(f_wrap2)
+
+# async def _dictmerge(foo1, foo2):
+# async for gen1, gen2 in zip(foo1(), foo2()):
+# if isinstance(gen1, types.AsyncGeneratorType) and \
+# isinstance(gen2, types.AsyncGeneratorType):
+# async for f1, f2 in zip(gen1, gen2):
+# ret = {}
+# ret.update(f1)
+# ret.update(f1)
+# yield ret
+# elif isinstance(gen1, types.AsyncGeneratorType):
+# async for f1 in gen1:
+# ret = {}
+# ret.update(f1)
+# ret.update(gen2)
+# yield ret
+# elif isinstance(gen2, types.AsyncGeneratorType):
+# async for f2 in gen2:
+# ret = {}
+# ret.update(gen1)
+# ret.update(f2)
+# yield ret
+# else:
+# ret = {}
+# ret.update(gen1)
+# ret.update(gen2)
+# yield ret
+
+# return _wrap(_dictmerge, dict(foo1=f_wrap1, foo2=f_wrap2), name='DictMerge', wraps=(f_wrap1, f_wrap2), share=None)
+
+
+# def Reduce(*f_wraps):
+# f_wraps = list(f_wraps)
+# for i, f_wrap in enumerate(f_wraps):
+# if not isinstance(f_wrap, types.FunctionType):
+# f_wraps[i] = Const(f_wrap)
+# else:
+# f_wraps[i] = Foo(f_wrap)
+
+# async def _reduce(foos):
+# async for all_gens in zip(*[foo() for foo in foos]):
+# gens = []
+# vals = []
+# for gen in all_gens:
+# if isinstance(gen, types.AsyncGeneratorType):
+# gens.append(gen)
+# else:
+# vals.append(gen)
+# if gens:
+# for gens in zip(*gens):
+# ret = list(vals)
+# for gen in gens:
+# ret.append(next(gen))
+# yield ret
+# else:
+# yield vals
+
+# return _wrap(_reduce, dict(foos=f_wraps), name='Reduce', wraps=tuple(f_wraps), share=None)
diff --git a/tributary/symbolic/__init__.py b/tributary/symbolic/__init__.py
index 4dbe5e6..8e0e5c6 100644
--- a/tributary/symbolic/__init__.py
+++ b/tributary/symbolic/__init__.py
@@ -1,5 +1,4 @@
import tributary.lazy as tl
-import tributary.reactive as tr
from sympy.utilities.lambdify import lambdify
from sympy.parsing.sympy_parser import parse_expr, standard_transformations as _st, implicit_multiplication_application as _ima
@@ -73,32 +72,32 @@ def evaluate(self):
return Lazy
-def construct_streaming(expr, modules=None):
- '''Construct Lazy tributary class from sympy expression
+# def construct_streaming(expr, modules=None):
+# '''Construct Lazy tributary class from sympy expression
- Args:
- expr (sympy expression): A Sympy expression
- modules (list): a list of modules to use for sympy's lambdify function
- Returns:
+# Args:
+# expr (sympy expression): A Sympy expression
+# modules (list): a list of modules to use for sympy's lambdify function
+# Returns:
- '''
- syms = list(symbols(expr))
- names = [s.name for s in syms]
- modules = modules or ["scipy", "numpy"]
+# '''
+# syms = list(symbols(expr))
+# names = [s.name for s in syms]
+# modules = modules or ["scipy", "numpy"]
- class Streaming(tr.StreamingGraph):
- def __init__(self, **kwargs):
- self._kwargs = {}
- for n in names:
- if n not in kwargs:
- raise Exception("Must provide input source for: {}".format(n))
- setattr(self, n, kwargs.get(n))
- self._kwargs[n] = kwargs.get(n)
+# class Streaming(tr.StreamingGraph):
+# def __init__(self, **kwargs):
+# self._kwargs = {}
+# for n in names:
+# if n not in kwargs:
+# raise Exception("Must provide input source for: {}".format(n))
+# setattr(self, n, kwargs.get(n))
+# self._kwargs[n] = kwargs.get(n)
- self._nodes = [getattr(self, n) for n in names]
-# self._function = tr.Foo(lambdify(syms, expr, modules=modules)(**self._kwargs))
- self._expr = expr
+# self._nodes = [getattr(self, n) for n in names]
+# # self._function = tr.Foo(lambdify(syms, expr, modules=modules)(**self._kwargs))
+# self._expr = expr
- super(Streaming, self).__init__(self._foo)
+# super(Streaming, self).__init__(self._foo)
- return Streaming
+# return Streaming
diff --git a/tributary/tests/streaming/calculations/test_rolling_streaming.py b/tributary/tests/streaming/calculations/test_rolling_streaming.py
new file mode 100644
index 0000000..9b63b8c
--- /dev/null
+++ b/tributary/tests/streaming/calculations/test_rolling_streaming.py
@@ -0,0 +1,16 @@
+import tributary.streaming as ts
+
+def foo():
+ yield 1
+ yield 1
+ yield 1
+ yield 1
+ yield 1
+
+
+class TestRolling:
+ def test_count(self):
+ assert ts.run(ts.Count(ts.Foo(foo))) == [1, 2, 3, 4, 5]
+
+ def test_sum(self):
+ assert ts.run(ts.Sum(ts.Foo(foo))) == [1, 2, 3, 4, 5]
diff --git a/tributary/tests/streaming/test_utils.py b/tributary/tests/streaming/test_utils.py
new file mode 100644
index 0000000..f9b1884
--- /dev/null
+++ b/tributary/tests/streaming/test_utils.py
@@ -0,0 +1,40 @@
+import time
+import tributary.streaming as ts
+
+
+def foo():
+ yield 1
+ yield 2
+
+
+def foo2():
+ yield [1, 2]
+ yield [3, 4]
+
+
+class TestUtils:
+ def test_delay(self):
+ out = ts.Delay(ts.Foo(foo), delay=5)
+ now = time.time()
+ ret = ts.run(out)
+ assert time.time() - now > 5
+ assert ret == [1, 2]
+
+ def test_apply(self):
+ def square(val):
+ return val ** 2
+
+ assert ts.run(ts.Apply(ts.Foo(foo), foo=square)) == [1, 4]
+
+ def test_window_any_size(self):
+ assert ts.run(ts.Window(ts.Foo(foo))) == [[1], [1, 2]]
+
+ def test_window_fixed_size(self):
+ assert ts.run(ts.Window(ts.Foo(foo), size=2)) == [[1], [1, 2]]
+
+ def test_window_fixed_size_full_only(self):
+ assert ts.run(ts.Window(ts.Foo(foo), size=2, full_only=True)) == [[1, 2]]
+
+ # def test_unroll(self):
+ # assert ts.run(ts.Unroll(ts.Foo(foo2))) == [1, 2, 3, 4]
+
From 6cd6a09614735b6a707e724ab60586f9da8f6eca Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:05:51 -0500
Subject: [PATCH 05/15] fix lint
---
tributary/streaming/base.py | 1 -
tributary/streaming/calculations/rolling.py | 2 ++
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/tributary/streaming/base.py b/tributary/streaming/base.py
index bda8722..8ecdcd2 100644
--- a/tributary/streaming/base.py
+++ b/tributary/streaming/base.py
@@ -2,7 +2,6 @@
import types
from aiostream.aiter_utils import anext
from asyncio import Queue, QueueEmpty as Empty
-from copy import deepcopy
from ..base import StreamEnd, StreamNone, StreamRepeat
diff --git a/tributary/streaming/calculations/rolling.py b/tributary/streaming/calculations/rolling.py
index 00d2d83..69b3023 100644
--- a/tributary/streaming/calculations/rolling.py
+++ b/tributary/streaming/calculations/rolling.py
@@ -1,5 +1,6 @@
from ..base import Node
+
class Count(Node):
'''Node to count inputs'''
def __init__(self, node, text=''):
@@ -14,6 +15,7 @@ def foo(val):
node._downstream.append((self, 0))
self._upstream.append(node)
+
class Sum(Node):
'''Node to sum inputs'''
def __init__(self, node, text=''):
From 72ac69dbb4a4d129d4a1cc0e044697b2100fdb45 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:14:56 -0500
Subject: [PATCH 06/15] rename reactive to streaming
---
.gitignore | 2 +-
README.md | 8 +-
docs/api.md | 36 +-
docs/examples/{reactive.md => streaming.md} | 16 +-
docs/img/{reactive => streaming}/example1.png | Bin
docs/img/{reactive => streaming}/example2.png | Bin
docs/img/{reactive => streaming}/example3.png | Bin
docs/img/{reactive => streaming}/example4.png | Bin
docs/img/{reactive => streaming}/http.png | Bin
docs/img/{reactive => streaming}/kafka.png | Bin
docs/img/{reactive => streaming}/sio.png | Bin
docs/img/{reactive => streaming}/ws.png | Bin
docs/index.md | 318 ---------
examples/lazy_excel.ipynb | 72 ++
examples/lazy_pricer.ipynb | 194 ++++++
examples/lazy_sigma.ipynb | 644 ++++++++++++++++++
examples/streaming_examples.ipynb | 134 ++++
...tive_kafka.ipynb => streaming_kafka.ipynb} | 0
...ve_stream.ipynb => streaming_stream.ipynb} | 0
...tive_sympy.ipynb => streaming_sympy.ipynb} | 0
..._sio.ipynb => streaming_ws_http_sio.ipynb} | 0
examples/tributary | 1 +
tributary/tests/helpers/dummy_ws.py | 2 +-
23 files changed, 1075 insertions(+), 352 deletions(-)
rename docs/examples/{reactive.md => streaming.md} (73%)
rename docs/img/{reactive => streaming}/example1.png (100%)
rename docs/img/{reactive => streaming}/example2.png (100%)
rename docs/img/{reactive => streaming}/example3.png (100%)
rename docs/img/{reactive => streaming}/example4.png (100%)
rename docs/img/{reactive => streaming}/http.png (100%)
rename docs/img/{reactive => streaming}/kafka.png (100%)
rename docs/img/{reactive => streaming}/sio.png (100%)
rename docs/img/{reactive => streaming}/ws.png (100%)
delete mode 100644 docs/index.md
create mode 100644 examples/lazy_excel.ipynb
create mode 100644 examples/lazy_pricer.ipynb
create mode 100644 examples/lazy_sigma.ipynb
create mode 100644 examples/streaming_examples.ipynb
rename examples/{reactive_kafka.ipynb => streaming_kafka.ipynb} (100%)
rename examples/{reactive_stream.ipynb => streaming_stream.ipynb} (100%)
rename examples/{reactive_sympy.ipynb => streaming_sympy.ipynb} (100%)
rename examples/{reactive_ws_http_sio.ipynb => streaming_ws_http_sio.ipynb} (100%)
create mode 120000 examples/tributary
diff --git a/.gitignore b/.gitignore
index 259a527..ade2304 100644
--- a/.gitignore
+++ b/.gitignore
@@ -173,4 +173,4 @@ docs/api/
tributary/tests/streaming/output/test_file_data.csv
python_junit.xml
tmps
-
+docs/index.md
\ No newline at end of file
diff --git a/README.md b/README.md
index 67e28a2..d71bb08 100644
--- a/README.md
+++ b/README.md
@@ -24,21 +24,17 @@ or from source
# Stream Types
Tributary offers several kinds of streams:
-## Reactive
+## Streaming
These are synchronous, reactive data streams, built using asynchronous python generators. They are designed to mimic complex event processors in terms of event ordering.
## Functional
These are functional streams, built by currying python functions (callbacks).
-## Event Loop
-TODO
-These function as tornado based event-loop based streams similar to streamz.
-
## Lazy
These are lazily-evaluated python streams, where outputs are propogated only as inputs change.
# Examples
-- [Reactive](docs/examples/reactive.md)
+- [Streaming](docs/examples/streaming.md)
- [Lazy](docs/examples/lazy.md)
# Math
diff --git a/docs/api.md b/docs/api.md
index 174b3e1..ddccd85 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -1,23 +1,23 @@
# API
-## Reactive
+## Streaming
```eval_rst
-.. automodule:: tributary.reactive
+.. automodule:: tributary.streaming
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.base
+.. automodule:: tributary.streaming.base
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.utils
+.. automodule:: tributary.streaming.utils
:members:
:undoc-members:
:show-inheritance:
@@ -27,42 +27,42 @@
```eval_rst
-.. automodule:: tributary.reactive.input
+.. automodule:: tributary.streaming.input
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.input.file
+.. automodule:: tributary.streaming.input.file
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.input.http
+.. automodule:: tributary.streaming.input.http
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.input.kafka
+.. automodule:: tributary.streaming.input.kafka
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.input.socketio
+.. automodule:: tributary.streaming.input.socketio
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.input.ws
+.. automodule:: tributary.streaming.input.ws
:members:
:undoc-members:
:show-inheritance:
@@ -72,35 +72,35 @@
```eval_rst
-.. automodule:: tributary.reactive.output
+.. automodule:: tributary.streaming.output
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.output.http
+.. automodule:: tributary.streaming.output.http
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.output.kafka
+.. automodule:: tributary.streaming.output.kafka
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.output.socketio
+.. automodule:: tributary.streaming.output.socketio
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.output.ws
+.. automodule:: tributary.streaming.output.ws
:members:
:undoc-members:
:show-inheritance:
@@ -111,21 +111,21 @@
```eval_rst
-.. automodule:: tributary.reactive.calculations
+.. automodule:: tributary.streaming.calculations
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.calculations.ops
+.. automodule:: tributary.streaming.calculations.ops
:members:
:undoc-members:
:show-inheritance:
```
```eval_rst
-.. automodule:: tributary.reactive.calculations.rolling
+.. automodule:: tributary.streaming.calculations.rolling
:members:
:undoc-members:
:show-inheritance:
diff --git a/docs/examples/reactive.md b/docs/examples/streaming.md
similarity index 73%
rename from docs/examples/reactive.md
rename to docs/examples/streaming.md
index 81c06d7..3d3cb87 100644
--- a/docs/examples/reactive.md
+++ b/docs/examples/streaming.md
@@ -1,28 +1,28 @@
## Simple Example
-
+
## More Complex Example
-
+
## Rolling Mean
-
+
## Custom Calculations and Window Functions
-
+
## Sources
### WebSocket
-
+
### HTTP
-
+
### SocketIO
-
+
### Kafka
-
+
## Sinks
### HTTP
diff --git a/docs/img/reactive/example1.png b/docs/img/streaming/example1.png
similarity index 100%
rename from docs/img/reactive/example1.png
rename to docs/img/streaming/example1.png
diff --git a/docs/img/reactive/example2.png b/docs/img/streaming/example2.png
similarity index 100%
rename from docs/img/reactive/example2.png
rename to docs/img/streaming/example2.png
diff --git a/docs/img/reactive/example3.png b/docs/img/streaming/example3.png
similarity index 100%
rename from docs/img/reactive/example3.png
rename to docs/img/streaming/example3.png
diff --git a/docs/img/reactive/example4.png b/docs/img/streaming/example4.png
similarity index 100%
rename from docs/img/reactive/example4.png
rename to docs/img/streaming/example4.png
diff --git a/docs/img/reactive/http.png b/docs/img/streaming/http.png
similarity index 100%
rename from docs/img/reactive/http.png
rename to docs/img/streaming/http.png
diff --git a/docs/img/reactive/kafka.png b/docs/img/streaming/kafka.png
similarity index 100%
rename from docs/img/reactive/kafka.png
rename to docs/img/streaming/kafka.png
diff --git a/docs/img/reactive/sio.png b/docs/img/streaming/sio.png
similarity index 100%
rename from docs/img/reactive/sio.png
rename to docs/img/streaming/sio.png
diff --git a/docs/img/reactive/ws.png b/docs/img/streaming/ws.png
similarity index 100%
rename from docs/img/reactive/ws.png
rename to docs/img/streaming/ws.png
diff --git a/docs/index.md b/docs/index.md
deleted file mode 100644
index 8f1b293..0000000
--- a/docs/index.md
+++ /dev/null
@@ -1,318 +0,0 @@
-# <a href="https://tributary.readthedocs.io"><img src="img/icon.png" width="300"></a>
-Python Data Streams
-
-[](https://travis-ci.org/timkpaine/tributary)
-[]()
-[](https://codecov.io/gh/timkpaine/tributary)
-[](https://bettercodehub.com/)
-[](https://pypi.python.org/pypi/tributary)
-[](https://pypi.python.org/pypi/tributary)
-[](https://tributary.readthedocs.io)
-
-
-
-
-
-# Installation
-Install from pip:
-
-`pip install tributary`
-
-or from source
-
-`python setup.py install`
-
-
-# Stream Types
-Tributary offers several kinds of streams:
-
-## Reactive
-These are synchronous, reactive data streams, built using asynchronous python generators. They are designed to mimic complex event processors in terms of event ordering.
-
-## Functional
-These are functional streams, built by currying python functions (callbacks).
-
-## Event Loop
-TODO
-These function as tornado based event-loop based streams similar to streamz.
-
-## Lazy
-These are lazily-evaluated python streams, where outputs are propogated only as inputs change.
-
-# Examples
-- [Reactive](examples/reactive.md)
-- [Lazy](examples/lazy.md)
-
-# Math
-`(Work in progress)`
-
-## Operations
-- unary operators/comparators
-- binary operators/comparators
-
-## Rolling
-- count
-- sum
-
-# Sources and Sinks
-`(Work in progress)`
-
-## Sources
-- file
-- kafka
-- websocket
-- http
-- socket io
-
-## Sinks
-- file
-- kafka
-- http
-- TODO websocket
-- TODO socket io
-# API Documentation
-
-# API
-
-## Reactive
-
-```eval_rst
-.. automodule:: tributary.reactive
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.base
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.utils
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-### Input
-
-
-```eval_rst
-.. automodule:: tributary.reactive.input
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.input.file
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.input.http
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.input.kafka
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.input.socketio
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.input.ws
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-### Output
-
-
-```eval_rst
-.. automodule:: tributary.reactive.output
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.output.http
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.output.kafka
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.output.socketio
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.output.ws
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-
-### Calculations
-
-
-```eval_rst
-.. automodule:: tributary.reactive.calculations
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.calculations.ops
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.reactive.calculations.rolling
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-
-## Functional
-
-```eval_rst
-.. automodule:: tributary.functional
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.functional.utils
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-### Input
-
-
-```eval_rst
-.. automodule:: tributary.functional.input
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-### Output
-
-
-```eval_rst
-.. automodule:: tributary.functional.output
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-## Lazy
-
-
-```eval_rst
-.. automodule:: tributary.lazy
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.lazy.base
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-### Input
-
-
-### Output
-
-
-### Calculations
-
-
-```eval_rst
-.. automodule:: tributary.lazy.calculations
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-
-```eval_rst
-.. automodule:: tributary.lazy.calculations.ops
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-
-## Symbolic
-
-
-```eval_rst
-.. automodule:: tributary.symbolic
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-## Common
-
-```eval_rst
-.. automodule:: tributary.base
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.thread
- :members:
- :undoc-members:
- :show-inheritance:
-```
-
-```eval_rst
-.. automodule:: tributary.utils
- :members:
- :undoc-members:
- :show-inheritance:
-```
\ No newline at end of file
diff --git a/examples/lazy_excel.ipynb b/examples/lazy_excel.ipynb
new file mode 100644
index 0000000..617742d
--- /dev/null
+++ b/examples/lazy_excel.ipynb
@@ -0,0 +1,72 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import tributary.symbolic as ts\n",
+ "import ipysheet as ips"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "fc1a6d7e0f1c423c91232825a42b3c46",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "MySheet()"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "class MySheet(ips.Sheet):\n",
+ " def __init__(self):\n",
+ " super(MySheet, self).__init__()\n",
+ "\n",
+ " \n",
+ "ms = MySheet()\n",
+ "ms"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/lazy_pricer.ipynb b/examples/lazy_pricer.ipynb
new file mode 100644
index 0000000..b9abe55
--- /dev/null
+++ b/examples/lazy_pricer.ipynb
@@ -0,0 +1,194 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "ModuleNotFoundError",
+ "evalue": "No module named 'tributary.symbolic'",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-42-a5dc375e66de>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtributary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlazy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtributary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msymbolic\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpyEX\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tributary.symbolic'"
+ ]
+ }
+ ],
+ "source": [
+ "import tributary.lazy as t\n",
+ "import pyEX as p\n",
+ "import pandas as pd\n",
+ "from datetime import datetime, timedelta"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "class MyPricer(t.BaseClass):\n",
+ " def __init__(self, symbol='AAPL'):\n",
+ " self.symbol = self.node('symbol', default_or_starting_value=symbol, trace=True)\n",
+ " self.date = self.node('date', default_or_starting_value=datetime.today()-timedelta(days=1), trace=True)\n",
+ " self.x = self.node('x', readonly=False, default_or_starting_value=1, trace=True)\n",
+ "\n",
+ " @t.node(trace=True)\n",
+ " def _fetch_data(self, symbol, date):\n",
+ " df = p.chartDF(symbol)\n",
+ " date = pd.Timestamp(date.date())\n",
+ " return df.loc[date].to_dict()\n",
+ " \n",
+ " @t.node(trace=True)\n",
+ " def fetch_data(self):\n",
+ " self._fetch_data().set(symbol=self.symbol(), date=self.date())\n",
+ " return self._fetch_data()\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m = MyPricer('AAPL')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/svg+xml": [
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+ "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+ " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+ "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
+ " -->\n",
+ "<!-- Title: _fetch_data Pages: 1 -->\n",
+ "<svg width=\"178pt\" height=\"116pt\"\n",
+ " viewBox=\"0.00 0.00 177.72 116.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+ "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 112)\">\n",
+ "<title>_fetch_data</title>\n",
+ "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-112 173.7185,-112 173.7185,4 -4,4\"/>\n",
+ "<!-- _fetch_data-0 -->\n",
+ "<g id=\"node1\" class=\"node\">\n",
+ "<title>_fetch_data-0</title>\n",
+ "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"90.5159\" cy=\"-18\" rx=\"58.9408\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"90.5159\" y=\"-13.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">_fetch_data-0</text>\n",
+ "</g>\n",
+ "<!-- symbol-1 -->\n",
+ "<g id=\"node2\" class=\"node\">\n",
+ "<title>symbol-1</title>\n",
+ "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"43.5159\" cy=\"-90\" rx=\"43.5319\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"43.5159\" y=\"-85.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">symbol-1</text>\n",
+ "</g>\n",
+ "<!-- symbol-1->_fetch_data-0 -->\n",
+ "<g id=\"edge1\" class=\"edge\">\n",
+ "<title>symbol-1->_fetch_data-0</title>\n",
+ "<path fill=\"none\" stroke=\"#000000\" d=\"M54.8933,-72.5708C60.4109,-64.1184 67.1562,-53.7851 73.2871,-44.3931\"/>\n",
+ "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"76.3814,-46.0559 78.9168,-35.7689 70.5197,-42.2295 76.3814,-46.0559\"/>\n",
+ "</g>\n",
+ "<!-- date-2 -->\n",
+ "<g id=\"node3\" class=\"node\">\n",
+ "<title>date-2</title>\n",
+ "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"137.5159\" cy=\"-90\" rx=\"32.4063\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"137.5159\" y=\"-85.8\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">date-2</text>\n",
+ "</g>\n",
+ "<!-- date-2->_fetch_data-0 -->\n",
+ "<g id=\"edge2\" class=\"edge\">\n",
+ "<title>date-2->_fetch_data-0</title>\n",
+ "<path fill=\"none\" stroke=\"#000000\" d=\"M126.3776,-72.937C120.7661,-64.3407 113.84,-53.7304 107.5755,-44.1338\"/>\n",
+ "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"110.4623,-42.153 102.0652,-35.6924 104.6006,-45.9794 110.4623,-42.153\"/>\n",
+ "</g>\n",
+ "</g>\n",
+ "</svg>\n"
+ ],
+ "text/plain": [
+ "<graphviz.dot.Digraph at 0x119813748>"
+ ]
+ },
+ "execution_count": 39,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "m.fetch_data().graphviz()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "recomputing: _fetch_data-4723070512\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "{'change': -1.95,\n",
+ " 'changeOverTime': 0.0720886184927969,\n",
+ " 'changePercent': -1.033,\n",
+ " 'close': 186.79,\n",
+ " 'high': 192.88,\n",
+ " 'label': 'Mar 26',\n",
+ " 'low': 184.58,\n",
+ " 'open': 191.664,\n",
+ " 'unadjustedVolume': 49800538,\n",
+ " 'volume': 49800538,\n",
+ " 'vwap': 189.3728}"
+ ]
+ },
+ "execution_count": 40,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "m.fetch_data()()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ " "
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.2"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/lazy_sigma.ipynb b/examples/lazy_sigma.ipynb
new file mode 100644
index 0000000..f0b1a0b
--- /dev/null
+++ b/examples/lazy_sigma.ipynb
@@ -0,0 +1,644 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import tributary.symbolic as ts\n",
+ "expr = ts.parse_expression(\"10sin**2 x**2 + 3xyz + tan theta\")\n",
+ "clz = ts.construct_lazy(expr)\n",
+ "x = clz(x=1, y=2, z=3, theta=4)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAALUAAAAPCAYAAACiAo66AAAACXBIWXMAAA7EAAAOxAGVKw4bAAAGfElEQVRoBeWajXEUNxTHfQwFGKcD04GBDpwOyFBBoAMYKmBIB6SDhHQAHSS4A+jAsTtwfr+NnuZpP+50570JM9GMLOnpr/e9knbPm7u7u5P/Q9lsNhfYepVthXbK+Az6t6CvjQu+a7e9eq4tV37/lWzkniP+kvo7MbtVl7myMakL+E0BPKX9m/pmnATBAPz76Jf2tyVsxvXKKbhXZa2JpzHvkfF5xM+5t9Rr6g9Uxx/HOGjaeFPmI7HFWp6Av/23uz4u+OYWXS4Yv0Ju2Jin1XWnf3vtkXHip58eU/VlfZAL5hi+NG478wr9umSDew6/j+q7UG6x65FzCjYRTqIy1qlu4ZdBs6WI/Zrp9BXyNePm+mC65IDTwA+ZB+OXVBV4PqI3OOco6tPgCl29rYJstfE08zsGbsw/yWh8Xujd/i029NjzBWz1hzaXtedZN2ir+hJ+XfEudnfJhqcx0x5j/GFU9cVgJ+0AmAuuO9vNyHAZNgow/mTNuLk+GBXZKQfMa2qTwK4rtC/Bm7GJ/jLG0RZsxSX6JIliLresXxWXedunaJ8BmMiB1u3fufUzsvTRZMOBZnLUmNFf3Zfw7I13t2x4NrkX9kL3AapzDxh4R7kpRwDdWjzqT6G7wCPMrd9jM44Tye7uP1qHwfY/XXJg4fXgttSBI/wdj4vH6E9j4vc8xof6r7Et9F3Bv8Eqt/onrluZ/ieDS2S6WViO4cveeO8j2w10rnidqlc5k9rk/baQODIIw727emeZSzBxu0qXHPh/pj6yDYYl4A49cqJooIH5lILjnLtQxkn7XsoL7Pp1QZn7+neOrYnl+9G4RAydtxzDl13x3kc2vvtj0Db9IfbG+10inTwEuLTbuau4E8eT7vgKJrYvqNdUnzKP0ZqAjGfLHnKa9cjT8Srui1VNCGUyp5GeIJ40niDq47E6MR66p41HXTykYj2ywj4hQ1kbJ1N4eu3Y9rDt7d9tejIXdg42Lfw5k34MX8KzK68OkR22YKO3CL9qNbeH+nLIRO0L1Fbqa+kUHWTHO19zj2Xs3bu+iGQ+u/qsa+RkfJkzEbybWVW+6hh96Ca8uoV+zQtQwrkb1Ts9fR3imvHL8Ko45RdZ1W+M9WO9U9Pf27+s2aon82GfR3PjN+Zm/Q59VV/2yi0+6pKdeaKvedHEb+CVQdEH6ItMvXjTD6cDmTjI3Wf4XDae2zUey1nCg3M3Njuah6fQdYYBNMhiJom6ha9rJi9SY3zhfTCO9U1iMV5KakQf7t+sJ/2epK56gdfHR/Ul/Ju8ClsPkc2awb7gkdvmCXaCYpLWhA4wNCcngYXmkd6dSInfrJyYH7fI8MFRzrDb0iq30ZNxJH/XQ+Z6qjxnd/fQ4T441qpnw59xk9TKofjnXv5lfbWHfmxENXGTPV7plBcn8dF9WXRr4lXsPkg2/HwAJ1+55PmAiVrKHc1Er2+SdZKXSfq3aTzu+uR0lW1ymLuwzjD6q9Di5UajmrsUenuX9q5sQAPnfdaXSRNpqYg/Bk6f+BA2P3QsKNHt3x57kBmxGmwbyQxa6LW6L7M89DVxl/KqW3bmSd8NbO4l+ORhABEs6HFOaGhDopag6IBtiRsOCpaz7TY5LFDJIfnA+QUkAtPwYs6gmCyTeXVl3uQ+S4ue0p9zwIBhTbwsro3TX8/Qx7tfLj6054Xulycfzn3826unL/BzMQvf+LJ9LF8O9sJ/Ma8AGJN94hg81Vm7Im4Dvf7BofZ18twx5YtaHPfDkSU+V+Y98pofafJ87oPrkePVob5ExXplUFU29HHcHOsJ6125ztGf2CaWIo/8I8SquNBn3Ba5jY3Quv0LtldPd8hJbKAZs3p0i6FWf2V9oR/ky+Lf3nh3yQ690GnwFe3kOlPkDhk/XOAFperu0tzxGGtgdSh9nxgdUl/gCs2MqU4rgnyydsoBYyDql4Ky1qddnpVOX8Pc1YckTwa7frgrJpqyGweIoap7XU9/VVzIH7fI0Zb6MMW8NOpW/4qldOlZsPp8HB/trl+T6B/Llz3x7pad/BT5UH0Vc7YbDDIxfKLmyhWgJ3mC48Q7kMlsOaO+A9McA2A0xh9R6t0cWrccsBqav3MaRA1ovoeDU++3VI+xKLPfzcHKI+7g6u0a/2nrNhbaro0b8XbT8OoQ/tYek/uXwCF/p3/F7qGnsdJH166jPKPOxWxVX6LfPvHulq0BxXb5/4zvJr9J/ANNwmUzU3tTNwAAAABJRU5ErkJggg==\n",
+ "text/latex": [
+ "$\\displaystyle 26.238555465085287$"
+ ],
+ "text/plain": [
+ "26.238555465085287"
+ ]
+ },
+ "execution_count": 2,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x.evaluate()()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAAPCAYAAAB0p1TfAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAGVklEQVRoBeWai3EVNxSGbU8KcJwKYjrg0YHTAQwVxHRAhgoy0AHpION0gDsguAPcAdgdON8ndDTSrnav7OzNJBPNyCsd/To6L0lHFw7v7u4O/i/l8PDwMfpe1fpCO6Z/Av066Fvjgu/W31E5t15Xfv/02odLgaogyPOK8Vdrio7iggf4U9q/5P5Tvl/tdwJIXKxtMNl/C+6Sbynwc+wN9Qv1B6r9iykOmsa9yeMRrGItT8Dffmtujwu+9XeX3Rh/W+Np/96x0ZA+8qn4aadHVG1ZNmfG7MOWQ/7O67/3m8sJ35+LX2gc9Cqgz1Qd3h0P+ihOPEWhG570dYiDZxVPDfY++nnuecY9n9AbXMZegG1wma5OVtfz69rHNb994Kb8qzUaW2S6NlK22h7q83nKJ+NG9PkEttiDtvZ13mnNk/6mtpQ/tdGR/pK/led1yEPbg9KNmGTsBqETqE5sFgkm8R3FVXgN3gsMBbqpcK5vME2NK+1ThTN4z6MfX2g6ouAq+qo++8IF3/gi26J9lZvaBAz9D9aYH19oO/UBo416QW7AFJ4Zt6kt4Tnqb3HF/5V+6p1scUSjKflKuoVoXSyjuAmDM/o3zDWQ6uJ1fgzdHWjxam5kQPiePF5hL5zwXylrdmPsOXp4kkRqlNRC95+sD9RR+0SqU7P4SOes8sU+bDnqb/Vu0pAsqHK70Q5mgQrtJUb5zcEdZRRXszEgrxeCTlwKYMYvqd/7jcnZiXbrPMYdp7E/VAYX42lR46T9W8qa3cy1b1fs8xAdDJavnYmx8R237MOWO/1d+a0no/l0erh9ZyMKk7ySdjp4FBd844sDlk4/TxHTkN7OV1CNafD5uCubiPYlY39Ad0d6UnsSeTJ4pUmfFTDu0LQh+Ir1apmtuzVOQeC5y77a4Qqc35fUePh4xZdNC72UNTkZCz0LvtM4kbYPW8JzyN/IqQhJDhtV8XFsOSk5Kh2v3ZKj0DZXmuVAoziELLzX2vDTKeaeJZEOfB7TueYwVn9emvGFHgl6ymHpN4+EmAPdU6PkyOKozikPF7GUTXGZ56p9WdOgCvmLH/Jcc/iSr4/qw5zQzxd+YzfGunaHvqktR9ZlzbUcVZucF+EVsGZKfylQh3A1r7U26/hoax4PPTwYT02FbhyW6RpXpxhgYqxN8PV4SqM4Z/bYmOL/Lo75q3ZjPAKVpWdB5S2Xfoqajk37tZy0RwK1yAVeG+/VlvCf+Ruauqtf8RltN5IBnHx+RMMryetw5MofwslzpLCua5qPxu+li9PAeJXfUi/iSsty+9Dwd1hzXx8ccd0U3CLTbwMm8afw0qlr5cG4LOeqfZFd3Sy9R4WHhs58KmBHqeXs5X0xPa5a04uIgb3aEjt0/Z11/xExXoB5TY3N8jELe30EUQd5HfYMlHFJkSFcmbCjwbrpNce6syBl7LG1w+LPTIsHgApNX8gGtLmnjg2cjvDBpcOXivh94O5jN30QAduTs2ymEX2q4E+6TRgGLfy+uS3r9ZB30d/ilNVYoL6jevCEHx2+9jGl8s9g5DFbFwPFk0a6ynhF7sS5CLjVAk+vmEdgS5BCCyd4CqSAgubLv+s4xjS0G2w2Du2acRWNU4NmOo16J0zCMCceVJ5aW+KG7Mv62k07hx1ozkoElQOjcvoI6/EM2/gg3ZctkwLwX/S3vkqg/h919Ma9LTkqnaYNwJxh9ph6KC7mwdMNUPKiiu6jKT10+HbXznRzlhq39nAqY8yZrenamWf9w/emuNBv+s3rNvaF5g0AdOYLr8zmB3H6o3J6kjVz5U+RZ/lHETHUYq9aBugeUmWM9tDaeZ0RfxvIrp/8mue5uRQ0PaCb4JwIJ6g4sB6r25lZg4PmDnV+MUS1eEqmGdNQUT21y4OGtsadvnxVRp6FTlvHpvxtIpPzm18R6Kt482gTQ+0ZaDNcLVfdZt2ufaEbFCUQaGtLZZw+Iof0yXbX5mV+xbP8igJtX7Yc8bd+EFcHqnYovp79pxSOaYPHI9edYPHqMBDfpV7+swvHuAs3DyVoBlXwrdnZvmKNJ0EEq+HiYSRZx+jA5vdEcPJ7Q62v6+7vjmDlEanJSZ5jPtSkD1vjWKcUeO+0LxjzRQPUopy/ImOkJonon3vIKS9t9MV5lGfUGU/4bWpL+N3H3+ocRXkbH/4FHtSUqElbOlsAAAAASUVORK5CYII=\n",
+ "text/latex": [
+ "$\\displaystyle 44.23855546508529$"
+ ],
+ "text/plain": [
+ "44.23855546508529"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x.x = 2\n",
+ "x.evaluate()()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/svg+xml": [
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+ "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+ " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+ "<!-- Generated by graphviz version 2.43.0 (0)\n",
+ " -->\n",
+ "<!-- Title: evaluate Pages: 1 -->\n",
+ "<svg width=\"671pt\" height=\"548pt\"\n",
+ " viewBox=\"0.00 0.00 671.47 548.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+ "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 544)\">\n",
+ "<title>evaluate</title>\n",
+ "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-544 667.47,-544 667.47,4 -4,4\"/>\n",
+ "<!-- evaluate (#0) -->\n",
+ "<g id=\"node1\" class=\"node\">\n",
+ "<title>evaluate (#0)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"442.34\" cy=\"-18\" rx=\"56.52\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"442.34\" y=\"-13.8\" font-family=\"Times,serif\" font-size=\"14.00\">evaluate (#0)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"node2\" class=\"node\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"442.34\" cy=\"-90\" rx=\"221.26\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"442.34\" y=\"-85.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)->evaluate (#0) -->\n",
+ "<g id=\"edge19\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)->evaluate (#0)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M442.34,-71.7C442.34,-63.98 442.34,-54.71 442.34,-46.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"445.84,-46.1 442.34,-36.1 438.84,-46.1 445.84,-46.1\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"node3\" class=\"node\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"309.34\" cy=\"-162\" rx=\"182.02\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"309.34\" y=\"-157.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"edge16\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M341.2,-144.23C359.24,-134.74 382.01,-122.75 401.37,-112.56\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"403.06,-115.63 410.28,-107.87 399.8,-109.43 403.06,-115.63\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z (#3) -->\n",
+ "<g id=\"node4\" class=\"node\">\n",
+ "<title>x*var(3)*y*z (#3)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"146.34\" cy=\"-306\" rx=\"74.88\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"146.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z (#3)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z (#3)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"edge7\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z (#3)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M152.92,-287.91C161.2,-268.36 177.06,-236.4 199.34,-216 214.19,-202.41 233.15,-191.61 251.14,-183.4\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"252.67,-186.55 260.43,-179.34 249.86,-180.14 252.67,-186.55\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y (#4) -->\n",
+ "<g id=\"node5\" class=\"node\">\n",
+ "<title>x*var(3)*y (#4)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"66.34\" cy=\"-378\" rx=\"66.18\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y (#4)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y (#4)->x*var(3)*y*z (#3) -->\n",
+ "<g id=\"edge5\" class=\"edge\">\n",
+ "<title>x*var(3)*y (#4)->x*var(3)*y*z (#3)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M85.3,-360.41C95.66,-351.34 108.66,-339.97 119.99,-330.06\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"122.3,-332.69 127.52,-323.47 117.69,-327.42 122.3,-332.69\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3) (#5) -->\n",
+ "<g id=\"node6\" class=\"node\">\n",
+ "<title>x*var(3) (#5)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"66.34\" cy=\"-450\" rx=\"57.5\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3) (#5)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3) (#5)->x*var(3)*y (#4) -->\n",
+ "<g id=\"edge3\" class=\"edge\">\n",
+ "<title>x*var(3) (#5)->x*var(3)*y (#4)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M66.34,-431.7C66.34,-423.98 66.34,-414.71 66.34,-406.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"69.84,-406.1 66.34,-396.1 62.84,-406.1 69.84,-406.1\"/>\n",
+ "</g>\n",
+ "<!-- x (#6) -->\n",
+ "<g id=\"node7\" class=\"node\">\n",
+ "<title>x (#6)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"185.34\" cy=\"-522\" rx=\"31.45\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"185.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">x (#6)</text>\n",
+ "</g>\n",
+ "<!-- x (#6)->x*var(3) (#5) -->\n",
+ "<g id=\"edge1\" class=\"edge\">\n",
+ "<title>x (#6)->x*var(3) (#5)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M164.07,-508.49C146.6,-498.21 121.5,-483.45 101.12,-471.46\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"102.81,-468.39 92.42,-466.34 99.26,-474.42 102.81,-468.39\"/>\n",
+ "</g>\n",
+ "<!-- x^var(2) (#13) -->\n",
+ "<g id=\"node14\" class=\"node\">\n",
+ "<title>x^var(2) (#13)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"296.34\" cy=\"-450\" rx=\"61.8\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"296.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">x^var(2) (#13)</text>\n",
+ "</g>\n",
+ "<!-- x (#6)->x^var(2) (#13) -->\n",
+ "<g id=\"edge8\" class=\"edge\">\n",
+ "<title>x (#6)->x^var(2) (#13)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M205.7,-508.16C221.69,-498.08 244.3,-483.82 262.94,-472.07\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"264.86,-474.99 271.45,-466.7 261.12,-469.07 264.86,-474.99\"/>\n",
+ "</g>\n",
+ "<!-- var(3) (#7) -->\n",
+ "<g id=\"node8\" class=\"node\">\n",
+ "<title>var(3) (#7)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"66.34\" cy=\"-522\" rx=\"48.81\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(3) (#7)</text>\n",
+ "</g>\n",
+ "<!-- var(3) (#7)->x*var(3) (#5) -->\n",
+ "<g id=\"edge2\" class=\"edge\">\n",
+ "<title>var(3) (#7)->x*var(3) (#5)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M66.34,-503.7C66.34,-495.98 66.34,-486.71 66.34,-478.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"69.84,-478.1 66.34,-468.1 62.84,-478.1 69.84,-478.1\"/>\n",
+ "</g>\n",
+ "<!-- y (#8) -->\n",
+ "<g id=\"node9\" class=\"node\">\n",
+ "<title>y (#8)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"173.34\" cy=\"-450\" rx=\"31.45\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"173.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">y (#8)</text>\n",
+ "</g>\n",
+ "<!-- y (#8)->x*var(3)*y (#4) -->\n",
+ "<g id=\"edge4\" class=\"edge\">\n",
+ "<title>y (#8)->x*var(3)*y (#4)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M153.47,-436C138.2,-426.01 116.78,-412 99,-400.37\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"100.9,-397.42 90.61,-394.88 97.06,-403.28 100.9,-397.42\"/>\n",
+ "</g>\n",
+ "<!-- z (#9) -->\n",
+ "<g id=\"node10\" class=\"node\">\n",
+ "<title>z (#9)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"181.34\" cy=\"-378\" rx=\"30.95\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"181.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">z (#9)</text>\n",
+ "</g>\n",
+ "<!-- z (#9)->x*var(3)*y*z (#3) -->\n",
+ "<g id=\"edge6\" class=\"edge\">\n",
+ "<title>z (#9)->x*var(3)*y*z (#3)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M173.05,-360.41C168.97,-352.25 163.95,-342.22 159.37,-333.07\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"162.42,-331.34 154.82,-323.96 156.16,-334.47 162.42,-331.34\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"node11\" class=\"node\">\n",
+ "<title>sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"342.34\" cy=\"-234\" rx=\"134.12\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"342.34\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2))^var(2)*var(10) (#10)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2)*var(10) (#10)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"edge15\" class=\"edge\">\n",
+ "<title>sin(x^var(2))^var(2)*var(10) (#10)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M334.18,-215.7C330.43,-207.73 325.89,-198.1 321.72,-189.26\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"324.83,-187.66 317.4,-180.1 318.5,-190.64 324.83,-187.66\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"node12\" class=\"node\">\n",
+ "<title>sin(x^var(2))^var(2) (#11)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"342.34\" cy=\"-306\" rx=\"103.07\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"342.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2))^var(2) (#11)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2) (#11)->sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"edge13\" class=\"edge\">\n",
+ "<title>sin(x^var(2))^var(2) (#11)->sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M342.34,-287.7C342.34,-279.98 342.34,-270.71 342.34,-262.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"345.84,-262.1 342.34,-252.1 338.84,-262.1 345.84,-262.1\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2)) (#12) -->\n",
+ "<g id=\"node13\" class=\"node\">\n",
+ "<title>sin(x^var(2)) (#12)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"308.34\" cy=\"-378\" rx=\"77.72\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"308.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2)) (#12)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2)) (#12)->sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"edge11\" class=\"edge\">\n",
+ "<title>sin(x^var(2)) (#12)->sin(x^var(2))^var(2) (#11)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M316.57,-360.05C320.44,-352.09 325.14,-342.41 329.47,-333.51\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"332.73,-334.8 333.95,-324.28 326.43,-331.74 332.73,-334.8\"/>\n",
+ "</g>\n",
+ "<!-- x^var(2) (#13)->sin(x^var(2)) (#12) -->\n",
+ "<g id=\"edge10\" class=\"edge\">\n",
+ "<title>x^var(2) (#13)->sin(x^var(2)) (#12)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M299.31,-431.7C300.63,-423.98 302.22,-414.71 303.69,-406.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"307.17,-406.55 305.41,-396.1 300.27,-405.37 307.17,-406.55\"/>\n",
+ "</g>\n",
+ "<!-- var(2) (#15) -->\n",
+ "<g id=\"node15\" class=\"node\">\n",
+ "<title>var(2) (#15)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"296.34\" cy=\"-522\" rx=\"53.15\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"296.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(2) (#15)</text>\n",
+ "</g>\n",
+ "<!-- var(2) (#15)->x^var(2) (#13) -->\n",
+ "<g id=\"edge9\" class=\"edge\">\n",
+ "<title>var(2) (#15)->x^var(2) (#13)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M296.34,-503.7C296.34,-495.98 296.34,-486.71 296.34,-478.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"299.84,-478.1 296.34,-468.1 292.84,-478.1 299.84,-478.1\"/>\n",
+ "</g>\n",
+ "<!-- var(2) (#16) -->\n",
+ "<g id=\"node16\" class=\"node\">\n",
+ "<title>var(2) (#16)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"457.34\" cy=\"-378\" rx=\"53.15\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"457.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(2) (#16)</text>\n",
+ "</g>\n",
+ "<!-- var(2) (#16)->sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"edge12\" class=\"edge\">\n",
+ "<title>var(2) (#16)->sin(x^var(2))^var(2) (#11)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M432.67,-361.98C416.82,-352.34 395.97,-339.64 378.29,-328.88\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"379.74,-325.67 369.38,-323.46 376.1,-331.65 379.74,-325.67\"/>\n",
+ "</g>\n",
+ "<!-- var(10) (#17) -->\n",
+ "<g id=\"node17\" class=\"node\">\n",
+ "<title>var(10) (#17)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"521.34\" cy=\"-306\" rx=\"57.5\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"521.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(10) (#17)</text>\n",
+ "</g>\n",
+ "<!-- var(10) (#17)->sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"edge14\" class=\"edge\">\n",
+ "<title>var(10) (#17)->sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M486.83,-291.5C460.24,-281.11 423.09,-266.58 393.12,-254.86\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"394.25,-251.54 383.66,-251.16 391.7,-258.06 394.25,-251.54\"/>\n",
+ "</g>\n",
+ "<!-- tan(theta) (#18) -->\n",
+ "<g id=\"node18\" class=\"node\">\n",
+ "<title>tan(theta) (#18)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"575.34\" cy=\"-162\" rx=\"65.21\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"575.34\" y=\"-157.8\" font-family=\"Times,serif\" font-size=\"14.00\">tan(theta) (#18)</text>\n",
+ "</g>\n",
+ "<!-- tan(theta) (#18)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"edge18\" class=\"edge\">\n",
+ "<title>tan(theta) (#18)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M546.48,-145.81C528.06,-136.12 503.9,-123.4 483.49,-112.66\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"484.89,-109.44 474.41,-107.88 481.63,-115.63 484.89,-109.44\"/>\n",
+ "</g>\n",
+ "<!-- theta (#19) -->\n",
+ "<g id=\"node19\" class=\"node\">\n",
+ "<title>theta (#19)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"575.34\" cy=\"-234\" rx=\"48.82\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"575.34\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\">theta (#19)</text>\n",
+ "</g>\n",
+ "<!-- theta (#19)->tan(theta) (#18) -->\n",
+ "<g id=\"edge17\" class=\"edge\">\n",
+ "<title>theta (#19)->tan(theta) (#18)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M575.34,-215.7C575.34,-207.98 575.34,-198.71 575.34,-190.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"578.84,-190.1 575.34,-180.1 571.84,-190.1 578.84,-190.1\"/>\n",
+ "</g>\n",
+ "</g>\n",
+ "</svg>\n"
+ ],
+ "text/plain": [
+ "<graphviz.dot.Digraph at 0x12327ca90>"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x.evaluate().graphviz()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/svg+xml": [
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
+ "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
+ " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
+ "<!-- Generated by graphviz version 2.43.0 (0)\n",
+ " -->\n",
+ "<!-- Title: evaluate Pages: 1 -->\n",
+ "<svg width=\"671pt\" height=\"548pt\"\n",
+ " viewBox=\"0.00 0.00 671.47 548.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
+ "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 544)\">\n",
+ "<title>evaluate</title>\n",
+ "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-544 667.47,-544 667.47,4 -4,4\"/>\n",
+ "<!-- evaluate (#0) -->\n",
+ "<g id=\"node1\" class=\"node\">\n",
+ "<title>evaluate (#0)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"442.34\" cy=\"-18\" rx=\"56.52\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"442.34\" y=\"-13.8\" font-family=\"Times,serif\" font-size=\"14.00\">evaluate (#0)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"node2\" class=\"node\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"442.34\" cy=\"-90\" rx=\"221.26\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"442.34\" y=\"-85.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)->evaluate (#0) -->\n",
+ "<g id=\"edge19\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)->evaluate (#0)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M442.34,-71.7C442.34,-63.98 442.34,-54.71 442.34,-46.11\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"445.84,-46.1 442.34,-36.1 438.84,-46.1 445.84,-46.1\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"node3\" class=\"node\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"309.34\" cy=\"-162\" rx=\"182.02\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"309.34\" y=\"-157.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"edge16\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M341.2,-144.23C359.24,-134.74 382.01,-122.75 401.37,-112.56\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"403.06,-115.63 410.28,-107.87 399.8,-109.43 403.06,-115.63\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z (#3) -->\n",
+ "<g id=\"node4\" class=\"node\">\n",
+ "<title>x*var(3)*y*z (#3)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"146.34\" cy=\"-306\" rx=\"74.88\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"146.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y*z (#3)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y*z (#3)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"edge7\" class=\"edge\">\n",
+ "<title>x*var(3)*y*z (#3)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M152.92,-287.91C161.2,-268.36 177.06,-236.4 199.34,-216 214.19,-202.41 233.15,-191.61 251.14,-183.4\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"252.67,-186.55 260.43,-179.34 249.86,-180.14 252.67,-186.55\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y (#4) -->\n",
+ "<g id=\"node5\" class=\"node\">\n",
+ "<title>x*var(3)*y (#4)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"66.34\" cy=\"-378\" rx=\"66.18\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3)*y (#4)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3)*y (#4)->x*var(3)*y*z (#3) -->\n",
+ "<g id=\"edge5\" class=\"edge\">\n",
+ "<title>x*var(3)*y (#4)->x*var(3)*y*z (#3)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M85.3,-360.41C95.66,-351.34 108.66,-339.97 119.99,-330.06\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"122.3,-332.69 127.52,-323.47 117.69,-327.42 122.3,-332.69\"/>\n",
+ "</g>\n",
+ "<!-- x*var(3) (#5) -->\n",
+ "<g id=\"node6\" class=\"node\">\n",
+ "<title>x*var(3) (#5)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"66.34\" cy=\"-450\" rx=\"57.5\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">x*var(3) (#5)</text>\n",
+ "</g>\n",
+ "<!-- x*var(3) (#5)->x*var(3)*y (#4) -->\n",
+ "<g id=\"edge3\" class=\"edge\">\n",
+ "<title>x*var(3) (#5)->x*var(3)*y (#4)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M66.34,-431.7C66.34,-423.98 66.34,-414.71 66.34,-406.11\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"69.84,-406.1 66.34,-396.1 62.84,-406.1 69.84,-406.1\"/>\n",
+ "</g>\n",
+ "<!-- x (#6) -->\n",
+ "<g id=\"node7\" class=\"node\">\n",
+ "<title>x (#6)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"185.34\" cy=\"-522\" rx=\"31.45\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"185.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">x (#6)</text>\n",
+ "</g>\n",
+ "<!-- x (#6)->x*var(3) (#5) -->\n",
+ "<g id=\"edge1\" class=\"edge\">\n",
+ "<title>x (#6)->x*var(3) (#5)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M164.07,-508.49C146.6,-498.21 121.5,-483.45 101.12,-471.46\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"102.81,-468.39 92.42,-466.34 99.26,-474.42 102.81,-468.39\"/>\n",
+ "</g>\n",
+ "<!-- x^var(2) (#13) -->\n",
+ "<g id=\"node14\" class=\"node\">\n",
+ "<title>x^var(2) (#13)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"296.34\" cy=\"-450\" rx=\"61.8\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"296.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">x^var(2) (#13)</text>\n",
+ "</g>\n",
+ "<!-- x (#6)->x^var(2) (#13) -->\n",
+ "<g id=\"edge8\" class=\"edge\">\n",
+ "<title>x (#6)->x^var(2) (#13)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M205.7,-508.16C221.69,-498.08 244.3,-483.82 262.94,-472.07\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"264.86,-474.99 271.45,-466.7 261.12,-469.07 264.86,-474.99\"/>\n",
+ "</g>\n",
+ "<!-- var(3) (#7) -->\n",
+ "<g id=\"node8\" class=\"node\">\n",
+ "<title>var(3) (#7)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"66.34\" cy=\"-522\" rx=\"48.81\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"66.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(3) (#7)</text>\n",
+ "</g>\n",
+ "<!-- var(3) (#7)->x*var(3) (#5) -->\n",
+ "<g id=\"edge2\" class=\"edge\">\n",
+ "<title>var(3) (#7)->x*var(3) (#5)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M66.34,-503.7C66.34,-495.98 66.34,-486.71 66.34,-478.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"69.84,-478.1 66.34,-468.1 62.84,-478.1 69.84,-478.1\"/>\n",
+ "</g>\n",
+ "<!-- y (#8) -->\n",
+ "<g id=\"node9\" class=\"node\">\n",
+ "<title>y (#8)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"173.34\" cy=\"-450\" rx=\"31.45\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"173.34\" y=\"-445.8\" font-family=\"Times,serif\" font-size=\"14.00\">y (#8)</text>\n",
+ "</g>\n",
+ "<!-- y (#8)->x*var(3)*y (#4) -->\n",
+ "<g id=\"edge4\" class=\"edge\">\n",
+ "<title>y (#8)->x*var(3)*y (#4)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M153.47,-436C138.2,-426.01 116.78,-412 99,-400.37\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"100.9,-397.42 90.61,-394.88 97.06,-403.28 100.9,-397.42\"/>\n",
+ "</g>\n",
+ "<!-- z (#9) -->\n",
+ "<g id=\"node10\" class=\"node\">\n",
+ "<title>z (#9)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"181.34\" cy=\"-378\" rx=\"30.95\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"181.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">z (#9)</text>\n",
+ "</g>\n",
+ "<!-- z (#9)->x*var(3)*y*z (#3) -->\n",
+ "<g id=\"edge6\" class=\"edge\">\n",
+ "<title>z (#9)->x*var(3)*y*z (#3)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M173.05,-360.41C168.97,-352.25 163.95,-342.22 159.37,-333.07\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"162.42,-331.34 154.82,-323.96 156.16,-334.47 162.42,-331.34\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"node11\" class=\"node\">\n",
+ "<title>sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"342.34\" cy=\"-234\" rx=\"134.12\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"342.34\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2))^var(2)*var(10) (#10)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2)*var(10) (#10)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2) -->\n",
+ "<g id=\"edge15\" class=\"edge\">\n",
+ "<title>sin(x^var(2))^var(2)*var(10) (#10)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10) (#2)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M334.18,-215.7C330.43,-207.73 325.89,-198.1 321.72,-189.26\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"324.83,-187.66 317.4,-180.1 318.5,-190.64 324.83,-187.66\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"node12\" class=\"node\">\n",
+ "<title>sin(x^var(2))^var(2) (#11)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"342.34\" cy=\"-306\" rx=\"103.07\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"342.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2))^var(2) (#11)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2))^var(2) (#11)->sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"edge13\" class=\"edge\">\n",
+ "<title>sin(x^var(2))^var(2) (#11)->sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M342.34,-287.7C342.34,-279.98 342.34,-270.71 342.34,-262.11\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"345.84,-262.1 342.34,-252.1 338.84,-262.1 345.84,-262.1\"/>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2)) (#12) -->\n",
+ "<g id=\"node13\" class=\"node\">\n",
+ "<title>sin(x^var(2)) (#12)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"red\" cx=\"308.34\" cy=\"-378\" rx=\"77.72\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"308.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">sin(x^var(2)) (#12)</text>\n",
+ "</g>\n",
+ "<!-- sin(x^var(2)) (#12)->sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"edge11\" class=\"edge\">\n",
+ "<title>sin(x^var(2)) (#12)->sin(x^var(2))^var(2) (#11)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M316.57,-360.05C320.44,-352.09 325.14,-342.41 329.47,-333.51\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"332.73,-334.8 333.95,-324.28 326.43,-331.74 332.73,-334.8\"/>\n",
+ "</g>\n",
+ "<!-- x^var(2) (#13)->sin(x^var(2)) (#12) -->\n",
+ "<g id=\"edge10\" class=\"edge\">\n",
+ "<title>x^var(2) (#13)->sin(x^var(2)) (#12)</title>\n",
+ "<path fill=\"none\" stroke=\"red\" d=\"M299.31,-431.7C300.63,-423.98 302.22,-414.71 303.69,-406.11\"/>\n",
+ "<polygon fill=\"red\" stroke=\"red\" points=\"307.17,-406.55 305.41,-396.1 300.27,-405.37 307.17,-406.55\"/>\n",
+ "</g>\n",
+ "<!-- var(2) (#15) -->\n",
+ "<g id=\"node15\" class=\"node\">\n",
+ "<title>var(2) (#15)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"296.34\" cy=\"-522\" rx=\"53.15\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"296.34\" y=\"-517.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(2) (#15)</text>\n",
+ "</g>\n",
+ "<!-- var(2) (#15)->x^var(2) (#13) -->\n",
+ "<g id=\"edge9\" class=\"edge\">\n",
+ "<title>var(2) (#15)->x^var(2) (#13)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M296.34,-503.7C296.34,-495.98 296.34,-486.71 296.34,-478.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"299.84,-478.1 296.34,-468.1 292.84,-478.1 299.84,-478.1\"/>\n",
+ "</g>\n",
+ "<!-- var(2) (#16) -->\n",
+ "<g id=\"node16\" class=\"node\">\n",
+ "<title>var(2) (#16)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"457.34\" cy=\"-378\" rx=\"53.15\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"457.34\" y=\"-373.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(2) (#16)</text>\n",
+ "</g>\n",
+ "<!-- var(2) (#16)->sin(x^var(2))^var(2) (#11) -->\n",
+ "<g id=\"edge12\" class=\"edge\">\n",
+ "<title>var(2) (#16)->sin(x^var(2))^var(2) (#11)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M432.67,-361.98C416.82,-352.34 395.97,-339.64 378.29,-328.88\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"379.74,-325.67 369.38,-323.46 376.1,-331.65 379.74,-325.67\"/>\n",
+ "</g>\n",
+ "<!-- var(10) (#17) -->\n",
+ "<g id=\"node17\" class=\"node\">\n",
+ "<title>var(10) (#17)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"521.34\" cy=\"-306\" rx=\"57.5\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"521.34\" y=\"-301.8\" font-family=\"Times,serif\" font-size=\"14.00\">var(10) (#17)</text>\n",
+ "</g>\n",
+ "<!-- var(10) (#17)->sin(x^var(2))^var(2)*var(10) (#10) -->\n",
+ "<g id=\"edge14\" class=\"edge\">\n",
+ "<title>var(10) (#17)->sin(x^var(2))^var(2)*var(10) (#10)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M486.83,-291.5C460.24,-281.11 423.09,-266.58 393.12,-254.86\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"394.25,-251.54 383.66,-251.16 391.7,-258.06 394.25,-251.54\"/>\n",
+ "</g>\n",
+ "<!-- tan(theta) (#18) -->\n",
+ "<g id=\"node18\" class=\"node\">\n",
+ "<title>tan(theta) (#18)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"575.34\" cy=\"-162\" rx=\"65.21\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"575.34\" y=\"-157.8\" font-family=\"Times,serif\" font-size=\"14.00\">tan(theta) (#18)</text>\n",
+ "</g>\n",
+ "<!-- tan(theta) (#18)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1) -->\n",
+ "<g id=\"edge18\" class=\"edge\">\n",
+ "<title>tan(theta) (#18)->x*var(3)*y*z+sin(x^var(2))^var(2)*var(10)+tan(theta) (#1)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M546.48,-145.81C528.06,-136.12 503.9,-123.4 483.49,-112.66\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"484.89,-109.44 474.41,-107.88 481.63,-115.63 484.89,-109.44\"/>\n",
+ "</g>\n",
+ "<!-- theta (#19) -->\n",
+ "<g id=\"node19\" class=\"node\">\n",
+ "<title>theta (#19)</title>\n",
+ "<ellipse fill=\"none\" stroke=\"black\" cx=\"575.34\" cy=\"-234\" rx=\"48.82\" ry=\"18\"/>\n",
+ "<text text-anchor=\"middle\" x=\"575.34\" y=\"-229.8\" font-family=\"Times,serif\" font-size=\"14.00\">theta (#19)</text>\n",
+ "</g>\n",
+ "<!-- theta (#19)->tan(theta) (#18) -->\n",
+ "<g id=\"edge17\" class=\"edge\">\n",
+ "<title>theta (#19)->tan(theta) (#18)</title>\n",
+ "<path fill=\"none\" stroke=\"black\" d=\"M575.34,-215.7C575.34,-207.98 575.34,-198.71 575.34,-190.11\"/>\n",
+ "<polygon fill=\"black\" stroke=\"black\" points=\"578.84,-190.1 575.34,-180.1 571.84,-190.1 578.84,-190.1\"/>\n",
+ "</g>\n",
+ "</g>\n",
+ "</svg>\n"
+ ],
+ "text/plain": [
+ "<graphviz.dot.Digraph at 0x1232a0ad0>"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x.x = 4\n",
+ "x.evaluate().graphviz()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAAPCAYAAAB0p1TfAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAGB0lEQVRoBeWai20cNxCGfYYLUJQO5A786EDpwIE7sDtw4AoCpwOng8DpQO4gsTqwO7ClDpT/o2aIIZd7O4vwkgAhwONwOJw3Xysd7u7uHvxfyuFweCJ7r6O9wp2pfy78F8fPpnO+s9usnrPlwu+fln3wRLWAvZUOX1W/VyWAHzT+UW1TRPvOENA+Vn0XA90Qdx3NvRDqJ0M/U/uNvub3CQTda6NDF/rIafTZqfeNeMDLZQFTnorv7T1YgjCVzvnGVno/Uf+15LqNcZhEcB87/reBj1J6wiDwW43ZiXyZjTexWM8/Gf+AqvLeYW+F+6D6wvtG9yniBCPgs+pFpBvB0KiS/EWm8SMgKHDpeMHwbPRR/5UqdL0+DZ3xXOhtePSkwocW2Wcu11sbm0bnfGNrMhpfMK6Cj5Ad/YE9n+N8o4Uuo2cqZuI11Zfil4q32XJUticpSfBq4AgS5pPjoVEdOYyAXzndWisaHD5KDHaGG58n+I1qk5TMM1yvz6bege8iMXwstpIzlS7yBlbBPhJsIUc4kqoJmvpX1AGfxfwBTSpm4p/KAecv+ozsbLw3ZT+UQArH948FOv4DjR+bkfIPdS7t6Ij4Hr4U4mZAx3F+JjwrkIKMW6v0WVD0+5LVu5/3r/VlI0d+Y5sro7EXghn3q1EZku0/UJ1uZ5uN2Sl8mY33pmxPVFYsiXalys7lhZ3yvXfUIvhb6DvoScT4sUJCfllJOuYV2Rr/qPodrTOzINKN+mT1djb/hfal7Pp1RRHuaLcad3+ukO1CZ2N2Cl+m4i1rNmU/wmQSQonwu0BWNDseK5os57gBz2U8JjCoUTkfIR0nXmu7NrsIeox2a2TjbBYNj48aZMGbesM3FvHimHFbsJFjdiF3Nh06iCdHflxooGPBD9eio32p6g8fjtm6aOOEY3pqzO2MU3q4xOwUvhTPVLxTskVU7qm0KiQDAJW7Un0gARuel3edY/NwLHPe9GNbfc1ZnWtjBJe7DpXPS41sk7+qd6TXfFZuvSMLdpvqw8X4TaUznsiq92nB+Lfe8wSTVO73SmdzucM3j8iMnprj9qVjpjlTfYmesYr/sXivyq5McIQpiXEEyhO2BFH9jNELh0QlR7D48qhoHg8rdOiHTk3A6Kti4FDvEa+I0zxsXTwQIw3w36XT/MY36q8lqsQtgssuXD5F9WN9P+opeFfMRH9yX0rGMN5bskuiiojjsEkWm0iE/Fudr/jG4ThKhaMZYNeOKnoC0MjtHR/7okUX5JRdUe2m3nH+CBYPdIBnPT1m05meDX/hmkRFpgo/i0UjHHYy1uz8W3qKPh0zk9HEQjjfHLKL5KgvxW8Y74zshyKisCP1L03uptzhMBYH3aql0O+L4+pfd3qCvm93KxbK4oM3dzRqP0f9Pw3HwqBs6n1PVu6HPBRJjrVSbBDNbDp2NRZWxjfQuJ9HesKrlIyeO2M23ZeuK630ZbEN4y30puxHYkCAcOTCQThX4yTsuSqFC311VsHc/8TxgB6D4slKfSz+NUmFc758VSgJJRwv/4VecNXYHr2Z8kx19MWi6C45/qCaTYddz6Uvd+xYWIgXhudLCBsFiep+iLQOx2TP6rkZsxP6sugt/lvx3s4/OQhmbO3N0QTexrjDlTG1rIr6YT7QsKXXD/GOH7WiI0Cj6wOPJj/S0ac+NJyPcP3Rn9Kb+SoLmYaHR/2YPpvOde9byVnYKFy5Qg1o8W/j9x16pmJm+mzmgPks5Uujzcb7qGxPRhzELlYSxR2lPkY29071uQzXB41gdjacXl/khiM7muRVn92iXKbVlvuKtew29W4mGLn9y5dVCc+KF7xHb2T3dzAWB7pXuwVPpXNf9q3kYEtdID4OTrUmgmD3b/U5tCopPY02E7NT+TIT703Z8Z9SyPy3qvF4XHy/s2MCuq+qlOeqP8shfnQWpOhQkA/38XhnMSBnVK5F+9QHNB/l43c4AkMAm++JokvpDV/RwsPv4ueCsZV/iGmuF7PpJKMW8WaBcmy7H7CHhP3FiUTDnY0EpaDnwr8M7NATXpmYTfWl9NsT76Oy/wKRX4CYwK0a1AAAAABJRU5ErkJggg==\n",
+ "text/latex": [
+ "$\\displaystyle 80.23855546508528$"
+ ],
+ "text/plain": [
+ "80.23855546508528"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x.evaluate()()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/examples/streaming_examples.ipynb b/examples/streaming_examples.ipynb
new file mode 100644
index 0000000..14a9b24
--- /dev/null
+++ b/examples/streaming_examples.ipynb
@@ -0,0 +1,134 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Simple Example"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import tributary as t\n",
+ "import random, time\n",
+ "\n",
+ "def foo():\n",
+ " return random.random()\n",
+ "\n",
+ "def long():\n",
+ " print('long called!')\n",
+ " time.sleep(1)\n",
+ " return 5\n",
+ "\n",
+ "\n",
+ "test = t.Timer(foo, {}, .5, 5)\n",
+ "\n",
+ "test2 = t.Negate(test)\n",
+ "\n",
+ "res2 = t.Add(test, test2)\n",
+ "\n",
+ "p2 = t.Print(res2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# t.GraphViz(p2, 'test1')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "x = t.run(p2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "InvalidStateError",
+ "evalue": "Result is not set.",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mInvalidStateError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-4-fb5cf2ba9423>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[0;31mInvalidStateError\u001b[0m: Result is not set."
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Print <function Print.<locals>._print at 0x11f7c8320>\n",
+ "Add <function bin.<locals>._bin at 0x11f7c8290>\n",
+ "Timer <function Timer.<locals>._repeater at 0x11f189680>\n",
+ "Negate <function unary.<locals>._unary at 0x11f7c8170>\n",
+ "Foo <function foo at 0x11e48b050>\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/usr/local/lib/python3.7/site-packages/aiostream/aiter_utils.py:115: UserWarning: AsyncIteratorContext is iterated outside of its context\n",
+ " \"AsyncIteratorContext is iterated outside of its context\")\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Foo <function foo at 0x11e48b050>\n",
+ "Foo <function foo at 0x11e48b050>\n",
+ "Foo <function foo at 0x11e48b050>\n",
+ "Foo <function foo at 0x11e48b050>\n"
+ ]
+ }
+ ],
+ "source": [
+ "x.result()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/examples/reactive_kafka.ipynb b/examples/streaming_kafka.ipynb
similarity index 100%
rename from examples/reactive_kafka.ipynb
rename to examples/streaming_kafka.ipynb
diff --git a/examples/reactive_stream.ipynb b/examples/streaming_stream.ipynb
similarity index 100%
rename from examples/reactive_stream.ipynb
rename to examples/streaming_stream.ipynb
diff --git a/examples/reactive_sympy.ipynb b/examples/streaming_sympy.ipynb
similarity index 100%
rename from examples/reactive_sympy.ipynb
rename to examples/streaming_sympy.ipynb
diff --git a/examples/reactive_ws_http_sio.ipynb b/examples/streaming_ws_http_sio.ipynb
similarity index 100%
rename from examples/reactive_ws_http_sio.ipynb
rename to examples/streaming_ws_http_sio.ipynb
diff --git a/examples/tributary b/examples/tributary
new file mode 120000
index 0000000..633e5d9
--- /dev/null
+++ b/examples/tributary
@@ -0,0 +1 @@
+/Users/theocean154/Programs/projects/tributary/tributary/tributary
\ No newline at end of file
diff --git a/tributary/tests/helpers/dummy_ws.py b/tributary/tests/helpers/dummy_ws.py
index 5bf7e27..5906c76 100644
--- a/tributary/tests/helpers/dummy_ws.py
+++ b/tributary/tests/helpers/dummy_ws.py
@@ -2,7 +2,7 @@
import tornado.web
import tornado.ioloop
import time
-from tributary.reactive.input import _gen
+from tributary.streaming.input import _gen
class DummyWebSocket(tornado.websocket.WebSocketHandler):
From 4979c75af77ed5c80c23e642fb0165db77ceec52 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:19:37 -0500
Subject: [PATCH 07/15] grab python version
---
azure-pipelines.yml | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6e687ca..a2545c2 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -95,7 +95,15 @@ jobs:
versionSpec: '$(python.version)'
displayName: 'Use Python $(python.version)'
- - script: |
+ - script: |
+ which python > python.txt
+ set /p PYTHON=<python.txt
+ ln -s %PYTHON% %PYTHON%$(python.version)
+ python --version
+ which python$(python.version)
+ displayName: "Which python"
+
+ - script: |
python -m pip install --upgrade pip
pip install -e .[dev]
displayName: 'Install dependencies'
From 805e2020c68363e138ec9b0ceac49ff37686bd73 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:19:57 -0500
Subject: [PATCH 08/15] .
---
Makefile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index a3b6761..d868971 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,11 @@
build: ## Build the repository
- python3 setup.py build
+ python3.7 setup.py build
buildpy2:
python2 setup.py build
tests: ## Clean and Make unit tests
- python3 -m pytest -v tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
+ python3.7 -m pytest -v tributary --cov=tributary --junitxml=python_junit.xml --cov-report=xml --cov-branch
notebooks: ## test execute the notebooks
./scripts/test_notebooks.sh
From b27ca59025fac93f0e25f11a9801077d94e1cdd4 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:23:13 -0500
Subject: [PATCH 09/15] .
---
azure-pipelines.yml | 176 ++++++++++++++++++++++----------------------
1 file changed, 88 insertions(+), 88 deletions(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index a2545c2..f0f9b5d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -12,34 +12,34 @@ jobs:
python.version: '3.7'
steps:
- - task: UsePythonVersion@0
- inputs:
- versionSpec: '$(python.version)'
- displayName: 'Use Python $(python.version)'
-
- - script: |
- python -m pip install --upgrade pip
- pip install -e .[dev]
- displayName: 'Install dependencies'
-
- - script: |
- make lint
- displayName: 'Lint'
-
- - script:
- make tests
- displayName: 'Test'
-
- - task: PublishTestResults@2
- condition: succeededOrFailed()
- inputs:
- testResultsFiles: 'python_junit.xml'
- testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
-
- - task: PublishCodeCoverageResults@1
- inputs:
- codeCoverageTool: Cobertura
- summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
+
+ - script: |
+ make lint
+ displayName: 'Lint'
+
+ - script:
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
- job: 'Mac'
pool:
@@ -51,34 +51,34 @@ jobs:
python.version: '3.7'
steps:
- - task: UsePythonVersion@0
- inputs:
- versionSpec: '$(python.version)'
- displayName: 'Use Python $(python.version)'
-
- - script: |
- python -m pip install --upgrade pip
- pip install -e .[dev]
- displayName: 'Install dependencies'
-
- - script: |
- make lint
- displayName: 'Lint'
-
- - script: |
- make tests
- displayName: 'Test'
-
- - task: PublishTestResults@2
- condition: succeededOrFailed()
- inputs:
- testResultsFiles: 'python_junit.xml'
- testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
-
- - task: PublishCodeCoverageResults@1
- inputs:
- codeCoverageTool: Cobertura
- summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
+
+ - script: |
+ make lint
+ displayName: 'Lint'
+
+ - script: |
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
- job: 'Windows'
pool:
@@ -90,39 +90,39 @@ jobs:
python.version: '3.7'
steps:
- - task: UsePythonVersion@0
- inputs:
- versionSpec: '$(python.version)'
- displayName: 'Use Python $(python.version)'
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '$(python.version)'
+ displayName: 'Use Python $(python.version)'
+
+ - script: |
+ which python > python.txt
+ set /p PYTHON=<python.txt
+ ln -s %PYTHON% %PYTHON%$(python.version)
+ python --version
+ which python$(python.version)
+ displayName: "Which python"
+
+ - script: |
+ python -m pip install --upgrade pip
+ pip install -e .[dev]
+ displayName: 'Install dependencies'
- script: |
- which python > python.txt
- set /p PYTHON=<python.txt
- ln -s %PYTHON% %PYTHON%$(python.version)
- python --version
- which python$(python.version)
- displayName: "Which python"
+ make lint
+ displayName: 'Lint'
- script: |
- python -m pip install --upgrade pip
- pip install -e .[dev]
- displayName: 'Install dependencies'
-
- - script: |
- make lint
- displayName: 'Lint'
-
- - script: |
- make tests
- displayName: 'Test'
-
- - task: PublishTestResults@2
- condition: succeededOrFailed()
- inputs:
- testResultsFiles: 'python_junit.xml'
- testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
-
- - task: PublishCodeCoverageResults@1
- inputs:
- codeCoverageTool: Cobertura
- summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
+ make tests
+ displayName: 'Test'
+
+ - task: PublishTestResults@2
+ condition: succeededOrFailed()
+ inputs:
+ testResultsFiles: 'python_junit.xml'
+ testRunTitle: 'Publish test results for Python $(python.version) $(manylinux_flag)'
+
+ - task: PublishCodeCoverageResults@1
+ inputs:
+ codeCoverageTool: Cobertura
+ summaryFileLocation: '$(System.DefaultWorkingDirectory)/*coverage.xml'
From 97858d5815bb7f065aeaf0ff739637f9861710b5 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:38:12 -0500
Subject: [PATCH 10/15] update badge
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index d71bb08..daa0f01 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# <a href="https://tributary.readthedocs.io"><img src="docs/img/icon.png" width="300"></a>
Python Data Streams
-[](https://travis-ci.org/timkpaine/tributary)
+[](https://dev.azure.com/tpaine154/tributary/_build/latest?definitionId=2&branchName=master)
[]()
[](https://codecov.io/gh/timkpaine/tributary)
[](https://bettercodehub.com/)
From 93e3af79c46bda0ed99860c11831e833c02dd865 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:46:03 -0500
Subject: [PATCH 11/15] update coverage
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index daa0f01..343b26b 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@ Python Data Streams
[](https://dev.azure.com/tpaine154/tributary/_build/latest?definitionId=2&branchName=master)
[]()
-[](https://codecov.io/gh/timkpaine/tributary)
+[]()
[](https://bettercodehub.com/)
[](https://pypi.python.org/pypi/tributary)
[](https://pypi.python.org/pypi/tributary)
From f647245fcc08386d25774a02c8b3db55fedc620b Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Sun, 16 Feb 2020 23:49:51 -0500
Subject: [PATCH 12/15] .
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 343b26b..d2bfaf3 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@ Python Data Streams
[](https://dev.azure.com/tpaine154/tributary/_build/latest?definitionId=2&branchName=master)
[]()
-[]()
+[]()
[](https://bettercodehub.com/)
[](https://pypi.python.org/pypi/tributary)
[](https://pypi.python.org/pypi/tributary)
From d0b7e19fd6c992cdd0f15e65e87d2a90580b157d Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Mon, 17 Feb 2020 17:33:12 -0500
Subject: [PATCH 13/15] parity with reactive
---
tributary/streaming/base.py | 112 ++++++--
tributary/streaming/input/input.py | 50 ++--
tributary/streaming/utils.py | 271 +++++++-----------
...{test_utils.py => test_utils_streaming.py} | 34 ++-
4 files changed, 262 insertions(+), 205 deletions(-)
rename tributary/tests/streaming/{test_utils.py => test_utils_streaming.py} (52%)
diff --git a/tributary/streaming/base.py b/tributary/streaming/base.py
index 8ecdcd2..fd42c6d 100644
--- a/tributary/streaming/base.py
+++ b/tributary/streaming/base.py
@@ -20,42 +20,98 @@ async def _agen_to_foo(generator):
class Node(object):
+ '''A representation of a node in the forward propogating graph.
+
+ Args:
+ foo (callable): the python callable to wrap in a forward propogating node, can be:
+ - function
+ - generator
+ - async function
+ - async generator
+ foo_kwargs (dict): kwargs for the wrapped callables, should be static call-to-call
+ name (str): name of the node
+ inputs (int): number of upstream inputs
+ kwargs (dict): extra kwargs:
+ - delay_interval (int/float): rate limit
+ - execution_max (int): max number of times to execute callable
+ '''
_id_ref = 0
- def __init__(self, foo, foo_kwargs=None, name=None, inputs=1):
+ def __init__(self, foo, foo_kwargs=None, name=None, inputs=1, **kwargs):
+ # Instances get an id but one id tracker for all nodes so we can
+ # uniquely identify them
+ # TODO different scheme
self._id = Node._id_ref
Node._id_ref += 1
+ # Every node gets a name so it can be uniquely identified in the graph
self._name = '{}#{}'.format(name or self.__class__.__name__, self._id)
+
+ # Inputs are async queues from upstream nodes
self._input = [Queue() for _ in range(inputs)]
+
+ # Active are currently valid inputs, since inputs
+ # may come at different rates
self._active = [StreamNone() for _ in range(inputs)]
+
+ # Downstream nodes so we can traverse graph, push
+ # results to downstream nodes
self._downstream = []
+
+ # Upstream nodes so we can traverse graph, plot and optimize
self._upstream = []
+ # The function we are wrapping, can be:
+ # - vanilla function
+ # - vanilla generator
+ # - async function
+ # - async generator
self._foo = foo
+
+ # Any kwargs necessary for the function.
+ # These should be static call-to-call.
self._foo_kwargs = foo_kwargs or {}
+
+ # Delay between executions, useful for rate-limiting
+ # default is no rate limiting
+ self._delay_interval = kwargs.get('delay_interval', 0)
+
+ # max number of times to execute callable
+ self._execution_max = kwargs.get('execution_max', 0)
+
+ # current execution count
+ self._execution_count = 0
+
+ # last value pushed downstream
self._last = StreamNone()
+
+ # stream is in a finished state, will only propogate StreamEnd instances
self._finished = False
def __repr__(self):
return '{}'.format(self._name)
async def _push(self, inp, index):
+ '''push value to downstream nodes'''
await self._input[index].put(inp)
async def _execute(self):
- if asyncio.iscoroutine(self._foo):
- _last = await self._foo(*self._active, **self._foo_kwargs)
- elif isinstance(self._foo, types.FunctionType):
- try:
- _last = self._foo(*self._active, **self._foo_kwargs)
- except ValueError:
- # Swap back to function
- self._foo = self._old_foo
- _last = self._foo(*self._active, **self._foo_kwargs)
-
- else:
- raise Exception('Cannot use type:{}'.format(type(self._foo)))
+ '''execute callable'''
+ valid = False
+ while not valid:
+ if asyncio.iscoroutine(self._foo):
+ _last = await self._foo(*self._active, **self._foo_kwargs)
+ elif isinstance(self._foo, types.FunctionType):
+ try:
+ _last = self._foo(*self._active, **self._foo_kwargs)
+ except ValueError:
+ # Swap back to function
+ self._foo = self._old_foo
+ continue
+ else:
+ raise Exception('Cannot use type:{}'.format(type(self._foo)))
+ valid = True
+ self._execution_count += 1
if isinstance(_last, types.AsyncGeneratorType):
async def _foo(g=_last):
@@ -77,25 +133,34 @@ async def _foo(g=_last):
self._active[i] = StreamNone()
async def _finish(self):
+ '''mark this node as finished'''
self._finished = True
self._last = StreamEnd()
await self._output(self._last)
def _backpressure(self):
- '''check if _downstream are all empty'''
+ '''check if _downstream are all empty, if not then don't propogate'''
ret = not all(n._input[i].empty() for n, i in self._downstream)
- if ret:
- print('backpressure!')
return ret
async def __call__(self):
+ '''execute the callable if possible, and propogate values downstream'''
+ # Downstream nodes can't process
if self._backpressure():
return StreamNone()
+ # Previously ended stream
if self._finished:
return await self._finish()
- print(self._input)
+ # Sleep if needed
+ if self._delay_interval:
+ await asyncio.sleep(self._delay_interval)
+
+ # Stop executing
+ if self._execution_max > 0 and self._execution_count >= self._execution_max:
+ self._foo = lambda: StreamEnd()
+ self._old_foo = lambda: StreamEnd()
ready = True
# iterate through inputs
@@ -110,7 +175,6 @@ async def __call__(self):
val = inp.get_nowait()
if isinstance(val, StreamEnd):
- print('here')
return await self._finish()
# set as active
@@ -125,12 +189,23 @@ async def __call__(self):
return await self._execute()
async def _output(self, ret):
+ '''output value to downstream nodes'''
# if downstreams, output
for down, i in self._downstream:
await down._push(ret, i)
return ret
def _deep_bfs(self, reverse=True):
+ '''get nodes by level in tree, reversed relative to output node.
+ e.g. given a tree that looks like:
+ A -> B -> D -> F
+ \-> C -> E /
+ the result will be: [[A], [B, C], [D, E], [F]]
+
+ This will be the order we synchronously execute, so that within a
+ level nodes' execution will be asynchronous but from level to level
+ they will be synchronous
+ '''
nodes = []
nodes.append([self])
@@ -147,4 +222,5 @@ def _deep_bfs(self, reverse=True):
return nodes
def value(self):
+ '''get value from node'''
return self._last
diff --git a/tributary/streaming/input/input.py b/tributary/streaming/input/input.py
index 436aa62..27cacd9 100644
--- a/tributary/streaming/input/input.py
+++ b/tributary/streaming/input/input.py
@@ -18,35 +18,46 @@ def _gen():
class Timer(Node):
+ '''Streaming wrapper to periodically call a callable `count` times
+ with a delay of `interval` in between
+
+ Arguments:
+ foo (callable): callable to call
+ foo_kwargs (dict): kwargs for callable
+ count (int): number of times to call, 0 means infinite (or until generator is complete)
+ interval (int/float): minimum delay between calls (can be more due to async scheduling)
+ '''
def __init__(self, foo, foo_kwargs=None, count=1, interval=0):
- self._count = count
- self._executed = 0
- self._interval = interval
-
- super().__init__(foo=foo, foo_kwargs=foo_kwargs, name='Timer[{}]'.format(foo.__name__), inputs=0)
-
- async def _execute(self):
- self._executed += 1
- await super()._execute()
-
- async def __call__(self):
- # sleep if needed
- if self._interval:
- await asyncio.sleep(self._interval)
-
- if self._count > 0 and self._executed >= self._count:
- self._foo = lambda: StreamEnd()
-
- return await self._execute()
+ super().__init__(foo=foo,
+ foo_kwargs=foo_kwargs,
+ name='Timer[{}]'.format(foo.__name__),
+ inputs=0,
+ execution_max=count,
+ delay_interval=interval)
class Const(Timer):
+ '''Streaming wrapper to return a scalar value
+
+ Arguments:
+ value (any): value to return
+ count (int): number of times to call, 0 means infinite
+ '''
def __init__(self, value, count=0):
super().__init__(foo=lambda: value, count=count, interval=0)
self._name = 'Const[{}]'.format(value)
class Foo(Timer):
+ '''Streaming wrapper to periodically call a function `count` times
+ with a delay of `interval` in between
+
+ Arguments:
+ foo (callable): callable to call
+ foo_kwargs (dict): kwargs for callable
+ count (int): number of times to call, 0 means infinite (or until generator is complete)
+ interval (int/float): minimum delay between calls (can be more due to async scheduling)
+ '''
def __init__(self, foo, foo_kwargs=None, count=0, interval=0):
super().__init__(foo=foo, foo_kwargs=foo_kwargs, count=count, interval=interval)
self._name = 'Foo[{}]'.format(foo.__name__)
@@ -59,7 +70,6 @@ class Random(Foo):
count (int): number of elements to yield
interval (float): interval to wait between yields
'''
-
def __init__(self, count=10, interval=0.1):
def _random(count=count, interval=interval):
step = 0
diff --git a/tributary/streaming/utils.py b/tributary/streaming/utils.py
index 600bb4f..0a4dd3c 100644
--- a/tributary/streaming/utils.py
+++ b/tributary/streaming/utils.py
@@ -101,7 +101,6 @@ def __init__(self, node):
self._count = 0
async def foo(value):
- print('foo got:', value)
# unrolled
if self._count > 0:
self._count -= 1
@@ -111,7 +110,6 @@ async def foo(value):
try:
for v in value:
self._count += 1
- print('pushing:', v)
await self._push(v, 0)
except TypeError:
return value
@@ -122,169 +120,112 @@ async def foo(value):
node._downstream.append((self, 0))
self._upstream.append(node)
-# class UnrollDataFrame(Node):
-# '''Streaming wrapper to unroll an iterable stream
+class UnrollDataFrame(Node):
+ '''Streaming wrapper to unroll a dataframe into a stream
-# Arguments:
-# node (node): input stream
-# '''
-# def __init__(self, node, json=False, wrap=False):
-# def foo(value, json=json, wrap=wrap):
-# for i in range(len(value)):
-# row = df.iloc[i]
-# if json:
-# data = row.to_dict()
-# data['index'] = row.name
-# yield data
-# else:
-# yield row
-
-# super().__init__(foo=foo, name='Unroll', inputs=1)
-# node._downstream.append((self, 0))
-# self._upstream.append(node)
+ Arguments:
+ node (node): input stream
+ '''
+ def __init__(self, node, json=False, wrap=False):
+ self._count = 0
+
+ async def foo(value, json=json, wrap=wrap):
+ # unrolled
+ if self._count > 0:
+ self._count -= 1
+ return value
+
+ # unrolling
+ try:
+ for i in range(len(value)):
+ row = value.iloc[i]
+
+ if json:
+ data = row.to_dict()
+ data['index'] = row.name
+ else:
+ data = row
+ self._count += 1
+ await self._push(data, 0)
+
+ except TypeError:
+ return value
+ else:
+ return StreamRepeat()
+ super().__init__(foo=foo, name='UnrollDF', inputs=1)
+ node._downstream.append((self, 0))
+ self._upstream.append(node)
+
+
+class Merge(Node):
+ '''Streaming wrapper to merge 2 inputs into a single output
+
+ Arguments:
+ node1 (node): input stream
+ node2 (node): input stream
+ '''
+ def __init__(self, node1, node2):
+ def foo(value1, value2):
+ return value1, value2
+
+ super().__init__(foo=foo, name='Merge', inputs=2)
+ node1._downstream.append((self, 0))
+ node2._downstream.append((self, 1))
+ self._upstream.append(node1)
+ self._upstream.append(node2)
+
+class ListMerge(Node):
+ '''Streaming wrapper to merge 2 input lists into a single output list
+
+ Arguments:
+ node1 (node): input stream
+ node2 (node): input stream
+ '''
+ def __init__(self, node1, node2):
+ def foo(value1, value2):
+ return list(value1) + list(value2)
-# def Merge(f_wrap1, f_wrap2):
-# if not isinstance(f_wrap1, FunctionWrapper):
-# if not isinstance(f_wrap1, types.FunctionType):
-# f_wrap1 = Const(f_wrap1)
-# else:
-# f_wrap1 = Foo(f_wrap1)
-
-# if not isinstance(f_wrap2, FunctionWrapper):
-# if not isinstance(f_wrap2, types.FunctionType):
-# f_wrap2 = Const(f_wrap2)
-# else:
-# f_wrap2 = Foo(f_wrap2)
-
-# async def _merge(foo1, foo2):
-# async for gen1, gen2 in zip(foo1(), foo2()):
-# if isinstance(gen1, types.AsyncGeneratorType) and \
-# isinstance(gen2, types.AsyncGeneratorType):
-# async for f1, f2 in zip(gen1, gen2):
-# yield [f1, f2]
-# elif isinstance(gen1, types.AsyncGeneratorType):
-# async for f1 in gen1:
-# yield [f1, gen2]
-# elif isinstance(gen2, types.AsyncGeneratorType):
-# async for f2 in gen2:
-# yield [gen1, f2]
-# else:
-# yield [gen1, gen2]
-
-# return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='Merge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-# def ListMerge(f_wrap1, f_wrap2):
-# if not isinstance(f_wrap1, FunctionWrapper):
-# if not isinstance(f_wrap1, types.FunctionType):
-# f_wrap1 = Const(f_wrap1)
-# else:
-# f_wrap1 = Foo(f_wrap1)
-
-# if not isinstance(f_wrap2, FunctionWrapper):
-# if not isinstance(f_wrap2, types.FunctionType):
-# f_wrap2 = Const(f_wrap2)
-# else:
-# f_wrap2 = Foo(f_wrap2)
-
-# async def _merge(foo1, foo2):
-# async for gen1, gen2 in zip(foo1(), foo2()):
-# if isinstance(gen1, types.AsyncGeneratorType) and \
-# isinstance(gen2, types.AsyncGeneratorType):
-# async for f1, f2 in zip(gen1, gen2):
-# ret = []
-# ret.extend(f1)
-# ret.extend(f1)
-# yield ret
-# elif isinstance(gen1, types.AsyncGeneratorType):
-# async for f1 in gen1:
-# ret = []
-# ret.extend(f1)
-# ret.extend(gen2)
-# yield ret
-# elif isinstance(gen2, types.AsyncGeneratorType):
-# async for f2 in gen2:
-# ret = []
-# ret.extend(gen1)
-# ret.extend(f2)
-# yield ret
-# else:
-# ret = []
-# ret.extend(gen1)
-# ret.extend(gen2)
-# yield ret
-
-# return _wrap(_merge, dict(foo1=f_wrap1, foo2=f_wrap2), name='ListMerge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-# def DictMerge(f_wrap1, f_wrap2):
-# if not isinstance(f_wrap1, FunctionWrapper):
-# if not isinstance(f_wrap1, types.FunctionType):
-# f_wrap1 = Const(f_wrap1)
-# else:
-# f_wrap1 = Foo(f_wrap1)
-
-# if not isinstance(f_wrap2, FunctionWrapper):
-# if not isinstance(f_wrap2, types.FunctionType):
-# f_wrap2 = Const(f_wrap2)
-# else:
-# f_wrap2 = Foo(f_wrap2)
-
-# async def _dictmerge(foo1, foo2):
-# async for gen1, gen2 in zip(foo1(), foo2()):
-# if isinstance(gen1, types.AsyncGeneratorType) and \
-# isinstance(gen2, types.AsyncGeneratorType):
-# async for f1, f2 in zip(gen1, gen2):
-# ret = {}
-# ret.update(f1)
-# ret.update(f1)
-# yield ret
-# elif isinstance(gen1, types.AsyncGeneratorType):
-# async for f1 in gen1:
-# ret = {}
-# ret.update(f1)
-# ret.update(gen2)
-# yield ret
-# elif isinstance(gen2, types.AsyncGeneratorType):
-# async for f2 in gen2:
-# ret = {}
-# ret.update(gen1)
-# ret.update(f2)
-# yield ret
-# else:
-# ret = {}
-# ret.update(gen1)
-# ret.update(gen2)
-# yield ret
-
-# return _wrap(_dictmerge, dict(foo1=f_wrap1, foo2=f_wrap2), name='DictMerge', wraps=(f_wrap1, f_wrap2), share=None)
-
-
-# def Reduce(*f_wraps):
-# f_wraps = list(f_wraps)
-# for i, f_wrap in enumerate(f_wraps):
-# if not isinstance(f_wrap, types.FunctionType):
-# f_wraps[i] = Const(f_wrap)
-# else:
-# f_wraps[i] = Foo(f_wrap)
-
-# async def _reduce(foos):
-# async for all_gens in zip(*[foo() for foo in foos]):
-# gens = []
-# vals = []
-# for gen in all_gens:
-# if isinstance(gen, types.AsyncGeneratorType):
-# gens.append(gen)
-# else:
-# vals.append(gen)
-# if gens:
-# for gens in zip(*gens):
-# ret = list(vals)
-# for gen in gens:
-# ret.append(next(gen))
-# yield ret
-# else:
-# yield vals
-
-# return _wrap(_reduce, dict(foos=f_wraps), name='Reduce', wraps=tuple(f_wraps), share=None)
+ super().__init__(foo=foo, name='ListMerge', inputs=2)
+ node1._downstream.append((self, 0))
+ node2._downstream.append((self, 1))
+ self._upstream.append(node1)
+ self._upstream.append(node2)
+
+
+class DictMerge(Node):
+ '''Streaming wrapper to merge 2 input dicts into a single output dict.
+ Preference is given to the second input (e.g. if keys overlap)
+
+ Arguments:
+ node1 (node): input stream
+ node2 (node): input stream
+ '''
+ def __init__(self, node1, node2):
+ def foo(value1, value2):
+ ret = {}
+ ret.update(value1)
+ ret.update(value2)
+ return ret
+
+ super().__init__(foo=foo, name='DictMerge', inputs=2)
+ node1._downstream.append((self, 0))
+ node2._downstream.append((self, 1))
+ self._upstream.append(node1)
+ self._upstream.append(node2)
+
+
+class Reduce(Node):
+ '''Streaming wrapper to merge any number of inputs
+
+ Arguments:
+ nodes (tuple): input streams
+ '''
+ def __init__(self, *nodes):
+ def foo(*values):
+ return values
+
+ super().__init__(foo=foo, name='Reduce', inputs=len(nodes))
+ for i, n in enumerate(nodes):
+ n._downstream.append((self, i))
+ self._upstream.append(n)
diff --git a/tributary/tests/streaming/test_utils.py b/tributary/tests/streaming/test_utils_streaming.py
similarity index 52%
rename from tributary/tests/streaming/test_utils.py
rename to tributary/tests/streaming/test_utils_streaming.py
index f9b1884..87c6766 100644
--- a/tributary/tests/streaming/test_utils.py
+++ b/tributary/tests/streaming/test_utils_streaming.py
@@ -35,6 +35,36 @@ def test_window_fixed_size(self):
def test_window_fixed_size_full_only(self):
assert ts.run(ts.Window(ts.Foo(foo), size=2, full_only=True)) == [[1, 2]]
- # def test_unroll(self):
- # assert ts.run(ts.Unroll(ts.Foo(foo2))) == [1, 2, 3, 4]
+ def test_unroll(self):
+ assert ts.run(ts.Unroll(ts.Foo(foo2))) == [1, 2, 3, 4]
+ def test_merge(self):
+ def foo1():
+ yield 1
+ yield 3
+
+ def foo2():
+ yield 2
+ yield 4
+ yield 6
+
+ out = ts.Merge(ts.Print(ts.Foo(foo1)), ts.Print(ts.Foo(foo2)))
+ assert ts.run(out) == [(1, 2), (3, 4)]
+
+ def test_reduce(self):
+ def foo1():
+ yield 1
+ yield 4
+
+ def foo2():
+ yield 2
+ yield 5
+ yield 7
+
+ def foo3():
+ yield 3
+ yield 6
+ yield 8
+
+ out = ts.Reduce(ts.Foo(foo1), ts.Foo(foo2), ts.Foo(foo3))
+ assert ts.run(out) == [(1, 2, 3), (4, 5, 6)]
From 34fc7c2c95112425ba2477f326b6febb425a1ee9 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Mon, 17 Feb 2020 17:38:46 -0500
Subject: [PATCH 14/15] update docs
---
README.md | 65 +++++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 51 insertions(+), 14 deletions(-)
diff --git a/README.md b/README.md
index d2bfaf3..7fd2f2a 100644
--- a/README.md
+++ b/README.md
@@ -37,21 +37,10 @@ These are lazily-evaluated python streams, where outputs are propogated only as
- [Streaming](docs/examples/streaming.md)
- [Lazy](docs/examples/lazy.md)
-# Math
-`(Work in progress)`
-
-## Operations
-- unary operators/comparators
-- binary operators/comparators
-
-## Rolling
-- count
-- sum
-
# Sources and Sinks
-`(Work in progress)`
-
## Sources
+- python function/generator/async function/async generator
+- random
- file
- kafka
- websocket
@@ -62,5 +51,53 @@ These are lazily-evaluated python streams, where outputs are propogated only as
- file
- kafka
- http
-- TODO websocket
+- websocket
- TODO socket io
+
+# Transforms
+- Delay - Streaming wrapper to delay a stream
+- Apply - Streaming wrapper to apply a function to an input stream
+- Window - Streaming wrapper to collect a window of values
+- Unroll - Streaming wrapper to unroll an iterable stream
+- UnrollDataFrame - Streaming wrapper to unroll a dataframe into a stream
+- Merge - Streaming wrapper to merge 2 inputs into a single output
+- ListMerge - Streaming wrapper to merge 2 input lists into a single output list
+- DictMerge - Streaming wrapper to merge 2 input dicts into a single output dict. Preference is given to the second input (e.g. if keys overlap)
+- Reduce - Streaming wrapper to merge any number of inputs
+
+# Calculations
+- Noop
+- Negate
+- Invert
+- Add
+- Sub
+- Mult
+- Div
+- RDiv
+- Mod
+- Pow
+- Not
+- And
+- Or
+- Equal
+- NotEqual
+- Less
+- LessOrEqual
+- Greater
+- GreaterOrEqual
+- Log
+- Sin
+- Cos
+- Tan
+- Arcsin
+- Arccos
+- Arctan
+- Sqrt
+- Abs
+- Exp
+- Erf
+- Int
+- Float
+- Bool
+- Str
+- Len
From c286fe3b14566188b476cd84dd654c9bb7647330 Mon Sep 17 00:00:00 2001
From: Tim Paine <t.paine154@gmail.com>
Date: Mon, 17 Feb 2020 17:39:39 -0500
Subject: [PATCH 15/15] fix lint
---
tributary/streaming/base.py | 2 +-
tributary/streaming/input/input.py | 2 --
tributary/streaming/utils.py | 2 ++
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/tributary/streaming/base.py b/tributary/streaming/base.py
index fd42c6d..c982a7a 100644
--- a/tributary/streaming/base.py
+++ b/tributary/streaming/base.py
@@ -199,7 +199,7 @@ def _deep_bfs(self, reverse=True):
'''get nodes by level in tree, reversed relative to output node.
e.g. given a tree that looks like:
A -> B -> D -> F
- \-> C -> E /
+ \\-> C -> E /
the result will be: [[A], [B, C], [D, E], [F]]
This will be the order we synchronously execute, so that within a
diff --git a/tributary/streaming/input/input.py b/tributary/streaming/input/input.py
index 27cacd9..d3bc3f4 100644
--- a/tributary/streaming/input/input.py
+++ b/tributary/streaming/input/input.py
@@ -1,9 +1,7 @@
-import asyncio
import math
import numpy as np
from ..base import Node
-from ...base import StreamEnd
def _gen():
diff --git a/tributary/streaming/utils.py b/tributary/streaming/utils.py
index 0a4dd3c..de043ce 100644
--- a/tributary/streaming/utils.py
+++ b/tributary/streaming/utils.py
@@ -120,6 +120,7 @@ async def foo(value):
node._downstream.append((self, 0))
self._upstream.append(node)
+
class UnrollDataFrame(Node):
'''Streaming wrapper to unroll a dataframe into a stream
@@ -175,6 +176,7 @@ def foo(value1, value2):
self._upstream.append(node1)
self._upstream.append(node2)
+
class ListMerge(Node):
'''Streaming wrapper to merge 2 input lists into a single output list