Skip to content
This repository has been archived by the owner on Feb 7, 2025. It is now read-only.

Sampling ui #61

Merged
merged 8 commits into from
Jan 23, 2020
Merged

Sampling ui #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2fdf343
Work in progress
okennedy Jan 21, 2020
7f1a70b
Fixing Mimir processor package
okennedy Jan 22, 2020
46d7836
Adding redirect from root to web UI path
okennedy Jan 22, 2020
9ee39ff
Ignoring temporary data directories
okennedy Jan 22, 2020
22e8239
Fixing redirect from root to web UI path
okennedy Jan 22, 2020
07f967c
Adding support for uniform sampling
okennedy Jan 22, 2020
5c1a3a7
Adding support for stratified sampling
okennedy Jan 22, 2020
f15d0bc
Automatic stratified sampling
okennedy Jan 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,6 @@ vizier-data/
vizier-data*/
vizier/env
resources/config/dev-config.yaml
/vizier/ds
/vizier/fs
/vizier/vt
273 changes: 273 additions & 0 deletions resources/packages/common/sample.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
{
"sampling": {
"command": [
{
"format" : [
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "CREATE"
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "sample_rate"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "SAMPLE"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "OF "
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "input_dataset"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "AS"
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "output_dataset"
}
],
"id": "basic_sample",
"name": "Basic Sample",
"parameter": [
{
"datatype": "dataset",
"hidden": false,
"id": "input_dataset",
"index": 0,
"name": "Input Dataset",
"required": true
},
{
"datatype": "decimal",
"hidden": false,
"id": "sample_rate",
"index": 1,
"name": "Sampling Rate (0.0-1.0)",
"required": true
},
{
"datatype": "string",
"hidden": false,
"id": "output_dataset",
"index": 2,
"name": "Output Dataset",
"required": false
}

]
},
{
"format" : [
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "CREATE"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "MANUALLY"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "STRATIFIED"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "SAMPLE"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "OF "
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "input_dataset"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "AS"
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "output_dataset"
}
],
"id": "manual_stratified_sample",
"name": "Manually Stratified Sample",
"parameter": [
{
"datatype": "dataset",
"hidden": false,
"id": "input_dataset",
"index": 0,
"name": "Input Dataset",
"required": true
},
{
"datatype": "colid",
"hidden": false,
"id": "stratification_column",
"index": 1,
"name": "Column",
"required": true
},
{
"datatype": "list",
"hidden": false,
"id": "strata",
"index": 2,
"name": "Strata",
"required": true
},
{
"datatype": "string",
"hidden": false,
"id": "stratum_value",
"index": 3,
"name": "Column Value",
"parent": "strata",
"required": true
},
{
"datatype": "decimal",
"hidden": false,
"id": "sample_rate",
"index": 4,
"name": "Sampling Rate (0.0-1.0)",
"parent": "strata",
"required": true
},
{
"datatype": "string",
"hidden": false,
"id": "output_dataset",
"index": 2,
"name": "Output Dataset",
"required": false
}

]
},
{
"format" : [
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "CREATE"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "STRATIFIED"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "SAMPLE"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "OF "
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "input_dataset"
},
{
"lspace": true,
"rspace": true,
"type": "const",
"value": "AS"
},
{
"lspace": true,
"rspace": true,
"type": "var",
"value": "output_dataset"
}
],
"id": "automatic_stratified_sample",
"name": "Automatically Stratified Sample",
"parameter": [
{
"datatype": "dataset",
"hidden": false,
"id": "input_dataset",
"index": 0,
"name": "Input Dataset",
"required": true
},
{
"datatype": "colid",
"hidden": false,
"id": "stratification_column",
"index": 1,
"name": "Column",
"required": true
},
{
"datatype": "decimal",
"hidden": false,
"id": "sample_rate",
"index": 2,
"name": "Total Sampling Rate (0.0-1.0)",
"required": true
},
{
"datatype": "string",
"hidden": false,
"id": "output_dataset",
"index": 3,
"name": "Output Dataset",
"required": false
}

]
}
],
"id": "sampling",
"category": "code"
}
}
5 changes: 5 additions & 0 deletions resources/processors/mimir/sampling.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
packages:
- sampling
engine:
className: 'SamplingProcessor'
moduleName: 'vizier.engine.packages.sample.processor'
10 changes: 9 additions & 1 deletion vizier/api/webservice/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import logging
import os

from flask import Flask, jsonify, make_response
from flask import Flask, jsonify, make_response, redirect
from flask_cors import CORS
from logging.handlers import RotatingFileHandler

Expand Down Expand Up @@ -124,4 +124,12 @@ def initialize():
if config.engine.identifier == const.MIMIR_ENGINE:
import vizier.mimir as mimir

root_redirect_path = "{}/web-ui/vizier-db".format(server.bp.url_prefix)
@app.route("/")
def handle_root():
"""Redirect users to the web UI
"""
return redirect(root_redirect_path)


return app
Empty file added vizier/engine/packages/sample/__init__.py
View file
Open in desktop
Empty file.
43 changes: 43 additions & 0 deletions vizier/engine/packages/sample/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Copyright (C) 2017-2019 New York University,
# University at Buffalo,
# Illinois Institute of Technology.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Specification of parameters for SQL cell."""

import vizier.engine.packages.base as pckg


"""Global constants."""

# Package name
PACKAGE_SAMPLE = 'sample'

# Sampling modes (must correspond w/ Mimir's mimir.algebra.sampling._)
SAMPLING_MODE_UNIFORM_PROBABILITY = 'uniform_probability'
SAMPLING_MODE_STRATIFIED_ON = 'stratified_on'

# Command identifier (unique within the package)
BASIC_SAMPLE = "basic_sample"
MANUAL_STRATIFIED_SAMPLE = "manual_stratified_sample"
AUTOMATIC_STRATIFIED_SAMPLE = "automatic_stratified_sample"

# SQL command parameters
PARA_INPUT_DATASET = 'input_dataset'
PARA_OUTPUT_DATASET = 'output_dataset'
PARA_SAMPLING_RATE = 'sample_rate'
PARA_STRATIFICATION_COLUMN = 'stratification_column'
PARA_STRATA = 'strata'
PARA_STRATUM_VALUE = 'stratum_value'

Loading