Skip to content

Commit

Permalink
Merge pull request #73 from lyft/remove_zip_fsa
Browse files Browse the repository at this point in the history 
Remove private commits related to ZIP code and FSA (Canada ZIP)
  • Loading branch information
Beto Dealmeida authored Oct 28, 2019
2 parents a25b3c9 + f60aeca commit e557c68
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 195 deletions.
50 changes: 0 additions & 50 deletions superset/assets/src/explore/components/controls/SpatialControl.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ const spatialTypes = {
latlong: 'latlong',
delimited: 'delimited',
geohash: 'geohash',
zipcode: 'zipcode',
fsa: 'fsa',
};

const propTypes = {
Expand Down Expand Up @@ -65,8 +63,6 @@ export default class SpatialControl extends React.Component {
lonlatCol: v.lonlatCol || defaultCol,
reverseCheckbox: v.reverseCheckbox || false,
geohashCol: v.geohashCol || defaultCol,
zipcodeCol: v.zipcodeCol || defaultCol,
fsaCol: v.fsaCol || defaultCol,
value: null,
errors: [],
};
Expand Down Expand Up @@ -101,18 +97,6 @@ export default class SpatialControl extends React.Component {
if (!value.geohashCol) {
errors.push(errMsg);
}
} else if (type === spatialTypes.zipcode) {
value.zipcodeCol = this.state.zipcodeCol;
value.reverseCheckbox = this.state.reverseCheckbox;
if (!value.zipcodeCol) {
errors.push(errMsg);
}
} else if (type === spatialTypes.fsa) {
value.fsaCol = this.state.fsaCol;
value.reverseCheckbox = this.state.reverseCheckbox;
if (!value.fsaCol) {
errors.push(errMsg);
}
}
this.setState({ value, errors });
this.props.onChange(value, errors);
Expand All @@ -136,10 +120,6 @@ export default class SpatialControl extends React.Component {
return `${this.state.lonlatCol}`;
} else if (this.state.type === spatialTypes.geohash) {
return `${this.state.geohashCol}`;
} else if (this.state.type === spatialTypes.zipcode) {
return `${this.state.zipcodeCol}`;
} else if (this.state.type === spatialTypes.fsa) {
return `${this.state.fsaCol}`;
}
return null;
}
Expand Down Expand Up @@ -219,36 +199,6 @@ export default class SpatialControl extends React.Component {
</Col>
</Row>
</PopoverSection>
<PopoverSection
title={t('ZIP code')}
isSelected={this.state.type === spatialTypes.zipcode}
onSelect={this.setType.bind(this, spatialTypes.zipcode)}
>
<Row>
<Col md={6}>
Column
{this.renderSelect('zipcodeCol', spatialTypes.zipcode)}
</Col>
<Col md={6}>
{this.renderReverseCheckbox()}
</Col>
</Row>
</PopoverSection>
<PopoverSection
title={t('FSA')}
isSelected={this.state.type === spatialTypes.fsa}
onSelect={this.setType.bind(this, spatialTypes.fsa)}
>
<Row>
<Col md={6}>
Column
{this.renderSelect('fsaCol', spatialTypes.fsa)}
</Col>
<Col md={6}>
{this.renderReverseCheckbox()}
</Col>
</Row>
</PopoverSection>
<div className="clearfix">
<Button
bsSize="small"
Expand Down
2 changes: 0 additions & 2 deletions superset/assets/src/explore/controls.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2213,8 +2213,6 @@ export const controls = {
['polyline', 'Polyline'],
['json', 'JSON'],
['geohash', 'geohash (square)'],
['zipcode', 'ZIP code'],
['fsa', 'FSA'],
],
},

Expand Down
9 changes: 3 additions & 6 deletions superset/assets/src/visualizations/deckgl/layers/Polygon/Polygon.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,7 @@ import sandboxedEval from '../../../../modules/sandbox';

const DOUBLE_CLICK_TRESHOLD = 250; // milliseconds

function getPoints(features, lineType) {
if (lineType === 'zipcode' || lineType === 'fsa') {
return features.map(d => d.polygon).flat().flat();
}
function getPoints(features) {
return features.map(d => d.polygon).flat();
}

Expand Down Expand Up @@ -97,7 +94,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip, selected, o
}
return baseColor;
};
const tooltipContentGenerator = (fd.line_column && fd.metric && ['geohash', 'zipcode', 'fsa'].indexOf(fd.line_type) >= 0)
const tooltipContentGenerator = (fd.line_column && fd.metric && fd.line_type === 'geohash')
? setTooltipContent(fd)
: undefined;
return new PolygonLayer({
Expand Down Expand Up @@ -170,7 +167,7 @@ class DeckGLPolygon extends React.Component {
} = getPlaySliderParams(timestamps, granularity);

const viewport = props.formData.autozoom
? fitViewport(props.viewport, getPoints(features, props.payload.form_data.line_type))
? fitViewport(props.viewport, getPoints(features))
: props.viewport;

return {
Expand Down
139 changes: 2 additions & 137 deletions superset/viz.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
from itertools import product
import logging
import math
import os
import pickle as pkl
import re
from typing import Any, Dict, List, Optional
Expand All @@ -46,7 +45,6 @@
from pandas.tseries.frequencies import to_offset
import polyline
import simplejson as json
import sqlalchemy

from superset import app, cache, get_css_manifest_files
from superset.exceptions import NullValueException, SpatialException
Expand Down Expand Up @@ -2132,10 +2130,6 @@ def get_spatial_columns(self, key):
return [spatial.get("lonlatCol")]
elif spatial.get("type") == "geohash":
return [spatial.get("geohashCol")]
elif spatial.get("type") == "zipcode":
return [spatial.get("zipcodeCol")]
elif spatial.get("type") == "fsa":
return [spatial.get("fsaCol")]

@staticmethod
def parse_coordinates(s):
Expand All @@ -2152,18 +2146,6 @@ def reverse_geohash_decode(geohash_code):
lat, lng = geohash.decode(geohash_code)
return (lng, lat)

@staticmethod
def reverse_zipcode_decode(zipcode):
raise NotImplementedError(
"No mapping from ZIP code to single latitude/longitude"
)

@staticmethod
def reverse_fsa_decode(fsa):
raise NotImplementedError(
"No mapping from FSA code to single latitude/longitude"
)

@staticmethod
def reverse_latlong(df, key):
df[key] = [tuple(reversed(o)) for o in df[key] if isinstance(o, (list, tuple))]
Expand All @@ -2187,12 +2169,6 @@ def process_spatial_data_obj(self, key, df):
elif spatial.get("type") == "geohash":
df[key] = df[spatial.get("geohashCol")].map(self.reverse_geohash_decode)
del df[spatial.get("geohashCol")]
elif spatial.get("type") == "zipcode":
df[key] = df[spatial.get("zipcodeCol")].map(self.reverse_zipcode_decode)
del df[spatial.get("zipcodeCol")]
elif spatial.get("type") == "fsa":
df[key] = df[spatial.get("fsaCol")].map(self.reverse_fsa_decode)
del df[spatial.get("fsaCol")]

if spatial.get("reverseCheckbox"):
self.reverse_latlong(df, key)
Expand Down Expand Up @@ -2382,106 +2358,6 @@ def geohash_to_json(geohash_code):
]


def zipcode_deser(zipcodes):
geojson = zipcodes_to_json(zipcodes)

def deser(zipcode):
if str(zipcode) in geojson:
return geojson[str(zipcode)]["coordinates"][0]

return deser


def fsa_deser(fsas):
geojson = fsas_to_json(fsas)

def deser(fsa):
if fsa in geojson:
return geojson[fsa][0][0]

return deser


def zipcodes_to_json(zipcodes):
user = os.environ.get("CREDENTIALS_LYFTPG_USER", "")
password = os.environ.get("CREDENTIALS_LYFTPG_PASSWORD", "")
url = (
"postgresql+psycopg2://"
"{user}:{password}"
"@analytics-platform-vpc.c067nfzisc99.us-east-1.rds.amazonaws.com:5432"
"/platform".format(user=user, password=password)
)

out = {}
missing = set()
for zipcode in zipcodes:
cache_key = "zipcode_geojson_{}".format(zipcode)
geojson = cache and cache.get(cache_key)
if geojson:
out[zipcode] = geojson
else:
missing.add(str(zipcode))

if not missing:
return out

# fetch missing geojson from
in_clause = ", ".join(["%s"] * len(missing))
query = "SELECT zipcode, geojson FROM zip_codes WHERE zipcode IN ({0})".format(
in_clause
)
conn = sqlalchemy.create_engine(url, client_encoding="utf8")
results = conn.execute(query, tuple(missing)).fetchall()

for zipcode, geojson in results:
out[zipcode] = geojson
if cache and len(results) < 10000: # avoid storing too much
cache_key = "zipcode_geojson_{}".format(zipcode)
try:
cache.set(cache_key, geojson, timeout=86400)
except Exception:
pass

return out


def fsas_to_json(fsas):
url = "presto://prestoproxy.lyft.net:8443/hive"

out = {}
missing = set()
for fsa in fsas:
cache_key = "fsa_geojson_{}".format(fsa)
geojson = cache and cache.get(cache_key)
if geojson:
out[fsa] = geojson
else:
missing.add(fsa)

if not missing:
return out

# fetch missing geojson from lyftpg
in_clause = ", ".join(["%s"] * len(missing))
query = "SELECT fsa, geo_json FROM jbridgem.test_geo_json_fsa WHERE fsa IN ({0})".format(
in_clause
)
conn = sqlalchemy.create_engine(url, connect_args={"protocol": "https"})
results = conn.execute(query, tuple(missing)).fetchall()

for fsa, geojson in results:
geojson = json.loads(geojson)
out[fsa] = geojson
if cache and len(results) < 10000: # avoid storing too much
cache_key = "fsa_geojson_{}".format(fsa)
try:
cache.set(cache_key, geojson, timeout=86400)
except Exception:
pass

return out


class DeckPathViz(BaseDeckGLViz):

"""deck.gl's PathLayer"""
Expand All @@ -2494,8 +2370,6 @@ class DeckPathViz(BaseDeckGLViz):
"json": json.loads,
"polyline": polyline.decode,
"geohash": geohash_to_json,
"zipcode": None, # per request
"fsa": None, # per request
}

def query_obj(self):
Expand All @@ -2521,23 +2395,14 @@ def get_properties(self, d):
if fd.get("reverse_long_lat"):
path = [(o[1], o[0]) for o in path]
d[self.deck_viz_key] = path
if line_type not in ["geohash", "zipcode", "fsa"]:
if line_type != "geohash":
del d[line_column]
d["__timestamp"] = d.get(DTTM_ALIAS) or d.get("__time")
return d

def get_data(self, df):
self.metric_label = utils.get_metric_name(self.metric)
fd = self.form_data
line_type = fd.get("line_type")
if line_type == "zipcode":
zipcodes = df[fd["line_column"]].unique()
self.deser_map["zipcode"] = zipcode_deser(zipcodes)
elif line_type == "fsa":
fsas = df[fd["line_column"]].unique()
self.deser_map["fsa"] = fsa_deser(fsas)

return super(DeckPathViz, self).get_data(df)
return super().get_data(df)


class DeckPolygon(DeckPathViz):
Expand Down

0 comments on commit e557c68

Please sign in to comment.