Skip to content

Commit

Permalink
Workaround pandas bug in datetimes with time zones (#3910)
Browse files Browse the repository at this point in the history
A bug in to_dict(orient="records") in pandas/core/frame.py prevents
datetimes with time zones to be worked with. This works around the
issue in superset by re-implementing the logic of pandas in the
correct way. Until pandas fixes the issue this code should stay.

pandas-dev/pandas#18372

This closes #1929
  • Loading branch information
bolkedebruin authored and mistercrunch committed Nov 20, 2017
1 parent 3c72e1f commit 4ae77ba
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 2 deletions.
6 changes: 5 additions & 1 deletion superset/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import numpy as np
import pandas as pd
from pandas.core.common import _maybe_box_datetimelike
from pandas.core.dtypes.dtypes import ExtensionDtype
from past.builtins import basestring

Expand Down Expand Up @@ -48,7 +49,10 @@ def size(self):

@property
def data(self):
return self.__df.to_dict(orient='records')
# work around for https://github.com/pandas-dev/pandas/issues/18372
return [dict((k, _maybe_box_datetimelike(v))
for k, v in zip(self.__df.columns, np.atleast_1d(row)))
for row in self.__df.values]

@classmethod
def db_type(cls, dtype):
Expand Down
21 changes: 20 additions & 1 deletion tests/core_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from __future__ import unicode_literals

import csv
import datetime
import doctest
import io
import json
Expand All @@ -13,9 +14,11 @@
import unittest

from flask import escape
import pandas as pd
import psycopg2
import sqlalchemy as sqla

from superset import appbuilder, db, jinja_context, sm, sql_lab, utils
from superset import appbuilder, dataframe, db, jinja_context, sm, sql_lab, utils
from superset.connectors.sqla.models import SqlaTable
from superset.models import core as models
from superset.models.sql_lab import Query
Expand Down Expand Up @@ -786,6 +789,22 @@ def test_viz_get_fillna_for_columns(self):
{'name': ' NULL', 'sum__num': 0},
)

def test_dataframe_timezone(self):
tz = psycopg2.tz.FixedOffsetTimezone(offset=60, name=None)
data = [(datetime.datetime(2017, 11, 18, 21, 53, 0, 219225, tzinfo=tz),),
(datetime.datetime(2017, 11, 18, 22, 6, 30, 61810, tzinfo=tz,),)]
df = dataframe.SupersetDataFrame(pd.DataFrame(data=list(data),
columns=['data', ]))
data = df.data
self.assertDictEqual(
data[0],
{'data': pd.Timestamp('2017-11-18 21:53:00.219225+0100', tz=tz), },
)
self.assertDictEqual(
data[1],
{'data': pd.Timestamp('2017-11-18 22:06:30.061810+0100', tz=tz), },
)


if __name__ == '__main__':
unittest.main()

0 comments on commit 4ae77ba

Please sign in to comment.