Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FIX sql handling of timedelta64 columns (GH6921) #7076

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions doc/source/io.rst
Original file line number Diff line number Diff line change
Expand Up @@ -809,8 +809,6 @@ two extra parameters:
String value 'infer' can be used to instruct the parser to try detecting
the column specifications from the first 100 rows of the data. Default
behaviour, if not specified, is to infer.
As with regular python slices, you can slice to the end of the line
with ``None``, e.g. ``colspecs = [(0, 1), (1, None)]``.
- ``widths``: A list of field widths which can be used instead of 'colspecs'
if the intervals are contiguous.

Expand Down Expand Up @@ -3235,6 +3233,13 @@ the database using :func:`~pandas.DataFrame.to_sql`.

data.to_sql('data', engine)

.. note::

Due to the limited support for timedelta's in the different database
flavors, columns with type ``timedelta64`` will be written as integer
values as nanoseconds to the database and a warning will be raised.


Reading Tables
~~~~~~~~~~~~~~

Expand Down
10 changes: 9 additions & 1 deletion pandas/io/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -730,7 +730,10 @@ def _sqlalchemy_type(self, arr_or_dtype):
except:
return DateTime
if com.is_timedelta64_dtype(arr_or_dtype):
return Interval
warnings.warn("the 'timedelta' type is not supported, and will be "
"written as integer values (ns frequency) to the "
"database.", UserWarning)
return Integer
elif com.is_float_dtype(arr_or_dtype):
return Float
elif com.is_integer_dtype(arr_or_dtype):
Expand Down Expand Up @@ -973,6 +976,11 @@ def _sql_type_name(self, dtype):
pytype_name = "text"
if issubclass(pytype, np.floating):
pytype_name = "float"
elif com.is_timedelta64_dtype(pytype):
warnings.warn("the 'timedelta' type is not supported, and will be "
"written as integer values (ns frequency) to the "
"database.", UserWarning)
pytype_name = "int"
elif issubclass(pytype, np.integer):
pytype_name = "int"
elif issubclass(pytype, np.datetime64) or pytype is datetime:
Expand Down
13 changes: 13 additions & 0 deletions pandas/io/tests/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@
from datetime import datetime

from pandas import DataFrame, Series, Index, MultiIndex, isnull
from pandas import to_timedelta
import pandas.compat as compat
from pandas.compat import StringIO, range, lrange
from pandas.core.datetools import format as date_format

import pandas.io.sql as sql
import pandas.util.testing as tm
from pandas import _np_version_under1p7


try:
Expand Down Expand Up @@ -480,6 +482,17 @@ def test_date_and_index(self):
self.assertTrue(issubclass(df.IntDateCol.dtype.type, np.datetime64),
"IntDateCol loaded with incorrect type")

def test_timedelta(self):
# see #6921
if _np_version_under1p7:
raise nose.SkipTest("test only valid in numpy >= 1.7")

df = to_timedelta(Series(['00:00:01', '00:00:03'], name='foo')).to_frame()
with tm.assert_produces_warning(UserWarning):
df.to_sql('test_timedelta', self.conn)
result = sql.read_sql_query('SELECT * FROM test_timedelta', self.conn)
tm.assert_series_equal(result['foo'], df['foo'].astype('int64'))

def test_to_sql_index_label(self):
temp_frame = DataFrame({'col1': range(4)})

Expand Down