From 36a6d81e2e21f28402b7bcd7f072d4b1fe6895ec Mon Sep 17 00:00:00 2001 From: ultra-andy Date: Sat, 19 Mar 2016 10:25:34 +1100 Subject: [PATCH 1/5] Fix to csv2rec bug for review Datetime strings in csv files in dayfirst or yearfirst format with nonzero hour&minute&second will not be properly represented. Date strings will, presuming the csv file contains no strings in that column that have nonzero hour/minute/second components in a datetime string. --- lib/matplotlib/mlab.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/matplotlib/mlab.py b/lib/matplotlib/mlab.py index 31154704dbd..c95c8102dcd 100644 --- a/lib/matplotlib/mlab.py +++ b/lib/matplotlib/mlab.py @@ -2833,7 +2833,13 @@ def mybool(x): raise ValueError('invalid bool') dateparser = dateutil.parser.parse - mydateparser = with_default_value(dateparser, datetime.date(1, 1, 1)) + + def mydateparser(x): + # try and return a datetime object + d = dateparser(x, dayfirst=dayfirst, yearfirst=yearfirst) + return d + mydateparser = with_default_value(mydateparser, datetime.datetime(1,1,1,0,0,0)) + myfloat = with_default_value(float, np.nan) myint = with_default_value(int, -1) mystr = with_default_value(str, '') From bc38288651213ecaec7bbf751e01214d7585d55a Mon Sep 17 00:00:00 2001 From: ultra-andy Date: Mon, 21 Mar 2016 03:40:47 +1100 Subject: [PATCH 2/5] Creation of fix to #6184 - csv2rec bug yearfirst & dayfirst date interpretation bug for datetime strings - csv2rec does not take yearfirst or dayfirst into account when interpreting datetime strings with a time component, so for ambiguous cases, the month and day get transposed. Test added also. --- lib/matplotlib/mlab.py | 3 ++- lib/matplotlib/tests/test_mlab.py | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/matplotlib/mlab.py b/lib/matplotlib/mlab.py index c95c8102dcd..d6b99427b9e 100644 --- a/lib/matplotlib/mlab.py +++ b/lib/matplotlib/mlab.py @@ -2838,7 +2838,8 @@ def mydateparser(x): # try and return a datetime object d = dateparser(x, dayfirst=dayfirst, yearfirst=yearfirst) return d - mydateparser = with_default_value(mydateparser, datetime.datetime(1,1,1,0,0,0)) + + mydateparser = with_default_value(mydateparser, datetime.datetime(1, 1, 1)) myfloat = with_default_value(float, np.nan) myint = with_default_value(int, -1) diff --git a/lib/matplotlib/tests/test_mlab.py b/lib/matplotlib/tests/test_mlab.py index c11f8ec450a..aed9e7cdc69 100644 --- a/lib/matplotlib/tests/test_mlab.py +++ b/lib/matplotlib/tests/test_mlab.py @@ -346,6 +346,50 @@ def test_csv2rec_names_with_comments(self): assert len(array) == 2 assert len(array.dtype) == 3 + def test_csv2rec_usdate(self): + self.fd.write('01/11/14\n' + + '03/05/76 12:00:01 AM\n' + + '07/09/83 5:17:34 PM\n' + + '06/20/2054 2:31:45 PM\n' + + '10/31/00 11:50:23 AM\n') + expected = [datetime.datetime(2014, 1, 11, 0, 0), + datetime.datetime(1976, 3, 5, 0, 0, 1), + datetime.datetime(1983, 7, 9, 17, 17, 34), + datetime.datetime(2054, 6, 20, 14, 31, 45), + datetime.datetime(2000, 10, 31, 11, 50, 23)] + self.fd.seek(0) + array = mlab.csv2rec(self.fd, names='a') + assert_array_equal(array['a'].tolist(), expected) + + def test_csv2rec_dayfirst(self): + self.fd.write('11/01/14\n' + + '05/03/76 12:00:01 AM\n' + + '09/07/83 5:17:34 PM\n' + + '20/06/2054 2:31:45 PM\n' + + '31/10/00 11:50:23 AM\n') + expected = [datetime.datetime(2014, 1, 11, 0, 0), + datetime.datetime(1976, 3, 5, 0, 0, 1), + datetime.datetime(1983, 7, 9, 17, 17, 34), + datetime.datetime(2054, 6, 20, 14, 31, 45), + datetime.datetime(2000, 10, 31, 11, 50, 23)] + self.fd.seek(0) + array = mlab.csv2rec(self.fd, names='a', dayfirst = True) + assert_array_equal(array['a'].tolist(), expected) + + def test_csv2rec_yearfirst(self): + self.fd.write('14/01/11\n' + + '76/03/05 12:00:01 AM\n' + + '83/07/09 5:17:34 PM\n' + + '2054/06/20 2:31:45 PM\n' + + '00/10/31 11:50:23 AM\n') + expected = [datetime.datetime(2014, 1, 11, 0, 0), + datetime.datetime(1976, 3, 5, 0, 0, 1), + datetime.datetime(1983, 7, 9, 17, 17, 34), + datetime.datetime(2054, 6, 20, 14, 31, 45), + datetime.datetime(2000, 10, 31, 11, 50, 23)] + self.fd.seek(0) + array = mlab.csv2rec(self.fd, names='a', yearfirst = True) + assert_array_equal(array['a'].tolist(), expected) class window_testcase(CleanupTestCase): def setUp(self): From 3d1d04394252495d0204bfc57a1340eea8e3102e Mon Sep 17 00:00:00 2001 From: ultra-andy Date: Mon, 21 Mar 2016 04:26:13 +1100 Subject: [PATCH 3/5] forgot the library...! adding in datetime library import --- lib/matplotlib/tests/test_mlab.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/matplotlib/tests/test_mlab.py b/lib/matplotlib/tests/test_mlab.py index aed9e7cdc69..5308818416f 100644 --- a/lib/matplotlib/tests/test_mlab.py +++ b/lib/matplotlib/tests/test_mlab.py @@ -8,6 +8,7 @@ from numpy.testing import assert_allclose, assert_array_equal import numpy.ma.testutils as matest import numpy as np +import datetime as datetime from nose.tools import (assert_equal, assert_almost_equal, assert_not_equal, assert_true, assert_raises) From 9ad1dc25c225f68a457c389a1fcffc34e5143c4c Mon Sep 17 00:00:00 2001 From: ultra-andy Date: Mon, 21 Mar 2016 04:29:41 +1100 Subject: [PATCH 4/5] more PEP-8 fixes... removal of spaces around "=" --- lib/matplotlib/tests/test_mlab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/matplotlib/tests/test_mlab.py b/lib/matplotlib/tests/test_mlab.py index 5308818416f..a11b5192f02 100644 --- a/lib/matplotlib/tests/test_mlab.py +++ b/lib/matplotlib/tests/test_mlab.py @@ -374,7 +374,7 @@ def test_csv2rec_dayfirst(self): datetime.datetime(2054, 6, 20, 14, 31, 45), datetime.datetime(2000, 10, 31, 11, 50, 23)] self.fd.seek(0) - array = mlab.csv2rec(self.fd, names='a', dayfirst = True) + array = mlab.csv2rec(self.fd, names='a', dayfirst=True) assert_array_equal(array['a'].tolist(), expected) def test_csv2rec_yearfirst(self): @@ -389,7 +389,7 @@ def test_csv2rec_yearfirst(self): datetime.datetime(2054, 6, 20, 14, 31, 45), datetime.datetime(2000, 10, 31, 11, 50, 23)] self.fd.seek(0) - array = mlab.csv2rec(self.fd, names='a', yearfirst = True) + array = mlab.csv2rec(self.fd, names='a', yearfirst=True) assert_array_equal(array['a'].tolist(), expected) class window_testcase(CleanupTestCase): From 5f7b1190bed9d7d5b9cc16123e95c64107f1ce94 Mon Sep 17 00:00:00 2001 From: ultra-andy Date: Mon, 21 Mar 2016 05:03:29 +1100 Subject: [PATCH 5/5] PEP-8 fix addition of extra blank line --- lib/matplotlib/tests/test_mlab.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/matplotlib/tests/test_mlab.py b/lib/matplotlib/tests/test_mlab.py index a11b5192f02..3631925cfd4 100644 --- a/lib/matplotlib/tests/test_mlab.py +++ b/lib/matplotlib/tests/test_mlab.py @@ -392,6 +392,7 @@ def test_csv2rec_yearfirst(self): array = mlab.csv2rec(self.fd, names='a', yearfirst=True) assert_array_equal(array['a'].tolist(), expected) + class window_testcase(CleanupTestCase): def setUp(self): np.random.seed(0)