[Python] Pandas numpy matplotlib OverflowError: date value out of range

I have code to crawl through every file in a directory and plot every csv file it comes across. Each CSV has a header of the form

Timestamp, P1rms (A), P2rms (A), P3rms (A), P4rms (A), P5rms (A), Vrms (V), P1 theta, P2 theta, P3 theta, P4 theta, P5 theta

Mon Sep 30 00:00:00 2013, 128, 128, 180, 177, 192, 43, 7, 7, 8, 8, 48
Mon Sep 30 00:00:01 2013, 127, 127, 182, 178, 193, 43, 8, 8, 8, 8, 49
# ect....


I am developing a fft visualization option, and I am running into an overflow error when I fft my data sets. here is my exact problem:

When I run my code:

#!/usr/bin/env python
from pandas import *
import matplotlib.pyplot as plt
import os
import sys
import platform
import numpy.fft as np

# name of plots folder
plotfold='plots'

# System specific info
if platform.system()=='Darwin':comsep="/"
else: comsep="\\"

# How many columns should I plot?
numcol=6

if len(sys.argv)<2:
print 'usage: ./canaryCrawler.py [-c] or [-f] rootdir'
quit()

if len(sys.argv)>2:
ylim=1500
root = sys.argv[2]
else:
ylim=1200
root = sys.argv[1]

for subdir, dirs, files in os.walk(root):

# plot each file
for file in files:

if str(file)[-4:]=='.csv':

print 'plotting '+str(file)+'...'
# load csv as data frame
df=pandas.io.parsers.read_csv(subdir+comsep+file)
for i in range(0,len(df.Timestamp)):
df.Timestamp = datetime.strptime(df.Timestamp, '%a %b %d %H:%M:%S %Y')

# We only want the first 6 columns
df = df.ix[:,0:numcol]

if len(sys.argv)>=2:
if sys.argv[1]=='-c' or sys.argv[1]=='-f':
plotfold='plots_Specialty'
df2 = df
df=pandas.DataFrame(df2.Timestamp)
df['Residence']=df2['P1rms (A)']+df2['P2rms (A)']
df['Specialty']=df2['P3rms (A)']+df2['P4rms (A)']
if sys.argv[1]=='-f':
df2['Residence']=np.fft(df['Residence'])
df2['Specialty']=np.fft(df['Specialty'])
df=df2
print 'Fourier Transformation Complete'
plotfold='plots_Specialty_fft'

# set up plot
plt.figure()
df.plot(df.Timestamp,alpha=0.6,linewidth=2.3) # add transparency to see overlapping colors
plt.tight_layout(pad=1.08)
plt.legend(loc='best') # add legend in non-intrusive location
plt.legend(loc=5,prop={'size':14}) #
plt.ylabel('Current')
plt.xlabel('Time')
plt.gcf().autofmt_xdate()
plt.gcf().set_size_inches(12.7,9.2)
plt.gca().set_ylim([0,ylim])

stamp = df.Timestamp[0]
day = datetime.strftime(stamp,'%a')
DOM=datetime.strftime(stamp,'%d')
month = datetime.strftime(stamp,'%b')
year = datetime.strftime(stamp,'%Y')

plt.title(subdir+' '+day+' '+month+' '+DOM+' '+year)

# keep plot

# check for existing plots folder,
# create one if it doesn't exist
if plotfold not in os.listdir(subdir):
print '** adding plots directory to ',subdir
os.mkdir(subdir+comsep+plotfold)

# save in plots directory
spsubs = str(subdir).split(comsep)
filnam=''
for piece in range(len(spsubs)-4,len(spsubs)-1):
filnam+='_'+spsubs[piece]
filnam+='_'+str(file)[:-4]
saveto=subdir+comsep+plotfold+comsep+filnam
print '**** saving plot to ',saveto
plt.savefig(saveto)
plt.close()


I get this error:

kilojoules$ ./canaryCrawler.py -f 35ca7/
/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/pandas-0.14.0-py2.7-macosx-10.9-x86_64.egg/pandas/io/excel.py:626: UserWarning: Installed openpyxl is not supported at this time. Use >=1.6.1 and <2.0.0.
.format(openpyxl_compat.start_ver, openpyxl_compat.stop_ver))
plotting 2014Aug04.csv...
Fourier Transformation Complete
/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/numpy-1.8.1-py2.7-macosx-10.9-x86_64.egg/numpy/core/numeric.py:460: ComplexWarning: Casting complex values to real discards the imaginary part
return array(a, dtype, copy=False, order=order)
/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/tight_layout.py:225: UserWarning: tight_layout : falling back to Agg renderer
warnings.warn("tight_layout : falling back to Agg renderer")
**** saving plot to 35ca7/2014/Aug/plots_Specialty_fft/_Aug_35ca7_2014_2014Aug04
plotting 2014Aug05.csv...
Fourier Transformation Complete
**** saving plot to 35ca7/2014/Aug/plots_Specialty_fft/_Aug_35ca7_2014_2014Aug05
plotting 2014Aug07.csv...
Fourier Transformation Complete
**** saving plot to 35ca7/2014/Aug/plots_Specialty_fft/_Aug_35ca7_2014_2014Aug07
Traceback (most recent call last):
File "./canaryCrawler.py", line 97, in <module>
plt.savefig(saveto)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/pyplot.py", line 561, in savefig
return fig.savefig(*args, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/figure.py", line 1421, in savefig
self.canvas.print_figure(*args, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/backend_bases.py", line 2220, in print_figure
**kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/backends/backend_agg.py", line 505, in print_png
FigureCanvasAgg.draw(self)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/backends/backend_agg.py", line 451, in draw
self.figure.draw(self.renderer)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/artist.py", line 55, in draw_wrapper
draw(artist, renderer, *args, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/figure.py", line 1034, in draw
func(*args)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/artist.py", line 55, in draw_wrapper
draw(artist, renderer, *args, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/axes.py", line 2086, in draw
a.draw(renderer)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/artist.py", line 55, in draw_wrapper
draw(artist, renderer, *args, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/lines.py", line 562, in draw
drawFunc(renderer, gc, tpath, affine.frozen())
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/lines.py", line 938, in _draw_lines
self._lineFunc(renderer, gc, path, trans)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/lines.py", line 978, in _draw_solid
renderer.draw_path(gc, path, trans)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/matplotlib/backends/backend_agg.py", line 145, in draw_path
self._renderer.draw_path(gc, path, transform, rgbFace)
OverflowError: Allocated too many blocks


I specified the agg.path.chunksize parameter in matplotlibrc to o agg.path.chunksize : 10000000. I only get this error when I run the -f fft option. How can I prevent this error?

Continue reading...