dorian
/
paramiko
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[project @ Arch-1:robey@lag.net--2003-public%secsh--dev--1.0--patch-59] 

speed up parts of BufferedFile
BufferedFile uses cStringIO for the write buffer now (i don't actually notice
any speed difference so this might revert later) and the default buffer size
has been upped from 1KB to 8KB.

when scanning for linefeeds (when writing to a line-buffered file), only scan
the newly-written bytes, since we know all the previously buffered data is
linefeed-free.  this was the #1 slowdown on the 1MB-file unit test.

also, limit the buffering on line-buffered files to whatever the default
buffer size is.  there's no reason to buffer 1MB waiting for a linefeed.
This commit is contained in:
Robey Pointer 2004-06-10 18:02:13 +00:00
parent cba104ce3f
commit f0ba3c482e
1 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
paramiko/file.py
View File

@ -15,13 +15,15 @@
# details. # details.
# #
# You should have received a copy of the GNU Lesser General Public License # You should have received a copy of the GNU Lesser General Public License
# along with Foobar; if not, write to the Free Software Foundation, Inc., # along with Paramiko; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
""" """
BufferedFile. BufferedFile.
""" """
from cStringIO import StringIO
_FLAG_READ = 0x1 _FLAG_READ = 0x1
_FLAG_WRITE = 0x2 _FLAG_WRITE = 0x2
_FLAG_APPEND = 0x4 _FLAG_APPEND = 0x4
@ -36,7 +38,7 @@ class BufferedFile (object):
simpler stream simpler stream
""" """
_DEFAULT_BUFSIZE = 1024 _DEFAULT_BUFSIZE = 8192
SEEK_SET = 0 SEEK_SET = 0
SEEK_CUR = 1 SEEK_CUR = 1
@ -45,7 +47,8 @@ class BufferedFile (object):
def __init__(self): def __init__(self):
self._flags = 0 self._flags = 0
self._bufsize = self._DEFAULT_BUFSIZE self._bufsize = self._DEFAULT_BUFSIZE
self._wbuffer = self._rbuffer = '' self._wbuffer = StringIO()
self._rbuffer = ''
self._at_trailing_cr = False self._at_trailing_cr = False
self._closed = False self._closed = False
# pos - position within the file, according to the user # pos - position within the file, according to the user
@ -80,8 +83,8 @@ class BufferedFile (object):
Write out any data in the write buffer. This may do nothing if write Write out any data in the write buffer. This may do nothing if write
buffering is not turned on. buffering is not turned on.
""" """
self._write_all(self._wbuffer) self._write_all(self._wbuffer.getvalue())
self._wbuffer = '' self._wbuffer = StringIO()
return return
def next(self): def next(self):
@ -300,16 +303,21 @@ class BufferedFile (object):
if not (self._flags & _FLAG_BUFFERED): if not (self._flags & _FLAG_BUFFERED):
self._write_all(data) self._write_all(data)
return return
self._wbuffer += data self._wbuffer.write(data)
if self._flags & _FLAG_LINE_BUFFERED: if self._flags & _FLAG_LINE_BUFFERED:
last_newline_pos = self._wbuffer.rfind('\n') # only scan the new data for linefeed, to avoid wasting time.
last_newline_pos = data.rfind('\n')
if last_newline_pos >= 0: if last_newline_pos >= 0:
self._write_all(self._wbuffer[:last_newline_pos + 1]) wbuf = self._wbuffer.getvalue()
self._wbuffer = self._wbuffer[last_newline_pos+1:] last_newline_pos += len(wbuf) - len(data)
else: self._write_all(wbuf[:last_newline_pos + 1])
if len(self._wbuffer) >= self._bufsize: self._wbuffer = StringIO()
self._write_all(self._wbuffer) self._wbuffer.write(wbuf[last_newline_pos+1:])
self._wbuffer = '' return
# even if we're line buffering, if the buffer has grown past the
# buffer size, force a flush.
if len(self._wbuffer.getvalue()) >= self._bufsize:
self.flush()
return return
def writelines(self, sequence): def writelines(self, sequence):