dorian
/
paramiko
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[project @ robey@lag.net-20060309081455-84be2ae54f98e897] 

move sftp big-file tests into a separate class and add one that does a prefetch, then seeks in random order
This commit is contained in:
Robey Pointer 2006-03-09 00:14:55 -08:00
parent de6315b9c5
commit 89e39a4559
3 changed files with 283 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
test.py
View File

@ -37,6 +37,7 @@ from test_kex import KexTest
from test_packetizer import PacketizerTest
from test_transport import TransportTest
from test_sftp import SFTPTest
from test_sftp_big import BigSFTPTest
default_host = 'localhost'
default_user = os.environ.get('USER', 'nobody')
@ -99,6 +100,8 @@ if options.use_transport:
suite.addTest(unittest.makeSuite(TransportTest))
if options.use_sftp:
suite.addTest(unittest.makeSuite(SFTPTest))
if options.use_big_file:
suite.addTest(unittest.makeSuite(BigSFTPTest))
verbosity = 1
if options.verbose:
verbosity = 2

196
tests/test_sftp.py
View File

@ -26,6 +26,7 @@ do test file operations in (so no existing files will be harmed).
import logging
import os
import random
import struct
import sys
import threading
import time
@ -69,6 +70,11 @@ tc = None
g_big_file_test = True
def get_sftp():
global sftp
return sftp
class SFTPTest (unittest.TestCase):
def init(hostname, username, keyfile, passwd):
@ -449,183 +455,7 @@ class SFTPTest (unittest.TestCase):
except:
pass
def test_E_lots_of_files(self):
"""
create a bunch of files over the same session.
"""
global g_big_file_test
if not g_big_file_test:
return
numfiles = 100
try:
for i in range(numfiles):
f = sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1)
f.write('this is file #%d.\n' % i)
f.close()
sftp.chmod('%s/file%d.txt' % (FOLDER, i), 0660)
# now make sure every file is there, by creating a list of filenmes
# and reading them in random order.
numlist = range(numfiles)
while len(numlist) > 0:
r = numlist[random.randint(0, len(numlist) - 1)]
f = sftp.open('%s/file%d.txt' % (FOLDER, r))
self.assertEqual(f.readline(), 'this is file #%d.\n' % r)
f.close()
numlist.remove(r)
finally:
for i in range(numfiles):
try:
sftp.remove('%s/file%d.txt' % (FOLDER, i))
except:
pass
def test_F_big_file(self):
"""
write a 1MB file with no buffering.
"""
global g_big_file_test
if not g_big_file_test:
return
kblob = (1024 * 'x')
start = time.time()
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
start = time.time()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
for n in range(1024):
data = f.read(1024)
self.assertEqual(data, kblob)
f.close()
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_G_big_file_pipelined(self):
"""
write a 1MB file, with no linefeeds, using pipelining.
"""
global g_big_file_test
if not g_big_file_test:
return
kblob = (1024 * 'x')
start = time.time()
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
f.set_pipelined(True)
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
start = time.time()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
for n in range(1024):
data = f.read(1024)
self.assertEqual(data, kblob)
f.close()
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_H_lots_of_prefetching(self):
"""
prefetch a 1MB file a bunch of times, discarding the file object
without using it, to verify that paramiko doesn't get confused.
"""
global g_big_file_test
if not g_big_file_test:
return
kblob = (1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
f.set_pipelined(True)
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
for i in range(10):
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
for n in range(1024):
data = f.read(1024)
self.assertEqual(data, kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_I_big_file_big_buffer(self):
"""
write a 1MB file, with no linefeeds, and a big buffer.
"""
global g_big_file_test
if not g_big_file_test:
return
mblob = (1024 * 1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
f.write(mblob)
f.close()
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_J_big_file_renegotiate(self):
"""
write a 1MB file, forcing key renegotiation in the middle.
"""
global g_big_file_test
if not g_big_file_test:
return
t = sftp.sock.get_transport()
t.packetizer.REKEY_BYTES = 512 * 1024
k32blob = (32 * 1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
for i in xrange(32):
f.write(k32blob)
f.close()
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
self.assertNotEquals(t.H, t.session_id)
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
t.packetizer.REKEY_BYTES = pow(2, 30)
def test_K_realpath(self):
def test_E_realpath(self):
"""
test that realpath is returning something non-empty and not an
error.
@ -636,7 +466,7 @@ class SFTPTest (unittest.TestCase):
self.assert_(len(f) > 0)
self.assertEquals(os.path.join(pwd, FOLDER), f)
def test_L_mkdir(self):
def test_F_mkdir(self):
"""
verify that mkdir/rmdir work.
"""
@ -659,7 +489,7 @@ class SFTPTest (unittest.TestCase):
except IOError:
pass
def test_M_chdir(self):
def test_G_chdir(self):
"""
verify that chdir/getcwd work.
"""
@ -696,7 +526,7 @@ class SFTPTest (unittest.TestCase):
except:
pass
def test_N_get_put(self):
def test_H_get_put(self):
"""
verify that get/put work.
"""
@ -725,7 +555,7 @@ class SFTPTest (unittest.TestCase):
os.unlink(localname)
sftp.unlink(FOLDER + '/bunny.txt')
def test_O_check(self):
def test_I_check(self):
"""
verify that file.check() works against our own server.
(it's an sftp extension that we support, and may be the only ones who
@ -747,7 +577,7 @@ class SFTPTest (unittest.TestCase):
finally:
sftp.unlink(FOLDER + '/kitty.txt')
def test_P_x_flag(self):
def test_J_x_flag(self):
"""
verify that the 'x' flag works when opening a file.
"""
@ -763,7 +593,7 @@ class SFTPTest (unittest.TestCase):
finally:
sftp.unlink(FOLDER + '/unusual.txt')
def test_Q_utf8(self):
def test_K_utf8(self):
"""
verify that unicode strings are encoded into utf8 correctly.
"""

267
tests/test_sftp_big.py Normal file
View File

@ -0,0 +1,267 @@
# Copyright (C) 2003-2005 Robey Pointer <robey@lag.net>
#
# This file is part of paramiko.
#
# Paramiko is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
"""
some unit tests to make sure sftp works well with large files.
a real actual sftp server is contacted, and a new folder is created there to
do test file operations in (so no existing files will be harmed).
"""
import logging
import os
import random
import struct
import sys
import threading
import time
import unittest
import paramiko
from stub_sftp import StubServer, StubSFTPServer
from loop import LoopSocket
from test_sftp import get_sftp
FOLDER = os.environ.get('TEST_FOLDER', 'temp-testing000')
class BigSFTPTest (unittest.TestCase):
def setUp(self):
global FOLDER
sftp = get_sftp()
for i in xrange(1000):
FOLDER = FOLDER[:-3] + '%03d' % i
try:
sftp.mkdir(FOLDER)
break
except (IOError, OSError):
pass
def tearDown(self):
sftp = get_sftp()
sftp.rmdir(FOLDER)
def test_1_lots_of_files(self):
"""
create a bunch of files over the same session.
"""
sftp = get_sftp()
numfiles = 100
try:
for i in range(numfiles):
f = sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1)
f.write('this is file #%d.\n' % i)
f.close()
sftp.chmod('%s/file%d.txt' % (FOLDER, i), 0660)
# now make sure every file is there, by creating a list of filenmes
# and reading them in random order.
numlist = range(numfiles)
while len(numlist) > 0:
r = numlist[random.randint(0, len(numlist) - 1)]
f = sftp.open('%s/file%d.txt' % (FOLDER, r))
self.assertEqual(f.readline(), 'this is file #%d.\n' % r)
f.close()
numlist.remove(r)
finally:
for i in range(numfiles):
try:
sftp.remove('%s/file%d.txt' % (FOLDER, i))
except:
pass
def test_2_big_file(self):
"""
write a 1MB file with no buffering.
"""
sftp = get_sftp()
kblob = (1024 * 'x')
start = time.time()
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
start = time.time()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
for n in range(1024):
data = f.read(1024)
self.assertEqual(data, kblob)
f.close()
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_3_big_file_pipelined(self):
"""
write a 1MB file, with no linefeeds, using pipelining.
"""
sftp = get_sftp()
kblob = ''.join([struct.pack('>H', n) for n in xrange(512)])
start = time.time()
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
f.set_pipelined(True)
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
start = time.time()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
# read on odd boundaries to make sure the bytes aren't getting scrambled
n = 0
k2blob = kblob + kblob
chunk = 629
size = 1024 * 1024
while n < size:
if n + chunk > size:
chunk = size - n
data = f.read(chunk)
offset = n % 1024
self.assertEqual(data, k2blob[offset:offset + chunk])
n += chunk
f.close()
end = time.time()
sys.stderr.write('%ds ' % round(end - start))
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_4_prefetch_seek(self):
sftp = get_sftp()
kblob = ''.join([struct.pack('>H', n) for n in xrange(512)])
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
f.set_pipelined(True)
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
k2blob = kblob + kblob
chunk = 793
for i in xrange(10):
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
base_offset = (512 * 1024) + 17 * random.randint(1000, 2000)
offsets = [base_offset + j * chunk for j in xrange(100)]
# randomly seek around and read them out
for j in xrange(100):
offset = offsets[random.randint(0, len(offsets) - 1)]
offsets.remove(offset)
f.seek(offset)
data = f.read(chunk)
n_offset = offset % 1024
self.assertEqual(data, k2blob[n_offset:n_offset + chunk])
offset += chunk
f.close()
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_5_lots_of_prefetching(self):
"""
prefetch a 1MB file a bunch of times, discarding the file object
without using it, to verify that paramiko doesn't get confused.
"""
sftp = get_sftp()
kblob = (1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
f.set_pipelined(True)
for n in range(1024):
f.write(kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
for i in range(10):
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
f.prefetch()
for n in range(1024):
data = f.read(1024)
self.assertEqual(data, kblob)
if n % 128 == 0:
sys.stderr.write('.')
f.close()
sys.stderr.write(' ')
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_6_big_file_big_buffer(self):
"""
write a 1MB file, with no linefeeds, and a big buffer.
"""
sftp = get_sftp()
mblob = (1024 * 1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
f.write(mblob)
f.close()
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
def test_7_big_file_renegotiate(self):
"""
write a 1MB file, forcing key renegotiation in the middle.
"""
sftp = get_sftp()
t = sftp.sock.get_transport()
t.packetizer.REKEY_BYTES = 512 * 1024
k32blob = (32 * 1024 * 'x')
try:
f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
for i in xrange(32):
f.write(k32blob)
f.close()
self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
self.assertNotEquals(t.H, t.session_id)
finally:
sftp.remove('%s/hongry.txt' % FOLDER)
t.packetizer.REKEY_BYTES = pow(2, 30)