[project @ robey@lag.net-20060309081455-84be2ae54f98e897]

move sftp big-file tests into a separate class and add one that does a prefetch, then seeks in random order

test.py

@@ -37,6 +37,7 @@ from test_kex import KexTest
 from test_packetizer import PacketizerTest
 from test_transport import TransportTest
 from test_sftp import SFTPTest
+from test_sftp_big import BigSFTPTest
 
 default_host = 'localhost'
 default_user = os.environ.get('USER', 'nobody')
@@ -99,6 +100,8 @@ if options.use_transport:
     suite.addTest(unittest.makeSuite(TransportTest))
 if options.use_sftp:
     suite.addTest(unittest.makeSuite(SFTPTest))
+if options.use_big_file:
+    suite.addTest(unittest.makeSuite(BigSFTPTest))
 verbosity = 1
 if options.verbose:
     verbosity = 2

tests/test_sftp.py

@@ -26,6 +26,7 @@ do test file operations in (so no existing files will be harmed).
 import logging
 import os
 import random
+import struct
 import sys
 import threading
 import time
@@ -69,6 +70,11 @@ tc = None
 g_big_file_test = True
 
 
+def get_sftp():
+    global sftp
+    return sftp
+
+
 class SFTPTest (unittest.TestCase):
 
     def init(hostname, username, keyfile, passwd):
@@ -449,183 +455,7 @@ class SFTPTest (unittest.TestCase):
             except:
                 pass
 
-    def test_E_lots_of_files(self):
+    def test_E_realpath(self):
-        """
-        create a bunch of files over the same session.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        numfiles = 100
-        try:
-            for i in range(numfiles):
-                f = sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1)
-                f.write('this is file #%d.\n' % i)
-                f.close()
-                sftp.chmod('%s/file%d.txt' % (FOLDER, i), 0660)
-
-            # now make sure every file is there, by creating a list of filenmes
-            # and reading them in random order.
-            numlist = range(numfiles)
-            while len(numlist) > 0:
-                r = numlist[random.randint(0, len(numlist) - 1)]
-                f = sftp.open('%s/file%d.txt' % (FOLDER, r))
-                self.assertEqual(f.readline(), 'this is file #%d.\n' % r)
-                f.close()
-                numlist.remove(r)
-        finally:
-            for i in range(numfiles):
-                try:
-                    sftp.remove('%s/file%d.txt' % (FOLDER, i))
-                except:
-                    pass
-
-    def test_F_big_file(self):
-        """
-        write a 1MB file with no buffering.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        kblob = (1024 * 'x')
-        start = time.time()
-        try:
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
-            for n in range(1024):
-                f.write(kblob)
-                if n % 128 == 0:
-                    sys.stderr.write('.')
-            f.close()
-            sys.stderr.write(' ')
-
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
-            end = time.time()
-            sys.stderr.write('%ds ' % round(end - start))
-            
-            start = time.time()
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
-            for n in range(1024):
-                data = f.read(1024)
-                self.assertEqual(data, kblob)
-            f.close()
-
-            end = time.time()
-            sys.stderr.write('%ds ' % round(end - start))
-        finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
-
-    def test_G_big_file_pipelined(self):
-        """
-        write a 1MB file, with no linefeeds, using pipelining.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        kblob = (1024 * 'x')
-        start = time.time()
-        try:
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
-            f.set_pipelined(True)
-            for n in range(1024):
-                f.write(kblob)
-                if n % 128 == 0:
-                    sys.stderr.write('.')
-            f.close()
-            sys.stderr.write(' ')
-
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
-            end = time.time()
-            sys.stderr.write('%ds ' % round(end - start))
-            
-            start = time.time()
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
-            f.prefetch()
-            for n in range(1024):
-                data = f.read(1024)
-                self.assertEqual(data, kblob)
-            f.close()
-
-            end = time.time()
-            sys.stderr.write('%ds ' % round(end - start))
-        finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
-
-    def test_H_lots_of_prefetching(self):
-        """
-        prefetch a 1MB file a bunch of times, discarding the file object
-        without using it, to verify that paramiko doesn't get confused.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        kblob = (1024 * 'x')
-        try:
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
-            f.set_pipelined(True)
-            for n in range(1024):
-                f.write(kblob)
-                if n % 128 == 0:
-                    sys.stderr.write('.')
-            f.close()
-            sys.stderr.write(' ')
-
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
-
-            for i in range(10):
-                f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
-                f.prefetch()
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
-            f.prefetch()
-            for n in range(1024):
-                data = f.read(1024)
-                self.assertEqual(data, kblob)
-                if n % 128 == 0:
-                    sys.stderr.write('.')
-            f.close()
-            sys.stderr.write(' ')
-        finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
-        
-    def test_I_big_file_big_buffer(self):
-        """
-        write a 1MB file, with no linefeeds, and a big buffer.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        mblob = (1024 * 1024 * 'x')
-        try:
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
-            f.write(mblob)
-            f.close()
-
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
-        finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
-    
-    def test_J_big_file_renegotiate(self):
-        """
-        write a 1MB file, forcing key renegotiation in the middle.
-        """
-        global g_big_file_test
-        if not g_big_file_test:
-            return
-        t = sftp.sock.get_transport()
-        t.packetizer.REKEY_BYTES = 512 * 1024
-        k32blob = (32 * 1024 * 'x')
-        try:
-            f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
-            for i in xrange(32):
-                f.write(k32blob)
-            f.close()
-            
-            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
-            self.assertNotEquals(t.H, t.session_id)
-        finally:
-            sftp.remove('%s/hongry.txt' % FOLDER)
-            t.packetizer.REKEY_BYTES = pow(2, 30)
-
-    def test_K_realpath(self):
         """
         test that realpath is returning something non-empty and not an
         error.
@@ -636,7 +466,7 @@ class SFTPTest (unittest.TestCase):
         self.assert_(len(f) > 0)
         self.assertEquals(os.path.join(pwd, FOLDER), f)
 
-    def test_L_mkdir(self):
+    def test_F_mkdir(self):
         """
         verify that mkdir/rmdir work.
         """
@@ -659,7 +489,7 @@ class SFTPTest (unittest.TestCase):
         except IOError:
             pass
     
-    def test_M_chdir(self):
+    def test_G_chdir(self):
         """
         verify that chdir/getcwd work.
         """
@@ -696,7 +526,7 @@ class SFTPTest (unittest.TestCase):
             except:
                 pass
 
-    def test_N_get_put(self):
+    def test_H_get_put(self):
         """
         verify that get/put work.
         """
@@ -725,7 +555,7 @@ class SFTPTest (unittest.TestCase):
         os.unlink(localname)
         sftp.unlink(FOLDER + '/bunny.txt')
 
-    def test_O_check(self):
+    def test_I_check(self):
         """
         verify that file.check() works against our own server.
         (it's an sftp extension that we support, and may be the only ones who
@@ -747,7 +577,7 @@ class SFTPTest (unittest.TestCase):
         finally:
             sftp.unlink(FOLDER + '/kitty.txt')
 
-    def test_P_x_flag(self):
+    def test_J_x_flag(self):
         """
         verify that the 'x' flag works when opening a file.
         """
@@ -763,7 +593,7 @@ class SFTPTest (unittest.TestCase):
         finally:
             sftp.unlink(FOLDER + '/unusual.txt')
     
-    def test_Q_utf8(self):
+    def test_K_utf8(self):
         """
         verify that unicode strings are encoded into utf8 correctly.
         """

tests/test_sftp_big.py

@@ -0,0 +1,267 @@
+# Copyright (C) 2003-2005 Robey Pointer <robey@lag.net>
+#
+# This file is part of paramiko.
+#
+# Paramiko is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+
+"""
+some unit tests to make sure sftp works well with large files.
+
+a real actual sftp server is contacted, and a new folder is created there to
+do test file operations in (so no existing files will be harmed).
+"""
+
+import logging
+import os
+import random
+import struct
+import sys
+import threading
+import time
+import unittest
+
+import paramiko
+from stub_sftp import StubServer, StubSFTPServer
+from loop import LoopSocket
+from test_sftp import get_sftp
+
+FOLDER = os.environ.get('TEST_FOLDER', 'temp-testing000')
+
+
+class BigSFTPTest (unittest.TestCase):
+
+    def setUp(self):
+        global FOLDER
+        sftp = get_sftp()
+        for i in xrange(1000):
+            FOLDER = FOLDER[:-3] + '%03d' % i
+            try:
+                sftp.mkdir(FOLDER)
+                break
+            except (IOError, OSError):
+                pass
+
+    def tearDown(self):
+        sftp = get_sftp()
+        sftp.rmdir(FOLDER)
+
+    def test_1_lots_of_files(self):
+        """
+        create a bunch of files over the same session.
+        """
+        sftp = get_sftp()
+        numfiles = 100
+        try:
+            for i in range(numfiles):
+                f = sftp.open('%s/file%d.txt' % (FOLDER, i), 'w', 1)
+                f.write('this is file #%d.\n' % i)
+                f.close()
+                sftp.chmod('%s/file%d.txt' % (FOLDER, i), 0660)
+
+            # now make sure every file is there, by creating a list of filenmes
+            # and reading them in random order.
+            numlist = range(numfiles)
+            while len(numlist) > 0:
+                r = numlist[random.randint(0, len(numlist) - 1)]
+                f = sftp.open('%s/file%d.txt' % (FOLDER, r))
+                self.assertEqual(f.readline(), 'this is file #%d.\n' % r)
+                f.close()
+                numlist.remove(r)
+        finally:
+            for i in range(numfiles):
+                try:
+                    sftp.remove('%s/file%d.txt' % (FOLDER, i))
+                except:
+                    pass
+
+    def test_2_big_file(self):
+        """
+        write a 1MB file with no buffering.
+        """
+        sftp = get_sftp()
+        kblob = (1024 * 'x')
+        start = time.time()
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
+            for n in range(1024):
+                f.write(kblob)
+                if n % 128 == 0:
+                    sys.stderr.write('.')
+            f.close()
+            sys.stderr.write(' ')
+
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            end = time.time()
+            sys.stderr.write('%ds ' % round(end - start))
+            
+            start = time.time()
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
+            for n in range(1024):
+                data = f.read(1024)
+                self.assertEqual(data, kblob)
+            f.close()
+
+            end = time.time()
+            sys.stderr.write('%ds ' % round(end - start))
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+
+    def test_3_big_file_pipelined(self):
+        """
+        write a 1MB file, with no linefeeds, using pipelining.
+        """
+        sftp = get_sftp()
+        kblob = ''.join([struct.pack('>H', n) for n in xrange(512)])
+        start = time.time()
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
+            f.set_pipelined(True)
+            for n in range(1024):
+                f.write(kblob)
+                if n % 128 == 0:
+                    sys.stderr.write('.')
+            f.close()
+            sys.stderr.write(' ')
+
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            end = time.time()
+            sys.stderr.write('%ds ' % round(end - start))
+            
+            start = time.time()
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
+            f.prefetch()
+
+            # read on odd boundaries to make sure the bytes aren't getting scrambled
+            n = 0
+            k2blob = kblob + kblob
+            chunk = 629
+            size = 1024 * 1024
+            while n < size:
+                if n + chunk > size:
+                    chunk = size - n
+                data = f.read(chunk)
+                offset = n % 1024
+                self.assertEqual(data, k2blob[offset:offset + chunk])
+                n += chunk
+            f.close()
+
+            end = time.time()
+            sys.stderr.write('%ds ' % round(end - start))
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+
+    def test_4_prefetch_seek(self):
+        sftp = get_sftp()
+        kblob = ''.join([struct.pack('>H', n) for n in xrange(512)])
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
+            f.set_pipelined(True)
+            for n in range(1024):
+                f.write(kblob)
+                if n % 128 == 0:
+                    sys.stderr.write('.')
+            f.close()
+            sys.stderr.write(' ')
+            
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            
+            k2blob = kblob + kblob
+            chunk = 793
+            for i in xrange(10):
+                f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
+                f.prefetch()
+                base_offset = (512 * 1024) + 17 * random.randint(1000, 2000)
+                offsets = [base_offset + j * chunk for j in xrange(100)]
+                # randomly seek around and read them out
+                for j in xrange(100):
+                    offset = offsets[random.randint(0, len(offsets) - 1)]
+                    offsets.remove(offset)
+                    f.seek(offset)
+                    data = f.read(chunk)
+                    n_offset = offset % 1024
+                    self.assertEqual(data, k2blob[n_offset:n_offset + chunk])
+                    offset += chunk
+                f.close()
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+                
+    def test_5_lots_of_prefetching(self):
+        """
+        prefetch a 1MB file a bunch of times, discarding the file object
+        without using it, to verify that paramiko doesn't get confused.
+        """
+        sftp = get_sftp()
+        kblob = (1024 * 'x')
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w')
+            f.set_pipelined(True)
+            for n in range(1024):
+                f.write(kblob)
+                if n % 128 == 0:
+                    sys.stderr.write('.')
+            f.close()
+            sys.stderr.write(' ')
+
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+
+            for i in range(10):
+                f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
+                f.prefetch()
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'r')
+            f.prefetch()
+            for n in range(1024):
+                data = f.read(1024)
+                self.assertEqual(data, kblob)
+                if n % 128 == 0:
+                    sys.stderr.write('.')
+            f.close()
+            sys.stderr.write(' ')
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+        
+    def test_6_big_file_big_buffer(self):
+        """
+        write a 1MB file, with no linefeeds, and a big buffer.
+        """
+        sftp = get_sftp()
+        mblob = (1024 * 1024 * 'x')
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
+            f.write(mblob)
+            f.close()
+
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+    
+    def test_7_big_file_renegotiate(self):
+        """
+        write a 1MB file, forcing key renegotiation in the middle.
+        """
+        sftp = get_sftp()
+        t = sftp.sock.get_transport()
+        t.packetizer.REKEY_BYTES = 512 * 1024
+        k32blob = (32 * 1024 * 'x')
+        try:
+            f = sftp.open('%s/hongry.txt' % FOLDER, 'w', 128 * 1024)
+            for i in xrange(32):
+                f.write(k32blob)
+            f.close()
+            
+            self.assertEqual(sftp.stat('%s/hongry.txt' % FOLDER).st_size, 1024 * 1024)
+            self.assertNotEquals(t.H, t.session_id)
+        finally:
+            sftp.remove('%s/hongry.txt' % FOLDER)
+            t.packetizer.REKEY_BYTES = pow(2, 30)