Merge with master

.gitignore

@@ -3,3 +3,4 @@ build/
 dist/
 paramiko.egg-info/
 test.log
+docs/

.travis.yml

@@ -9,6 +9,6 @@ install:
 script: python test.py
 notifications:
   irc:
-    channels: "irc.freenode.org#fabric"
+    channels: "irc.freenode.org#paramiko"
     on_success: change
     on_failure: change

Makefile

@@ -1,7 +1,7 @@
 release: docs
 	python setup.py sdist register upload
 
-docs:
+docs: paramiko/*
 	epydoc --no-private -o docs/ paramiko
 
 clean:

NEWS

@@ -12,12 +12,36 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/.
 Releases
 ========
 
-v1.9.0 (DD MM YYYY)
+v1.10.0 (DD MM YYYY)
--------------------
+--------------------
 
+* #71: Add `SFTPClient.putfo` and `.getfo` methods to allow direct
+  uploading/downloading of file-like objects. Thanks to Eric Buehl for the
+  patch.
+* #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting
+  of the command's internal channel object's timeout. Thanks to Cernov Vladimir
+  for the patch.
+* #94: Remove duplication of SSH port constant. Thanks to Olle Lundberg for the
+  catch.
+* #80: Expose the internal "is closed" property of the file transfer class
+  `BufferedFile` as `.closed`, better conforming to Python's file interface.
+  Thanks to `@smunaut` and James Hiscock for catch & patch.
 
-v1.8.1 (DD MM YYYY)
+v1.9.0 (6th Nov 2012)
--------------------
+---------------------
+
+* #97 (with a little #93): Improve config parsing of `ProxyCommand` directives
+  and provide a wrapper class to allow subprocess-driven proxy commands to be
+  used as `sock=` arguments for `SSHClient.connect`.
+* #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter
+  overriding creation of an internal, implicit socket object.
+* Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven
+  Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi, Olle
+  Lundberg, and Github user `@acrish` for the various and sundry patches
+  leading to the above changes.
+
+v1.8.1 (6th Nov 2012)
+---------------------
 
 * #90: Ensure that callbacks handed to `SFTPClient.get()` always fire at least
   once, even for zero-length files downloaded. Thanks to Github user `@enB` for
@@ -32,6 +56,8 @@ v1.8.1 (DD MM YYYY)
 v1.8.0 (3rd Oct 2012)
 ---------------------
 
+* #17 ('ssh' 28): Fix spurious `NoneType has no attribute 'error'` and similar
+  exceptions that crop up on interpreter exit.
 * 'ssh' 32: Raise a more useful error explaining which `known_hosts` key line was
   problematic, when encountering `binascii` issues decoding known host keys.
   Thanks to `@thomasvs` for catch & patch.

paramiko/__init__.py

@@ -55,7 +55,7 @@ if sys.version_info < (2, 2):
 
 
 __author__ = "Jeff Forcier <jeff@bitprophet.org>"
-__version__ = "1.8.0"
+__version__ = "1.10.0"
 __license__ = "GNU Lesser General Public License (LGPL)"
 
 
@@ -65,7 +65,7 @@ from auth_handler import AuthHandler
 from channel import Channel, ChannelFile
 from ssh_exception import SSHException, PasswordRequiredException, \
     BadAuthenticationType, ChannelException, BadHostKeyException, \
-    AuthenticationException
+    AuthenticationException, ProxyCommandFailure
 from server import ServerInterface, SubsystemHandler, InteractiveQuery
 from rsakey import RSAKey
 from dsskey import DSSKey
@@ -83,6 +83,7 @@ from agent import Agent, AgentKey
 from pkey import PKey
 from hostkeys import HostKeys
 from config import SSHConfig
+from proxy import ProxyCommand
 
 # fix module names for epydoc
 for c in locals().values():
@@ -119,6 +120,8 @@ __all__ = [ 'Transport',
             'BadAuthenticationType',
             'ChannelException',
             'BadHostKeyException',
+            'ProxyCommand',
+            'ProxyCommandFailure',
             'SFTP',
             'SFTPFile',
             'SFTPHandle',

paramiko/client.py

@@ -28,6 +28,7 @@ import warnings
 
 from paramiko.agent import Agent
 from paramiko.common import *
+from paramiko.config import SSH_PORT
 from paramiko.dsskey import DSSKey
 from paramiko.hostkeys import HostKeys
 from paramiko.resource import ResourceManager
@@ -37,8 +38,6 @@ from paramiko.transport import Transport
 from paramiko.util import retry_on_signal
 
 
-SSH_PORT = 22
-
 class MissingHostKeyPolicy (object):
     """
     Interface for defining the policy that L{SSHClient} should use when the
@@ -229,7 +228,7 @@ class SSHClient (object):
 
     def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None,
                 key_filename=None, timeout=None, allow_agent=True, look_for_keys=True,
-                compress=False):
+                compress=False, sock=None):
         """
         Connect to an SSH server and authenticate to it.  The server's host key
         is checked against the system host keys (see L{load_system_host_keys})
@@ -272,6 +271,9 @@ class SSHClient (object):
         @type look_for_keys: bool
         @param compress: set to True to turn on compression
         @type compress: bool
+        @param sock: an open socket or socket-like object (such as a
+            L{Channel}) to use for communication to the target host
+        @type sock: socket
 
         @raise BadHostKeyException: if the server's host key could not be
             verified
@@ -280,21 +282,23 @@ class SSHClient (object):
             establishing an SSH session
         @raise socket.error: if a socket error occurred while connecting
         """
-        for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
+        if not sock:
-            if socktype == socket.SOCK_STREAM:
+            for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM):
-                af = family
+                if socktype == socket.SOCK_STREAM:
-                addr = sockaddr
+                    af = family
-                break
+                    addr = sockaddr
-        else:
+                    break
-            # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :(
+            else:
-            af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
+                # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :(
-        sock = socket.socket(af, socket.SOCK_STREAM)
+                af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
-        if timeout is not None:
+            sock = socket.socket(af, socket.SOCK_STREAM)
-            try:
+            if timeout is not None:
-                sock.settimeout(timeout)
+                try:
-            except:
+                    sock.settimeout(timeout)
-                pass
+                except:
-        retry_on_signal(lambda: sock.connect(addr))
+                    pass
+            retry_on_signal(lambda: sock.connect(addr))
+
         t = self._transport = Transport(sock)
         t.use_compression(compress=compress)
         if self._log_channel is not None:
@@ -345,7 +349,7 @@ class SSHClient (object):
             self._agent.close()
             self._agent = None
 
-    def exec_command(self, command, bufsize=-1):
+    def exec_command(self, command, bufsize=-1, timeout=None):
         """
         Execute a command on the SSH server.  A new L{Channel} is opened and
         the requested command is executed.  The command's input and output
@@ -356,12 +360,15 @@ class SSHClient (object):
         @type command: str
         @param bufsize: interpreted the same way as by the built-in C{file()} function in python
         @type bufsize: int
+        @param timeout: set command's channel timeout. See L{Channel.settimeout}.settimeout
+        @type timeout: int
         @return: the stdin, stdout, and stderr of the executing command
         @rtype: tuple(L{ChannelFile}, L{ChannelFile}, L{ChannelFile})
 
         @raise SSHException: if the server fails to execute the command
         """
         chan = self._transport.open_session()
+        chan.settimeout(timeout)
         chan.exec_command(command)
         stdin = chan.makefile('wb', bufsize)
         stdout = chan.makefile('rb', bufsize)

paramiko/config.py

@@ -22,9 +22,12 @@ L{SSHConfig}.
 
 import fnmatch
 import os
+import re
 import socket
 
-SSH_PORT=22
+SSH_PORT = 22
+proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I)
+
 
 class SSHConfig (object):
     """
@@ -56,8 +59,13 @@ class SSHConfig (object):
             if (line == '') or (line[0] == '#'):
                 continue
             if '=' in line:
-                key, value = line.split('=', 1)
+                # Ensure ProxyCommand gets properly split
-                key = key.strip().lower()
+                if line.lower().strip().startswith('proxycommand'):
+                    match = proxy_re.match(line)
+                    key, value = match.group(1).lower(), match.group(2)
+                else:
+                    key, value = line.split('=', 1)
+                    key = key.strip().lower()
             else:
                 # find first whitespace, and split there
                 i = 0
@@ -149,26 +157,30 @@ class SSHConfig (object):
         host = socket.gethostname().split('.')[0]
         fqdn = socket.getfqdn()
         homedir = os.path.expanduser('~')
-        replacements = {'controlpath' :
+        replacements = {
-                [
+            'controlpath': [
-                    ('%h', config['hostname']),
+                ('%h', config['hostname']),
-                    ('%l', fqdn),
+                ('%l', fqdn),
-                    ('%L', host),
+                ('%L', host),
-                    ('%n', hostname),
+                ('%n', hostname),
-                    ('%p', port),
+                ('%p', port),
-                    ('%r', remoteuser),
+                ('%r', remoteuser),
-                    ('%u', user)
+                ('%u', user)
-                ],
+            ],
-                'identityfile' :
+            'identityfile': [
-                [
+                ('~', homedir),
-                    ('~', homedir),
+                ('%d', homedir),
-                    ('%d', homedir),
+                ('%h', config['hostname']),
-                    ('%h', config['hostname']),
+                ('%l', fqdn),
-                    ('%l', fqdn),
+                ('%u', user),
-                    ('%u', user),
+                ('%r', remoteuser)
-                    ('%r', remoteuser)
+            ],
-                ]
+            'proxycommand': [
-                }
+                ('%h', config['hostname']),
+                ('%p', port),
+                ('%r', remoteuser),
+            ],
+        }
         for k in config:
             if k in replacements:
                 for find, replace in replacements[k]:

paramiko/file.py

@@ -354,6 +354,10 @@ class BufferedFile (object):
         """
         return self
 
+    @property
+    def closed(self):
+        return self._closed
+
 
     ###  overrides...
 

paramiko/packet.py

@@ -29,7 +29,7 @@ import time
 
 from paramiko.common import *
 from paramiko import util
-from paramiko.ssh_exception import SSHException
+from paramiko.ssh_exception import SSHException, ProxyCommandFailure
 from paramiko.message import Message
 
 
@@ -254,6 +254,8 @@ class Packetizer (object):
                     retry_write = True
                 else:
                     n = -1
+            except ProxyCommandFailure:
+                raise # so it doesn't get swallowed by the below catchall
             except Exception:
                 # could be: (32, 'Broken pipe')
                 n = -1

paramiko/proxy.py

@@ -0,0 +1,91 @@
+# Copyright (C) 2012  Yipit, Inc <coders@yipit.com>
+#
+# This file is part of paramiko.
+#
+# Paramiko is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+
+"""
+L{ProxyCommand}.
+"""
+
+import os
+from shlex import split as shlsplit
+import signal
+from subprocess import Popen, PIPE
+
+from paramiko.ssh_exception import ProxyCommandFailure
+
+
+class ProxyCommand(object):
+    """
+    Wraps a subprocess running ProxyCommand-driven programs.
+
+    This class implements a the socket-like interface needed by the
+    L{Transport} and L{Packetizer} classes. Using this class instead of a
+    regular socket makes it possible to talk with a Popen'd command that will
+    proxy traffic between the client and a server hosted in another machine.
+    """
+    def __init__(self, command_line):
+        """
+        Create a new CommandProxy instance. The instance created by this
+        class can be passed as an argument to the L{Transport} class.
+
+        @param command_line: the command that should be executed and
+            used as the proxy.
+        @type command_line: str
+        """
+        self.cmd = shlsplit(command_line)
+        self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+
+    def send(self, content):
+        """
+        Write the content received from the SSH client to the standard
+        input of the forked command.
+
+        @param content: string to be sent to the forked command
+        @type content: str
+        """
+        try:
+            self.process.stdin.write(content)
+        except IOError, e:
+            # There was a problem with the child process. It probably
+            # died and we can't proceed. The best option here is to
+            # raise an exception informing the user that the informed
+            # ProxyCommand is not working.
+            raise BadProxyCommand(' '.join(self.cmd), e.strerror)
+        return len(content)
+
+    def recv(self, size):
+        """
+        Read from the standard output of the forked program.
+
+        @param size: how many chars should be read
+        @type size: int
+
+        @return: the length of the read content
+        @rtype: int
+        """
+        try:
+            return os.read(self.process.stdout.fileno(), size)
+        except IOError, e:
+            raise BadProxyCommand(' '.join(self.cmd), e.strerror)
+
+    def close(self):
+        os.kill(self.process.pid, signal.SIGTERM)
+
+    def settimeout(self, timeout):
+        # Timeouts are meaningless for this implementation, but are part of the
+        # spec, so must be present.
+        pass

paramiko/sftp_client.py

@@ -533,6 +533,56 @@ class SFTPClient (BaseSFTP):
         """
         return self._cwd
 
+    def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True):
+        """
+        Copy the contents of an open file object (C{fl}) to the SFTP server as
+        C{remotepath}. Any exception raised by operations will be passed through.
+
+        The SFTP operations use pipelining for speed.
+
+        @param fl: opened file or file-like object to copy
+        @type localpath: object
+        @param remotepath: the destination path on the SFTP server
+        @type remotepath: str
+        @param file_size: optional size parameter passed to callback. If none is
+            specified, size defaults to 0
+        @type file_size: int
+        @param callback: optional callback function that accepts the bytes
+            transferred so far and the total bytes to be transferred
+            (since 1.7.4)
+        @type callback: function(int, int)
+        @param confirm: whether to do a stat() on the file afterwards to
+            confirm the file size (since 1.7.7)
+        @type confirm: bool
+
+        @return: an object containing attributes about the given file
+            (since 1.7.4)
+        @rtype: SFTPAttributes
+
+        @since: 1.4
+        """
+        fr = self.file(remotepath, 'wb')
+        fr.set_pipelined(True)
+        size = 0
+        try:
+            while True:
+                data = fl.read(32768)
+                fr.write(data)
+                size += len(data)
+                if callback is not None:
+                    callback(size, file_size)
+                if len(data) == 0:
+                    break
+        finally:
+            fr.close()
+        if confirm and file_size:
+            s = self.stat(remotepath)
+            if s.st_size != size:
+                raise IOError('size mismatch in put!  %d != %d' % (s.st_size, size))
+        else:
+            s = SFTPAttributes()
+        return s
+
     def put(self, localpath, remotepath, callback=None, confirm=True):
         """
         Copy a local file (C{localpath}) to the SFTP server as C{remotepath}.
@@ -562,29 +612,46 @@ class SFTPClient (BaseSFTP):
         file_size = os.stat(localpath).st_size
         fl = file(localpath, 'rb')
         try:
-            fr = self.file(remotepath, 'wb')
+            return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm)
-            fr.set_pipelined(True)
-            size = 0
-            try:
-                while True:
-                    data = fl.read(32768)
-                    if len(data) == 0:
-                        break
-                    fr.write(data)
-                    size += len(data)
-                    if callback is not None:
-                        callback(size, file_size)
-            finally:
-                fr.close()
         finally:
             fl.close()
-        if confirm:
+
-            s = self.stat(remotepath)
+    def getfo(self, remotepath, fl, callback=None):
-            if s.st_size != size:
+        """
-                raise IOError('size mismatch in put!  %d != %d' % (s.st_size, size))
+        Copy a remote file (C{remotepath}) from the SFTP server and write to
-        else:
+        an open file or file-like object, C{fl}.  Any exception raised by
-            s = SFTPAttributes()
+        operations will be passed through.  This method is primarily provided
-        return s
+        as a convenience.
+
+        @param remotepath: opened file or file-like object to copy to
+        @type remotepath: object
+        @param fl: the destination path on the local host or open file
+            object
+        @type localpath: str
+        @param callback: optional callback function that accepts the bytes
+            transferred so far and the total bytes to be transferred
+            (since 1.7.4)
+        @type callback: function(int, int)
+        @return: the number of bytes written to the opened file object
+
+        @since: 1.4
+        """
+        fr = self.file(remotepath, 'rb')
+        file_size = self.stat(remotepath).st_size
+        fr.prefetch()
+        try:
+            size = 0
+            while True:
+                data = fr.read(32768)
+                fl.write(data)
+                size += len(data)
+                if callback is not None:
+                    callback(size, file_size)
+                if len(data) == 0:
+                    break
+        finally:
+            fr.close()
+        return size
 
     def get(self, remotepath, localpath, callback=None):
         """
@@ -603,25 +670,12 @@ class SFTPClient (BaseSFTP):
 
         @since: 1.4
         """
-        fr = self.file(remotepath, 'rb')
         file_size = self.stat(remotepath).st_size
-        fr.prefetch()
+        fl = file(localpath, 'wb')
         try:
-            fl = file(localpath, 'wb')
+            size = self.getfo(remotepath, fl, callback)
-            try:
-                size = 0
-                while True:
-                    data = fr.read(32768)
-                    fl.write(data)
-                    size += len(data)
-                    if callback is not None:
-                        callback(size, file_size)
-                    if len(data) == 0:
-                        break
-            finally:
-                fl.close()
         finally:
-            fr.close()
+            fl.close()
         s = os.stat(localpath)
         if s.st_size != size:
             raise IOError('size mismatch in get!  %d != %d' % (s.st_size, size))

paramiko/ssh_exception.py

@@ -113,3 +113,20 @@ class BadHostKeyException (SSHException):
         self.key = got_key
         self.expected_key = expected_key
 
+
+class ProxyCommandFailure (SSHException):
+    """
+    The "ProxyCommand" found in the .ssh/config file returned an error.
+
+    @ivar command: The command line that is generating this exception.
+    @type command: str
+    @ivar error: The error captured from the proxy command output.
+    @type error: str
+    """
+    def __init__(self, command, error):
+        SSHException.__init__(self,
+            '"ProxyCommand (%s)" returned non-zero exit status: %s' % (
+                command, error
+            )
+        )
+        self.error = error

paramiko/transport.py

@@ -44,7 +44,8 @@ from paramiko.primes import ModulusPack
 from paramiko.rsakey import RSAKey
 from paramiko.server import ServerInterface
 from paramiko.sftp_client import SFTPClient
-from paramiko.ssh_exception import SSHException, BadAuthenticationType, ChannelException
+from paramiko.ssh_exception import (SSHException, BadAuthenticationType,
+    ChannelException, ProxyCommandFailure)
 from paramiko.util import retry_on_signal
 
 from Crypto import Random
@@ -1674,6 +1675,8 @@ class Transport (threading.Thread):
                 timeout = 2
             try:
                 buf = self.packetizer.readline(timeout)
+            except ProxyCommandFailure:
+                raise
             except Exception, x:
                 raise SSHException('Error reading SSH protocol banner' + str(x))
             if buf[:4] == 'SSH-':

setup.py

@@ -52,7 +52,7 @@ if sys.platform == 'darwin':
 
 
 setup(name = "paramiko",
-      version = "1.8.0",
+      version = "1.10.0",
       description = "SSH2 protocol library",
       author = "Jeff Forcier",
       author_email = "jeff@bitprophet.org",

tests/test_util.py

@@ -27,6 +27,7 @@ import os
 import unittest
 from Crypto.Hash import SHA
 import paramiko.util
+from paramiko.util import lookup_ssh_host_config as host_config
 
 from util import ParamikoTest
 
@@ -151,7 +152,7 @@ class UtilTest(ParamikoTest):
         x = rng.read(32)
         self.assertEquals(len(x), 32)
         
-    def test_7_host_config_expose_issue_33(self):
+    def test_7_host_config_expose_ssh_issue_33(self):
         test_config_file = """
 Host www13.*
     Port 22
@@ -194,3 +195,48 @@ Host *
             raise AssertionError('foo')
         self.assertRaises(AssertionError,
                           lambda: paramiko.util.retry_on_signal(raises_other_exception))
+
+    def test_9_proxycommand_config_equals_parsing(self):
+        """
+        ProxyCommand should not split on equals signs within the value.
+        """
+        conf = """
+Host space-delimited
+    ProxyCommand foo bar=biz baz
+
+Host equals-delimited
+    ProxyCommand=foo bar=biz baz
+"""
+        f = cStringIO.StringIO(conf)
+        config = paramiko.util.parse_ssh_config(f)
+        for host in ('space-delimited', 'equals-delimited'):
+            self.assertEquals(
+                host_config(host, config)['proxycommand'],
+                'foo bar=biz baz'
+            )
+
+    def test_10_proxycommand_interpolation(self):
+        """
+        ProxyCommand should perform interpolation on the value
+        """
+        config = paramiko.util.parse_ssh_config(cStringIO.StringIO("""
+Host *
+    Port 25
+    ProxyCommand host %h port %p
+
+Host specific
+    Port 37
+    ProxyCommand host %h port %p lol
+
+Host portonly
+    Port 155
+"""))
+        for host, val in (
+            ('foo.com', "host foo.com port 25"),
+            ('specific', "host specific port 37 lol"),
+            ('portonly', "host portonly port 155"),
+        ):
+            self.assertEquals(
+                host_config(host, config)['proxycommand'],
+                val
+            )