From c78a5856e88a67344cb47e170ec8e237df1fc2c6 Mon Sep 17 00:00:00 2001 From: James Hiscock Date: Wed, 17 Oct 2012 14:25:22 +1200 Subject: [PATCH 01/50] Update paramiko/file.py Added a closed property as an alternative accessor to BufferedFile's _closed property. --- paramiko/file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paramiko/file.py b/paramiko/file.py index d4aec8e..7e2904e 100644 --- a/paramiko/file.py +++ b/paramiko/file.py @@ -354,6 +354,10 @@ class BufferedFile (object): """ return self + @property + def closed(self): + return self._closed + ### overrides... From 23f3099b6f7808934400b96b707e122ed2dd302c Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:44:34 +0300 Subject: [PATCH 02/50] Make send() and recv() fail when channel is closed ``sendall()`` was checking if the channel has been closed, and failed accordingly, but ``send()`` and ``recv()`` did not. This meant that ``chan.send("foo")`` when the channel was already closed, just blocked forever. --- paramiko/channel.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 534f8d7..35991de 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -605,6 +605,10 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -655,6 +659,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -708,6 +716,10 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -745,6 +757,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -783,9 +799,6 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 668870aa83e095c81371ffce8d69404ac29e536f Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:46:28 +0300 Subject: [PATCH 03/50] Forgot to import errno --- paramiko/channel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paramiko/channel.py b/paramiko/channel.py index 35991de..9a1bb87 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,6 +25,7 @@ import sys import time import threading import socket +import errno import os from paramiko.common import * From fd5e29b5a8aff1fe9f11f9f7bee5f0eb85ae569a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 11:22:07 -0800 Subject: [PATCH 04/50] Somehow missed a pretty important change in the changelog --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index ed9fd00..30bca13 100644 --- a/NEWS +++ b/NEWS @@ -32,6 +32,8 @@ v1.8.1 (DD MM YYYY) v1.8.0 (3rd Oct 2012) --------------------- +* #17 ('ssh' 28): Fix spurious `NoneType has no attribute 'error'` and similar + exceptions that crop up on interpreter exit. * 'ssh' 32: Raise a more useful error explaining which `known_hosts` key line was problematic, when encountering `binascii` issues decoding known host keys. Thanks to `@thomasvs` for catch & patch. From 31ea4f0734a086f2345aaea57fd6fc1c3ea4a87e Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Mon, 15 Oct 2012 04:09:33 -0700 Subject: [PATCH 05/50] SSHClient: add 'sock' parameter to connect() for tunneling Re #77 This parameter, if set, can be used to make Paramiko wrap an existing socket connected to a remote SSH server. For instance, you could set up another SSHClient directly connected to a "gateway" host, and then create a direct-tcpip tunnel to a "target" host directly accessible from the gateway's perspective (e.g. think of trying to establish an SSH connection to hosts behind a NAT). The gateway host would then establish a TCP connection to the target host directly, and a channel is exposed on the client side. This channel could be wrapped by an SSHClient class using the connect() function, avoiding the need to establish a new TCP connnection. This effectively allows you to create tunneled SSH connections. Based on work by Oskari Saarenmaa , in Paramiko pull request #39. Signed-off-by: Steven Noonan --- paramiko/client.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/paramiko/client.py b/paramiko/client.py index 0f40897..0a9ced7 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -229,7 +229,7 @@ class SSHClient (object): def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None, key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, - compress=False): + compress=False, sock=None): """ Connect to an SSH server and authenticate to it. The server's host key is checked against the system host keys (see L{load_system_host_keys}) @@ -272,6 +272,9 @@ class SSHClient (object): @type look_for_keys: bool @param compress: set to True to turn on compression @type compress: bool + @param sock: an open socket or direct-tcpip channel from another + SSHClient class to use for communication with the target host. + @type channel: socket @raise BadHostKeyException: if the server's host key could not be verified @@ -280,21 +283,23 @@ class SSHClient (object): establishing an SSH session @raise socket.error: if a socket error occurred while connecting """ - for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if socktype == socket.SOCK_STREAM: - af = family - addr = sockaddr - break - else: - # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( - af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - sock = socket.socket(af, socket.SOCK_STREAM) - if timeout is not None: - try: - sock.settimeout(timeout) - except: - pass - retry_on_signal(lambda: sock.connect(addr)) + if not sock: + for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if socktype == socket.SOCK_STREAM: + af = family + addr = sockaddr + break + else: + # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( + af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + sock = socket.socket(af, socket.SOCK_STREAM) + if timeout is not None: + try: + sock.settimeout(timeout) + except: + pass + retry_on_signal(lambda: sock.connect(addr)) + t = self._transport = Transport(sock) t.use_compression(compress=compress) if self._log_channel is not None: From f9b7ce902ff4532aaa1ea51911a3f798b9c9d653 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 13:29:57 -0800 Subject: [PATCH 06/50] Tweak docstring re #77 --- paramiko/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paramiko/client.py b/paramiko/client.py index 0a9ced7..07560a3 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -272,9 +272,9 @@ class SSHClient (object): @type look_for_keys: bool @param compress: set to True to turn on compression @type compress: bool - @param sock: an open socket or direct-tcpip channel from another - SSHClient class to use for communication with the target host. - @type channel: socket + @param sock: an open socket or socket-like object (such as a + L{Channel}) to use for communication to the target host + @type sock: socket @raise BadHostKeyException: if the server's host key could not be verified From 31244a2ccbf9e2e8607236e859cbf38ddafd6060 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 13:30:20 -0800 Subject: [PATCH 07/50] Changelog re #77 --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 30bca13..f13cb9a 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ Releases v1.9.0 (DD MM YYYY) ------------------- +* #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter + overriding creation of an internal, implicit socket object. v1.8.1 (DD MM YYYY) ------------------- From 8e8dcea2953dfd888602f37330c9f26e3f151244 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 15:34:46 -0800 Subject: [PATCH 08/50] Add in big attribution big in prep for having ProxyCommand done --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index f13cb9a..02ec142 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,10 @@ v1.9.0 (DD MM YYYY) * #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter overriding creation of an internal, implicit socket object. +* Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven + Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi and Github + user `@acrish` for the various and sundry patches leading to the above + changes. v1.8.1 (DD MM YYYY) ------------------- From fb5d245b3148250bb3d7d6b46c2ec2b7914982a7 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:04:25 -0800 Subject: [PATCH 09/50] More attributions --- NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 02ec142..83a524c 100644 --- a/NEWS +++ b/NEWS @@ -18,9 +18,9 @@ v1.9.0 (DD MM YYYY) * #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter overriding creation of an internal, implicit socket object. * Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven - Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi and Github - user `@acrish` for the various and sundry patches leading to the above - changes. + Noonan, Vladimir Lazarenko, Lincoln de Sousa, Valentino Volonghi, Olle + Lundberg, and Github user `@acrish` for the various and sundry patches + leading to the above changes. v1.8.1 (DD MM YYYY) ------------------- From 928c06274816669a94753b80f493a2e4e1b9357a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:04:45 -0800 Subject: [PATCH 10/50] Add failing test(s) re ProxyCommand config parsing --- tests/test_util.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index 458709b..83de004 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -27,6 +27,7 @@ import os import unittest from Crypto.Hash import SHA import paramiko.util +from paramiko.util import lookup_ssh_host_config as host_config from util import ParamikoTest @@ -151,7 +152,7 @@ class UtilTest(ParamikoTest): x = rng.read(32) self.assertEquals(len(x), 32) - def test_7_host_config_expose_issue_33(self): + def test_7_host_config_expose_ssh_issue_33(self): test_config_file = """ Host www13.* Port 22 @@ -194,3 +195,22 @@ Host * raise AssertionError('foo') self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) + + def test_9_proxycommand_config_parsing(self): + """ + ProxyCommand should not split on equals signs within the value. + """ + conf = """ +Host space-delimited + ProxyCommand foo bar=biz baz + +Host equals-delimited + ProxyCommand=foo bar=biz baz +""" + f = cStringIO.StringIO(conf) + config = paramiko.util.parse_ssh_config(f) + for host in ('space-delimited', 'equals-delimited'): + self.assertEquals( + host_config(host, config)['proxycommand'], + 'foo bar=biz baz' + ) From 270bb94a46d94298c2fbe3926578650e3f21ae5d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:18:48 -0800 Subject: [PATCH 11/50] Fix ProxyCommand equals splitting. Uses regex approach from @lndbrg --- paramiko/config.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 458d5dd..cf35cdc 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -22,9 +22,12 @@ L{SSHConfig}. import fnmatch import os +import re import socket SSH_PORT=22 +proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) + class SSHConfig (object): """ @@ -56,8 +59,13 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: - key, value = line.split('=', 1) - key = key.strip().lower() + # Ensure ProxyCommand gets properly split + if line.lower().strip().startswith('proxycommand'): + match = proxy_re.match(line) + key, value = match.group(1).lower(), match.group(2) + else: + key, value = line.split('=', 1) + key = key.strip().lower() else: # find first whitespace, and split there i = 0 From 7cd2f2715bb7bca536a1b8d85a2d62f64831bd3a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:25:03 -0800 Subject: [PATCH 12/50] Initial port of ProxyCommand class from @clarete --- paramiko/proxy.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 paramiko/proxy.py diff --git a/paramiko/proxy.py b/paramiko/proxy.py new file mode 100644 index 0000000..537e17b --- /dev/null +++ b/paramiko/proxy.py @@ -0,0 +1,86 @@ +# Copyright (C) 2012 Yipit, Inc +# +# This file is part of ssh. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# 'ssh' is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with 'ssh'; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA. + +import os +from shlex import split as shlsplit +from subprocess import Popen, PIPE + +from ssh.ssh_exception import BadProxyCommand + + +class ProxyCommand(object): + """ + A proxy based on the program informed in the ProxyCommand config. + + This class implements a the interface needed by both L{Transport} + and L{Packetizer} classes. Using this class instead of a regular + socket makes it possible to talk with a popened command that will + proxy all the conversation between the client and a server hosted in + another machine. + """ + + def __init__(self, command_line): + """ + Create a new CommandProxy instance. The instance created by this + class should be passed as argument to the L{Transport} class. + + @param command_line: the command that should be executed and + used as the proxy. + @type command_line: str + """ + self.cmd = shlsplit(command_line) + self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + + def send(self, content): + """ + Write the content received from the SSH client to the standard + input of the forked command. + + @param content: string to be sent to the forked command + @type content: str + """ + try: + self.process.stdin.write(content) + except IOError, exc: + # There was a problem with the child process. It probably + # died and we can't proceed. The best option here is to + # raise an exception informing the user that the informed + # ProxyCommand is not working. + raise BadProxyCommand(' '.join(self.cmd), exc.strerror) + return len(content) + + def recv(self, size): + """ + Read from the standard output of the forked program + + @param size: how many chars should be read + @type size: int + + @return: the length of the readed content + @rtype: int + """ + try: + return os.read(self.process.stdout.fileno(), size) + except IOError, exc: + raise BadProxyCommand(' '.join(self.cmd), exc.strerror) + + def close(self): + self.process.terminate() + + def settimeout(self, timeout): + pass From 27271fa455228f14b1e16b576f0c8c40f532c227 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:26:47 -0800 Subject: [PATCH 13/50] Post-import edits --- paramiko/proxy.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/paramiko/proxy.py b/paramiko/proxy.py index 537e17b..e91a11a 100644 --- a/paramiko/proxy.py +++ b/paramiko/proxy.py @@ -1,46 +1,48 @@ # Copyright (C) 2012 Yipit, Inc # -# This file is part of ssh. +# This file is part of paramiko. # # Paramiko is free software; you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation; either version 2.1 of the License, or (at your option) # any later version. # -# 'ssh' is distrubuted in the hope that it will be useful, but WITHOUT ANY +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more # details. # # You should have received a copy of the GNU Lesser General Public License -# along with 'ssh'; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA. +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{ProxyCommand}. +""" import os from shlex import split as shlsplit from subprocess import Popen, PIPE -from ssh.ssh_exception import BadProxyCommand +from ssh.ssh_exception import ProxyCommandFailure class ProxyCommand(object): """ - A proxy based on the program informed in the ProxyCommand config. + Wraps a subprocess running ProxyCommand-driven programs. - This class implements a the interface needed by both L{Transport} - and L{Packetizer} classes. Using this class instead of a regular - socket makes it possible to talk with a popened command that will - proxy all the conversation between the client and a server hosted in - another machine. + This class implements a the socket-like interface needed by the + L{Transport} and L{Packetizer} classes. Using this class instead of a + regular socket makes it possible to talk with a Popen'd command that will + proxy traffic between the client and a server hosted in another machine. """ - def __init__(self, command_line): """ Create a new CommandProxy instance. The instance created by this - class should be passed as argument to the L{Transport} class. + class can be passed as an argument to the L{Transport} class. @param command_line: the command that should be executed and - used as the proxy. + used as the proxy. @type command_line: str """ self.cmd = shlsplit(command_line) @@ -56,31 +58,33 @@ class ProxyCommand(object): """ try: self.process.stdin.write(content) - except IOError, exc: + except IOError, e: # There was a problem with the child process. It probably # died and we can't proceed. The best option here is to # raise an exception informing the user that the informed # ProxyCommand is not working. - raise BadProxyCommand(' '.join(self.cmd), exc.strerror) + raise BadProxyCommand(' '.join(self.cmd), e.strerror) return len(content) def recv(self, size): """ - Read from the standard output of the forked program + Read from the standard output of the forked program. @param size: how many chars should be read @type size: int - @return: the length of the readed content + @return: the length of the read content @rtype: int """ try: return os.read(self.process.stdout.fileno(), size) - except IOError, exc: - raise BadProxyCommand(' '.join(self.cmd), exc.strerror) + except IOError, e: + raise BadProxyCommand(' '.join(self.cmd), e.strerror) def close(self): self.process.terminate() def settimeout(self, timeout): + # Timeouts are meaningless for this implementation, but are part of the + # spec, so must be present. pass From 5d15467ad4f14b13e2d68f8f3b0c8d33dde4afeb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:29:23 -0800 Subject: [PATCH 14/50] Import BadProxyCommand --- paramiko/ssh_exception.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/paramiko/ssh_exception.py b/paramiko/ssh_exception.py index 68924d0..3fe2739 100644 --- a/paramiko/ssh_exception.py +++ b/paramiko/ssh_exception.py @@ -113,3 +113,18 @@ class BadHostKeyException (SSHException): self.key = got_key self.expected_key = expected_key + +class BadProxyCommand (SSHException): + """ + The "ProxyCommand" found in the .ssh/config file returned an error. + + @ivar command: The command line that is generating this exception + @type command: str + @ivar error: The error captured from the proxy command output + @type error: str + """ + def __init__(self, command, error): + SSHException.__init__(self, + '"ProxyCommand (%s)" returned non-zero exit status: %s' % ( + command, error)) + self.error = error From 394ab2699e254145e4904e1408036bb1683c14ca Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:29:32 -0800 Subject: [PATCH 15/50] Post-import edits --- paramiko/ssh_exception.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/paramiko/ssh_exception.py b/paramiko/ssh_exception.py index 3fe2739..f2406dc 100644 --- a/paramiko/ssh_exception.py +++ b/paramiko/ssh_exception.py @@ -114,17 +114,19 @@ class BadHostKeyException (SSHException): self.expected_key = expected_key -class BadProxyCommand (SSHException): +class ProxyCommandFailure (SSHException): """ The "ProxyCommand" found in the .ssh/config file returned an error. - @ivar command: The command line that is generating this exception + @ivar command: The command line that is generating this exception. @type command: str - @ivar error: The error captured from the proxy command output + @ivar error: The error captured from the proxy command output. @type error: str """ def __init__(self, command, error): SSHException.__init__(self, '"ProxyCommand (%s)" returned non-zero exit status: %s' % ( - command, error)) + command, error + ) + ) self.error = error From 0a276ac34bac3c9729992e3fdb3622d8916c5096 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:31:17 -0800 Subject: [PATCH 16/50] Bubble up ProxyCommandFailure in packetizer --- paramiko/packet.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paramiko/packet.py b/paramiko/packet.py index 9782061..5d918e2 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -29,7 +29,7 @@ import time from paramiko.common import * from paramiko import util -from paramiko.ssh_exception import SSHException +from paramiko.ssh_exception import SSHException, ProxyCommandFailure from paramiko.message import Message @@ -254,6 +254,8 @@ class Packetizer (object): retry_write = True else: n = -1 + except ProxyCommandFailure: + raise # so it doesn't get swallowed by the below catchall except Exception: # could be: (32, 'Broken pipe') n = -1 From 0981c25cd8ef651e6274feae8b3b6acd0869b680 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:34:49 -0800 Subject: [PATCH 17/50] Formatting --- paramiko/config.py | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index cf35cdc..8daee18 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -157,26 +157,25 @@ class SSHConfig (object): host = socket.gethostname().split('.')[0] fqdn = socket.getfqdn() homedir = os.path.expanduser('~') - replacements = {'controlpath' : - [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile' : - [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ] - } + replacements = { + 'controlpath': [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ] + } for k in config: if k in replacements: for find, replace in replacements[k]: From 191a5fc08cb8ce08917ad4e8739d7d5efbec2be2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:44:25 -0800 Subject: [PATCH 18/50] Implement (& test for) ProxyCommand interpolation. Forgot this earlier. --- paramiko/config.py | 7 ++++++- tests/test_util.py | 28 +++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 8daee18..2828d90 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -174,7 +174,12 @@ class SSHConfig (object): ('%l', fqdn), ('%u', user), ('%r', remoteuser) - ] + ], + 'proxycommand': [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser), + ], } for k in config: if k in replacements: diff --git a/tests/test_util.py b/tests/test_util.py index 83de004..093a215 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -196,7 +196,7 @@ Host * self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) - def test_9_proxycommand_config_parsing(self): + def test_9_proxycommand_config_equals_parsing(self): """ ProxyCommand should not split on equals signs within the value. """ @@ -214,3 +214,29 @@ Host equals-delimited host_config(host, config)['proxycommand'], 'foo bar=biz baz' ) + + def test_10_proxycommand_interpolation(self): + """ + ProxyCommand should perform interpolation on the value + """ + config = paramiko.util.parse_ssh_config(cStringIO.StringIO(""" +Host * + Port 25 + ProxyCommand host %h port %p + +Host specific + Port 37 + ProxyCommand host %h port %p lol + +Host portonly + Port 155 +""")) + for host, val in ( + ('foo.com', "host foo.com port 25"), + ('specific', "host specific port 37 lol"), + ('portonly', "host portonly port 155"), + ): + self.assertEquals( + host_config(host, config)['proxycommand'], + val + ) From fd392d6b2090d227937add287a2ad0f8eb5cba59 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:47:33 -0800 Subject: [PATCH 19/50] One more patch from @clarete's work --- paramiko/transport.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/paramiko/transport.py b/paramiko/transport.py index 04680a9..c801031 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -44,7 +44,8 @@ from paramiko.primes import ModulusPack from paramiko.rsakey import RSAKey from paramiko.server import ServerInterface from paramiko.sftp_client import SFTPClient -from paramiko.ssh_exception import SSHException, BadAuthenticationType, ChannelException +from paramiko.ssh_exception import (SSHException, BadAuthenticationType, + ChannelException, ProxyCommandFailure) from paramiko.util import retry_on_signal from Crypto import Random @@ -1674,6 +1675,8 @@ class Transport (threading.Thread): timeout = 2 try: buf = self.packetizer.readline(timeout) + except ProxyCommandFailure: + raise except Exception, x: raise SSHException('Error reading SSH protocol banner' + str(x)) if buf[:4] == 'SSH-': From 7a3cb790a6561f391be6118b75b9f74513820517 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 17:55:37 -0800 Subject: [PATCH 20/50] Changelog re #97 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 83a524c..75dda2c 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,9 @@ Releases v1.9.0 (DD MM YYYY) ------------------- +* #97 (with a little #93): Improve config parsing of `ProxyCommand` directives + and provide a wrapper class to allow subprocess-driven proxy commands to be + used as `sock=` arguments for `SSHClient.connect`. * #77: Allow `SSHClient.connect()` to take an explicit `sock` parameter overriding creation of an internal, implicit socket object. * Thanks in no particular order to Erwin Bolwidt, Oskari Saarenmaa, Steven From 308c5f57d9a06e925001ca11df323a78d443d4b2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 23:09:52 -0800 Subject: [PATCH 21/50] Add ProxyCommand classes to top level API --- paramiko/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/paramiko/__init__.py b/paramiko/__init__.py index 8c15853..b79b4d7 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -65,7 +65,7 @@ from auth_handler import AuthHandler from channel import Channel, ChannelFile from ssh_exception import SSHException, PasswordRequiredException, \ BadAuthenticationType, ChannelException, BadHostKeyException, \ - AuthenticationException + AuthenticationException, ProxyCommandFailure from server import ServerInterface, SubsystemHandler, InteractiveQuery from rsakey import RSAKey from dsskey import DSSKey @@ -83,6 +83,7 @@ from agent import Agent, AgentKey from pkey import PKey from hostkeys import HostKeys from config import SSHConfig +from proxy import ProxyCommand # fix module names for epydoc for c in locals().values(): @@ -119,6 +120,8 @@ __all__ = [ 'Transport', 'BadAuthenticationType', 'ChannelException', 'BadHostKeyException', + 'ProxyCommand', + 'ProxyCommandFailure', 'SFTP', 'SFTPFile', 'SFTPHandle', From e7ab3c068fe39cb7be783127039113213fdb2be4 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 23:10:05 -0800 Subject: [PATCH 22/50] Fix broken import --- paramiko/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/proxy.py b/paramiko/proxy.py index e91a11a..292e361 100644 --- a/paramiko/proxy.py +++ b/paramiko/proxy.py @@ -24,7 +24,7 @@ import os from shlex import split as shlsplit from subprocess import Popen, PIPE -from ssh.ssh_exception import ProxyCommandFailure +from paramiko.ssh_exception import ProxyCommandFailure class ProxyCommand(object): From ebd007b21757d3a88b7f769fa6000662fae2eb9a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 5 Nov 2012 23:10:13 -0800 Subject: [PATCH 23/50] Python 2.5 compat --- paramiko/proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paramiko/proxy.py b/paramiko/proxy.py index 292e361..218b76e 100644 --- a/paramiko/proxy.py +++ b/paramiko/proxy.py @@ -22,6 +22,7 @@ L{ProxyCommand}. import os from shlex import split as shlsplit +import signal from subprocess import Popen, PIPE from paramiko.ssh_exception import ProxyCommandFailure @@ -82,7 +83,7 @@ class ProxyCommand(object): raise BadProxyCommand(' '.join(self.cmd), e.strerror) def close(self): - self.process.terminate() + os.kill(self.process.pid, signal.SIGTERM) def settimeout(self, timeout): # Timeouts are meaningless for this implementation, but are part of the From a3b44c7ed9a26231fcdf0e795305311da8814f92 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 13:06:08 -0800 Subject: [PATCH 24/50] Bump to 1.9.0 for release --- paramiko/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paramiko/__init__.py b/paramiko/__init__.py index b79b4d7..29e470a 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -55,7 +55,7 @@ if sys.version_info < (2, 2): __author__ = "Jeff Forcier " -__version__ = "1.8.0" +__version__ = "1.9.0" __license__ = "GNU Lesser General Public License (LGPL)" diff --git a/setup.py b/setup.py index 73407b0..1bb1a71 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ if sys.platform == 'darwin': setup(name = "paramiko", - version = "1.8.0", + version = "1.9.0", description = "SSH2 protocol library", author = "Jeff Forcier", author_email = "jeff@bitprophet.org", From 7f4c26f8601b2335b210896d2cb06348957e7aae Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 13:08:41 -0800 Subject: [PATCH 25/50] Cut 1.8.1 --- NEWS | 4 ++-- paramiko/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index adcc1b9..26382ff 100644 --- a/NEWS +++ b/NEWS @@ -12,8 +12,8 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== -v1.8.1 (DD MM YYYY) -------------------- +v1.8.1 (6th Nov 2012) +--------------------- * #90: Ensure that callbacks handed to `SFTPClient.get()` always fire at least once, even for zero-length files downloaded. Thanks to Github user `@enB` for diff --git a/paramiko/__init__.py b/paramiko/__init__.py index 8c15853..cd4dbe0 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -55,7 +55,7 @@ if sys.version_info < (2, 2): __author__ = "Jeff Forcier " -__version__ = "1.8.0" +__version__ = "1.8.1" __license__ = "GNU Lesser General Public License (LGPL)" diff --git a/setup.py b/setup.py index 73407b0..06597c6 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ if sys.platform == 'darwin': setup(name = "paramiko", - version = "1.8.0", + version = "1.8.1", description = "SSH2 protocol library", author = "Jeff Forcier", author_email = "jeff@bitprophet.org", From 65de2529a9e21f66c72f6335886db5a7f2ea6b75 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 13:10:03 -0800 Subject: [PATCH 26/50] Update changelog date for 1.9.0 --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index d7d3709..5542046 100644 --- a/NEWS +++ b/NEWS @@ -12,8 +12,8 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== -v1.9.0 (DD MM YYYY) -------------------- +v1.9.0 (6th Nov 2012) +--------------------- * #97 (with a little #93): Improve config parsing of `ProxyCommand` directives and provide a wrapper class to allow subprocess-driven proxy commands to be From a32addcfb781199b2216d7b87cf3178b97620809 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 13:13:00 -0800 Subject: [PATCH 27/50] Tweak travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 312a184..90dbb80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,8 @@ install: - pip install -e . script: python test.py notifications: + email: + on_failure: change irc: channels: "irc.freenode.org#fabric" on_success: change From b42c73356c2c5afbfbd4bf0b1278368d5910ea5d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 16:28:35 -0800 Subject: [PATCH 28/50] Git ignore built docs dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3283ff3..5f9c3d7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/ dist/ paramiko.egg-info/ test.log +docs/ From d7aa342c20b67f1846c7458b09fdd3b0f308dd71 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 16:28:35 -0800 Subject: [PATCH 29/50] Git ignore built docs dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3283ff3..5f9c3d7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/ dist/ paramiko.egg-info/ test.log +docs/ From 71f8c5c9f53e3eda550ddb68ba573eed289d74ec Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 6 Nov 2012 16:28:35 -0800 Subject: [PATCH 30/50] Git ignore built docs dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3283ff3..5f9c3d7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/ dist/ paramiko.egg-info/ test.log +docs/ From c4d4818cddf851665c8e9777355209d1aab8d4bb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 28 Nov 2012 22:18:31 -0800 Subject: [PATCH 31/50] Make docs target build whenever Paramiko files change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3c12990..572f867 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ release: docs python setup.py sdist register upload -docs: +docs: paramiko/* epydoc --no-private -o docs/ paramiko clean: From 10c51e27263afb62c395ca2905195482d7c3b630 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 28 Nov 2012 22:18:44 -0800 Subject: [PATCH 32/50] Bump dev version to 1.10 --- paramiko/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paramiko/__init__.py b/paramiko/__init__.py index 29e470a..7d7dcbf 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -55,7 +55,7 @@ if sys.version_info < (2, 2): __author__ = "Jeff Forcier " -__version__ = "1.9.0" +__version__ = "1.10.0" __license__ = "GNU Lesser General Public License (LGPL)" diff --git a/setup.py b/setup.py index 1bb1a71..9812a4f 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ if sys.platform == 'darwin': setup(name = "paramiko", - version = "1.9.0", + version = "1.10.0", description = "SSH2 protocol library", author = "Jeff Forcier", author_email = "jeff@bitprophet.org", From 2403504b44de773c3f566e7d647bc0e8661af918 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 28 Nov 2012 22:22:33 -0800 Subject: [PATCH 33/50] Fix #113: add timeout passthru to exec_command --- NEWS | 7 +++++++ paramiko/client.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 5542046..0627ad8 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,13 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== +v1.10.0 (DD MM YYYY) +-------------------- + +* #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting + of the command's internal channel object's timeout. Thanks to Cernov Vladimir + for the patch. + v1.9.0 (6th Nov 2012) --------------------- diff --git a/paramiko/client.py b/paramiko/client.py index 07560a3..97ae449 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -350,7 +350,7 @@ class SSHClient (object): self._agent.close() self._agent = None - def exec_command(self, command, bufsize=-1): + def exec_command(self, command, bufsize=-1, timeout=None): """ Execute a command on the SSH server. A new L{Channel} is opened and the requested command is executed. The command's input and output @@ -361,12 +361,15 @@ class SSHClient (object): @type command: str @param bufsize: interpreted the same way as by the built-in C{file()} function in python @type bufsize: int + @param timeout: set command's channel timeout. See L{Channel.settimeout}.settimeout + @type timeout: int @return: the stdin, stdout, and stderr of the executing command @rtype: tuple(L{ChannelFile}, L{ChannelFile}, L{ChannelFile}) @raise SSHException: if the server fails to execute the command """ chan = self._transport.open_session() + chan.settimeout(timeout) chan.exec_command(command) stdin = chan.makefile('wb', bufsize) stdout = chan.makefile('rb', bufsize) From 2832f3c60fdf078066a266cd32027a0b8ed65ce9 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 28 Nov 2012 22:18:31 -0800 Subject: [PATCH 34/50] Make docs target build whenever Paramiko files change --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3c12990..572f867 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ release: docs python setup.py sdist register upload -docs: +docs: paramiko/* epydoc --no-private -o docs/ paramiko clean: From 2575b3efc48cb9f42c3105015284b9afd9c67625 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 08:52:39 -0800 Subject: [PATCH 35/50] Fix #94 --- NEWS | 2 ++ paramiko/client.py | 3 +-- paramiko/config.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 0627ad8..2dba8e0 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,8 @@ v1.10.0 (DD MM YYYY) * #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting of the command's internal channel object's timeout. Thanks to Cernov Vladimir for the patch. +* #94: Remove duplication of SSH port constant. Thanks to Olle Lundberg for the + catch. v1.9.0 (6th Nov 2012) --------------------- diff --git a/paramiko/client.py b/paramiko/client.py index 97ae449..a777b45 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -28,6 +28,7 @@ import warnings from paramiko.agent import Agent from paramiko.common import * +from paramiko.config import SSH_PORT from paramiko.dsskey import DSSKey from paramiko.hostkeys import HostKeys from paramiko.resource import ResourceManager @@ -37,8 +38,6 @@ from paramiko.transport import Transport from paramiko.util import retry_on_signal -SSH_PORT = 22 - class MissingHostKeyPolicy (object): """ Interface for defining the policy that L{SSHClient} should use when the diff --git a/paramiko/config.py b/paramiko/config.py index 2828d90..d1ce949 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -25,7 +25,7 @@ import os import re import socket -SSH_PORT=22 +SSH_PORT = 22 proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) From f6ed6a8bbff9aabc58176b04e61c84b0509ecb7b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 08:55:43 -0800 Subject: [PATCH 36/50] Changelog re #80, fixes #80 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 2dba8e0..1a6427a 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,9 @@ v1.10.0 (DD MM YYYY) for the patch. * #94: Remove duplication of SSH port constant. Thanks to Olle Lundberg for the catch. +* #80: Expose the internal "is closed" property of the file transfer class + `BufferedFile` as `.closed`, better conforming to Python's file interface. + Thanks to `@smunaut` and James Hiscock for catch & patch. v1.9.0 (6th Nov 2012) --------------------- From bc3674d0f0c61b5c7af9cfbc9248cf574d0998b0 Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:44:34 +0300 Subject: [PATCH 37/50] Make send() and recv() fail when channel is closed ``sendall()`` was checking if the channel has been closed, and failed accordingly, but ``send()`` and ``recv()`` did not. This meant that ``chan.send("foo")`` when the channel was already closed, just blocked forever. --- paramiko/channel.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 534f8d7..35991de 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -605,6 +605,10 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -655,6 +659,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -708,6 +716,10 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -745,6 +757,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -783,9 +799,6 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 537f95dbb36c578e65cfd9cbc1a1eabc03f38428 Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:46:28 +0300 Subject: [PATCH 38/50] Forgot to import errno --- paramiko/channel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paramiko/channel.py b/paramiko/channel.py index 35991de..9a1bb87 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,6 +25,7 @@ import sys import time import threading import socket +import errno import os from paramiko.common import * From 8496eff0b7afc07675e1f42815f83633d87097ee Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:44:34 +0300 Subject: [PATCH 39/50] Make send() and recv() fail when channel is closed ``sendall()`` was checking if the channel has been closed, and failed accordingly, but ``send()`` and ``recv()`` did not. This meant that ``chan.send("foo")`` when the channel was already closed, just blocked forever. --- paramiko/channel.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 534f8d7..35991de 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -605,6 +605,10 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -655,6 +659,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -708,6 +716,10 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -745,6 +757,10 @@ class Channel (object): @since: 1.1 """ + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error(errno.EBADF, 'Socket is closed') + size = len(s) self.lock.acquire() try: @@ -783,9 +799,6 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 203c7379ac1211cc393d127ab83be774faa0f913 Mon Sep 17 00:00:00 2001 From: Tomer Filiba Date: Fri, 26 Oct 2012 15:46:28 +0300 Subject: [PATCH 40/50] Forgot to import errno --- paramiko/channel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paramiko/channel.py b/paramiko/channel.py index 35991de..9a1bb87 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,6 +25,7 @@ import sys import time import threading import socket +import errno import os from paramiko.common import * From 7a4d3c4e424e6669d091be495eb8a000b4815ebe Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:18:40 -0800 Subject: [PATCH 41/50] Revert "Make send() and recv() fail when channel is closed" This reverts commit bc3674d0f0c61b5c7af9cfbc9248cf574d0998b0. --- paramiko/channel.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 9a1bb87..1c9ae4b 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -606,10 +606,6 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -660,10 +656,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -717,10 +709,6 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -758,10 +746,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -800,6 +784,9 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 682a3eff8447ef83c10b0917c06f595803db97bb Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:18:48 -0800 Subject: [PATCH 42/50] Revert "Forgot to import errno" This reverts commit 537f95dbb36c578e65cfd9cbc1a1eabc03f38428. --- paramiko/channel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 1c9ae4b..534f8d7 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,7 +25,6 @@ import sys import time import threading import socket -import errno import os from paramiko.common import * From bda161330ffe6556db63afd3742a1727b969acb6 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:19:07 -0800 Subject: [PATCH 43/50] Revert "Make send() and recv() fail when channel is closed" This reverts commit 8496eff0b7afc07675e1f42815f83633d87097ee. --- paramiko/channel.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 9a1bb87..1c9ae4b 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -606,10 +606,6 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -660,10 +656,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -717,10 +709,6 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -758,10 +746,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -800,6 +784,9 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 287f9c3423eaa5a48357bcccd5892437f764706e Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:19:15 -0800 Subject: [PATCH 44/50] Revert "Forgot to import errno" This reverts commit 203c7379ac1211cc393d127ab83be774faa0f913. --- paramiko/channel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 1c9ae4b..534f8d7 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,7 +25,6 @@ import sys import time import threading import socket -import errno import os from paramiko.common import * From 2223aa10ccc473c440de7233e9daa69019a8bb01 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:19:50 -0800 Subject: [PATCH 45/50] Revert "Forgot to import errno" This reverts commit 668870aa83e095c81371ffce8d69404ac29e536f. --- paramiko/channel.py | 1 - 1 file changed, 1 deletion(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 9a1bb87..35991de 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -25,7 +25,6 @@ import sys import time import threading import socket -import errno import os from paramiko.common import * From 531606b0d6ebb46f8f36de554a571d96790ce3f8 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:19:56 -0800 Subject: [PATCH 46/50] Revert "Make send() and recv() fail when channel is closed" This reverts commit 23f3099b6f7808934400b96b707e122ed2dd302c. --- paramiko/channel.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 35991de..534f8d7 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -605,10 +605,6 @@ class Channel (object): @raise socket.timeout: if no data is ready before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -659,10 +655,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - try: out = self.in_stderr_buffer.read(nbytes, self.timeout) except PipeTimeout, e: @@ -716,10 +708,6 @@ class Channel (object): @raise socket.timeout: if no data could be sent before the timeout set by L{settimeout}. """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -757,10 +745,6 @@ class Channel (object): @since: 1.1 """ - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error(errno.EBADF, 'Socket is closed') - size = len(s) self.lock.acquire() try: @@ -799,6 +783,9 @@ class Channel (object): This is irritating, but identically follows python's API. """ while s: + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error('Socket is closed') sent = self.send(s) s = s[sent:] return None From 7255dcf042d3099d8b8f7d618da0fbc19e401008 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 15:37:22 -0800 Subject: [PATCH 47/50] Update Travis settings: * Don't email me, I'll see it on IRC if I'm online * Ping #paramiko, not #fabric, as it now exists --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90dbb80..6896b89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,7 @@ install: - pip install -e . script: python test.py notifications: - email: - on_failure: change irc: - channels: "irc.freenode.org#fabric" + channels: "irc.freenode.org#paramiko" on_success: change on_failure: change From 9c0d467667568fa67d45ef288cc1f6ee8a7ebfe6 Mon Sep 17 00:00:00 2001 From: Eric Buehl Date: Mon, 4 Jun 2012 11:14:34 -0700 Subject: [PATCH 48/50] allow uploading of files from an open file object --- paramiko/sftp_client.py | 128 ++++++++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 37 deletions(-) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 3eaefc9..2184134 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -533,6 +533,56 @@ class SFTPClient (BaseSFTP): """ return self._cwd + def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True): + """ + Copy the contents of an open file object (C{fl}) to the SFTP server as + C{remotepath}. Any exception raised by operations will be passed through. + + The SFTP operations use pipelining for speed. + + @param fl: opened file or file-like object to copy + @type localpath: object + @param remotepath: the destination path on the SFTP server + @type remotepath: str + @param file_size: optional size parameter passed to callback. If none is + specified, size defaults to 0 + @type file_size: int + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @param confirm: whether to do a stat() on the file afterwards to + confirm the file size (since 1.7.7) + @type confirm: bool + + @return: an object containing attributes about the given file + (since 1.7.4) + @rtype: SFTPAttributes + + @since: 1.4 + """ + fr = self.file(remotepath, 'wb') + fr.set_pipelined(True) + size = 0 + try: + while True: + data = fl.read(32768) + if len(data) == 0: + break + fr.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + finally: + fr.close() + if confirm and file_size: + s = self.stat(remotepath) + if s.st_size != size: + raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) + else: + s = SFTPAttributes() + return s + def put(self, localpath, remotepath, callback=None, confirm=True): """ Copy a local file (C{localpath}) to the SFTP server as C{remotepath}. @@ -562,29 +612,46 @@ class SFTPClient (BaseSFTP): file_size = os.stat(localpath).st_size fl = file(localpath, 'rb') try: - fr = self.file(remotepath, 'wb') - fr.set_pipelined(True) - size = 0 - try: - while True: - data = fl.read(32768) - if len(data) == 0: - break - fr.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - finally: - fr.close() + return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm) finally: fl.close() - if confirm: - s = self.stat(remotepath) - if s.st_size != size: - raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) - else: - s = SFTPAttributes() - return s + + def getfo(self, remotepath, fl, callback=None): + """ + Copy a remote file (C{remotepath}) from the SFTP server and write to + an open file or file-like object, C{fl}. Any exception raised by + operations will be passed through. This method is primarily provided + as a convenience. + + @param remotepath: opened file or file-like object to copy to + @type remotepath: object + @param fl: the destination path on the local host or open file + object + @type localpath: str + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @return: the number of bytes written to the opened file object + + @since: 1.4 + """ + fr = self.file(remotepath, 'rb') + file_size = self.stat(remotepath).st_size + fr.prefetch() + try: + size = 0 + while True: + data = fr.read(32768) + fl.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + if len(data) == 0: + break + finally: + fr.close() + return size def get(self, remotepath, localpath, callback=None): """ @@ -603,25 +670,12 @@ class SFTPClient (BaseSFTP): @since: 1.4 """ - fr = self.file(remotepath, 'rb') file_size = self.stat(remotepath).st_size - fr.prefetch() + fl = file(localpath, 'wb') try: - fl = file(localpath, 'wb') - try: - size = 0 - while True: - data = fr.read(32768) - fl.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - if len(data) == 0: - break - finally: - fl.close() + size = self.getfo(remotepath, fl, callback) finally: - fr.close() + fl.close() s = os.stat(localpath) if s.st_size != size: raise IOError('size mismatch in get! %d != %d' % (s.st_size, size)) From 2cbe38308097f920cec0aa87fed85f2cfe6d9612 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 16:16:35 -0800 Subject: [PATCH 49/50] Apply put() version of #90 --- paramiko/sftp_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 2184134..8cb8cea 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -567,12 +567,12 @@ class SFTPClient (BaseSFTP): try: while True: data = fl.read(32768) - if len(data) == 0: - break fr.write(data) size += len(data) if callback is not None: callback(size, file_size) + if len(data) == 0: + break finally: fr.close() if confirm and file_size: From 0ae0e9800c7bfb3f8ea40fa0d33ebf6dff49f759 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 29 Nov 2012 18:06:38 -0800 Subject: [PATCH 50/50] Changelog re #71 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 1a6427a..1cfaa7e 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,9 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #71: Add `SFTPClient.putfo` and `.getfo` methods to allow direct + uploading/downloading of file-like objects. Thanks to Eric Buehl for the + patch. * #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting of the command's internal channel object's timeout. Thanks to Cernov Vladimir for the patch.