From 3174b6c894b125426a7a4bae7f934a0dbe64d5b1 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:52:21 +0200 Subject: [PATCH 01/56] Updated tests for new ssh config format. --- tests/test_util.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 7e56245..6d68af7 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -112,23 +112,32 @@ class UtilTest (unittest.TestCase): f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) self.assertEquals(config._config, - [ {'identityfile': '~/.ssh/id_rsa', 'host': '*', 'user': 'robey', - 'crazy': 'something dumb '}, - {'host': '*.example.com', 'user': 'bjork', 'port': '3333'}, - {'host': 'spoo.example.com', 'crazy': 'something else'}]) + [{'host': ['*'], 'config': {}}, {'host': ['*'], 'config': {'identityfile': ['~/.ssh/id_rsa'], 'user': 'robey'}}, + {'host': ['*.example.com'], 'config': {'user': 'bjork', 'port': '3333'}}, + {'host': ['*'], 'config': {'crazy': 'something dumb '}}, + {'host': ['spoo.example.com'], 'config': {'crazy': 'something else'}}]) def test_3_host_config(self): global test_config_file f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) + for host, values in { - 'irc.danger.com': {'user': 'robey', 'crazy': 'something dumb '}, - 'irc.example.com': {'user': 'bjork', 'crazy': 'something dumb ', 'port': '3333'}, - 'spoo.example.com': {'user': 'bjork', 'crazy': 'something else', 'port': '3333'} + 'irc.danger.com': {'crazy': 'something dumb ', + 'hostname': 'irc.danger.com', + 'user': 'robey'}, + 'irc.example.com': {'crazy': 'something dumb ', + 'hostname': 'irc.example.com', + 'user': 'robey', + 'port': '3333'}, + 'spoo.example.com': {'crazy': 'something dumb ', + 'hostname': 'spoo.example.com', + 'user': 'robey', + 'port': '3333'} }.items(): values = dict(values, hostname=host, - identityfile=os.path.expanduser("~/.ssh/id_rsa") + identityfile=[os.path.expanduser("~/.ssh/id_rsa")] ) self.assertEquals( paramiko.util.lookup_ssh_host_config(host, config), @@ -159,7 +168,7 @@ class UtilTest (unittest.TestCase): # just verify that we can pull out 32 bytes and not get an exception. x = rng.read(32) self.assertEquals(len(x), 32) - + def test_7_host_config_expose_issue_33(self): test_config_file = """ Host www13.* From f33481cc44fb57a788b5726b529383f300d06b36 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:53:06 +0200 Subject: [PATCH 02/56] Add test for host negation. --- tests/test_util.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 6d68af7..9890cfc 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -212,3 +212,26 @@ Host * raise AssertionError('foo') self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) + + + def test_9_host_config_test_negation(self): + test_config_file = """ +Host www13.* !*.example.com + Port 22 + +Host *.example.com !www13.* + Port 2222 + +Host www13.* + Port 8080 + +Host * + Port 3333 + """ + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + host = 'www13.example.com' + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + {'hostname': host, 'port': '8080'} + ) From ad587fa0ef32eeeb633245c7ae68759765c0e439 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:54:23 +0200 Subject: [PATCH 03/56] Add host negation support to paramiko config. This is a rewrite of the SSHConfig class to conform with the rules specified by the manpage for ssh_config. This change also adds support for negation according to the rules introduced by OpenSSH 5.9. Reference: http://www.openssh.com/txt/release-5.9 --- paramiko/config.py | 80 +++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 458d5dd..802d915 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -1,4 +1,5 @@ # Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg # # This file is part of paramiko. # @@ -41,7 +42,8 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ - self._config = [ { 'host': '*' } ] + self._config = [] + def parse(self, file_obj): """ @@ -50,7 +52,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - configs = [self._config[0]] + host = {"host":['*'],"config":{}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -69,20 +71,20 @@ class SSHConfig (object): value = line[i:].lstrip() if key == 'host': - del configs[:] - # the value may be multiple hosts, space-delimited - for host in value.split(): - # do we have a pre-existing host config to append to? - matches = [c for c in self._config if c['host'] == host] - if len(matches) > 0: - configs.append(matches[0]) - else: - config = { 'host': host } - self._config.append(config) - configs.append(config) - else: - for config in configs: - config[key] = value + self._config.append(host) + value = value.split() + host = {key:value,'config':{}} + #identitifile is a special case, since it is allowed to be + # specified multiple times and they should be tried in order + # of specification. + elif key == 'identityfile': + if key in host['config']: + host['config']['identityfile'].append(value) + else: + host['config']['identityfile'] = [value] + elif key not in host['config']: + host['config'].update({key:value}) + self._config.append(host) def lookup(self, hostname): """ @@ -97,31 +99,45 @@ class SSHConfig (object): will win out. The keys in the returned dict are all normalized to lowercase (look for - C{"port"}, not C{"Port"}. No other processing is done to the keys or - values. + C{"port"}, not C{"Port"}. The values are processed according to the + rules for substitution variable expansion in C{ssh_config}. @param hostname: the hostname to lookup @type hostname: str """ - matches = [x for x in self._config if fnmatch.fnmatch(hostname, x['host'])] - # Move * to the end - _star = matches.pop(0) - matches.append(_star) + + matches = [ config for config in self._config if + self._allowed(hostname,config['host']) ] + ret = {} - for m in matches: - for k,v in m.iteritems(): - if not k in ret: - ret[k] = v + for match in matches: + for key in match['config']: + value = match['config'][key] + if key == 'identityfile': + if key in ret: + ret['identityfile'].extend(value) + else: + ret['identityfile'] = value + elif key not in ret: + ret[key] = value ret = self._expand_variables(ret, hostname) - del ret['host'] return ret - def _expand_variables(self, config, hostname ): + def _allowed(self, hostname, hosts): + match = False + for host in hosts: + if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]): + return False + elif fnmatch.fnmatch(hostname, host): + match = True + return match + + def _expand_variables(self, config, hostname): """ Return a dict of config options with expanded substitutions for a given hostname. - Please refer to man ssh_config(5) for the parameters that + Please refer to man C{ssh_config} for the parameters that are replaced. @param config: the config for the hostname @@ -172,5 +188,9 @@ class SSHConfig (object): for k in config: if k in replacements: for find, replace in replacements[k]: - config[k] = config[k].replace(find, str(replace)) + if isinstance(config[k],list): + for item in range(len(config[k])): + config[k][item] = config[k][item].replace(find, str(replace)) + else: + config[k] = config[k].replace(find, str(replace)) return config From 2dd74f953d0a789a9c65662492f8340d66246e5b Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 14:52:27 +0200 Subject: [PATCH 04/56] Spelling --- paramiko/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 802d915..baf1040 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -74,7 +74,7 @@ class SSHConfig (object): self._config.append(host) value = value.split() host = {key:value,'config':{}} - #identitifile is a special case, since it is allowed to be + #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. elif key == 'identityfile': From b22c11ab1b889032345a80b89af6ebee50ef7904 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 15:00:02 +0200 Subject: [PATCH 05/56] Pep8 fixes --- paramiko/config.py | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index baf1040..26a94a1 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -25,7 +25,8 @@ import fnmatch import os import socket -SSH_PORT=22 +SSH_PORT = 22 + class SSHConfig (object): """ @@ -44,7 +45,6 @@ class SSHConfig (object): """ self._config = [] - def parse(self, file_obj): """ Read an OpenSSH config from the given file object. @@ -52,7 +52,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - host = {"host":['*'],"config":{}} + host = {"host": ['*'], "config": {}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -73,7 +73,7 @@ class SSHConfig (object): if key == 'host': self._config.append(host) value = value.split() - host = {key:value,'config':{}} + host = {key: value, 'config': {}} #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. @@ -83,7 +83,7 @@ class SSHConfig (object): else: host['config']['identityfile'] = [value] elif key not in host['config']: - host['config'].update({key:value}) + host['config'].update({key: value}) self._config.append(host) def lookup(self, hostname): @@ -106,8 +106,8 @@ class SSHConfig (object): @type hostname: str """ - matches = [ config for config in self._config if - self._allowed(hostname,config['host']) ] + matches = [config for config in self._config if + self._allowed(hostname, config['host'])] ret = {} for match in matches: @@ -147,7 +147,7 @@ class SSHConfig (object): """ if 'hostname' in config: - config['hostname'] = config['hostname'].replace('%h',hostname) + config['hostname'] = config['hostname'].replace('%h', hostname) else: config['hostname'] = hostname @@ -165,32 +165,32 @@ class SSHConfig (object): host = socket.gethostname().split('.')[0] fqdn = socket.getfqdn() homedir = os.path.expanduser('~') - replacements = {'controlpath' : - [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile' : - [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ] - } + replacements = {'controlpath': + [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': + [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ]} for k in config: if k in replacements: for find, replace in replacements[k]: - if isinstance(config[k],list): + if isinstance(config[k], list): for item in range(len(config[k])): - config[k][item] = config[k][item].replace(find, str(replace)) + config[k][item] = config[k][item].\ + replace(find, str(replace)) else: config[k] = config[k].replace(find, str(replace)) return config From d66d75f277cfd49b8adc84498b0b09d71092bd0b Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:09 +0200 Subject: [PATCH 06/56] Add tests for proxycommand parsing. --- tests/test_util.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 9890cfc..d1776db 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -235,3 +235,30 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) + def test_10_host_config_test_proxycommand(self): + test_config_file = """ +Host proxy-with-equal-divisor-and-space +ProxyCommand = foo=bar + +Host proxy-with-equal-divisor-and-no-space +ProxyCommand=foo=bar + +Host proxy-without-equal-divisor +ProxyCommand foo=bar:%h-%p + """ + for host, values in { + 'proxy-with-equal-divisor-and-space' :{'hostname': 'proxy-with-equal-divisor-and-space', + 'proxycommand': 'foo=bar'}, + 'proxy-with-equal-divisor-and-no-space':{'hostname': 'proxy-with-equal-divisor-and-no-space', + 'proxycommand': 'foo=bar'}, + 'proxy-without-equal-divisor' :{'hostname': 'proxy-without-equal-divisor', + 'proxycommand': + 'foo=bar:proxy-without-equal-divisor-22'} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From 7ce9875ed7d3e3738ed775ffda4ded1ce12d35f2 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:38 +0200 Subject: [PATCH 07/56] Implement support for parsing proxycommand. --- paramiko/config.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 26a94a1..832f42e 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -23,6 +23,7 @@ L{SSHConfig}. import fnmatch import os +import re import socket SSH_PORT = 22 @@ -43,6 +44,7 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ + self._proxyregex = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) self._config = [] def parse(self, file_obj): @@ -58,8 +60,15 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: - key, value = line.split('=', 1) - key = key.strip().lower() + if not line.lower().startswith('proxycommand'): + key, value = line.split('=', 1) + key = key.strip().lower() + else: + #ProxyCommand have been specified with an equal + # sign. Eat that and split in two groups. + match = self._proxyregex.match(line) + key = match.group(1).lower() + value = match.group(2) else: # find first whitespace, and split there i = 0 @@ -183,7 +192,14 @@ class SSHConfig (object): ('%l', fqdn), ('%u', user), ('%r', remoteuser) - ]} + ], + 'proxycommand': + [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser) + ] + } for k in config: if k in replacements: for find, replace in replacements[k]: From 04cc4d55102c111d3e054ef8d83a0a62c34de561 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:54:44 +0200 Subject: [PATCH 08/56] Be more pythonic. --- paramiko/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 832f42e..bd1c2f6 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -120,8 +120,7 @@ class SSHConfig (object): ret = {} for match in matches: - for key in match['config']: - value = match['config'][key] + for key, value in match['config'].iteritems(): if key == 'identityfile': if key in ret: ret['identityfile'].extend(value) From 221131fa21e624a30d0d7cd5ed29e2749ddfd246 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 17:02:04 +0200 Subject: [PATCH 09/56] Whitespace fixes. --- tests/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index d1776db..9cbc439 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -213,7 +213,6 @@ Host * self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) - def test_9_host_config_test_negation(self): test_config_file = """ Host www13.* !*.example.com @@ -235,6 +234,7 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) + def test_10_host_config_test_proxycommand(self): test_config_file = """ Host proxy-with-equal-divisor-and-space From 79dffacf4e4ec63a4db8869c03aae33bbbbfa0b5 Mon Sep 17 00:00:00 2001 From: Lincoln de Sousa Date: Tue, 6 Nov 2012 18:02:10 -0500 Subject: [PATCH 10/56] Adding tox info (and a requirements file) --- .gitignore | 1 + requirements.txt | 2 ++ tox.ini | 6 ++++++ 3 files changed, 9 insertions(+) create mode 100644 requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 3283ff3..6f46f85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc build/ dist/ +.tox/ paramiko.egg-info/ test.log diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..75112a2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pycrypto +tox diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..6cb8001 --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py25,py26,py27 + +[testenv] +commands = pip install --use-mirrors -q -r requirements.txt + python test.py From 78654e82ecfceba30c61974b18fd4809b1ea9899 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 00:45:32 +0100 Subject: [PATCH 11/56] DRY up the code for populating the return list --- paramiko/config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index bd1c2f6..135e009 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -121,13 +121,10 @@ class SSHConfig (object): ret = {} for match in matches: for key, value in match['config'].iteritems(): - if key == 'identityfile': - if key in ret: - ret['identityfile'].extend(value) - else: - ret['identityfile'] = value - elif key not in ret: + if key not in ret: ret[key] = value + elif key == 'identityfile': + ret[key].extend(value) ret = self._expand_variables(ret, hostname) return ret From 5670e111c95612fc1c96135ca0e942ed89ec5420 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:42:29 +0100 Subject: [PATCH 12/56] Add tests for identityfile parsing. --- tests/test_util.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 9cbc439..9a155bf 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -262,3 +262,33 @@ ProxyCommand foo=bar:%h-%p paramiko.util.lookup_ssh_host_config(host, config), values ) + + def test_11_host_config_test_identityfile(self): + test_config_file = """ + +IdentityFile id_dsa0 + +Host * +IdentityFile id_dsa1 + +Host dsa2 +IdentityFile id_dsa2 + +Host dsa2* +IdentityFile id_dsa22 + """ + for host, values in { + 'foo' :{'hostname': 'foo', + 'identityfile': ['id_dsa0', 'id_dsa1']}, + 'dsa2' :{'hostname': 'dsa2', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa2', 'id_dsa22']}, + 'dsa22' :{'hostname': 'dsa22', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa22']} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From a07a339006259eb7ad60993415493d2d25a760d3 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:43:40 +0100 Subject: [PATCH 13/56] Create a copy of the identityfile list. The copy is needed else the original identityfile list is in the internal config list is updated when we modify the return dictionary. --- paramiko/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 135e009..c5a5804 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -122,7 +122,11 @@ class SSHConfig (object): for match in matches: for key, value in match['config'].iteritems(): if key not in ret: - ret[key] = value + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + ret[key] = value[:] elif key == 'identityfile': ret[key].extend(value) ret = self._expand_variables(ret, hostname) From cd51bfc031eb9163fd6c85bb3d4ec23476bb2090 Mon Sep 17 00:00:00 2001 From: Michael van der Kolff Date: Fri, 30 Nov 2012 22:02:09 +1100 Subject: [PATCH 14/56] Add support for get_pty to SSHClient.exec_command() --- paramiko/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paramiko/client.py b/paramiko/client.py index a777b45..f863806 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -349,7 +349,7 @@ class SSHClient (object): self._agent.close() self._agent = None - def exec_command(self, command, bufsize=-1, timeout=None): + def exec_command(self, command, bufsize=-1, timeout=None, get_pty=False): """ Execute a command on the SSH server. A new L{Channel} is opened and the requested command is executed. The command's input and output @@ -368,6 +368,8 @@ class SSHClient (object): @raise SSHException: if the server fails to execute the command """ chan = self._transport.open_session() + if(get_pty): + chan.get_pty() chan.settimeout(timeout) chan.exec_command(command) stdin = chan.makefile('wb', bufsize) From 3bbcf808d8da43a379cee5ce3d004d3c6eb6e1b7 Mon Sep 17 00:00:00 2001 From: Maarten Date: Fri, 30 Nov 2012 15:14:49 +0100 Subject: [PATCH 15/56] Limit memory allocation of get_bytes to 1MB If get_bytes() can pad unlimited, a RSA pub key could be crafted that would allocate GB's of nulls, thereby forming a DoS-vector. --- paramiko/message.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paramiko/message.py b/paramiko/message.py index 366c43c..47acc34 100644 --- a/paramiko/message.py +++ b/paramiko/message.py @@ -110,7 +110,8 @@ class Message (object): @rtype: string """ b = self.packet.read(n) - if len(b) < n: + max_pad_size = 1<<20 # Limit padding to 1 MB + if len(b) < n and n < max_pad_size: return b + '\x00' * (n - len(b)) return b From d5edad63a37c36aae83be0d5ddd61a2ec816f1b1 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 31 Dec 2012 16:53:51 -0500 Subject: [PATCH 16/56] New year --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 9e9cc3a..c0c2fe0 100644 --- a/README +++ b/README @@ -5,7 +5,7 @@ paramiko :Paramiko: Python SSH module :Copyright: Copyright (c) 2003-2009 Robey Pointer -:Copyright: Copyright (c) 2012 Jeff Forcier +:Copyright: Copyright (c) 2013 Jeff Forcier :License: LGPL :Homepage: https://github.com/paramiko/paramiko/ From 602250fdf9515a8127cd567d79afa8134d4cf923 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Fri, 4 Jan 2013 23:39:48 +0000 Subject: [PATCH 17/56] Turn SFTPFile into a context manager --- paramiko/sftp_file.py | 6 ++++++ tests/test_sftp.py | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index 8c5c7ac..c9fc76f 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -474,3 +474,9 @@ class SFTPFile (BufferedFile): x = self._saved_exception self._saved_exception = None raise x + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 2eadabc..39c8aa8 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -188,6 +188,17 @@ class SFTPTest (unittest.TestCase): finally: sftp.remove(FOLDER + '/duck.txt') + def test_3_sftp_file_can_be_used_as_context_manager(self): + """ + verify that an opened file can be used as a context manager + """ + try: + with sftp.open(FOLDER + '/duck.txt', 'w') as f: + f.write(ARTICLE) + self.assertEqual(sftp.stat(FOLDER + '/duck.txt').st_size, 1483) + finally: + sftp.remove(FOLDER + '/duck.txt') + def test_4_append(self): """ verify that a file can be opened for append, and tell() still works. From 0b6aebb8a95d44102968a3e0caf94cd234369e5d Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sat, 5 Jan 2013 00:05:58 +0000 Subject: [PATCH 18/56] Verify Python version >= 2.6 before running context manager test --- tests/test_sftp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 39c8aa8..7ce3e68 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -192,6 +192,10 @@ class SFTPTest (unittest.TestCase): """ verify that an opened file can be used as a context manager """ + major, minor, micro, releaselevel, serial = sys.version_info + if (major, minor) <= (2, 5): + return + try: with sftp.open(FOLDER + '/duck.txt', 'w') as f: f.write(ARTICLE) From 08109136b4217b3fc620436819f4c92434189955 Mon Sep 17 00:00:00 2001 From: Michael Williamson Date: Sat, 5 Jan 2013 00:15:26 +0000 Subject: [PATCH 19/56] Replace useless version check with import from __future__ --- tests/test_sftp.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_sftp.py b/tests/test_sftp.py index 7ce3e68..f95da69 100755 --- a/tests/test_sftp.py +++ b/tests/test_sftp.py @@ -23,6 +23,8 @@ a real actual sftp server is contacted, and a new folder is created there to do test file operations in (so no existing files will be harmed). """ +from __future__ import with_statement + from binascii import hexlify import logging import os @@ -192,10 +194,6 @@ class SFTPTest (unittest.TestCase): """ verify that an opened file can be used as a context manager """ - major, minor, micro, releaselevel, serial = sys.version_info - if (major, minor) <= (2, 5): - return - try: with sftp.open(FOLDER + '/duck.txt', 'w') as f: f.write(ARTICLE) From bf87cd124d1e973e1ba80de6cba0585d217801d8 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 3 Feb 2013 11:52:11 -0800 Subject: [PATCH 20/56] Update explicitly-stated Python version reqs to 2.5+ --- README | 18 ++---------------- paramiko/__init__.py | 6 +++--- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/README b/README index c0c2fe0..310a7f0 100644 --- a/README +++ b/README @@ -20,7 +20,7 @@ What ---- "paramiko" is a combination of the esperanto words for "paranoid" and -"friend". it's a module for python 2.2+ that implements the SSH2 protocol +"friend". it's a module for python 2.5+ that implements the SSH2 protocol for secure (encrypted and authenticated) connections to remote machines. unlike SSL (aka TLS), SSH2 protocol does not require hierarchical certificates signed by a powerful central authority. you may know SSH2 as @@ -39,8 +39,7 @@ that should have come with this archive. Requirements ------------ - - python 2.3 or better - (python 2.2 is also supported, but not recommended) + - python 2.5 or better - pycrypto 2.1 or better If you have setuptools, you can build and install paramiko and all its @@ -58,19 +57,6 @@ should also work on Windows, though i don't test it as frequently there. if you run into Windows problems, send me a patch: portability is important to me. -python 2.2 may work, thanks to some patches from Roger Binns. things to -watch out for: - - * sockets in 2.2 don't support timeouts, so the 'select' module is - imported to do polling. - * logging is mostly stubbed out. it works just enough to let paramiko - create log files for debugging, if you want them. to get real logging, - you can backport python 2.3's logging package. Roger has done that - already: - http://sourceforge.net/project/showfiles.php?group_id=75211&package_id=113804 - -you really should upgrade to python 2.3. laziness is no excuse! :) - some python distributions don't include the utf-8 string encodings, for reasons of space (misdirected as that is). if your distribution is missing encodings, you'll see an error like this:: diff --git a/paramiko/__init__.py b/paramiko/__init__.py index cd4dbe0..58f4b09 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -18,7 +18,7 @@ """ I{Paramiko} (a combination of the esperanto words for "paranoid" and "friend") -is a module for python 2.3 or greater that implements the SSH2 protocol for +is a module for python 2.5 or greater that implements the SSH2 protocol for secure (encrypted and authenticated) connections to remote machines. Unlike SSL (aka TLS), the SSH2 protocol does not require hierarchical certificates signed by a powerful central authority. You may know SSH2 as the protocol that @@ -50,8 +50,8 @@ Website: U{https://github.com/paramiko/paramiko/} import sys -if sys.version_info < (2, 2): - raise RuntimeError('You need python 2.2 for this module.') +if sys.version_info < (2, 5): + raise RuntimeError('You need python 2.5+ for this module.') __author__ = "Jeff Forcier " From e761502e8ec5958c27228006314e423057ff8c9d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Sun, 3 Feb 2013 12:54:53 -0800 Subject: [PATCH 21/56] Add changelog entry re #115 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 1cfaa7e..0fe99f5 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,9 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #115: Add convenience `get_pty` kwarg to `Client.exec_command` so users not + manually controlling a channel object can still toggle PTY creation. Thanks + to Michael van der Kolff for the patch. * #71: Add `SFTPClient.putfo` and `.getfo` methods to allow direct uploading/downloading of file-like objects. Thanks to Eric Buehl for the patch. From ac9370d3e0e4a645f893028c9dee0ece20c4f0d3 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 15:32:19 -0800 Subject: [PATCH 22/56] Changelog re #116 --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 0fe99f5..10b2bf5 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #116: Limit `Message.get_bytes` to an upper bound of 1MB to protect against + potential DoS vectors. Thanks to `@mvschaik` for catch & patch. * #115: Add convenience `get_pty` kwarg to `Client.exec_command` so users not manually controlling a channel object can still toggle PTY creation. Thanks to Michael van der Kolff for the patch. From 8e697988af494f4cd499911c9a15ee25d6e2a492 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 15:56:09 -0800 Subject: [PATCH 23/56] Changelog + docs re #127 --- NEWS | 2 ++ paramiko/sftp_client.py | 2 +- paramiko/sftp_file.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 10b2bf5..cd98477 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #127: Turn `SFTPFile` into a context manager. Thanks to Michael Williamson +* for the patch. * #116: Limit `Message.get_bytes` to an upper bound of 1MB to protect against potential DoS vectors. Thanks to `@mvschaik` for catch & patch. * #115: Add convenience `get_pty` kwarg to `Client.exec_command` so users not diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 8cb8cea..7df643f 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -198,7 +198,7 @@ class SFTPClient (BaseSFTP): Open a file on the remote server. The arguments are the same as for python's built-in C{file} (aka C{open}). A file-like object is returned, which closely mimics the behavior of a normal python file - object. + object, including the ability to be used as a context manager. The mode indicates how the file is to be opened: C{'r'} for reading, C{'w'} for writing (truncating an existing file), C{'a'} for appending, diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index c9fc76f..d4ecb89 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -34,6 +34,9 @@ from paramiko.sftp_attr import SFTPAttributes class SFTPFile (BufferedFile): """ Proxy object for a file on the remote server, in client mode SFTP. + + Instances of this class may be used as context managers in the same way + that built-in Python file objects are. """ # Some sftp servers will choke if you send read/write requests larger than From adad068b132d0f0c64174b75fb72ee238c289992 Mon Sep 17 00:00:00 2001 From: Kent Gibson Date: Sun, 4 Nov 2012 13:58:04 +0800 Subject: [PATCH 24/56] Don't random pad packets for SDCTR ciphers --- paramiko/packet.py | 12 +++++++----- paramiko/transport.py | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/paramiko/packet.py b/paramiko/packet.py index 5d918e2..5972ab2 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -87,6 +87,7 @@ class Packetizer (object): self.__mac_size_in = 0 self.__block_engine_out = None self.__block_engine_in = None + self.__sdctr_out = False self.__mac_engine_out = None self.__mac_engine_in = None self.__mac_key_out = '' @@ -110,11 +111,12 @@ class Packetizer (object): """ self.__logger = log - def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key): + def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key, sdctr): """ Switch outbound data cipher. """ self.__block_engine_out = block_engine + self.__sdctr_out = sdctr self.__block_size_out = block_size self.__mac_engine_out = mac_engine self.__mac_size_out = mac_size @@ -490,12 +492,12 @@ class Packetizer (object): padding = 3 + bsize - ((len(payload) + 8) % bsize) packet = struct.pack('>IB', len(payload) + padding + 1, padding) packet += payload - if self.__block_engine_out is not None: - packet += rng.read(padding) - else: - # cute trick i caught openssh doing: if we're not encrypting, + if self.__sdctr_out or self.__block_engine_out is None: + # cute trick i caught openssh doing: if we're not encrypting or SDCTR mode (RFC4344), # don't waste random bytes for the padding packet += (chr(0) * padding) + else: + packet += rng.read(padding) return packet def _trigger_rekey(self): diff --git a/paramiko/transport.py b/paramiko/transport.py index c801031..fd6dab7 100644 --- a/paramiko/transport.py +++ b/paramiko/transport.py @@ -1885,7 +1885,8 @@ class Transport (threading.Thread): mac_key = self._compute_key('F', mac_engine.digest_size) else: mac_key = self._compute_key('E', mac_engine.digest_size) - self.packetizer.set_outbound_cipher(engine, block_size, mac_engine, mac_size, mac_key) + sdctr = self.local_cipher.endswith('-ctr') + self.packetizer.set_outbound_cipher(engine, block_size, mac_engine, mac_size, mac_key, sdctr) compress_out = self._compression_info[self.local_compression][0] if (compress_out is not None) and ((self.local_compression != 'zlib@openssh.com') or self.authenticated): self._log(DEBUG, 'Switching on outbound compression ...') From 7e5911a1ff67c1d7be58e3cb8faa456156a455ed Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 18:44:06 -0800 Subject: [PATCH 25/56] Give sdctr a default value for backwards+test compat Re #102 --- paramiko/packet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/packet.py b/paramiko/packet.py index 5972ab2..38a6d4b 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -111,7 +111,7 @@ class Packetizer (object): """ self.__logger = log - def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key, sdctr): + def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key, sdctr=False): """ Switch outbound data cipher. """ From e034a24f87ca24171fceb9f9069149f3fc76d51b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 18:50:19 -0800 Subject: [PATCH 26/56] Add changelog entry re #102 --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index cd98477..6a70e86 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,10 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #102: Forego random padding for packets when running under `*-ctr` ciphers. + This corrects some slowdowns on platforms where random byte generation is + inefficient (e.g. Windows). Thanks to `@warthog618` for catch & patch, and + Michael van der Kolff for code/technique review. * #127: Turn `SFTPFile` into a context manager. Thanks to Michael Williamson * for the patch. * #116: Limit `Message.get_bytes` to an upper bound of 1MB to protect against From 2f1daad1b9dfb2a8ff38795d5b3a1d95765a0ea4 Mon Sep 17 00:00:00 2001 From: Parantapa Bhattacharya Date: Sun, 6 Jan 2013 23:41:54 +0530 Subject: [PATCH 27/56] Compute host's FQDN on demand only --- paramiko/config.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index d1ce949..9c15e18 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -28,6 +28,18 @@ import socket SSH_PORT = 22 proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) +class LazyFqdn(object): + """ + Returns the host's fqdn on request as string. + """ + + def __init__(self): + self.fqdn = None + + def __str__(self): + if self.fqdn is None: + self.fqdn = socket.getfqdn() + return self.fqdn class SSHConfig (object): """ @@ -155,7 +167,7 @@ class SSHConfig (object): remoteuser = user host = socket.gethostname().split('.')[0] - fqdn = socket.getfqdn() + fqdn = LazyFqdn() homedir = os.path.expanduser('~') replacements = { 'controlpath': [ @@ -184,5 +196,6 @@ class SSHConfig (object): for k in config: if k in replacements: for find, replace in replacements[k]: + if find in config[k]: config[k] = config[k].replace(find, str(replace)) return config From bf4b535920eee426ceae1ade30a62e7afd698619 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 19:40:09 -0800 Subject: [PATCH 28/56] Changelog re #128 --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 6a70e86..53d87fd 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,9 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #128: Defer FQDN resolution until needed, when parsing SSH config files. This + will speed up some operations on systems whose local hostnames are not in + DNS. Thanks to Parantapa Bhattacharya for catch & patch. * #102: Forego random padding for packets when running under `*-ctr` ciphers. This corrects some slowdowns on platforms where random byte generation is inefficient (e.g. Windows). Thanks to `@warthog618` for catch & patch, and From 9d2fb82284a42de81bcb35d2536e682687d1bd81 Mon Sep 17 00:00:00 2001 From: John Hensley Date: Mon, 19 Nov 2012 15:32:08 -0500 Subject: [PATCH 29/56] Document SSHConfig FQDN logic. Merged with pre-picked changes re #128. Conflicts: paramiko/config.py --- paramiko/config.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 9c15e18..2782b6a 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -33,12 +33,46 @@ class LazyFqdn(object): Returns the host's fqdn on request as string. """ - def __init__(self): + def __init__(self, config): self.fqdn = None + self.config = config def __str__(self): if self.fqdn is None: - self.fqdn = socket.getfqdn() + # + # If the SSH config contains AddressFamily, use that when determining + # the local host's FQDN. Using socket.getfqdn() from the standard + # library is the most general solution, but can result in noticeable + # delays on some platforms when IPv6 is misconfigured or not available, + # as it calls getaddrinfo with no address family specified, so both + # IPv4 and IPv6 are checked. + # + + address_family = self.config.get('addressfamily', 'any').lower() + + if address_family == 'inet': + ipv4_results = socket.getaddrinfo(host, None, socket.AF_INET, + socket.SOCK_DGRAM, socket.IPPROTO_IP, + socket.AI_CANONNAME) + for res in ipv4_results: + af, socktype, proto, canonname, sa = res + if canonname and '.' in canonname: + fqdn = canonname + break + elif address_family == 'inet6': + ipv6_results = socket.getaddrinfo(host, None, socket.AF_INET6, + socket.SOCK_DGRAM, + socket.IPPROTO_IP, + socket.AI_CANONNAME) + for res in ipv6_results: + af, socktype, proto, canonname, sa = res + if canonname and '.' in canonname: + fqdn = canonname + break + + if fqdn is None: + fqdn = socket.getfqdn() + self.fqdn = fqdn return self.fqdn class SSHConfig (object): @@ -167,7 +201,7 @@ class SSHConfig (object): remoteuser = user host = socket.gethostname().split('.')[0] - fqdn = LazyFqdn() + fqdn = LazyFqdn(self) homedir = os.path.expanduser('~') replacements = { 'controlpath': [ From b9242c654abd3ec5137f482436f57a306b151573 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 19:49:29 -0800 Subject: [PATCH 30/56] Changelog re #110 --- NEWS | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 53d87fd..7b45d62 100644 --- a/NEWS +++ b/NEWS @@ -15,9 +15,10 @@ Releases v1.10.0 (DD MM YYYY) -------------------- -* #128: Defer FQDN resolution until needed, when parsing SSH config files. This - will speed up some operations on systems whose local hostnames are not in - DNS. Thanks to Parantapa Bhattacharya for catch & patch. +* #110: Honor SSH config `AddressFamily` setting when looking up local + host's FQDN. Thanks to John Hensley for the patch. +* #128: Defer FQDN resolution until needed, when parsing SSH config files. + Thanks to Parantapa Bhattacharya for catch & patch. * #102: Forego random padding for packets when running under `*-ctr` ciphers. This corrects some slowdowns on platforms where random byte generation is inefficient (e.g. Windows). Thanks to `@warthog618` for catch & patch, and From 3563fca9942a752697cb17af71b179c4f6cc4361 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 27 Feb 2013 19:54:22 -0800 Subject: [PATCH 31/56] Refactor duplicative code re #110 --- paramiko/config.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 2782b6a..a080533 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -25,6 +25,7 @@ import os import re import socket + SSH_PORT = 22 proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) @@ -48,30 +49,21 @@ class LazyFqdn(object): # IPv4 and IPv6 are checked. # + # Handle specific option address_family = self.config.get('addressfamily', 'any').lower() - - if address_family == 'inet': - ipv4_results = socket.getaddrinfo(host, None, socket.AF_INET, - socket.SOCK_DGRAM, socket.IPPROTO_IP, - socket.AI_CANONNAME) - for res in ipv4_results: + if address_family != 'any': + family = socket.AF_INET if address_family == 'inet' else socket.AF_INET6 + results = socket.getaddrinfo(host, None, family, + socket.SOCK_DGRAM, socket.IPPROTO_IP, socket.AI_CANONNAME) + for res in results: af, socktype, proto, canonname, sa = res if canonname and '.' in canonname: fqdn = canonname break - elif address_family == 'inet6': - ipv6_results = socket.getaddrinfo(host, None, socket.AF_INET6, - socket.SOCK_DGRAM, - socket.IPPROTO_IP, - socket.AI_CANONNAME) - for res in ipv6_results: - af, socktype, proto, canonname, sa = res - if canonname and '.' in canonname: - fqdn = canonname - break - + # Handle 'any' / unspecified if fqdn is None: fqdn = socket.getfqdn() + # Cache self.fqdn = fqdn return self.fqdn From 98ae4e975dd8fd25af4b615bbbd034cc0831e1fa Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:52:21 +0200 Subject: [PATCH 32/56] Updated tests for new ssh config format. --- tests/test_util.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 093a215..348b796 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -104,23 +104,32 @@ class UtilTest(ParamikoTest): f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) self.assertEquals(config._config, - [ {'identityfile': '~/.ssh/id_rsa', 'host': '*', 'user': 'robey', - 'crazy': 'something dumb '}, - {'host': '*.example.com', 'user': 'bjork', 'port': '3333'}, - {'host': 'spoo.example.com', 'crazy': 'something else'}]) + [{'host': ['*'], 'config': {}}, {'host': ['*'], 'config': {'identityfile': ['~/.ssh/id_rsa'], 'user': 'robey'}}, + {'host': ['*.example.com'], 'config': {'user': 'bjork', 'port': '3333'}}, + {'host': ['*'], 'config': {'crazy': 'something dumb '}}, + {'host': ['spoo.example.com'], 'config': {'crazy': 'something else'}}]) def test_3_host_config(self): global test_config_file f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) + for host, values in { - 'irc.danger.com': {'user': 'robey', 'crazy': 'something dumb '}, - 'irc.example.com': {'user': 'bjork', 'crazy': 'something dumb ', 'port': '3333'}, - 'spoo.example.com': {'user': 'bjork', 'crazy': 'something else', 'port': '3333'} + 'irc.danger.com': {'crazy': 'something dumb ', + 'hostname': 'irc.danger.com', + 'user': 'robey'}, + 'irc.example.com': {'crazy': 'something dumb ', + 'hostname': 'irc.example.com', + 'user': 'robey', + 'port': '3333'}, + 'spoo.example.com': {'crazy': 'something dumb ', + 'hostname': 'spoo.example.com', + 'user': 'robey', + 'port': '3333'} }.items(): values = dict(values, hostname=host, - identityfile=os.path.expanduser("~/.ssh/id_rsa") + identityfile=[os.path.expanduser("~/.ssh/id_rsa")] ) self.assertEquals( paramiko.util.lookup_ssh_host_config(host, config), @@ -151,8 +160,8 @@ class UtilTest(ParamikoTest): # just verify that we can pull out 32 bytes and not get an exception. x = rng.read(32) self.assertEquals(len(x), 32) - - def test_7_host_config_expose_ssh_issue_33(self): + + def test_7_host_config_expose_issue_33(self): test_config_file = """ Host www13.* Port 22 From 21689d964798fbbd037e7b995088cee883796225 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:53:06 +0200 Subject: [PATCH 33/56] Add test for host negation. --- tests/test_util.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 348b796..4bb34da 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -249,3 +249,25 @@ Host portonly host_config(host, config)['proxycommand'], val ) + + def test_11_host_config_test_negation(self): + test_config_file = """ +Host www13.* !*.example.com + Port 22 + +Host *.example.com !www13.* + Port 2222 + +Host www13.* + Port 8080 + +Host * + Port 3333 + """ + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + host = 'www13.example.com' + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + {'hostname': host, 'port': '8080'} + ) From 57d776b31851bf6f50e46b3c41d064fff90d4385 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:54:23 +0200 Subject: [PATCH 34/56] Add host negation support to paramiko config. This is a rewrite of the SSHConfig class to conform with the rules specified by the manpage for ssh_config. This change also adds support for negation according to the rules introduced by OpenSSH 5.9. Reference: http://www.openssh.com/txt/release-5.9 --- paramiko/config.py | 81 ++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index a080533..7908a52 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -1,4 +1,5 @@ # Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg # # This file is part of paramiko. # @@ -82,7 +83,8 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ - self._config = [ { 'host': '*' } ] + self._config = [] + def parse(self, file_obj): """ @@ -91,7 +93,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - configs = [self._config[0]] + host = {"host":['*'],"config":{}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -115,20 +117,20 @@ class SSHConfig (object): value = line[i:].lstrip() if key == 'host': - del configs[:] - # the value may be multiple hosts, space-delimited - for host in value.split(): - # do we have a pre-existing host config to append to? - matches = [c for c in self._config if c['host'] == host] - if len(matches) > 0: - configs.append(matches[0]) - else: - config = { 'host': host } - self._config.append(config) - configs.append(config) - else: - for config in configs: - config[key] = value + self._config.append(host) + value = value.split() + host = {key:value,'config':{}} + #identitifile is a special case, since it is allowed to be + # specified multiple times and they should be tried in order + # of specification. + elif key == 'identityfile': + if key in host['config']: + host['config']['identityfile'].append(value) + else: + host['config']['identityfile'] = [value] + elif key not in host['config']: + host['config'].update({key:value}) + self._config.append(host) def lookup(self, hostname): """ @@ -143,31 +145,45 @@ class SSHConfig (object): will win out. The keys in the returned dict are all normalized to lowercase (look for - C{"port"}, not C{"Port"}. No other processing is done to the keys or - values. + C{"port"}, not C{"Port"}. The values are processed according to the + rules for substitution variable expansion in C{ssh_config}. @param hostname: the hostname to lookup @type hostname: str """ - matches = [x for x in self._config if fnmatch.fnmatch(hostname, x['host'])] - # Move * to the end - _star = matches.pop(0) - matches.append(_star) + + matches = [ config for config in self._config if + self._allowed(hostname,config['host']) ] + ret = {} - for m in matches: - for k,v in m.iteritems(): - if not k in ret: - ret[k] = v + for match in matches: + for key in match['config']: + value = match['config'][key] + if key == 'identityfile': + if key in ret: + ret['identityfile'].extend(value) + else: + ret['identityfile'] = value + elif key not in ret: + ret[key] = value ret = self._expand_variables(ret, hostname) - del ret['host'] return ret - def _expand_variables(self, config, hostname ): + def _allowed(self, hostname, hosts): + match = False + for host in hosts: + if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]): + return False + elif fnmatch.fnmatch(hostname, host): + match = True + return match + + def _expand_variables(self, config, hostname): """ Return a dict of config options with expanded substitutions for a given hostname. - Please refer to man ssh_config(5) for the parameters that + Please refer to man C{ssh_config} for the parameters that are replaced. @param config: the config for the hostname @@ -222,6 +238,9 @@ class SSHConfig (object): for k in config: if k in replacements: for find, replace in replacements[k]: - if find in config[k]: - config[k] = config[k].replace(find, str(replace)) + if isinstance(config[k],list): + for item in range(len(config[k])): + config[k][item] = config[k][item].replace(find, str(replace)) + else: + config[k] = config[k].replace(find, str(replace)) return config From 85551dffd6d6008cf0409ee776d6792a8e51468f Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 14:52:27 +0200 Subject: [PATCH 35/56] Spelling --- paramiko/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 7908a52..3f2703e 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -120,7 +120,7 @@ class SSHConfig (object): self._config.append(host) value = value.split() host = {key:value,'config':{}} - #identitifile is a special case, since it is allowed to be + #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. elif key == 'identityfile': From 42d77483e846db5e43296d7055d059e43168dab7 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 15:00:02 +0200 Subject: [PATCH 36/56] Pep8 fixes --- paramiko/config.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 3f2703e..a31dffc 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -26,7 +26,6 @@ import os import re import socket - SSH_PORT = 22 proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) @@ -85,7 +84,6 @@ class SSHConfig (object): """ self._config = [] - def parse(self, file_obj): """ Read an OpenSSH config from the given file object. @@ -93,7 +91,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - host = {"host":['*'],"config":{}} + host = {"host": ['*'], "config": {}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -119,7 +117,7 @@ class SSHConfig (object): if key == 'host': self._config.append(host) value = value.split() - host = {key:value,'config':{}} + host = {key: value, 'config': {}} #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. @@ -129,7 +127,7 @@ class SSHConfig (object): else: host['config']['identityfile'] = [value] elif key not in host['config']: - host['config'].update({key:value}) + host['config'].update({key: value}) self._config.append(host) def lookup(self, hostname): @@ -152,8 +150,8 @@ class SSHConfig (object): @type hostname: str """ - matches = [ config for config in self._config if - self._allowed(hostname,config['host']) ] + matches = [config for config in self._config if + self._allowed(hostname, config['host'])] ret = {} for match in matches: @@ -193,7 +191,7 @@ class SSHConfig (object): """ if 'hostname' in config: - config['hostname'] = config['hostname'].replace('%h',hostname) + config['hostname'] = config['hostname'].replace('%h', hostname) else: config['hostname'] = hostname @@ -235,12 +233,14 @@ class SSHConfig (object): ('%r', remoteuser), ], } + for k in config: if k in replacements: for find, replace in replacements[k]: - if isinstance(config[k],list): + if isinstance(config[k], list): for item in range(len(config[k])): - config[k][item] = config[k][item].replace(find, str(replace)) + config[k][item] = config[k][item].\ + replace(find, str(replace)) else: config[k] = config[k].replace(find, str(replace)) return config From b3d5156019566c972a3dbbea851cffed3cf08514 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:09 +0200 Subject: [PATCH 37/56] Add tests for proxycommand parsing. --- tests/test_util.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 4bb34da..c97c1d5 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -271,3 +271,30 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) + def test_10_host_config_test_proxycommand(self): + test_config_file = """ +Host proxy-with-equal-divisor-and-space +ProxyCommand = foo=bar + +Host proxy-with-equal-divisor-and-no-space +ProxyCommand=foo=bar + +Host proxy-without-equal-divisor +ProxyCommand foo=bar:%h-%p + """ + for host, values in { + 'proxy-with-equal-divisor-and-space' :{'hostname': 'proxy-with-equal-divisor-and-space', + 'proxycommand': 'foo=bar'}, + 'proxy-with-equal-divisor-and-no-space':{'hostname': 'proxy-with-equal-divisor-and-no-space', + 'proxycommand': 'foo=bar'}, + 'proxy-without-equal-divisor' :{'hostname': 'proxy-without-equal-divisor', + 'proxycommand': + 'foo=bar:proxy-without-equal-divisor-22'} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From ac1310c4a171587ae8fd15268a1981eaf139be57 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:38 +0200 Subject: [PATCH 38/56] Implement support for parsing proxycommand. --- paramiko/config.py | 66 +++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index a31dffc..e9c1aaf 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -82,6 +82,7 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ + self._proxyregex = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) self._config = [] def parse(self, file_obj): @@ -97,6 +98,7 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: +<<<<<<< HEAD # Ensure ProxyCommand gets properly split if line.lower().strip().startswith('proxycommand'): match = proxy_re.match(line) @@ -104,6 +106,20 @@ class SSHConfig (object): else: key, value = line.split('=', 1) key = key.strip().lower() +||||||| merged common ancestors + key, value = line.split('=', 1) + key = key.strip().lower() +======= + if not line.lower().startswith('proxycommand'): + key, value = line.split('=', 1) + key = key.strip().lower() + else: + #ProxyCommand have been specified with an equal + # sign. Eat that and split in two groups. + match = self._proxyregex.match(line) + key = match.group(1).lower() + value = match.group(2) +>>>>>>> Implement support for parsing proxycommand. else: # find first whitespace, and split there i = 0 @@ -209,30 +225,32 @@ class SSHConfig (object): host = socket.gethostname().split('.')[0] fqdn = LazyFqdn(self) homedir = os.path.expanduser('~') - replacements = { - 'controlpath': [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile': [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ], - 'proxycommand': [ - ('%h', config['hostname']), - ('%p', port), - ('%r', remoteuser), - ], - } + replacements = {'controlpath': + [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': + [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ], + 'proxycommand': + [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser) + ] + } for k in config: if k in replacements: From 32424ba109f2c9a3843b69c6768e04616fceb5fd Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:54:44 +0200 Subject: [PATCH 39/56] Be more pythonic. --- paramiko/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index e9c1aaf..91b691a 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -171,8 +171,7 @@ class SSHConfig (object): ret = {} for match in matches: - for key in match['config']: - value = match['config'][key] + for key, value in match['config'].iteritems(): if key == 'identityfile': if key in ret: ret['identityfile'].extend(value) From c79e6a3f92938af07915c23ea1d7cbb75dac6d89 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 17:02:04 +0200 Subject: [PATCH 40/56] Whitespace fixes. --- paramiko/config.py | 16 ---------------- tests/test_util.py | 4 +++- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 91b691a..143223d 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -82,7 +82,6 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ - self._proxyregex = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) self._config = [] def parse(self, file_obj): @@ -98,7 +97,6 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: -<<<<<<< HEAD # Ensure ProxyCommand gets properly split if line.lower().strip().startswith('proxycommand'): match = proxy_re.match(line) @@ -106,20 +104,6 @@ class SSHConfig (object): else: key, value = line.split('=', 1) key = key.strip().lower() -||||||| merged common ancestors - key, value = line.split('=', 1) - key = key.strip().lower() -======= - if not line.lower().startswith('proxycommand'): - key, value = line.split('=', 1) - key = key.strip().lower() - else: - #ProxyCommand have been specified with an equal - # sign. Eat that and split in two groups. - match = self._proxyregex.match(line) - key = match.group(1).lower() - value = match.group(2) ->>>>>>> Implement support for parsing proxycommand. else: # find first whitespace, and split there i = 0 diff --git a/tests/test_util.py b/tests/test_util.py index c97c1d5..e6f417d 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -205,6 +205,7 @@ Host * self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) +<<<<<<< HEAD def test_9_proxycommand_config_equals_parsing(self): """ ProxyCommand should not split on equals signs within the value. @@ -271,7 +272,8 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) - def test_10_host_config_test_proxycommand(self): + + def test_12_host_config_test_proxycommand(self): test_config_file = """ Host proxy-with-equal-divisor-and-space ProxyCommand = foo=bar From ea3c3f53b662eafa5dfa8e25d9543a568b81b353 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 00:45:32 +0100 Subject: [PATCH 41/56] DRY up the code for populating the return list --- paramiko/config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 143223d..5e1cd88 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -156,13 +156,10 @@ class SSHConfig (object): ret = {} for match in matches: for key, value in match['config'].iteritems(): - if key == 'identityfile': - if key in ret: - ret['identityfile'].extend(value) - else: - ret['identityfile'] = value - elif key not in ret: + if key not in ret: ret[key] = value + elif key == 'identityfile': + ret[key].extend(value) ret = self._expand_variables(ret, hostname) return ret From 109d2b200a1e419f139166380fd0d136f4d57321 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:42:29 +0100 Subject: [PATCH 42/56] Add tests for identityfile parsing. --- tests/test_util.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index e6f417d..c4ad9d8 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -205,7 +205,6 @@ Host * self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) -<<<<<<< HEAD def test_9_proxycommand_config_equals_parsing(self): """ ProxyCommand should not split on equals signs within the value. @@ -300,3 +299,33 @@ ProxyCommand foo=bar:%h-%p paramiko.util.lookup_ssh_host_config(host, config), values ) + + def test_11_host_config_test_identityfile(self): + test_config_file = """ + +IdentityFile id_dsa0 + +Host * +IdentityFile id_dsa1 + +Host dsa2 +IdentityFile id_dsa2 + +Host dsa2* +IdentityFile id_dsa22 + """ + for host, values in { + 'foo' :{'hostname': 'foo', + 'identityfile': ['id_dsa0', 'id_dsa1']}, + 'dsa2' :{'hostname': 'dsa2', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa2', 'id_dsa22']}, + 'dsa22' :{'hostname': 'dsa22', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa22']} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From f41fc8fd28c9fd7a41fd6c291d507a7a1b3cde56 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:43:40 +0100 Subject: [PATCH 43/56] Create a copy of the identityfile list. The copy is needed else the original identityfile list is in the internal config list is updated when we modify the return dictionary. --- paramiko/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 5e1cd88..f5486b9 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -157,7 +157,11 @@ class SSHConfig (object): for match in matches: for key, value in match['config'].iteritems(): if key not in ret: - ret[key] = value + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + ret[key] = value[:] elif key == 'identityfile': ret[key].extend(value) ret = self._expand_variables(ret, hostname) From 38767982cd1a5181536a5f24f830e71933f7fb10 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Thu, 28 Feb 2013 12:36:03 +0100 Subject: [PATCH 44/56] Fix broken test. --- tests/test_util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index c4ad9d8..efda9b2 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -229,16 +229,16 @@ Host equals-delimited ProxyCommand should perform interpolation on the value """ config = paramiko.util.parse_ssh_config(cStringIO.StringIO(""" -Host * - Port 25 - ProxyCommand host %h port %p - Host specific Port 37 ProxyCommand host %h port %p lol Host portonly Port 155 + +Host * + Port 25 + ProxyCommand host %h port %p """)) for host, val in ( ('foo.com', "host foo.com port 25"), From 93dce43e8643c9dc79fadd546426d9434ae6e534 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Thu, 28 Feb 2013 12:36:21 +0100 Subject: [PATCH 45/56] Fix argument passed to LazyFqdn --- paramiko/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index f5486b9..2bac6c2 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -50,6 +50,7 @@ class LazyFqdn(object): # # Handle specific option + fqdn = None address_family = self.config.get('addressfamily', 'any').lower() if address_family != 'any': family = socket.AF_INET if address_family == 'inet' else socket.AF_INET6 @@ -207,7 +208,7 @@ class SSHConfig (object): remoteuser = user host = socket.gethostname().split('.')[0] - fqdn = LazyFqdn(self) + fqdn = LazyFqdn(config) homedir = os.path.expanduser('~') replacements = {'controlpath': [ From 06f970482060c707d210561159c3d9e5ba5a5f1f Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Thu, 28 Feb 2013 12:45:07 +0100 Subject: [PATCH 46/56] Pep8 fixes --- paramiko/config.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 2bac6c2..e41bae4 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -29,6 +29,7 @@ import socket SSH_PORT = 22 proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) + class LazyFqdn(object): """ Returns the host's fqdn on request as string. @@ -41,21 +42,26 @@ class LazyFqdn(object): def __str__(self): if self.fqdn is None: # - # If the SSH config contains AddressFamily, use that when determining - # the local host's FQDN. Using socket.getfqdn() from the standard - # library is the most general solution, but can result in noticeable - # delays on some platforms when IPv6 is misconfigured or not available, - # as it calls getaddrinfo with no address family specified, so both - # IPv4 and IPv6 are checked. + # If the SSH config contains AddressFamily, use that when + # determining the local host's FQDN. Using socket.getfqdn() from + # the standard library is the most general solution, but can + # result in noticeable delays on some platforms when IPv6 is + # misconfigured or not available, as it calls getaddrinfo with no + # address family specified, so both IPv4 and IPv6 are checked. # # Handle specific option fqdn = None address_family = self.config.get('addressfamily', 'any').lower() if address_family != 'any': - family = socket.AF_INET if address_family == 'inet' else socket.AF_INET6 - results = socket.getaddrinfo(host, None, family, - socket.SOCK_DGRAM, socket.IPPROTO_IP, socket.AI_CANONNAME) + family = socket.AF_INET if address_family == 'inet' \ + else socket.AF_INET6 + results = socket.getaddrinfo(host, + None, + family, + socket.SOCK_DGRAM, + socket.IPPROTO_IP, + socket.AI_CANONNAME) for res in results: af, socktype, proto, canonname, sa = res if canonname and '.' in canonname: @@ -68,6 +74,7 @@ class LazyFqdn(object): self.fqdn = fqdn return self.fqdn + class SSHConfig (object): """ Representation of config information as stored in the format used by From 1903ee1432676f1d3a51daba7ae2edffdd4a28cd Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Thu, 28 Feb 2013 12:52:01 +0100 Subject: [PATCH 47/56] Pep8 fixes --- paramiko/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paramiko/config.py b/paramiko/config.py index 8e45ad5..e41bae4 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -74,6 +74,7 @@ class LazyFqdn(object): self.fqdn = fqdn return self.fqdn + class SSHConfig (object): """ Representation of config information as stored in the format used by From 277526315e18c7190a92471eab61d3610331c7df Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 1 Mar 2013 09:42:09 -0800 Subject: [PATCH 48/56] Changelog re #93 --- NEWS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 7b45d62..e30e433 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,9 @@ Releases v1.10.0 (DD MM YYYY) -------------------- +* #93: Overhaul SSH config parsing to be in line with `man ssh_config` (& the + behavior of `ssh` itself), including addition of parameter expansion within + config values. Thanks to Olle Lundberg for the patch. * #110: Honor SSH config `AddressFamily` setting when looking up local host's FQDN. Thanks to John Hensley for the patch. * #128: Defer FQDN resolution until needed, when parsing SSH config files. @@ -24,7 +27,7 @@ v1.10.0 (DD MM YYYY) inefficient (e.g. Windows). Thanks to `@warthog618` for catch & patch, and Michael van der Kolff for code/technique review. * #127: Turn `SFTPFile` into a context manager. Thanks to Michael Williamson -* for the patch. + for the patch. * #116: Limit `Message.get_bytes` to an upper bound of 1MB to protect against potential DoS vectors. Thanks to `@mvschaik` for catch & patch. * #115: Add convenience `get_pty` kwarg to `Client.exec_command` so users not From edc9eaf4f2993858b10594dbda3d321c45444417 Mon Sep 17 00:00:00 2001 From: Phillip Heller Date: Tue, 20 Nov 2012 16:10:16 -0500 Subject: [PATCH 49/56] Added width_pixel and height_pixel parameters to Channel.get_pty() and resize_pty(), and Client.invoke_shell(). Perhaps useless, but more RFC compliant. Updated methods to include these parameters in server messages. Adjusted Channel.resize_pty() to neither request nor wait for a response, as per RFC 4254 6.7 (A response SHOULD NOT be sent to this message.) This is necessary as certain hosts have been observed to not acknowledge this type of channel request (Cisco IOS XR), which causes paramiko to end the session. --- paramiko/channel.py | 24 ++++++++++++++++-------- paramiko/client.py | 9 +++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/paramiko/channel.py b/paramiko/channel.py index 534f8d7..0c603c6 100644 --- a/paramiko/channel.py +++ b/paramiko/channel.py @@ -122,7 +122,8 @@ class Channel (object): out += '>' return out - def get_pty(self, term='vt100', width=80, height=24): + def get_pty(self, term='vt100', width=80, height=24, width_pixels=0, + height_pixels=0): """ Request a pseudo-terminal from the server. This is usually used right after creating a client channel, to ask the server to provide some @@ -136,6 +137,10 @@ class Channel (object): @type width: int @param height: height (in characters) of the terminal screen @type height: int + @param width_pixels: width (in pixels) of the terminal screen + @type width_pixels: int + @param height_pixels: height (in pixels) of the terminal screen + @type height_pixels: int @raise SSHException: if the request was rejected or the channel was closed @@ -150,8 +155,8 @@ class Channel (object): m.add_string(term) m.add_int(width) m.add_int(height) - # pixel height, width (usually useless) - m.add_int(0).add_int(0) + m.add_int(width_pixels) + m.add_int(height_pixels) m.add_string('') self._event_pending() self.transport._send_user_message(m) @@ -239,7 +244,7 @@ class Channel (object): self.transport._send_user_message(m) self._wait_for_event() - def resize_pty(self, width=80, height=24): + def resize_pty(self, width=80, height=24, width_pixels=0, height_pixels=0): """ Resize the pseudo-terminal. This can be used to change the width and height of the terminal emulation created in a previous L{get_pty} call. @@ -248,6 +253,10 @@ class Channel (object): @type width: int @param height: new height (in characters) of the terminal screen @type height: int + @param width_pixels: new width (in pixels) of the terminal screen + @type width_pixels: int + @param height_pixels: new height (in pixels) of the terminal screen + @type height_pixels: int @raise SSHException: if the request was rejected or the channel was closed @@ -258,13 +267,12 @@ class Channel (object): m.add_byte(chr(MSG_CHANNEL_REQUEST)) m.add_int(self.remote_chanid) m.add_string('window-change') - m.add_boolean(True) + m.add_boolean(False) m.add_int(width) m.add_int(height) - m.add_int(0).add_int(0) - self._event_pending() + m.add_int(width_pixels) + m.add_int(height_pixels) self.transport._send_user_message(m) - self._wait_for_event() def exit_status_ready(self): """ diff --git a/paramiko/client.py b/paramiko/client.py index f863806..5b71958 100644 --- a/paramiko/client.py +++ b/paramiko/client.py @@ -377,7 +377,8 @@ class SSHClient (object): stderr = chan.makefile_stderr('rb', bufsize) return stdin, stdout, stderr - def invoke_shell(self, term='vt100', width=80, height=24): + def invoke_shell(self, term='vt100', width=80, height=24, width_pixels=0, + height_pixels=0): """ Start an interactive shell session on the SSH server. A new L{Channel} is opened and connected to a pseudo-terminal using the requested @@ -389,13 +390,17 @@ class SSHClient (object): @type width: int @param height: the height (in characters) of the terminal window @type height: int + @param width_pixels: the width (in pixels) of the terminal window + @type width_pixels: int + @param height_pixels: the height (in pixels) of the terminal window + @type height_pixels: int @return: a new channel connected to the remote shell @rtype: L{Channel} @raise SSHException: if the server fails to invoke a shell """ chan = self._transport.open_session() - chan.get_pty(term, width, height) + chan.get_pty(term, width, height, width_pixels, height_pixels) chan.invoke_shell() return chan From dd7edd8ec85e339b2ad6dd7d64abf25c8396b0ea Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 1 Mar 2013 10:34:35 -0800 Subject: [PATCH 50/56] Changelog re #133, and date fix for 1.10 --- NEWS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index e30e433..3f6add6 100644 --- a/NEWS +++ b/NEWS @@ -12,9 +12,13 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== -v1.10.0 (DD MM YYYY) +v1.10.0 (1st Mar 2013) -------------------- +* #133: Fix handling of window-change events to be on-spec and not + attempt to wait for a response from the remote sshd; this fixes problems with + less common targets such as some Cisco devices. Thanks to Phillip Heller for + catch & patch. * #93: Overhaul SSH config parsing to be in line with `man ssh_config` (& the behavior of `ssh` itself), including addition of parameter expansion within config values. Thanks to Olle Lundberg for the patch. From bd1a97a0453afb07466ff3cacc2df9b3ee065224 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Fri, 1 Mar 2013 18:09:28 +0100 Subject: [PATCH 51/56] Speed up the write operation by bulk calling read. Bulk check the ACKs from the server every 32MB (or every write request). This way you gain speed but also making sure not to get the error too late in a large transfer. This works for smaller files too, since there is a cleanup routine being called when the file has been transfered. --- paramiko/sftp_file.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index d4ecb89..e056d70 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -21,6 +21,7 @@ L{SFTPFile} """ from binascii import hexlify +from collections import deque import socket import threading import time @@ -54,6 +55,7 @@ class SFTPFile (BufferedFile): self._prefetch_data = {} self._prefetch_reads = [] self._saved_exception = None + self._reqs = deque() def __del__(self): self._close(async=True) @@ -163,12 +165,14 @@ class SFTPFile (BufferedFile): def _write(self, data): # may write less than requested if it would exceed max packet size chunk = min(len(data), self.MAX_REQUEST_SIZE) - req = self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk])) - if not self.pipelined or self.sftp.sock.recv_ready(): - t, msg = self.sftp._read_response(req) - if t != CMD_STATUS: - raise SFTPError('Expected status') - # convert_status already called + self._reqs.append(self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk]))) + if not self.pipelined or (len(self._reqs) > 100 and self.sftp.sock.recv_ready()): + while len(self._reqs): + req = self._reqs.popleft() + t, msg = self.sftp._read_response(req) + if t != CMD_STATUS: + raise SFTPError('Expected status') + # convert_status already called return chunk def settimeout(self, timeout): From 721f74d8c240071e444ed1cdfff432c75339e8b2 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 1 Mar 2013 12:01:35 -0800 Subject: [PATCH 52/56] Changelog re #66, re #141. Fixes #66 --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 3f6add6..f34a876 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,8 @@ Releases v1.10.0 (1st Mar 2013) -------------------- +* #66: Batch SFTP writes to help speed up file transfers. Thanks to Olle + Lundberg for the patch. * #133: Fix handling of window-change events to be on-spec and not attempt to wait for a response from the remote sshd; this fixes problems with less common targets such as some Cisco devices. Thanks to Phillip Heller for From 9858ccf207f4d2df5b006fcf114d3ee303789afb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 1 Nov 2012 10:03:46 -0400 Subject: [PATCH 53/56] Remove test for presence of ctypes (assumed present in global imports). --- paramiko/win_pageant.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/paramiko/win_pageant.py b/paramiko/win_pageant.py index d77d58f..b7a3fcf 100644 --- a/paramiko/win_pageant.py +++ b/paramiko/win_pageant.py @@ -29,20 +29,14 @@ import array import platform import ctypes.wintypes -# if you're on windows, you should have one of these, i guess? -# ctypes is part of standard library since Python 2.5 +# if pywin32 is available, use it _has_win32all = False -_has_ctypes = False try: # win32gui is preferred over win32ui to avoid MFC dependencies import win32gui _has_win32all = True except ImportError: - try: - import ctypes - _has_ctypes = True - except ImportError: - pass + pass _AGENT_COPYDATA_ID = 0x804e50ba _AGENT_MAX_MSGLEN = 8192 @@ -58,7 +52,7 @@ def _get_pageant_window_object(): return hwnd except win32gui.error: pass - elif _has_ctypes: + else: # Return 0 if there is no Pageant window. return ctypes.windll.user32.FindWindowA('Pageant', 'Pageant') return None @@ -71,9 +65,7 @@ def can_talk_to_agent(): This checks both if we have the required libraries (win32all or ctypes) and if there is a Pageant currently running. """ - if (_has_win32all or _has_ctypes) and _get_pageant_window_object(): - return True - return False + return bool(_get_pageant_window_object()) ULONG_PTR = ctypes.c_uint64 if platform.architecture()[0] == '64bit' else ctypes.c_uint32 class COPYDATASTRUCT(ctypes.Structure): @@ -114,10 +106,8 @@ def _query_pageant(msg): # win32gui.SendMessage should also allow the same pattern as # ctypes, but let's keep it like this for now... response = win32gui.SendMessage(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.addressof(cds)) - elif _has_ctypes: - response = ctypes.windll.user32.SendMessageA(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) else: - response = 0 + response = ctypes.windll.user32.SendMessageA(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) if response > 0: datalen = pymap.read(4) From c3056914926e0940ed5716fcdaf2f4a0a8b71fb7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 1 Nov 2012 10:09:41 -0400 Subject: [PATCH 54/56] Remove dependency on pywin32. Just use ctypes for simplicity. --- paramiko/win_pageant.py | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/paramiko/win_pageant.py b/paramiko/win_pageant.py index b7a3fcf..2c9ac99 100644 --- a/paramiko/win_pageant.py +++ b/paramiko/win_pageant.py @@ -29,15 +29,6 @@ import array import platform import ctypes.wintypes -# if pywin32 is available, use it -_has_win32all = False -try: - # win32gui is preferred over win32ui to avoid MFC dependencies - import win32gui - _has_win32all = True -except ImportError: - pass - _AGENT_COPYDATA_ID = 0x804e50ba _AGENT_MAX_MSGLEN = 8192 # Note: The WM_COPYDATA value is pulled from win32con, as a workaround @@ -46,16 +37,7 @@ win32con_WM_COPYDATA = 74 def _get_pageant_window_object(): - if _has_win32all: - try: - hwnd = win32gui.FindWindow('Pageant', 'Pageant') - return hwnd - except win32gui.error: - pass - else: - # Return 0 if there is no Pageant window. - return ctypes.windll.user32.FindWindowA('Pageant', 'Pageant') - return None + return ctypes.windll.user32.FindWindowA('Pageant', 'Pageant') def can_talk_to_agent(): @@ -102,12 +84,7 @@ def _query_pageant(msg): # Create a string to use for the SendMessage function call cds = COPYDATASTRUCT(_AGENT_COPYDATA_ID, char_buffer_size, char_buffer_address) - if _has_win32all: - # win32gui.SendMessage should also allow the same pattern as - # ctypes, but let's keep it like this for now... - response = win32gui.SendMessage(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.addressof(cds)) - else: - response = ctypes.windll.user32.SendMessageA(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) + response = ctypes.windll.user32.SendMessageA(hwnd, win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) if response > 0: datalen = pymap.read(4) From 0cc6bb970f54467841084f5a6199985d0ec045b1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 2 Dec 2012 06:52:37 -0500 Subject: [PATCH 55/56] Updated NEWS --- NEWS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index f34a876..5d989f0 100644 --- a/NEWS +++ b/NEWS @@ -50,6 +50,10 @@ v1.10.0 (1st Mar 2013) * #80: Expose the internal "is closed" property of the file transfer class `BufferedFile` as `.closed`, better conforming to Python's file interface. Thanks to `@smunaut` and James Hiscock for catch & patch. +* #100: Remove use of PyWin32 in `win_pageant` module. Module was already + dependent on ctypes for constructing appropriate structures and had ctypes + implementations of all functionality. Thanks to Jason R. Coombs for the + patch. v1.9.0 (6th Nov 2012) --------------------- @@ -325,7 +329,7 @@ v1.5 (paras) 02oct05 separation * demo scripts fixed to have a better chance of loading the host keys correctly on windows/cygwin - + v1.4 (oddish) 17jul05 --------------------- * added SSH-agent support (for posix) from john rochester From d5db60329701df2423909773af9cc9aa5df4d4f6 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 19 Mar 2013 13:25:25 -0700 Subject: [PATCH 56/56] Move changelog re #100 to new release chunk --- NEWS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 5d989f0..b99645d 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,13 @@ Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. Releases ======== +v1.11.0 (DD MM YYYY) +-------------------- +* #100: Remove use of PyWin32 in `win_pageant` module. Module was already + dependent on ctypes for constructing appropriate structures and had ctypes + implementations of all functionality. Thanks to Jason R. Coombs for the + patch. + v1.10.0 (1st Mar 2013) -------------------- @@ -50,10 +57,6 @@ v1.10.0 (1st Mar 2013) * #80: Expose the internal "is closed" property of the file transfer class `BufferedFile` as `.closed`, better conforming to Python's file interface. Thanks to `@smunaut` and James Hiscock for catch & patch. -* #100: Remove use of PyWin32 in `win_pageant` module. Module was already - dependent on ctypes for constructing appropriate structures and had ctypes - implementations of all functionality. Thanks to Jason R. Coombs for the - patch. v1.9.0 (6th Nov 2012) ---------------------