From 3174b6c894b125426a7a4bae7f934a0dbe64d5b1 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:52:21 +0200 Subject: [PATCH 01/12] Updated tests for new ssh config format. --- tests/test_util.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 7e56245..6d68af7 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -112,23 +112,32 @@ class UtilTest (unittest.TestCase): f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) self.assertEquals(config._config, - [ {'identityfile': '~/.ssh/id_rsa', 'host': '*', 'user': 'robey', - 'crazy': 'something dumb '}, - {'host': '*.example.com', 'user': 'bjork', 'port': '3333'}, - {'host': 'spoo.example.com', 'crazy': 'something else'}]) + [{'host': ['*'], 'config': {}}, {'host': ['*'], 'config': {'identityfile': ['~/.ssh/id_rsa'], 'user': 'robey'}}, + {'host': ['*.example.com'], 'config': {'user': 'bjork', 'port': '3333'}}, + {'host': ['*'], 'config': {'crazy': 'something dumb '}}, + {'host': ['spoo.example.com'], 'config': {'crazy': 'something else'}}]) def test_3_host_config(self): global test_config_file f = cStringIO.StringIO(test_config_file) config = paramiko.util.parse_ssh_config(f) + for host, values in { - 'irc.danger.com': {'user': 'robey', 'crazy': 'something dumb '}, - 'irc.example.com': {'user': 'bjork', 'crazy': 'something dumb ', 'port': '3333'}, - 'spoo.example.com': {'user': 'bjork', 'crazy': 'something else', 'port': '3333'} + 'irc.danger.com': {'crazy': 'something dumb ', + 'hostname': 'irc.danger.com', + 'user': 'robey'}, + 'irc.example.com': {'crazy': 'something dumb ', + 'hostname': 'irc.example.com', + 'user': 'robey', + 'port': '3333'}, + 'spoo.example.com': {'crazy': 'something dumb ', + 'hostname': 'spoo.example.com', + 'user': 'robey', + 'port': '3333'} }.items(): values = dict(values, hostname=host, - identityfile=os.path.expanduser("~/.ssh/id_rsa") + identityfile=[os.path.expanduser("~/.ssh/id_rsa")] ) self.assertEquals( paramiko.util.lookup_ssh_host_config(host, config), @@ -159,7 +168,7 @@ class UtilTest (unittest.TestCase): # just verify that we can pull out 32 bytes and not get an exception. x = rng.read(32) self.assertEquals(len(x), 32) - + def test_7_host_config_expose_issue_33(self): test_config_file = """ Host www13.* From f33481cc44fb57a788b5726b529383f300d06b36 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:53:06 +0200 Subject: [PATCH 02/12] Add test for host negation. --- tests/test_util.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 6d68af7..9890cfc 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -212,3 +212,26 @@ Host * raise AssertionError('foo') self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) + + + def test_9_host_config_test_negation(self): + test_config_file = """ +Host www13.* !*.example.com + Port 22 + +Host *.example.com !www13.* + Port 2222 + +Host www13.* + Port 8080 + +Host * + Port 3333 + """ + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + host = 'www13.example.com' + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + {'hostname': host, 'port': '8080'} + ) From ad587fa0ef32eeeb633245c7ae68759765c0e439 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 13:54:23 +0200 Subject: [PATCH 03/12] Add host negation support to paramiko config. This is a rewrite of the SSHConfig class to conform with the rules specified by the manpage for ssh_config. This change also adds support for negation according to the rules introduced by OpenSSH 5.9. Reference: http://www.openssh.com/txt/release-5.9 --- paramiko/config.py | 80 +++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 458d5dd..802d915 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -1,4 +1,5 @@ # Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg # # This file is part of paramiko. # @@ -41,7 +42,8 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ - self._config = [ { 'host': '*' } ] + self._config = [] + def parse(self, file_obj): """ @@ -50,7 +52,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - configs = [self._config[0]] + host = {"host":['*'],"config":{}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -69,20 +71,20 @@ class SSHConfig (object): value = line[i:].lstrip() if key == 'host': - del configs[:] - # the value may be multiple hosts, space-delimited - for host in value.split(): - # do we have a pre-existing host config to append to? - matches = [c for c in self._config if c['host'] == host] - if len(matches) > 0: - configs.append(matches[0]) - else: - config = { 'host': host } - self._config.append(config) - configs.append(config) - else: - for config in configs: - config[key] = value + self._config.append(host) + value = value.split() + host = {key:value,'config':{}} + #identitifile is a special case, since it is allowed to be + # specified multiple times and they should be tried in order + # of specification. + elif key == 'identityfile': + if key in host['config']: + host['config']['identityfile'].append(value) + else: + host['config']['identityfile'] = [value] + elif key not in host['config']: + host['config'].update({key:value}) + self._config.append(host) def lookup(self, hostname): """ @@ -97,31 +99,45 @@ class SSHConfig (object): will win out. The keys in the returned dict are all normalized to lowercase (look for - C{"port"}, not C{"Port"}. No other processing is done to the keys or - values. + C{"port"}, not C{"Port"}. The values are processed according to the + rules for substitution variable expansion in C{ssh_config}. @param hostname: the hostname to lookup @type hostname: str """ - matches = [x for x in self._config if fnmatch.fnmatch(hostname, x['host'])] - # Move * to the end - _star = matches.pop(0) - matches.append(_star) + + matches = [ config for config in self._config if + self._allowed(hostname,config['host']) ] + ret = {} - for m in matches: - for k,v in m.iteritems(): - if not k in ret: - ret[k] = v + for match in matches: + for key in match['config']: + value = match['config'][key] + if key == 'identityfile': + if key in ret: + ret['identityfile'].extend(value) + else: + ret['identityfile'] = value + elif key not in ret: + ret[key] = value ret = self._expand_variables(ret, hostname) - del ret['host'] return ret - def _expand_variables(self, config, hostname ): + def _allowed(self, hostname, hosts): + match = False + for host in hosts: + if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]): + return False + elif fnmatch.fnmatch(hostname, host): + match = True + return match + + def _expand_variables(self, config, hostname): """ Return a dict of config options with expanded substitutions for a given hostname. - Please refer to man ssh_config(5) for the parameters that + Please refer to man C{ssh_config} for the parameters that are replaced. @param config: the config for the hostname @@ -172,5 +188,9 @@ class SSHConfig (object): for k in config: if k in replacements: for find, replace in replacements[k]: - config[k] = config[k].replace(find, str(replace)) + if isinstance(config[k],list): + for item in range(len(config[k])): + config[k][item] = config[k][item].replace(find, str(replace)) + else: + config[k] = config[k].replace(find, str(replace)) return config From 2dd74f953d0a789a9c65662492f8340d66246e5b Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 14:52:27 +0200 Subject: [PATCH 04/12] Spelling --- paramiko/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 802d915..baf1040 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -74,7 +74,7 @@ class SSHConfig (object): self._config.append(host) value = value.split() host = {key:value,'config':{}} - #identitifile is a special case, since it is allowed to be + #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. elif key == 'identityfile': From b22c11ab1b889032345a80b89af6ebee50ef7904 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 15:00:02 +0200 Subject: [PATCH 05/12] Pep8 fixes --- paramiko/config.py | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index baf1040..26a94a1 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -25,7 +25,8 @@ import fnmatch import os import socket -SSH_PORT=22 +SSH_PORT = 22 + class SSHConfig (object): """ @@ -44,7 +45,6 @@ class SSHConfig (object): """ self._config = [] - def parse(self, file_obj): """ Read an OpenSSH config from the given file object. @@ -52,7 +52,7 @@ class SSHConfig (object): @param file_obj: a file-like object to read the config file from @type file_obj: file """ - host = {"host":['*'],"config":{}} + host = {"host": ['*'], "config": {}} for line in file_obj: line = line.rstrip('\n').lstrip() if (line == '') or (line[0] == '#'): @@ -73,7 +73,7 @@ class SSHConfig (object): if key == 'host': self._config.append(host) value = value.split() - host = {key:value,'config':{}} + host = {key: value, 'config': {}} #identityfile is a special case, since it is allowed to be # specified multiple times and they should be tried in order # of specification. @@ -83,7 +83,7 @@ class SSHConfig (object): else: host['config']['identityfile'] = [value] elif key not in host['config']: - host['config'].update({key:value}) + host['config'].update({key: value}) self._config.append(host) def lookup(self, hostname): @@ -106,8 +106,8 @@ class SSHConfig (object): @type hostname: str """ - matches = [ config for config in self._config if - self._allowed(hostname,config['host']) ] + matches = [config for config in self._config if + self._allowed(hostname, config['host'])] ret = {} for match in matches: @@ -147,7 +147,7 @@ class SSHConfig (object): """ if 'hostname' in config: - config['hostname'] = config['hostname'].replace('%h',hostname) + config['hostname'] = config['hostname'].replace('%h', hostname) else: config['hostname'] = hostname @@ -165,32 +165,32 @@ class SSHConfig (object): host = socket.gethostname().split('.')[0] fqdn = socket.getfqdn() homedir = os.path.expanduser('~') - replacements = {'controlpath' : - [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile' : - [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ] - } + replacements = {'controlpath': + [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': + [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ]} for k in config: if k in replacements: for find, replace in replacements[k]: - if isinstance(config[k],list): + if isinstance(config[k], list): for item in range(len(config[k])): - config[k][item] = config[k][item].replace(find, str(replace)) + config[k][item] = config[k][item].\ + replace(find, str(replace)) else: config[k] = config[k].replace(find, str(replace)) return config From d66d75f277cfd49b8adc84498b0b09d71092bd0b Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:09 +0200 Subject: [PATCH 06/12] Add tests for proxycommand parsing. --- tests/test_util.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 9890cfc..d1776db 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -235,3 +235,30 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) + def test_10_host_config_test_proxycommand(self): + test_config_file = """ +Host proxy-with-equal-divisor-and-space +ProxyCommand = foo=bar + +Host proxy-with-equal-divisor-and-no-space +ProxyCommand=foo=bar + +Host proxy-without-equal-divisor +ProxyCommand foo=bar:%h-%p + """ + for host, values in { + 'proxy-with-equal-divisor-and-space' :{'hostname': 'proxy-with-equal-divisor-and-space', + 'proxycommand': 'foo=bar'}, + 'proxy-with-equal-divisor-and-no-space':{'hostname': 'proxy-with-equal-divisor-and-no-space', + 'proxycommand': 'foo=bar'}, + 'proxy-without-equal-divisor' :{'hostname': 'proxy-without-equal-divisor', + 'proxycommand': + 'foo=bar:proxy-without-equal-divisor-22'} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From 7ce9875ed7d3e3738ed775ffda4ded1ce12d35f2 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:38:38 +0200 Subject: [PATCH 07/12] Implement support for parsing proxycommand. --- paramiko/config.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 26a94a1..832f42e 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -23,6 +23,7 @@ L{SSHConfig}. import fnmatch import os +import re import socket SSH_PORT = 22 @@ -43,6 +44,7 @@ class SSHConfig (object): """ Create a new OpenSSH config object. """ + self._proxyregex = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) self._config = [] def parse(self, file_obj): @@ -58,8 +60,15 @@ class SSHConfig (object): if (line == '') or (line[0] == '#'): continue if '=' in line: - key, value = line.split('=', 1) - key = key.strip().lower() + if not line.lower().startswith('proxycommand'): + key, value = line.split('=', 1) + key = key.strip().lower() + else: + #ProxyCommand have been specified with an equal + # sign. Eat that and split in two groups. + match = self._proxyregex.match(line) + key = match.group(1).lower() + value = match.group(2) else: # find first whitespace, and split there i = 0 @@ -183,7 +192,14 @@ class SSHConfig (object): ('%l', fqdn), ('%u', user), ('%r', remoteuser) - ]} + ], + 'proxycommand': + [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser) + ] + } for k in config: if k in replacements: for find, replace in replacements[k]: From 04cc4d55102c111d3e054ef8d83a0a62c34de561 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 16:54:44 +0200 Subject: [PATCH 08/12] Be more pythonic. --- paramiko/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index 832f42e..bd1c2f6 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -120,8 +120,7 @@ class SSHConfig (object): ret = {} for match in matches: - for key in match['config']: - value = match['config'][key] + for key, value in match['config'].iteritems(): if key == 'identityfile': if key in ret: ret['identityfile'].extend(value) From 221131fa21e624a30d0d7cd5ed29e2749ddfd246 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 16 Oct 2012 17:02:04 +0200 Subject: [PATCH 09/12] Whitespace fixes. --- tests/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util.py b/tests/test_util.py index d1776db..9cbc439 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -213,7 +213,6 @@ Host * self.assertRaises(AssertionError, lambda: paramiko.util.retry_on_signal(raises_other_exception)) - def test_9_host_config_test_negation(self): test_config_file = """ Host www13.* !*.example.com @@ -235,6 +234,7 @@ Host * paramiko.util.lookup_ssh_host_config(host, config), {'hostname': host, 'port': '8080'} ) + def test_10_host_config_test_proxycommand(self): test_config_file = """ Host proxy-with-equal-divisor-and-space From 78654e82ecfceba30c61974b18fd4809b1ea9899 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 00:45:32 +0100 Subject: [PATCH 10/12] DRY up the code for populating the return list --- paramiko/config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/paramiko/config.py b/paramiko/config.py index bd1c2f6..135e009 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -121,13 +121,10 @@ class SSHConfig (object): ret = {} for match in matches: for key, value in match['config'].iteritems(): - if key == 'identityfile': - if key in ret: - ret['identityfile'].extend(value) - else: - ret['identityfile'] = value - elif key not in ret: + if key not in ret: ret[key] = value + elif key == 'identityfile': + ret[key].extend(value) ret = self._expand_variables(ret, hostname) return ret From 5670e111c95612fc1c96135ca0e942ed89ec5420 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:42:29 +0100 Subject: [PATCH 11/12] Add tests for identityfile parsing. --- tests/test_util.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index 9cbc439..9a155bf 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -262,3 +262,33 @@ ProxyCommand foo=bar:%h-%p paramiko.util.lookup_ssh_host_config(host, config), values ) + + def test_11_host_config_test_identityfile(self): + test_config_file = """ + +IdentityFile id_dsa0 + +Host * +IdentityFile id_dsa1 + +Host dsa2 +IdentityFile id_dsa2 + +Host dsa2* +IdentityFile id_dsa22 + """ + for host, values in { + 'foo' :{'hostname': 'foo', + 'identityfile': ['id_dsa0', 'id_dsa1']}, + 'dsa2' :{'hostname': 'dsa2', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa2', 'id_dsa22']}, + 'dsa22' :{'hostname': 'dsa22', + 'identityfile': ['id_dsa0', 'id_dsa1', 'id_dsa22']} + }.items(): + + f = cStringIO.StringIO(test_config_file) + config = paramiko.util.parse_ssh_config(f) + self.assertEquals( + paramiko.util.lookup_ssh_host_config(host, config), + values + ) From a07a339006259eb7ad60993415493d2d25a760d3 Mon Sep 17 00:00:00 2001 From: Olle Lundberg Date: Tue, 20 Nov 2012 12:43:40 +0100 Subject: [PATCH 12/12] Create a copy of the identityfile list. The copy is needed else the original identityfile list is in the internal config list is updated when we modify the return dictionary. --- paramiko/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paramiko/config.py b/paramiko/config.py index 135e009..c5a5804 100644 --- a/paramiko/config.py +++ b/paramiko/config.py @@ -122,7 +122,11 @@ class SSHConfig (object): for match in matches: for key, value in match['config'].iteritems(): if key not in ret: - ret[key] = value + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + ret[key] = value[:] elif key == 'identityfile': ret[key].extend(value) ret = self._expand_variables(ret, hostname)