From 974294ad7d083c49bffe9189cac0d5b08984b1ad Mon Sep 17 00:00:00 2001 From: Richard Kettlewell Date: Mon, 29 Aug 2011 15:50:35 +0100 Subject: [PATCH 01/33] Fix issue 34 (SFTPFile prefetch assumes response order matches requests) SFTPFile._async_response gets a new 'num' parameter giving the request number. This can be matched up with the return value of SFTPClient._async_request() to retrieve data specific to that request. The prefetch queue SFTPFile._prefetch_reads is replaced with the dict _prefetch_extents, which maps request numbers to (offset,length) tuples. A lock is used to exclude the case where a response arrives in _async_response before _prefetch_thread has updated it. --- paramiko/sftp_client.py | 2 +- paramiko/sftp_file.py | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/paramiko/sftp_client.py b/paramiko/sftp_client.py index 79a7761..189caa4 100644 --- a/paramiko/sftp_client.py +++ b/paramiko/sftp_client.py @@ -682,7 +682,7 @@ class SFTPClient (BaseSFTP): self._convert_status(msg) return t, msg if fileobj is not type(None): - fileobj._async_response(t, msg) + fileobj._async_response(t, msg, num) if waitfor is None: # just doing a single check break diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index 8c5c7ac..2d4d431 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -49,7 +49,8 @@ class SFTPFile (BufferedFile): self._prefetching = False self._prefetch_done = False self._prefetch_data = {} - self._prefetch_reads = [] + self._prefetch_extents = {} + self._prefetch_lock = threading.Lock() self._saved_exception = None def __del__(self): @@ -86,7 +87,7 @@ class SFTPFile (BufferedFile): pass def _data_in_prefetch_requests(self, offset, size): - k = [i for i in self._prefetch_reads if i[0] <= offset] + k = [self._prefetch_extents[i] for i in self._prefetch_extents if self._prefetch_extents[i][0] <= offset] if len(k) == 0: return False k.sort(lambda x, y: cmp(x[0], y[0])) @@ -440,7 +441,6 @@ class SFTPFile (BufferedFile): def _start_prefetch(self, chunks): self._prefetching = True self._prefetch_done = False - self._prefetch_reads.extend(chunks) t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) t.setDaemon(True) @@ -450,9 +450,11 @@ class SFTPFile (BufferedFile): # do these read requests in a temporary thread because there may be # a lot of them, so it may block. for offset, length in chunks: - self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) + with self._prefetch_lock: + num = self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) + self._prefetch_extents[num] = (offset, length) - def _async_response(self, t, msg): + def _async_response(self, t, msg, num): if t == CMD_STATUS: # save exception and re-raise it on next file operation try: @@ -463,10 +465,12 @@ class SFTPFile (BufferedFile): if t != CMD_DATA: raise SFTPError('Expected data') data = msg.get_string() - offset, length = self._prefetch_reads.pop(0) - self._prefetch_data[offset] = data - if len(self._prefetch_reads) == 0: - self._prefetch_done = True + with self._prefetch_lock: + offset,length = self._prefetch_extents[num] + self._prefetch_data[offset] = data + del self._prefetch_extents[num] + if len(self._prefetch_extents) == 0: + self._prefetch_done = True def _check_exception(self): "if there's a saved exception, raise & clear it" From 87cd72c144c54fc9c6f05ff7482bd6e9e7622730 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 23 Jan 2014 11:43:34 -0800 Subject: [PATCH 02/33] Set up Intersphinx so www can ref docs --- sites/shared_conf.py | 2 +- sites/www/conf.py | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/sites/shared_conf.py b/sites/shared_conf.py index 89e0a56..c48dcdc 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -42,7 +42,7 @@ html_sidebars = { # Regular settings project = u'Paramiko' year = datetime.now().year -copyright = u'%d Jeff Forcier, 2003-2012 Robey Pointer' % year +copyright = u'2013-%d Jeff Forcier, 2003-2012 Robey Pointer' % year master_doc = 'index' templates_path = ['_templates'] exclude_trees = ['_build'] diff --git a/sites/www/conf.py b/sites/www/conf.py index c144b5b..b2f9618 100644 --- a/sites/www/conf.py +++ b/sites/www/conf.py @@ -1,15 +1,30 @@ # Obtain shared config values -import os, sys -sys.path.append(os.path.abspath('..')) +import sys +import os +from os.path import abspath, join, dirname + +sys.path.append(abspath(join(dirname(__file__), '..'))) from shared_conf import * -# Add local blog extension -sys.path.append(os.path.abspath('.')) +# Local blog extension +sys.path.append(abspath('.')) extensions = ['blog'] rss_link = 'http://paramiko.org' rss_description = 'Paramiko project news' -# Add Releases changelog extension +# Releases changelog extension extensions.append('releases') releases_release_uri = "https://github.com/paramiko/paramiko/tree/%s" releases_issue_uri = "https://github.com/paramiko/paramiko/issues/%s" + +# Intersphinx for referencing API/usage docs +extensions.append('sphinx.ext.intersphinx') +# Default is 'local' building, but reference the public docs site when building +# under RTD. +target = join(dirname(__file__), '..', 'docs', '_build') +if os.environ.get('READTHEDOCS') == 'True': + # TODO: switch to docs.paramiko.org post go-live + target = 'http://paramiko-docs.readthedocs.org/en/latest/' +intersphinx_mapping = { + 'docs': (target, None), +} From 69d40710dc55373f9bbeb9f2721c6942a5250bbd Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 08:46:21 -0800 Subject: [PATCH 03/33] Tweak logo settings (& rely on new alabaster defaults) --- dev-requirements.txt | 2 +- sites/shared_conf.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index a2b9a4e..a967765 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,5 +3,5 @@ invoke>=0.6.1 invocations>=0.4.4 sphinx>=1.1.3 -alabaster>=0.1.0 +alabaster>=0.2.0 releases>=0.2.4 diff --git a/sites/shared_conf.py b/sites/shared_conf.py index c48dcdc..d6852cc 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -11,8 +11,6 @@ html_theme_path = [alabaster.get_path()] html_static_path = ['../_shared_static'] html_theme = 'alabaster' html_theme_options = { - 'logo': 'logo.png', - 'logo_name': 'true', 'description': "A Python implementation of SSHv2.", 'github_user': 'paramiko', 'github_repo': 'paramiko', From da51cd8b14231a3fa9850d0d1b939168ae7b0534 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 12:50:19 -0800 Subject: [PATCH 04/33] Update to new alabaster-driven nav sidebar --- sites/shared_conf.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sites/shared_conf.py b/sites/shared_conf.py index d6852cc..267b3bc 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -17,21 +17,18 @@ html_theme_options = { 'gittip_user': 'bitprophet', 'analytics_id': 'UA-18486793-2', + 'extra_nav_links': { + "API Docs": 'http://docs.paramiko.org', + }, + 'link': '#3782BE', 'link_hover': '#3782BE', } html_sidebars = { - # Landing page (no ToC) - 'index': [ - 'about.html', - 'searchbox.html', - 'donate.html', - ], - # Inner pages get a ToC '**': [ 'about.html', - 'localtoc.html', + 'navigation.html', 'searchbox.html', 'donate.html', ] From 7b690707dd81a80c7550c1f537b870805418d6df Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 12:50:24 -0800 Subject: [PATCH 05/33] Don't display blog for now --- sites/www/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/sites/www/index.rst b/sites/www/index.rst index f8db6fd..43c060b 100644 --- a/sites/www/index.rst +++ b/sites/www/index.rst @@ -12,7 +12,6 @@ usage and API documentation can be found at our code documentation site, `docs.paramiko.org `_. .. toctree:: - blog changelog installing contributing From 414fec8f273e6807a8ee97253a5a82d0c8b787ab Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 15:12:33 -0800 Subject: [PATCH 06/33] Bump Invoke requirement to newly released version w/ better Collection.from_module --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index a967765..2fe1e77 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # For newer tasks like building Sphinx docs. # NOTE: Requires Python >=2.6 -invoke>=0.6.1 +invoke>=0.7.0 invocations>=0.4.4 sphinx>=1.1.3 alabaster>=0.2.0 From fee04a39ec45baefff84c8a58540fa3401b4a826 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 15:23:11 -0800 Subject: [PATCH 07/33] Add sites/docs building to Travis --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 29e44e5..98a5482 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,12 @@ install: # Dev (doc/test running) requirements - pip install coveralls # For coveralls.io specifically - pip install -r dev-requirements.txt -script: coverage run --source=paramiko test.py --verbose +script: + # Main tests, with coverage! + - coverage run --source=paramiko test.py --verbose + # Ensure documentation & invoke pipeline run OK + - invoke www + - invoke docs notifications: irc: channels: "irc.freenode.org#paramiko" From d1d65b4ddde17a97d25a79aeaad6c3068846cb36 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 15:53:26 -0800 Subject: [PATCH 08/33] Stricter/more useful doc builds in Travis --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 98a5482..df7c225 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,9 +11,12 @@ install: script: # Main tests, with coverage! - coverage run --source=paramiko test.py --verbose - # Ensure documentation & invoke pipeline run OK - - invoke www - - invoke docs + # Ensure documentation & invoke pipeline run OK. + # Run 'docs' first since its objects.inv is referred to by 'www'. + # Also force warnings to be errors since most of them tend to be actual + # problems. + - invoke docs -o -W + - invoke www -o -W notifications: irc: channels: "irc.freenode.org#paramiko" From b60075c7cd2fb4dd67701382f953e454d3db1848 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 28 Jan 2014 16:00:03 -0800 Subject: [PATCH 09/33] Prevent warning about unlinked blog page --- sites/www/index.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sites/www/index.rst b/sites/www/index.rst index 43c060b..7fefedd 100644 --- a/sites/www/index.rst +++ b/sites/www/index.rst @@ -17,6 +17,13 @@ usage and API documentation can be found at our code documentation site, contributing contact +.. Hide blog in hidden toctree for now (to avoid warnings.) + +.. toctree:: + :hidden: + + blog + .. rubric:: Footnotes From 2b0f834c1660c8865300e27b46a85de2edf4d556 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 29 Jan 2014 14:23:46 -0800 Subject: [PATCH 10/33] Move API doc sister link to www site only, derp --- sites/shared_conf.py | 5 ----- sites/www/conf.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sites/shared_conf.py b/sites/shared_conf.py index 267b3bc..2b98654 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -17,13 +17,8 @@ html_theme_options = { 'gittip_user': 'bitprophet', 'analytics_id': 'UA-18486793-2', - 'extra_nav_links': { - "API Docs": 'http://docs.paramiko.org', - }, - 'link': '#3782BE', 'link_hover': '#3782BE', - } html_sidebars = { '**': [ diff --git a/sites/www/conf.py b/sites/www/conf.py index b2f9618..58132d7 100644 --- a/sites/www/conf.py +++ b/sites/www/conf.py @@ -28,3 +28,8 @@ if os.environ.get('READTHEDOCS') == 'True': intersphinx_mapping = { 'docs': (target, None), } + +# Sister-site links to API docs +html_theme_options['extra_nav_links'] = { + "API Docs": 'http://docs.paramiko.org', +} From b8d1724f5714c27ad02ae013e87f5531aec041ea Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 29 Jan 2014 14:24:54 -0800 Subject: [PATCH 11/33] Comment out intersphinx pending #256 --- sites/www/conf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sites/www/conf.py b/sites/www/conf.py index 58132d7..1f11c1e 100644 --- a/sites/www/conf.py +++ b/sites/www/conf.py @@ -23,11 +23,11 @@ extensions.append('sphinx.ext.intersphinx') # under RTD. target = join(dirname(__file__), '..', 'docs', '_build') if os.environ.get('READTHEDOCS') == 'True': - # TODO: switch to docs.paramiko.org post go-live + # TODO: switch to docs.paramiko.org post go-live of sphinx API docs target = 'http://paramiko-docs.readthedocs.org/en/latest/' -intersphinx_mapping = { - 'docs': (target, None), -} +#intersphinx_mapping = { +# 'docs': (target, None), +#} # Sister-site links to API docs html_theme_options['extra_nav_links'] = { From 34320dfd61ba97f7f302166365b244400abc0857 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 29 Jan 2014 15:05:52 -0800 Subject: [PATCH 12/33] Forgot to update backticks for Sphinx, derp --- sites/www/changelog.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index bba7894..7134461 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -14,7 +14,7 @@ Changelog * :bug:`179` Fix a missing variable causing errors when an ssh_config file has a non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch & patch. -* :bug:`200` Fix an exception-causing typo in `demo_simple.py`. Thanks to Alex +* :bug:`200` Fix an exception-causing typo in ``demo_simple.py``. Thanks to Alex Buchanan for catch & Dave Foster for patch. * :bug:`199` Typo fix in the license header cross-project. Thanks to Armin Ronacher for catch & patch. @@ -27,7 +27,7 @@ Changelog and 'remoteforward' keys. Thanks to Emre Yılmaz for the patch. * :release:`1.10.2 <2013-07-26>` * :bug:`153` (also :issue:`67`) Warn on parse failure when reading known_hosts - file. Thanks to `@glasserc` for patch. + file. Thanks to ``@glasserc`` for patch. * :bug:`146` Indentation fixes for readability. Thanks to Abhinav Upadhyay for catch & patch. * :release:`1.10.1 <2013-04-05>` @@ -46,32 +46,32 @@ Changelog attempt to wait for a response from the remote sshd; this fixes problems with less common targets such as some Cisco devices. Thanks to Phillip Heller for catch & patch. -* :feature:`93` Overhaul SSH config parsing to be in line with `man - ssh_config` (& the behavior of `ssh` itself), including addition of parameter +* :feature:`93` Overhaul SSH config parsing to be in line with ``man + ssh_config`` (& the behavior of ``ssh`` itself), including addition of parameter expansion within config values. Thanks to Olle Lundberg for the patch. -* :feature:`110` Honor SSH config `AddressFamily` setting when looking up +* :feature:`110` Honor SSH config ``AddressFamily`` setting when looking up local host's FQDN. Thanks to John Hensley for the patch. * :feature:`128` Defer FQDN resolution until needed, when parsing SSH config files. Thanks to Parantapa Bhattacharya for catch & patch. * :bug:`102 major` Forego random padding for packets when running under - `*-ctr` ciphers. This corrects some slowdowns on platforms where random byte - generation is inefficient (e.g. Windows). Thanks to `@warthog618` for catch + ``*-ctr`` ciphers. This corrects some slowdowns on platforms where random byte + generation is inefficient (e.g. Windows). Thanks to ``@warthog618` for catch & patch, and Michael van der Kolff for code/technique review. -* :feature:`127` Turn `SFTPFile` into a context manager. Thanks to Michael +* :feature:`127` Turn ``SFTPFile`` into a context manager. Thanks to Michael Williamson for the patch. -* :feature:`116` Limit `Message.get_bytes` to an upper bound of 1MB to protect - against potential DoS vectors. Thanks to `@mvschaik` for catch & patch. -* :feature:`115` Add convenience `get_pty` kwarg to `Client.exec_command` so +* :feature:`116` Limit ``Message.get_bytes`` to an upper bound of 1MB to protect + against potential DoS vectors. Thanks to ``@mvschaik`` for catch & patch. +* :feature:`115` Add convenience ``get_pty`` kwarg to ``Client.exec_command`` so users not manually controlling a channel object can still toggle PTY creation. Thanks to Michael van der Kolff for the patch. -* :feature:`71` Add `SFTPClient.putfo` and `.getfo` methods to allow direct +* :feature:`71` Add ``SFTPClient.putfo`` and ``.getfo`` methods to allow direct uploading/downloading of file-like objects. Thanks to Eric Buehl for the patch. -* :feature:`113` Add `timeout` parameter to `SSHClient.exec_command` for +* :feature:`113` Add ``timeout`` parameter to ``SSHClient.exec_command`` for easier setting of the command's internal channel object's timeout. Thanks to Cernov Vladimir for the patch. * :support:`94` Remove duplication of SSH port constant. Thanks to Olle Lundberg for the catch. * :feature:`80` Expose the internal "is closed" property of the file transfer - class `BufferedFile` as `.closed`, better conforming to Python's file - interface. Thanks to `@smunaut` and James Hiscock for catch & patch. + class ``BufferedFile`` as ``.closed``, better conforming to Python's file + interface. Thanks to ``@smunaut`` and James Hiscock for catch & patch. From 176493823c4a580bbfd602a19e3fee5d84c5e080 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Wed, 29 Jan 2014 15:16:04 -0800 Subject: [PATCH 13/33] Missed a spot w/ SnR --- sites/www/changelog.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 7134461..eaf380d 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -54,9 +54,9 @@ Changelog * :feature:`128` Defer FQDN resolution until needed, when parsing SSH config files. Thanks to Parantapa Bhattacharya for catch & patch. * :bug:`102 major` Forego random padding for packets when running under - ``*-ctr`` ciphers. This corrects some slowdowns on platforms where random byte - generation is inefficient (e.g. Windows). Thanks to ``@warthog618` for catch - & patch, and Michael van der Kolff for code/technique review. + ``*-ctr`` ciphers. This corrects some slowdowns on platforms where random + byte generation is inefficient (e.g. Windows). Thanks to ``@warthog618`` for + catch & patch, and Michael van der Kolff for code/technique review. * :feature:`127` Turn ``SFTPFile`` into a context manager. Thanks to Michael Williamson for the patch. * :feature:`116` Limit ``Message.get_bytes`` to an upper bound of 1MB to protect From 70bcde95c1df9858c9e7dbe5310c8bfc0b4b5d5b Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 30 Jan 2014 10:55:10 -0800 Subject: [PATCH 14/33] Backport changelog bugfix from 1.12 --- sites/www/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index eaf380d..0539285 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -* :release:`1.10.6 <2014-01-21>` +* :release:`1.10.6 <2014-01-21>` 193 * :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`): Fix SSH agent problems present on Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle Lundberg for the patches. @@ -10,7 +10,7 @@ Changelog * :bug:`176` Fix AttributeError bugs in known_hosts file (re)loading. Thanks to Nathan Scowcroft for the patch & Martin Blumenstingl for the initial test case. -* :release:`1.10.4 <2013-09-27>` +* :release:`1.10.4 <2013-09-27>` 199, 200, 179 * :bug:`179` Fix a missing variable causing errors when an ssh_config file has a non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch & patch. From c563f06954fa7825ee74b9bdba8449b35c975cab Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 30 Jan 2014 10:59:15 -0800 Subject: [PATCH 15/33] Nuke sphinx-changelogg'd material --- NEWS | 98 ++++-------------------------------------------------------- 1 file changed, 5 insertions(+), 93 deletions(-) diff --git a/NEWS b/NEWS index f66a014..761f8e4 100644 --- a/NEWS +++ b/NEWS @@ -9,102 +9,14 @@ Issues noted as "'ssh' #NN" can be found at https://github.com/bitprophet/ssh/. Issues noted as "Fabric #NN" can be found at https://github.com/fabric/fabric/. + +**PLEASE NOTE:** For changes in 1.10.x and newer releases, please see +www.paramiko.org's changelog page, or the source file, sites/www/changelog.rst + + Releases ======== -v1.10.6 (21st Jan 2014) ------------------------ - -* #193 (and its attentant PRs #230 & #253): Fix SSH agent problems present on - Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle - Lundberg for the patches. - -v1.10.5 (8th Jan 2014) ----------------------- - -* #176: Fix AttributeError bugs in known_hosts file (re)loading. Thanks to - Nathan Scowcroft for the patch & Martin Blumenstingl for the initial test - case. - -v1.10.4 (27th Sep 2013) ------------------------ - -* #179: Fix a missing variable causing errors when an ssh_config file has a - non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch - & patch. -* #200: Fix an exception-causing typo in `demo_simple.py`. Thanks to Alex - Buchanan for catch & Dave Foster for patch. -* #199: Typo fix in the license header cross-project. Thanks to Armin Ronacher - for catch & patch. - -v1.10.3 (20th Sep 2013) ------------------------ - -* #162: Clean up HMAC module import to avoid deadlocks in certain uses of - SSHClient. Thanks to Gernot Hillier for the catch & suggested - fix. -* #36: Fix the port-forwarding demo to avoid file descriptor errors. Thanks to - Jonathan Halcrow for catch & patch. -* #168: Update config handling to properly handle multiple 'localforward' and - 'remoteforward' keys. Thanks to Emre Yılmaz for the patch. - -v1.10.2 (26th Jul 2013) ------------------------ - -* #153, #67: Warn on parse failure when reading known_hosts file. Thanks to - `@glasserc` for patch. -* #146: Indentation fixes for readability. Thanks to Abhinav Upadhyay for catch - & patch. - -v1.10.1 (5th Apr 2013) ----------------------- - -* #142: (Fabric #811) SFTP put of empty file will still return the attributes - of the put file. Thanks to Jason R. Coombs for the patch. -* #154: (Fabric #876) Forwarded SSH agent connections left stale local pipes - lying around, which could cause local (and sometimes remote or network) - resource starvation when running many agent-using remote commands. Thanks to - Kevin Tegtmeier for catch & patch. - -v1.10.0 (1st Mar 2013) --------------------- - -* #66: Batch SFTP writes to help speed up file transfers. Thanks to Olle - Lundberg for the patch. -* #133: Fix handling of window-change events to be on-spec and not - attempt to wait for a response from the remote sshd; this fixes problems with - less common targets such as some Cisco devices. Thanks to Phillip Heller for - catch & patch. -* #93: Overhaul SSH config parsing to be in line with `man ssh_config` (& the - behavior of `ssh` itself), including addition of parameter expansion within - config values. Thanks to Olle Lundberg for the patch. -* #110: Honor SSH config `AddressFamily` setting when looking up local - host's FQDN. Thanks to John Hensley for the patch. -* #128: Defer FQDN resolution until needed, when parsing SSH config files. - Thanks to Parantapa Bhattacharya for catch & patch. -* #102: Forego random padding for packets when running under `*-ctr` ciphers. - This corrects some slowdowns on platforms where random byte generation is - inefficient (e.g. Windows). Thanks to `@warthog618` for catch & patch, and - Michael van der Kolff for code/technique review. -* #127: Turn `SFTPFile` into a context manager. Thanks to Michael Williamson - for the patch. -* #116: Limit `Message.get_bytes` to an upper bound of 1MB to protect against - potential DoS vectors. Thanks to `@mvschaik` for catch & patch. -* #115: Add convenience `get_pty` kwarg to `Client.exec_command` so users not - manually controlling a channel object can still toggle PTY creation. Thanks - to Michael van der Kolff for the patch. -* #71: Add `SFTPClient.putfo` and `.getfo` methods to allow direct - uploading/downloading of file-like objects. Thanks to Eric Buehl for the - patch. -* #113: Add `timeout` parameter to `SSHClient.exec_command` for easier setting - of the command's internal channel object's timeout. Thanks to Cernov Vladimir - for the patch. -* #94: Remove duplication of SSH port constant. Thanks to Olle Lundberg for the - catch. -* #80: Expose the internal "is closed" property of the file transfer class - `BufferedFile` as `.closed`, better conforming to Python's file interface. - Thanks to `@smunaut` and James Hiscock for catch & patch. - v1.9.0 (6th Nov 2012) --------------------- From 52183257d0ac71c72263331c3542d36a2b49ab94 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 30 Jan 2014 14:20:04 -0800 Subject: [PATCH 16/33] Remove blog link for now --- sites/www/contact.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/www/contact.rst b/sites/www/contact.rst index b479f17..2b6583f 100644 --- a/sites/www/contact.rst +++ b/sites/www/contact.rst @@ -8,4 +8,4 @@ following ways: * IRC: ``#paramiko`` on Freenode * Mailing list: ``paramiko@librelist.com`` (see `the LibreList homepage `_ for usage details). -* This website's :doc:`blog `. +* This website - a blog section is forthcoming. From 55722240b5b676277ef26a20b4b03ef81be3b372 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 30 Jan 2014 14:20:13 -0800 Subject: [PATCH 17/33] Fix broken link format --- sites/www/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/www/contributing.rst b/sites/www/contributing.rst index b121e64..2b752cc 100644 --- a/sites/www/contributing.rst +++ b/sites/www/contributing.rst @@ -6,7 +6,7 @@ How to get the code =================== Our primary Git repository is on Github at `paramiko/paramiko -`; please follow their instruction for +`_; please follow their instructions for cloning to your local system. (If you intend to submit patches/pull requests, we recommend forking first, then cloning your fork. Github has excellent documentation for all this.) From 4aa966865393d52ab548ca4b3b12e84260d82f8d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Thu, 30 Jan 2014 14:31:59 -0800 Subject: [PATCH 18/33] Don't preseed releases, ugh --- sites/www/changelog.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 0539285..235978d 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,7 +2,6 @@ Changelog ========= -* :release:`1.10.6 <2014-01-21>` 193 * :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`): Fix SSH agent problems present on Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle Lundberg for the patches. From 0f2fb262870d080d872bb4dd9689d2faf75545ee Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 3 Feb 2014 14:09:56 -0800 Subject: [PATCH 19/33] Update to new Alabaster version for footer/extension stuff --- dev-requirements.txt | 2 +- sites/shared_conf.py | 3 ++- sites/www/conf.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2fe1e77..e9770f4 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,5 +3,5 @@ invoke>=0.7.0 invocations>=0.4.4 sphinx>=1.1.3 -alabaster>=0.2.0 +alabaster>=0.3.0 releases>=0.2.4 diff --git a/sites/shared_conf.py b/sites/shared_conf.py index 2b98654..ca555ca 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -5,8 +5,9 @@ import sys import alabaster -# Alabaster theme +# Alabaster theme + mini-extension html_theme_path = [alabaster.get_path()] +extensions = ['alabaster'] # Paths relative to invoking conf.py - not this shared file html_static_path = ['../_shared_static'] html_theme = 'alabaster' diff --git a/sites/www/conf.py b/sites/www/conf.py index 1f11c1e..481acdf 100644 --- a/sites/www/conf.py +++ b/sites/www/conf.py @@ -8,7 +8,7 @@ from shared_conf import * # Local blog extension sys.path.append(abspath('.')) -extensions = ['blog'] +extensions.append('blog') rss_link = 'http://paramiko.org' rss_description = 'Paramiko project news' From cdf62655cbddbc68b42f58984eb60906a253cca0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 3 Feb 2014 14:10:23 -0800 Subject: [PATCH 20/33] Simplify www page footer. Robey copyright remains in source code but this site is brand new & the footer was real noisy. Eh. --- sites/shared_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/shared_conf.py b/sites/shared_conf.py index ca555ca..86ecdfe 100644 --- a/sites/shared_conf.py +++ b/sites/shared_conf.py @@ -33,7 +33,7 @@ html_sidebars = { # Regular settings project = u'Paramiko' year = datetime.now().year -copyright = u'2013-%d Jeff Forcier, 2003-2012 Robey Pointer' % year +copyright = u'%d Jeff Forcier' % year master_doc = 'index' templates_path = ['_templates'] exclude_trees = ['_build'] From fe3d3beca4985d7c99a306a269cdbd76091ead6a Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 17:59:25 -0800 Subject: [PATCH 21/33] Fix extra colon in changelog --- sites/www/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 235978d..2378b99 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -* :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`): Fix SSH agent +* :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`) Fix SSH agent problems present on Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle Lundberg for the patches. * :release:`1.10.5 <2014-01-08>` From 0a2887a8cca93eb5278ff79ed4b014e0ea24c53c Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 17:59:32 -0800 Subject: [PATCH 22/33] Changelog re #35 --- sites/www/changelog.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 2378b99..1a82758 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,9 @@ Changelog ========= +* :bug:`34` (PR :issue:`35`) Fix SFTP prefetching incompatibility with some + SFTP servers regarding request/response ordering. Thanks to Richard + Kettlewell for catch & patch. * :bug:`193` (and its attentant PRs :issue:`230` & :issue:`253`) Fix SSH agent problems present on Windows. Thanks to David Hobbs for initial report and to Aarni Koskela & Olle Lundberg for the patches. From 9fa2cfee9d182eefe918c0303c7966667d9673c9 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 18:03:25 -0800 Subject: [PATCH 23/33] Replace incorrect import of generic test runner w/ custom task --- tasks.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tasks.py b/tasks.py index c716415..e402599 100644 --- a/tasks.py +++ b/tasks.py @@ -1,7 +1,7 @@ from os.path import join -from invoke import Collection -from invocations import docs as _docs, testing +from invoke import Collection, task +from invocations import docs as _docs d = 'sites' @@ -20,4 +20,11 @@ www = Collection.from_module(_docs, name='www', config={ 'sphinx.target': join(path, '_build'), }) -ns = Collection(testing.test, docs=docs, www=www) + +# Until we move to spec-based testing +@task +def test(ctx): + ctx.run("python test.py --verbose") + + +ns = Collection(test, docs=docs, www=www) From aed86f26bff44106b3c564fb29b816a8b5da42b0 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 18:10:52 -0800 Subject: [PATCH 24/33] Future import for with: under py25 --- paramiko/sftp_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index b7cf850..ac8ceaf 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -20,6 +20,8 @@ L{SFTPFile} """ +from __future__ import with_statement + from binascii import hexlify from collections import deque import socket From 280c240685b6c5d66bde1e81cb160c24c8212387 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 18:11:29 -0800 Subject: [PATCH 25/33] Whitespace --- paramiko/sftp_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index ac8ceaf..c426d6e 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -475,7 +475,7 @@ class SFTPFile (BufferedFile): raise SFTPError('Expected data') data = msg.get_string() with self._prefetch_lock: - offset,length = self._prefetch_extents[num] + offset, length = self._prefetch_extents[num] self._prefetch_data[offset] = data del self._prefetch_extents[num] if len(self._prefetch_extents) == 0: From 1d87a0ceb9b6689e3969cc2b528f842b51e68f88 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 18:21:29 -0800 Subject: [PATCH 26/33] Herp and a derp --- tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks.py b/tasks.py index e402599..79cb94f 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,6 @@ from os.path import join -from invoke import Collection, task +from invoke import Collection, ctask as task from invocations import docs as _docs From 675e30986e3aa2882093b8c983a93619f61a019d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Mon, 10 Feb 2014 18:21:47 -0800 Subject: [PATCH 27/33] Clean up a list-comp (also fixes race condition) --- paramiko/sftp_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paramiko/sftp_file.py b/paramiko/sftp_file.py index c426d6e..a39b1f4 100644 --- a/paramiko/sftp_file.py +++ b/paramiko/sftp_file.py @@ -94,7 +94,7 @@ class SFTPFile (BufferedFile): pass def _data_in_prefetch_requests(self, offset, size): - k = [self._prefetch_extents[i] for i in self._prefetch_extents if self._prefetch_extents[i][0] <= offset] + k = [x for x in self._prefetch_extents.values() if x[0] <= offset] if len(k) == 0: return False k.sort(lambda x, y: cmp(x[0], y[0])) From 2d3b13e91705a2940f8af0c3d363190bee85ea0d Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 11 Feb 2014 09:31:43 -0800 Subject: [PATCH 28/33] Add coverage command as Invoke task --- .gitignore | 1 + .travis.yml | 2 +- tasks.py | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9e1febf..e149bb8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ test.log docs/ !sites/docs _build +.coverage diff --git a/.travis.yml b/.travis.yml index df7c225..97165c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip install -r dev-requirements.txt script: # Main tests, with coverage! - - coverage run --source=paramiko test.py --verbose + - invoke coverage # Ensure documentation & invoke pipeline run OK. # Run 'docs' first since its objects.inv is referred to by 'www'. # Also force warnings to be errors since most of them tend to be actual diff --git a/tasks.py b/tasks.py index 79cb94f..f8f4017 100644 --- a/tasks.py +++ b/tasks.py @@ -26,5 +26,9 @@ www = Collection.from_module(_docs, name='www', config={ def test(ctx): ctx.run("python test.py --verbose") +@task +def coverage(ctx): + ctx.run("coverage run --source=paramiko test.py --verbose") -ns = Collection(test, docs=docs, www=www) + +ns = Collection(test, coverage, docs=docs, www=www) From c1424546219a769540fc70c54f268e284c0b2b85 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 11 Feb 2014 14:23:15 -0800 Subject: [PATCH 29/33] Start reverting explicit-release stuff in favor of release-line specifiers on bugs. --- dev-requirements.txt | 2 +- sites/www/changelog.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index e9770f4..6c242c0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,4 +4,4 @@ invoke>=0.7.0 invocations>=0.4.4 sphinx>=1.1.3 alabaster>=0.3.0 -releases>=0.2.4 +releases>=0.5.0 diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index 1a82758..af4c69e 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -12,7 +12,7 @@ Changelog * :bug:`176` Fix AttributeError bugs in known_hosts file (re)loading. Thanks to Nathan Scowcroft for the patch & Martin Blumenstingl for the initial test case. -* :release:`1.10.4 <2013-09-27>` 199, 200, 179 +* :release:`1.10.4 <2013-09-27>` * :bug:`179` Fix a missing variable causing errors when an ssh_config file has a non-default AddressFamily set. Thanks to Ed Marshall & Tomaz Muraus for catch & patch. From 2690a90ccb7a19b0676900f8804fb3da5f3130a6 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Tue, 11 Feb 2014 14:51:28 -0800 Subject: [PATCH 30/33] Bump to releases bugfix --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 6c242c0..331e38c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,4 +4,4 @@ invoke>=0.7.0 invocations>=0.4.4 sphinx>=1.1.3 alabaster>=0.3.0 -releases>=0.5.0 +releases>=0.5.1 From 457a34f55b1d69c3f7699b3bc055f6c6d90371cd Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 14 Feb 2014 09:36:33 -0800 Subject: [PATCH 31/33] Cut 1.10.6 --- paramiko/__init__.py | 2 +- setup.py | 2 +- sites/www/changelog.rst | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/paramiko/__init__.py b/paramiko/__init__.py index 924e8bb..7f60b1f 100644 --- a/paramiko/__init__.py +++ b/paramiko/__init__.py @@ -55,7 +55,7 @@ if sys.version_info < (2, 5): __author__ = "Jeff Forcier " -__version__ = "1.10.5" +__version__ = "1.10.6" __version_info__ = tuple([ int(d) for d in __version__.split(".") ]) __license__ = "GNU Lesser General Public License (LGPL)" diff --git a/setup.py b/setup.py index 8ec4a0e..bc5f9e6 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ if sys.platform == 'darwin': setup(name = "paramiko", - version = "1.10.5", + version = "1.10.6", description = "SSH2 protocol library", author = "Jeff Forcier", author_email = "jeff@bitprophet.org", diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index af4c69e..c26b996 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,7 @@ Changelog ========= +* :release:`1.10.6 <2014-02-14>` * :bug:`34` (PR :issue:`35`) Fix SFTP prefetching incompatibility with some SFTP servers regarding request/response ordering. Thanks to Richard Kettlewell for catch & patch. From 9d7aeff7b19aabacecdb42d86af15bdb45e01e20 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 14 Feb 2014 11:52:00 -0800 Subject: [PATCH 32/33] Use constant time hash comparisons for improved security. See e.g. http://codahale.com/a-lesson-in-timing-attacks/ Mega thanks to Alex Gaynor for the original patch. --- paramiko/hostkeys.py | 4 ++-- paramiko/packet.py | 2 +- paramiko/util.py | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/paramiko/hostkeys.py b/paramiko/hostkeys.py index f967a3d..c34b173 100644 --- a/paramiko/hostkeys.py +++ b/paramiko/hostkeys.py @@ -28,7 +28,7 @@ import UserDict from paramiko.common import * from paramiko.dsskey import DSSKey from paramiko.rsakey import RSAKey -from paramiko.util import get_logger +from paramiko.util import get_logger, constant_time_bytes_eq class InvalidHostKey(Exception): @@ -243,7 +243,7 @@ class HostKeys (UserDict.DictMixin): entries = [] for e in self._entries: for h in e.hostnames: - if (h.startswith('|1|') and (self.hash_host(hostname, h) == h)) or (h == hostname): + if h.startswith('|1|') and constant_time_bytes_eq(self.hash_host(hostname, h), h) or h == hostname: entries.append(e) if len(entries) == 0: return None diff --git a/paramiko/packet.py b/paramiko/packet.py index 3f85d66..dce4383 100644 --- a/paramiko/packet.py +++ b/paramiko/packet.py @@ -358,7 +358,7 @@ class Packetizer (object): mac = post_packet[:self.__mac_size_in] mac_payload = struct.pack('>II', self.__sequence_number_in, packet_size) + packet my_mac = compute_hmac(self.__mac_key_in, mac_payload, self.__mac_engine_in)[:self.__mac_size_in] - if my_mac != mac: + if not util.constant_time_bytes_eq(my_mac, mac): raise SSHException('Mismatched MAC') padding = ord(packet[0]) payload = packet[1:packet_size - padding] diff --git a/paramiko/util.py b/paramiko/util.py index 85ee6b0..9db512d 100644 --- a/paramiko/util.py +++ b/paramiko/util.py @@ -309,3 +309,12 @@ class Counter (object): def new(cls, nbits, initial_value=1L, overflow=0L): return cls(nbits, initial_value=initial_value, overflow=overflow) new = classmethod(new) + + +def constant_time_bytes_eq(a, b): + if len(a) != len(b): + return False + res = 0 + for i in xrange(len(a)): + res |= ord(a[i]) ^ ord(b[i]) + return res == 0 From 30518280f1356465f0acfdb4816843b61303a633 Mon Sep 17 00:00:00 2001 From: Jeff Forcier Date: Fri, 14 Feb 2014 11:53:42 -0800 Subject: [PATCH 33/33] Changelog re hash comparison bugfix --- sites/www/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sites/www/changelog.rst b/sites/www/changelog.rst index c26b996..fa58c5f 100644 --- a/sites/www/changelog.rst +++ b/sites/www/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +* :bug:`-` Use constant-time hash comparison operations where possible, to + protect against `timing-based attacks + `_. Thanks to Alex Gaynor + for the patch. * :release:`1.10.6 <2014-02-14>` * :bug:`34` (PR :issue:`35`) Fix SFTP prefetching incompatibility with some SFTP servers regarding request/response ordering. Thanks to Richard