Merged 0.12 changes to stable

--HG--
branch : stable

Author: Menno Smits

.hgignore

@@ -6,3 +6,6 @@ doc/html/
 .tox/
 ^mock-.+.egg.*
 ^setuptools-.+
+^unittest2-.+
+^six-.+
+^argparse-.+\.egg$

NEWS.rst

@@ -1,3 +1,53 @@
+======
+ 0.12
+======
+
+Fixed unicode handling [API CHANGE]
+-----------------------------------
+During the work to support Python 3, IMAPClient was changed to do
+return unicode for most responses. This was a bad decision, especially
+because it effectively breaks content that uses multiple encodings
+(e.g. RFC822 responses). This release includes major changes so that
+most responses are returned as bytes (Python 3) or str (Python
+2). This means that correct handling of response data is now possible
+by code using IMAPClient.
+
+Folder name handling has also been cleaned up as part of this work. If
+the ``folder_encode`` attribute is ``True`` (the default) then folder
+names will **always** be returned as unicode. If ``folder_encode`` is
+False then folder names will always be returned as bytes/strs.
+
+Code using IMAPClient will most likely need to be updated to account
+these unicode handling changes.
+
+Many thanks to Inbox (https://www.inboxapp.com/) for sponsoring this
+work.
+
+Extra __init__ keyword args are passed through [NEW]
+----------------------------------------------------
+Any unused keyword arguments passed to the IMAPClient initialiser will
+now be passed through to the underlying imaplib IMAP4, IMAP4_SSL or
+IMAP4_stream class. This is specifically to allow the use of imaplib
+features that control certificate validation (if available with the
+version of Python being used).
+
+Thanks to Chris Arndt for this change.
+
+MODSEQ parts in SEARCH responses are now handled
+------------------------------------------------
+If the CONDSTORE extension is supported by a server and a MODSEQ
+criteria was used with search(), a TypeError could occur. This has now
+been fixed and the MODSEQ value returned by the server is now
+available via an attribute on the returned list of ids.
+
+Minor Changes
+-------------
+* Small tweaks to support Python 3.4.
+* The deprecated get_folder_delimiter() method has been removed.
+* More control over OAUTH2 parameters. Thanks to Phil Peterson for
+  this.
+* Fixed livetest/interact OAUTH handling under Python 3.
+
 ========
  0.11.1
 ========

README.rst

@@ -17,7 +17,7 @@ Features:
 - Convenience methods are provided for commonly used functionality.
 - Exceptions are raised when errors occur.
 
-Python versions 2.6, 2.7, 3.2 and 3.3 are officially supported.
+Python versions 2.6, 2.7, 3.2, 3.3 and 3.4 are officially supported.
 
 Why IMAPClient?
 ---------------

THANKS

@@ -44,8 +44,13 @@ John Louis del Rosario
      Gmail search support (X-GM-RAW).
 
 Naveen Nathan
-     ENVELOPE FETCH response parsing
-     
+     ENVELOPE FETCH response parsing.
+
+Chris Arndt
+     Pass-through imaplib kwargs.
+
+Phil Peterson
+     OAUTH2 improvements.
 
 Bug Reports
 ------------
@@ -59,7 +64,6 @@ Erik Quaeghebeur
 Benjamin Morrise
 Brandon Rhodes
 
-
 External Projects
 -----------------
 IMAPClient uses Michael's excellent 'Mock' package.
@@ -69,5 +73,7 @@ IMAPClient includes Benjamin Peterson's 'six' library to help achieve
 cross-version Python compatibility.
 (https://pypi.python.org/pypi/six)
 
+External Contributions
+----------------------
 The project was started while the author was working at NetBox Blue.
-(http://netboxblue.com/)
+(http://netboxblue.com/)

doc/src/conf.py

@@ -17,7 +17,10 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode',
+    'sphinx.ext.intersphinx']
+
+intersphinx_mapping = {'python':('http://docs.python.org/3', None)}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

doc/src/index.rst

@@ -31,7 +31,7 @@ explains IMAP in detail. Other RFCs also apply to various extensions
 to the base protocol. These are referred to in the documentation below
 where relevant.
 
-Python versions 2.6, 2.7, 3.2 and 3.3 are officially supported.
+Python versions 2.6, 2.7, 3.2, 3.3 and 3.4 are officially supported.
 
 A Simple Example
 ----------------
@@ -53,18 +53,18 @@ The output from this example could look something like this
     75 messages that aren't deleted
 
     Messages:
-       ID 38273: 1775 bytes, flags=('NonJunk',)
-       ID 36459: 2833 bytes, flags=('\\Flagged', '\\Seen')
-       ID 34693: 2874 bytes, flags=('\\Flagged', '\\Seen')
-       ID 38066: 5368 bytes, flags=('\\Flagged', '\\Seen')
-       ID 38154: 9079 bytes, flags=('\\Seen', 'NonJunk')
-       ID 14099: 3322 bytes, flags=('\\Flagged', '\\Seen', '$Label1')
-       ID 34196: 9163 bytes, flags=('\\Answered', '\\Seen')
-       ID 35349: 4266 bytes, flags=('\\Flagged', '\\Seen')
-       ID 29335: 5617 bytes, flags=('\\Flagged', '\\Seen', 'NonJunk')
-       ID 38041: 7649 bytes, flags=('\\Seen', 'NonJunk')
-       ID 22310: 976108 bytes, flags=('\\Flagged', '\\Seen', '$Label1')
-       ID 6439: 3792 bytes, flags=('\\Flagged', '\\Seen', '$Label1', 'Junk')
+       ID 38273: 1775 bytes, flags=(b'NonJunk',)
+       ID 36459: 2833 bytes, flags=(b'\\Flagged', v'\\Seen')
+       ID 34693: 2874 bytes, flags=(b'\\Flagged', v'\\Seen')
+       ID 38066: 5368 bytes, flags=(b'\\Flagged', v'\\Seen')
+       ID 38154: 9079 bytes, flags=(b'\\Seen', b'NonJunk')
+       ID 14099: 3322 bytes, flags=(b'\\Flagged', b'\\Seen', b'$Label1')
+       ID 34196: 9163 bytes, flags=(b'\\Answered', b'\\Seen')
+       ID 35349: 4266 bytes, flags=(b'\\Flagged', b'\\Seen')
+       ID 29335: 5617 bytes, flags=(b'\\Flagged', b'\\Seen', b'NonJunk')
+       ID 38041: 7649 bytes, flags=(b'\\Seen', b'NonJunk')
+       ID 22310: 976108 bytes, flags=(b'\\Flagged', b'\\Seen', b'$Label1')
+       ID 6439: 3792 bytes, flags=(b'\\Flagged', b'\\Seen', b'$Label1', b'Junk')
 
 
 Concepts
@@ -126,16 +126,17 @@ modified UTF-7 as specified by :rfc:`3501#section-5.1.3`.  This allows
 for arbitrary unicode characters (eg. non-English characters) to be
 used in folder names.
 
-All folder names returned by IMAPClient are always returned as unicode
-strings.
-
 The ampersand character ("&") has special meaning in IMAP folder
 names. IMAPClient automatically escapes and unescapes this character
 so that the caller doesn't have to.
 
 Automatic folder name encoding and decoding can be enabled or disabled
 with the *folder_encode* attribute. It defaults to True.
 
+If *folder_encode* is True, all folder names returned by IMAPClient
+are always returned as unicode strings. If *folder_encode* is False,
+folder names are returned as str (Python 2) or bytes (Python 3).
+
 Exceptions
 ~~~~~~~~~~
 The IMAP related exceptions that will be raised by this class are:
@@ -183,6 +184,8 @@ Various options are available to specify the IMAP server details. See
 the help (--help) for more details. You'll be prompted for a username
 and password if one isn't provided on the command line.
 
+The connected IMAPClient instance is available as the variable "c".
+
 If installed, IPython will be used as the embedded shell. Otherwise
 the basic built-in Python shell will be used.
 
@@ -195,26 +198,24 @@ Here's an example session::
     IMAPClient instance is "c"
     In [1]: c.select_folder('inbox')
     Out[1]: 
-    {'EXISTS': 2,
-     'FLAGS': ('\\Answered',
-      '\\Flagged',
-      '\\Deleted',
-      '\\Seen',
-      '\\Draft'),
-     'PERMANENTFLAGS': ('\\Answered',
-      '\\Flagged',
-      '\\Deleted',
-      '\\Seen',
-      '\\Draft'),
-     'READ-WRITE': True,
-     'RECENT': 0,
-     'UIDNEXT': 1339,
-     'UIDVALIDITY': 1239278212}
+    {b'EXISTS': 2,
+     b'FLAGS': (b'\\Answered',
+         b'\\Flagged',
+         b'\\Deleted',
+         b'\\Seen',
+         b'\\Draft'),
+     b'PERMANENTFLAGS': (b'\\Answered',
+         b'\\Flagged',
+         b'\\Deleted',
+         b'\\Seen',
+         b'\\Draft'),
+     b'READ-WRITE': True,
+     b'RECENT': 0,
+     b'UIDNEXT': 1339,
+     b'UIDVALIDITY': 1239278212}
 
     In [2]: c.search()
     Out[2]: [1123, 1233]
 
     In [3]: c.logout()
-    Out[3]: 'Logging out'
-
-Note that the connected IMAPClient instance is available as the variable "c".
+    Out[3]: b'Logging out'

imapclient/__init__.py

@@ -7,7 +7,7 @@
 
 from __future__ import unicode_literals
 
-version_info = (0, 11, 0, 'final')
+version_info = (0, 12, 0, 'final')
 
 def _imapclient_version_string(vinfo):
     major, minor, micro, releaselevel = vinfo

imapclient/config.py

@@ -6,7 +6,8 @@
     from configparser import SafeConfigParser, NoOptionError
 
 import imapclient
-import urllib
+from .six.moves.urllib.request import urlopen
+from .six.moves.urllib.parse import urlencode
 
 try:
     import json
@@ -66,13 +67,13 @@ def parse_config_file(path):
 def refresh_oauth2_token(client_id, client_secret, refresh_token):
     if not json:
         raise RuntimeError("livetest OAUTH2 functionality relies on 'json' module")
-    post = dict(client_id=client_id,
-                client_secret=client_secret,
-                refresh_token=refresh_token,
-                grant_type='refresh_token')
-    response = urllib.urlopen('https://accounts.google.com/o/oauth2/token',
-                              urllib.urlencode(post)).read()
-    return json.loads(response)['access_token']
+    post = dict(client_id=client_id.encode('ascii'),
+                client_secret=client_secret.encode('ascii'),
+                refresh_token=refresh_token.encode('ascii'),
+                grant_type=b'refresh_token')
+    response = urlopen('https://accounts.google.com/o/oauth2/token',
+                       urlencode(post).encode('ascii')).read()
+    return json.loads(response.decode('ascii'))['access_token']
 
 # Tokens are expensive to refresh so use the same one for the duration of the process.
 _oauth2_cache = {}

imapclient/datetime_util.py

@@ -19,6 +19,7 @@ def parse_to_datetime(timestamp, normalise=True):
     If normalise is False, then the returned datetime will be
     unadjusted but will contain timezone information as per the input.
     """
+    timestamp = timestamp.decode('latin-1')  # parsedate_tz only works with strings
     time_tuple = parsedate_tz(timestamp)
     if time_tuple == None:
         raise ValueError("couldn't parse datetime %r" % timestamp)

imapclient/examples/example.py

@@ -25,5 +25,5 @@
 response = server.fetch(messages, ['FLAGS', 'RFC822.SIZE'])
 for msgid, data in response.iteritems():
     print('   ID %d: %d bytes, flags=%s' % (msgid,
-                                            data['RFC822.SIZE'],
-                                            data['FLAGS']))
+                                            data[b'RFC822.SIZE'],
+                                            data[b'FLAGS']))