initial commit

Author: gusdan

.gitignore

@@ -34,3 +34,4 @@ nosetests.xml
 .mr.developer.cfg
 .project
 .pydevproject
+.idea

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014 gusdan
+Copyright (c) 2014 Danil Gusev
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -0,0 +1,59 @@
+Amazon ElastiCache backend for Django
+=====================================
+
+Simple Django cache backend for Amazon ElastiCache (memcached based). It uses
+`pylibmc <http://github.com/lericson/pylibmc>`_ and setup connection to each
+node in cluster using
+`Auto Discovery <http://docs.aws.amazon.com/AmazonElastiCache/latest/UserGuide/AutoDiscovery.html>`_
+function.
+
+
+Requirements
+------------
+
+* pylibmc
+* Django 1.3+.
+
+It was written and tested on Python 2.7.
+
+Installation
+------------
+
+Get it from `pypi <http://pypi.python.org/pypi/django-elasticache>`_::
+
+    pip install django-pylibmc
+
+or `github <http://github.com/gusdan/django-elasticache>`_::
+
+    pip install -e git://github.com/gusdan/django-elasticache.git#egg=django-elasticache
+
+
+Usage
+-----
+
+Your cache backend should look something like this::
+
+    CACHES = {
+        'default': {
+            'BACKEND': 'django_elasticache.memcached.ElasctiCache',
+            'LOCATION': 'cache-c.drtgf.cfg.use1.cache.amazonaws.com:11211',
+        }
+    }
+
+By the first call to cache it connect to cluster (using LOCATION),
+get list of all nodes and setup pylibmc client using full
+list of nodes. As result your cache will work with all nodes and
+automatically detect new nodes in cluster. List of nodes are stored in class-level
+cached, so any changes in cluster take affect only after restart working process.
+But if you're using gunicorn or mod_wsgi you usually have max_request settings which
+restart process after some count of processed requests.
+
+Django-elascticache changes default pylibmc params to increase performance.
+
+
+Testing
+-------
+
+Run the tests like this::
+
+    nosetest

README.rst

@@ -0,0 +1,2 @@
+VERSION = (0, 0, 1)
+__version__ = '.'.join(map(str, VERSION))

django_elasticache/__init__.py

@@ -0,0 +1,58 @@
+"""
+utils for discovery cluster
+"""
+from distutils.version import StrictVersion
+from telnetlib import Telnet
+
+
+class WrongProtocolData(ValueError):
+    """
+    Exception for raising when we get something unexpected
+    in telnet protocol
+    """
+    def __init__(self, cmd, response):
+        super(WrongProtocolData, self).__init__(
+            'Unexpected response {} for command {}'.format(response, cmd))
+
+
+def get_cluster_info(host, port):
+    """
+    return dict with info about nodes in cluster and current version
+    {
+        'nodes': [
+            'IP:port',
+            'IP:port',
+        ],
+        'version': '1.4.4'
+    }
+    """
+    client = Telnet(host, port)
+    client.write('version\n')
+    res = client.read_until('\r\n').strip()
+    version_list = res.split(' ')
+    if len(version_list) != 2 or version_list[0] != 'VERSION':
+        raise WrongProtocolData('version', res)
+    version = version_list[1]
+    if StrictVersion(version) >= StrictVersion('1.4.14'):
+        cmd = 'config get cluster\n'
+    else:
+        cmd = 'get AmazonElastiCache:cluster\n'
+    client.write(cmd)
+    res = client.read_until('\r\n').strip()
+    try:
+        version = int(res)
+    except ValueError:
+        raise WrongProtocolData(cmd, res)
+    res = client.read_until('\r\n').strip()
+    client.close()
+    nodes = []
+    try:
+        for node in res.split(' '):
+            host, ip, port = node.split('|')
+            nodes.append('{}:{}'.format(ip or host, port))
+    except ValueError:
+        raise WrongProtocolData(cmd, res)
+    return {
+        'version': version,
+        'nodes': nodes
+    }

django_elasticache/cluster_utils.py

@@ -0,0 +1,65 @@
+"""
+Backend for django cache
+"""
+from django.core.cache import InvalidCacheBackendError
+from django.core.cache.backends.memcached import PyLibMCCache
+from django.utils.functional import cached_property
+from .cluster_utils import get_cluster_info
+
+
+class ElastiCache(PyLibMCCache):
+    """
+    backend for Amazon ElastiCache (memcached) with auto discovery mode
+    it used pylibmc in binary mode
+    """
+    def __init__(self, server, params):
+        self.update_params(params)
+        super(ElastiCache, self).__init__(server, params)
+        if len(self._servers) > 1:
+            raise InvalidCacheBackendError(
+                'ElastiCache should be configured with only one server '
+                '(Configuration Endpoint)')
+        if len(self._servers[0].split(':')) != 2:
+            raise InvalidCacheBackendError(
+                'Server configuration should be in format IP:port')
+
+    def update_params(self, params):
+        """
+        update connection params to maximize performance
+        """
+        if not params.get('BINARY', True):
+            raise Warning('To increase performance please use ElastiCache'
+                          ' in binary mode')
+        else:
+            params['BINARY'] = True  # patch params, set binary mode
+        if not 'OPTIONS' in params:
+            # set special 'behaviors' pylibmc attributes
+            params['OPTIONS'] = {
+                'tcp_nodelay': True,
+                'ketama': True
+            }
+
+    @cached_property
+    def get_cluster_nodes(self):
+        """
+        return list with all nodes in cluster
+        """
+        server, port = self._servers[0].split(':')
+        return get_cluster_info(server, port)['nodes']
+
+    @property
+    def _cache(self):
+        # PylibMC uses cache options as the 'behaviors' attribute.
+        # It also needs to use threadlocals, because some versions of
+        # PylibMC don't play well with the GIL.
+        client = getattr(self._local, 'client', None)
+        if client:
+            return client
+
+        client = self._lib.Client(self.get_cluster_nodes)
+        if self._options:
+            client.behaviors = self._options
+
+        self._local.client = client
+
+        return client

django_elasticache/memcached.py

@@ -0,0 +1,29 @@
+from setuptools import setup
+
+import django_elasticache
+
+
+setup(
+    name='django-elasticache',
+    version=django_elasticache.__version__,
+    description='Django cache backend for Amazon ElastiCache memcached',
+    long_description=open('README.rst').read(),
+    author='Danil Gusev',
+    author_email='[email protected]',
+    url='http://github.com/gusdan/django-elasticache',
+    license='MIT',
+    keywords='elasticache amazon cache pylibmc memcached aws',
+    packages=['django_elasticache'],
+    install_requires=['pylibmc', 'Django>=1.3'],
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Environment :: Web Environment',
+        'Environment :: Web Environment :: Mozilla',
+        'Framework :: Django',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: BSD License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+    ],
+)

setup.py

@@ -0,0 +1,42 @@
+from django.conf import global_settings
+from mock import patch, Mock
+from nose.tools import eq_, raises
+
+
+@patch('django.conf.settings', global_settings)
+def test_patch_params():
+    from django_elasticache.memcached import ElastiCache
+    params = {}
+    ElastiCache('qew:12', params)
+    eq_(params['BINARY'], True)
+    eq_(params['OPTIONS']['tcp_nodelay'], True)
+    eq_(params['OPTIONS']['ketama'], True)
+
+
+@raises(Exception)
+@patch('django.conf.settings', global_settings)
+def test_wrong_params():
+    from django_elasticache.memcached import ElastiCache
+    ElastiCache('qew', {})
+
+
+@raises(Warning)
+@patch('django.conf.settings', global_settings)
+def test_wrong_params_warning():
+    from django_elasticache.memcached import ElastiCache
+    ElastiCache('qew', {'BINARY': False})
+
+
+@patch('django.conf.settings', global_settings)
+@patch('django_elasticache.memcached.get_cluster_info')
+def test_split_servers(get_cluster_info):
+    from django_elasticache.memcached import ElastiCache
+    backend = ElastiCache('h:0', {})
+    servers = ['h1:p', 'h2:p']
+    get_cluster_info.return_value = {
+        'nodes': servers
+    }
+    backend._lib.Client = Mock()
+    assert backend._cache
+    get_cluster_info.assert_called_once_with('h', '0')
+    backend._lib.Client.assert_called_once_with(servers)

tests/__init__.py

@@ -0,0 +1,53 @@
+from mock import patch, call
+from django_elasticache.cluster_utils import (
+    get_cluster_info, WrongProtocolData)
+from nose.tools import eq_, raises
+
+TEST_PROTOCOL_1 = [
+    'VERSION 1.4.14',
+    '1',
+    'hostname|ip-address|port hostname||port'
+]
+
+TEST_PROTOCOL_2 = [
+    'VERSION 1.4.13',
+    '1',
+    'hostname|ip-address|port hostname||port'
+]
+
+
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_happy_path(Telnet):
+    client = Telnet.return_value
+    client.read_until.side_effect = TEST_PROTOCOL_1
+    info = get_cluster_info('', 0)
+    eq_(info['version'], 1)
+    eq_(info['nodes'], ['ip-address:port', 'hostname:port'])
+
+
+@raises(WrongProtocolData)
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_bad_protocol(Telnet):
+    get_cluster_info('', 0)
+
+
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_last_versions(Telnet):
+    client = Telnet.return_value
+    client.read_until.side_effect = TEST_PROTOCOL_1
+    get_cluster_info('', 0)
+    client.write.assert_has_calls([
+        call('version\n'),
+        call('config get cluster\n'),
+    ])
+
+
+@patch('django_elasticache.cluster_utils.Telnet')
+def test_prev_versions(Telnet):
+    client = Telnet.return_value
+    client.read_until.side_effect = TEST_PROTOCOL_2
+    get_cluster_info('', 0)
+    client.write.assert_has_calls([
+        call('version\n'),
+        call('get AmazonElastiCache:cluster\n'),
+    ])