Merge branch 'master' of github.com:globocom/database-as-a-service

Author: ricardosdias

.travis.yml

@@ -1,24 +1,30 @@
+sudo: required
+
 language: python
 
 python:
     - "2.7"
 
-before_install:
-    - mysql -e 'CREATE DATABASE dbaas;'
-
-install:
-    - pip install -r requirements_test.txt
-
-cache:
-    - pip
-
 services:
-    - mysql
-    - mongodb
-    - redis-server
+    - docker
+
+env:
+  DOCKER_COMPOSE_VERSION: 1.11.2
 
-before_script:
-    - make create_admin_mongo_user
+before_install:
+  - make docker_build
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
+  - docker-compose --version
+  - docker-compose pull mysqldb56 redisdb mongodb
+  - docker images
+  - docker ps
+  - sudo /etc/init.d/mysql stop
 
 script:
-    - make test
+  - docker-compose run test
+
+notifications:
+    email: false

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:2.7
+
+ADD . /code
+WORKDIR /code
+RUN apt-get update
+RUN apt-get install -y libsasl2-dev python-dev libldap2-dev libssl-dev mysql-client
+RUN easy_install ipython==5.1.0 ipdb==0.10.1
+RUN pip install -r requirements_test.txt
+ENTRYPOINT /code/tests.sh

Makefile

@@ -61,14 +61,26 @@ reset_data: db_reset # drop and create database and insert sample data
 run_migrate: # run all migrations
 	@cd dbaas && python manage.py syncdb --migrate --noinput --no-initial-data
 
-create_admin_mongo_user:
-	@mongo admin --eval 'db.addUser({user: "admin", pwd: "123456", roles: ["userAdminAnyDatabase", "clusterAdmin", "readWriteAnyDatabase", "dbAdminAnyDatabase"]});'
-
 test: # run tests
-	# @echo "create database IF NOT EXISTS dbaas;" | mysql -u root
-	@mysqladmin -uroot -p$(DBAAS_DATABASE_PASSWORD) -f drop test_dbaas -h$(DBAAS_DATABASE_HOST); true
 	@cd dbaas && python manage.py test --settings=dbaas.settings_test --traceback $(filter-out $@,$(MAKECMDGOALS))
 
+docker_build:
+	docker build -t dbaas_test .
+
+test_with_docker:
+	docker-compose run test
+
+docker_mysql_55:
+	docker-compose run --publish="3306:3306" mysqldb55
+
+docker_mysql_56:
+	docker-compose run --publish="3306:3306" mysqldb56
+
+docker_mysql_57:
+	docker-compose run --publish="3306:3306" mysqldb57
+
+kill_mysql:
+	@ps aux | grep mysqldb | grep -v 'grep mysqldb' | awk '{print $$2}' | xargs kill
 
 run: # run local server
 	@cd dbaas && python manage.py runserver 0.0.0.0:8000 $(filter-out $@,$(MAKECMDGOALS))

README.md

@@ -33,6 +33,11 @@ DBaaS requires the following:
 * and all packages in requirements.txt file (there is a shortcut to install them)
 
 
+Setup docker and docker-compose(optional)
+=========================================
+
+* Install [Docker](https://docs.docker.com/engine/installation/)
+
 Setup your local environment
 ============================
 
@@ -58,23 +63,13 @@ some minimum operational data on DB.
 
     make reset_data
 
-## Running all tests
-
-Before running the test, makes sure that you have mongod running and a user admin created with password 123456.
-
-    db = db.getSiblingDB('admin')
+## Running all tests on local enviroment
 
-    db.createUser( { user: "admin",
-                  pwd: "123456",
-                  roles: [ "userAdminAnyDatabase", "clusterAdmin", "readWriteAnyDatabase", "dbAdminAnyDatabase" ] } )
-
-Then install all the required packages
-
-    make pip
+    make pip && make test
 
-Run it!
+## Running all test with docker( To run this the and docker-compose must be avaliable)
 
-    make test
+    make docker_build && make test_with_docker
 
 ## Running the project
 

add_user_admin_on_mongo.py

@@ -0,0 +1,7 @@
+from pymongo import MongoClient
+
+client = MongoClient("mongodb://mongodb:27017")
+
+db = client.admin
+
+db.add_user('admin', '123456', roles=["userAdminAnyDatabase", "clusterAdmin", "readWriteAnyDatabase", "dbAdminAnyDatabase"])

dbaas/dbaas/celeryconfig.py

@@ -1,8 +1,10 @@
 import os
 
+
 REDIS_PORT = os.getenv('DBAAS_NOTIFICATION_BROKER_PORT', '6379')
+REDIS_HOST = os.getenv('REDIS_HOST', 'localhost')
 BROKER_URL = os.getenv(
-    'DBAAS_NOTIFICATION_BROKER_URL', 'redis://localhost:%s/0' % REDIS_PORT)
+    'DBAAS_NOTIFICATION_BROKER_URL', 'redis://{}:{}/0'.format(REDIS_HOST, REDIS_PORT))
 CELERYD_TASK_TIME_LIMIT = 10800
 CELERY_TRACK_STARTED = True
 CELERY_IGNORE_RESULT = False

dbaas/drivers/tests/test_driver_mysql.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
-import mock
 import logging
 from django.test import TestCase
 from drivers import DriverFactory
@@ -16,10 +15,15 @@
 class AbstractTestDriverMysql(TestCase):
 
     def setUp(self):
+        mysql_host = settings.DB_HOST
+        mysql_port = settings.DB_PORT or 3306
+        self.mysql_endpoint = '{}:{}'.format(mysql_host, mysql_port)
         self.databaseinfra = factory_physical.DatabaseInfraFactory(
-            user="root", password=settings.DB_PASSWORD, endpoint="127.0.0.1:3306")
+            user="root", password=settings.DB_PASSWORD,
+            endpoint=self.mysql_endpoint)
         self.instance = factory_physical.InstanceFactory(
-            databaseinfra=self.databaseinfra, port=3306)
+            databaseinfra=self.databaseinfra, address=mysql_host,
+            port=mysql_port)
         self.driver = MySQL(databaseinfra=self.databaseinfra)
         self._mysql_client = None
 
@@ -53,7 +57,7 @@ def test_instantiate_mysqldb_using_engine_factory(self):
 
     def test_connection_string(self):
         self.assertEqual(
-            "mysql://<user>:<password>@127.0.0.1:3306", self.driver.get_connection())
+            "mysql://<user>:<password>@{}".format(self.mysql_endpoint), self.driver.get_connection())
 
     def test_get_user(self):
         self.assertEqual(self.databaseinfra.user, self.driver.get_user())
@@ -68,7 +72,7 @@ def test_get_default_port(self):
     def test_connection_with_database(self):
         self.database = factory_logical.DatabaseFactory(
             name="my_db_url_name", databaseinfra=self.databaseinfra)
-        self.assertEqual("mysql://<user>:<password>@127.0.0.1:3306/my_db_url_name",
+        self.assertEqual("mysql://<user>:<password>@{}/my_db_url_name".format(self.mysql_endpoint),
                          self.driver.get_connection(database=self.database))
 
 

dbaas/drivers/tests/test_driver_pymongo.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
+import os
 import mock
 from django.test import TestCase
 from drivers import DriverFactory
@@ -12,9 +13,12 @@
 class AbstractTestDriverMongo(TestCase):
 
     def setUp(self):
+        mongo_host = os.getenv('TESTS_MONGODB_HOST', '127.0.0.1')
+        mongo_port = os.getenv('TESTS_MONGODB_PORT', '27017')
+        self.mongo_endpoint = '{}:{}'.format(mongo_host, mongo_port)
         self.databaseinfra = factory_physical.DatabaseInfraFactory()
         self.instance = factory_physical.InstanceFactory(
-            databaseinfra=self.databaseinfra)
+            databaseinfra=self.databaseinfra, address=mongo_host)
         self.driver = MongoDB(databaseinfra=self.databaseinfra)
         self._mongo_client = None
 
@@ -48,7 +52,7 @@ def test_instantiate_mongodb_using_engine_factory(self):
 
     def test_connection_string(self):
         self.assertEqual(
-            "mongodb://<user>:<password>@127.0.0.1:27017", self.driver.get_connection())
+            "mongodb://<user>:<password>@{}".format(self.mongo_endpoint), self.driver.get_connection())
 
     def test_get_user(self):
         self.assertEqual(self.databaseinfra.user, self.driver.get_user())
@@ -65,14 +69,21 @@ def test_connection_string_when_in_replica_set(self, get_replica_name):
         self.instance = factory_physical.InstanceFactory(
             databaseinfra=self.databaseinfra, address='127.0.0.2', port=27018)
         get_replica_name.return_value = 'my_repl'
-        self.assertEqual(
-            "mongodb://<user>:<password>@127.0.0.1:27017,127.0.0.2:27018?replicaSet=my_repl", self.driver.get_connection())
+
+        expected_conn = ("mongodb://<user>:<password>"
+                          "@{},127.0.0.2:27018"
+                          "?replicaSet=my_repl").format(self.mongo_endpoint)
+
+        self.assertEqual(expected_conn, self.driver.get_connection())
 
     def test_connection_with_database(self):
         self.database = factory_logical.DatabaseFactory(
             name="my_db_url_name", databaseinfra=self.databaseinfra)
-        self.assertEqual("mongodb://<user>:<password>@127.0.0.1:27017/my_db_url_name",
-                         self.driver.get_connection(database=self.database))
+
+        expected_conn = ("mongodb://<user>:<password>"
+                         "@{}/my_db_url_name").format(self.mongo_endpoint)
+
+        self.assertEqual(expected_conn, self.driver.get_connection(database=self.database))
 
     @mock.patch.object(MongoDB, 'get_replica_name')
     def test_connection_with_database_and_replica(self, get_replica_name):
@@ -81,8 +92,12 @@ def test_connection_with_database_and_replica(self, get_replica_name):
         get_replica_name.return_value = 'my_repl'
         self.database = factory_logical.DatabaseFactory(
             name="my_db_url_name", databaseinfra=self.databaseinfra)
-        self.assertEqual("mongodb://<user>:<password>@127.0.0.1:27017,127.0.0.2:27018/my_db_url_name?replicaSet=my_repl",
-                         self.driver.get_connection(database=self.database))
+
+        expected_conn = ("mongodb://<user>:<password>"
+                         "@{},127.0.0.2:27018/my_db_url_name"
+                         "?replicaSet=my_repl").format(self.mongo_endpoint)
+
+        self.assertEqual(expected_conn, self.driver.get_connection(database=self.database))
 
 
 class ManageDatabaseMongoDBTestCase(AbstractTestDriverMongo):

dbaas/drivers/tests/test_driver_redis.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
-import mock
 import logging
 from django.test import TestCase
+from django.conf import settings
 from drivers import DriverFactory
 from physical.tests import factory as factory_physical
 from logical.tests import factory as factory_logical
@@ -15,10 +15,13 @@
 class AbstractTestDriverRedis(TestCase):
 
     def setUp(self):
+        redis_host = settings.REDIS_HOST
+        redis_port = settings.REDIS_PORT
+        self.endpoint = "{}:{}".format(redis_host, redis_port)
         self.databaseinfra = factory_physical.DatabaseInfraFactory(
-            password="OPlpplpooi", endpoint="127.0.0.1:6379")
+            password="OPlpplpooi", endpoint=self.endpoint)
         self.instance = factory_physical.InstanceFactory(
-            databaseinfra=self.databaseinfra, port=6379, instance_type=4)
+            databaseinfra=self.databaseinfra, port=redis_port, instance_type=4, address=redis_host)
         self.driver = Redis(databaseinfra=self.databaseinfra)
         self._redis_client = None
 
@@ -50,7 +53,7 @@ def test_instantiate_redis_using_engine_factory(self):
 
     def test_connection_string(self):
         self.assertEqual(
-            "redis://:<password>@127.0.0.1:6379/0", self.driver.get_connection())
+            "redis://:<password>@{}/0".format(self.endpoint), self.driver.get_connection())
 
     def test_get_password(self):
         self.assertEqual(
@@ -62,7 +65,7 @@ def test_get_default_port(self):
     def test_connection_with_database(self):
         self.database = factory_logical.DatabaseFactory(
             name="my_db_url_name", databaseinfra=self.databaseinfra)
-        self.assertEqual("redis://:<password>@127.0.0.1:6379/0",
+        self.assertEqual("redis://:<password>@{}/0".format(self.endpoint),
                          self.driver.get_connection(database=self.database))
 
 
@@ -73,7 +76,7 @@ class ManageDatabaseRedisTestCase(AbstractTestDriverRedis):
     def setUp(self):
         super(ManageDatabaseRedisTestCase, self).setUp()
         self.database = factory_logical.DatabaseFactory(
-            databaseinfra=self.databaseinfra)
+            databaseinfra=self.databaseinfra, address=settings.REDIS_HOST)
         # ensure database is dropped
         # get fake driver
         driver = self.databaseinfra.get_driver()

docker-compose.yml

@@ -0,0 +1,58 @@
+version: '2.1'
+
+services:
+
+    mysqldb55:
+        image: mysql:5.5
+        environment:
+            - MYSQL_ROOT_PASSWORD=123
+            - MYSQL_USER=rute
+            - MYSQL_PASSWORD=123
+            - MYSQL_DATABASE=dbaas
+        ports:
+            - "3306:3306"
+
+    mysqldb56:
+        image: mysql:5.6
+        environment:
+            - MYSQL_ROOT_PASSWORD=123
+            - MYSQL_USER=rute
+            - MYSQL_PASSWORD=123
+            - MYSQL_DATABASE=dbaas
+        ports:
+            - "3306:3306"
+
+    mysqldb57:
+        image: mysql:5.7
+        environment:
+            - MYSQL_ROOT_PASSWORD=123
+            - MYSQL_USER=rute
+            - MYSQL_PASSWORD=123
+            - MYSQL_DATABASE=dbaas
+        ports:
+            - "3306:3306"
+
+    mongodb:
+        image: mongo:3.4
+
+    redisdb:
+        image: redis:3.2
+
+    test:
+        image: dbaas_test
+        volumes:
+            - .:/code
+        depends_on:
+            - mysqldb56
+            - mongodb
+            - redisdb
+        links:
+            - mysqldb56
+            - mongodb
+            - redisdb
+
+        environment:
+            - REDIS_HOST=redisdb
+            - DBAAS_DATABASE_HOST=mysqldb56
+            - DBAAS_DATABASE_PASSWORD=123
+            - TESTS_MONGODB_HOST=mongodb

tests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+echo -n 'waiting mysql start'
+while ! mysql -h $DBAAS_DATABASE_HOST -uroot -p$DBAAS_DATABASE_PASSWORD -e "SHOW DATABASES"
+do
+  echo -n .
+  sleep 1
+done
+
+echo 'MYSQL STARTED!!!!'
+
+# Create DATABASE
+echo "create database IF NOT EXISTS dbaas;" | mysql -h $DBAAS_DATABASE_HOST -uroot -p$DBAAS_DATABASE_PASSWORD
+
+# Create user on mongo
+python add_user_admin_on_mongo.py
+
+# Run tests
+make test
+# bash