fixing issue ps0uth#9, adding travis-ci

Author: ps0uth

.gitignore

@@ -13,7 +13,7 @@ MANIFEST
 .\#*
 *~
 *.egg-info
+mysql2pgsql.yml
 *.yaml
-*.yml
 docs/_build
 tests/reports

.travis.yml

@@ -0,0 +1,21 @@
+language: python
+python:
+  - "2.6"
+  - "2.7"
+install:
+  - pip install . --use-mirrors
+  - pip install -r requirements.txt --use-mirrors
+script: nosetests
+before_script:
+  - mysql -e 'create database mysql2pgsql;'
+  - psql -c 'create database mysql2pgsql;' -U postgres
+env:
+  - DB=mysql
+  - DB=postgres
+branches:
+  only:
+    - master
+    - develop
+notifications:
+  email:
+    - [email protected]

bin/py-mysql2pgsql

@@ -1,54 +1,33 @@
 #! /usr/bin/env python
 import os
 import sys
+import argparse
 import mysql2pgsql
 from mysql2pgsql.lib.errors import ConfigurationFileInitialized
 
 if __name__ == '__main__':
     description = 'Tool for migrating/converting data from mysql to postgresql.'
     epilog = 'https://github.com/philipsoutham/py-mysql2pgsql'
-    try:
-        import argparse
-        parser = argparse.ArgumentParser(
-            description=description,
-            epilog=epilog)
-        parser.add_argument(
-            '-v', '--verbose',
-            action='store_true',
-            help='Show progress of data migration.'
-            )
-        parser.add_argument(
-            '-f', '--file',
-            default='mysql2pgsql.yml',
-            help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.',
-            )
-        parser.add_argument(
-            '-V', '--version',
-            action='store_true',
-            help='Print version and exit.'
-            )
-        options = parser.parse_args()
-    except ImportError:
-        import optparse
-        parser = optparse.OptionParser(
-            description=description,
-            epilog=epilog)
-        parser.add_argument(
-            '-v', '--verbose',
-            action='store_true',
-            help='Show progress of data migration.'
-            )
-        parser.add_argument(
-            '-f', '--file',
-            default='mysql2pgsql.yml',
-            help='Location of configuration file (default: %default). If none exists at that path, one will be created for you.',
-            )
-        parser.add_argument(
-            '-V', '--version',
-            action='store_true',
-            help='Print version and exit.'
-            )
-        options, args = parser.parse_args()
+
+    parser = argparse.ArgumentParser(
+        description=description,
+        epilog=epilog)
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help='Show progress of data migration.'
+        )
+    parser.add_argument(
+        '-f', '--file',
+        default='mysql2pgsql.yml',
+        help='Location of configuration file (default: %(default)s). If none exists at that path, one will be created for you.',
+        )
+    parser.add_argument(
+        '-V', '--version',
+        action='store_true',
+        help='Print version and exit.'
+        )
+    options = parser.parse_args()
 
     if options.version:
         # Someone wants to know the version, print and exit

mysql2pgsql/lib/postgres_db_writer.py

@@ -52,7 +52,7 @@ def readline(self, *args, **kwargs):
                 try:
                     return '%s\n' % ('\t'.join(row))
                 except UnicodeDecodeError:
-                    return '%s\n' % ('\t'.join(row)).decode('utf-8')
+                    return '%s\n' % ('\t'.join(r.decode('utf8') for r in row))
             finally:
                 if self.verbose:
                     if (self.idx % 20000) == 0:
@@ -69,6 +69,7 @@ def read(self, *args, **kwargs):
             return self.readline(*args, **kwargs)
 
     def __init__(self, db_options, verbose=False):
+        super(PostgresDbWriter, self).__init__()
         self.verbose = verbose
         self.db_options = {
             'host': db_options['hostname'],
@@ -133,7 +134,7 @@ def truncate(self, table):
 
         Returns None
         """
-        truncate_sql, serial_key_sql = super(self.__class__, self).truncate(table)
+        truncate_sql, serial_key_sql = super(PostgresDbWriter, self).truncate(table)
         self.execute(truncate_sql)
         if serial_key_sql:
             self.execute(serial_key_sql)
@@ -147,7 +148,7 @@ def write_table(self, table):
 
         Returns None
         """
-        table_sql, serial_key_sql = super(self.__class__, self).write_table(table)
+        table_sql, serial_key_sql = super(PostgresDbWriter, self).write_table(table)
         for sql in serial_key_sql + table_sql:
             self.execute(sql)
 
@@ -160,7 +161,7 @@ def write_indexes(self, table):
 
         Returns None
         """
-        index_sql = super(self.__class__, self).write_indexes(table)
+        index_sql = super(PostgresDbWriter, self).write_indexes(table)
         for sql in index_sql:
             self.execute(sql)
 
@@ -173,7 +174,7 @@ def write_constraints(self, table):
 
         Returns None
         """
-        constraint_sql = super(self.__class__, self).write_constraints(table)
+        constraint_sql = super(PostgresDbWriter, self).write_constraints(table)
         for sql in constraint_sql:
             self.execute(sql)
 

mysql2pgsql/lib/postgres_file_writer.py

@@ -21,6 +21,7 @@ class PostgresFileWriter(PostgresWriter):
     verbose = None
 
     def __init__(self, output_file, verbose=False):
+        super(PostgresFileWriter, self).__init__()
         self.verbose = verbose
         self.f = output_file
         self.f.write("""
@@ -40,7 +41,7 @@ def truncate(self, table):
 
         Returns None
         """
-        truncate_sql, serial_key_sql = super(self.__class__, self).truncate(table)
+        truncate_sql, serial_key_sql = super(PostgresFileWriter, self).truncate(table)
         self.f.write("""
 -- TRUNCATE %(table_name)s;
 %(truncate_sql)s
@@ -61,7 +62,7 @@ def write_table(self, table):
 
         Returns None
         """
-        table_sql, serial_key_sql = super(self.__class__, self).write_table(table)
+        table_sql, serial_key_sql = super(PostgresFileWriter, self).write_table(table)
         if serial_key_sql:
             self.f.write("""
 %(serial_key_sql)s
@@ -86,7 +87,7 @@ def write_indexes(self, table):
 
         Returns None
         """
-        self.f.write('\n'.join(super(self.__class__, self).write_indexes(table)))
+        self.f.write('\n'.join(super(PostgresFileWriter, self).write_indexes(table)))
 
     @status_logger
     def write_constraints(self, table):
@@ -97,7 +98,7 @@ def write_constraints(self, table):
 
         Returns None
         """
-        self.f.write('\n'.join(super(self.__class__, self).write_constraints(table)))
+        self.f.write('\n'.join(super(PostgresFileWriter, self).write_constraints(table)))
 
     @status_logger
     def write_contents(self, table, reader):
@@ -129,22 +130,22 @@ def write_contents(self, table, reader):
             start_time = tt()
             prev_val_len = 0
             prev_row_count = 0
-        for i, row in enumerate(reader.read(table)):
+        for i, row in enumerate(reader.read(table), 1):
             row = list(row)
             pr(table, row)
             try:
-                f_write('%s\n' % ('\t'.join(row)))
+                f_write(u'%s\n' % (u'\t'.join(row)))
             except UnicodeDecodeError:
-                f_write('%s\n' % ('\t'.join(row)).decode('utf-8'))
+                f_write(u'%s\n' % (u'\t'.join(r.decode('utf-8') for r in row)))
             if verbose:
-                if ((i + 1) % 20000) == 0:
+                if (i % 20000) == 0:
                     now = tt()
                     elapsed = now - start_time
-                    val = '%.2f rows/sec [%s] ' % (((i + 1) - prev_row_count) / elapsed, (i + 1))
+                    val = '%.2f rows/sec [%s] ' % ((i - prev_row_count) / elapsed, i)
                     print_row_progress('%s%s' % (("\b" * prev_val_len), val))
                     prev_val_len = len(val) + 3
                     start_time = now
-                    prev_row_count = i + 1
+                    prev_row_count = i
 
         f_write("\\.\n\n")
         if verbose:

mysql2pgsql/lib/postgres_writer.py

@@ -6,18 +6,21 @@
 
 from psycopg2.extensions import QuotedString, Binary, AsIs
 
-from .writer import Writer
 
-
-class PostgresWriter(Writer):
+class PostgresWriter(object):
     """Base class for :py:class:`mysql2pgsql.lib.postgres_file_writer.PostgresFileWriter`
     and :py:class:`mysql2pgsql.lib.postgres_db_writer.PostgresDbWriter`.
     """
+    def __init__(self):
+        self.column_types = {}
+
     def column_description(self, column):
         return '"%s" %s' % (column['name'], self.column_type_info(column))
 
     def column_type(self, column):
-        return self.column_type_info(column).split(" ")[0]
+        hash_key = hash(frozenset(column.items()))
+        self.column_types[hash_key] = self.column_type_info(column).split(" ")[0]
+        return self.column_types[hash_key]
 
     def column_type_info(self, column):
         """
@@ -110,14 +113,15 @@ def process_row(self, table, row):
         sending to PostgreSQL via the copy command
         """
         for index, column in enumerate(table.columns):
-            column_type = self.column_type(column)
+            hash_key = hash(frozenset(column.items()))
+            column_type = self.column_types[hash_key] if hash_key in self.column_types else self.column_type(column)
             if row[index] == None and ('timestamp' not in column_type or not column['default']):
                 row[index] = '\N'
             elif row[index] == None and column['default']:
                 row[index] = '1970-01-01 00:00:00'
             elif 'bit' in column_type:
                 row[index] = bin(ord(row[index]))[2:]
-            elif row[index].__class__ in (str, unicode):
+            elif isinstance(row[index], (str, unicode, basestring)):
                 if column_type == 'bytea':
                     row[index] = Binary(row[index]).getquoted()[1:-8] if row[index] else row[index]
                 elif 'text[' in column_type:
@@ -127,9 +131,9 @@ def process_row(self, table, row):
             elif column_type == 'boolean':
                 # We got here because you used a tinyint(1), if you didn't want a bool, don't use that type
                 row[index] = 't' if row[index] not in (None, 0) else 'f' if row[index] == 0 else row[index]
-            elif row[index].__class__ in (date, datetime):
+            elif isinstance(row[index], (date, datetime)):
                 row[index] = row[index].isoformat()
-            elif row[index].__class__ is timedelta:
+            elif isinstance(row[index], timedelta):
                 row[index] = datetime.utcfromtimestamp(row[index].total_seconds()).time().isoformat()
             else:
                 row[index] = AsIs(row[index]).getquoted()

mysql2pgsql/lib/writer.py

@@ -0,0 +1,5 @@
+mysql-python
+psycopg2
+pyyaml
+termcolor
+argparse

requirements.txt

@@ -5,14 +5,15 @@
     'mysql-python>=1.2.3', 
     'psycopg2>=2.4.2',
     'pyyaml>=3.10.0',
+    'argparse',
 ]
 
 if os.name == 'posix':
     install_requires.append('termcolor>=1.1.0')
 
 setup(
     name='py-mysql2pgsql',
-    version='0.1.3',
+    version='0.1.4',
     description='Tool for migrating/converting from mysql to postgresql.',
     long_description=open('README.rst').read(),
     license='MIT License',

setup.py

@@ -0,0 +1,38 @@
+
+# if a socket is specified we will use that
+# if tcp is chosen you can use compression
+mysql:
+ hostname: 127.0.0.1
+ port: 3306
+ socket:
+ username: root
+ password: 
+ database: mysql2pgsql
+ compress: false
+destination:
+ # if file is given, output goes to file, else postgres
+ file: 
+ postgres:
+  hostname: 127.0.0.1
+  port: 5432
+  username: postgres
+  password: 
+  database: mysql2pgsql
+
+# if tables is given, only the listed tables will be converted.  leave empty to convert all tables.
+#only_tables:
+#- table1
+#- table2
+# if exclude_tables is given, exclude the listed tables from the conversion.
+#exclude_tables:
+#- table3
+#- table4
+
+# if supress_data is true, only the schema definition will be exported/migrated, and not the data
+supress_data: false
+
+# if supress_ddl is true, only the data will be exported/imported, and not the schema
+supress_ddl: false
+
+# if force_truncate is true, forces a table truncate before table loading
+force_truncate: false