Made json to csv converter more memory efficient

Author: Scott Clark

json_to_csv_converter.py

@@ -10,18 +10,26 @@
 import simplejson as json
 
 
-def read_file(file_path):
-    """Read in the json dataset file and return a list of python dicts."""
-    file_contents = []
+def read_and_write_file(json_file_path, csv_file_path, column_names):
+    """Read in the json dataset file and write it out to a csv file, given the column names."""
+    with open(csv_file_path, 'wb+') as fout:
+        csv_file = csv.writer(fout)
+        csv_file.writerow(list(column_names))
+        with open(json_file_path) as fin:
+            for line in fin:
+                line_contents = json.loads(line)
+                csv_file.writerow(get_row(line_contents, column_names))
+
+def get_superset_of_column_names_from_file(json_file_path):
+    """Read in the json dataset file and return the superset of column names."""
     column_names = set()
-    with open(file_path) as fin:
+    with open(json_file_path) as fin:
         for line in fin:
             line_contents = json.loads(line)
             column_names.update(
                     set(get_column_names(line_contents).keys())
                     )
-            file_contents.append(line_contents)
-    return file_contents, column_names
+    return column_names
 
 def get_column_names(line_contents, parent_key=''):
     """Return a list of flattened key names given a dict.
@@ -93,15 +101,6 @@ def get_row(line_contents, column_names):
             row.append('')
     return row
 
-def write_file(file_path, file_contents, column_names):
-    """Create and write a csv file given file_contents of our json dataset file and column names."""
-    csv_file = csv.writer(open('file_path', 'wb+'))
-    with open(file_path, 'wb+') as fin:
-        csv_file = csv.writer(fin)
-        csv_file.writerow(list(column_names))
-        for line_contents in file_contents:
-            csv_file.writerow(get_row(line_contents, column_names))
-
 if __name__ == '__main__':
     """Convert a yelp dataset file from json to csv."""
 
@@ -120,5 +119,5 @@ def write_file(file_path, file_contents, column_names):
     json_file = args.json_file
     csv_file = '{0}.csv'.format(json_file.split('.json')[0])
 
-    file_contents, column_names = read_file(json_file)
-    write_file(csv_file, file_contents, column_names)
+    column_names = get_superset_of_column_names_from_file(json_file)
+    read_and_write_file(json_file, csv_file, column_names)