BUG: Fix bug in type conversion of arrays; add array and struct tests (googleapis#101)

* Fix array bug in type conversion; add array and struct tests

* Update changelog

Author: jasonqng

docs/source/changelog.rst

@@ -5,6 +5,7 @@ Changelog
 ------------------
 
 - Use the `google-cloud-bigquery <https://googlecloudplatform.github.io/google-cloud-python/latest/bigquery/usage.html>`__ library for API calls. The ``google-cloud-bigquery`` package is a new dependency, and dependencies on ``google-api-python-client`` and ``httplib2`` are removed. See the `installation guide <https://pandas-gbq.readthedocs.io/en/latest/install.html#dependencies>`__ for more details.  (:issue:`93`)
+- Structs and arrays are now named properly (:issue:`23`) and BigQuery functions like ``array_agg`` no longer run into errors during type conversion (:issue:`22`).
 - :func:`to_gbq` now uses a load job instead of the streaming API. Remove ``StreamingInsertError`` class, as it is no longer used by :func:`to_gbq`. (:issue:`7`, :issue:`75`)
 
 0.2.1 / 2017-11-27

pandas_gbq/gbq.py

@@ -867,11 +867,12 @@ def read_gbq(query, project_id=None, index_col=None, col_order=None,
             )
 
     # cast BOOLEAN and INTEGER columns from object to bool/int
-    # if they dont have any nulls
+    # if they dont have any nulls AND field mode is not repeated (i.e., array)
     type_map = {'BOOLEAN': bool, 'INTEGER': int}
     for field in schema['fields']:
         if field['type'].upper() in type_map and \
-                final_df[field['name']].notnull().all():
+                final_df[field['name']].notnull().all() and \
+                field['mode'] != 'repeated':
             final_df[field['name']] = \
                 final_df[field['name']].astype(type_map[field['type'].upper()])
 

pandas_gbq/tests/test_gbq.py

@@ -920,6 +920,56 @@ def test_query_response_bytes(self):
         assert self.gbq_connector.sizeof_fmt(1.208926E24) == "1.0 YB"
         assert self.gbq_connector.sizeof_fmt(1.208926E28) == "10000.0 YB"
 
+    def test_struct(self):
+        query = """SELECT 1 int_field,
+                   STRUCT("a" as letter, 1 as num) struct_field"""
+        df = gbq.read_gbq(query, project_id=_get_project_id(),
+                          private_key=_get_private_key_path(),
+                          dialect='standard')
+        tm.assert_frame_equal(df, DataFrame([[1, {"letter": "a", "num": 1}]],
+                              columns=["int_field", "struct_field"]))
+
+    def test_array(self):
+        query = """select ["a","x","b","y","c","z"] as letters"""
+        df = gbq.read_gbq(query, project_id=_get_project_id(),
+                          private_key=_get_private_key_path(),
+                          dialect='standard')
+        tm.assert_frame_equal(df, DataFrame([[["a", "x", "b", "y", "c", "z"]]],
+                              columns=["letters"]))
+
+    def test_array_length_zero(self):
+        query = """WITH t as (
+                   SELECT "a" letter, [""] as array_field
+                   UNION ALL
+                   SELECT "b" letter, [] as array_field)
+
+                   select letter, array_field, array_length(array_field) len
+                   from t
+                   order by letter ASC"""
+        df = gbq.read_gbq(query, project_id=_get_project_id(),
+                          private_key=_get_private_key_path(),
+                          dialect='standard')
+        tm.assert_frame_equal(df, DataFrame([["a", [""], 1], ["b", [], 0]],
+                              columns=["letter", "array_field", "len"]))
+
+    def test_array_agg(self):
+        query = """WITH t as (
+                SELECT "a" letter, 1 num
+                UNION ALL
+                SELECT "b" letter, 2 num
+                UNION ALL
+                SELECT "a" letter, 3 num)
+
+                select letter, array_agg(num order by num ASC) numbers
+                from t
+                group by letter
+                order by letter ASC"""
+        df = gbq.read_gbq(query, project_id=_get_project_id(),
+                          private_key=_get_private_key_path(),
+                          dialect='standard')
+        tm.assert_frame_equal(df, DataFrame([["a", [1, 3]], ["b", [2]]],
+                              columns=["letter", "numbers"]))
+
 
 class TestToGBQIntegrationWithServiceAccountKeyPath(object):
     # Changes to BigQuery table schema may take up to 2 minutes as of May 2015