Merge pull request turi-code#350 from hoytak/internal-merge-by-hoyt-and-a-squirrel

Merge of updates in internal repository.

Author: Yucheng Low

oss_src/unity/extensions/json/decoder.cpp

@@ -78,7 +78,10 @@ static variant_type _list_from_serializable(const flexible_type& data, const sch
     _check_type(schema, JSON::types::VECTOR);
     flex_vec ret;
     for (const auto& value : data_list) {
-      ret.push_back(value);
+      schema_t value_schema;
+      value_schema.insert(std::make_pair("type", JSON::types::FLOAT));
+      variant_type deserialized_value = _any_from_serializable(value, value_schema);
+      ret.push_back(variant_get_value<flex_float>(deserialized_value));
     }
     return ret;
   } else {

oss_src/unity/extensions/json/encoder.cpp

@@ -41,7 +41,10 @@ static void _to_serializable(flexible_type& data, schema_t& schema, const flex_v
   schema.insert(std::make_pair("type", JSON::types::VECTOR));
   flex_list ret;
   for (const auto& value : input) {
-    ret.push_back(value);
+    flexible_type serialized_value;
+    schema_t serialized_schema;
+    _any_to_serializable(serialized_value, serialized_schema, value);
+    ret.push_back(serialized_value);
   }
   data = ret;
 }

oss_src/unity/lib/version_number.hpp

@@ -6,4 +6,4 @@
  * of the BSD license. See the LICENSE file for details.
  */
 
-#define __UNITY_VERSION__ "1.9"//#{{VERSION_STRING}}
+#define __UNITY_VERSION__ "1.10"//#{{VERSION_STRING}}

oss_src/unity/python/sframe/cython/cy_flexible_type.pxd

@@ -88,6 +88,7 @@ cdef extern from "<flexible_type/flexible_type.hpp>" namespace "graphlab":
         void reset()
 
     cdef flexible_type FLEX_UNDEFINED
+
     cdef void swap "std::swap"(flexible_type&, flexible_type&)
 
     cdef int TIMEZONE_RESOLUTION_IN_SECONDS "graphlab::flex_date_time::TIMEZONE_RESOLUTION_IN_SECONDS"
@@ -102,6 +103,7 @@ ctypedef map[string, flexible_type] gl_options_map
 # If we just want to work with the enum types, use these.
 cdef flex_type_enum flex_type_enum_from_pytype(type t) except *
 cdef type pytype_from_flex_type_enum(flex_type_enum e)
+cpdef type pytype_from_type_name(str)
 cpdef type pytype_from_dtype(object dt)
 cdef flex_type_enum flex_type_from_dtype(object dt)
 cpdef type pytype_from_array_typecode(str a)

oss_src/unity/python/sframe/cython/cy_flexible_type.pyx

@@ -427,6 +427,34 @@ _type_lookup_by_type_enum[<int>IMAGE]     = _image_type
 cdef type pytype_from_flex_type_enum(flex_type_enum e):
     return _type_lookup_by_type_enum[<int> e]
 
+################################################################################
+# Name of enum type only
+
+cdef dict _type_lookup_by_type_enum_name = {
+    "integer"    : int,
+    "datetime"   : datetime_type,
+    "dictionary" : dict,
+    "float"      : float,
+    "string"     : str,
+    "array"      : array_type,
+    "list"       : list,
+    "image"      : _image_type,
+    "undefined"  : none_type}
+    
+# Also add in the names of each of the types in order to recognize
+# them as well.
+for _t in list(_type_lookup_by_type_enum_name.values()):
+    _type_lookup_by_type_enum_name[_t.__name__.lower()] = _t
+    
+cpdef type pytype_from_type_name(str s):
+    global _type_lookup_by_type_enum_name
+    try:
+        return _type_lookup_by_type_enum_name[s.lower()]
+    except KeyError:
+        raise ValueError("'%s' not a recogizable type name; valid names are %s."
+                         % (s, ','.join(sorted(set(_type_lookup_by_type_enum_name.keys())))))
+
+
 ################################################################################
 # Looking up the translation code to enum type
 

oss_src/unity/python/sframe/data_structures/image.py

@@ -6,9 +6,9 @@
 of the BSD license. See the LICENSE file for details.
 '''
 try:
-    import io as _StringIO
+    from io import BytesIO as _StringIO
 except ImportError:
-    import StringIO as _StringIO
+    from StringIO import StringIO as _StringIO
 
 from ..deps import numpy as _np, HAS_NUMPY as _HAS_NUMPY
 import array as _array
@@ -175,8 +175,6 @@ def pixel_data(self):
         """
 
         try:
-            global _PIL_image
-            from PIL import Image as _PIL_image
             pil_img = self._to_pil_image()
             if _HAS_NUMPY:
                 return _np.asarray(pil_img)
@@ -209,17 +207,18 @@ def __repr__(self):
         return ret
 
     def _to_pil_image(self):
+        from PIL import Image as _PIL_image
         if self._format_enum == _format[_RAW]:
             if self.channels == 1:
-                img = _PIL_image.frombytes('L', (self._width, self._height), str(self._image_data))
+                img = _PIL_image.frombytes('L', (self._width, self._height), bytes(self._image_data))
             elif self.channels == 3:
-                img = _PIL_image.frombytes('RGB', (self._width, self._height), str(self._image_data))
+                img = _PIL_image.frombytes('RGB', (self._width, self._height), bytes(self._image_data))
             elif self.channels == 4:
-                img = _PIL_image.frombytes('RGBA', (self._width, self._height), str(self._image_data))
+                img = _PIL_image.frombytes('RGBA', (self._width, self._height), bytes(self._image_data))
             else:
                 raise ValueError('Unsupported channel size: ' + str(self.channels))
         else:
-            img = _PIL_image.open(_StringIO.StringIO(self._image_data))
+            img = _PIL_image.open(_StringIO(self._image_data))
         return img
 
     def show(self):
@@ -248,8 +247,6 @@ def show(self):
 @_show_dispatch(Image)
 def show(obj):
     try:
-        global _PIL_image
-        from PIL import Image as _PIL_image
         img = obj._to_pil_image()
         img.show()
     except ImportError:

oss_src/unity/python/sframe/data_structures/sarray.py

@@ -562,13 +562,13 @@ def from_avro(cls, filename):
     @classmethod
     def where(cls, condition, istrue, isfalse, dtype=None):
         """
-        Selects elements from either istrue or isfalse depending on the value
+        Selects elements from either istrue or isfalse depending on the value 
         of the condition SArray.
 
         Parameters
         ----------
         condition : SArray
-        An SArray of values such that for each value, if non-zero, yields a
+        An SArray of values such that for each value, if non-zero, yields a 
         value from istrue, otherwise from isfalse.
 
         istrue : SArray or constant
@@ -585,8 +585,8 @@ def where(cls, condition, istrue, isfalse, dtype=None):
 
         Examples
         --------
-
-        Returns an SArray with the same values as g with values above 10
+        
+        Returns an SArray with the same values as g with values above 10 
         clipped to 10
 
         >>> g = SArray([6,7,8,9,10,11,12,13])
@@ -595,7 +595,7 @@ def where(cls, condition, istrue, isfalse, dtype=None):
         Rows: 8
         [6, 7, 8, 9, 10, 10, 10, 10]
 
-        Returns an SArray with the same values as g with values below 10
+        Returns an SArray with the same values as g with values below 10 
         clipped to 10
 
         >>> SArray.where(g > 10, g, 10)
@@ -832,15 +832,6 @@ def contains(self, item):
         If the current SArray contains dictionaries, this produces a 1
         for each row if 'item' is a key in the dictionary.
 
-        If the current SArray contains strings and item is a string. Produces a 1
-        for each row if 'item' is a substring of the row and 0 otherwise.
-
-        If the current SArray contains list or arrays, this produces a 1
-        for each row if 'item' is an element of the list or array.
-
-        If the current SArray contains dictionaries, this produces a 1
-        for each row if 'item' is a key in the dictionary.
-
         Parameters
         ----------
         item : any type
@@ -849,7 +840,7 @@ def contains(self, item):
         Returns
         -------
         out : SArray
-            A binary SArray where a non-zero value denotes that the item
+            A binary SArray where a non-zero value denotes that the item 
             was found in the row. And 0 if it is not found.
 
         Examples
@@ -869,7 +860,7 @@ def contains(self, item):
 
         See Also
         --------
-        is_in
+        is_in 
         """
         return SArray(_proxy = self.__proxy__.left_scalar_operator(item, 'in'))
 
@@ -881,7 +872,7 @@ def is_in(self, other):
         Conceptually equivalent to:
 
         >>> sa.apply(lambda x: x in other)
-
+        
         If the current SArray contains strings and other is a string. Produces a 1
         for each row if the row is a substring of 'other', and 0 otherwise.
 
@@ -1991,7 +1982,7 @@ def hash(self, seed=0):
         Parameters
         ----------
         seed : int
-            Defaults to 0. Can be changed to different values to get
+            Defaults to 0. Can be changed to different values to get 
             different hash results.
 
         Returns

oss_src/unity/python/sframe/test/test_flexible_type.py

@@ -18,6 +18,7 @@
 from ..cython.cy_flexible_type import infer_type_of_list
 from ..cython.cy_flexible_type import _get_inferred_column_type, _all_convertable
 from ..cython.cy_flexible_type import _check_ft_pyobject_hint_path
+from ..cython.cy_flexible_type import pytype_from_type_name
 from ..util.timezone import GMT
 import datetime
 from itertools import product
@@ -525,3 +526,16 @@ def test_flexible_type_hint(self):
         _check_ft_pyobject_hint_path({1:1}, dict)
         _check_ft_pyobject_hint_path(array.array('i', [1,2]), array.array)
         _check_ft_pyobject_hint_path(array.array('d', [1,2]), array.array)
+
+    def test_pytype_from_type_name(self):
+
+        self.assertEquals(pytype_from_type_name("str"), str)
+        self.assertEquals(pytype_from_type_name("string"), str)
+        self.assertEquals(pytype_from_type_name("float"), float)
+        self.assertEquals(pytype_from_type_name("datetime"), datetime.datetime)
+        self.assertEquals(pytype_from_type_name("image"), image.Image)
+        self.assertEquals(pytype_from_type_name("list"), list)
+        self.assertEquals(pytype_from_type_name("undefined"), type(None))
+
+        self.assertRaises(ValueError, lambda: pytype_from_type_name("happiness"))
+        

oss_src/unity/python/sframe/test/test_image_type.py

@@ -198,6 +198,17 @@ def test_lambda(self):
         for i in range(len(sa_width)):
             self.assertEqual(sa[i].width, sa_width[i])
 
+        # Lambda returning height
+        sa_height = sa.apply(lambda x: x.height)
+        for i in range(len(sa_height)):
+            self.assertEqual(sa[i].height, sa_height[i])
+
+        # Lambda returning channels
+        sa_channels = sa.apply(lambda x: x.channels)
+        for i in range(len(sa_channels)):
+            self.assertEqual(sa[i].channels, sa_channels[i])
+
+
         # Lambda returning resized self
         sa_resized = sa.apply(lambda x: image_analysis.resize(x, int(x.width / 2), int(x.height / 2)))
         for i in range(len(sa_resized)):
@@ -218,3 +229,15 @@ def test_generate_mean(self):
 
         self.assertEqual(average, img3)
         self.assertEqual(average2, img2)
+
+    def test_pixel_data(self):
+        fifties = bytearray([50] * 100)
+        img = image.Image(_image_data=fifties, _channels=1, _height=1, _width=100, _image_data_size=100, _format_enum=2)
+        pixel_data = img.pixel_data.flatten()
+        self.assertEqual(pixel_data.shape, (100,))
+
+        self.assertEqual(len(pixel_data), len(fifties))
+        for p in range(len(pixel_data)):
+            self.assertEqual(pixel_data[p], 50)
+
+

oss_src/unity/python/sframe/test/test_json.py

@@ -10,14 +10,15 @@
 
 import array
 import datetime
+import json # Python built-in JSON module
 import math
 import os
 import pytz
 import sys
 import unittest
 
 from . import util
-from .. import _json
+from .. import _json # graphlab._json
 
 if sys.version_info.major == 3:
     long = int
@@ -74,6 +75,11 @@ def _run_test_case(self, value):
         # test that JSON serialization is invertible with respect to both
         # value and type.
         (data, schema) = _json.to_serializable(value)
+
+        # ensure that resulting value is actually naively serializable
+        data = json.loads(json.dumps(data, allow_nan=False))
+        schema = json.loads(json.dumps(schema, allow_nan=False))
+
         #print("----------------------------------")
         #print("Value: %s" % value)
         #print("Serializable Data: %s" % data)
@@ -125,6 +131,7 @@ def test_vec_to_json(self):
             array.array('d'),
             array.array('d', [1.5]),
             array.array('d', [2.1,2.5,3.1]),
+            array.array('d', [float('-inf'), float('inf')]),
         ]]
 
     def test_list_to_json(self):
@@ -139,6 +146,7 @@ def test_list_to_json(self):
             ["hello", 3, None],
             [3.14159, None],
             [{}, {'x': 1, 'y': 2}],
+            ["hello", float('-inf'), float('inf')],
         ]]
 
     def test_dict_to_json(self):