【add】highway_conv2d-layer

Author: imhuay

code/my_tensorflow/demo.py

@@ -1,3 +1,10 @@
+from typing import (
+    TypeVar, Iterator, Iterable, overload, Container,
+    Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic,
+    Set, AbstractSet, FrozenSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat,
+    SupportsBytes, SupportsAbs, SupportsRound, IO, Union, ItemsView, KeysView, ValuesView,
+    ByteString, Optional, AnyStr, Type,
+)
 from src.utils import tf_dtype
 
 print(tf_dtype)
@@ -6,8 +13,10 @@
 import numpy as np
 
 from src.layers import multi_dense, Dense
-from tensorlayer.layers import DenseLayer
-from keras.layers import Dense
+
+from tensorlayer.layers import DenseLayer, Conv2dLayer
+
+from keras.layers import Dense, Conv2D
 
 import keras.initializers
 
@@ -25,3 +34,22 @@
 
 sess.run(o)
 
+
+def f(x):
+    """
+
+    Args:
+        x(function):
+
+    Returns:
+
+    """
+
+
+def b():
+    """"""
+
+
+f(1)
+
+sorted()

code/my_tensorflow/src/layers/__init__.py

@@ -1,2 +1,7 @@
+"""常见层与常用层的写法
+"""
+
 from .dense import *
+from .cnn import *
 from .highway import *
+

code/my_tensorflow/src/layers/cnn.py

@@ -0,0 +1,53 @@
+"""卷积层
+
+References:
+    tensorlayer.layers.Conv2dLayer
+"""
+import tensorflow as tf
+
+from ..activations import relu
+from ..utils import get_wb
+
+
+def conv2d(x, kernel_size, out_channels,
+           act_fn=relu,
+           strides=1,
+           padding="SAME",
+           name=None,
+           reuse=False):
+    """2-D 卷积层
+    Input shape:  [batch_size, in_w, in_h, in_channels]
+    Output shape: [batch_size, out_w, out_h, out_channels]
+
+    Args:
+        x(tf.Tensor):
+        kernel_size(int or list of int):
+        out_channels(int):
+        act_fn(function):
+        strides(int or list of int):
+        padding(str):
+        name(str):
+        reuse(bool):
+
+    Returns:
+
+    """
+    if isinstance(kernel_size, int):
+        kernel_size = [kernel_size] * 2
+    if isinstance(strides, int):
+        strides = [strides] * 4
+
+    assert len(kernel_size) == 2
+    assert len(strides) == 4
+
+    in_channels = int(x.get_shape()[-1])
+    kernel_shape = list(kernel_size) + [in_channels, out_channels]
+
+    with tf.variable_scope(name or "conv2d", reuse=reuse):
+        W, b = get_wb(kernel_shape)
+
+        o = tf.nn.conv2d(x, W, strides=strides, padding=padding) + b
+        o = act_fn(o)
+
+    return o
+

code/my_tensorflow/src/layers/highway.py

@@ -1,4 +1,6 @@
-"""高速网络
+"""高速网络 Highway Network
+
+注意 x 经过 Highway 之后维度应该保持不变
 
 References:
     https://github.com/fomorians/highway-fcn
@@ -18,27 +20,66 @@ def highway_dense(x, act_fn=relu, carry_bias=-1.0, name=None):
         `o = H(x, W)T(x, W) + x(1 - T(x, W))`
     """
     n_input = int(x.get_shape()[-1])
-    with tf.variable_scope(name or "highway"):
+    with tf.variable_scope(name or "highway_dense"):
         W, b = get_wb([n_input, n_input])
 
         with tf.variable_scope("transform"):
             W_T, b_T = get_wb([n_input, n_input], b_initializer=tf.initializers.constant(carry_bias))
 
         H = act_fn(tf.matmul(x, W) + b)
         T = sigmoid(tf.matmul(x, W_T) + b_T)
-
         o = tf.multiply(H, T) + tf.multiply(x, (1. - T))
 
     return o
 
 
 def multi_highway_dense(x, n_layer, act_fn=relu, carry_bias=-1.0, name=None):
-    """多层 highway
+    """多层 highway_dense
     Input shape:  [batch_size, n_input]
     Output shape: [batch_size, n_input]
     """
-    name = name or "highway"
+    name = name or "highway_dense"
     for i in range(n_layer):
         x = highway_dense(x, act_fn=act_fn, carry_bias=carry_bias, name="{}-{}".format(name, i))
 
     return x
+
+
+def highway_conv2d(x, kernel_size,
+                   carry_bias=-1.0,
+                   act_fn=relu,
+                   strides=1,
+                   padding="SAME",
+                   name=None):
+    """用于 conv2d 的 highway
+    Input shape:  [batch_size, in_w, in_h, in_channels]
+    Output shape: [batch_size, in_w, in_w, in_channels]
+
+    公式
+        `o = H(x, W)T(x, W) + x(1 - T(x, W))`
+    """
+    if isinstance(kernel_size, int):
+        kernel_size = [kernel_size] * 2
+    if isinstance(strides, int):
+        strides = [strides] * 4
+
+    assert len(kernel_size) == 2
+    assert len(strides) == 4
+
+    in_channels = int(x.get_shape()[-1])
+    kernel_shape = list(kernel_size) + [in_channels, in_channels]
+
+    with tf.variable_scope(name or "highway_conv2d"):
+        W, b = get_wb(kernel_shape, b_initializer=tf.initializers.constant(carry_bias))
+
+        with tf.variable_scope("transform"):
+            W_T, b_T = get_wb(kernel_shape)
+
+        H = act_fn(tf.nn.conv2d(x, W, strides=strides, padding=padding) + b)
+        T = sigmoid(tf.nn.conv2d(x, W_T, strides=strides, padding=padding) + b_T)
+        o = tf.multiply(H, T) + tf.multiply(x, (1. - T))
+    return o
+
+
+# TODO(huay): 因为卷积的参数比较复杂，如果需要多层 highway_conv2d，不如单独设置参数
+# def multi_highway_conv2d():

code/my_tensorflow/src/utils/__init__.py

@@ -23,10 +23,9 @@ def get_wb(shape,
            w_regularizer=l2_regularizer,
            b_regularizer=l2_regularizer):
     """"""
-    n_in, n_unit = shape
-    W = tf.get_variable('W', shape=[n_in, n_unit],
+    W = tf.get_variable('W', shape=shape,
                         dtype=tf_dtype, initializer=w_initializer, regularizer=w_regularizer)
-    b = tf.get_variable('b', shape=[n_unit],
+    b = tf.get_variable('b', shape=shape[-1:],
                         dtype=tf_dtype, initializer=b_initializer, regularizer=b_regularizer)
     return W, b
 
@@ -48,7 +47,7 @@ def get_params_dict():
     return param_dict
 
 
-def print_get_params_dict():
+def print_params_dict():
     """"""
     param_dict = get_params_dict()
     # pprint(param_dict, indent=2)