Neuron.py

import pandas as pd
from numpy import *
from Logging import Log

from numpy import eye, zeros, ones, dot, exp
from numpy.random import rand, randn
from numpy.linalg import inv


###############################################################
#                                                             #
#                            LNU                              #
#                                                             #
###############################################################

class LNUGD:

  w = []

  def countSerie(self, Y, X, **kwargs):

    # optional parameters
    prediction =  kwargs.get('prediction', 1)
    mu = kwargs.get('mu', 0.1)
    self.w = kwargs.get('weigths', self.w)
    logging = kwargs.get('logging', False)

    log = Log(logging, "LNU.count()")
    log.message("start")

    
    # for counting
    nx = len(X) * 4 + 1 #lenX * X width + 1
    nw = nx
    N=len(Y)
    e=zeros(N) # "prazdna " array pro vypocet chyb v delce dat

    yn = Y.copy() # init vystup neuronu (site)
    yn[0] = float('nan')
    log.message(yn[1])


    x = [1.]
    x = array(x)
    if len(self.w) < 1:
      self.w = random.randn(nw)/nw
      print self.w

      self.w = random.randn(nw)
      print self.w

    # for record
    Wall = []
    Wall.append(self.w) #initial weigth
    MSE = []


    for k in range(N-prediction):
      ## input vector
      x = [1.]
      for i in range(len(X)):
        x = concatenate((x, [ X[i][k], X[i][k-1], X[i][k-2], X[i][k-3] ]))

      # neuron count
      yn[k+prediction]=dot(self.w,x)


      if not isnan(yn[k]):
        e[k]=Y[k]-yn[k]

        #update weights
        dydw=x     # pro LNU dy/dw = x
        dw=mu/(1+sum(x**2))*e[k]*dydw  # normalizovany GD
        self.w=self.w+dw

      #for plotting
      Wall.append(self.w)
    
    MSE.append(sum(e**2)/N)    
    
    log.message("done \n")
    return yn, self.w, Wall, MSE, e


  def train(self, Y_train, X_train, **kwargs):

    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)

    log = Log(logging, "LNU.train()")
    log.message("start")

    Wall = []
    MSE = []
    w = []

    # Training
    for epoch in range(epochs):

      yn, w, Wall0, MSE0, e = self.countSerie(Y_train, X_train, prediction = prediction)

      Wall.append(Wall0)
      MSE.append(MSE0)

    log.message("done \n")
    return yn, w, e, Wall, MSE


###############################################################
#                                                             #
#                            QNU                              #
#                                                             #
###############################################################


class QNULM:

  w = []

  def countSerie(self, Y, X, **kwargs):

     # optional parameters
    prediction =  kwargs.get('prediction', 1)
    mu = kwargs.get('mu', 0.9)
    self.w = kwargs.get('weigths', self.w)
    logging = kwargs.get('logging', False)

    log = Log(logging, "MLP.phiCell()")
    log.message("start")


    N=len(Y)   # X.shape[0]
    e=zeros(N)
    yn=Y.copy()  # init output neuronu (site)
    nx = len(X) * 3 + 1 #lenX * X width + 1
    nw=(nx*nx+nx)/2
    if len(self.w) < 1:
      self.w = random.randn(nw)/nw
    x=[]
    colx=[]
    J=zeros((N,nw))
    I=eye(nw)

    # for record
    MSE = [] #mean squer error


    for k in range(N-prediction):

      #input vector
      x = [1.]
      for i in range(len(X)):
        x = concatenate((x, [ X[i][k], X[i][k-1] , X[i][k-2]]))#, X[i][k-3], X[i][k-4], X[i][k-5] ]))
      
      colx = []
      for i in range(nx):
        for j in range(i,nx):
          colx.append(x[i]*x[j])

      yn[k+prediction]=dot(self.w,colx)

      e[k]=Y[k]-yn[k]
      dydw=colx       # for QNU and HONU (Higher Order Neural Units)
      J[k,:]=dydw

    print "___"
    print len(dot(linalg.inv(dot(J.T,J)+1.0/mu*I),J.T))
    print len(e)
    print len(inv(dot(J.T,J)+1./mu*I))
    dw=dot(dot(linalg.inv(dot(J.T,J)+1.0/mu*I),J.T),e)
    self.w=self.w+dw
    MSE.append(sum(e**2)/N)  
    
    return yn, self.w, MSE, e


  def train(self, Y_train, X_train, **kwargs):

    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)

    log = Log(logging, "QNU.train()")
    log.message("start")

    Wall = []
    MSE = []

    # Training
    for epoch in range(epochs):

      yn, w, MSE0, e = self.countSerie(Y_train, X_train, prediction = prediction)

      Wall.append(w)
      MSE.append(MSE0)

    log.message("done \n")
    return yn, w, e, Wall, MSE


###############################################################
#                                                             #
#                            RBF                              #
#                                                             #
###############################################################

class RBF:

  def fphi(self, nu,beta):
    phi=exp(-nu/beta)
    return(phi)

  def fnu(self, w,x):
    return(sqrt(sum((w-x)**2)))

  def train(self, Y_train, X_train, **kwargs):

    prediction =  kwargs.get('prediction', 1)
    logging = kwargs.get('logging', False)
    log = Log(logging, "RBF.train()")
    log.message("start")

    self.W = []
    self.Y = []

    # every row is RBF neuron center
    for k in range(len(X_train[0])):
      self.W.append([])
      for i in range(len(X_train)):
        l = k - prediction
        self.W[k].append([ X_train[i][l] ]) #, X_train[i][l-1], X_train[i][l-2], X_train[i][l-3], X_train[i][l-4] ])
      self.Y.append(Y_train[k])

    log.message("done \n")

  def count(self, Y, X, **kwargs):

    prediction =  kwargs.get('prediction', 1)
    beta =  kwargs.get('beta', 0.1)
    logging = kwargs.get('logging', False)

    log = Log(logging, "RBF.count()")
    log.message("start")

    N=len(Y)
    self.Yn=Y.copy()
    e=zeros(N)
    colx = []
    allColx = []

    for j in range(N-prediction):

      log.message(j, conditioned=True)

      #update neuronu LIFO (moving window) start updating after prediction lag
      if (j > 0 and j > prediction):
        self.W = self.W[1:] #remove first
        self.W.append(allColx[-prediction]) 
        self.Y = self.Y[1:] #remove first
        self.Y.append(Y[j])

      colx = []
      for i in range(len(X)):
        colx.append([X[i][j] ]) #, X[i][j-1], X[i][j-2], X[i][j-3], X[i][j-4]])

      allColx.append(colx)

      Nw=len(self.W) 
      phi=zeros(Nw)
      nu=zeros(Nw)
      for i in range(Nw):
          nu[i]=self.fnu(asarray(self.W[i]),asarray(colx)) #nu[i]=fnu(W[i,:],x)
      phi=self.fphi(nu,beta) # output of RBF

      self.Yn[j+prediction]=sum(asarray(phi)*asarray(self.Y))/sum(asarray(phi))

    log.message("done \n")
    return self.Yn


###############################################################
#                                                             #
#                            MLPML                            #
#                                                             #
###############################################################

class MLPLM:

  w = []
  v = []

  def phi(self, ny):
    bz = 2. / ( 1. + exp( -ny ) ) -1.
    return bz

  def dphidny(self, ny):
    bzz = 2. * exp( -ny ) / ( 1. + exp( -ny ) ) **2
    return bzz

  def count(self, Y, X, **kwargs):


    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)

    log = Log(logging, "MLPGD.count()")
    log.message("start")
   
    yr = Y

    muw = 0.05
    muv = 0.01

    N = len(Y)
    n1 = 2
    nx = len(X) * 4 + 1
    nv = 1 + n1
    nxi = nv

    if len(self.w) < 1:
      self.w = randn( n1, nx ) / nx

    if len(self.v) < 1:
      self.v = randn( nv ) / nv
      
    e = zeros( N )
    y = Y.copy()
    #x = ones( nx )
    xi = ones( nxi )
    dxidny = zeros ( n1 + 1 )
    dydv = zeros(( N, nv ))
    dydw = zeros( ( N, nx, n1 ) )
    Lv = eye( nv )
    Lw = eye( nx )
    SSE = zeros( epochs )

    for epoch in range( epochs ):
      for k in range ( N - prediction):

        #input vector
        x = [1.]
        for i in range(len(X)):
          x = concatenate((x, [ X[i][k], X[i][k-1], X[i][k-2], X[i][k-3] ]))

        ny = dot( self.w, x )


        xi[1:] = self.phi(ny)
        y[ k + prediction ] = dot( self.v, xi )
        e[ k ] = yr[ k ] - y[ k - prediction]


        #weights of input neuron
        dydv[k, :] = xi

        #weights of hidden nodes
        dxidny[1:] = self.dphidny(ny)
        for i in range(1, n1 + 1 ):
          dydw[k, :, i-1 ] = self.v[ i ] * dxidny[ i ] * x

      Jv = dydv
      dv = dot( dot( inv( dot( Jv.T, Jv ) + 1. / muv * Lv ), Jv.T ), e )
      self.v = self.v + dv

      for i in range( 1, n1 + 1 ):
        Jw = dydw[ :, :, i - 1 ]
        dw = dot( dot( inv( dot( Jw.T, Jw ) + 1. / muw * Lw ), Jw.T ), e )
        self.w[ i - 1, : ] = self.w[ i - 1, : ] + dw

      SSE[ epoch ] = dot( e, e )
        


    return y


###############################################################
#                                                             #
#                            MLPMLWL                          #
#                                                             #
###############################################################

class MLPLMWL:

  w = []
  v = []

  def phi(self, ny):
    bz = 2. / ( 1. + exp( -ny ) ) -1.
    return bz

  def dphidny(self, ny):
    bzz = 2. * exp( -ny ) / ( 1. + exp( -ny ) ) **2
    return bzz

  def count(self, Y, X, **kwargs):


    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)
    lw = kwargs.get('learningWindow', 0)
    ol = kwargs.get('overLearn', 0)

    log = Log(logging, "MLPGD.count()")
    log.message("start")
   
    yr = Y

    muw = 0.05
    muv = 0.01

    N = len(Y)
    nx = len(X) * 4 + 1
    n1 = 5
    nv = 1 + n1
    nxi = nv

    if len(self.w) < 1:
      self.w = randn( n1, nx ) / nx

    if len(self.v) < 1:
      self.v = randn( nv ) / nv
      
    e = zeros( lw )
    y = Y.copy()
    #x = ones( nx )
    xi = ones( nxi )
    dxidny = zeros ( n1 + 1 )
    dydv = zeros(( lw, nv ))
    dydw = zeros(( lw, nx, n1 ))
    Lv = eye( nv )
    Lw = eye( nx )
    SSE = zeros( epochs )

    for epoch in range( epochs ):
      for k in range ( N - prediction):

        #input vector
        x = [1.]
        for i in range(len(X)):
          x = concatenate((x, [ X[i][k], X[i][k-1], X[i][k-2], X[i][k-3] ]))

        ny = dot( self.w, x )


        xi[1:] = self.phi(ny)
        y[ k + prediction ] = dot( self.v, xi )
        ek = yr[ k ] - y[ k ]

        e = concatenate(( e, [ek] ))
        e = delete(e, 0, 0)


        #weights of output neuron
        dydv = delete(dydv, 0, 0)
        dydv = vstack(( dydv, xi ))

        #vweights of hidden nodes
        dxidny[1:] = self.dphidny(ny)
        dydwk = zeros( ( 1, nx, n1 ) )
        for i in range(1, n1 + 1 ):
          dydwk[0, :,i-1] = self.v[ i ] * dxidny[ i ] * x
        dydw = delete(dydw, 0, 0)
        dydw = vstack(( dydw, dydwk))

        if k > lw and k % ol == 0:
          print k
          Jv = dydv
          dv = dot( dot( inv( dot( Jv.T, Jv ) + 1. / muv * Lv ), Jv.T ), e )
          self.v = self.v + dv

          for i in range( 1, n1 + 1 ):
            Jw = dydw[ :, :, i - 1 ]
            dw = dot( dot( inv( dot( Jw.T, Jw ) + 1. / muw * Lw ), Jw.T ), e )
            self.w[ i - 1, : ] = self.w[ i - 1, : ] + dw

      SSE[ epoch ] = dot( e, e )
        


    return y

###############################################################
#                                                             #
#                            MLPGD                            #
#                                                             #
###############################################################

class MLPGD:

  w = []
  v = []

  def phi(self, ny):
    bz = 2. / ( 1. + exp( -ny ) ) -1.
    return bz

  def dphidny(self, ny):
    bzz = 2. * exp( -ny ) / ( 1. + exp( -ny ) ) **2
    return bzz

  def count(self, Y, X, **kwargs):


    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)

    log = Log(logging, "MLPGD.count()")
    log.message("start")
   
    yr = Y

    muw = 0.05
    muv = 0.01

    N = len(Y)
    n1 = 100
    nx = len(X) * 4 + 1
    nv = 1 + n1
    nxi = nv

    if len(self.w) < 1:
      self.w = randn( n1, nx ) / nx

    if len(self.v) < 1:
      self.v = randn( nv ) / nv
      
    e = zeros( N )
    y = Y.copy()
    #x = ones( nx )
    xi = ones( nxi )
    dxidny = zeros ( n1 + 1 )
    dydv = zeros(( N, nv ))
    dydw = zeros( ( N, nx, n1 ) )
    SSE = zeros( epochs )

    for epoch in range( epochs ):
      for k in range ( N - prediction):

        #input vector
        x = [1.]
        for i in range(len(X)):
          x = concatenate((x, [ X[i][k], X[i][k-1], X[i][k-2], X[i][k-3] ]))

        ny = dot( self.w, x )

        xi[1:] = self.phi(ny)
        y[ k + prediction ] = dot( self.v, xi )
        e[ k ] = yr[ k ] - y[ k ]

        #weights of output neuron
        dydv[k, :] = xi
        dv = muv/(1+sum(xi**2))*e[k]*dydv[k]  # normalizovany GD
        self.v = self.v + dv

        #weights of hiddne nodes
        dxidny[1:] = self.dphidny(ny)
        for i in range(1, n1 + 1 ):
          dydw[k, :, i-1 ] = self.v[ i ] * dxidny[ i ] * x
        
        for i in range( 1, n1 + 1 ):
          dw = muw/(1+sum(x**2))*e[k]*dydw[ k, :, i - 1 ]
          self.w[ i - 1, : ] = self.w[ i - 1, : ] + dw

    return y


class MLPELM:

  w = []
  v = []

  def phi(self, ny):
    bz = 2. / ( 1. + exp( -ny ) ) -1.
    return bz

  def dphidny(self, ny):
    bzz = 2. * exp( -ny ) / ( 1. + exp( -ny ) ) **2
    return bzz

  def count(self, Y, X, **kwargs):


    prediction =  kwargs.get('prediction', 1)
    epochs = kwargs.get('epochs', 1)
    logging = kwargs.get('logging', False)
    learning = kwargs.get('learning', True)

    log = Log(logging, "MLPGD.count()")
    log.message("start")
   
    yr = Y

    #muw = 0.05
    #muv = 0.01
    muv = 0.005

    N = len(Y)
    n1 = 400
    nx = len(X) * 4 + 1
    nv = 1 + n1
    nxi = nv

    if len(self.w) < 1:
      self.w = randn( n1, nx ) / nx

    if len(self.v) < 1:
      self.v = randn( nv ) / nv
      
    e = zeros( N )
    y = Y.copy()
    #x = ones( nx )
    xi = ones( nxi )
    dxidny = zeros ( n1 + 1 )
    dydv = zeros(( N, nv ))
    dydw = zeros( ( N, nx, n1 ) )
    SSE = zeros( epochs )

    for epoch in range( epochs ):
      for k in range ( N - prediction):

        #input vector
        x = [1.]
        for i in range(len(X)):
          x = concatenate((x, [ X[i][k], X[i][k-1], X[i][k-2], X[i][k-3] ]))

        ny = dot( self.w, x )

        xi[1:] = self.phi(ny)
        y[ k + prediction ] = dot( self.v, xi )
        e[ k ] = yr[ k ] - y[ k ]

        #weights of output neuron
        dydv[k, :] = xi
        dv = muv/(1+sum(xi**2))*e[k]*dydv[k]  # normalized GD
        if learning:
          self.v = self.v + dv


    return y