new video

Author: l2k2

videos/lstm-classifier/download-imdb.py

@@ -0,0 +1,26 @@
+import os
+
+def load_imdb():
+    X_train = []
+    y_train = []
+
+    path = './aclImdb/train/pos/'
+    X_train.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_train.extend([1 for _ in range(12500)])
+
+    path = './aclImdb/train/neg/'
+    X_train.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_train.extend([0 for _ in range(12500)])
+
+    X_test = []
+    y_test = []
+    
+    path = './aclImdb/test/pos/'
+    X_test.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_test.extend([1 for _ in range(12500)])
+
+    path = './aclImdb/test/neg/'
+    X_test.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_test.extend([0 for _ in range(12500)])
+    
+    return (X_train, y_train), (X_test, y_test)

videos/lstm-classifier/imdb-lstm.py

@@ -0,0 +1,54 @@
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Activation
+from keras.layers import Embedding, LSTM, Bidirectional
+from keras.layers import Conv1D, Flatten
+from keras.datasets import imdb
+import wandb
+from wandb.keras import WandbCallback
+import imdb
+import numpy as np
+from keras.preprocessing import text
+
+wandb.init()
+config = wandb.config
+
+# set parameters:
+config.vocab_size = 1000
+config.maxlen = 300
+config.batch_size = 32
+config.embedding_dims = 50
+config.filters = 10
+config.kernel_size = 3
+config.hidden_dims = 10
+config.epochs = 10
+
+(X_train, y_train), (X_test, y_test) = imdb.load_imdb()
+
+tokenizer = text.Tokenizer(num_words=config.vocab_size)
+tokenizer.fit_on_texts(X_train)
+X_train = tokenizer.texts_to_matrix(X_train)
+X_test = tokenizer.texts_to_matrix(X_test)
+
+X_train = sequence.pad_sequences(X_train, maxlen=config.maxlen)
+X_test = sequence.pad_sequences(X_test, maxlen=config.maxlen)
+
+model = Sequential()
+model.add(Embedding(config.vocab_size,
+                    config.embedding_dims,
+                    input_length=config.maxlen))
+model.add(Conv1D(config.filters,
+                 config.kernel_size,
+                 padding='valid',
+                 activation='relu'))
+model.add(MaxPooling1D())
+model.add(LSTM(config.hidden_dims, activation="sigmoid"))
+model.add(Dense(1, activation='sigmoid'))
+model.compile(loss='binary_crossentropy',
+              optimizer='rmsprop',
+              metrics=['accuracy'])
+
+model.fit(X_train, y_train,
+          batch_size=config.batch_size,
+          epochs=config.epochs,
+          validation_data=(X_test, y_test), callbacks=[WandbCallback()])

videos/lstm-classifier/imdb.py

@@ -0,0 +1,26 @@
+import os
+
+def load_imdb():
+    X_train = []
+    y_train = []
+
+    path = './aclImdb/train/pos/'
+    X_train.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_train.extend([1 for _ in range(12500)])
+
+    path = './aclImdb/train/neg/'
+    X_train.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_train.extend([0 for _ in range(12500)])
+
+    X_test = []
+    y_test = []
+    
+    path = './aclImdb/test/pos/'
+    X_test.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_test.extend([1 for _ in range(12500)])
+
+    path = './aclImdb/test/neg/'
+    X_test.extend([open(path + f).read() for f in os.listdir(path) if f.endswith('.txt')])
+    y_test.extend([0 for _ in range(12500)])
+    
+    return (X_train, y_train), (X_test, y_test)