script to make alphabet/lm/trie on cluster

Author: josh

.compute

@@ -14,20 +14,25 @@ pip install $(python3 util/taskcluster.py --decoder)
 mkdir -p ../keep/summaries
 
 LANG="sl"
-cv="$SHARED_DIR/data/mozilla/CommonVoice/v2.0-alpha2.0/$LANG"
+CV="${SHARED_DIR}/data/mozilla/CommonVoice/v2.0-alpha2.0/${LANG}"
+
+# the *.csv on cluster have old paths
+cp ${CV}/*.csv .
+sed -Ei 's/snakepit/data\/ro/g' cv_${LANG}_valid_*.csv
+
 
 python -u DeepSpeech.py \
-  --train_files '$CV/cv_${LANG}_valid_train.csv' \
-  --dev_files '$CV/cv_${LANG}_valid_dev.csv' \
-  --test_files '$CV/cv_${LANG}_valid_test.csv' \
+  --train_files "cv_${LANG}_valid_train.csv" \
+  --dev_files "cv_${LANG}_valid_dev.csv" \
+  --test_files "cv_${LANG}_valid_test.csv" \
   --train_batch_size 24 \
   --dev_batch_size 48 \
   --test_batch_size 48 \
   --noearly_stop \
   --n_hidden 2048 \
   --learning_rate 0.0001 \
-  --dropout_rate 0.15 \
-  --epoch 1 \
+  --dropout_rate 0.2 \
+  --epoch 30 \
   --display_step 0 \
   --validation_step 1 \
   --checkpoint_dir "../keep" \

util/create_alpha_lm_trie.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+LANG=$1
+TEXT=$2
+
+echo "$0: Looking for CSV transcripts at cv_${LANG}_valid_{train/dev/test}.csv"
+echo "$0: Looking for text training corpus at ${TEXT}"
+
+# kenlm Dependencies
+apt-get install -y build-essential cmake libboost-all-dev zlib1g-dev libbz2-dev liblzma-dev libeigen3-dev
+
+# Install Kenlm #
+
+wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz
+mkdir kenlm/build
+cd kenlm/build
+cmake ..
+make -j `nproc`
+cd ../..
+
+#################
+### CREATE LM ###
+#################
+
+# Make alphabet.txt #
+
+python3 util/check_characters.py \
+        -csv "cv_${LANG}_valid_train.csv","cv_${LANG}_valid_train.csv","cv_${LANG}_valid_train.csv" \
+        -alpha \
+    | data/alphabet.txt
+
+# Make lm.arpa #
+
+kenlm/build/bin/lmplz \
+    --order 2 \
+    --text ${TEXT} \
+    --arpa /tmp/lm.arpa
+
+# Make lm.binary #
+
+kenlm/build/bin/build_binary \
+    -a 255 \
+    -q 8 trie \
+    /tmp/lm.arpa \
+    data/lm/lm.binary
+
+# Make trie #
+
+native_client/generate_trie \
+    data/alphabet.txt \
+    data/lm/lm.binary \
+    data/lm/trie
+
+rm /tmp/lm.arpa
+
+