german

Author: josh

.compute

@@ -3,89 +3,72 @@
 set -xe
 
 apt-get install -y python3-venv
-_LANG="zh-TW"
-CV="${SHARED_DIR}/data/mozilla/CommonVoice/v2.0-alpha2.0/${_LANG}"
+_LANG="en"
+CV="${SHARED_DIR}/data/mozilla/CommonVoice/v2.0"
 
-# venv
-apt-get update -y
-apt-get install -y python3-venv swig
-python3 -m venv /tmp/venv
-source /tmp/venv/bin/activate
 # check HTTP_PROXY
 if ! (( $( env | grep -iq "^http_proxy=" ) )); then
     source /etc/profile
 fi
 
+# venv
+apt-get update -y
+apt-get install -y python3-venv swig
+python3 -m venv /tmp/venv
+source /tmp/venv/bin/activate
 pip install -r <(grep -v tensorflow requirements.txt)
 pip install tensorflow-gpu==1.13.0-rc2
 pip install wheel
 
-pushd ../src/native_client/ctcdecode
-make clean
-make NUM_PROCESSES=16
-pip install dist/*.whl
-popd
-
-
 ###############################
 ### INSTALL KENLM + DECODER ###
 ###############################
 
-# pip install "/data/rw/home/ds_ctcdecoder-0.5.0a1-cp36-cp36m-manylinux1_x86_64.whl"
-# pip install $(python util/taskcluster.py --decoder)
-# python util/taskcluster.py --arch gpu --target ../tmp/native_client
+pip install "/data/rw/home/ds_ctcdecoder-0.5.0a1-cp36-cp36m-manylinux1_x86_64.whl"
+python util/taskcluster.py --arch gpu --target ../tmp/native_client
 
 
 # kenlm Dependencies
-# apt-get install -y build-essential cmake libboost-all-dev zlib1g-dev libbz2-dev liblzma-dev libeigen3-dev
+apt-get install -y build-essential cmake libboost-all-dev zlib1g-dev libbz2-dev liblzma-dev libeigen3-dev
 
 
-###################################
-### CREATE ALPHABET / LM / TRIE ###
-###################################
 
-# # alphabet.txt
-# python util/check_characters.py \
-#         -csv "${CV}/cv_${_LANG}_valid_train.csv","${CV}/cv_${_LANG}_valid_train.csv","${CV}/cv_${_LANG}_valid_train.csv" \
-#         -alpha \
-#     > /data/rw/home/zh-TW/alphabet.txt
+CREATE ALPHABET / LM / TRIE ###
 
-# python util/check_characters.py \
-#         -csv "/data/rw/home/ky/clips/train.csv","/data/rw/home/ky/clips/test.csv","/data/rw/home/ky/clips/dev.csv" \
-#         -alpha \
-#     > /data/rw/home/ky/alphabet.txt
 
+# # alphabet.txt
+mkdir /data/rw/home/${_LANG}
+python util/check_characters.py \
+        -csv "${CV}/${_LANG}/clips/train.csv","${CV}/${_LANG}/clips/dev.csv","${CV}/${_LANG}/clips/test.csv" \
+        -alpha \
+    > /data/rw/home/${_LANG}/alphabet.txt
 
 # lm.arpa
 #TEXT="${SHARED_DIR}/data/wikipedia/${_LANG}/wiki.txt"
-# cut -d',' -f3 /data/rw/home/ky/clips/train.csv > /data/rw/home/ky/text.txt
-# TEXT="/data/rw/home/ky/text.txt"
-# TEXT="${SHARED_DIR}/data/wikipedia/${_LANG}/wiki.txt"
-# TEXT="/data/rw/home/CHAR_GRAMS_ZH_TW"
+cut -d',' -f3 "${CV}/${_LANG}/clips/train.csv" > /data/rw/home/${_LANG}/text.txt
+TEXT="/data/rw/home/${_LANG}/text.txt"
 
+/data/rw/home/kenlm/build/bin/lmplz \
+    --skip_symbols \
+    --order 2 \
+    --text "${TEXT}" \
+    --arpa /tmp/lm.arpa
 
-# /data/rw/home/kenlm/build/bin/lmplz \
-#     --skip_symbols \
-#     --order 2 \
-#     --text "${TEXT}" \
-#     --arpa lm.arpa
+# # lm.binary
+/data/rw/home/kenlm/build/bin/build_binary \
+    -a 255 \
+    -q 8 \
+    trie \
+    /tmp/lm.arpa \
+    /data/rw/home/${_LANG}/lm.binary
 
-# # # lm.binary
-# /data/rw/home/kenlm/build/bin/build_binary \
-#     -a 255 \
-#     -q 8 \
-#     trie \
-#     lm.arpa \
-#     /data/rw/home/ky/lm.binary
+# # trie
+/data/rw/home/generate_trie \
+    /data/rw/home/${_LANG}/alphabet.txt \
+    /data/rw/home/${_LANG}/lm.binary \
+    /data/rw/home/${_LANG}/trie_utf8
 
-# # # trie
-# # ../tmp/native_client/generate_trie \
-# /data/rw/home/generate_trie \
-#     /data/rw/home/ky/alphabet.txt \
-#     /data/rw/home/ky/lm.binary \
-#     /data/rw/home/ky/trie_utf8
-
-# rm lm.arpa
+rm /tmp/lm.arpa
 
 
 ########################
@@ -94,14 +77,11 @@ popd
 
 mkdir -p ../keep/summaries
 
-# --train_files "${CV}/cv_${_LANG}_valid_dev.csv" \
-  # --dev_files "${CV}/cv_${_LANG}_valid_test.csv" \
-  # --test_files "${CV}/cv_${_LANG}_valid_train.csv" \
-  
+
 python -u DeepSpeech.py \
-  --train_files "/data/rw/home/ky/clips/train.csv" \
-  --dev_files "/data/rw/home/ky/clips/dev.csv" \
-  --test_files "/data/rw/home/ky/clips/test.csv" \
+  --train_files "${CV}/${_LANG}/clips/train.csv" \
+  --dev_files "${CV}/${_LANG}/clips/dev.csv" \
+  --test_files "${CV}/${_LANG}/clips/test.csv" \
   --train_batch_size 24 \
   --dev_batch_size 48 \
   --test_batch_size 48 \
@@ -116,6 +96,6 @@ python -u DeepSpeech.py \
   --summary_dir "../keep/summaries" \
   --report_count 100 \
   --test_output_file "../keep/RESULTS.json" \
-  --lm-binary-path "/data/rw/home/ky/lm.binary" \
-  --lm-trie-path "/data/rw/home/ky/trie_utf8" \
-  --alphabet-config-path "/data/rw/home/ky/alphabet.txt"
+  --lm-binary-path "/data/rw/home/${_LANG}/lm.binary" \
+  --lm-trie-path "/data/rw/home/${_LANG}/trie_utf8" \
+  --alphabet-config-path "/data/rw/home/${_LANG}/alphabet.txt"