Add lora finetuning script

training/lora/README.md

@@ -0,0 +1,84 @@
+## Fine-tuning with DeeperSpeed
+### Install dependencies
+
+`mamba install -c conda-forge cudatoolkit-dev`
+
+`pip install evaluate datasets peft git+https://github.com/huggingface/transformers.git git+https://github.com/EleutherAI/DeeperSpeed.git`
+
+`pip install 'transformers[sklearn]'`
+
+#### Install bitsandbytes if loading in 8-bit
+`pip install bitsandbytes`
+
+### Start...
+
+`export CUDA_HOME=$CONDA_PREFIX`
+
+`cd training/lora`
+
+## Examples
+#### From HuggingFace dataset:
+```
+deepspeed --num_gpus=1 finetune.py \
+--deepspeed config.json \
+--model_name_or_path togethercomputer/RedPajama-INCITE-Base-3B-v1 \
+--dataset_name imdb \
+--do_train \
+--do_eval \
+--fp16 \
+--overwrite_cache \
+--evaluation_strategy="steps" \
+--output_dir finetuned \
+--num_train_epochs 1 \
+--eval_steps 15 \
+--gradient_accumulation_steps 1 \
+--per_device_train_batch_size 4 \
+--use_fast_tokenizer True \
+--learning_rate 1e-5 \
+--warmup_steps 10
+```
+#### From train and validation files:
+```
+deepspeed --num_gpus=1 finetune.py \
+--deepspeed config.json \
+--model_name_or_path togethercomputer/RedPajama-INCITE-Base-3B-v1 \
+--train_file train.csv \
+--validation_file validation.csv \
+--do_train \
+--do_eval \
+--fp16 \
+--overwrite_cache \
+--evaluation_strategy="steps" \
+--output_dir finetuned \
+--num_train_epochs 1 \
+--eval_steps 15 \
+--gradient_accumulation_steps 1 \
+--per_device_train_batch_size 4 \
+--use_fast_tokenizer True \
+--learning_rate 1e-5 \
+--warmup_steps 10
+```
+
+#### In 8-bit
+** Change `fp16.enabled` to `false` in `config.json` **
+```
+deepspeed --num_gpus=1 finetune.py \
+--deepspeed config.json \
+--model_name_or_path togethercomputer/RedPajama-INCITE-Base-3B-v1 \
+--dataset_name imdb \
+--do_train \
+--do_eval \
+--int8 \
+--low_cpu_mem_usage \
+--overwrite_cache \
+--evaluation_strategy="steps" \
+--output_dir finetuned \
+--num_train_epochs 1 \
+--eval_steps 15 \
+--gradient_accumulation_steps 1 \
+--per_device_train_batch_size 4 \
+--use_fast_tokenizer True \
+--learning_rate 1e-5 \
+--warmup_steps 10 \
+--no_cache
+```

training/lora/config.json

@@ -0,0 +1,39 @@
+{
+  "train_batch_size": "auto",
+  "fp16": {
+    "enabled": true,
+    "min_loss_scale": 1,
+    "opt_level": "O2"
+  },
+  "zero_optimization": {
+    "stage": 2,
+    "offload_param": {
+      "device": "cpu"
+    },
+    "offload_optimizer": {
+      "device": "cpu"
+    },
+    "allgather_partitions": true,
+    "allgather_bucket_size": 5e8,
+    "contiguous_gradients": true
+  },
+  "optimizer": {
+    "type": "AdamW",
+    "params": {
+      "lr": "auto",
+      "betas": [
+        0.9,
+        0.999
+      ],
+      "eps": 1e-08
+    }
+  },
+  "scheduler": {
+    "type": "WarmupLR",
+    "params": {
+      "warmup_min_lr": 0,
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
+    }
+  }
+}