add test scripts

Author: bryanyzhu

README.md

@@ -1 +1,58 @@
-# two-stream-pytorch
+# PyTorch implementation of popular two-stream frameworks for video action recognition
+============================
+
+Current release is the PyTorch implementation of the "Towards Good Practices for Very Deep Two-Stream ConvNets". You can refer to paper for more details at [Arxiv](https://arxiv.org/abs/1507.02159).
+
+For future, I will add PyTorch implementation for the following papers:
+
+```
+Temporal Segment Networks: Towards Good Practices for Deep Action Recognition,
+Limin Wang, Yuanjun Xiong, Zhe Wang, Yu Qiao, Dahua Lin, Xiaoou Tang, Luc Van Gool
+ECCV 2016
+
+Deep Temporal Linear Encoding Networks
+Ali Diba, Vivek Sharma, Luc Van Gool
+https://arxiv.org/abs/1611.06678
+
+Hidden Two-Stream Convolutional Networks for Action Recognition
+Yi Zhu, Zhenzhong Lan, Shawn Newsam, Alexander G. Hauptmann
+https://arxiv.org/abs/1704.00389
+```
+
+Install
+=========
+
+Tested on PyTorch:
+
+```
+OS: Ubuntu 16.04
+Package manager: Conda
+Python: 3.5
+CUDA: 8.0
+```
+
+Code also works for Python 2.7.
+
+Training
+========
+
+Simply run:
+
+`python main_single_gpu.py DATA_PATH`
+
+`DATA_PATH` is where you store RGB frames or optical flow images. Change the parameters passing to argparse as you need.
+
+Testing
+========
+
+`Will release soon.`
+
+Related Projects
+====================
+
+[TSN](https://github.com/yjxiong/temporal-segment-networks): Temporal Segment Networks: Towards Good Practices for Deep Action Recognition
+
+[Hidden Two-Stream](https://github.com/bryanyzhu/Hidden-Two-Stream): Hidden Two-Stream Convolutional Networks for Action Recognition
+
+
+

datasets/ucf101.py

@@ -19,7 +19,7 @@ def make_dataset(root, source):
         sys.exit()
     else:
         clips = []
-        with open(source) as split_f:  
+        with open(source) as split_f:
             data = split_f.readlines()
             for line in data:
                 line_info = line.split()
@@ -46,14 +46,14 @@ def ReadSegmentRGB(path, offsets, new_height, new_width, new_length, is_color, n
             cv_img_origin = cv2.imread(frame_path, cv_read_flag)
             if cv_img_origin is None:
                print("Could not load file %s" % (frame_path))
-               sys.exit() 
+               sys.exit()
                # TODO: error handling here
             if new_width > 0 and new_height > 0:
                 cv_img = cv2.resize(cv_img_origin, (new_width, new_height), interpolation)
             else:
                 cv_img = cv_img_origin
             cv_img = cv2.cvtColor(cv_img, cv2.COLOR_BGR2RGB)
-            sampled_list.append(cv_img)    
+            sampled_list.append(cv_img)
     clip_input = np.concatenate(sampled_list, axis=2)
     return clip_input
 
@@ -62,7 +62,7 @@ def ReadSegmentFlow(path, offsets, new_height, new_width, new_length, is_color,
         cv_read_flag = cv2.IMREAD_COLOR         # > 0
     else:
         cv_read_flag = cv2.IMREAD_GRAYSCALE     # = 0
-    interpolation = cv2.INTER_LINEAR  
+    interpolation = cv2.INTER_LINEAR
 
     sampled_list = []
     for offset_id in range(len(offsets)):
@@ -76,50 +76,50 @@ def ReadSegmentFlow(path, offsets, new_height, new_width, new_length, is_color,
             cv_img_origin_y = cv2.imread(frame_path_y, cv_read_flag)
             if cv_img_origin_x is None or cv_img_origin_y is None:
                print("Could not load file %s or %s" % (frame_path_x, frame_path_y))
-               sys.exit() 
+               sys.exit()
                # TODO: error handling here
             if new_width > 0 and new_height > 0:
                 cv_img_x = cv2.resize(cv_img_origin_x, (new_width, new_height), interpolation)
                 cv_img_y = cv2.resize(cv_img_origin_y, (new_width, new_height), interpolation)
             else:
                 cv_img_x = cv_img_origin_x
                 cv_img_y = cv_img_origin_y
-            sampled_list.append(np.expand_dims(cv_img_x, 2))  
-            sampled_list.append(np.expand_dims(cv_img_y, 2))      
+            sampled_list.append(np.expand_dims(cv_img_x, 2))
+            sampled_list.append(np.expand_dims(cv_img_y, 2))
 
     clip_input = np.concatenate(sampled_list, axis=2)
     return clip_input
 
 
 class ucf101(data.Dataset):
 
-    def __init__(self, 
-                 root, 
-                 source, 
-                 phase, 
+    def __init__(self,
+                 root,
+                 source,
+                 phase,
                  modality,
                  name_pattern=None,
-                 is_color=True, 
+                 is_color=True,
                  num_segments=1,
-                 new_length=1, 
+                 new_length=1,
                  new_width=0,
                  new_height=0,
-                 transform=None, 
-                 target_transform=None, 
+                 transform=None,
+                 target_transform=None,
                  video_transform=None):
 
         classes, class_to_idx = find_classes(root)
         clips = make_dataset(root, source)
-        
+
         if len(clips) == 0:
             raise(RuntimeError("Found 0 video clips in subfolders of: " + root + "\n"
                                "Check your data directory."))
-        
+
         self.root = root
         self.source = source
         self.phase = phase
         self.modality = modality
-        
+
         self.classes = classes
         self.class_to_idx = class_to_idx
         self.clips = clips
@@ -131,13 +131,13 @@ def __init__(self,
                 self.name_pattern = "image_%04d.jpg"
             elif self.modality == "flow":
                 self.name_pattern = "flow_%s_%04d.jpg"
-        
+
         self.is_color = is_color
         self.num_segments = num_segments
         self.new_length = new_length
         self.new_width = new_width
         self.new_height = new_height
-        
+
         self.transform = transform
         self.target_transform = target_transform
         self.video_transform = video_transform
@@ -165,24 +165,24 @@ def __getitem__(self, index):
 
         if self.modality == "rgb":
             clip_input = ReadSegmentRGB(path,
-                                        offsets, 
+                                        offsets,
                                         self.new_height,
-                                        self.new_width, 
-                                        self.new_length, 
-                                        self.is_color, 
+                                        self.new_width,
+                                        self.new_length,
+                                        self.is_color,
                                         self.name_pattern
                                         )
         elif self.modality == "flow":
             clip_input = ReadSegmentFlow(path,
-                                        offsets, 
+                                        offsets,
                                         self.new_height,
-                                        self.new_width, 
-                                        self.new_length, 
-                                        self.is_color, 
+                                        self.new_width,
+                                        self.new_length,
+                                        self.is_color,
                                         self.name_pattern
                                         )
         else:
-            print("No such modality %s" % (self.modality))  
+            print("No such modality %s" % (self.modality))
 
         if self.transform is not None:
             clip_input = self.transform(clip_input)

main_single_gpu.py

@@ -28,7 +28,7 @@
                     help='path to dataset')
 parser.add_argument('--settings', metavar='DIR', default='./settings',
                     help='path to datset setting files')
-parser.add_argument('--modality', '-m', metavar='MODALITY', default='rgb',
+parser.add_argument('--modality', '-m', metavar='MODALITY', default='flow',
                     choices=["rgb", "flow"],
                     help='modality: rgb | flow')
 parser.add_argument('--dataset', '-d', default='ucf101',
@@ -43,21 +43,25 @@
                     help='which split of data to work on (default: 1)')
 parser.add_argument('-j', '--workers', default=4, type=int, metavar='N',
                     help='number of data loading workers (default: 4)')
+<<<<<<< HEAD
 parser.add_argument('--epochs', default=250, type=int, metavar='N',
+=======
+parser.add_argument('--epochs', default=750, type=int, metavar='N',
+>>>>>>> f388aed3118e80cf51805afefbc78da8405f9e6a
                     help='number of total epochs to run')
 parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                     help='manual epoch number (useful on restarts)')
 parser.add_argument('-b', '--batch-size', default=25, type=int,
                     metavar='N', help='mini-batch size (default: 50)')
 parser.add_argument('--iter-size', default=5, type=int,
                     metavar='I', help='iter size as in Caffe to reduce memory usage (default: 5)')
-parser.add_argument('--new_length', default=1, type=int,
+parser.add_argument('--new_length', default=10, type=int,
                     metavar='N', help='length of sampled video frames (default: 1)')
 parser.add_argument('--new_width', default=340, type=int,
                     metavar='N', help='resize width (default: 340)')
 parser.add_argument('--new_height', default=256, type=int,
                     metavar='N', help='resize height (default: 256)')
-parser.add_argument('--lr', '--learning-rate', default=0.001, type=float,
+parser.add_argument('--lr', '--learning-rate', default=0.005, type=float,
                     metavar='LR', help='initial learning rate')
 parser.add_argument('--lr_steps', default=[100, 200], type=float, nargs="+",
                     metavar='LRSteps', help='epochs to decay learning rate by 10')
@@ -87,23 +91,24 @@ def main():
     model = build_model()
     print("Model %s is loaded. " % (args.arch))
 
-    if not os.path.exists(args.resume):
-        os.makedirs(args.resume)
-    print("Saving everything to directory %s." % (args.resume))
-
     # define loss function (criterion) and optimizer
     criterion = nn.CrossEntropyLoss().cuda()
 
     optimizer = torch.optim.SGD(model.parameters(), args.lr,
                                 momentum=args.momentum,
                                 weight_decay=args.weight_decay)
 
+    if not os.path.exists(args.resume):
+        os.makedirs(args.resume)
+    print("Saving everything to directory %s." % (args.resume))
+
     cudnn.benchmark = True
 
     # Data transforming
     if args.modality == "rgb":
         is_color = True
         scale_ratios = [1.0, 0.875, 0.75, 0.66]
+<<<<<<< HEAD
         clip_mean = [0.485, 0.456, 0.406] * args.new_length
         clip_std = [0.229, 0.224, 0.225] * args.new_length
     elif args.modality == "flow":
@@ -116,7 +121,21 @@ def main():
 
     normalize = video_transforms.Normalize(mean=clip_mean,
                                      std=clip_std)
+=======
+        is_color = True
+        clip_mean = [0.485, 0.456, 0.406] * args.new_length
+        clip_std = [0.229, 0.224, 0.225] * args.new_length
+    elif args.modality == "flow":
+        scale_ratios = [1.0, 0.875, 0.75]
+        is_color = False
+        clip_mean = [0.5, 0.5] * args.new_length
+        clip_std = [0.5, 0.5] * args.new_length
+    else:
+        print("No such modality. Only rgb and flow supported.")
+>>>>>>> f388aed3118e80cf51805afefbc78da8405f9e6a
 
+    normalize = video_transforms.Normalize(mean=clip_mean,
+                                     std=clip_std)
     train_transform = video_transforms.Compose([
             # video_transforms.Scale((256)),
             video_transforms.MultiScaleCrop((224, 224), scale_ratios),
@@ -258,6 +277,10 @@ def train(train_loader, model, criterion, optimizer, epoch):
                       'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                       'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                        epoch, i+1, len(train_loader)+1, batch_time=batch_time, loss=losses, top1=top1))
+<<<<<<< HEAD
+=======
+
+>>>>>>> f388aed3118e80cf51805afefbc78da8405f9e6a
 
 def validate(val_loader, model, criterion):
     batch_time = AverageMeter()
@@ -329,9 +352,14 @@ def update(self, val, n=1):
 
 def adjust_learning_rate(optimizer, epoch):
     """Sets the learning rate to the initial LR decayed by 10 every 150 epochs"""
+<<<<<<< HEAD
     decay = 0.1 ** (sum(epoch >= np.array(args.lr_steps)))
     lr = args.lr * decay
     print("Current learning rate is %4.6f:" % lr)
+=======
+    lr = args.lr * (0.1 ** (epoch // 250))
+    print(lr)
+>>>>>>> f388aed3118e80cf51805afefbc78da8405f9e6a
     for param_group in optimizer.param_groups:
         param_group['lr'] = lr
 

models/flow_vgg16.py

@@ -91,7 +91,7 @@ def change_key_names(old_params, in_channels):
                 new_params[layer_key] = old_params[layer_key]
                 layer_count += 1
                 # print(layer_key, new_params[layer_key].size())
-    
+
     return new_params
 
 def flow_vgg16(pretrained=False, **kwargs):
@@ -112,7 +112,7 @@ def flow_vgg16(pretrained=False, **kwargs):
         # 1. filter out unnecessary keys
         new_pretrained_dict = {k: v for k, v in new_pretrained_dict.items() if k in model_dict}
         # 2. overwrite entries in the existing state dict
-        model_dict.update(new_pretrained_dict) 
+        model_dict.update(new_pretrained_dict)
         # 3. load the new state dict
         model.load_state_dict(model_dict)