Merge pull request #1331 from FCP-INDI/develop

Merge 1.7.0-development branch into Master

Author: sgiavasis

.circleci/config.yml

@@ -1,36 +1,93 @@
 version: 2
 jobs:
-  pytest:
-    docker:
-      - image: fcpindi/c-pac:python3
+  pytest-docker:
+    machine: true
     steps:
-      - checkout
+      - attach_workspace:
+          # Must be absolute path or relative path from working_directory
+          at: /home/circleci/
+      - run:
+          name: "Set Python Version"
+          command: |
+            pyenv install 3.6.3
+            pyenv global 3.6.3
       - run:
           name: Get Sample BIDS Data
           command: git clone https://github.com/bids-standard/bids-examples.git
       - run:
           name: pytest
           command: |
-            pip install -r dev/circleci_data/requirements.txt
-            coverage run -m pytest --junitxml=test-results/junit.xml --doctest-ignore-import-errors --continue-on-collection-errors --doctest-modules --ignore=CPAC/cwas/tests/test_mdmr_cython.py dev/circleci_data/
-            coverage report --include="./*" --omit="/home/circleci/.local/*"
-            coverage html
+            docker load < cpac-docker-image.tar.gz
+            docker run -dit -P -v /home/circleci/project/test-results:/code/test-results -v /home/circleci/project/htmlcov:/code/htmlcov --entrypoint=/bin/bash --name docker_test fcpindi/c-pac:${CIRCLE_BRANCH//\//_}
+            docker exec docker_test /bin/bash ./code/dev/circleci_data/test_in_image.sh
       - store_test_results:
           path: test-results
       - store_artifacts:
           path: test-results
       - store_artifacts:
           path: htmlcov
+          no_output_timeout: 5h
+  pytest-singularity:
+    machine: true
+    steps:
+      - attach_workspace:
+          # Must be absolute path or relative path from working_directory
+          at: /home/circleci/
+      - run:
+          name: "Set Python Version"
+          command: |
+            pyenv install 3.6.3
+            pyenv global 3.6.3
+      - run:
+          name: test_install
+          command: |
+            sudo apt-get update && sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 libarchive-dev libgpgme11-dev libseccomp-dev -y
+            cd singularity
+            ./autogen.sh
+            ./configure --prefix=/usr/local --sysconfdir=/etc
+            make
+            sudo make install
+            cd ..
+            pip install -r dev/circleci_data/requirements.txt
+            coverage run -m pytest --junitxml=test-results/junit.xml --continue-on-collection-errors  dev/circleci_data/test_install.py
   build:
     machine: true
     steps:
       - checkout
       - run:
-          name: build
+          name: "Set Python Version"
+          command: |
+            pyenv install 3.6.3
+            pyenv global 3.6.3
+      - run:
+          name: build-docker
           command: |
-            docker build -t cpac:${CIRCLE_BRANCH//\//_} .
-            docker save cpac:${CIRCLE_BRANCH//\//_} | gzip > cpac-docker-image.tar.gz
+            docker build -t fcpindi/c-pac:${CIRCLE_BRANCH//\//_} .
+            docker save fcpindi/c-pac:${CIRCLE_BRANCH//\//_} | gzip > cpac-docker-image.tar.gz
           # Persist the specified paths (workspace/echo-output) into the workspace for use in downstream job.
+      - run:
+          name: local-registry
+          command: docker run -d -p 5000:5000 --restart=always --name registry registry:2
+      - run:
+          name: build-singularity
+          command: |
+            sudo apt-get update && sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 libarchive-dev libgpgme11-dev libseccomp-dev -y
+            git clone -b 2.5.2 https://github.com/sylabs/singularity
+            cd singularity
+            ./autogen.sh
+            ./configure --prefix=/usr/local --sysconfdir=/etc
+            make
+            sudo make install
+            cd ..
+            docker load < cpac-docker-image.tar.gz
+            docker tag fcpindi/c-pac:${CIRCLE_BRANCH//\//_} localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}
+            docker push localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_}
+            SINGULARITY_NOHTTPS=1 singularity build C-PAC-CI.simg docker://localhost:5000/fcpindi/c-pac:${CIRCLE_BRANCH//\//_} 
+      - store_artifacts:
+          path: cpac-docker-image.tar.gz
+      - store_artifacts:
+          path: C-PAC-CI.simg
+          destination: cpac-singularity-image.simg
       - persist_to_workspace:
           # Must be an absolute path, or relative path from working_directory. This is a directory on the container which is
           # taken to be the root directory of the workspace.
@@ -43,6 +100,11 @@ jobs:
       - attach_workspace:
           # Must be absolute path or relative path from working_directory
           at: /home/circleci/
+      - run:
+          name: "Set Python Version"
+          command: |
+            pyenv install 3.6.3
+            pyenv global 3.6.3
       - run:
           name: ≟
           command: python dev/circleci_data/scripts/compare.py
@@ -52,6 +114,11 @@ jobs:
       - attach_workspace:
           # Must be absolute path or relative path from working_directory
           at: /home/circleci/
+      - run:
+          name: "Set Python Version"
+          command: |
+            pyenv install 3.6.3
+            pyenv global 3.6.3
       - run:
           name: Run C-PAC
           command: |
@@ -84,6 +151,9 @@ workflows:
           filters:
             branches:
               only: feature/auto-heatmaps
-  unit_tests:
-    jobs:
-      - pytest
+      - pytest-docker:
+          requires:
+            - build
+      - pytest-singularity:
+          requires:
+            - build

CPAC/anat_preproc/anat_preproc.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import os
 from nipype.interfaces import afni
 from nipype.interfaces import ants
 from nipype.interfaces import fsl
@@ -9,6 +10,23 @@
 from CPAC.utils.datasource import create_check_for_s3_node
 from CPAC.unet.function import predict_volumes
 
+def patch_cmass_output(lst, index=0):
+    """
+    Parameters
+    ----------
+    lst : list of tuples
+        output of afni.CenterMass()
+    index : int
+        index in the list of tuples
+
+    Returns
+    -------
+        tuple
+            one set of center of mass coordinates
+    """
+    if len(lst) <= index:
+        raise IndexError("lst index out of range")
+    return lst[index]
 
 def create_anat_preproc(method='afni', already_skullstripped=False, config=None, wf_name='anat_preproc'):
     """The main purpose of this workflow is to process T1 scans. Raw mprage file is deobliqued, reoriented
@@ -82,15 +100,20 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
 
     preproc = pe.Workflow(name=wf_name)
 
-    inputnode = pe.Node(util.IdentityInterface(
-        fields=['anat', 'brain_mask']), name='inputspec')
+    inputnode = pe.Node(util.IdentityInterface(fields=['anat', 
+                                                       'brain_mask',
+                                                       'template_brain_only_for_anat',
+                                                       'template_skull_for_anat',
+                                                       'template_cmass']), 
+                        name='inputspec')
 
     outputnode = pe.Node(util.IdentityInterface(fields=['refit',
                                                         'reorient',
                                                         'skullstrip',
                                                         'brain',
-                                                        'brain_mask']),
-                         name='outputspec')
+                                                        'brain_mask',
+                                                        'center_of_mass']),
+                        name='outputspec')
 
     anat_deoblique = pe.Node(interface=afni.Refit(),
                              name='anat_deoblique')
@@ -241,6 +264,7 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
 
             preproc.connect(anat_reorient, 'out_file',
                             anat_skullstrip, 'in_file')
+
             preproc.connect(skullstrip_args, 'expr',
                             anat_skullstrip, 'args')
 
@@ -295,7 +319,7 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
             anat_skullstrip = pe.Node(
                 interface=fsl.BET(), name='anat_skullstrip')
             anat_skullstrip.inputs.output_type = 'NIFTI_GZ'
-
+            
             preproc.connect(anat_reorient, 'out_file',
                             anat_skullstrip, 'in_file')
 
@@ -422,29 +446,28 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
             # flirt -v -dof 6 -in brain.nii.gz -ref NMT_SS_0.5mm.nii.gz -o brain_rot2atl -omat brain_rot2atl.mat -interp sinc
             # TODO: antsRegistration -z 0 -d 3 -r [NMT_SS_0.5mm.nii.gz,brain.nii.gz,0] -o [transform,brain_rot2atl.nii.gz,brain_inv_rot2atl.nii.gz] -t Rigid[0.1] -m MI[NMT_SS_0.5mm.nii.gz,brain.nii.gz,1,32,Regular,0.25] -c [1000x500x250x100,1e-08,10] -s 3.0x2.0x1.0x0.0 -f 8x4x2x1 -u 1 -t Affine[0.1] -m MI[NMT_SS_0.5mm.nii.gz,brain.nii.gz,1,32,Regular,0.25] -c [1000x500x250x100,1e-08,10] -s 3.0x2.0x1.0x0.0 -f 8x4x2x1 -u 1
             native_brain_to_template_brain = pe.Node(interface=fsl.FLIRT(), name='native_brain_to_template_brain')
-            native_brain_to_template_brain.inputs.reference = config.template_brain_only_for_anat
             native_brain_to_template_brain.inputs.dof = 6
             native_brain_to_template_brain.inputs.interp = 'sinc'
             preproc.connect(unet_masked_brain, 'out_file', native_brain_to_template_brain, 'in_file')
+            preproc.connect(inputnode, 'template_brain_only_for_anat', native_brain_to_template_brain, 'reference')
 
             # flirt -in head.nii.gz -ref NMT_0.5mm.nii.gz -o head_rot2atl -applyxfm -init brain_rot2atl.mat
             # TODO: antsApplyTransforms -d 3 -i head.nii.gz -r NMT_0.5mm.nii.gz -n Linear -o head_rot2atl.nii.gz -v -t transform1Rigid.mat -t transform2Affine.mat -t transform0DerivedInitialMovingTranslation.mat 
             native_head_to_template_head = pe.Node(interface=fsl.FLIRT(), name='native_head_to_template_head')
-            native_head_to_template_head.inputs.reference = config.template_skull_for_anat
             native_head_to_template_head.inputs.apply_xfm = True
             preproc.connect(anat_reorient, 'out_file', native_head_to_template_head, 'in_file')
             preproc.connect(native_brain_to_template_brain, 'out_matrix_file', native_head_to_template_head, 'in_matrix_file')
-            
+            preproc.connect(inputnode, 'template_skull_for_anat', native_head_to_template_head, 'reference')
+
             # fslmaths NMT_SS_0.5mm.nii.gz -bin templateMask.nii.gz
             template_brain_mask = pe.Node(interface=fsl.maths.MathsCommand(), name='template_brain_mask')
-            template_brain_mask.inputs.in_file = config.template_brain_only_for_anat
             template_brain_mask.inputs.args = '-bin'
+            preproc.connect(inputnode, 'template_brain_only_for_anat', template_brain_mask, 'in_file')
 
             # ANTS 3 -m  CC[head_rot2atl.nii.gz,NMT_0.5mm.nii.gz,1,5] -t SyN[0.25] -r Gauss[3,0] -o atl2T1rot -i 60x50x20 --use-Histogram-Matching  --number-of-affine-iterations 10000x10000x10000x10000x10000 --MI-option 32x16000
             ants_template_head_to_template = pe.Node(interface=ants.Registration(), name='template_head_to_template')
             ants_template_head_to_template.inputs.metric = ['CC']
             ants_template_head_to_template.inputs.metric_weight = [1,5]
-            ants_template_head_to_template.inputs.moving_image = config.template_skull_for_anat
             ants_template_head_to_template.inputs.transforms = ['SyN']
             ants_template_head_to_template.inputs.transform_parameters = [(0.25,)]
             ants_template_head_to_template.inputs.interpolation = 'NearestNeighbor'
@@ -453,7 +476,7 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
             ants_template_head_to_template.inputs.shrink_factors = [[4,2,1]] 
             ants_template_head_to_template.inputs.convergence_threshold = [1.e-8]
             preproc.connect(native_head_to_template_head, 'out_file', ants_template_head_to_template, 'fixed_image')
-
+            preproc.connect(inputnode, 'template_skull_for_anat', ants_template_head_to_template, 'moving_image')
             # antsApplyTransforms -d 3 -i templateMask.nii.gz -t atl2T1rotWarp.nii.gz atl2T1rotAffine.txt -r brain_rot2atl.nii.gz -o brain_rot2atl_mask.nii.gz
             template_head_transform_to_template = pe.Node(interface=ants.ApplyTransforms(), name='template_head_transform_to_template')
             template_head_transform_to_template.inputs.dimension = 3
@@ -491,3 +514,4 @@ def create_anat_preproc(method='afni', already_skullstripped=False, config=None,
             preproc.connect(refined_brain, 'out_file', outputnode, 'brain')
 
     return preproc
+