Run Unittest on CircleCI

.circleci/config.yml

@@ -0,0 +1,306 @@
+version: 2.1
+
+# How to test the Linux jobs:
+#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
+#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8
+#     - Replace binary_linux_wheel_py3.8 with the name of the job you want to test.
+#       Job names are 'name:' key.
+
+orbs:
+  win: circleci/[email protected]
+
+binary_common: &binary_common
+  parameters:
+    # Edit these defaults to do a release
+    build_version:
+      description: "version number of release binary; by default, build a nightly"
+      type: string
+      default: ""
+    pytorch_version:
+      description: "PyTorch version to build against; by default, use a nightly"
+      type: string
+      default: ""
+    # Don't edit these
+    python_version:
+      description: "Python version to build against (e.g., 3.8)"
+      type: string
+  environment:
+    PYTHON_VERSION: << parameters.python_version >>
+    BUILD_VERSION: << parameters.build_version >>
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    CU_VERSION: cpu
+
+smoke_test_common: &smoke_test_common
+  <<: *binary_common
+  docker:
+    - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtext/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
+      aws_auth:
+        aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
+        aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}
+
+jobs:
+  circleci_consistency:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - checkout
+      - run:
+          command: |
+            pip install --user --progress-bar off jinja2 pyyaml
+            python .circleci/regenerate.py
+            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)
+
+  binary_linux_wheel:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+
+  binary_linux_conda:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/conda-cuda"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - run: packaging/build_conda.sh
+      - store_artifacts:
+          path: /opt/conda/conda-bld/linux-64
+      - persist_to_workspace:
+          root: /opt/conda/conda-bld/linux-64
+          paths:
+            - "*"
+
+  binary_win_conda:
+    <<: *binary_common
+    executor:
+      name: win/default
+      shell: bash.exe
+    steps:
+      - checkout
+      - run:
+          command: |
+            choco install miniconda3
+            (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
+            conda activate base
+            conda install -yq conda-build "conda-package-handling!=1.5.0"
+            bash packaging/build_conda.sh
+          shell: powershell.exe
+
+  binary_macos_wheel:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          # Cannot easily deduplicate this as source'ing activate
+          # will set environment variables which we need to propagate
+          # to build_wheel.sh
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            packaging/build_wheel.sh
+      - store_artifacts:
+          path: dist
+      - persist_to_workspace:
+          root: dist
+          paths:
+            - "*"
+
+  binary_macos_conda:
+    <<: *binary_common
+    macos:
+      xcode: "9.0"
+    steps:
+      - checkout
+      - run:
+          command: |
+            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
+            sh conda.sh -b
+            source $HOME/miniconda3/bin/activate
+            conda install -yq conda-build
+            packaging/build_conda.sh
+      - store_artifacts:
+          path: /Users/distiller/miniconda3/conda-bld/osx-64
+      - persist_to_workspace:
+          root: /Users/distiller/miniconda3/conda-bld/osx-64
+          paths:
+            - "*"
+
+  # Requires org-member context
+  binary_conda_upload:
+    docker:
+      - image: continuumio/miniconda
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run:
+          command: |
+            # Prevent credential from leaking
+            conda install -yq anaconda-client
+            set -x
+            anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force
+
+  # Requires org-member context
+  binary_wheel_upload:
+    docker:
+      - image: circleci/python:3.8
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - checkout
+      - run:
+          command: |
+            pip install --user awscli
+            export PATH="$HOME/.local/bin:$PATH"
+            # Prevent credential from leaking
+            set +x
+            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
+            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
+            set -x
+            for pkg in ~/workspace/*.whl; do
+              aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
+            done
+
+  smoke_test_linux_conda:
+    <<: *smoke_test_common
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run:
+          name: install binaries
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            conda install -v -y -c pytorch-nightly pytorch
+            conda install -v -y $(ls ~/workspace/torchtext*.tar.bz2)
+      - run:
+          name: smoke test
+          command: |
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            python -c "import torchtext"
+
+  smoke_test_linux_pip:
+    <<: *smoke_test_common
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run:
+          name: install binaries
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            pip install $(ls ~/workspace/torchtext*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html
+      - run:
+          name: smoke test
+          command: |
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            python -c "import torchtext"
+
+  smoke_test_docker_image_build:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: large
+    environment:
+      image_name: torchtext/smoke_test
+    steps:
+      - checkout
+      - run:
+          name: build_docker image
+          no_output_timeout: "1h"
+          command: |
+            cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
+      - run:
+          name: upload docker image
+          no_output_timeout: "1h"
+          command: |
+            set +x
+            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
+            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
+            eval $(aws ecr get-login --region us-east-1 --no-include-email)
+            set -x
+            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
+            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
+
+  unittest_linux:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - restore_cache:
+
+          keys:
+            - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
+
+      - run:
+          name: Setup
+          command: .circleci/unittest/scripts/setup_env.sh
+      - save_cache:
+
+          key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }}
+
+          paths:
+            - conda
+            - env
+      - run:
+          name: Install torchtext
+          command: .circleci/unittest/scripts/install.sh
+      - run:
+          name: Run tests
+          command: .circleci/unittest/scripts/run_test.sh
+      - run:
+          name: Post process
+          command: .circleci/unittest/scripts/post_process.sh
+      - store_test_results:
+          path: test-results
+
+
+workflows:
+  # build:
+  #   jobs:
+  #     - circleci_consistency
+  #     
+  #     - binary_win_conda:
+  #         name: torchtext_win_py3.6
+  #         python_version: "3.6"
+  unittest:
+    jobs:
+      - unittest_linux:
+          name: unittest_linux_py3.6
+          python_version: '3.6'
+      - unittest_linux:
+          name: unittest_linux_py3.7
+          python_version: '3.7'
+      - unittest_linux:
+          name: unittest_linux_py3.8
+          python_version: '3.8'
+  nightly:
+    jobs:
+      - circleci_consistency:
+          filters:
+            branches:
+              only: nightly
+      # 
+  docker_build:
+    triggers:
+      - schedule:
+          cron: "0 10 * * 0"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - smoke_test_docker_image_build