From ff9db0c37cf0be4fdf76a0ceb4f1f1b397e7d284 Mon Sep 17 00:00:00 2001 From: Christian Puhrsch Date: Thu, 23 Apr 2020 21:33:55 -0400 Subject: [PATCH 1/5] Cherrypick Christian's work --- .circleci/config.yml | 508 +++++++++++++++++++++++++++++++ .circleci/config.yml.in | 258 ++++++++++++++++ .circleci/docker/Dockerfile | 36 +++ .circleci/regenerate.py | 119 ++++++++ .circleci/test/test_sort_yaml.py | 14 + 5 files changed, 935 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .circleci/config.yml.in create mode 100644 .circleci/docker/Dockerfile create mode 100755 .circleci/regenerate.py create mode 100755 .circleci/test/test_sort_yaml.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..67a9905bc9 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,508 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8 +# - Replace binary_linux_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +orbs: + win: circleci/windows@2.0.0 + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + CU_VERSION: cpu + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtext/smoke_test:d26af7d0-8458-face-book-84239b5c75c0 + aws_auth: + aws_access_key_id: ${ECR_AWS_ACCESS_KEY} + aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY} + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 pyyaml + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_linux_conda: + <<: *binary_common + docker: + - image: "pytorch/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" + + binary_win_conda: + <<: *binary_common + executor: + name: win/default + shell: bash.exe + steps: + - checkout + - run: + command: | + choco install miniconda3 + (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + bash packaging/build_conda.sh + shell: powershell.exe + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set -x + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.8 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done + + smoke_test_linux_conda: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + conda install -v -y -c pytorch-nightly pytorch + conda install -v -y $(ls ~/workspace/torchtext*.tar.bz2) + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchtext" + + smoke_test_linux_pip: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + pip install $(ls ~/workspace/torchtext*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchtext" + + docker_image_build: + machine: + image: ubuntu-1604:201903-01 + resource_class: large + environment: + image_name: torchtext/smoke_test + steps: + - checkout + - run: + name: build_docker image + no_output_timeout: "1h" + command: | + cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + - run: + name: upload docker image + no_output_timeout: "1h" + command: | + set +x + export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY} + export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY} + eval $(aws ecr get-login --region us-east-1 --no-include-email) + set -x + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + +workflows: + build: + jobs: + - circleci_consistency + - binary_linux_wheel: + name: binary_linux_wheel_py3.6 + python_version: '3.6' + - binary_linux_wheel: + name: binary_linux_wheel_py3.7 + python_version: '3.7' + - binary_linux_wheel: + name: binary_linux_wheel_py3.8 + python_version: '3.8' + - binary_macos_wheel: + name: binary_macos_wheel_py3.6 + python_version: '3.6' + - binary_macos_wheel: + name: binary_macos_wheel_py3.7 + python_version: '3.7' + - binary_macos_wheel: + name: binary_macos_wheel_py3.8 + python_version: '3.8' + - binary_linux_conda: + name: binary_linux_conda_py3.6 + python_version: '3.6' + - binary_linux_conda: + name: binary_linux_conda_py3.7 + python_version: '3.7' + - binary_linux_conda: + name: binary_linux_conda_py3.8 + python_version: '3.8' + - binary_macos_conda: + name: binary_macos_conda_py3.6 + python_version: '3.6' + - binary_macos_conda: + name: binary_macos_conda_py3.7 + python_version: '3.7' + - binary_macos_conda: + name: binary_macos_conda_py3.8 + python_version: '3.8' + - binary_win_conda: + name: torchtext_win_py3.6 + python_version: "3.6" + + nightly: + jobs: + - circleci_consistency + - binary_linux_wheel: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.6 + python_version: '3.6' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.6_upload + requires: + - nightly_binary_linux_wheel_py3.6 + - smoke_test_linux_pip: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.6_smoke_test_pip + python_version: '3.6' + requires: + - nightly_binary_linux_wheel_py3.6_upload + - binary_linux_wheel: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.7 + python_version: '3.7' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.7_upload + requires: + - nightly_binary_linux_wheel_py3.7 + - smoke_test_linux_pip: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.7_smoke_test_pip + python_version: '3.7' + requires: + - nightly_binary_linux_wheel_py3.7_upload + - binary_linux_wheel: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.8 + python_version: '3.8' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.8_upload + requires: + - nightly_binary_linux_wheel_py3.8 + - smoke_test_linux_pip: + filters: + branches: + only: nightly + name: nightly_binary_linux_wheel_py3.8_smoke_test_pip + python_version: '3.8' + requires: + - nightly_binary_linux_wheel_py3.8_upload + - binary_macos_wheel: + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.6 + python_version: '3.6' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.6_upload + requires: + - nightly_binary_macos_wheel_py3.6 + - binary_macos_wheel: + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.7 + python_version: '3.7' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.7_upload + requires: + - nightly_binary_macos_wheel_py3.7 + - binary_macos_wheel: + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.8 + python_version: '3.8' + - binary_wheel_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_wheel_py3.8_upload + requires: + - nightly_binary_macos_wheel_py3.8 + - binary_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.6 + python_version: '3.6' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.6_upload + requires: + - nightly_binary_linux_conda_py3.6 + - smoke_test_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.6_smoke_test_conda + python_version: '3.6' + requires: + - nightly_binary_linux_conda_py3.6_upload + - binary_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.7 + python_version: '3.7' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.7_upload + requires: + - nightly_binary_linux_conda_py3.7 + - smoke_test_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.7_smoke_test_conda + python_version: '3.7' + requires: + - nightly_binary_linux_conda_py3.7_upload + - binary_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.8 + python_version: '3.8' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.8_upload + requires: + - nightly_binary_linux_conda_py3.8 + - smoke_test_linux_conda: + filters: + branches: + only: nightly + name: nightly_binary_linux_conda_py3.8_smoke_test_conda + python_version: '3.8' + requires: + - nightly_binary_linux_conda_py3.8_upload + - binary_macos_conda: + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.6 + python_version: '3.6' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.6_upload + requires: + - nightly_binary_macos_conda_py3.6 + - binary_macos_conda: + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.7 + python_version: '3.7' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.7_upload + requires: + - nightly_binary_macos_conda_py3.7 + - binary_macos_conda: + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.8 + python_version: '3.8' + - binary_conda_upload: + context: org-member + filters: + branches: + only: nightly + name: nightly_binary_macos_conda_py3.8_upload + requires: + - nightly_binary_macos_conda_py3.8 + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - master + jobs: + - docker_image_build \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in new file mode 100644 index 0000000000..270aa72856 --- /dev/null +++ b/.circleci/config.yml.in @@ -0,0 +1,258 @@ +version: 2.1 + +# How to test the Linux jobs: +# - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/ +# - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.8 +# - Replace binary_linux_wheel_py3.8 with the name of the job you want to test. +# Job names are 'name:' key. + +orbs: + win: circleci/windows@2.0.0 + +binary_common: &binary_common + parameters: + # Edit these defaults to do a release + build_version: + description: "version number of release binary; by default, build a nightly" + type: string + default: "" + pytorch_version: + description: "PyTorch version to build against; by default, use a nightly" + type: string + default: "" + # Don't edit these + python_version: + description: "Python version to build against (e.g., 3.8)" + type: string + environment: + PYTHON_VERSION: << parameters.python_version >> + BUILD_VERSION: << parameters.build_version >> + PYTORCH_VERSION: << parameters.pytorch_version >> + CU_VERSION: cpu + +smoke_test_common: &smoke_test_common + <<: *binary_common + docker: + - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtext/smoke_test:d26af7d0-8458-face-book-84239b5c75c0 + aws_auth: + aws_access_key_id: ${ECR_AWS_ACCESS_KEY} + aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY} + +jobs: + circleci_consistency: + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: + command: | + pip install --user --progress-bar off jinja2 pyyaml + python .circleci/regenerate.py + git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1) + + binary_linux_wheel: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_linux_conda: + <<: *binary_common + docker: + - image: "pytorch/conda-cuda" + resource_class: 2xlarge+ + steps: + - checkout + - run: packaging/build_conda.sh + - store_artifacts: + path: /opt/conda/conda-bld/linux-64 + - persist_to_workspace: + root: /opt/conda/conda-bld/linux-64 + paths: + - "*" + + binary_win_conda: + <<: *binary_common + executor: + name: win/default + shell: bash.exe + steps: + - checkout + - run: + command: | + choco install miniconda3 + (& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression + conda activate base + conda install -yq conda-build "conda-package-handling!=1.5.0" + bash packaging/build_conda.sh + shell: powershell.exe + + binary_macos_wheel: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + # Cannot easily deduplicate this as source'ing activate + # will set environment variables which we need to propagate + # to build_wheel.sh + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + packaging/build_wheel.sh + - store_artifacts: + path: dist + - persist_to_workspace: + root: dist + paths: + - "*" + + binary_macos_conda: + <<: *binary_common + macos: + xcode: "9.0" + steps: + - checkout + - run: + command: | + curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh + sh conda.sh -b + source $HOME/miniconda3/bin/activate + conda install -yq conda-build + packaging/build_conda.sh + - store_artifacts: + path: /Users/distiller/miniconda3/conda-bld/osx-64 + - persist_to_workspace: + root: /Users/distiller/miniconda3/conda-bld/osx-64 + paths: + - "*" + + # Requires org-member context + binary_conda_upload: + docker: + - image: continuumio/miniconda + steps: + - attach_workspace: + at: ~/workspace + - run: + command: | + # Prevent credential from leaking + conda install -yq anaconda-client + set -x + anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force + + # Requires org-member context + binary_wheel_upload: + docker: + - image: circleci/python:3.8 + steps: + - attach_workspace: + at: ~/workspace + - checkout + - run: + command: | + pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + # Prevent credential from leaking + set +x + export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" + set -x + for pkg in ~/workspace/*.whl; do + aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read + done + + smoke_test_linux_conda: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + conda install -v -y -c pytorch-nightly pytorch + conda install -v -y $(ls ~/workspace/torchtext*.tar.bz2) + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchtext" + + smoke_test_linux_pip: + <<: *smoke_test_common + steps: + - attach_workspace: + at: ~/workspace + - run: + name: install binaries + command: | + set -x + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + pip install $(ls ~/workspace/torchtext*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html + - run: + name: smoke test + command: | + source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} + python -c "import torchtext" + + docker_image_build: + machine: + image: ubuntu-1604:201903-01 + resource_class: large + environment: + image_name: torchtext/smoke_test + steps: + - checkout + - run: + name: build_docker image + no_output_timeout: "1h" + command: | + cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + - run: + name: upload docker image + no_output_timeout: "1h" + command: | + set +x + export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY} + export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY} + eval $(aws ecr get-login --region us-east-1 --no-include-email) + set -x + docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + +workflows: + build: + jobs: + - circleci_consistency + {{ workflows() }} + - binary_win_conda: + name: torchtext_win_py3.6 + python_version: "3.6" + + nightly: + jobs: + - circleci_consistency + {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} + docker_build: + triggers: + - schedule: + cron: "0 10 * * 0" + filters: + branches: + only: + - master + jobs: + - docker_image_build diff --git a/.circleci/docker/Dockerfile b/.circleci/docker/Dockerfile new file mode 100644 index 0000000000..e98e83fb71 --- /dev/null +++ b/.circleci/docker/Dockerfile @@ -0,0 +1,36 @@ +# this Dockerfile is for torchtext smoke test, it will be created periodically via CI system +# if you need to do it locally, follow below steps once you have Docker installed +# assuming you're within the directory where this Dockerfile located +# $ docker build . -t torchtext/smoketest + +# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run +# $ eval $(aws ecr get-login --region us-east-1 --no-include-email) +# $ export MYTAG=localbuild ## you can choose whatever tag you like +# $ docker tag torchtext/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtext/smoke_test:${MYTAG} +# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchtext/smoke_test:${MYTAG} + +FROM ubuntu:latest + +RUN apt-get -qq update && apt-get -qq -y install curl bzip2 sox libsox-dev libsox-fmt-all \ + && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -bfp /usr/local \ + && rm -rf /tmp/miniconda.sh \ + && conda install -y python=3 \ + && conda update conda \ + && apt-get -qq -y remove curl bzip2 \ + && apt-get -qq -y autoremove \ + && apt-get autoclean \ + && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \ + && conda clean --all --yes + +ENV PATH /opt/conda/bin:$PATH + +RUN conda create -y --name python3.5 python=3.5 +RUN conda create -y --name python3.6 python=3.6 +RUN conda create -y --name python3.7 python=3.7 +SHELL [ "/bin/bash", "-c" ] +RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc +RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.5 && conda install -y -c conda-forge sox && conda install -y numpy +RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.6 && conda install -y -c conda-forge sox && conda install -y numpy +RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.7 && conda install -y -c conda-forge sox && conda install -y numpy +CMD [ "/bin/bash"] diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py new file mode 100755 index 0000000000..6120101641 --- /dev/null +++ b/.circleci/regenerate.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +""" +This script should use a very simple, functional programming style. +Avoid Jinja macros in favor of native Python functions. + +Don't go overboard on code generation; use Python only to generate +content that can't be easily declared statically using CircleCI's YAML API. + +Data declarations (e.g. the nested loops for defining the configuration matrix) +should be at the top of the file for easy updating. + +See this comment for design rationale: +https://github.com/pytorch/vision/pull/1321#issuecomment-531033978 +""" + +import jinja2 +import yaml +import os.path + + +def workflows(prefix='', upload=False, filter_branch=None, indentation=6): + w = [] + for btype in ["wheel", "conda"]: + for os_type in ["linux", "macos"]: + for python_version in ["3.6", "3.7", "3.8"]: + w += workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) + + return indent(indentation, w) + + +def workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False): + + w = [] + base_workflow_name = "{prefix}binary_{os_type}_{btype}_py{python_version}".format( + prefix=prefix, + os_type=os_type, + btype=btype, + python_version=python_version, + ) + + w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype)) + + if upload: + + is_py3_linux = os_type == 'linux' and not python_version.startswith("2.") + + w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype)) + + if filter_branch == 'nightly' and is_py3_linux: + pydistro = 'pip' if btype == 'wheel' else 'conda' + w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version)) + + return w + + +def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype): + + d = { + "name": base_workflow_name, + "python_version": python_version, + } + + if filter_branch: + d["filters"] = gen_filter_branch_tree(filter_branch) + + return {"binary_{os_type}_{btype}".format(os_type=os_type, btype=btype): d} + + +def gen_filter_branch_tree(branch_name): + return {"branches": {"only": branch_name}} + + +def generate_upload_workflow(base_workflow_name, filter_branch, btype): + d = { + "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name), + "context": "org-member", + "requires": [base_workflow_name], + } + + if filter_branch: + d["filters"] = gen_filter_branch_tree(filter_branch) + + return {"binary_{btype}_upload".format(btype=btype): d} + + +def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version): + + required_build_suffix = "_upload" + required_build_name = base_workflow_name + required_build_suffix + + smoke_suffix = "smoke_test_{pydistro}".format(pydistro=pydistro) + d = { + "name": "{base_workflow_name}_{smoke_suffix}".format( + base_workflow_name=base_workflow_name, smoke_suffix=smoke_suffix), + "requires": [required_build_name], + "python_version": python_version, + } + + if filter_branch: + d["filters"] = gen_filter_branch_tree(filter_branch) + + return {"smoke_test_linux_{pydistro}".format(pydistro=pydistro): d} + + +def indent(indentation, data_list): + return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines()) + + +if __name__ == "__main__": + d = os.path.dirname(__file__) + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(d), + lstrip_blocks=True, + autoescape=False, + ) + + with open(os.path.join(d, 'config.yml'), 'w') as f: + f.write(env.get_template('config.yml.in').render(workflows=workflows)) diff --git a/.circleci/test/test_sort_yaml.py b/.circleci/test/test_sort_yaml.py new file mode 100755 index 0000000000..dc6db481dd --- /dev/null +++ b/.circleci/test/test_sort_yaml.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +""" +To compare new version with previous: + + ./regenerate.sh + meld <(git show HEAD:./config.yml | ./sort-yaml.py) <(cat config.yml | ./sort-yaml.py) +""" + + +import sys +import yaml + +sys.stdout.write(yaml.dump(yaml.load(sys.stdin, Loader=yaml.FullLoader), sort_keys=True)) From 930766b588bb76882e53df6e6eb74c73b59008a1 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 29 Apr 2020 19:00:30 +0000 Subject: [PATCH 2/5] Add unittest --- .circleci/config.yml | 58 +++++++++++++++++-- .circleci/config.yml.in | 54 +++++++++++++++--- .circleci/regenerate.py | 59 +++++++++++--------- .circleci/{ => smoke_test}/docker/Dockerfile | 0 .circleci/unittest/scripts/environment.yml | 18 ++++++ .circleci/unittest/scripts/install.sh | 17 ++++++ .circleci/unittest/scripts/post_process.sh | 8 +++ .circleci/unittest/scripts/run_test.sh | 10 ++++ .circleci/unittest/scripts/setup_env.sh | 38 +++++++++++++ .circleci/{test => utils}/test_sort_yaml.py | 0 10 files changed, 223 insertions(+), 39 deletions(-) rename .circleci/{ => smoke_test}/docker/Dockerfile (100%) create mode 100644 .circleci/unittest/scripts/environment.yml create mode 100755 .circleci/unittest/scripts/install.sh create mode 100755 .circleci/unittest/scripts/post_process.sh create mode 100755 .circleci/unittest/scripts/run_test.sh create mode 100755 .circleci/unittest/scripts/setup_env.sh rename .circleci/{test => utils}/test_sort_yaml.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 67a9905bc9..3455404192 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ jobs: source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} python -c "import torchtext" - docker_image_build: + smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 resource_class: large @@ -220,7 +220,7 @@ jobs: name: build_docker image no_output_timeout: "1h" command: | - cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - run: name: upload docker image no_output_timeout: "1h" @@ -233,6 +233,41 @@ jobs: docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + unittest_linux: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - restore_cache: + + keys: + - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }} + + - run: + name: Setup + command: .circleci/unittest/scripts/setup_env.sh + - save_cache: + + key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }} + + paths: + - conda + - env + - run: + name: Install torchtext + command: .circleci/unittest/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/scripts/post_process.sh + - store_test_results: + path: test-results + + workflows: build: jobs: @@ -276,10 +311,23 @@ workflows: - binary_win_conda: name: torchtext_win_py3.6 python_version: "3.6" - + unittest: + jobs: + - unittest_linux: + name: unittest_linux_py3.6 + python_version: '3.6' + - unittest_linux: + name: unittest_linux_py3.7 + python_version: '3.7' + - unittest_linux: + name: unittest_linux_py3.8 + python_version: '3.8' nightly: jobs: - - circleci_consistency + - circleci_consistency: + filters: + branches: + only: nightly - binary_linux_wheel: filters: branches: @@ -505,4 +553,4 @@ workflows: only: - master jobs: - - docker_image_build \ No newline at end of file + - smoke_test_docker_image_build \ No newline at end of file diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 270aa72856..89c568fec1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -208,7 +208,7 @@ jobs: source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION} python -c "import torchtext" - docker_image_build: + smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 resource_class: large @@ -220,7 +220,7 @@ jobs: name: build_docker image no_output_timeout: "1h" command: | - cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} + cd .circleci/smoke_test/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID} - run: name: upload docker image no_output_timeout: "1h" @@ -233,19 +233,59 @@ jobs: docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID} + unittest_linux: + <<: *binary_common + docker: + - image: "pytorch/manylinux-cuda100" + resource_class: 2xlarge+ + steps: + - checkout + - restore_cache: + {% raw %} + keys: + - env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }} + {% endraw %} + - run: + name: Setup + command: .circleci/unittest/scripts/setup_env.sh + - save_cache: + {% raw %} + key: env-v1-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/scripts/environment.yml" }} + {% endraw %} + paths: + - conda + - env + - run: + name: Install torchtext + command: .circleci/unittest/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/scripts/post_process.sh + - store_test_results: + path: test-results + + workflows: build: jobs: - circleci_consistency - {{ workflows() }} + {{ build_workflows() }} - binary_win_conda: name: torchtext_win_py3.6 python_version: "3.6" - + unittest: + jobs: + {{ unittest_workflows() }} nightly: jobs: - - circleci_consistency - {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} + - circleci_consistency: + filters: + branches: + only: nightly + {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} docker_build: triggers: - schedule: @@ -255,4 +295,4 @@ workflows: only: - master jobs: - - docker_image_build + - smoke_test_docker_image_build diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 6120101641..3374fadbea 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -19,43 +19,33 @@ import os.path -def workflows(prefix='', upload=False, filter_branch=None, indentation=6): +PYTHON_VERSIONS = ["3.6", "3.7", "3.8"] + + +def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): w = [] for btype in ["wheel", "conda"]: for os_type in ["linux", "macos"]: - for python_version in ["3.6", "3.7", "3.8"]: - w += workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) + for python_version in PYTHON_VERSIONS: + w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) return indent(indentation, w) -def workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False): - +def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False): w = [] - base_workflow_name = "{prefix}binary_{os_type}_{btype}_py{python_version}".format( - prefix=prefix, - os_type=os_type, - btype=btype, - python_version=python_version, - ) - + base_workflow_name = f"{prefix}binary_{os_type}_{btype}_py{python_version}" w.append(generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype)) if upload: - - is_py3_linux = os_type == 'linux' and not python_version.startswith("2.") - w.append(generate_upload_workflow(base_workflow_name, filter_branch, btype)) - - if filter_branch == 'nightly' and is_py3_linux: + if filter_branch == 'nightly' and os_type == 'linux': pydistro = 'pip' if btype == 'wheel' else 'conda' w.append(generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version)) - return w def generate_base_workflow(base_workflow_name, python_version, filter_branch, os_type, btype): - d = { "name": base_workflow_name, "python_version": python_version, @@ -64,7 +54,7 @@ def generate_base_workflow(base_workflow_name, python_version, filter_branch, os if filter_branch: d["filters"] = gen_filter_branch_tree(filter_branch) - return {"binary_{os_type}_{btype}".format(os_type=os_type, btype=btype): d} + return {f"binary_{os_type}_{btype}": d} def gen_filter_branch_tree(branch_name): @@ -73,7 +63,7 @@ def gen_filter_branch_tree(branch_name): def generate_upload_workflow(base_workflow_name, filter_branch, btype): d = { - "name": "{base_workflow_name}_upload".format(base_workflow_name=base_workflow_name), + "name": f"{base_workflow_name}_upload", "context": "org-member", "requires": [base_workflow_name], } @@ -81,7 +71,7 @@ def generate_upload_workflow(base_workflow_name, filter_branch, btype): if filter_branch: d["filters"] = gen_filter_branch_tree(filter_branch) - return {"binary_{btype}_upload".format(btype=btype): d} + return {f"binary_{btype}_upload": d} def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, python_version): @@ -89,10 +79,9 @@ def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, pyt required_build_suffix = "_upload" required_build_name = base_workflow_name + required_build_suffix - smoke_suffix = "smoke_test_{pydistro}".format(pydistro=pydistro) + smoke_suffix = f"smoke_test_{pydistro}" d = { - "name": "{base_workflow_name}_{smoke_suffix}".format( - base_workflow_name=base_workflow_name, smoke_suffix=smoke_suffix), + "name": f"{base_workflow_name}_{smoke_suffix}", "requires": [required_build_name], "python_version": python_version, } @@ -100,13 +89,26 @@ def generate_smoketest_workflow(pydistro, base_workflow_name, filter_branch, pyt if filter_branch: d["filters"] = gen_filter_branch_tree(filter_branch) - return {"smoke_test_linux_{pydistro}".format(pydistro=pydistro): d} + return {f"smoke_test_linux_{pydistro}": d} def indent(indentation, data_list): return ("\n" + " " * indentation).join(yaml.dump(data_list).splitlines()) +def unittest_workflows(indentation=6): + w = [] + for os_type in ["linux"]: + for python_version in PYTHON_VERSIONS: + w.append({ + f"unittest_{os_type}": { + "name": f"unittest_{os_type}_py{python_version}", + "python_version": python_version, + } + }) + return indent(indentation, w) + + if __name__ == "__main__": d = os.path.dirname(__file__) env = jinja2.Environment( @@ -116,4 +118,7 @@ def indent(indentation, data_list): ) with open(os.path.join(d, 'config.yml'), 'w') as f: - f.write(env.get_template('config.yml.in').render(workflows=workflows)) + f.write(env.get_template('config.yml.in').render( + build_workflows=build_workflows, + unittest_workflows=unittest_workflows, + )) diff --git a/.circleci/docker/Dockerfile b/.circleci/smoke_test/docker/Dockerfile similarity index 100% rename from .circleci/docker/Dockerfile rename to .circleci/smoke_test/docker/Dockerfile diff --git a/.circleci/unittest/scripts/environment.yml b/.circleci/unittest/scripts/environment.yml new file mode 100644 index 0000000000..15e6078e15 --- /dev/null +++ b/.circleci/unittest/scripts/environment.yml @@ -0,0 +1,18 @@ +channels: + - defaults +dependencies: + - flake8 + - codecov + - pip + - pip: + - nltk + - requests + - revtok + - pytest + - pytest-cov + - pytest-pythonpath + - sacremoses + - spacy + - sphinx + - sphinx-rtd-theme + - tqdm diff --git a/.circleci/unittest/scripts/install.sh b/.circleci/unittest/scripts/install.sh new file mode 100755 index 0000000000..e6682818e0 --- /dev/null +++ b/.circleci/unittest/scripts/install.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +printf "* Installing PyTorch nightly build" +conda install -y -c pytorch-nightly pytorch cpuonly + +printf "* Installing torchtext\n" +python setup.py develop diff --git a/.circleci/unittest/scripts/post_process.sh b/.circleci/unittest/scripts/post_process.sh new file mode 100755 index 0000000000..a84a0dea55 --- /dev/null +++ b/.circleci/unittest/scripts/post_process.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +codecov diff --git a/.circleci/unittest/scripts/run_test.sh b/.circleci/unittest/scripts/run_test.sh new file mode 100755 index 0000000000..0bf7f3be0c --- /dev/null +++ b/.circleci/unittest/scripts/run_test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +python -m torch.utils.collect_env +flake8 +pytest --cov=torchtext --junitxml=test-results/junit.xml -v test diff --git a/.circleci/unittest/scripts/setup_env.sh b/.circleci/unittest/scripts/setup_env.sh new file mode 100755 index 0000000000..7b208630d4 --- /dev/null +++ b/.circleci/unittest/scripts/setup_env.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchtext here, otherwise they also get cached. + +set -e + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash ./miniconda.sh -b -f -p "${conda_dir}" +fi +eval "$(${conda_dir}/bin/conda shell.bash hook)" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +conda env update --file "${this_dir}/environment.yml" --prune + +# 4. Download +printf "* SpaCy English models\n" +python -m spacy download en diff --git a/.circleci/test/test_sort_yaml.py b/.circleci/utils/test_sort_yaml.py similarity index 100% rename from .circleci/test/test_sort_yaml.py rename to .circleci/utils/test_sort_yaml.py From 6d5870237a6385350c9599c263b8832d6d521ce0 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 29 Apr 2020 13:28:56 -0700 Subject: [PATCH 3/5] Fix flake8, disable binary --- .circleci/config.yml | 266 +------------------------ .circleci/config.yml.in | 16 +- .circleci/regenerate.py | 4 +- .circleci/unittest/scripts/run_test.sh | 2 +- 4 files changed, 20 insertions(+), 268 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3455404192..c44a1bcdd2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -269,48 +269,13 @@ jobs: workflows: - build: - jobs: - - circleci_consistency - - binary_linux_wheel: - name: binary_linux_wheel_py3.6 - python_version: '3.6' - - binary_linux_wheel: - name: binary_linux_wheel_py3.7 - python_version: '3.7' - - binary_linux_wheel: - name: binary_linux_wheel_py3.8 - python_version: '3.8' - - binary_macos_wheel: - name: binary_macos_wheel_py3.6 - python_version: '3.6' - - binary_macos_wheel: - name: binary_macos_wheel_py3.7 - python_version: '3.7' - - binary_macos_wheel: - name: binary_macos_wheel_py3.8 - python_version: '3.8' - - binary_linux_conda: - name: binary_linux_conda_py3.6 - python_version: '3.6' - - binary_linux_conda: - name: binary_linux_conda_py3.7 - python_version: '3.7' - - binary_linux_conda: - name: binary_linux_conda_py3.8 - python_version: '3.8' - - binary_macos_conda: - name: binary_macos_conda_py3.6 - python_version: '3.6' - - binary_macos_conda: - name: binary_macos_conda_py3.7 - python_version: '3.7' - - binary_macos_conda: - name: binary_macos_conda_py3.8 - python_version: '3.8' - - binary_win_conda: - name: torchtext_win_py3.6 - python_version: "3.6" + # build: + # jobs: + # - circleci_consistency + # + # - binary_win_conda: + # name: torchtext_win_py3.6 + # python_version: "3.6" unittest: jobs: - unittest_linux: @@ -328,222 +293,7 @@ workflows: filters: branches: only: nightly - - binary_linux_wheel: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.6 - python_version: '3.6' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.6_upload - requires: - - nightly_binary_linux_wheel_py3.6 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.6_smoke_test_pip - python_version: '3.6' - requires: - - nightly_binary_linux_wheel_py3.6_upload - - binary_linux_wheel: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.7 - python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.7_upload - requires: - - nightly_binary_linux_wheel_py3.7 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.7_smoke_test_pip - python_version: '3.7' - requires: - - nightly_binary_linux_wheel_py3.7_upload - - binary_linux_wheel: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.8 - python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.8_upload - requires: - - nightly_binary_linux_wheel_py3.8 - - smoke_test_linux_pip: - filters: - branches: - only: nightly - name: nightly_binary_linux_wheel_py3.8_smoke_test_pip - python_version: '3.8' - requires: - - nightly_binary_linux_wheel_py3.8_upload - - binary_macos_wheel: - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.6 - python_version: '3.6' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.6_upload - requires: - - nightly_binary_macos_wheel_py3.6 - - binary_macos_wheel: - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.7 - python_version: '3.7' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.7_upload - requires: - - nightly_binary_macos_wheel_py3.7 - - binary_macos_wheel: - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.8 - python_version: '3.8' - - binary_wheel_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_wheel_py3.8_upload - requires: - - nightly_binary_macos_wheel_py3.8 - - binary_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.6 - python_version: '3.6' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.6_upload - requires: - - nightly_binary_linux_conda_py3.6 - - smoke_test_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.6_smoke_test_conda - python_version: '3.6' - requires: - - nightly_binary_linux_conda_py3.6_upload - - binary_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.7 - python_version: '3.7' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.7_upload - requires: - - nightly_binary_linux_conda_py3.7 - - smoke_test_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.7_smoke_test_conda - python_version: '3.7' - requires: - - nightly_binary_linux_conda_py3.7_upload - - binary_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.8 - python_version: '3.8' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.8_upload - requires: - - nightly_binary_linux_conda_py3.8 - - smoke_test_linux_conda: - filters: - branches: - only: nightly - name: nightly_binary_linux_conda_py3.8_smoke_test_conda - python_version: '3.8' - requires: - - nightly_binary_linux_conda_py3.8_upload - - binary_macos_conda: - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.6 - python_version: '3.6' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.6_upload - requires: - - nightly_binary_macos_conda_py3.6 - - binary_macos_conda: - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.7 - python_version: '3.7' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.7_upload - requires: - - nightly_binary_macos_conda_py3.7 - - binary_macos_conda: - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.8 - python_version: '3.8' - - binary_conda_upload: - context: org-member - filters: - branches: - only: nightly - name: nightly_binary_macos_conda_py3.8_upload - requires: - - nightly_binary_macos_conda_py3.8 + # docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 89c568fec1..c86dbbeca1 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -269,13 +269,13 @@ jobs: workflows: - build: - jobs: - - circleci_consistency - {{ build_workflows() }} - - binary_win_conda: - name: torchtext_win_py3.6 - python_version: "3.6" + # build: + # jobs: + # - circleci_consistency + # {{ build_workflows() }} + # - binary_win_conda: + # name: torchtext_win_py3.6 + # python_version: "3.6" unittest: jobs: {{ unittest_workflows() }} @@ -285,7 +285,7 @@ workflows: filters: branches: only: nightly - {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} + # {{ build_workflows(prefix="nightly_", filter_branch="nightly", upload=True) }} docker_build: triggers: - schedule: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 3374fadbea..1c57596ad5 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -23,6 +23,7 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): + ''' w = [] for btype in ["wheel", "conda"]: for os_type in ["linux", "macos"]: @@ -30,7 +31,8 @@ def build_workflows(prefix='', upload=False, filter_branch=None, indentation=6): w += build_workflow_pair(btype, os_type, python_version, filter_branch, prefix, upload) return indent(indentation, w) - + ''' + return '' def build_workflow_pair(btype, os_type, python_version, filter_branch, prefix='', upload=False): w = [] diff --git a/.circleci/unittest/scripts/run_test.sh b/.circleci/unittest/scripts/run_test.sh index 0bf7f3be0c..aaad175425 100755 --- a/.circleci/unittest/scripts/run_test.sh +++ b/.circleci/unittest/scripts/run_test.sh @@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -flake8 +flake8 torchtext test pytest --cov=torchtext --junitxml=test-results/junit.xml -v test From 1f500dc1b48579610abfa990ab9e6b9ada7d360d Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 29 Apr 2020 14:09:02 -0700 Subject: [PATCH 4/5] Fix typo --- .circleci/unittest/scripts/install.sh | 2 +- .circleci/unittest/scripts/setup_env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/unittest/scripts/install.sh b/.circleci/unittest/scripts/install.sh index e6682818e0..f02440942e 100755 --- a/.circleci/unittest/scripts/install.sh +++ b/.circleci/unittest/scripts/install.sh @@ -10,7 +10,7 @@ set -e eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env -printf "* Installing PyTorch nightly build" +printf "* Installing PyTorch nightly build\n" conda install -y -c pytorch-nightly pytorch cpuonly printf "* Installing torchtext\n" diff --git a/.circleci/unittest/scripts/setup_env.sh b/.circleci/unittest/scripts/setup_env.sh index 7b208630d4..f64c4f49d8 100755 --- a/.circleci/unittest/scripts/setup_env.sh +++ b/.circleci/unittest/scripts/setup_env.sh @@ -34,5 +34,5 @@ printf "* Installing dependencies (except PyTorch)\n" conda env update --file "${this_dir}/environment.yml" --prune # 4. Download -printf "* SpaCy English models\n" +printf "* Downloading SpaCy English models\n" python -m spacy download en From 270eeb8ae9bc1c45d32b565598826d38b14dfc25 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 29 Apr 2020 14:09:42 -0700 Subject: [PATCH 5/5] Run flake8 after unittest --- .circleci/unittest/scripts/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest/scripts/run_test.sh b/.circleci/unittest/scripts/run_test.sh index aaad175425..19649530b5 100755 --- a/.circleci/unittest/scripts/run_test.sh +++ b/.circleci/unittest/scripts/run_test.sh @@ -6,5 +6,5 @@ eval "$(./conda/bin/conda shell.bash hook)" conda activate ./env python -m torch.utils.collect_env -flake8 torchtext test pytest --cov=torchtext --junitxml=test-results/junit.xml -v test +flake8 torchtext test