build-support/travis/travis.yml.mustache

{{{header}}}

# Conditions are documented here: https://docs.travis-ci.com/user/conditions-v1
conditions: v1

if: commit_message !~ SKIP_FULL_CI

# -------------------------------------------------------------------------
# Global setup
# -------------------------------------------------------------------------

env:
  global:
    - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini"
    - LC_ALL="en_US.UTF-8"
    - BOOTSTRAPPED_PEX_BUCKET=ci-public.pantsbuild.org
    - BOOTSTRAPPED_PEX_KEY_PREFIX=${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex
    - BOOTSTRAPPED_PEX_URL_PREFIX=s3://${BOOTSTRAPPED_PEX_BUCKET}/${BOOTSTRAPPED_PEX_KEY_PREFIX}
    - PYENV_PY27_VERSION=2.7.15
    - PYENV_PY36_VERSION=3.6.8
    - PYENV_PY37_VERSION=3.7.2
    # NB: Linux shards use Pyenv to pre-install Python. We must not override
    # PYENV_ROOT on those shards, or their Python will no longer work.
    - PYENV_ROOT="${PYENV_ROOT:-${HOME}/.pants_pyenv}"
    - PATH="${PYENV_ROOT}/shims:${PATH}"
    - AWS_CLI_ROOT="${HOME}/.aws_cli"

# Stages are documented here: https://docs.travis-ci.com/user/build-stages
stages:
  - name: &bootstrap Bootstrap Pants
    if: type != cron
  - name: &bootstrap_cron Bootstrap Pants (Cron)
    if: type = cron
  - name: &test Test Pants
    if: type != cron
  - name: &test_cron Test Pants (Cron)
    if: type = cron
  - name: &build_stable Deploy Pants Pex
    if: tag IS present AND tag =~ ^release_.*$
  - name: &build_unstable Deploy Pants Pex Unstable
    if: tag IS NOT present AND type NOT IN (pull_request, cron)

# -------------------------------------------------------------------------
# Cache config
# -------------------------------------------------------------------------

# Travis cache config for jobs that build the native engine.
native_engine_cache_config: &native_engine_cache_config
  before_cache:
    # Ensure permissions to do the below removals, which happen with or without caching enabled.
    - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}"
    # Kill all python bytecode in our cached venvs.  Some files appear to
    # get bytecode compiled in non-yet-understood circumstances leading to
    # a full cache re-pack due to new bytecode files.
    - find build-support -name "*.py[co]" -delete
  cache:
    # The default timeout is 180 seconds, and our larger cache uploads exceed this.
    # TODO: Figure out why we have such large caches (2-7GB) and try to trim them.
    timeout: 500
    directories:
      - ${AWS_CLI_ROOT}
      - ${PYENV_ROOT}
      - ${HOME}/.cache/pants/rust/cargo
      - build-support/pants_dev_deps.py27.venv
      - build-support/pants_dev_deps.py36.venv
      - build-support/pants_dev_deps.py37.venv
      - src/rust/engine/target

# Travis cache config for jobs that run a bootstrapped pants.pex.
pants_run_cache_config: &pants_run_cache_config
  before_cache:
    # Ensure permissions to do the below removals, which happen with or without caching enabled.
    - sudo chown -R travis:travis "${HOME}" "${TRAVIS_BUILD_DIR}"
    # The `ivydata-*.properties` & root level `*.{properties,xml}` files'
    # effect on resolution time is in the noise, but they are
    # re-timestamped in internal comments and fields on each run and this
    # leads to travis-ci cache thrash.  Kill these files before the cache
    # check to avoid un-needed cache re-packing and re-upload (a ~100s
    # operation).
    - find ${HOME}/.ivy2/pants -type f -name "ivydata-*.properties" -delete
    - rm -f ${HOME}/.ivy2/pants/*.{css,properties,xml,xsl}
    # We have several tests that do local file:// url resolves for
    # com.example artifacts, these disrupt the cache but are fast since
    # they're resolved from local files when omitted from the cache.
    - rm -rf ${HOME}/.ivy2/pants/com.example
    # Render a summary to assist with further tuning the cache.
    - du -m -d2 ${HOME}/.cache/pants | sort -r -n
  cache:
    # The default timeout is 180 seconds, and our larger cache uploads exceed this.
    # TODO: Figure out why we have such large caches (2-7GB) and try to trim them.
    timeout: 500
    directories:
      - ${AWS_CLI_ROOT}
      - ${PYENV_ROOT}
      - ${HOME}/.cache/pants/tools
      - ${HOME}/.cache/pants/zinc
      - ${HOME}/.ivy2/pants
      # TODO(John Sirois): Update this to ~/.npm/pants when pants starts
      # using its own isolated cache:
      #   https://github.com/pantsbuild/pants/issues/2485
      - ${HOME}/.npm

# -------------------------------------------------------------------------
# AWS
# -------------------------------------------------------------------------

# We use AWS S3 to avoid unnecessary work in CI. Specifically, the bootstrap
# shards create a pants.pex, and then upload it to S3 for all of the test
# shards to pull down.

aws_deploy_pants_pex: &aws_deploy_pants_pex >
  aws --no-sign-request --region us-east-1 s3 cp ${TRAVIS_BUILD_DIR}/pants.pex ${BOOTSTRAPPED_PEX_URL_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}

aws_get_pants_pex: &aws_get_pants_pex >
  ./build-support/bin/get_ci_bootstrapped_pants_pex.sh ${BOOTSTRAPPED_PEX_BUCKET} ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}

# -------------------------------------------------------------------------
# Generic shard setups
# -------------------------------------------------------------------------

run_tests_under_pantsd: &run_tests_under_pantsd >
  USE_PANTSD_FOR_INTEGRATION_TESTS="true"

base_linux_config: &base_linux_config
  os: linux
  dist: xenial
  sudo: required
  python:
    - "2.7"
    - "3.6"
    - "3.7"
  addons:
    apt:
      packages:
        - lib32stdc++6
        - lib32z1
        - lib32z1-dev
        - gcc-multilib
        - python-dev
        - openssl
        - libssl-dev
        - jq
        - unzip
  language: python
  before_install:
    - ./build-support/bin/install_aws_cli_for_ci.sh
    # TODO(John Sirois): Get rid of this in favor of explicitly adding pyenv versions to the PATH:
    #   https://github.com/pantsbuild/pants/issues/7601
    - pyenv global 2.7.15 3.6.7 3.7.1
  after_failure:
    - ./build-support/bin/ci-failure.sh

py27_linux_config: &py27_linux_config
  <<: *base_linux_config

py36_linux_config: &py36_linux_config
  <<: *base_linux_config

py37_linux_config: &py37_linux_config
  <<: *base_linux_config

base_linux_test_config: &base_linux_test_config
  <<: *base_linux_config
  <<: *pants_run_cache_config
  before_install:
    {{>before_install_linux}}
  before_script:
    - *aws_get_pants_pex

py27_linux_test_config: &py27_linux_test_config
  <<: *py27_linux_config
  <<: *base_linux_test_config
  stage: *test_cron
  env:
    - &py27_linux_test_config_env >
      BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.linux

py36_linux_test_config: &py36_linux_test_config
  <<: *py36_linux_config
  <<: *base_linux_test_config
  stage: *test
  env:
    - &py36_linux_test_config_env >
      BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux

py37_linux_test_config: &py37_linux_test_config
  <<: *py37_linux_config
  <<: *base_linux_test_config
  stage: *test_cron
  env:
    # TODO(https://github.com/tensorflow/tensorflow/issues/27078): tensorflow==1.13.1 on python 3.7.2 for Linux uses the new C++ ABI, which may be an error.
    - &py37_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]"

base_osx_config: &base_osx_config
  os: osx
  language: generic
  addons:
    brew:
      packages:
      - openssl

py27_osx_config: &py27_osx_config
  <<: *base_osx_config
  # NB: We ensure sane python2 and python3 interpreters are available on the PATH for scripting
  # against.
  env:
    - &py27_osx_config_env >
      {{>env_osx_with_pyenv}}
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin:${PATH}"
  before_install:
    {{>before_install_osx}}
    - ./build-support/bin/install_python_for_ci.sh "${PYENV_PY27_VERSION}" "${PYENV_PY36_VERSION}"

py36_osx_config: &py36_osx_config
  <<: *base_osx_config
  # NB: We ensure sane python2 and python3 interpreters are available on the PATH for scripting
  # against.
  env:
    - &py36_osx_config_env >
      {{>env_osx_with_pyenv}}
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin:${PATH}"
  before_install:
    {{>before_install_osx}}
    - ./build-support/bin/install_python_for_ci.sh "${PYENV_PY27_VERSION}" "${PYENV_PY36_VERSION}"

py37_osx_config: &py37_osx_config
  <<: *base_osx_config
  # NB: We ensure sane python2 and python3 interpreters are available on the PATH for scripting
  # against.
  env:
    - &py37_osx_config_env >
      {{>env_osx_with_pyenv}}
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY37_VERSION}/bin:${PATH}"
  before_install:
    {{>before_install_osx}}
    - ./build-support/bin/install_python_for_ci.sh "${PYENV_PY27_VERSION}" "${PYENV_PY37_VERSION}"

base_osx_test_config: &base_osx_test_config
  <<: *pants_run_cache_config
  before_script:
    - ulimit -c unlimited
    - ulimit -n 8192
    - *aws_get_pants_pex

py27_osx_test_config: &py27_osx_test_config
  <<: *py27_osx_config
  <<: *base_osx_test_config
  stage: *test_cron
  env:
    - &py27_osx_test_config_env >
      {{>env_osx_with_pyenv}}
      BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.osx
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin:${PATH}"

py36_osx_test_config: &py36_osx_test_config
  <<: *py36_osx_config
  <<: *base_osx_test_config
  stage: *test
  env:
    - &py36_osx_test_config_env >
      {{>env_osx_with_pyenv}}
      BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.osx
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin:${PATH}"

py37_osx_test_config: &py37_osx_test_config
  <<: *py37_osx_config
  <<: *base_osx_test_config
  stage: *test_cron
  env:
    - &py37_osx_test_config_env >
      {{>env_osx_with_pyenv}}
      BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY37_VERSION}/bin:${PATH}"

linux_with_fuse: &linux_with_fuse
  os: linux
  dist: xenial
  sudo: required
  before_install:
    {{>before_install_linux}}
    - sudo apt-get install -y pkg-config fuse libfuse-dev
    - sudo modprobe fuse
    - sudo chmod 666 /dev/fuse
    - sudo chown root:$USER /etc/fuse.conf
  python:
    - "2.7"
    - "3.6"
    - "3.7"

travis_docker_image: &travis_docker_image
  services:
    - docker
  before_script:
    - ulimit -c unlimited

# -------------------------------------------------------------------------
# Bootstrap engine shards
# -------------------------------------------------------------------------

# Note for each platform, we have the Python 3.6 shard also create fs_util and
# upload to S3, to take advantage of the Rust code built during
# bootstrapping. We use the Python 3.6 shard, as it runs during both daily and
# nightly CI. This requires setting PREPARE_DEPLOY=1.

base_linux_build_engine: &base_linux_build_engine
  <<: *native_engine_cache_config
  <<: *travis_docker_image
  stage: *bootstrap
  # Callers of this anchor are expected to provide values in their `env` for
  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
  script:
    - >
      {{>docker_build_image}}
    - >
      {{>docker_run_image}}
    - *aws_deploy_pants_pex

py27_linux_build_engine: &py27_linux_build_engine
  <<: *py27_linux_config
  <<: *base_linux_build_engine
  name: "Build Linux native engine and pants.pex (Py2.7 PEX)"
  env:
    - docker_image_name=travis_ci
    # TODO: While this image shouldn't have any cache to fetch anything from, it fails to find
    # libpython2.7.so.1 during the cargo build, despite the image being built with --enable-shared,
    # unless the -x argument is added. This isn't expected to affect build time too much.
    - docker_run_command="./build-support/bin/ci.sh -2bx"
    - CACHE_NAME=linuxpexbuild.py27
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.linux

py36_linux_build_engine: &py36_linux_build_engine
  <<: *py36_linux_config
  <<: *base_linux_build_engine
  name: "Build Linux native engine and pants.pex (Py3.6 PEX)"
  env:
    - docker_image_name=travis_ci
    - docker_run_command="./build-support/bin/ci.sh -b && ./build-support/bin/release.sh -f"
    - PREPARE_DEPLOY=1
    - CACHE_NAME=linuxpexbuild.py36
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux

# NB: we do not use Docker for Py37 because Centos6 has too outdated of OpenSSL.
# See https://github.com/pantsbuild/pants/issues/7421. We will want to change this
# shard back to Docker once we have a Centos7 base image.
py37_linux_build_engine: &py37_linux_build_engine
  <<: *py37_linux_config
  <<: *native_engine_cache_config
  stage: *bootstrap_cron
  name: "Build Linux native engine and pants.pex (Py3.7 PEX)"
  env:
    - CACHE_NAME=linuxpexbuild.py37
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux
  script:
    - ./build-support/bin/ci.sh -7b
    - *aws_deploy_pants_pex

base_osx_build_engine: &base_osx_build_engine
  <<: *native_engine_cache_config
  stage: *bootstrap
  # We request the oldest image we can (corresponding to OSX 10.11) for maximum compatibility.
  # We use 10.11 as a minimum to avoid https://github.com/rust-lang/regex/issues/489.
  # See: https://docs.travis-ci.com/user/reference/osx/#OS-X-Version
  osx_image: xcode8
  after_failure:
    - ./build-support/bin/ci-failure.sh

py27_osx_build_engine: &py27_osx_build_engine
  <<: *py27_osx_config
  <<: *base_osx_build_engine
  name: "Build OSX native engine and pants.pex (Py2.7 PEX)"
  env:
    - *py27_osx_config_env
    - CACHE_NAME=osxpexbuild.py27
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py27.osx
  script:
    - ./build-support/bin/ci.sh -2b
    - *aws_deploy_pants_pex

py36_osx_build_engine: &py36_osx_build_engine
  <<: *py36_osx_config
  <<: *base_osx_build_engine
  name: "Build OSX native engine and pants.pex (Py3.6 PEX)"
  env:
    - *py36_osx_config_env
    - PREPARE_DEPLOY=1
    - CACHE_NAME=osxpexbuild.py36
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.osx
  script:
    - ./build-support/bin/ci.sh -b
    - ./build-support/bin/release.sh -f
    - *aws_deploy_pants_pex

py37_osx_build_engine: &py37_osx_build_engine
  <<: *py37_osx_config
  <<: *base_osx_build_engine
  stage: *bootstrap_cron
  name: "Build OSX native engine and pants.pex (Py3.7 PEX)"
  env:
    - *py37_osx_config_env
    - CACHE_NAME=osxpexbuild.py37
    - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx
  script:
    - ./build-support/bin/ci.sh -7b
    - *aws_deploy_pants_pex

# -------------------------------------------------------------------------
# Lint
# -------------------------------------------------------------------------

py27_lint: &py27_lint
  <<: *py27_linux_test_config
  name: "Self-checks and lint (Py2.7 PEX)"
  stage: *test
  env:
    - *py27_linux_test_config_env
    - CACHE_NAME=linuxselfchecks.py27
  script:
    - ./build-support/bin/ci.sh -fmrt2

py36_lint: &py36_lint
  <<: *py36_linux_test_config
  name: "Self-checks and lint (Py3.6 PEX)"
  env:
    - *py36_linux_test_config_env
    - CACHE_NAME=linuxselfchecks.py36
  script:
    - ./build-support/bin/ci.sh -fmrt

py37_lint: &py37_lint
  <<: *py37_linux_test_config
  name: "Self-checks and lint (Py3.7 PEX)"
  env:
    - *py37_linux_test_config_env
    - CACHE_NAME=linuxselfchecks.py37
  script:
    - ./build-support/bin/ci.sh -fmrt7

# -------------------------------------------------------------------------
# Rust lints
# -------------------------------------------------------------------------

base_rust_lints: &base_rust_lints
  <<: *linux_with_fuse

linux_rust_clippy: &linux_rust_clippy
  <<: *base_rust_lints
  <<: *native_engine_cache_config
  name: "Linux Rust Clippy (No PEX)"
  env:
    - CACHE_NAME=linuxclippy
  stage: *test
  before_script:
    - ulimit -c unlimited
    - ulimit -n 8192
  script:
    - ./build-support/bin/ci.sh -s

cargo_audit: &cargo_audit
  <<: *base_rust_lints
  name: "Cargo audit (No PEX)"
  env:
    - CACHE_NAME=linuxcargoaudit
  stage: *test_cron
  script:
    - ./build-support/bin/ci.sh -a

# -------------------------------------------------------------------------
# Build wheels
# -------------------------------------------------------------------------

# N.B. With Python 2, we must build pantsbuild.pants with both UCS2 and UCS4 to provide full
# compatibility for end users. This is because we constrain our ABI due to the native engine.
# See https://www.python.org/dev/peps/pep-0513/#ucs-2-vs-ucs-4-builds. Note this distinction is
# not necessary with Python 3.3+ due to flexible storage of Unicode strings (https://www.python.org/dev/peps/pep-0393/).
#
# We treat both Linux UCS4 and OSX UCS2 normally, as these are the defaults for those environments.
# The Linux UCS2 and OSX UCS4 shards, however, must rebuild Python with
# `PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs{2,4}` set, along with bootstrapping Pants again rather
# than pulling the PEX from AWS.

base_build_wheels: &base_build_wheels
  stage: *test
  env:
    - &base_build_wheels_env PREPARE_DEPLOY=1

base_linux_build_wheels: &base_linux_build_wheels
  # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize compatibility.
  <<: *travis_docker_image
  <<: *base_build_wheels
  # Callers of this anchor are expected to provide values in their `env` for
  # `docker_image_name` and `docker_run_command` (i.e. the bash command(s) to run).
  script:
    - >
      {{>docker_build_image}}
    - >
      {{>docker_run_image}}

py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2
  <<: *py27_linux_config
  <<: *base_linux_build_wheels
  <<: *native_engine_cache_config
  name: "Build wheels - Linux and cp27m (UCS2)"
  env:
    - *base_build_wheels_env
    - docker_image_name=travis_ci_py27_ucs2
    - docker_run_command="./build-support/bin/ci.sh -2b
                          && ./build-support/bin/check_pants_pex_abi.py cp27m
                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -2n"
    - CACHE_NAME=linuxwheelsbuild.ucs2

py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4
  <<: *base_linux_build_wheels
  <<: *py27_linux_test_config
  # `py27_linux_test_config` overrides the stage set by `base_build_wheels`, so we re-override it.
  stage: *test
  name: "Build wheels - Linux and cp27mu (UCS4)"
  env:
    - *py27_linux_test_config_env
    - *base_build_wheels_env
    - docker_image_name=travis_ci
    - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp27mu
                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -2n"
    - CACHE_NAME=linuxwheelsbuild.ucs4

py36_linux_build_wheels: &py36_linux_build_wheels
  <<: *base_linux_build_wheels
  <<: *py36_linux_test_config
  name: "Build wheels - Linux and abi3 (Py3.6+)"
  env:
    - *py36_linux_test_config_env
    - *base_build_wheels_env
    - docker_image_name=travis_ci
    - docker_run_command="./build-support/bin/check_pants_pex_abi.py abi3 cp36m
                          && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n"
    - CACHE_NAME=linuxwheelsbuild.abi3

base_osx_build_wheels: &base_osx_build_wheels
  <<: *base_build_wheels
  osx_image: xcode8

py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2
  <<: *py27_osx_test_config
  <<: *base_osx_build_wheels
  name: "Build wheels - OSX and cp27m (UCS2)"
  env:
    - *py27_osx_test_config_env
    - *base_build_wheels_env
    - CACHE_NAME=osxwheelsbuild.ucs2
    - PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs2
    # We ensure selection of the the pyenv interpreter by PY aware scripts and pants.pex with these
    # env vars.
    - PY=${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin/python
    - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==${PYENV_PY27_VERSION}']"
  script:
    - ./build-support/bin/check_pants_pex_abi.py cp27m
    - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -2n

py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4
  <<: *py27_osx_config
  <<: *base_osx_build_wheels
  <<: *native_engine_cache_config
  name: "Build wheels - OSX and cp27mu (UCS4)"
  env:
    - *py27_osx_config_env
    - *base_build_wheels_env
    - CACHE_NAME=osxwheelsbuild.ucs4
    - PYTHON_CONFIGURE_OPTS=--enable-unicode=ucs4
    # We ensure selection of the the pyenv interpreter by PY aware scripts and pants.pex with these
    # env vars.
    - PY=${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin/python
    - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==${PYENV_PY27_VERSION}']"
  script:
    - ./build-support/bin/ci.sh -2b
    - ./build-support/bin/check_pants_pex_abi.py cp27mu
    - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -2n

py36_osx_build_wheels: &py36_osx_build_wheels
  <<: *py36_osx_test_config
  <<: *base_osx_build_wheels
  name: "Build wheels - OSX and abi3 (Py3.6+)"
  env:
    - *py36_osx_test_config_env
    - *base_build_wheels_env
    - CACHE_NAME=osxwheelsbuild.abi3
    # We ensure selection of the the pyenv interpreter by PY aware scripts and pants.pex with these
    # env vars.
    - PY=${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin/python
    - PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="['CPython==${PYENV_PY36_VERSION}']"
  script:
    - ./build-support/bin/check_pants_pex_abi.py abi3 cp36m
    - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n

# -------------------------------------------------------------------------
# Rust tests
# -------------------------------------------------------------------------

base_rust_tests: &base_rust_tests
  <<: *native_engine_cache_config
  stage: *test
  before_script:
    - ulimit -c unlimited
    - ulimit -n 8192
  script:
    - ./build-support/bin/ci.sh -e

linux_rust_tests: &linux_rust_tests
  <<: *base_rust_tests
  <<: *linux_with_fuse
  name: "Rust tests - Linux (No PEX)"
  env:
    - CACHE_NAME=linuxrusttests

osx_rust_tests: &osx_rust_tests
  <<: *base_rust_tests
  name: "Rust tests - OSX (No PEX)"
  os: osx
  # Fuse actually works on this image. It hangs on many others.
  osx_image: xcode8.3
  addons:
    homebrew:
      casks:
      - openssl
      - osxfuse
  before_install:
    - ./build-support/bin/install_python_for_ci.sh "${PYENV_PY27_VERSION}" "${PYENV_PY36_VERSION}"
  env:
    - >
      {{>env_osx_with_pyenv}}
      PATH="${PYENV_ROOT}/versions/${PYENV_PY27_VERSION}/bin:${PATH}"
      PATH="${PYENV_ROOT}/versions/${PYENV_PY36_VERSION}/bin:${PATH}"
    - CACHE_NAME=macosrusttests

# -------------------------------------------------------------------------
# OSX sanity checks
# -------------------------------------------------------------------------

base_osx_sanity_check: &base_osx_sanity_check
  script:
    - MODE=debug ./build-support/bin/ci.sh -m

# TODO: Update this to use 10.14 once it is available
base_osx_10_12_sanity_check: &base_osx_10_12_sanity_check
  <<: *base_osx_sanity_check
  osx_image: xcode9.2

py27_osx_10_12_sanity_check: &py27_osx_10_12_sanity_check
  <<: *py27_osx_test_config
  <<: *base_osx_10_12_sanity_check
  name: "OSX 10.12 sanity check (Py2.7 PEX)"
  env:
    - *py27_osx_test_config_env
    - CACHE_NAME=macos10.12sanity.py27

py36_osx_10_12_sanity_check: &py36_osx_10_12_sanity_check
  <<: *py36_osx_test_config
  <<: *base_osx_10_12_sanity_check
  name: "OSX 10.12 sanity check (Py3.6 PEX)"
  env:
    - *py36_osx_test_config_env
    - CACHE_NAME=macos10.12sanity.py36

py37_osx_10_12_sanity_check: &py37_osx_10_12_sanity_check
  <<: *py37_osx_test_config
  <<: *base_osx_10_12_sanity_check
  name: "OSX 10.12 sanity check (Py3.7 PEX)"
  env:
    - *py37_osx_test_config_env
    - CACHE_NAME=macos10.12sanity.py37

base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check
  <<: *base_osx_sanity_check
  osx_image: xcode10.1

py27_osx_10_13_sanity_check: &py27_osx_10_13_sanity_check
  <<: *py27_osx_test_config
  <<: *base_osx_10_13_sanity_check
  name: "OSX 10.13 sanity check (Py2.7 PEX)"
  env:
    - *py27_osx_test_config_env
    - CACHE_NAME=macos10.13sanity.py27

py36_osx_10_13_sanity_check: &py36_osx_10_13_sanity_check
  <<: *py36_osx_test_config
  <<: *base_osx_10_13_sanity_check
  name: "OSX 10.13 sanity check (Py3.6 PEX)"
  env:
    - *py36_osx_test_config_env
    - CACHE_NAME=macos10.13sanity.py36

py37_osx_10_13_sanity_check: &py37_osx_10_13_sanity_check
  <<: *py37_osx_test_config
  <<: *base_osx_10_13_sanity_check
  name: "OSX 10.13 sanity check (Py3.7 PEX)"
  env:
    - *py37_osx_test_config_env
    - CACHE_NAME=macos10.13sanity.py37

# -------------------------------------------------------------------------
# Platform specific tests
# -------------------------------------------------------------------------

py27_osx_platform_tests: &py27_osx_platform_tests
  <<: *py27_osx_test_config
  name: "OSX platform-specific tests (Py2.7 PEX)"
  env:
    - *py27_osx_test_config_env
    - CACHE_NAME=macosplatformtests.py27
  script:
    - ./build-support/bin/ci.sh -z2

py36_osx_platform_tests: &py36_osx_platform_tests
  <<: *py36_osx_test_config
  name: "OSX platform-specific tests (Py3.6 PEX)"
  env:
    - *py36_osx_test_config_env
    - CACHE_NAME=macosplatformtests.py36
  script:
    - ./build-support/bin/ci.sh -z

py37_osx_platform_tests: &py37_osx_platform_tests
  <<: *py37_osx_test_config
  name: "OSX platform-specific tests (Py3.7 PEX)"
  env:
    - *py37_osx_test_config_env
    - CACHE_NAME=macosplatformtests.py37
  script:
    - ./build-support/bin/ci.sh -z7

# -------------------------------------------------------------------------
# JVM tests
# -------------------------------------------------------------------------

base_jvm_tests: &base_jvm_tests
  <<: *linux_with_fuse

py27_jvm_tests: &py27_jvm_tests
  <<: *py27_linux_test_config
  <<: *base_jvm_tests
  name: "JVM tests (Py2.7 PEX)"
  env:
    - *py27_linux_test_config_env
    - CACHE_NAME=linuxjvmtests.py27
  script:
    - ./build-support/bin/ci.sh -j2

py36_jvm_tests: &py36_jvm_tests
  <<: *py36_linux_test_config
  <<: *base_jvm_tests
  name: "JVM tests (Py3.6 PEX)"
  env:
    - *py36_linux_test_config_env
    - CACHE_NAME=linuxjvmtests.py36
  script:
    - ./build-support/bin/ci.sh -j

py37_jvm_tests: &py37_jvm_tests
  <<: *py37_linux_test_config
  <<: *base_jvm_tests
  name: "JVM tests (Py3.7 PEX)"
  env:
    - *py37_linux_test_config_env
    - CACHE_NAME=linuxjvmtests.py37
  script:
    - ./build-support/bin/ci.sh -j7

# -------------------------------------------------------------------------
# Deploy
# -------------------------------------------------------------------------

base_deploy: &base_deploy
  os: linux
  dist: trusty
  language: python
  python:
    - "2.7"
    - "3.6"
  before_install:
    # TODO(John Sirois): Get rid of this in favor of explicitly adding pyenv versions to the PATH:
    #   https://github.com/pantsbuild/pants/issues/7601
    - pyenv global 2.7.14 3.6.3
  env:
    - &base_deploy_env RUN_PANTS_FROM_PEX=1

base_deploy_stable_muliplatform_pex: &base_deploy_stable_muliplatform_pex
  <<: *base_deploy
  stage: *build_stable
  env:
    - &base_deploy_stable_env PANTS_PEX_RELEASE=stable
  script:
    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
  deploy:
    # See https://docs.travis-ci.com/user/deployment/releases/
    provider: releases
    # The pantsbuild-ci-bot OAuth token, see the pantsbuild vault for details.
    api_key:
      secure: "u0aCsiuVGOg28YxG0sQUovuUm29kKwQfFgHbNz2TT5L+cGoHxGl4aoVOCtuwWYEtbNGmYc8/3WRS3C/jOiqQj6JEgHUzWOsnfKUObEqNhisAmXbzBbKc0wPQTL8WNK+DKFh32sD3yPYcw+a5PTLO56+o7rqlI25LK7A17WesHC4="
    file_glob: true
    file: dist/deploy/pex/*
    skip_cleanup: true
    on:
      # We only release a pex for Pants releases, which are tagged.
      tags: true
      repo: pantsbuild/pants

py27_deploy_stable_multiplatform_pex: &py27_deploy_stable_multiplatform_pex
  <<: *base_deploy_stable_muliplatform_pex
  name: "Deploy stable multiplatform pants.pex (Py2.7 PEX)"
  env:
    - *base_deploy_env
    - *base_deploy_stable_env
    - RELEASE_ARGS="-2"
    - CACHE_NAME=linuxpexdeploystable.py27

py36_deploy_stable_multiplatform_pex: &py36_deploy_stable_multiplatform_pex
  <<: *base_deploy_stable_muliplatform_pex
  name: "Deploy stable multiplatform pants.pex (Py3.6 PEX)"
  env:
    - *base_deploy_env
    - *base_deploy_stable_env
    - RELEASE_ARGS=""
    - CACHE_NAME=linuxpexdeploystable.py36

base_deploy_unstable_multiplatform_pex: &base_deploy_unstable_multiplatform_pex
  <<: *base_deploy
  stage: *build_unstable
  env:
    - &base_deploy_unstable_env PREPARE_DEPLOY=1
  script:
    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
    - mkdir -p dist/deploy/pex/
    - mv dist/pants*.pex dist/deploy/pex/

py27_deploy_unstable_multiplatform_pex: &py27_deploy_unstable_multiplatform_pex
  <<: *base_deploy_unstable_multiplatform_pex
  name: "Deploy unstable multiplatform pants.pex (Py2.7 PEX)"
  env:
    - *base_deploy_env
    - *base_deploy_unstable_env
    - RELEASE_ARGS="-2"
    - CACHE_NAME=linuxpexdeployunstable.py27

py36_deploy_unstable_multiplatform_pex: &py36_deploy_unstable_multiplatform_pex
  <<: *base_deploy_unstable_multiplatform_pex
  name: "Deploy unstable multiplatform pants.pex (Py3.6 PEX)"
  env:
    - *base_deploy_env
    - *base_deploy_unstable_env
    - RELEASE_ARGS=""
    - CACHE_NAME=linuxpexdeployunstable.py36

# -------------------------------------------------------------------------
# Test matrix
# -------------------------------------------------------------------------

matrix:
  include:
    - <<: *py27_linux_build_engine
    - <<: *py27_linux_build_engine
      stage: *bootstrap_cron
    - <<: *py36_linux_build_engine
    - <<: *py36_linux_build_engine
      stage: *bootstrap_cron
    - <<: *py37_linux_build_engine

    - <<: *py27_osx_build_engine
    - <<: *py27_osx_build_engine
      stage: *bootstrap_cron
    - <<: *py36_osx_build_engine
    - <<: *py36_osx_build_engine
      stage: *bootstrap_cron
    - <<: *py37_osx_build_engine

    - <<: *py27_lint
    - <<: *py36_lint
    - <<: *py37_lint

    - <<: *linux_rust_clippy
    - <<: *cargo_audit

    - <<: *py27_linux_test_config
      name: "Unit tests (Py2.7 PEX)"
      stage: *test
      env:
        - *py27_linux_test_config_env
        - CACHE_NAME=linuxunittests.py27
      script:
        - ./build-support/bin/ci.sh -2lp

    - <<: *py36_linux_test_config
      name: "Unit tests (Py3.6 PEX)"
      env:
        - *py36_linux_test_config_env
        - CACHE_NAME=linuxunittests.py36
      script:
        - ./build-support/bin/ci.sh -lp

    - <<: *py37_linux_test_config
      name: "Unit tests (Py3.7 PEX)"
      env:
        - *py37_linux_test_config_env
        - CACHE_NAME=linuxunittests.py37
      script:
        - ./build-support/bin/ci.sh -7lp

    - <<: *py27_linux_build_wheels_ucs2
    - <<: *py27_linux_build_wheels_ucs4
    - <<: *py36_linux_build_wheels

    - <<: *py27_osx_build_wheels_ucs2
    - <<: *py27_osx_build_wheels_ucs4
    - <<: *py36_osx_build_wheels

{{#integration_shards}}
    - <<: *py36_linux_test_config
      name: "Integration tests - shard {{.}} (Py3.6 PEX)"
      env:
        - *py36_linux_test_config_env
        - CACHE_NAME=integrationshard{{.}}
      script:
        - ./build-support/bin/ci.sh -c -i {{.}}/{{integration_shards_length}}

{{/integration_shards}}
{{#integration_shards}}
    - <<: *py36_linux_test_config
      name: "Integration tests with Pantsd - shard {{.}} (Py3.6 PEX)"
      stage: *test_cron
      env:
        - *py36_linux_test_config_env
        - *run_tests_under_pantsd
        - CACHE_NAME=integrationshard{{.}}_pantsd
      script:
        - ./build-support/bin/ci.sh -c -i {{.}}/{{integration_shards_length}}

{{/integration_shards}}
{{#integration_shards}}
    - <<: *py37_linux_test_config
      name: "Integration tests - shard {{.}} (Py3.7 PEX)"
      env:
        - *py37_linux_test_config_env
        - CACHE_NAME=integrationshard{{.}}
      script:
        - ./build-support/bin/ci.sh -c7 -i {{.}}/{{integration_shards_length}}

{{/integration_shards}}
{{#integration_shards}}
    - <<: *py27_linux_test_config
      name: "Integration tests - shard {{.}} (Py2.7 PEX)"
      env:
        - *py27_linux_test_config_env
        - CACHE_NAME=cronshard{{.}}
      script:
        - ./build-support/bin/ci.sh -c2 -i {{.}}/{{integration_shards_length}}

{{/integration_shards}}

    - <<: *linux_rust_tests
    - <<: *osx_rust_tests

    - <<: *py27_linux_test_config
      name: "Python contrib tests (Py2.7 PEX)"
      stage: *test
      env:
        - *py27_linux_test_config_env
        - CACHE_NAME=linuxcontribtests.py27
      script:
        - ./build-support/bin/ci.sh -2n

    - <<: *py36_linux_test_config
      name: "Python contrib tests (Py3.6 PEX)"
      env:
        - *py36_linux_test_config_env
        - CACHE_NAME=linuxcontribtests.py36
      script:
        - ./build-support/bin/ci.sh -n

    - <<: *py36_linux_test_config
      name: "Python contrib tests with Pantsd (Py3.6 PEX)"
      stage: *test_cron
      env:
        - *py36_linux_test_config_env
        - *run_tests_under_pantsd
        - CACHE_NAME=linuxcontribtests.py36
      script:
        - ./build-support/bin/ci.sh -n

    - <<: *py37_linux_test_config
      name: "Python contrib tests (Py3.7 PEX)"
      env:
        - *py37_linux_test_config_env
        - CACHE_NAME=linuxcontribtests.py37
      script:
        - ./build-support/bin/ci.sh -7n

    - <<: *py27_osx_10_12_sanity_check
    - <<: *py36_osx_10_12_sanity_check
    - <<: *py37_osx_10_12_sanity_check

    - <<: *py27_osx_10_13_sanity_check
    - <<: *py36_osx_10_13_sanity_check
    - <<: *py37_osx_10_13_sanity_check

    - <<: *py27_osx_platform_tests
    - <<: *py36_osx_platform_tests
    - <<: *py37_osx_platform_tests

    - <<: *py27_jvm_tests
    - <<: *py36_jvm_tests
    - <<: *py37_jvm_tests

    - <<: *py27_deploy_stable_multiplatform_pex
    - <<: *py36_deploy_stable_multiplatform_pex

    - <<: *py27_deploy_unstable_multiplatform_pex
    - <<: *py36_deploy_unstable_multiplatform_pex

deploy:
  # Deploy whatever a previous stage has left in dist/deploy.
  # See: https://docs.travis-ci.com/user/deployment/s3/
  provider: s3
  access_key_id: AKIAIWOKBXVU3JLY6EGQ
  secret_access_key:
    secure: "UBVbpdYJ81OsDGKlPRBw6FlPJGlxosnFQ4A1xBbU5GwEBfv90GoKc6J0UwF+I4CDwytj/BlAks1XbW0zYX0oeIlXDnl1Vfikm1k4hfIr6VCLHKppiU69FlEs+ph0Dktz8+aUWhrvJzICZs6Gu08kTBQ5++3ulDWDeTHqjr713YM="
  bucket: binaries.pantsbuild.org
  local_dir: dist/deploy
  # Otherwise travis will stash dist/deploy and the deploy will fail.
  skip_cleanup: true
  acl: public_read
  on:
    condition: $PREPARE_DEPLOY = 1
    # NB: We mainly want deploys for `master` commits; but we also need new binaries for stable
    # release branches; eg `1.3.x`
    all_branches: true
    repo: pantsbuild/pants