Add ci scripts (#1)

* Add ci scripts

* Move config files to ci/kokoro/docker

* Remove dump-logs etc

* Simplified grpc installation

* Add Ubuntu 16.04 kokoro config

* Address code review
This commit is contained in:
Takashi Matsuo 2019-07-11 13:11:32 -07:00 committed by GitHub
parent 620940a411
commit aefb8c98b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 834 additions and 49 deletions

View file

@ -0,0 +1,51 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG DISTRO_VERSION=7
FROM centos:${DISTRO_VERSION}
# Add /usr/local/lib
# Search paths tweak for the build
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig
ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64
ENV PATH=/usr/local/bin:${PATH}
# First install the development tools and OpenSSL. The development tools
# distributed with CentOS (notably CMake) are too old to build
# `google-cloud-cpp`. In these instructions, we use `cmake3` obtained from
# [Software Collections](https://www.softwarecollections.org/).
RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
RUN yum install -y centos-release-scl
RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
RUN yum makecache && \
yum install -y automake cmake3 curl-devel gcc gcc-c++ git libtool \
make openssl-devel pkgconfig tar wget which zlib-devel
RUN ln -sf /usr/bin/cmake3 /usr/bin/cmake && ln -sf /usr/bin/ctest3 /usr/bin/ctest
# Install c-ares
RUN mkdir -p /var/tmp/Downloads; \
cd /var/tmp/Downloads; \
wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \
tar -xf cares-1_15_0.tar.gz; \
cd /var/tmp/Downloads/c-ares-cares-1_15_0; \
./buildconf && ./configure && make -j $(nproc); \
make install; \
ldconfig
# Install grpc from source
WORKDIR /var/tmp/ci
COPY install-grpc.sh /var/tmp/ci
RUN /var/tmp/ci/install-grpc.sh

View file

@ -0,0 +1,38 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG DISTRO_VERSION=30
FROM fedora:${DISTRO_VERSION}
# Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
# for `google-cloud-cpp`. Install these packages and additional development
# tools to compile the dependencies:
RUN dnf makecache && \
dnf install -y clang clang-tools-extra cmake doxygen findutils gcc-c++ git \
grpc-devel grpc-plugins libcxx-devel libcxxabi-devel libcurl-devel \
make openssl-devel pkgconfig protobuf-compiler python-pip ShellCheck \
tar wget zlib-devel
# Install the the buildifier tool, which does not compile with the default
# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
RUN chmod 755 /usr/bin/buildifier
# Install cmake_format to automatically format the CMake list files.
# https://github.com/cheshirekow/cmake_format
# Pin this to an specific version because the formatting changes when the
# "latest" version is updated, and we do not want the builds to break just
# because some third party changed something.
RUN pip install --upgrade pip
RUN pip install numpy cmake_format==0.5.2

View file

@ -0,0 +1,81 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG DISTRO_VERSION=18.04
FROM ubuntu:${DISTRO_VERSION}
RUN apt update && \
apt install -y \
build-essential \
clang \
cmake \
curl \
doxygen \
gawk \
git \
gcc \
golang \
g++ \
libc-ares-dev \
libc-ares2 \
libssl-dev \
make \
ninja-build \
pkg-config \
python-pip \
shellcheck \
tar \
unzip \
wget \
zlib1g-dev
# Install newer c-ares on Ubuntu 16.04.
RUN if grep -q 16.04 /etc/lsb-release; then \
apt remove libc-ares-dev libc-ares2; \
apt install -y automake libtool; \
mkdir -p /var/tmp/Downloads; \
cd /var/tmp/Downloads; \
wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \
tar -xf cares-1_15_0.tar.gz; \
cd /var/tmp/Downloads/c-ares-cares-1_15_0; \
./buildconf && ./configure && make -j $(nproc); \
make install; \
ldconfig; \
fi
# By default, Ubuntu 18.04 does not install the alternatives for clang-format
# and clang-tidy, so we need to manually install those.
RUN if grep -q 18.04 /etc/lsb-release; then \
apt update && apt install -y clang-tidy clang-format-7; \
update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-6.0 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-7 100; \
fi
# Install the the buildifier tool, which does not compile with the default
# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
RUN chmod 755 /usr/bin/buildifier
# Install cmake_format to automatically format the CMake list files.
# https://github.com/cheshirekow/cmake_format
# Pin this to an specific version because the formatting changes when the
# "latest" version is updated, and we do not want the builds to break just
# because some third party changed something.
RUN pip install --upgrade pip
RUN pip install numpy cmake_format==0.5.2
# Install grpc from source
WORKDIR /var/tmp/ci
COPY install-grpc.sh /var/tmp/ci
RUN /var/tmp/ci/install-grpc.sh

View file

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
# Create a Docker image with all the dependencies necessary to build the
# project.
if [[ -z "${PROJECT_ROOT+x}" ]]; then
readonly PROJECT_ROOT="$(cd "$(dirname "$0")/../.."; pwd)"
fi
source "${PROJECT_ROOT}/ci/kokoro/define-docker-variables.sh"
cd "${PROJECT_ROOT}"
sudo docker build -t "${IMAGE}:tip" \
--build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
-f "ci/kokoro/Dockerfile.${DISTRO}" ci

View file

@ -0,0 +1,24 @@
#!/usr/bin/env bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
if [[ -n "${IMAGE+x}" ]]; then
echo "IMAGE is already defined."
else
readonly IMAGE="apisci-${DISTRO}-${DISTRO_VERSION}"
readonly BUILD_OUTPUT="cmake-out/${IMAGE}-${BUILD_NAME}"
readonly BUILD_HOME="cmake-out/home/${IMAGE}-${BUILD_NAME}"
fi

View file

@ -0,0 +1,108 @@
#!/usr/bin/env bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
if [[ $# != 2 ]]; then
echo "Usage: $(basename "$0") <source-directory> <binary-directory>"
exit 1
fi
readonly SOURCE_DIR="$1"
readonly BINARY_DIR="$2"
# This script is supposed to run inside a Docker container, see
# ci/kokoro/cmake/installed-dependencies/build.sh for the expected setup. The
# /v directory is a volume pointing to a (clean-ish) checkout of the project:
if [[ -z "${PROJECT_ROOT+x}" ]]; then
readonly PROJECT_ROOT="/v"
fi
source "${PROJECT_ROOT}/ci/colors.sh"
echo
echo "${COLOR_YELLOW}Starting docker build $(date) with $(nproc) cores${COLOR_RESET}"
echo
echo "================================================================"
echo "Verify formatting $(date)"
(cd "${PROJECT_ROOT}" ; ./ci/check-style.sh)
echo "================================================================"
echo "================================================================"
echo "Compiling on $(date)"
echo "================================================================"
cd "${PROJECT_ROOT}"
cmake_flags=()
if [[ "${CLANG_TIDY:-}" = "yes" ]]; then
cmake_flags+=("-DGOOGLE_CLOUD_CPP_CLANG_TIDY=yes")
fi
if [[ "${GOOGLE_CLOUD_CPP_CXX_STANDARD:-}" != "" ]]; then
cmake_flags+=(
"-DGOOGLE_CLOUD_CPP_CXX_STANDARD=${GOOGLE_CLOUD_CPP_CXX_STANDARD}")
fi
if [[ "${CODE_COVERAGE:-}" == "yes" ]]; then
cmake_flags+=(
"-DCMAKE_BUILD_TYPE=Coverage")
fi
# Avoid unbound variable error with older bash
if [[ "${#cmake_flags[@]}" == 0 ]]; then
cmake "-H${SOURCE_DIR}" "-B${BINARY_DIR}"
else
cmake "-H${SOURCE_DIR}" "-B${BINARY_DIR}" "${cmake_flags[@]}"
fi
cmake --build "${BINARY_DIR}" -- -j "$(nproc)"
# When user a super-build the tests are hidden in a subdirectory. We can tell
# that ${BINARY_DIR} does not have the tests by checking for this file:
if [[ -r "${BINARY_DIR}/CTestTestfile.cmake" ]]; then
echo "================================================================"
# It is Okay to skip the tests in this case because the super build
# automatically runs them.
echo "Running the unit tests $(date)"
env -C "${BINARY_DIR}" ctest \
-LE integration-tests \
--output-on-failure -j "$(nproc)"
echo "================================================================"
fi
if [[ "${GENERATE_DOCS:-}" = "yes" ]]; then
echo "================================================================"
echo "Validate Doxygen documentation $(date)"
cmake --build "${BINARY_DIR}" --target doxygen-docs
echo "================================================================"
fi
if [[ ${RUN_INTEGRATION_TESTS} == "yes" ]]; then
echo "================================================================"
echo "Running the integration tests $(date)"
echo "================================================================"
# shellcheck disable=SC1091
source /c/spanner-integration-tests-config.sh
export GOOGLE_APPLICATION_CREDENTIALS=/c/spanner-credentials.json
# Run the integration tests too.
env -C "${BINARY_DIR}" ctest \
-L integration-tests \
--output-on-failure
echo "================================================================"
fi
echo "================================================================"
echo "Build finished at $(date)"
echo "================================================================"
exit 0

218
ci/kokoro/docker/build.sh Executable file
View file

@ -0,0 +1,218 @@
#!/usr/bin/env bash
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
export CC=gcc
export CXX=g++
export DISTRO=ubuntu
export DISTRO_VERSION=18.04
export CMAKE_SOURCE_DIR="."
in_docker_script="ci/kokoro/docker/build-in-docker-cmake.sh"
if [[ $# -eq 1 ]]; then
export BUILD_NAME="${1}"
elif [[ -n "${KOKORO_JOB_NAME:-}" ]]; then
# Kokoro injects the KOKORO_JOB_NAME environment variable, the value of this
# variable is cloud-cpp/spanner/<config-file-name-without-cfg> (or more
# generally <path/to/config-file-without-cfg>). By convention we name these
# files `$foo.cfg` for continuous builds and `$foo-presubmit.cfg` for
# presubmit builds. Here we extract the value of "foo" and use it as the build
# name.
BUILD_NAME="$(basename "${KOKORO_JOB_NAME}" "-presubmit")"
export BUILD_NAME
else
echo "Aborting build as the build name is not defined."
echo "If you are invoking this script via the command line use:"
echo " $0 <build-name>"
echo
echo "If this script is invoked by Kokoro, the CI system is expected to set"
echo "the KOKORO_JOB_NAME environment variable."
exit 1
fi
if [[ "${BUILD_NAME}" = "clang-tidy" ]]; then
# Compile with clang-tidy(1) turned on. The build treats clang-tidy warnings
# as errors.
export DISTRO=fedora-install
export DISTRO_VERSION=30
export CC=clang
export CXX=clang++
export CHECK_STYLE=yes
export CLANG_TIDY=yes
elif [[ "${BUILD_NAME}" = "ubuntu-18.04" ]]; then
export CC=gcc
export CXX=g++
elif [[ "${BUILD_NAME}" = "ubuntu-16.04" ]]; then
export DISTRO_VERSION=16.04
export CC=gcc
export CXX=g++
elif [[ "${BUILD_NAME}" = "gcc-4.8" ]]; then
# The oldest version of GCC we support is 4.8, this build checks the code
# against that version. The use of CentOS 7 for that build is not a
# coincidence: the reason we support GCC 4.8 is to support this distribution
# (and its commercial cousin: RHEL 7).
export CC=gcc
export CXX=g++
export DISTRO=centos
export DISTRO_VERSION=7
elif [[ "${BUILD_NAME}" = "clang-3.8" ]]; then
# The oldest version of Clang we actively test is 3.8. There is nothing
# particularly interesting about that version. It is simply the version
# included with Ubuntu:16.04, and the oldest version tested by
# google-cloud-cpp.
export DISTRO=ubuntu
export DISTRO_VERSION=16.04
export CC=clang
export CXX=clang++
else
echo "Unknown BUILD_NAME (${BUILD_NAME}). Fix the Kokoro .cfg file."
exit 1
fi
if [[ -z "${PROJECT_ROOT+x}" ]]; then
readonly PROJECT_ROOT="$(cd "$(dirname "$0")/../../.."; pwd)"
fi
source "${PROJECT_ROOT}/ci/kokoro/define-docker-variables.sh"
echo "================================================================"
cd "${PROJECT_ROOT}"
echo "Building with $(nproc) cores $(date) on ${PWD}."
echo "================================================================"
echo "Capture Docker version to troubleshoot $(date)."
sudo docker version
echo "================================================================"
echo "================================================================"
echo "Creating Docker image with all the development tools $(date)."
# We do not want to print the log unless there is an error, so disable the -e
# flag. Later, we will want to print out the emulator(s) logs *only* if there
# is an error, so disabling from this point on is the right choice.
set +e
mkdir -p "${BUILD_OUTPUT}"
readonly CREATE_DOCKER_IMAGE_LOG="${BUILD_OUTPUT}/create-build-docker-image.log"
echo "Logging to ${CREATE_DOCKER_IMAGE_LOG}"
if ! "${PROJECT_ROOT}/ci/retry-command.sh" \
"${PROJECT_ROOT}/ci/kokoro/create-docker-image.sh" \
>"${CREATE_DOCKER_IMAGE_LOG}" 2>&1 </dev/null; then
cat "${CREATE_DOCKER_IMAGE_LOG}"
exit 1
fi
echo "Docker image created $(date)."
sudo docker image ls
echo "================================================================"
echo "================================================================"
echo "Running the full build $(date)."
# The default user for a Docker container has uid 0 (root). To avoid creating
# root-owned files in the build directory we tell docker to use the current
# user ID, if known.
docker_uid="${UID:-0}"
docker_user="${USER:-root}"
docker_home_prefix="${PWD}/cmake-out/home"
if [[ "${docker_uid}" == "0" ]]; then
docker_home_prefix="${PWD}/cmake-out/root"
fi
# Make sure the user has a $HOME directory inside the Docker container.
mkdir -p "${BUILD_HOME}"
# We use an array for the flags so they are easier to document.
docker_flags=(
# Grant the PTRACE capability to the Docker container running the build,
# this is needed by tools like AddressSanitizer.
"--cap-add" "SYS_PTRACE"
# The name and version of the distribution, this is used to call
# define-docker-variables.sh and determine the Docker image built, and the
# output directory for any artifacts.
"--env" "DISTRO=${DISTRO}"
"--env" "DISTRO_VERSION=${DISTRO_VERSION}"
# The C++ and C compiler, both Bazel and CMake use this environment variable
# to select the compiler binary.
"--env" "CXX=${CXX}"
"--env" "CC=${CC}"
# If set to 'yes', the build script will run the style checks, including
# clang-format, cmake-format, and buildifier.
"--env" "CHECK_STYLE=${CHECK_STYLE:-}"
# If set to 'yes', the build script will configure clang-tidy. Currently
# only the CMake builds use this flag.
"--env" "CLANG_TIDY=${CLANG_TIDY:-}"
# If set to 'yes', run the integration tests. Currently only the Bazel
# builds use this flag.
"--env" "RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-}"
# If set to 'yes', run compile with code coverage flags. Currently only the
# CMake builds use this flag.
"--env" "CODE_COVERAGE=${CODE_COVERAGE:-}"
# If set, pass -DGOOGLE_CLOUD_CPP_CXX_STANDARD=<value> to CMake.
"--env" "GOOGLE_CLOUD_CPP_CXX_STANDARD=${GOOGLE_CLOUD_CPP_CXX_STANDARD:-}"
# When running the integration tests this directory contains the
# configuration files needed to run said tests. Make it available inside
# the Docker container.
"--volume" "${KOKORO_GFILE_DIR:-/dev/shm}:/c"
# Let the Docker image script know what kind of terminal we are using, that
# produces properly colorized error messages.
"--env" "TERM=${TERM:-dumb}"
# Run the docker script and this user id. Because the docker image gets to
# write in ${PWD} you typically want this to be your user id.
"--user" "${docker_uid}"
# Bazel needs this environment variable to work correctly.
"--env" "USER=${docker_user}"
# We give Bazel and CMake a fake $HOME inside the docker image. Bazel caches
# build byproducts in this directory. CMake (when ccache is enabled) uses
# it to store $HOME/.ccache
"--env" "HOME=/h"
"--volume" "${PWD}/${BUILD_HOME}:/h"
# Mount the current directory (which is the top-level directory for the
# project) as `/v` inside the docker image, and move to that directory.
"--volume" "${PWD}:/v"
"--workdir" "/v"
# Mask any other builds that may exist at the same time. That is, these
# directories appear as empty inside the Docker container, this prevents the
# container from writing into other builds, or to get confused by the output
# of other builds. In the CI system this does not matter, as each build runs
# on a completely separate VM. This is useful when running multiple builds
# in your workstation.
"--volume" "/v/cmake-out/home"
"--volume" "/v/cmake-out"
"--volume" "${PWD}/${BUILD_OUTPUT}:/v/${BUILD_OUTPUT}"
)
# When running the builds from the command-line they get a tty, and the scripts
# running inside the Docker container can produce nicer output. On Kokoro the
# script does not get a tty, and Docker terminates the program if we pass the
# `-it` flag.
if [[ -t 0 ]]; then
docker_flags+=("-it")
fi
sudo docker run "${docker_flags[@]}" "${IMAGE}:tip" \
"/v/${in_docker_script}" "${CMAKE_SOURCE_DIR}" \
"${BUILD_OUTPUT}"

View file

View file

View file

View file

@ -0,0 +1,17 @@
# Format: //devtools/kokoro/config/proto/build.proto
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
build_file: "cpp-cmakefiles/ci/kokoro/docker/build.sh"
timeout_mins: 120

View file

View file

View file

View file