From b6678a002cfa00be2068d499c5b7ce9c515bbc4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20de=20la=20Pe=C3=B1a?= Date: Tue, 28 Sep 2021 17:45:56 +0100 Subject: [PATCH] Add option for environment variables to CircleCI config generation script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Andrés de la Peña; reviewed by Berenguer Blasi, Ekaterina Dimitrova and Ruslan Fomkin for CASSANDRA-16989 --- .circleci/config-2_1.yml | 8 +++ .circleci/generate.sh | 115 ++++++++++++++++++++++++++++++++++----- .circleci/readme.md | 30 ++++++++++ 3 files changed, 138 insertions(+), 15 deletions(-) diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml index 9595495e46..c5e2583a10 100644 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@ -19,6 +19,14 @@ version: 2.1 default_env_vars: &default_env_vars + + # The values of some of these environment variables are meant to be frequently changed by developers. + # The generate.sh script contains a list of accepted environment variables that should contain some of + # these variables. Also, some variables are mentioned in the documentation, at least in + # .circleci/readme.md and in doc/source/development/testing.rst. + # If you modify these variables, or if you add new variables whose values are meant to be changed frequently, + # please remember to modify the generate.sh script and the documentation accordingly. + JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 ANT_HOME: /usr/share/ant LANG: en_US.UTF-8 diff --git a/.circleci/generate.sh b/.circleci/generate.sh index db7d772b24..f9d5ddfb89 100755 --- a/.circleci/generate.sh +++ b/.circleci/generate.sh @@ -22,41 +22,109 @@ BASEDIR=`dirname $0` die () { echo "ERROR: $*" - echo "Usage: $0 [-l|-m|-h]" - echo " -l Generate config.yml using low resources" - echo " -m Generate config.yml using mid resources" - echo " -h Generate config.yml using high resources" - echo " No flags generates the default config.yml using low resources and the three" - echo " templates (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES)" + print_help exit 1 } +print_help() +{ + echo "Usage: $0 [-l|-m|-h|-f|-e]" + echo " -a Generate the default config.yml using low resources and the three templates" + echo " (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES). Use this for" + echo " permanent changes in config-2_1.yml that will be committed to the main repo." + echo " -l Generate config.yml using low resources" + echo " -m Generate config.yml using mid resources" + echo " -h Generate config.yml using high resources" + echo " -e Environment variables to be used in the generated config.yml, e.g.:" + echo " -e DTEST_BRANCH=CASSANDRA-8272" + echo " -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git" + echo " -e REPEATED_UTEST_TARGET=testsome" + echo " -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest" + echo " -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable" + echo " -e REPEATED_UTEST_COUNT=100" + echo " -e REPEATED_UTEST_STOP_ON_FAILURE=false" + echo " -e REPEATED_DTEST_NAME=cqlsh_tests/test_cqlsh.py::TestCqlshSmoke" + echo " -e REPEATED_DTEST_VNODES=false" + echo " -e REPEATED_DTEST_COUNT=100" + echo " -e REPEATED_DTEST_STOP_ON_FAILURE=false" + echo " For the complete list of environment variables, please check the" + echo " list of examples in config-2_1.yml and/or the documentation." + echo " If you want to specify multiple environment variables simply add" + echo " multiple -e options. The flags -l/-m/-h should be used when using -e." + echo " -f Stop checking that the environment variables are known" +} + +all=false lowres=false midres=false highres=false -while getopts ":lmh" opt; do +env_vars="" +has_env_vars=false +check_env_vars=true +while getopts "e:almhf" opt; do case $opt in - l ) ($midres || $highres) && die "Cannot specify option -l after specifying options -m or -h" - lowres=true + a ) all=true ;; - m ) ($lowres || $highres) && die "Cannot specify option -m after specifying options -l or -h" - midres=true + l ) lowres=true ;; - h ) ($lowres || $midres) && die "Cannot specify option -h after specifying options -l or -m" - highres=true + m ) midres=true + ;; + h ) highres=true + ;; + e ) if (!($has_env_vars)); then + env_vars="$OPTARG" + else + env_vars="$env_vars|$OPTARG" + fi + has_env_vars=true + ;; + f ) check_env_vars=false ;; \?) die "Invalid option: -$OPTARG" ;; esac done +shift $((OPTIND-1)) +if [ "$#" -ne 0 ]; then + die "Unexpected arguments" +fi + +# validate environment variables +if $has_env_vars && $check_env_vars; then + for entry in $(echo $env_vars | tr "|" "\n"); do + key=$(echo $entry | tr "=" "\n" | head -n 1) + if [ "$key" != "DTEST_REPO" ] && + [ "$key" != "DTEST_BRANCH" ] && + [ "$key" != "REPEATED_UTEST_TARGET" ] && + [ "$key" != "REPEATED_UTEST_CLASS" ] && + [ "$key" != "REPEATED_UTEST_METHODS" ] && + [ "$key" != "REPEATED_UTEST_COUNT" ] && + [ "$key" != "REPEATED_UTEST_STOP_ON_FAILURE" ] && + [ "$key" != "REPEATED_DTEST_NAME" ] && + [ "$key" != "REPEATED_DTEST_VNODES" ] && + [ "$key" != "REPEATED_DTEST_COUNT" ] && + [ "$key" != "REPEATED_DTEST_STOP_ON_FAILURE" ] && + [ "$key" != "REPEATED_UPGRADE_DTEST_NAME" ] && + [ "$key" != "REPEATED_UPGRADE_DTEST_COUNT" ] && + [ "$key" != "REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_CLASS" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_METHODS" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_COUNT" ] && + [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE" ]; then + die "Unrecognised environment variable name: $key" + fi + done +fi if $lowres; then + ($all || $midres || $highres) && die "Cannot use option -l with options -a, -m or -h" echo "Generating new config.yml file with low resources from config-2_1.yml" circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml rm $BASEDIR/config.yml.LOWRES.tmp elif $midres; then + ($all || $lowres || $highres) && die "Cannot use option -m with options -a, -l or -h" echo "Generating new config.yml file with middle resources from config-2_1.yml" patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp @@ -64,13 +132,15 @@ elif $midres; then rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp elif $highres; then + ($all || $lowres || $midres) && die "Cannot use option -h with options -a, -l or -m" echo "Generating new config.yml file with high resources from config-2_1.yml" patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp -else +elif $all; then + ($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option -a with options -l, -m, -h or -e" echo "Generating new config.yml file with low resources and LOWRES/MIDRES/HIGHRES templates from config-2_1.yml" # setup lowres @@ -92,7 +162,22 @@ else # copy lower into config.yml to make sure this gets updated cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml + +elif (!($has_env_vars)); then + print_help +fi + +# replace environment variables +if $has_env_vars; then + IFS='=' + echo "$env_vars" | tr '|' '\n' | while read entry; do + set -- $entry + key=$1 + val=$2 + echo "Setting environment variable $key: $val" + sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml + done + unset IFS fi - diff --git a/.circleci/readme.md b/.circleci/readme.md index e4e32f5345..044adfc0e6 100644 --- a/.circleci/readme.md +++ b/.circleci/readme.md @@ -42,6 +42,36 @@ This script validates and applies any changes to the `config-2_1.yml` file, and requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be installed. +## Setting environment variables +Both `config-2_1.yml` and `config.yml` files contain a set of environment variables +defining things like what dtest repo and branch to use, what tests could be repeatedly +run, etc. + +These environment variables can be directly edited in the `config.yml` file, although if +you do this you should take into account that the entire set of env vars is repeated on +every job. + +A probably better approach is editing them in `config-2_1.yml` and then regenerate the +`config.yml` file using the `generate.sh` script. You can also directly pass environment +variable values to the `generate.sh` script with the `-e` flag. For example, to set the +dtest repo and branch with MIDRES config you can run: + +``` +generate.sh -m \ + -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git \ + -e DTEST_BRANCH=CASSANDRA-8272 +``` + +Or you can set the test multiplexer for repeating a specific test with HIGHRES: + +``` +generate.sh -h \ + -e REPEATED_UTEST_TARGET=testsome \ + -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest \ + -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable \ + -e REPEATED_UTEST_COUNT=100 +``` + ## Updating the config For configuration changes meant to be permanent in the Apache repo you should never edit the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then