Add option for environment variables to CircleCI config generation script

patch by Andrés de la Peña; reviewed by Berenguer Blasi, Ekaterina Dimitrova and Ruslan Fomkin for CASSANDRA-16989

.circleci/config-2_1.yml

@@ -19,6 +19,14 @@
 version: 2.1
 
 default_env_vars: &default_env_vars
+
+    # The values of some of these environment variables are meant to be frequently changed by developers.
+    # The generate.sh script contains a list of accepted environment variables that should contain some of
+    # these variables. Also, some variables are mentioned in the documentation, at least in
+    # .circleci/readme.md and in doc/source/development/testing.rst.
+    # If you modify these variables, or if you add new variables whose values are meant to be changed frequently,
+    # please remember to modify the generate.sh script and the documentation accordingly.
+
     JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
     ANT_HOME: /usr/share/ant
     LANG: en_US.UTF-8

.circleci/generate.sh

@@ -22,41 +22,109 @@ BASEDIR=`dirname $0`
 die ()
 {
   echo "ERROR: $*"
-  echo "Usage: $0 [-l|-m|-h]"
-  echo "   -l Generate config.yml using low resources"
-  echo "   -m Generate config.yml using mid resources"
-  echo "   -h Generate config.yml using high resources"
-  echo "   No flags generates the default config.yml using low resources and the three"
-  echo "   templates (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES)"
+  print_help
   exit 1
 }
 
+print_help()
+{
+  echo "Usage: $0 [-l|-m|-h|-f|-e]"
+  echo "   -a Generate the default config.yml using low resources and the three templates"
+  echo "      (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES). Use this for"
+  echo "      permanent changes in config-2_1.yml that will be committed to the main repo."
+  echo "   -l Generate config.yml using low resources"
+  echo "   -m Generate config.yml using mid resources"
+  echo "   -h Generate config.yml using high resources"
+  echo "   -e <key=value> Environment variables to be used in the generated config.yml, e.g.:"
+  echo "                   -e DTEST_BRANCH=CASSANDRA-8272"
+  echo "                   -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git"
+  echo "                   -e REPEATED_UTEST_TARGET=testsome"
+  echo "                   -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest"
+  echo "                   -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable"
+  echo "                   -e REPEATED_UTEST_COUNT=100"
+  echo "                   -e REPEATED_UTEST_STOP_ON_FAILURE=false"
+  echo "                   -e REPEATED_DTEST_NAME=cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
+  echo "                   -e REPEATED_DTEST_VNODES=false"
+  echo "                   -e REPEATED_DTEST_COUNT=100"
+  echo "                   -e REPEATED_DTEST_STOP_ON_FAILURE=false"
+  echo "                  For the complete list of environment variables, please check the"
+  echo "                  list of examples in config-2_1.yml and/or the documentation."
+  echo "                  If you want to specify multiple environment variables simply add"
+  echo "                  multiple -e options. The flags -l/-m/-h should be used when using -e."
+  echo "   -f Stop checking that the environment variables are known"
+}
+
+all=false
 lowres=false
 midres=false
 highres=false
-while getopts ":lmh" opt; do
+env_vars=""
+has_env_vars=false
+check_env_vars=true
+while getopts "e:almhf" opt; do
   case $opt in
-      l ) ($midres || $highres) && die "Cannot specify option -l after specifying options -m or -h"
-          lowres=true
+      a ) all=true
           ;;
-      m ) ($lowres || $highres) && die "Cannot specify option -m after specifying options -l or -h"
-          midres=true
+      l ) lowres=true
           ;;
-      h ) ($lowres || $midres) && die "Cannot specify option -h after specifying options -l or -m"
-          highres=true
+      m ) midres=true
+          ;;
+      h ) highres=true
+          ;;
+      e ) if (!($has_env_vars)); then
+            env_vars="$OPTARG"
+          else
+            env_vars="$env_vars|$OPTARG"
+          fi
+          has_env_vars=true
+          ;;
+      f ) check_env_vars=false
           ;;
       \?) die "Invalid option: -$OPTARG"
           ;;
   esac
 done
+shift $((OPTIND-1))
+if [ "$#" -ne 0 ]; then
+    die "Unexpected arguments"
+fi
+
+# validate environment variables
+if $has_env_vars && $check_env_vars; then
+  for entry in $(echo $env_vars | tr "|" "\n"); do
+    key=$(echo $entry | tr "=" "\n" | head -n 1)
+    if [ "$key" != "DTEST_REPO" ] &&
+       [ "$key" != "DTEST_BRANCH" ] &&
+       [ "$key" != "REPEATED_UTEST_TARGET" ] &&
+       [ "$key" != "REPEATED_UTEST_CLASS" ] &&
+       [ "$key" != "REPEATED_UTEST_METHODS" ] &&
+       [ "$key" != "REPEATED_UTEST_COUNT" ] &&
+       [ "$key" != "REPEATED_UTEST_STOP_ON_FAILURE" ] &&
+       [ "$key" != "REPEATED_DTEST_NAME" ] &&
+       [ "$key" != "REPEATED_DTEST_VNODES" ] &&
+       [ "$key" != "REPEATED_DTEST_COUNT" ] &&
+       [ "$key" != "REPEATED_DTEST_STOP_ON_FAILURE" ] &&
+       [ "$key" != "REPEATED_UPGRADE_DTEST_NAME" ] &&
+       [ "$key" != "REPEATED_UPGRADE_DTEST_COUNT" ] &&
+       [ "$key" != "REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE" ] &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_CLASS" ] &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_METHODS" ] &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_COUNT" ] &&
+       [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE" ]; then
+      die "Unrecognised environment variable name: $key"
+    fi
+  done
+fi
 
 if $lowres; then
+  ($all || $midres || $highres) && die "Cannot use option -l with options -a, -m or -h"
   echo "Generating new config.yml file with low resources from config-2_1.yml"
   circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp
   cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml
   rm $BASEDIR/config.yml.LOWRES.tmp
 
 elif $midres; then
+  ($all || $lowres || $highres) && die "Cannot use option -m with options -a, -l or -h"
   echo "Generating new config.yml file with middle resources from config-2_1.yml"
   patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch
   circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp
@@ -64,13 +132,15 @@ elif $midres; then
   rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
 
 elif $highres; then
+  ($all || $lowres || $midres) && die "Cannot use option -h with options -a, -l or -m"
   echo "Generating new config.yml file with high resources from config-2_1.yml"
   patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch
   circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp
   cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml
   rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
 
-else
+elif $all; then
+  ($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option -a with options -l, -m, -h or -e"
   echo "Generating new config.yml file with low resources and LOWRES/MIDRES/HIGHRES templates from config-2_1.yml"
 
   # setup lowres
@@ -92,7 +162,22 @@ else
 
   # copy lower into config.yml to make sure this gets updated
   cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml
+
+elif (!($has_env_vars)); then
+  print_help
+fi
+
+# replace environment variables
+if $has_env_vars; then
+  IFS='='
+  echo "$env_vars" | tr '|' '\n' | while read entry; do
+    set -- $entry
+    key=$1
+    val=$2
+    echo "Setting environment variable $key: $val"
+    sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml
+  done
+  unset IFS
 fi
 
 
-

.circleci/readme.md

@@ -42,6 +42,36 @@ This script validates and applies any changes to the `config-2_1.yml` file, and
 requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be
 installed.
 
+## Setting environment variables
+Both `config-2_1.yml` and `config.yml` files contain a set of environment variables 
+defining things like what dtest repo and branch to use, what tests could be repeatedly 
+run, etc.
+
+These environment variables can be directly edited in the `config.yml` file, although if 
+you do this you should take into account that the entire set of env vars is repeated on 
+every job. 
+
+A probably better approach is editing them in `config-2_1.yml` and then regenerate the 
+`config.yml` file using the `generate.sh` script. You can also directly pass environment
+variable values to the `generate.sh` script with the `-e` flag. For example, to set the 
+dtest repo and branch with MIDRES config you can run:
+
+```
+generate.sh -m \
+  -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git \
+  -e DTEST_BRANCH=CASSANDRA-8272 
+```
+
+Or you can set the test multiplexer for repeating a specific test with HIGHRES:
+
+```
+generate.sh -h \
+  -e REPEATED_UTEST_TARGET=testsome \
+  -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest \
+  -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable \
+  -e REPEATED_UTEST_COUNT=100
+```
+
 ## Updating the config
 For configuration changes meant to be permanent in the Apache repo you should never edit
 the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then