natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CircleCI config J17 + J11 

patch by Ekaterina Dimitrova; reviewed by Michael Semb Wever and Brandon Williams for CASSANDRA-18247
This commit is contained in:
Ekaterina Dimitrova 2023-02-15 18:21:32 -05:00
parent 3eb605b4db
commit 0dea9c268a
7 changed files with 32852 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9561
.circleci/config_11_and_17.yml Normal file
View File

File diff suppressed because it is too large Load Diff

9561
.circleci/config_11_and_17.yml.FREE Normal file
View File

File diff suppressed because it is too large Load Diff

9561
.circleci/config_11_and_17.yml.PAID Normal file
View File

File diff suppressed because it is too large Load Diff

3451
.circleci/config_template_11_and_17.yml Normal file
View File

File diff suppressed because it is too large Load Diff

374
.circleci/config_template_11_and_17.yml.PAID.patch Normal file
View File

@ -0,0 +1,374 @@
--- config_template_11_and_17.yml 2023-03-15 21:34:57.000000000 -0400
+++ config_template_11_and_17.yml.PAID 2023-03-15 21:37:25.000000000 -0400
@@ -140,14 +140,14 @@
j11_par_executor: &j11_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
j11_small_par_executor: &j11_small_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: large
+ parallelism: 10
j11_small_executor: &j11_small_executor
executor:
@@ -155,30 +155,41 @@
exec_resource_class: medium
parallelism: 1
+j11_large_par_executor: &j11_large_par_executor
+ executor:
+ name: java11-executor
+ exec_resource_class: large
+ parallelism: 50
+
+j11_very_large_par_executor: &j11_very_large_par_executor
+ executor:
+ name: java11-executor
+ exec_resource_class: large
+ parallelism: 100
+
j11_medium_par_executor: &j11_medium_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 4
j11_seq_executor: &j11_seq_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
exec_resource_class: medium
parallelism: 1 # sequential, single container tests: no parallelism benefits
j17_par_executor: &j17_par_executor
executor:
name: java17-executor
- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
j17_small_par_executor: &j17_small_par_executor
executor:
name: java17-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: large
+ parallelism: 10
j17_small_executor: &j17_small_executor
executor:
@@ -189,34 +200,44 @@
j17_medium_par_executor: &j17_medium_par_executor
executor:
name: java17-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 4
+
+j17_large_par_executor: &j17_large_par_executor
+ executor:
+ name: java17-executor
+ exec_resource_class: large
+ parallelism: 50
j17_seq_executor: &j17_seq_executor
executor:
name: java17-executor
- #exec_resource_class: xlarge
+ exec_resource_class: medium
parallelism: 1 # sequential, single container tests: no parallelism benefits
j11_repeated_utest_executor: &j11_repeated_utest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
j11_repeated_dtest_executor: &j11_repeated_dtest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: large
+ parallelism: 25
j17_repeated_utest_executor: &j17_repeated_utest_executor
executor:
name: java17-executor
- parallelism: 4
+ exec_resource_class: medium
+ parallelism: 25
j17_repeated_dtest_executor: &j17_repeated_dtest_executor
executor:
name: java17-executor
- parallelism: 4
+ exec_resource_class: large
+ parallelism: 25
j11_separate_jobs: &j11_separate_jobs
jobs:
@@ -1808,7 +1829,7 @@
target: testclasslist-system-keyspace-directory
j11_dtests_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1822,7 +1843,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j11_dtests_offheap:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1836,7 +1857,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j17_dtests_vnode:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1851,7 +1872,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --skip-resource-intensive-tests'
j17_dtests_offheap:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1866,7 +1887,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --use-off-heap-memtables --skip-resource-intensive-tests'
j11_dtests:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1880,7 +1901,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j17_dtests:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1895,7 +1916,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j11_cqlsh_dtests_py3_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1910,7 +1931,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py3_offheap:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1925,7 +1946,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1943,7 +1964,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_vnode:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1961,7 +1982,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py38_offheap:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1979,7 +2000,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311_offheap:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1997,7 +2018,7 @@
python_version: '3.11'
j11_cqlsh_dtests_py3:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2012,7 +2033,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j11_cqlsh_dtests_py38:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2030,7 +2051,7 @@
python_version: '3.8'
j11_cqlsh_dtests_py311:
- <<: *j11_par_executor
+ <<: *j11_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2048,7 +2069,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py3_vnode:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2063,7 +2084,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py3_offheap:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2078,7 +2099,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py38_vnode:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2096,7 +2117,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311_vnode:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2114,7 +2135,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py38_offheap:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2132,7 +2153,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311_offheap:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2150,7 +2171,7 @@
python_version: '3.11'
j17_cqlsh_dtests_py3:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2165,7 +2186,7 @@
extra_env_args: 'CQLSH_PYTHON=/usr/bin/python3.6'
j17_cqlsh_dtests_py38:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2183,7 +2204,7 @@
python_version: '3.8'
j17_cqlsh_dtests_py311:
- <<: *j17_par_executor
+ <<: *j17_large_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2201,7 +2222,7 @@
python_version: '3.11'
j11_dtests_large_vnode:
- <<: *j11_par_executor
+ <<: *j11_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2215,7 +2236,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large:
- <<: *j11_par_executor
+ <<: *j11_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2229,7 +2250,7 @@
pytest_extra_args: '--only-resource-intensive-tests --force-resource-intensive-tests'
j17_dtests_large_vnode:
- <<: *j17_par_executor
+ <<: *j17_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -2243,7 +2264,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j17_dtests_large:
- <<: *j17_par_executor
+ <<: *j17_medium_par_executor
steps:
- attach_workspace:
at: /home/cassandra

306
.circleci/generate_11_and_17.sh Executable file
View File

@ -0,0 +1,306 @@
#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
BASEDIR=`dirname $0`
BASE_BRANCH=trunk
set -e
die ()
{
echo "ERROR: $*"
print_help
exit 1
}
print_help()
{
echo "Usage: $0 [-f|-p|-a|-e|-i]"
echo " -a Generate the config_11_and_17.yml, config_11_and_17.yml.FREE and config_11_and_17.yml.PAID expanded configuration"
echo " files from the main config_template.yml reusable configuration file."
echo " Use this for permanent changes in config_11_and_17.yml that will be committed to the main repo."
echo " -f Generate config.yml for tests compatible with the CircleCI free tier resources"
echo " -p Generate config.yml for tests compatible with the CircleCI paid tier resources"
echo " -e <key=value> Environment variables to be used in the generated config_11_and_17.yml, e.g.:"
echo " -e DTEST_BRANCH=CASSANDRA-8272"
echo " -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git"
echo " -e REPEATED_TESTS_STOP_ON_FAILURE=false"
echo " -e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest#testCountersTable"
echo " -e REPEATED_UTESTS_COUNT=500"
echo " -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest"
echo " -e REPEATED_UTESTS_FQLTOOL_COUNT=500"
echo " -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest"
echo " -e REPEATED_UTESTS_LONG_COUNT=100"
echo " -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest"
echo " -e REPEATED_UTESTS_STRESS_COUNT=500"
echo " -e REPEATED_SIMULATOR_DTESTS=org.apache.cassandra.simulator.test.TrivialSimulationTest"
echo " -e REPEATED_SIMULATOR_DTESTS_COUNT=500"
echo " -e REPEATED_JVM_DTESTS=org.apache.cassandra.distributed.test.PagingTest"
echo " -e REPEATED_JVM_DTESTS_COUNT=500"
echo " -e REPEATED_DTESTS=cdc_test.py cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
echo " -e REPEATED_DTESTS_COUNT=500"
echo " -e REPEATED_LARGE_DTESTS=replace_address_test.py::TestReplaceAddress::test_replace_stopped_node"
echo " -e REPEATED_LARGE_DTESTS=100"
echo " -e REPEATED_ANT_TEST_TARGET=testsome"
echo " -e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest"
echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
echo " -e REPEATED_ANT_TEST_VNODES=false"
echo " -e REPEATED_ANT_TEST_COUNT=500"
echo " For the complete list of environment variables, please check the"
echo " list of examples in config_template_11_and_17.yml and/or the documentation."
echo " If you want to specify multiple environment variables simply add"
echo " multiple -e options. The flags -f/-p should be used when using -e."
echo " -i Ignore unknown environment variables"
}
all=false
free=false
paid=false
env_vars=""
has_env_vars=false
check_env_vars=true
while getopts "e:afpi" opt; do
case $opt in
a ) all=true
;;
f ) free=true
;;
p ) paid=true
;;
e ) if (! ($has_env_vars)); then
env_vars="$OPTARG"
else
env_vars="$env_vars|$OPTARG"
fi
has_env_vars=true
;;
i ) check_env_vars=false
;;
\?) die "Invalid option: -$OPTARG"
;;
esac
done
shift $((OPTIND-1))
if [ "$#" -ne 0 ]; then
die "Unexpected arguments"
fi
# validate environment variables
if $has_env_vars && $check_env_vars; then
for entry in $(echo $env_vars | tr "|" "\n"); do
key=$(echo $entry | tr "=" "\n" | head -n 1)
if [ "$key" != "DTEST_REPO" ] &&
[ "$key" != "DTEST_BRANCH" ] &&
[ "$key" != "REPEATED_TESTS_STOP_ON_FAILURE" ] &&
[ "$key" != "REPEATED_UTESTS" ] &&
[ "$key" != "REPEATED_UTESTS_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_FQLTOOL" ] &&
[ "$key" != "REPEATED_UTESTS_FQLTOOL_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_LONG" ] &&
[ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_STRESS" ] &&
[ "$key" != "REPEATED_UTESTS_STRESS_COUNT" ] &&
[ "$key" != "REPEATED_SIMULATOR_DTESTS" ] &&
[ "$key" != "REPEATED_SIMULATOR_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_JVM_DTESTS" ] &&
[ "$key" != "REPEATED_JVM_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_DTESTS" ] &&
[ "$key" != "REPEATED_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS" ] &&
[ "$key" != "REPEATED_LARGE_DTESTS_COUNT" ] &&
[ "$key" != "REPEATED_ANT_TEST_TARGET" ] &&
[ "$key" != "REPEATED_ANT_TEST_CLASS" ] &&
[ "$key" != "REPEATED_ANT_TEST_METHODS" ] &&
[ "$key" != "REPEATED_ANT_TEST_VNODES" ] &&
[ "$key" != "REPEATED_ANT_TEST_COUNT" ]; then
die "Unrecognised environment variable name: $key"
fi
done
fi
if $free; then
($all || $paid) && die "Cannot use option -f with options -a or -p"
echo "Generating new config.yml file for free tier from config_template_11_and_17.yml"
circleci config process $BASEDIR/config_template_11_and_17.yml > $BASEDIR/config_11_and_17.yml.FREE.tmp
cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.FREE.tmp > $BASEDIR/config.yml
cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
rm $BASEDIR/config_11_and_17.yml.FREE.tmp
elif $paid; then
($all || $free) && die "Cannot use option -p with options -a or -f"
echo "Generating new config.yml file for paid tier from config_template_11_and_17.yml"
patch -o $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_template_11_and_17.yml $BASEDIR/config_template_11_and_17.yml.PAID.patch
circleci config process $BASEDIR/config_template_11_and_17.yml.PAID > $BASEDIR/config_11_and_17.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.PAID.tmp > $BASEDIR/config.yml
cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
rm $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_11_and_17.yml.PAID.tmp
elif $all; then
($free || $paid || $has_env_vars) && die "Cannot use option -a with options -f, -p or -e"
echo "Generating new default config_11_and_17.yml file for free tier and FREE/PAID templates from config_template_11_and_17.yml."
echo "Make sure you commit the newly generated config_11_and_17.yml, config_11_and_17.yml.FREE and config_11_and_17.yml.PAID files"
echo "after running this command if you want them to persist."
# setup config for free tier
circleci config process $BASEDIR/config_template_11_and_17.yml > $BASEDIR/config_11_and_17.yml.FREE.tmp
cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.FREE.tmp > $BASEDIR/config_11_and_17.yml.FREE
rm $BASEDIR/config_11_and_17.yml.FREE.tmp
# setup config for paid tier
patch -o $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_template_11_and_17.yml $BASEDIR/config_template_11_and_17.yml.PAID.patch
circleci config process $BASEDIR/config_template_11_and_17.yml.PAID > $BASEDIR/config_11_and_17.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config_11_and_17.yml.PAID.tmp > $BASEDIR/config_11_and_17.yml.PAID
rm $BASEDIR/config_template_11_and_17.yml.PAID $BASEDIR/config_11_and_17.yml.PAID.tmp
# copy free tier into config_11_and_17.yml to make sure this gets updated
cp $BASEDIR/config_11_and_17.yml.FREE $BASEDIR/config_11_and_17.yml
elif (! ($has_env_vars)); then
print_help
exit 0
fi
# add new or modified tests to the sets of tests to be repeated
if (! ($all)); then
add_diff_tests ()
{
dir="${BASEDIR}/../${2}"
diff=$(git --no-pager diff --name-only --diff-filter=AMR ${BASE_BRANCH}...HEAD ${dir})
tests=$( echo "$diff" \
| grep "Test\\.java" \
| sed -e "s/\\.java//" \
| sed -e "s,^${2},," \
| tr '/' '.' \
| grep ${3} )\
|| : # avoid execution interruptions due to grep return codes and set -e
for test in $tests; do
echo " $test"
has_env_vars=true
if echo "$env_vars" | grep -q "${1}="; then
env_vars=$(echo "$env_vars" | sed -e "s/${1}=/${1}=${test},/")
elif [ -z "$env_vars" ]; then
env_vars="${1}=${test}"
else
env_vars="$env_vars|${1}=${test}"
fi
done
}
echo
echo "Detecting new or modified tests with git diff --diff-filter=AMR ${BASE_BRANCH}...HEAD:"
add_diff_tests "REPEATED_UTESTS" "test/unit/" "org.apache.cassandra"
add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra"
add_diff_tests "REPEATED_UTESTS_STRESS" "tools/stress/test/unit/" "org.apache.cassandra.stress"
add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
fi
# replace environment variables
if $has_env_vars; then
echo
echo "Setting environment variables:"
IFS='='
echo "$env_vars" | tr '|' '\n' | while read entry; do
set -- $entry
key=$1
val=$2
echo " $key: $val"
sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml
cp $BASEDIR/config.yml $BASEDIR/config_11_and_17.yml
done
unset IFS
fi
# Define function to remove unneeded jobs.
# The first argument is the file name, and the second arguemnt is the job name.
delete_job()
{
delete_yaml_block()
{
sed -Ei.bak "/^ - ${2}/,/^ [^[:space:]]+|^ [^[:space:]]+/{//!d;}" "$1"
sed -Ei.bak "/^ - ${2}/d" "$1"
}
file="$BASEDIR/$1"
delete_yaml_block "$file" "${2}"
delete_yaml_block "$file" "start_${2}"
}
# Define function to remove any unneeded repeated jobs.
# The first and only argument is the file name.
delete_repeated_jobs()
{
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS=" )); then
delete_job "$1" "j11_unit_tests_repeat"
delete_job "$1" "j17_unit_tests_repeat"
delete_job "$1" "j11_utests_cdc_repeat"
delete_job "$1" "j17_utests_cdc_repeat"
delete_job "$1" "j11_utests_compression_repeat"
delete_job "$1" "j17_utests_compression_repeat"
delete_job "$1" "j11_utests_trie_repeat"
delete_job "$1" "j17_utests_trie_repeat"
delete_job "$1" "j11_utests_system_keyspace_directory_repeat"
delete_job "$1" "j17_utests_system_keyspace_directory_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_LONG=")); then
delete_job "$1" "j11_utests_long_repeat"
delete_job "$1" "j17_utests_long_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_STRESS=")); then
delete_job "$1" "j11_utests_stress_repeat"
delete_job "$1" "j17_utests_stress_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_FQLTOOL=")); then
delete_job "$1" "j11_utests_fqltool_repeat"
delete_job "$1" "j17_utests_fqltool_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_SIMULATOR_DTESTS=")); then
delete_job "$1" "j11_simulator_dtests_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_JVM_DTESTS=")); then
delete_job "$1" "j11_jvm_dtests_repeat"
delete_job "$1" "j11_jvm_dtests_vnode_repeat"
delete_job "$1" "j17_jvm_dtests_repeat"
delete_job "$1" "j17_jvm_dtests_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_DTESTS=")); then
delete_job "$1" "j11_dtests_repeat"
delete_job "$1" "j11_dtests_vnode_repeat"
delete_job "$1" "j11_dtests_offheap_repeat"
delete_job "$1" "j17_dtests_repeat"
delete_job "$1" "j17_dtests_vnode_repeat"
delete_job "$1" "j17_dtests_offheap_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_LARGE_DTESTS=")); then
delete_job "$1" "j11_dtests_large_repeat"
delete_job "$1" "j11_dtests_large_vnode_repeat"
delete_job "$1" "j17_dtests_large_repeat"
delete_job "$1" "j17_dtests_large_vnode_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_ANT_TEST_CLASS=")); then
delete_job "$1" "j11_repeated_ant_test"
delete_job "$1" "j17_repeated_ant_test"
fi
}
delete_repeated_jobs "config.yml"
delete_repeated_jobs "config_11_and_17.yml"
if $all; then
delete_repeated_jobs "config_11_and_17.yml.FREE"
delete_repeated_jobs "config_11_and_17.yml.PAID"
fi

54
.circleci/readme.md
View File

@ -23,12 +23,18 @@
This directory contains the configuration for CircleCI continous integration platform.
The file `config.yml` is the configuration file that is read by CircleCI. This file is
automatically generated by the `generate.sh` script from the `config_template.yml` file.
Experimental JDK17 configuration is added for test purposes to enable easier testing
while working on JDK17 related issues. The file `config_11_and_17.yml` is the configuration
file that we can copy over `config.yml` to test JDK11+17 workflows. This file is automatically
generated by the `generate_11_and_17.sh` script from the `config_template_11_and_17.yml` file.
The provided `config.yml` file uses low resources so users of the CircleCI free tier can
use it. Additionally, there are two versions of this file using different resources so
users who have access to premium CircleCI resources can use larger instances and more
parallelism. These files are `config.yml.FREE` and `config.yml.PAID`.
The default `config.yml` file is just a copy of `config.yml.FREE`.
For the JDK11+17 workflows we have equivalent files - `config_11_and_17.yml.FREE` and
`config_11_and_17.yml.PAID`.
## Switching to higher resource settings
This directory contains generated files for free and paid resource settings.
@ -37,29 +43,41 @@ For example, for using higher resources benefiting from CircleCI's paid tier:
`cp .circleci/config.yml.PAID .circleci/config.yml`
respectively for JDK11+17:
`cp .circleci/config_11_and_17.yml.PAID .circleci/config.yml`
And for using lower resources comaptible with CircleCI's free tier:
`cp .circleci/config.yml.FREE .circleci/config.yml`
respectively for JDK11+17:
`cp .circleci/config_11_and_17.yml.FREE .circleci/config.yml`
Alternatively, you can run the `generate.sh` script with the flags `-f`/`-p`
to regenerate the `config.yml` file from `config_template.yml` using free or paid resources.
This script validates and applies any changes to the `config_template.yml` file, and it
requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be
installed.
For JDK11 and 17 you can run the `generate_11_and_17.sh` script with the flags
`-f`/`-p` to regenerate the `config.yml` file from `config_template_11_and_17.yml`
using free or paid resources. The two scripts validate and apply any changes to the
`config_template.yml`and `config_template_11_and_17.yml` files, and they require the
[CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be installed.
## Setting environment variables
Both `config_template.yml` and `config.yml` files contain a set of environment variables
defining things like what dtest repo and branch to use, what tests could be repeatedly
run, etc.
run, etc. Same applies for `config_template_11_and_17.yml` and `config_11_and_17.yml`.
These environment variables can be directly edited in the `config.yml` file, although if
you do this you should take into account that the entire set of env vars is repeated on
every job.
A probably better approach is editing them in `config_template.yml` and then regenerate the
`config.yml` file using the `generate.sh` script. You can also directly pass environment
variable values to the `generate.sh` script with the `-e` flag. For example, to set the
dtest repo and branch with MIDRES config you can run:
A probably better approach is editing them in `config_template.yml` or respectively
`config_template_11_and_17.yml` and then regenerate the `config.yml` and `config_11_and_17.yml`
files using the `generate.sh` or respectively the `generate_11_and_17.yml` script. You can also
directly pass environment variable values to the `generate.sh` and `generate_11_and_17.sh` scripts
with the `-e` flag. For example, to set the dtest repo and branch with PAID config
you can run:
```
generate.sh -p \
@ -69,10 +87,10 @@ generate.sh -p \
```
## Running tests in a loop
Running the `generate.sh` script with use `git diff` to find the new or modified tests.
The script will then create jobs to run each of these new or modified tests for a certain
number of times, to verify that they are stable. You can use environment variables to
specify the number of iterations of each type of test:
Running the `generate.sh` or `generate_11_and_17.sh` script will use `git diff` to find the
new or modified tests. The scripts will then create jobs to run each of these new or modified
tests for a certain number of times, to verify that they are stable. You can use environment
variables to specify the number of iterations of each type of test:
```
generate.sh -p \
-e REPEATED_UTESTS_COUNT=500 \
@ -157,16 +175,18 @@ the `config.yml` file manually. Instead, you should edit the `config_template.ym
regenerate the `config.yml`, `config.yml.FREE` and `config.yml.PAID`
files by runnining the `generate.sh` script with `-a` flag. For using this script you
need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install).
Same applies for the equivalent JDK11+17 config files and `generate_11_and_17.sh`.
As for temporal changes done while working in a patch, such as pointing to you dtest repo or
running a test repeatedly, you can either directly edit `config.yml` or edit `config_template.yml`
and then regenerate `config.yml` with the `generate.sh` script using a `-f`/`-p` flag.
When this flag is used only the `config.yml` will be generated.
When this flag is used only the `config.yml` will be generated. Same workflow applies to the respective
JDK11+17 CircleCI configuration files.
Please note that any previous swapping or edition of the generated files will be overriden
by running `generate.sh` with `-a` argument, returning `config.yml` to the default FREE. So if
you previously swapped your `config.yml` to MIDRES you would need to either swap it
again or use the `-f`/`-p` script flags.
by running `generate.sh`/`generate_11_and_17.sh` with `-a` argument, returning `config.yml`
to the default FREE. So if you previously swapped your `config.yml` to PAID you would need to
either swap it again or use the `-f`/`-p` script flags.
Read below for details how to generate the files manually without the `generate.sh` script:
@ -188,3 +208,5 @@ Read below for details how to generate the files manually without the `generate.
7. add the Apache license header to the newly created PAID file:
`cat license.yml config.yml.PAID > config.yml.PAID.new && mv config.yml.PAID.new config.yml.PAID`
8. finally, remember to update the config.yml
The process is equivalent for `generate_11_and_17.sh` and the respective JDK11+17 configuration files.