natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.11' into cassandra-4.0

This commit is contained in:
Andrés de la Peña 2023-03-10 11:51:25 +00:00
parent a874e02eec 4b3cb0a772
commit 2aedb58123
8 changed files with 93 additions and 11897 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
.circleci/config-2_1.yml.high_res.patch
View File

@ -1,155 +0,0 @@
--- config-2_1.yml 2023-02-07 21:00:52.000000000 -0500
+++ config-2_1.yml.HIGHRES 2023-02-07 21:03:37.000000000 -0500
@@ -145,14 +145,14 @@
j8_par_executor: &j8_par_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j8_small_par_executor: &j8_small_par_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 5
j8_small_executor: &j8_small_executor
executor:
@@ -163,74 +163,80 @@
j8_medium_par_executor: &j8_medium_par_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 2
j8_seq_executor: &j8_seq_executor
executor:
name: java8-executor
- #exec_resource_class: xlarge
+ exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
j11_par_executor: &j11_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j11_small_par_executor: &j11_small_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 5
j11_small_executor: &j11_small_executor
executor:
name: java11-executor
- #exec_resource_class: medium
+ exec_resource_class: medium
parallelism: 1
j11_medium_par_executor: &j11_medium_par_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
- parallelism: 1
+ exec_resource_class: xlarge
+ parallelism: 2
j11_seq_executor: &j11_seq_executor
executor:
name: java11-executor
- #exec_resource_class: xlarge
+ exec_resource_class: xlarge
parallelism: 1 # sequential, single container tests: no parallelism benefits
j8_repeated_utest_executor: &j8_repeated_utest_executor
executor:
name: java8-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j8_repeated_dtest_executor: &j8_repeated_dtest_executor
executor:
name: java8-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j8_repeated_upgrade_dtest_executor: &j8_repeated_upgrade_dtest_executor
executor:
name: java8-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j8_repeated_jvm_upgrade_dtest_executor: &j8_repeated_jvm_upgrade_dtest_executor
executor:
name: java8-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j11_repeated_utest_executor: &j11_repeated_utest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j11_repeated_dtest_executor: &j11_repeated_dtest_executor
executor:
name: java11-executor
- parallelism: 4
+ exec_resource_class: xlarge
+ parallelism: 100
j8_separate_jobs: &j8_separate_jobs
jobs:
@@ -1787,7 +1793,7 @@
pytest_extra_args: '--skip-resource-intensive-tests'
j8_dtests_large_vnode:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1801,7 +1807,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j8_dtests_large:
- <<: *j8_par_executor
+ <<: *j8_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1815,7 +1821,7 @@
pytest_extra_args: '--num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large_vnode:
- <<: *j11_par_executor
+ <<: *j11_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
@@ -1829,7 +1835,7 @@
pytest_extra_args: '--use-vnodes --num-tokens=16 --only-resource-intensive-tests --force-resource-intensive-tests'
j11_dtests_large:
- <<: *j11_par_executor
+ <<: *j11_seq_executor
steps:
- attach_workspace:
at: /home/cassandra

0
.circleci/config.yml.LOWRES → .circleci/config.yml.FREE
View File

11630
.circleci/config.yml.HIGHRES
View File

File diff suppressed because it is too large Load Diff

0
.circleci/config.yml.MIDRES → .circleci/config.yml.PAID
View File

0
.circleci/config-2_1.yml → .circleci/config_template.yml
View File

0
.circleci/config-2_1.yml.mid_res.patch → .circleci/config_template.yml.PAID.patch
View File

107
.circleci/generate.sh
View File

@ -30,13 +30,12 @@ die ()
print_help()
{
echo "Usage: $0 [-l|-m|-h|-f|-e]"
echo " -a Generate the default config.yml using low resources and the three templates"
echo " (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES). Use this for"
echo " permanent changes in config-2_1.yml that will be committed to the main repo."
echo " -l Generate config.yml using low resources"
echo " -m Generate config.yml using mid resources"
echo " -h Generate config.yml using high resources"
echo "Usage: $0 [-f|-p|-a|-e|-i]"
echo " -a Generate the config.yml, config.yml.FREE and config.yml.PAID expanded configuration"
echo " files from the main config_template.yml reusable configuration file."
echo " Use this for permanent changes in config that will be committed to the main repo."
echo " -f Generate config.yml for tests compatible with the CircleCI free tier resources"
echo " -p Generate config.yml for tests compatible with the CircleCI paid tier resources"
echo " -e <key=value> Environment variables to be used in the generated config.yml, e.g.:"
echo " -e DTEST_BRANCH=CASSANDRA-8272"
echo " -e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git"
@ -64,28 +63,25 @@ print_help()
echo " -e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable"
echo " -e REPEATED_ANT_TEST_COUNT=500"
echo " For the complete list of environment variables, please check the"
echo " list of examples in config-2_1.yml and/or the documentation."
echo " list of examples in config_template.yml and/or the documentation."
echo " If you want to specify multiple environment variables simply add"
echo " multiple -e options. The flags -l/-m/-h should be used when using -e."
echo " -f Stop checking that the environment variables are known"
echo " multiple -e options. The flags -f/-p should be used when using -e."
echo " -i Ignore unknown environment variables"
}
all=false
lowres=false
midres=false
highres=false
free=false
paid=false
env_vars=""
has_env_vars=false
check_env_vars=true
while getopts "e:almhf" opt; do
while getopts "e:afpi" opt; do
case $opt in
a ) all=true
;;
l ) lowres=true
f ) free=true
;;
m ) midres=true
;;
h ) highres=true
p ) paid=true
;;
e ) if (!($has_env_vars)); then
env_vars="$OPTARG"
@ -94,7 +90,7 @@ while getopts "e:almhf" opt; do
fi
has_env_vars=true
;;
f ) check_env_vars=false
i ) check_env_vars=false
;;
\?) die "Invalid option: -$OPTARG"
;;
@ -139,52 +135,40 @@ if $has_env_vars && $check_env_vars; then
done
fi
if $lowres; then
($all || $midres || $highres) && die "Cannot use option -l with options -a, -m or -h"
echo "Generating new config.yml file with low resources from config-2_1.yml"
circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml
rm $BASEDIR/config.yml.LOWRES.tmp
if $free; then
($all || $paid) && die "Cannot use option -f with options -a or -p"
echo "Generating new config.yml file for free tier from config_template.yml"
circleci config process $BASEDIR/config_template.yml > $BASEDIR/config.yml.FREE.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp > $BASEDIR/config.yml
rm $BASEDIR/config.yml.FREE.tmp
elif $midres; then
($all || $lowres || $highres) && die "Cannot use option -m with options -a, -l or -h"
echo "Generating new config.yml file with middle resources from config-2_1.yml"
patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch
circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml
rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
elif $highres; then
($all || $lowres || $midres) && die "Cannot use option -h with options -a, -l or -m"
echo "Generating new config.yml file with high resources from config-2_1.yml"
patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch
circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml
rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
elif $paid; then
($all || $free) && die "Cannot use option -p with options -a or -f"
echo "Generating new config.yml file for paid tier from config_template.yml"
patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml
rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
elif $all; then
($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option -a with options -l, -m, -h or -e"
echo "Generating new config.yml file with low resources and LOWRES/MIDRES/HIGHRES templates from config-2_1.yml"
($free || $paid || $has_env_vars) && die "Cannot use option -a with options -f, -p or -e"
echo "Generating new default config.yml file for free tier and FREE/PAID templates from config_template.yml."
echo "Make sure you commit the newly generated config.yml, config.yml.FREE and config.yml.PAID files"
echo "after running this command if you want them to persist."
# setup lowres
circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml.LOWRES
rm $BASEDIR/config.yml.LOWRES.tmp
# setup config for free tier
circleci config process $BASEDIR/config_template.yml > $BASEDIR/config.yml.FREE.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.FREE.tmp > $BASEDIR/config.yml.FREE
rm $BASEDIR/config.yml.FREE.tmp
# setup midres
patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch
circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.MIDRES.tmp > $BASEDIR/config.yml.MIDRES
rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
# setup config for paid tier
patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml.PAID
rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
# setup highres
patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch
circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml.HIGHRES
rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
# copy lower into config.yml to make sure this gets updated
cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml
# copy free tier into config.yml to make sure this gets updated
cp $BASEDIR/config.yml.FREE $BASEDIR/config.yml
elif (!($has_env_vars)); then
print_help
@ -317,7 +301,6 @@ delete_repeated_jobs()
delete_repeated_jobs "config.yml"
if $all; then
delete_repeated_jobs "config.yml.LOWRES"
delete_repeated_jobs "config.yml.MIDRES"
delete_repeated_jobs "config.yml.HIGHRES"
delete_repeated_jobs "config.yml.FREE"
delete_repeated_jobs "config.yml.PAID"
fi

98
.circleci/readme.md
View File

@ -22,42 +22,47 @@
This directory contains the configuration for CircleCI continous integration platform.
The file `config.yml` is the configuration file that is read by CircleCI. This file is
automatically generated by the `generate.sh` script from the `config-2_1.yml` file.
automatically generated by the `generate.sh` script from the `config_template.yml` file.
The provided `config.yml` file uses low resources so users of the CircleCI free tier can
use it. Additionally, there are three versions of this file using different resources so
use it. Additionally, there are two versions of this file using different resources so
users who have access to premium CircleCI resources can use larger instances and more
parallelism. These files are `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES`.
The default `config.yml` file is just a copy of `config.yml.LOWRES`.
parallelism. These files are `config.yml.FREE` and `config.yml.PAID`.
The default `config.yml` file is just a copy of `config.yml.FREE`.
## Switching to higher resource settings
This directory contains generated files for low, middle and high resource settings.
Switch between them by copying the correct file to `config.yml` and committing the result:
This directory contains generated files for free and paid resource settings.
Switch between them by copying the correct file to `config.yml` and committing the result.
For example, for using higher resources benefiting from CircleCI's paid tier:
`cp .circleci/config.yml.HIGHRES .circleci/config.yml`
`cp .circleci/config.yml.PAID .circleci/config.yml`
Alternatively, you can run the `generate.sh` script with the flags `-l`/`-m`/`-h`
to regenerate the `config.yml` file from `config-2_1.yml` using LOWRES/MIDRES/HIGHRES.
This script validates and applies any changes to the `config-2_1.yml` file, and it
And for using lower resources comaptible with CircleCI's free tier:
`cp .circleci/config.yml.FREE .circleci/config.yml`
Alternatively, you can run the `generate.sh` script with the flags `-f`/`-p`
to regenerate the `config.yml` file from `config_template.yml` using free or paid resources.
This script validates and applies any changes to the `config_template.yml` file, and it
requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be
installed.
## Setting environment variables
Both `config-2_1.yml` and `config.yml` files contain a set of environment variables
Both `config_template.yml` and `config.yml` files contain a set of environment variables
defining things like what dtest repo and branch to use, what tests could be repeatedly
run, etc.
These environment variables can be directly edited in the `config.yml` file, although if
you do this you should take into account that the entire set of env vars is repeated on
every job.
every job.
A probably better approach is editing them in `config-2_1.yml` and then regenerate the
A probably better approach is editing them in `config_template.yml` and then regenerate the
`config.yml` file using the `generate.sh` script. You can also directly pass environment
variable values to the `generate.sh` script with the `-e` flag. For example, to set the
dtest repo and branch with MIDRES config you can run:
```
generate.sh -m \
generate.sh -p \
-e DTEST_REPO=https://github.com/adelapena/cassandra-dtest.git \
-e DTEST_BRANCH=CASSANDRA-8272
@ -69,7 +74,7 @@ The script will then create jobs to run each of these new or modified tests for
number of times, to verify that they are stable. You can use environment variables to
specify the number of iterations of each type of test:
```
generate.sh -m \
generate.sh -p \
-e REPEATED_UTESTS_COUNT=500 \
-e REPEATED_UTESTS_FQLTOOL_COUNT=500 \
-e REPEATED_UTESTS_LONG_COUNT=100 \
@ -84,12 +89,12 @@ generate.sh -m \
```
You can also specify whether the iteration should fail on the first test failure:
```
generate.sh -m -e REPEATED_TESTS_STOP_ON_FAILURE=false
generate.sh -p -e REPEATED_TESTS_STOP_ON_FAILURE=false
```
In addition to the automatically detected tests, it's also possible to provide lists of
specific tests to be repeated:
```
generate.sh -m \
generate.sh -p \
-e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \
-e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest \
-e REPEATED_UTESTS_LONG=org.apache.cassandra.io.sstable.CQLSSTableWriterLongTest#testWideRow \
@ -103,7 +108,7 @@ generate.sh -m \
For particular Ant test targets that are not included in the regular test suites, you can
use the `run_repeated_utest` job:
```
generate.sh -m \
generate.sh -p \
-e REPEATED_ANT_TEST_TARGET=test-cdc \
-e REPEATED_ANT_TEST_CLASS=org.apache.cassandra.cql3.ViewTest \
-e REPEATED_ANT_TEST_METHODS=testCompoundPartitionKey,testStaticTable \
@ -111,7 +116,7 @@ generate.sh -m \
```
Putting all together, you can have runs as complex as:
```
generate.sh -m \
generate.sh -p \
-e REPEATED_TESTS_STOP_ON_FAILURE=true \
-e REPEATED_UTESTS=org.apache.cassandra.cql3.ViewTest,org.apache.cassandra.db.CellTest \
-e REPEATED_UTESTS_COUNT=500 \
@ -143,45 +148,38 @@ default medium config and Large containers.
## Updating the config
For configuration changes meant to be permanent in the Apache repo you should never edit
the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then
regenerate the `config.yml`, `config.yml.LOWRES`, `config.yml.MIDRES` and `config.yml.HIGHRES`
the `config.yml` file manually. Instead, you should edit the `config_template.yml` file and then
regenerate the `config.yml`, `config.yml.FREE` and `config.yml.PAID`
files by runnining the `generate.sh` script with `-a` flag. For using this script you
need to install the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install).
As for temporal changes done while working in a patch, such as pointing to you dtest repo or
running a test repeatedly, you can either directly edit `config.yml` or edit `config-2_1.yml`
and then regenerate `config.yml` with the `generate.sh` script using a `-l`/`-m`/`-h` flag.
running a test repeatedly, you can either directly edit `config.yml` or edit `config_template.yml`
and then regenerate `config.yml` with the `generate.sh` script using a `-f`/`-p` flag.
When this flag is used only the `config.yml` will be generated.
Please note that any previous swapping or edition of the generated files will be overriden
by running `generate.sh` with `-a` argument, returning `config.yml` to the default LOWRES. So if
you previously swapped your `config.yml` to MIDRES or HIGHRES you would need to either swap it
again or use the `-l`/`-m`/`-h` script flags.
by running `generate.sh` with `-a` argument, returning `config.yml` to the default FREE. So if
you previously swapped your `config.yml` to MIDRES you would need to either swap it
again or use the `-f`/`-p` script flags.
Read below for details how to generate the files manually without the `generate.sh` script:
1. make your edits to config-2_1.yml - let it stay at lowres settings
2. generate a valid LOWRES file:
`circleci config process config-2_1.yml > config.yml.LOWRES`
3. add the Apache license header to the newly created LOWRES file:
`cat license.yml config.yml.LOWRES > config.yml.LOWRES.new && mv config.yml.LOWRES.new config.yml.LOWRES`
4. then apply the highres patch to config-2_1.yml;
`patch -o config-2_1.yml.HIGHRES config-2_1.yml config-2_1.yml.high_res.patch`
(this creates a new file `config-2_1.yml.HIGHRES` instead of in-place patching
config-2_1.yml)
Note that if the patch no longer applies to `config-2_1.yml` a new patch file
is needed, do this by manually making `config-2_1.yml` high resource and create
1. make your edits to config_template.yml - let it stay at free tier settings
2. generate a valid FREE file:
`circleci config process config_template.yml > config.yml.FREE`
3. add the Apache license header to the newly created FREE file:
`cat license.yml config.yml.FREE > config.yml.FREE.new && mv config.yml.FREE.new config.yml.FREE`
4. then apply the PAID patch to config_template.yml;
`patch -o config_template.yml.PAID config_template.yml config_template.yml.PAID.patch`
(this creates a new file `config_template.yml.PAID` instead of in-place patching
config_template.yml).
Note that if the patch no longer applies to `config_template.yml` a new patch file
is needed, do this by manually creating and editing `config_template.yml.PAID`, and create
the patch file based on the diff (don't commit it though).
5. generate the HIGHRES file:
`circleci config process config-2_1.yml.HIGHRES > config.yml.HIGHRES`
6. remove the temporary patched HIGHRES file: `rm config-2_1.yml.HIGHRES`
7. add the Apache license header to the newly created HIGHRES file:
`cat license.yml config.yml.HIGHRES > config.yml.HIGHRES.new && mv config.yml.HIGHRES.new config.yml.HIGHRES`
8. repeat the last steps to generate the MIDRES file:
```
patch -o config-2_1.yml.MIDRES config-2_1.yml config-2_1.yml.mid_res.patch
circleci config process config-2_1.yml.MIDRES > config.yml.MIDRES
rm config-2_1.yml.MIDRES
cat license.yml config.yml.MIDRES > config.yml.MIDRES.new && mv config.yml.MIDRES.new config.yml.MIDRES
```
9. finally, remember to update the config.yml
5. generate the PAID file:
`circleci config process config_template.yml.PAID > config.yml.PAID`
6. remove the temporary patched PAID file: `rm config_template.yml.PAID`
7. add the Apache license header to the newly created PAID file:
`cat license.yml config.yml.PAID > config.yml.PAID.new && mv config.yml.PAID.new config.yml.PAID`
8. finally, remember to update the config.yml