natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Exclude intensive dtests when not running dtest-upgrade*-large 

Also add dtest-large-latest

 patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-19963
This commit is contained in:
Mick Semb Wever 2024-09-28 17:59:56 +03:00
parent 07c1f93b06
commit 2fb578263c
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
3 changed files with 26 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.build/docker/run-tests.sh
View File

@ -149,7 +149,7 @@ case ${target} in
[[ ${mem} -gt $((15 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 16g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
docker_flags="-m 15g --memory-swap 15g"
;;
"dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large")
"dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-large-latest" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large")
[ -f "${cassandra_dtest_dir}/dtest.py" ] || { error 1 "${cassandra_dtest_dir}/dtest.py not found. please specify 'cassandra_dtest_dir' to point to the local cassandra-dtest source"; }
test_script="run-python-dtests.sh"
docker_mounts="${docker_mounts} -v ${cassandra_dtest_dir}:/home/cassandra/cassandra-dtest"

44
.build/run-python-dtests.sh
View File

@ -64,6 +64,8 @@ command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1
command -v virtualenv >/dev/null 2>&1 || { echo >&2 "virtualenv needs to be installed"; exit 1; }
[ -f "${CASSANDRA_DIR}/build.xml" ] || { echo >&2 "${CASSANDRA_DIR}/build.xml must exist"; exit 1; }
[ -d "${DIST_DIR}" ] || { mkdir -p "${DIST_DIR}" ; }
ALLOWED_DTEST_VARIANTS="novnode|large|latest|upgrade"
[[ "${DTEST_TARGET}" =~ ^dtest(-(${ALLOWED_DTEST_VARIANTS}))*$ ]] || { echo >&2 "Unknown dtest target: ${DTEST_TARGET}. Allowed variants are ${ALLOWED_DTEST_VARIANTS}"; exit 1; }
java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
version=$(grep 'property\s*name=\"base.version\"' ${CASSANDRA_DIR}/build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
@ -114,31 +116,26 @@ pip3 freeze
cd ${CASSANDRA_DTEST_DIR}
set +e # disable immediate exit from this point
if [ "${DTEST_TARGET}" = "dtest" ]; then
DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --skip-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-novnode" ]; then
DTEST_ARGS="--skip-resource-intensive-tests --keep-failed-test-dir"
elif [ "${DTEST_TARGET}" = "dtest-latest" ]; then
DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --configuration-yaml=cassandra_latest.yaml --skip-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-large" ]; then
DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --only-resource-intensive-tests --force-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-large-novnode" ]; then
DTEST_ARGS="--only-resource-intensive-tests --force-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-upgrade" ]; then
DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --execute-upgrade-tests --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all"
elif [ "${DTEST_TARGET}" = "dtest-upgrade-novnode" ]; then
DTEST_ARGS="--execute-upgrade-tests --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all"
elif [ "${DTEST_TARGET}" = "dtest-upgrade-large" ]; then
DTEST_ARGS="--use-vnodes --num-tokens=${NUM_TOKENS} --execute-upgrade-tests --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --only-resource-intensive-tests --force-resource-intensive-tests"
elif [ "${DTEST_TARGET}" = "dtest-upgrade-novnode-large" ]; then
DTEST_ARGS="--execute-upgrade-tests --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all --only-resource-intensive-tests --force-resource-intensive-tests"
DTEST_ARGS="--keep-failed-test-dir"
# Check for specific keywords in DTEST_TARGET and append corresponding options
if [[ "${DTEST_TARGET}" == *"-large"* ]]; then
DTEST_ARGS+=" --only-resource-intensive-tests --force-resource-intensive-tests"
else
echo "Unknown dtest target: ${DTEST_TARGET}"
exit 1
DTEST_ARGS+=" --skip-resource-intensive-tests"
fi
if [[ "${DTEST_TARGET}" != *"-novnode"* ]]; then
DTEST_ARGS+=" --use-vnodes --num-tokens=${NUM_TOKENS}"
fi
if [[ "${DTEST_TARGET}" == *"-latest"* ]]; then
DTEST_ARGS+=" --configuration-yaml=cassandra_latest.yaml"
fi
if [[ "${DTEST_TARGET}" == *"-upgrade"* ]]; then
DTEST_ARGS+=" --execute-upgrade-tests --execute-upgrade-tests-only --upgrade-target-version-only --upgrade-version-selection all"
fi
touch ${DIST_DIR}/test_list.txt
./run_dtests.py --cassandra-dir=${CASSANDRA_DIR} ${DTEST_ARGS} --dtest-print-tests-only --dtest-print-tests-output=${DIST_DIR}/test_list.txt 2>&1 > ${DIST_DIR}/test_stdout.txt
[[ $? -eq 0 ]] || { cat ${DIST_DIR}/test_stdout.txt ; exit 1; }
if [[ "${DTEST_SPLIT_CHUNK}" =~ ^[0-9]+/[0-9]+$ ]]; then
@ -159,11 +156,16 @@ elif [[ "x" != "x${DTEST_SPLIT_CHUNK}" ]] ; then
else
SPLIT_TESTS=$(cat ${DIST_DIR}/test_list.txt)
fi
SPLIT_TESTS="${SPLIT_TESTS//$'\n'/ }"
pytest_results_file="${DIST_DIR}/test/output/nosetests.xml"
pytest_opts="-vv --log-cli-level=DEBUG --junit-xml=${pytest_results_file} --junit-prefix=${DTEST_TARGET} -s"
pytest ${pytest_opts} --cassandra-dir=${CASSANDRA_DIR} --keep-failed-test-dir ${DTEST_ARGS} ${SPLIT_TESTS} 2>&1 | tee -a ${DIST_DIR}/test_stdout.txt
echo ""
echo "pytest ${pytest_opts} --cassandra-dir=${CASSANDRA_DIR} --keep-failed-test-dir ${DTEST_ARGS} ${SPLIT_TESTS}"
echo ""
pytest ${pytest_opts} --cassandra-dir=${CASSANDRA_DIR} --keep-failed-test-dir ${DTEST_ARGS} ${SPLIT_TESTS} 2>&1 | tee -a ${DIST_DIR}/test_stdout.txt
# tar up any ccm logs for easy retrieval
if ls ${TMPDIR}/*/test/*/logs/* &>/dev/null ; then

3
.jenkins/Jenkinsfile vendored
View File

@ -129,7 +129,7 @@ def pipelineProfiles() {
'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'],
'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
'custom': []
]
}
@ -176,6 +176,7 @@ def tasks() {
'dtest-latest': [splits: 64, size: 'large'],
'dtest-large': [splits: 8, size: 'large'],
'dtest-large-novnode': [splits: 8, size: 'large'],
'dtest-large-latest': [splits: 8, size: 'large'],
'dtest-upgrade': [splits: 64, size: 'large'],
'dtest-upgrade-novnode': [splits: 64, size: 'large'],
'dtest-upgrade-large': [splits: 32, size: 'large'],