diff --git a/pylib/cassandra-cqlsh-tests.sh b/pylib/cassandra-cqlsh-tests.sh
index 1fb3aa0c85..81746367ab 100755
--- a/pylib/cassandra-cqlsh-tests.sh
+++ b/pylib/cassandra-cqlsh-tests.sh
@@ -22,6 +22,7 @@ export CCM_HEAP_NEWSIZE="200M"
 export CCM_CONFIG_DIR=${WORKSPACE}/.ccm
 export NUM_TOKENS="32"
 export CASSANDRA_DIR=${WORKSPACE}
+export TESTSUITE_NAME="cqlshlib.python2.jdk8"
 
 # Loop to prevent failure due to maven-ant-tasks not downloading a jar..
 for x in $(seq 1 3); do
@@ -45,9 +46,12 @@ pip install -r ${CASSANDRA_DIR}/pylib/requirements.txt
 pip freeze
 
 if [ "$cython" = "yes" ]; then
+    TESTSUITE_NAME="${TESTSUITE_NAME}.cython"
     pip install "Cython>=0.20,<0.25"
     cd pylib/; python setup.py build_ext --inplace
     cd ${WORKSPACE}
+else
+    TESTSUITE_NAME="${TESTSUITE_NAME}.no_cython"
 fi
 
 ################################
@@ -85,6 +89,9 @@ set +e # disable immediate exit from this point
 nosetests
 
 ccm remove
+# hack around --xunit-prefix-with-testsuite-name not being available in nose 1.3.7
+sed -i "s/testsuite name=\"nosetests\"/testsuite name=\"${TESTSUITE_NAME}\"/g" nosetests.xml
+sed -i "s/testcase classname=\"cqlshlib./testcase classname=\"${TESTSUITE_NAME}./g" nosetests.xml
 mv nosetests.xml ${WORKSPACE}/cqlshlib.xml
 
 ################################