!48 sysbench code for Redis

Merge pull request !48 from 熊小军/master
This commit is contained in:
opengauss-bot 2023-05-30 09:11:03 +00:00 committed by Gitee
commit 635d2e5235
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
8 changed files with 654 additions and 19 deletions

View File

@ -0,0 +1,9 @@
1. complile sysbench binary by README.md
2. cd src/lua
3. modify oltp_common_redis.lua. change require path to your real path, and lua script for redis is in Sysbench4RedisAndMot/lib
4. start redis server, and set IP and PORT in oltp_common_redis.lua, besides, you should start mysql or openGauss server
4. you can run oltp_read_write.lua for test, and you can change its require lua script to 'oltp_common.lua' when you want to run sysbench without redis

View File

@ -477,6 +477,20 @@ int db_bind_result(db_stmt_t *stmt, db_bind_t *results, size_t len)
return con->driver->ops.bind_result(stmt, results, len);
}
/* ADD for redis, for point_select, when get data from redis, we should increment READ counter */
int db_counter_inc(db_stmt_t *stmt, int inc)
{
db_conn_t *con = stmt->connection;
if (con->state == DB_CONN_INVALID)
{
log_text(LOG_ALERT, "attempt to use an already closed connection");
return 1;
}
sb_counter_inc(con->thread_id, inc);
return 0;
}
/* Execute prepared statement */

View File

@ -317,6 +317,8 @@ int db_bulk_insert_done(db_conn_t *);
void db_report_intermediate(sb_stat_t *);
void db_report_cumulative(sb_stat_t *);
int db_counter_inc(db_stmt_t *, int);
/* DB drivers registrars */
#ifdef USE_MYSQL

View File

@ -67,6 +67,8 @@ db_free_results = sysbench.db.free_results
db_close = sysbench.db.close
db_counter_inc = sysbench.db.counter_inc
DB_ERROR_NONE = sysbench.db.DB_ERROR_NONE
DB_ERROR_RESTART_TRANSACTION = sysbench.db.DB_ERROR_RESTART_TRANSACTION
DB_ERROR_FAILED = sysbench.db.DB_ERROR_FAILED

View File

@ -146,7 +146,7 @@ int db_bind_param(sql_statement *stmt, sql_bind *params, size_t len);
int db_bind_result(sql_statement *stmt, sql_bind *results, size_t len);
sql_result *db_execute(sql_statement *stmt);
int db_close(sql_statement *stmt);
int db_counter_inc(sql_statement *stmt, int inc);
int db_free_results(sql_result *);
]]
@ -435,6 +435,10 @@ function statement_methods.close(self)
return ffi.C.db_close(self)
end
function statement_methods.counter_inc(self, inc)
return ffi.C.db_counter_inc(self, inc)
end
-- sql_statement metatable
local statement_mt = {
__index = statement_methods,

View File

@ -0,0 +1,601 @@
-- Copyright (C) 2006-2018 Alexey Kopytov <akopytov@gmail.com>
-- This program is free software; you can redistribute it and/or modify
-- it under the terms of the GNU General Public License as published by
-- the Free Software Foundation; either version 2 of the License, or
-- (at your option) any later version.
-- This program is distributed in the hope that it will be useful,
-- but WITHOUT ANY WARRANTY; without even the implied warranty of
-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-- GNU General Public License for more details.
-- You should have received a copy of the GNU General Public License
-- along with this program; if not, write to the Free Software
-- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-- -----------------------------------------------------------------------------
-- Common code for OLTP benchmarks.
-- -----------------------------------------------------------------------------
package.path = package.path..";../../lib/redis-lua-version-2.0/src/?.lua"
local redis = require 'redis'
function init()
assert(event ~= nil,
"this script is meant to be included by other OLTP scripts and " ..
"should not be called directly.")
end
if sysbench.cmdline.command == nil then
error("Command is required. Supported commands: prepare, prewarm, run, " ..
"cleanup, help")
end
-- Command line options
sysbench.cmdline.options = {
table_size =
{"Number of rows per table", 10000},
range_size =
{"Range size for range SELECT queries", 100},
tables =
{"Number of tables", 1},
point_selects =
{"Number of point SELECT queries per transaction", 10},
simple_ranges =
{"Number of simple range SELECT queries per transaction", 1},
sum_ranges =
{"Number of SELECT SUM() queries per transaction", 1},
order_ranges =
{"Number of SELECT ORDER BY queries per transaction", 1},
distinct_ranges =
{"Number of SELECT DISTINCT queries per transaction", 1},
index_updates =
{"Number of UPDATE index queries per transaction", 1},
non_index_updates =
{"Number of UPDATE non-index queries per transaction", 1},
delete_inserts =
{"Number of DELETE/INSERT combinations per transaction", 1},
range_selects =
{"Enable/disable all range SELECT queries", true},
auto_inc =
{"Use AUTO_INCREMENT column as Primary Key (for MySQL), " ..
"or its alternatives in other DBMS. When disabled, use " ..
"client-generated IDs", true},
skip_trx =
{"Don't start explicit transactions and execute all queries " ..
"in the AUTOCOMMIT mode", false},
secondary =
{"Use a secondary index in place of the PRIMARY KEY", false},
create_secondary =
{"Create a secondary index in addition to the PRIMARY KEY", true},
mysql_storage_engine =
{"Storage engine, if MySQL is used", "innodb"},
pgsql_variant =
{"Use this PostgreSQL variant when running with the " ..
"PostgreSQL driver. The only currently supported " ..
"variant is 'redshift'. When enabled, " ..
"create_secondary is automatically disabled, and " ..
"delete_inserts is set to 0"}
}
function redis_connection()
local redis_con = {}
redis_con[0] = redis.connect('127.0.0.1', 6379)
redis_con[1] = redis.connect('127.0.0.1', 6389)
redis_con[2] = redis.connect('127.0.0.1', 6399)
redis_con[3] = redis.connect('127.0.0.1', 6309)
for i=0, 3 do
local response = redis_con[i]:ping()
if not response then
print("failed to connect redis".. i)
return false,nil
end
end
return true, redis_con
end
function redis_close(redis_con)
for i=0, 3 do
redis_con[i]:close()
end
end
function redis_cleanup(redis_con)
for i=0, 3 do
redis_con[i]:flushall()
end
end
-- Prepare the dataset. This command supports parallel execution, i.e. will
-- benefit from executing with --threads > 1 as long as --tables > 1
function cmd_prepare()
local drv = sysbench.sql.driver()
local con = drv:connect()
local res, redis_con = redis_connection()
if not res then
print("failed to connect redis")
return
end
for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables,
sysbench.opt.threads do
create_table(drv, con, redis_con, i)
end
end
-- Preload the dataset into the server cache. This command supports parallel
-- execution, i.e. will benefit from executing with --threads > 1 as long as
-- --tables > 1
--
-- PS. Currently, this command is only meaningful for MySQL/InnoDB benchmarks
function cmd_prewarm()
local drv = sysbench.sql.driver()
local con = drv:connect()
assert(drv:name() == "mysql", "prewarm is currently MySQL only")
-- Do not create on disk tables for subsequent queries
con:query("SET tmp_table_size=2*1024*1024*1024")
con:query("SET max_heap_table_size=2*1024*1024*1024")
for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables,
sysbench.opt.threads do
local t = "sbtest" .. i
print("Prewarming table " .. t)
con:query("ANALYZE TABLE sbtest" .. i)
con:query(string.format(
"SELECT AVG(id) FROM " ..
"(SELECT * FROM %s FORCE KEY (PRIMARY) " ..
"LIMIT %u) t",
t, sysbench.opt.table_size))
con:query(string.format(
"SELECT COUNT(*) FROM " ..
"(SELECT * FROM %s WHERE k LIKE '%%0%%' LIMIT %u) t",
t, sysbench.opt.table_size))
end
end
-- Implement parallel prepare and prewarm commands
sysbench.cmdline.commands = {
prepare = {cmd_prepare, sysbench.cmdline.PARALLEL_COMMAND},
prewarm = {cmd_prewarm, sysbench.cmdline.PARALLEL_COMMAND}
}
-- Template strings of random digits with 11-digit groups separated by dashes
-- 10 groups, 119 characters
local c_value_template = "###########-###########-###########-" ..
"###########-###########-###########-" ..
"###########-###########-###########-" ..
"###########"
-- 5 groups, 59 characters
local pad_value_template = "###########-###########-###########-" ..
"###########-###########"
function get_c_value()
return sysbench.rand.string(c_value_template)
end
function get_pad_value()
return sysbench.rand.string(pad_value_template)
end
function create_table(drv, con, redis_con, table_num)
local id_index_def, id_def
local engine_def = ""
local extra_table_options = ""
local query
if sysbench.opt.secondary then
id_index_def = "KEY xid"
else
id_index_def = "PRIMARY KEY"
end
if drv:name() == "mysql" or drv:name() == "attachsql" or
drv:name() == "drizzle"
then
if sysbench.opt.auto_inc then
id_def = "INTEGER NOT NULL AUTO_INCREMENT"
else
id_def = "INTEGER NOT NULL"
end
engine_def = "/*! ENGINE = " .. sysbench.opt.mysql_storage_engine .. " */"
extra_table_options = mysql_table_options or ""
elseif drv:name() == "pgsql"
then
if not sysbench.opt.auto_inc then
id_def = "INTEGER NOT NULL"
elseif pgsql_variant == 'redshift' then
id_def = "INTEGER IDENTITY(1,1)"
else
id_def = "SERIAL"
end
else
error("Unsupported database driver:" .. drv:name())
end
print(string.format("Creating table 'sbtest%d'...", table_num))
query = string.format([[
CREATE TABLE sbtest%d(
id %s,
k INTEGER DEFAULT '0' NOT NULL,
c CHAR(120) DEFAULT '' NOT NULL,
pad CHAR(60) DEFAULT '' NOT NULL,
%s (id)
) %s %s]],
table_num, id_def, id_index_def, engine_def, extra_table_options)
con:query(query)
if (sysbench.opt.table_size > 0) then
print(string.format("Inserting %d records into 'sbtest%d'",
sysbench.opt.table_size, table_num))
end
if sysbench.opt.auto_inc then
query = "INSERT INTO sbtest" .. table_num .. "(k, c, pad) VALUES"
else
query = "INSERT INTO sbtest" .. table_num .. "(id, k, c, pad) VALUES"
end
con:bulk_insert_init(query)
local c_val
local pad_val
local redis_key
local redis_value
local k_val
for i = 1, sysbench.opt.table_size do
c_val = get_c_value()
pad_val = get_pad_value()
k_val = sb_rand(1, sysbench.opt.table_size)
if (sysbench.opt.auto_inc) then
query = string.format("(%d, '%s', '%s')",
k_val, c_val,
pad_val)
else
query = string.format("(%d, %d, '%s', '%s')",
i, k_val, c_val,
pad_val)
end
con:bulk_insert_next(query)
redis_key = table_num .. ":" .. i
redis_value = i .. "," .. k_val .. "," .. c_val .. "," .. pad_val
local redis_idx = i % 4
redis_con[redis_idx]:set(redis_key, redis_value)
end
con:bulk_insert_done()
if sysbench.opt.create_secondary then
print(string.format("Creating a secondary index on 'sbtest%d'...",
table_num))
con:query(string.format("CREATE INDEX k_%d ON sbtest%d(k)",
table_num, table_num))
end
end
local t = sysbench.sql.type
local stmt_defs = {
point_selects = {
"SELECT c FROM sbtest%u WHERE id=?",
t.INT},
simple_ranges = {
"SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ?",
t.INT, t.INT},
sum_ranges = {
"SELECT SUM(k) FROM sbtest%u WHERE id BETWEEN ? AND ?",
t.INT, t.INT},
order_ranges = {
"SELECT c FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY c",
t.INT, t.INT},
distinct_ranges = {
"SELECT DISTINCT c FROM sbtest%u WHERE id BETWEEN ? AND ? ORDER BY c",
t.INT, t.INT},
index_updates = {
"UPDATE sbtest%u SET k=k+1 WHERE id=?",
t.INT},
non_index_updates = {
"UPDATE sbtest%u SET c=? WHERE id=?",
{t.CHAR, 120}, t.INT},
deletes = {
"DELETE FROM sbtest%u WHERE id=?",
t.INT},
inserts = {
"INSERT INTO sbtest%u (id, k, c, pad) VALUES (?, ?, ?, ?)",
t.INT, t.INT, {t.CHAR, 120}, {t.CHAR, 60}},
}
function prepare_begin()
stmt.begin = con:prepare("BEGIN")
end
function prepare_commit()
stmt.commit = con:prepare("COMMIT")
end
function prepare_for_each_table(key)
for t = 1, sysbench.opt.tables do
stmt[t][key] = con:prepare(string.format(stmt_defs[key][1], t))
local nparam = #stmt_defs[key] - 1
if nparam > 0 then
param[t][key] = {}
end
for p = 1, nparam do
local btype = stmt_defs[key][p+1]
local len
if type(btype) == "table" then
len = btype[2]
btype = btype[1]
end
if btype == sysbench.sql.type.VARCHAR or
btype == sysbench.sql.type.CHAR then
param[t][key][p] = stmt[t][key]:bind_create(btype, len)
else
param[t][key][p] = stmt[t][key]:bind_create(btype)
end
end
if nparam > 0 then
stmt[t][key]:bind_param(unpack(param[t][key]))
end
end
end
function prepare_point_selects()
prepare_for_each_table("point_selects")
end
function prepare_simple_ranges()
prepare_for_each_table("simple_ranges")
end
function prepare_sum_ranges()
prepare_for_each_table("sum_ranges")
end
function prepare_order_ranges()
prepare_for_each_table("order_ranges")
end
function prepare_distinct_ranges()
prepare_for_each_table("distinct_ranges")
end
function prepare_index_updates()
prepare_for_each_table("index_updates")
end
function prepare_non_index_updates()
prepare_for_each_table("non_index_updates")
end
function prepare_delete_inserts()
prepare_for_each_table("deletes")
prepare_for_each_table("inserts")
end
function thread_init()
drv = sysbench.sql.driver()
con = drv:connect()
res, redis_con = redis_connection()
if not res then
print("failed to connect redis")
return
end
-- Create global nested tables for prepared statements and their
-- parameters. We need a statement and a parameter set for each combination
-- of connection/table/query
stmt = {}
param = {}
for t = 1, sysbench.opt.tables do
stmt[t] = {}
param[t] = {}
end
-- This function is a 'callback' defined by individual benchmark scripts
prepare_statements()
end
-- Close prepared statements
function close_statements()
for t = 1, sysbench.opt.tables do
for k, s in pairs(stmt[t]) do
stmt[t][k]:close()
end
end
if (stmt.begin ~= nil) then
stmt.begin:close()
end
if (stmt.commit ~= nil) then
stmt.commit:close()
end
end
function thread_done()
close_statements()
con:disconnect()
end
function cleanup()
local drv = sysbench.sql.driver()
local con = drv:connect()
for i = 1, sysbench.opt.tables do
print(string.format("Dropping table 'sbtest%d'...", i))
con:query("DROP TABLE IF EXISTS sbtest" .. i )
end
local res, redis_con = redis_connection()
if not res then
print("failed to connect redis")
return
end
redis_cleanup(redis_con)
end
local function get_table_num()
return sysbench.rand.uniform(1, sysbench.opt.tables)
end
local function get_id()
return sysbench.rand.default(1, sysbench.opt.table_size)
end
function begin()
stmt.begin:execute()
end
function commit()
stmt.commit:execute()
end
function execute_point_selects()
local tnum = get_table_num()
local i
local id
for i = 1, sysbench.opt.point_selects do
id = get_id()
param[tnum].point_selects[1]:set(id)
local redis_idx = id % 4
value = redis_con[redis_idx]:get(tnum .. ":" .. id)
if value == nil then
stmt[tnum].point_selects:execute()
--print("get data from db, redis_idx is " .. redis_idx .. ", key is " .. tnum .. ":" .. id)
else
stmt[tnum].point_selects:counter_inc(1)
end
end
end
local function execute_range(key)
local tnum = get_table_num()
for i = 1, sysbench.opt[key] do
local id = get_id()
param[tnum][key][1]:set(id)
param[tnum][key][2]:set(id + sysbench.opt.range_size - 1)
stmt[tnum][key]:execute()
end
end
function execute_simple_ranges()
execute_range("simple_ranges")
end
function execute_sum_ranges()
execute_range("sum_ranges")
end
function execute_order_ranges()
execute_range("order_ranges")
end
function execute_distinct_ranges()
execute_range("distinct_ranges")
end
function split(str,reps)
local resultStrList = {}
string.gsub(str,'[^'..reps..']+',function (w)
table.insert(resultStrList,w)
end)
return resultStrList
end
function execute_index_updates()
local tnum = get_table_num()
local id
for i = 1, sysbench.opt.index_updates do
id = get_id()
param[tnum].index_updates[1]:set(id)
stmt[tnum].index_updates:execute()
local redis_idx = id % 4
value = redis_con[redis_idx]:get(tnum .. ":" .. id)
if value ~= nil then
local sp = split(value, ',')
local k = tonumber(sp[2]) + 1
local new_value = sp[1] .. "," .. k .. "," .. sp[3] .. "," ..sp[4]
redis_con[redis_idx]:set(tnum .. ":" .. id, new_value)
end
end
end
function execute_non_index_updates()
local tnum = get_table_num()
local id
for i = 1, sysbench.opt.non_index_updates do
id = get_id()
local c_val = sb_rand_str(c_value_template)
param[tnum].non_index_updates[1]:set(c_val)
param[tnum].non_index_updates[2]:set(id)
stmt[tnum].non_index_updates:execute()
local redis_idx = id % 4
value = redis_con[redis_idx]:get(tnum .. ":" .. id)
if value ~= nil then
local sp = split(value, ',')
local new_value = sp[1] .. "," .. sp[2] .. "," .. c_val .. "," ..sp[4]
redis_con[redis_idx]:set(tnum .. ":" .. id, new_value)
end
end
end
function execute_delete_inserts()
local tnum = get_table_num()
for i = 1, sysbench.opt.delete_inserts do
local id = get_id()
local k = get_id()
param[tnum].deletes[1]:set(id)
stmt[tnum].deletes:execute()
local redis_idx = id % 4
redis_con[redis_idx]:del(tnum .. ":" .. id)
local c_val = sb_rand_str(c_value_template)
local pad_val = sb_rand_str(pad_value_template)
param[tnum].inserts[1]:set(id)
param[tnum].inserts[2]:set(k)
param[tnum].inserts[3]:set(c_val)
param[tnum].inserts[4]:set(pad_val)
stmt[tnum].inserts:execute()
local new_value = id .. "," .. k .. "," .. c_val .. "," .. pad_val
redis_con[redis_idx]:set(tnum .. ":" .. id, new_value)
end
end
-- Re-prepare statements if we have reconnected, which is possible when some of
-- the listed error codes are in the --mysql-ignore-errors list
function sysbench.hooks.before_restart_event(errdesc)
if errdesc.sql_errno == 2013 or -- CR_SERVER_LOST
errdesc.sql_errno == 2055 or -- CR_SERVER_LOST_EXTENDED
errdesc.sql_errno == 2006 or -- CR_SERVER_GONE_ERROR
errdesc.sql_errno == 2011 -- CR_TCP_CONNECTION
then
close_statements()
prepare_statements()
end
end

View File

@ -19,7 +19,7 @@
-- Read/Write OLTP benchmark
-- ----------------------------------------------------------------------
require("oltp_common")
require("oltp_common_redis")
function prepare_statements()
if not sysbench.opt.skip_trx then
@ -29,13 +29,6 @@ function prepare_statements()
prepare_point_selects()
if sysbench.opt.range_selects then
prepare_simple_ranges()
prepare_sum_ranges()
prepare_order_ranges()
prepare_distinct_ranges()
end
prepare_index_updates()
prepare_non_index_updates()
prepare_delete_inserts()
@ -48,18 +41,12 @@ function event()
execute_point_selects()
if sysbench.opt.range_selects then
execute_simple_ranges()
execute_sum_ranges()
execute_order_ranges()
execute_distinct_ranges()
end
execute_index_updates()
execute_non_index_updates()
execute_delete_inserts()
if not sysbench.opt.skip_trx then
commit()
commit()
end
end

View File

@ -178,7 +178,8 @@ static int sb_lua_db_execute(lua_State *);
static int sb_lua_db_close(lua_State *);
static int sb_lua_db_store_results(lua_State *);
static int sb_lua_db_free_results(lua_State *);
static int sb_lua_db_counter_inc(lua_State *);
static unsigned int sb_lua_table_size(lua_State *, int);
static int read_cmdline_options(lua_State *L);
@ -731,7 +732,8 @@ static lua_State *sb_lua_new_state(void)
sb_lua_var_func(L, "close", sb_lua_db_close);
sb_lua_var_func(L, "store_results", sb_lua_db_store_results);
sb_lua_var_func(L, "free_results", sb_lua_db_free_results);
sb_lua_var_func(L, "counter_inc", sb_lua_db_counter_inc);
sb_lua_var_number(L, "DB_ERROR_NONE", DB_ERROR_NONE);
sb_lua_var_number(L, "DB_ERROR_RESTART_TRANSACTION", DB_ERROR_IGNORABLE);
sb_lua_var_number(L, "DB_ERROR_FAILED", DB_ERROR_FATAL);
@ -1355,6 +1357,20 @@ unsigned int sb_lua_table_size(lua_State *L, int index)
return i;
}
int sb_lua_db_counter_inc(lua_State *L)
{
int cnt;
size_t len = 0;
check_connection(L, &tls_lua_ctxt);
db_conn_t* const con = tls_lua_ctxt.con;
cnt = luaL_checkinteger(L, 1 &len);
sb_counter_inc(con->thread_id, cnt);
luaL_error(L, "counter inc cnt=%d", cnt);
return 0;
}
/* Check if a specified hook exists */
static bool sb_lua_hook_defined(lua_State *L, const char *name)