forked from TencentOS/TencentOS-kernel
tqos/rqm: Tencent Quality Monitor Buffer
Providing back up buffer for Quality Monitor, can be used to catch key context when abnormal jitters occur. And application can also use it to detect system env exception. Signed-off-by: Bauerchen <bauerchen@tencent.com> Reviewed-by: Jiang Biao <benbjiang@tencent.com> Reviewed-by: Bin Lai <robinlai@tencent.com>
This commit is contained in:
Bauerchen
kaixuxiakx
parent
589e8a96bc
commit
7600f2ca44
|
|
@ -21,6 +21,7 @@
|
|||
#include <linux/workqueue.h>
|
||||
#include <linux/bpf-cgroup.h>
|
||||
#include <linux/psi_types.h>
|
||||
#include <linux/mbuf.h>
|
||||
|
||||
#ifdef CONFIG_CGROUPS
|
||||
|
||||
|
|
@ -487,6 +488,9 @@ struct cgroup {
|
|||
/* Used to store internal freezer state */
|
||||
struct cgroup_freezer_state freezer;
|
||||
|
||||
/* cgroup log subsys*/
|
||||
struct mbuf_slot *mbuf;
|
||||
|
||||
/* ids of the ancestors at each level including self */
|
||||
int ancestor_ids[];
|
||||
};
|
||||
|
|
|
|||
|
|
@ -959,4 +959,5 @@ static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
|
|||
|
||||
#endif /* CONFIG_CGROUP_BPF */
|
||||
|
||||
ssize_t mbuf_print(struct cgroup *cgrp, const char *fmt, ...);
|
||||
#endif /* _LINUX_CGROUP_H */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
/*
|
||||
* Copyright (C) 2021 bauerchen <bauerchen@tencent.com>
|
||||
*/
|
||||
|
||||
#include <linux/cgroup.h>
|
||||
#include <linux/rwsem.h>
|
||||
|
||||
#ifndef _CGROUP_MBUF_H
|
||||
#define _CGROUP_MBUF_H
|
||||
|
||||
struct mbuf_struct {
|
||||
u32 mbuf_len;
|
||||
u32 mbuf_max_slots;
|
||||
u32 mbuf_frees;
|
||||
u32 mbuf_next_id;
|
||||
u32 mbuf_size_per_cg;
|
||||
spinlock_t mbuf_lock;
|
||||
char *mbuf;
|
||||
unsigned long *mbuf_bitmap;
|
||||
};
|
||||
|
||||
struct mbuf_ring_desc {
|
||||
/* timestamp of this message */
|
||||
u64 ts_ns;
|
||||
/* message total len, ring_item + ->len = next_item */
|
||||
u16 len;
|
||||
/* text len text_len + sizeof(ring) = len */
|
||||
u16 text_len;
|
||||
};
|
||||
|
||||
struct mbuf_ring {
|
||||
u32 base_idx;
|
||||
u32 first_idx;
|
||||
u64 first_seq;
|
||||
u32 next_idx;
|
||||
u64 next_seq;
|
||||
u32 end_idx;
|
||||
};
|
||||
|
||||
struct mbuf_user_desc {
|
||||
u64 user_seq;
|
||||
u32 user_idx;
|
||||
char buf[1024];
|
||||
};
|
||||
|
||||
/* each cgroup has a mbuf_slot struct */
|
||||
struct mbuf_slot {
|
||||
u32 idx;
|
||||
/* write op must hold this lock */
|
||||
spinlock_t slot_lock;
|
||||
/* rate limit */
|
||||
struct ratelimit_state ratelimit;
|
||||
struct cgroup *owner;
|
||||
const struct mbuf_operations *ops;
|
||||
struct mbuf_ring *mring;
|
||||
struct mbuf_user_desc *udesc;
|
||||
};
|
||||
|
||||
struct mbuf_operations {
|
||||
/* read message */
|
||||
ssize_t (*read) (struct mbuf_slot *, struct mbuf_user_desc *);
|
||||
/* get next available idx */
|
||||
u32 (*next) (struct mbuf_ring *, u32);
|
||||
/* write message */
|
||||
ssize_t (*write) (struct cgroup *, const char *, va_list);
|
||||
} ____cacheline_aligned;
|
||||
|
||||
|
||||
void __init mbuf_bmap_init(void);
|
||||
void __init setup_mbuf(void);
|
||||
struct mbuf_slot *mbuf_slot_alloc(struct cgroup *cg);
|
||||
void mbuf_free(struct cgroup *cg);
|
||||
#endif
|
||||
|
|
@ -46,6 +46,7 @@
|
|||
#include <linux/cpu.h>
|
||||
#include <linux/cpuset.h>
|
||||
#include <linux/cgroup.h>
|
||||
#include <linux/mbuf.h>
|
||||
#include <linux/efi.h>
|
||||
#include <linux/tick.h>
|
||||
#include <linux/sched/isolation.h>
|
||||
|
|
@ -623,11 +624,13 @@ asmlinkage __visible void __init start_kernel(void)
|
|||
* kmem_cache_init()
|
||||
*/
|
||||
setup_log_buf(0);
|
||||
setup_mbuf();
|
||||
vfs_caches_init_early();
|
||||
sort_main_extable();
|
||||
trap_init();
|
||||
mm_init();
|
||||
|
||||
mbuf_bmap_init();
|
||||
ftrace_init();
|
||||
|
||||
/* trace_printk can be enabled here */
|
||||
|
|
|
|||
|
|
@ -6,3 +6,4 @@ obj-$(CONFIG_CGROUP_PIDS) += pids.o
|
|||
obj-$(CONFIG_CGROUP_RDMA) += rdma.o
|
||||
obj-$(CONFIG_CPUSETS) += cpuset.o
|
||||
obj-$(CONFIG_CGROUP_DEBUG) += debug.o
|
||||
obj-y += mbuf.o
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
#include <linux/pid_namespace.h>
|
||||
#include <linux/cgroupstats.h>
|
||||
#include <linux/fs_parser.h>
|
||||
|
||||
#include <trace/events/cgroup.h>
|
||||
|
||||
#define cg_invalf(fc, fmt, ...) invalf(fc, fmt, ## __VA_ARGS__)
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@
|
|||
#include <linux/sched/cputime.h>
|
||||
#include <linux/psi.h>
|
||||
#include <net/sock.h>
|
||||
#include <linux/mbuf.h>
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/cgroup.h>
|
||||
|
|
@ -180,6 +181,8 @@ static u16 cgrp_dfl_threaded_ss_mask;
|
|||
LIST_HEAD(cgroup_roots);
|
||||
static int cgroup_root_count;
|
||||
|
||||
int sysctl_qos_mbuf_enable = 0;
|
||||
|
||||
/* hierarchy ID allocation and mapping, protected by cgroup_mutex */
|
||||
static DEFINE_IDR(cgroup_hierarchy_idr);
|
||||
|
||||
|
|
@ -3683,6 +3686,107 @@ static void cgroup_pressure_release(struct kernfs_open_file *of)
|
|||
}
|
||||
#endif /* CONFIG_PSI */
|
||||
|
||||
static void *mbuf_start(struct seq_file *s, loff_t *pos)
|
||||
{
|
||||
struct cgroup *cgrp = seq_css(s)->cgroup;
|
||||
struct mbuf_slot *mb = cgrp->mbuf;
|
||||
u32 index;
|
||||
|
||||
if (!mb)
|
||||
return NULL;
|
||||
|
||||
index = *pos;
|
||||
/* If already reach end, just return */
|
||||
if (index && index == mb->mring->next_idx)
|
||||
return NULL;
|
||||
|
||||
if (!mb->udesc) {
|
||||
mb->udesc = kmalloc(sizeof(struct mbuf_user_desc), GFP_KERNEL);
|
||||
|
||||
if (!mb->udesc)
|
||||
goto out;
|
||||
|
||||
mb->udesc->user_idx = mb->mring->first_idx;
|
||||
mb->udesc->user_seq = mb->mring->first_seq;
|
||||
}
|
||||
|
||||
/* Maybe reach end or empty */
|
||||
if (mb->udesc->user_idx == mb->mring->next_idx)
|
||||
return NULL;
|
||||
|
||||
out:
|
||||
return mb->udesc;
|
||||
}
|
||||
|
||||
static void *mbuf_next(struct seq_file *s, void *v, loff_t *pos)
|
||||
{
|
||||
struct mbuf_user_desc *udesc = (struct mbuf_user_desc *)v;
|
||||
struct cgroup *cgrp = seq_css(s)->cgroup;
|
||||
struct mbuf_slot *mb = cgrp->mbuf;
|
||||
|
||||
udesc->user_idx = mb->ops->next(mb->mring, udesc->user_idx);
|
||||
*pos = udesc->user_idx;
|
||||
|
||||
if (udesc->user_idx == mb->mring->next_idx)
|
||||
return NULL;
|
||||
|
||||
return udesc;
|
||||
}
|
||||
|
||||
static void mbuf_stop(struct seq_file *s, void *v)
|
||||
{
|
||||
struct cgroup *cgrp = seq_css(s)->cgroup;
|
||||
struct mbuf_user_desc *desc;
|
||||
|
||||
if (cgrp->mbuf) {
|
||||
desc = cgrp->mbuf->udesc;
|
||||
if(desc && desc->user_idx == cgrp->mbuf->mring->next_idx) {
|
||||
kfree(cgrp->mbuf->udesc);
|
||||
cgrp->mbuf->udesc = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int mbuf_show(struct seq_file *s, void *v)
|
||||
{
|
||||
ssize_t ret;
|
||||
struct mbuf_user_desc *udesc = (struct mbuf_user_desc *)v;
|
||||
struct cgroup *cgrp = seq_css(s)->cgroup;
|
||||
struct mbuf_slot *mb = cgrp->mbuf;
|
||||
|
||||
memset(udesc->buf, 0, sizeof(udesc->buf));
|
||||
ret = mb->ops->read(mb, udesc);
|
||||
|
||||
if (ret > 0)
|
||||
seq_printf(s, "%s", udesc->buf);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
ssize_t mbuf_print(struct cgroup *cgrp, const char *fmt, ...)
|
||||
{
|
||||
struct mbuf_slot *mb;
|
||||
va_list args;
|
||||
|
||||
mb = cgrp->mbuf;
|
||||
if(!sysctl_qos_mbuf_enable || !mb)
|
||||
goto out;
|
||||
|
||||
if(!__ratelimit(&mb->ratelimit)) {
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (mb->ops) {
|
||||
va_start(args, fmt);
|
||||
mb->ops->write(cgrp, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
out:
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(mbuf_print);
|
||||
|
||||
static int cgroup_freeze_show(struct seq_file *seq, void *v)
|
||||
{
|
||||
struct cgroup *cgrp = seq_css(seq)->cgroup;
|
||||
|
|
@ -4941,6 +5045,14 @@ static struct cftype cgroup_base_files[] = {
|
|||
.release = cgroup_pressure_release,
|
||||
},
|
||||
#endif /* CONFIG_PSI */
|
||||
{
|
||||
.name = "mbuf",
|
||||
.flags = CFTYPE_NOT_ON_ROOT,
|
||||
.seq_show = mbuf_show,
|
||||
.seq_start = mbuf_start,
|
||||
.seq_next = mbuf_next,
|
||||
.seq_stop = mbuf_stop,
|
||||
},
|
||||
{ } /* terminate */
|
||||
};
|
||||
|
||||
|
|
@ -5258,6 +5370,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
|
|||
cgrp->self.parent = &parent->self;
|
||||
cgrp->root = root;
|
||||
cgrp->level = level;
|
||||
cgrp->mbuf = NULL;
|
||||
|
||||
ret = psi_cgroup_alloc(cgrp);
|
||||
if (ret)
|
||||
|
|
@ -5368,6 +5481,33 @@ fail:
|
|||
return ret;
|
||||
}
|
||||
|
||||
/* mbuf only support cpu/memory/io/net cgroup */
|
||||
static inline bool cgroup_need_mbuf(struct cgroup *cgrp)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_CGROUP_SCHED)
|
||||
if (cgroup_css(cgrp, cgroup_subsys[cpu_cgrp_id]))
|
||||
return true;
|
||||
#endif
|
||||
|
||||
#if IS_ENABLED(CONFIG_BLK_CGROUP)
|
||||
if (cgroup_css(cgrp, cgroup_subsys[io_cgrp_id]))
|
||||
return true;
|
||||
#endif
|
||||
|
||||
#if IS_ENABLED(CONFIG_MEMCG)
|
||||
if (cgroup_css(cgrp, cgroup_subsys[memory_cgrp_id]))
|
||||
return true;
|
||||
#endif
|
||||
|
||||
#if IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)
|
||||
if (cgroup_css(cgrp, cgroup_subsys[net_cls_cgrp_id]))
|
||||
return true;
|
||||
#endif
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
|
||||
{
|
||||
struct cgroup *parent, *cgrp;
|
||||
|
|
@ -5406,7 +5546,6 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
|
|||
* that @cgrp->kn is always accessible.
|
||||
*/
|
||||
kernfs_get(kn);
|
||||
|
||||
ret = cgroup_kn_set_ugid(kn);
|
||||
if (ret)
|
||||
goto out_destroy;
|
||||
|
|
@ -5419,6 +5558,9 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
|
|||
if (ret)
|
||||
goto out_destroy;
|
||||
|
||||
if(sysctl_qos_mbuf_enable && cgroup_on_dfl(cgrp) && cgroup_need_mbuf(cgrp))
|
||||
cgrp->mbuf = mbuf_slot_alloc(cgrp);
|
||||
|
||||
TRACE_CGROUP_PATH(mkdir, cgrp);
|
||||
|
||||
/* let's create and online css's */
|
||||
|
|
@ -5601,6 +5743,9 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
|
|||
|
||||
cgroup_bpf_offline(cgrp);
|
||||
|
||||
if (cgrp->mbuf)
|
||||
mbuf_free(cgrp);
|
||||
|
||||
/* put the base reference */
|
||||
percpu_ref_kill(&cgrp->self.refcnt);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,419 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/*
|
||||
* Quality Monitor Buffer
|
||||
* Aim to provide backup buffer for RQM to record critical message.
|
||||
* Could be used to catch critical context when abnormal jitters occur.
|
||||
*
|
||||
* Author: bauerchen <bauerchen@tencent.com>
|
||||
* Copyright (C) 2021 Tencent, Inc
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/cgroup.h>
|
||||
#include <linux/memblock.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <linux/mbuf.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/sched/clock.h>
|
||||
#include <linux/ratelimit.h>
|
||||
|
||||
/* Define max mbuf len is 8M, and min is 2M */
|
||||
#define MBUF_LEN_MAX (1 << 23)
|
||||
#define MBUF_LEN_MIN (1 << 21)
|
||||
#define MBUF_LEN_DEF MBUF_LEN_MIN
|
||||
|
||||
#define MBUF_MSG_LEN_MAX 1024
|
||||
|
||||
/* Monitor buffer support max 1024 items */
|
||||
#define MBUF_SLOTS_MAX 1024
|
||||
#define MBUF_SLOTS_MIN 256
|
||||
#define MBUF_SLOTS_DEF 512
|
||||
|
||||
/* Global mbuf metadata struct */
|
||||
static struct mbuf_struct g_mbuf = {
|
||||
.mbuf_len = MBUF_LEN_DEF,
|
||||
.mbuf_max_slots = MBUF_SLOTS_DEF,
|
||||
.mbuf_next_id = 0,
|
||||
.mbuf_size_per_cg = 0,
|
||||
.mbuf = NULL,
|
||||
.mbuf_bitmap = NULL,
|
||||
};
|
||||
|
||||
static void __init mbuf_len_update(u64 size)
|
||||
{
|
||||
if (size)
|
||||
size = roundup_pow_of_two(size);
|
||||
|
||||
if (size > MBUF_LEN_MAX) {
|
||||
size = (u64)MBUF_LEN_MAX;
|
||||
pr_warn("mbuf: monitor buffer over [ %llu ] is not supported.\n",
|
||||
(u64)MBUF_LEN_MAX);
|
||||
}
|
||||
|
||||
if (size < MBUF_LEN_MIN){
|
||||
size = (u64) MBUF_LEN_MIN;
|
||||
pr_warn("mbuf: monitor buffer less [ %llu ] is not supported.\n",
|
||||
(u64) MBUF_LEN_MIN);
|
||||
}
|
||||
|
||||
g_mbuf.mbuf_len = size;
|
||||
}
|
||||
|
||||
static int __init mbuf_len_setup(char *str)
|
||||
{
|
||||
|
||||
u64 size;
|
||||
|
||||
if (!str)
|
||||
return -EINVAL;
|
||||
|
||||
size = memparse(str, &str);
|
||||
|
||||
mbuf_len_update(size);
|
||||
|
||||
return 0;
|
||||
|
||||
}
|
||||
early_param("mbuf_len", mbuf_len_setup);
|
||||
|
||||
static int __init mbuf_max_items_setup(char *str)
|
||||
{
|
||||
int num;
|
||||
|
||||
if (!str)
|
||||
return -EINVAL;
|
||||
|
||||
if (!get_option(&str, &num))
|
||||
return -EINVAL;
|
||||
|
||||
if (num)
|
||||
num = roundup_pow_of_two(num);
|
||||
|
||||
if (num > MBUF_SLOTS_MAX)
|
||||
num = MBUF_SLOTS_MAX;
|
||||
|
||||
if (num < MBUF_SLOTS_MIN)
|
||||
num = MBUF_SLOTS_MIN;
|
||||
|
||||
g_mbuf.mbuf_max_slots = num;
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_param("mbuf_max_items", mbuf_max_items_setup);
|
||||
|
||||
/* Alloc mbuf global bitmap, each bit stands for a mbuf slot. */
|
||||
void __init mbuf_bmap_init(void)
|
||||
{
|
||||
size_t alloc_size;
|
||||
void *mbuf_bitmap;
|
||||
|
||||
alloc_size = max_t(size_t, g_mbuf.mbuf_max_slots / BITS_PER_BYTE + 1,
|
||||
L1_CACHE_BYTES);
|
||||
mbuf_bitmap = kmalloc(alloc_size, __GFP_HIGH|__GFP_ZERO);
|
||||
|
||||
if(!mbuf_bitmap){
|
||||
pr_err("mbuf: alloc mbuf_bitmap failed!\n");
|
||||
return;
|
||||
}
|
||||
g_mbuf.mbuf_bitmap = mbuf_bitmap;
|
||||
g_mbuf.mbuf_size_per_cg = g_mbuf.mbuf_len / g_mbuf.mbuf_max_slots;
|
||||
}
|
||||
|
||||
/* Called by start_kernel() */
|
||||
void __init setup_mbuf(void)
|
||||
{
|
||||
/* mbuf has been alloced */
|
||||
if (g_mbuf.mbuf)
|
||||
return;
|
||||
|
||||
g_mbuf.mbuf = memblock_alloc(g_mbuf.mbuf_len, PAGE_SIZE);
|
||||
if (unlikely(!g_mbuf.mbuf)) {
|
||||
pr_err("mbuf: memblock_alloc [ %u ] bytes failed\n",
|
||||
g_mbuf.mbuf_len);
|
||||
return;
|
||||
}
|
||||
|
||||
g_mbuf.mbuf_frees = g_mbuf.mbuf_max_slots;
|
||||
spin_lock_init(&g_mbuf.mbuf_lock);
|
||||
|
||||
pr_info("mbuf: mbuf_len:%u\n", g_mbuf.mbuf_len);
|
||||
}
|
||||
|
||||
/* Get mbuf ring desc text pointer */
|
||||
static char *mbuf_text(struct mbuf_ring_desc *desc)
|
||||
{
|
||||
return (char *)desc + sizeof(struct mbuf_ring_desc);
|
||||
}
|
||||
|
||||
/* Get next mbuf_slot idx */
|
||||
static u32 mbuf_next(struct mbuf_ring *mring, u32 curr_idx)
|
||||
{
|
||||
struct mbuf_ring_desc *cdesc, *ndesc;
|
||||
u32 frees, next_idx;
|
||||
|
||||
cdesc = (struct mbuf_ring_desc *)(g_mbuf.mbuf + curr_idx);
|
||||
next_idx = curr_idx + cdesc->len;
|
||||
/*
|
||||
* If frees are not enough to store mbuf_ring_desc struct,
|
||||
* just goto head
|
||||
*/
|
||||
frees = mring->end_idx - next_idx;
|
||||
if(frees < sizeof(struct mbuf_ring_desc)){
|
||||
next_idx = mring->base_idx;
|
||||
goto next;
|
||||
}
|
||||
|
||||
ndesc = (struct mbuf_ring_desc *)(g_mbuf.mbuf + next_idx);
|
||||
if (!ndesc->len && next_idx != mring->next_idx)
|
||||
next_idx = mring->base_idx;
|
||||
|
||||
next:
|
||||
return next_idx;
|
||||
}
|
||||
|
||||
static inline struct mbuf_ring_desc *get_ring_desc_from_idx(
|
||||
struct mbuf_ring *ring, u32 idx)
|
||||
{
|
||||
return (struct mbuf_ring_desc *)(g_mbuf.mbuf + idx);
|
||||
}
|
||||
|
||||
/* Read mbuf message according to its idx */
|
||||
static ssize_t mbuf_read(struct mbuf_slot *mb, struct mbuf_user_desc *udesc)
|
||||
{
|
||||
struct mbuf_ring *mring;
|
||||
struct mbuf_ring_desc *desc;
|
||||
ssize_t ret;
|
||||
size_t i, len, tbuf_len;
|
||||
|
||||
tbuf_len = sizeof(udesc->buf);
|
||||
mring = mb->mring;
|
||||
|
||||
if (udesc->user_seq < mring->first_seq) {
|
||||
udesc->user_seq = mring->first_seq;
|
||||
udesc->user_idx = mring->first_idx;
|
||||
ret = -1;
|
||||
goto out;
|
||||
}
|
||||
|
||||
desc = get_ring_desc_from_idx(mring, udesc->user_idx);
|
||||
|
||||
len = sprintf(udesc->buf, "%llu:", desc->ts_ns);
|
||||
for (i = 0; i < desc->text_len; i++) {
|
||||
unsigned char c = mbuf_text(desc)[i];
|
||||
|
||||
if (c < ' ' || c >= 127 || c == '\\')
|
||||
continue;
|
||||
else
|
||||
udesc->buf[len++] = c;
|
||||
|
||||
if (len >= tbuf_len)
|
||||
break;
|
||||
}
|
||||
|
||||
len = len >= tbuf_len ? tbuf_len - 1 : len;
|
||||
udesc->buf[len] = '\n';
|
||||
udesc->user_seq++;
|
||||
ret = len;
|
||||
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int mbuf_prepare(struct mbuf_ring *mring, u32 msg_size)
|
||||
{
|
||||
u32 frees;
|
||||
|
||||
if (unlikely(msg_size > MBUF_MSG_LEN_MAX)) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
while (mring->first_seq < mring->next_seq) {
|
||||
|
||||
if (mring->first_idx < mring->next_idx)
|
||||
frees = max(mring->end_idx - mring->next_idx,
|
||||
mring->first_idx - mring->base_idx);
|
||||
else
|
||||
frees = mring->first_idx - mring->next_idx;
|
||||
|
||||
if (frees > msg_size)
|
||||
break;
|
||||
|
||||
/* Drop old message until se have enough contiguous space */
|
||||
mring->first_idx = mbuf_next(mring, mring->first_idx);
|
||||
mring->first_seq++;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Write monitor buffer message */
|
||||
static ssize_t do_mbuf_write(struct cgroup *cg, char *buffer, size_t size)
|
||||
{
|
||||
struct mbuf_ring *mring;
|
||||
struct mbuf_ring_desc *desc;
|
||||
size_t len;
|
||||
unsigned long flags;
|
||||
|
||||
if (size >= g_mbuf.mbuf_size_per_cg){
|
||||
pr_err("mbuf: write message need less than [ %u ] bytes\n",
|
||||
g_mbuf.mbuf_size_per_cg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
mring = cg->mbuf->mring;
|
||||
len = sizeof(struct mbuf_ring_desc) + size;
|
||||
|
||||
spin_lock_irqsave(&cg->mbuf->slot_lock, flags);
|
||||
|
||||
if (mbuf_prepare(mring, len)){
|
||||
spin_unlock_irqrestore(&cg->mbuf->slot_lock, flags);
|
||||
pr_err("mbuf: Can not find enough space.\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (mring->next_idx + len >= mring->end_idx) {
|
||||
/* Set remain buffer to 0 if we go to head */
|
||||
memset(g_mbuf.mbuf + mring->next_idx, 0, mring->end_idx - mring->next_idx);
|
||||
mring->next_idx = mring->base_idx;
|
||||
}
|
||||
|
||||
desc = (struct mbuf_ring_desc *)(g_mbuf.mbuf + mring->next_idx);
|
||||
memcpy(mbuf_text(desc), buffer, size);
|
||||
desc->len = size + sizeof(struct mbuf_ring_desc);
|
||||
desc->text_len = size;
|
||||
desc->ts_ns = local_clock();
|
||||
mring->next_idx += desc->len;
|
||||
mring->next_seq++;
|
||||
|
||||
spin_unlock_irqrestore(&cg->mbuf->slot_lock, flags);
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
void mbuf_reset(struct mbuf_ring *mring)
|
||||
{
|
||||
mring->first_idx = mring->base_idx;
|
||||
mring->first_seq = 0;
|
||||
mring->next_idx = mring->base_idx;
|
||||
mring->next_seq = 0;
|
||||
}
|
||||
|
||||
static ssize_t mbuf_write(struct cgroup *cg, const char *fmt, va_list args)
|
||||
{
|
||||
static char buf[MBUF_MSG_LEN_MAX];
|
||||
char *text = buf;
|
||||
size_t t_len;
|
||||
ssize_t ret;
|
||||
/* Store string to buffer */
|
||||
t_len = vscnprintf(text, sizeof(buf), fmt, args);
|
||||
|
||||
/* Write string to mbuf */
|
||||
ret = do_mbuf_write(cg, text, t_len);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
const struct mbuf_operations mbuf_ops = {
|
||||
.read = mbuf_read,
|
||||
.next = mbuf_next,
|
||||
.write = mbuf_write,
|
||||
};
|
||||
|
||||
static int get_next_mbuf_id(unsigned long *addr, u32 start)
|
||||
{
|
||||
u32 index;
|
||||
|
||||
index = find_next_zero_bit(addr, g_mbuf.mbuf_max_slots, start);
|
||||
if (unlikely(index >= g_mbuf.mbuf_max_slots))
|
||||
return g_mbuf.mbuf_max_slots;
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
static void mbuf_slot_init(struct mbuf_slot *mb, struct cgroup *cg, u32 index)
|
||||
{
|
||||
mb->owner = cg;
|
||||
mb->idx = index;
|
||||
mb->ops = &mbuf_ops;
|
||||
spin_lock_init(&mb->slot_lock);
|
||||
ratelimit_default_init(&mb->ratelimit);
|
||||
|
||||
mb->mring = (struct mbuf_ring *)((char *)mb + sizeof(struct mbuf_slot));
|
||||
mb->mring->base_idx = index *
|
||||
g_mbuf.mbuf_size_per_cg + sizeof(struct mbuf_slot) + sizeof(struct mbuf_ring);
|
||||
mb->mring->end_idx = (index + 1) * g_mbuf.mbuf_size_per_cg - 1;
|
||||
|
||||
mbuf_reset(mb->mring);
|
||||
}
|
||||
|
||||
struct mbuf_slot *mbuf_slot_alloc(struct cgroup *cg)
|
||||
{
|
||||
struct mbuf_slot *mb;
|
||||
u32 index = 0;
|
||||
u32 try_times;
|
||||
unsigned long *m_bitmap;
|
||||
unsigned long flags;
|
||||
|
||||
/* If mbuf_bitmap or mbuf not ready, just return NULL. */
|
||||
if (!g_mbuf.mbuf_bitmap || !g_mbuf.mbuf) {
|
||||
pr_warn_ratelimited("mbuf: mbuf bitmap or mbuf pointer is NULL, alloc failed\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
spin_lock_irqsave(&g_mbuf.mbuf_lock, flags);
|
||||
|
||||
if (g_mbuf.mbuf_frees == 0) {
|
||||
pr_warn_ratelimited("mbuf: reached max num, alloc failed\n");
|
||||
spin_unlock_irqrestore(&g_mbuf.mbuf_lock, flags);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Alloc a free mbuf_slot from global mbuf, according mbuf_bitmap */
|
||||
m_bitmap = g_mbuf.mbuf_bitmap;
|
||||
|
||||
try_times = 1;
|
||||
again:
|
||||
index = get_next_mbuf_id(m_bitmap, g_mbuf.mbuf_next_id);
|
||||
|
||||
/* Rescan next avail idx from head if current idx reach end */
|
||||
if (index == g_mbuf.mbuf_max_slots && try_times--) {
|
||||
g_mbuf.mbuf_next_id = 0;
|
||||
goto again;
|
||||
}
|
||||
|
||||
if (unlikely(index == g_mbuf.mbuf_max_slots)) {
|
||||
/*
|
||||
* Just a protection mechanism, its must be a bug
|
||||
* if function reached here.
|
||||
*/
|
||||
pr_warn_ratelimited("mbuf: frees and bitmap not coincident, just return\n");
|
||||
spin_unlock_irqrestore(&g_mbuf.mbuf_lock, flags);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
__set_bit(index, m_bitmap);
|
||||
g_mbuf.mbuf_next_id = index;
|
||||
|
||||
mb = (struct mbuf_slot *)(g_mbuf.mbuf + index * g_mbuf.mbuf_size_per_cg);
|
||||
mbuf_slot_init(mb, cg, index);
|
||||
g_mbuf.mbuf_frees--;
|
||||
|
||||
spin_unlock_irqrestore(&g_mbuf.mbuf_lock, flags);
|
||||
|
||||
return mb;
|
||||
}
|
||||
|
||||
void mbuf_free(struct cgroup *cg)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&g_mbuf.mbuf_lock, flags);
|
||||
|
||||
/* Make current idx the next available buffer */
|
||||
g_mbuf.mbuf_next_id = cg->mbuf->idx;
|
||||
__clear_bit(g_mbuf.mbuf_next_id, g_mbuf.mbuf_bitmap);
|
||||
|
||||
g_mbuf.mbuf_frees++;
|
||||
spin_unlock_irqrestore(&g_mbuf.mbuf_lock, flags);
|
||||
}
|
||||
|
||||
|
|
@ -121,6 +121,7 @@ extern unsigned int sysctl_nr_open_min, sysctl_nr_open_max;
|
|||
extern int sysctl_nr_trim_pages;
|
||||
#endif
|
||||
|
||||
extern int sysctl_qos_mbuf_enable;
|
||||
extern int sysctl_min_epoll_wait_time;
|
||||
extern unsigned int sysctl_softirq_accel_target;
|
||||
extern int sysctl_softirq_accel_mask;
|
||||
|
|
@ -1448,6 +1449,15 @@ static struct ctl_table kern_table[] = {
|
|||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{
|
||||
.procname = "qos_mbuf_enable",
|
||||
.data = &sysctl_qos_mbuf_enable,
|
||||
.maxlen = sizeof(int),
|
||||
.mode = 0600,
|
||||
.proc_handler = proc_dointvec_minmax,
|
||||
.extra1 = SYSCTL_ZERO,
|
||||
.extra2 = SYSCTL_ONE,
|
||||
},
|
||||
{ }
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue