From c7997e144ac227c7c4e808fef41630d4aa3a01bb Mon Sep 17 00:00:00 2001 From: Zhiping du Date: Tue, 12 Oct 2021 17:00:10 +0800 Subject: [PATCH] iavf: update intel ethernet iavf driver to 4.2.7 Signed-off-by: Zhiping Du --- drivers/net/ethernet/intel/iavf/Makefile | 19 +- .../net/ethernet/intel/iavf/Module.supported | 1 + drivers/net/ethernet/intel/iavf/common.mk | 352 +++ drivers/net/ethernet/intel/iavf/iavf.h | 512 ++++- drivers/net/ethernet/intel/iavf/iavf_adminq.h | 4 +- drivers/net/ethernet/intel/iavf/iavf_client.c | 585 ----- drivers/net/ethernet/intel/iavf/iavf_client.h | 169 -- drivers/net/ethernet/intel/iavf/iavf_common.c | 858 ++++++- .../net/ethernet/intel/iavf/iavf_ethtool.c | 307 ++- .../ethernet/intel/iavf/iavf_ethtool_stats.h | 158 ++ drivers/net/ethernet/intel/iavf/iavf_helper.h | 1 + drivers/net/ethernet/intel/iavf/iavf_main.c | 1989 +++++++++++++---- .../net/ethernet/intel/iavf/iavf_prototype.h | 4 +- drivers/net/ethernet/intel/iavf/iavf_ptp.c | 1041 +++++++++ drivers/net/ethernet/intel/iavf/iavf_ptp.h | 80 + drivers/net/ethernet/intel/iavf/iavf_trace.h | 2 +- drivers/net/ethernet/intel/iavf/iavf_txrx.c | 1202 ++++++++-- drivers/net/ethernet/intel/iavf/iavf_txrx.h | 81 +- drivers/net/ethernet/intel/iavf/iavf_type.h | 87 +- .../net/ethernet/intel/iavf/iavf_virtchnl.c | 1454 ++++++++++-- drivers/net/ethernet/intel/iavf/kcompat.c | 211 +- drivers/net/ethernet/intel/iavf/kcompat.h | 871 ++++++-- .../net/ethernet/intel/iavf/kcompat_impl.h | 411 ++++ .../ethernet/intel/iavf/kcompat_rhel_defs.h | 77 + .../ethernet/intel/iavf/kcompat_sles_defs.h | 149 ++ .../ethernet/intel/iavf/kcompat_std_defs.h | 114 + .../ethernet/intel/iavf/kcompat_ubuntu_defs.h | 28 + drivers/net/ethernet/intel/iavf/kcompat_vfd.c | 845 ++++++- drivers/net/ethernet/intel/iavf/kcompat_vfd.h | 50 +- drivers/net/ethernet/intel/iavf/virtchnl.h | 1256 ++++++++++- .../intel/iavf/virtchnl_inline_ipsec.h | 199 +- .../ethernet/intel/iavf/virtchnl_lan_desc.h | 525 +++++ 32 files changed, 11557 insertions(+), 2085 deletions(-) create mode 100644 drivers/net/ethernet/intel/iavf/Module.supported create mode 100644 drivers/net/ethernet/intel/iavf/common.mk delete mode 100644 drivers/net/ethernet/intel/iavf/iavf_client.c delete mode 100644 drivers/net/ethernet/intel/iavf/iavf_client.h create mode 100644 drivers/net/ethernet/intel/iavf/iavf_ptp.c create mode 100644 drivers/net/ethernet/intel/iavf/iavf_ptp.h create mode 100644 drivers/net/ethernet/intel/iavf/kcompat_impl.h create mode 100644 drivers/net/ethernet/intel/iavf/kcompat_rhel_defs.h create mode 100644 drivers/net/ethernet/intel/iavf/kcompat_sles_defs.h create mode 100644 drivers/net/ethernet/intel/iavf/kcompat_std_defs.h create mode 100644 drivers/net/ethernet/intel/iavf/kcompat_ubuntu_defs.h create mode 100644 drivers/net/ethernet/intel/iavf/virtchnl_lan_desc.h diff --git a/drivers/net/ethernet/intel/iavf/Makefile b/drivers/net/ethernet/intel/iavf/Makefile index ca0a73016..3bbb5b400 100644 --- a/drivers/net/ethernet/intel/iavf/Makefile +++ b/drivers/net/ethernet/intel/iavf/Makefile @@ -1,16 +1,17 @@ # SPDX-License-Identifier: GPL-2.0 -# Copyright(c) 2013 - 2018 Intel Corporation. -# -# Makefile for the Intel(R) Ethernet Adaptive Virtual Function (iavf) -# driver -# -# +# Copyright (c) 2013, Intel Corporation. ccflags-y += -I$(src) subdir-ccflags-y += -I$(src) obj-$(CONFIG_IAVF) += iavf.o -iavf-objs := iavf_main.o iavf_ethtool.o iavf_virtchnl.o \ - iavf_txrx.o iavf_common.o iavf_adminq.o iavf_client.o \ - kcompat_vfd.o kcompat.o +iavf-y := iavf_main.o \ + iavf_ethtool.o \ + iavf_virtchnl.o \ + iavf_adminq.o \ + iavf_common.o \ + iavf_txrx.o \ + kcompat.o + +iavf-$(CONFIG_PTP_1588_CLOCK:m=y) += iavf_ptp.o diff --git a/drivers/net/ethernet/intel/iavf/Module.supported b/drivers/net/ethernet/intel/iavf/Module.supported new file mode 100644 index 000000000..1ccaeee86 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/Module.supported @@ -0,0 +1 @@ +iavf.ko external diff --git a/drivers/net/ethernet/intel/iavf/common.mk b/drivers/net/ethernet/intel/iavf/common.mk new file mode 100644 index 000000000..ae2b10e6c --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/common.mk @@ -0,0 +1,352 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2013, Intel Corporation. + +# +# common Makefile rules useful for out-of-tree Linux driver builds +# +# Usage: include common.mk +# +# After including, you probably want to add a minimum_kver_check call +# +# Required Variables: +# DRIVER +# -- Set to the lowercase driver name + +##################### +# Helpful functions # +##################### + +readlink = $(shell readlink -f ${1}) + +# helper functions for converting kernel version to version codes +get_kver = $(or $(word ${2},$(subst ., ,${1})),0) +get_kvercode = $(shell [ "${1}" -ge 0 -a "${1}" -le 255 2>/dev/null ] && \ + [ "${2}" -ge 0 -a "${2}" -le 255 2>/dev/null ] && \ + [ "${3}" -ge 0 -a "${3}" -le 255 2>/dev/null ] && \ + printf %d $$(( ( ${1} << 16 ) + ( ${2} << 8 ) + ( ${3} ) )) ) + +################ +# depmod Macro # +################ + +cmd_depmod = /sbin/depmod $(if ${SYSTEM_MAP_FILE},-e -F ${SYSTEM_MAP_FILE}) \ + $(if $(strip ${INSTALL_MOD_PATH}),-b ${INSTALL_MOD_PATH}) \ + -a ${KVER} + +################ +# dracut Macro # +################ + +cmd_initrd := $(shell \ + if which dracut > /dev/null 2>&1 ; then \ + echo "dracut --force"; \ + elif which update-initramfs > /dev/null 2>&1 ; then \ + echo "update-initramfs -u"; \ + fi ) + +##################### +# Environment tests # +##################### + +DRIVER_UPPERCASE := $(shell echo ${DRIVER} | tr "[:lower:]" "[:upper:]") + +ifeq (,${BUILD_KERNEL}) +BUILD_KERNEL=$(shell uname -r) +endif + +# Kernel Search Path +# All the places we look for kernel source +KSP := /lib/modules/${BUILD_KERNEL}/source \ + /lib/modules/${BUILD_KERNEL}/build \ + /usr/src/linux-${BUILD_KERNEL} \ + /usr/src/linux-$(${BUILD_KERNEL} | sed 's/-.*//') \ + /usr/src/kernel-headers-${BUILD_KERNEL} \ + /usr/src/kernel-source-${BUILD_KERNEL} \ + /usr/src/linux-$(${BUILD_KERNEL} | sed 's/\([0-9]*\.[0-9]*\)\..*/\1/') \ + /usr/src/linux \ + /usr/src/kernels/${BUILD_KERNEL} \ + /usr/src/kernels + +# prune the list down to only values that exist and have an include/linux +# sub-directory. We can't use include/config because some older kernels don't +# have this. +test_dir = $(shell [ -e ${dir}/include/linux ] && echo ${dir}) +KSP := $(foreach dir, ${KSP}, ${test_dir}) + +# we will use this first valid entry in the search path +ifeq (,${KSRC}) + KSRC := $(firstword ${KSP}) +endif + +ifeq (,${KSRC}) + $(warning *** Kernel header files not in any of the expected locations.) + $(warning *** Install the appropriate kernel development package, e.g.) + $(error kernel-devel, for building kernel modules and try again) +else +ifeq (/lib/modules/${BUILD_KERNEL}/source, ${KSRC}) + KOBJ := /lib/modules/${BUILD_KERNEL}/build +else + KOBJ := ${KSRC} +endif +endif + +# Version file Search Path +VSP := ${KOBJ}/include/generated/utsrelease.h \ + ${KOBJ}/include/linux/utsrelease.h \ + ${KOBJ}/include/linux/version.h \ + ${KOBJ}/include/generated/uapi/linux/version.h \ + /boot/vmlinuz.version.h + +# Config file Search Path +CSP := ${KOBJ}/include/generated/autoconf.h \ + ${KOBJ}/include/linux/autoconf.h \ + /boot/vmlinuz.autoconf.h + +# System.map Search Path (for depmod) +MSP := ${KSRC}/System.map \ + /boot/System.map-${BUILD_KERNEL} + +# prune the lists down to only files that exist +test_file = $(shell [ -f ${file} ] && echo ${file}) +VSP := $(foreach file, ${VSP}, ${test_file}) +CSP := $(foreach file, ${CSP}, ${test_file}) +MSP := $(foreach file, ${MSP}, ${test_file}) + + +# and use the first valid entry in the Search Paths +ifeq (,${VERSION_FILE}) + VERSION_FILE := $(firstword ${VSP}) +endif + +ifeq (,${CONFIG_FILE}) + CONFIG_FILE := $(firstword ${CSP}) +endif + +ifeq (,${SYSTEM_MAP_FILE}) + SYSTEM_MAP_FILE := $(firstword ${MSP}) +endif + +ifeq (,$(wildcard ${VERSION_FILE})) + $(error Linux kernel source not configured - missing version header file) +endif + +ifeq (,$(wildcard ${CONFIG_FILE})) + $(error Linux kernel source not configured - missing autoconf.h) +endif + +ifeq (,$(wildcard ${SYSTEM_MAP_FILE})) + $(warning Missing System.map file - depmod will not check for missing symbols during module installation) +endif + +ifneq ($(words $(subst :, ,$(CURDIR))), 1) + $(error Sources directory '$(CURDIR)' cannot contain spaces nor colons. Rename directory or move sources to another path) +endif + +######################## +# Extract config value # +######################## + +get_config_value = $(shell ${CC} -E -dM ${CONFIG_FILE} 2> /dev/null |\ + grep -m 1 ${1} | awk '{ print $$3 }') + +######################## +# Check module signing # +######################## + +CONFIG_MODULE_SIG_ALL := $(call get_config_value,CONFIG_MODULE_SIG_ALL) +CONFIG_MODULE_SIG_FORCE := $(call get_config_value,CONFIG_MODULE_SIG_FORCE) +CONFIG_MODULE_SIG_KEY := $(call get_config_value,CONFIG_MODULE_SIG_KEY) + +SIG_KEY_SP := ${KOBJ}/${CONFIG_MODULE_SIG_KEY} \ + ${KOBJ}/certs/signing_key.pem + +SIG_KEY_FILE := $(firstword $(foreach file, ${SIG_KEY_SP}, ${test_file})) + +# print a warning if the kernel configuration attempts to sign modules but +# the signing key can't be found. +ifneq (${SIG_KEY_FILE},) +warn_signed_modules := : ; +else +warn_signed_modules := +ifeq (${CONFIG_MODULE_SIG_ALL},1) +warn_signed_modules += \ + echo "*** The target kernel has CONFIG_MODULE_SIG_ALL enabled, but" ; \ + echo "*** the signing key cannot be found. Module signing has been" ; \ + echo "*** disabled for this build." ; +endif # CONFIG_MODULE_SIG_ALL=y +ifeq (${CONFIG_MODULE_SIG_FORCE},1) + echo "warning: The target kernel has CONFIG_MODULE_SIG_FORCE enabled," ; \ + echo "warning: but the signing key cannot be found. The module must" ; \ + echo "warning: be signed manually using 'scripts/sign-file'." ; +endif # CONFIG_MODULE_SIG_FORCE +DISABLE_MODULE_SIGNING := Yes +endif + +####################### +# Linux Version Setup # +####################### + +# The following command line parameter is intended for development of KCOMPAT +# against upstream kernels such as net-next which have broken or non-updated +# version codes in their Makefile. They are intended for debugging and +# development purpose only so that we can easily test new KCOMPAT early. If you +# don't know what this means, you do not need to set this flag. There is no +# arcane magic here. + +# Convert LINUX_VERSION into LINUX_VERSION_CODE +ifneq (${LINUX_VERSION},) + LINUX_VERSION_CODE=$(call get_kvercode,$(call get_kver,${LINUX_VERSION},1),$(call get_kver,${LINUX_VERSION},2),$(call get_kver,${LINUX_VERSION},3)) +endif + +# Honor LINUX_VERSION_CODE +ifneq (${LINUX_VERSION_CODE},) + $(warning Forcing target kernel to build with LINUX_VERSION_CODE of ${LINUX_VERSION_CODE}$(if ${LINUX_VERSION}, from LINUX_VERSION=${LINUX_VERSION}). Do this at your own risk.) + KVER_CODE := ${LINUX_VERSION_CODE} + EXTRA_CFLAGS += -DLINUX_VERSION_CODE=${LINUX_VERSION_CODE} +endif + +# Determine SLE_KERNEL_REVISION for SuSE SLE >= 11 (needed by kcompat) +# This assumes SuSE will continue setting CONFIG_LOCALVERSION to the string +# appended to the stable kernel version on which their kernel is based with +# additional versioning information (up to 3 numbers), a possible abbreviated +# git SHA1 commit id and a kernel type, e.g. CONFIG_LOCALVERSION=-1.2.3-default +# or CONFIG_LOCALVERSION=-999.gdeadbee-default +# +# SLE_LOCALVERSION_CODE is also exported to support legacy kcompat.h +# definitions. +ifeq (1,$(call get_config_value,CONFIG_SUSE_KERNEL)) + +ifneq (10,$(call get_config_value,CONFIG_SLE_VERSION)) + + CONFIG_LOCALVERSION := $(call get_config_value,CONFIG_LOCALVERSION) + LOCALVERSION := $(shell echo ${CONFIG_LOCALVERSION} | \ + cut -d'-' -f2 | sed 's/\.g[[:xdigit:]]\{7\}//') + LOCALVER_A := $(shell echo ${LOCALVERSION} | cut -d'.' -f1) + LOCALVER_B := $(shell echo ${LOCALVERSION} | cut -s -d'.' -f2) + LOCALVER_C := $(shell echo ${LOCALVERSION} | cut -s -d'.' -f3) + SLE_LOCALVERSION_CODE := $(shell expr ${LOCALVER_A} \* 65536 + \ + 0${LOCALVER_B} \* 256 + 0${LOCALVER_C}) + EXTRA_CFLAGS += -DSLE_LOCALVERSION_CODE=${SLE_LOCALVERSION_CODE} + EXTRA_CFLAGS += -DSLE_KERNEL_REVISION=${LOCALVER_A} +endif +endif + +EXTRA_CFLAGS += ${CFLAGS_EXTRA} + +# get the kernel version - we use this to find the correct install path +KVER := $(shell ${CC} ${EXTRA_CFLAGS} -E -dM ${VERSION_FILE} | grep UTS_RELEASE | \ + awk '{ print $$3 }' | sed 's/\"//g') + +# assume source symlink is the same as build, otherwise adjust KOBJ +ifneq (,$(wildcard /lib/modules/${KVER}/build)) + ifneq (${KSRC},$(call readlink,/lib/modules/${KVER}/build)) + KOBJ=/lib/modules/${KVER}/build + endif +endif + +ifeq (${KVER_CODE},) + KVER_CODE := $(shell ${CC} ${EXTRA_CFLAGS} -E -dM ${VSP} 2> /dev/null |\ + grep -m 1 LINUX_VERSION_CODE | awk '{ print $$3 }' | sed 's/\"//g') +endif + +# minimum_kver_check +# +# helper function to provide uniform output for different drivers to abort the +# build based on kernel version check. Usage: "$(call minimum_kver_check,2,6,XX)". +define _minimum_kver_check +ifeq (0,$(shell [ ${KVER_CODE} -lt $(call get_kvercode,${1},${2},${3}) ]; echo "$$?")) + $$(warning *** Aborting the build.) + $$(error This driver is not supported on kernel versions older than ${1}.${2}.${3}) +endif +endef +minimum_kver_check = $(eval $(call _minimum_kver_check,${1},${2},${3})) + +################ +# Manual Pages # +################ + +MANSECTION = 7 + +ifeq (,${MANDIR}) + # find the best place to install the man page + MANPATH := $(shell (manpath 2>/dev/null || echo $MANPATH) | sed 's/:/ /g') + ifneq (,${MANPATH}) + # test based on inclusion in MANPATH + test_dir = $(findstring ${dir}, ${MANPATH}) + else + # no MANPATH, test based on directory existence + test_dir = $(shell [ -e ${dir} ] && echo ${dir}) + endif + # our preferred install path + # should /usr/local/man be in here ? + MANDIR := /usr/share/man /usr/man + MANDIR := $(foreach dir, ${MANDIR}, ${test_dir}) + MANDIR := $(firstword ${MANDIR}) +endif +ifeq (,${MANDIR}) + # fallback to /usr/man + MANDIR := /usr/man +endif + +#################### +# CCFLAGS variable # +#################### + +# set correct CCFLAGS variable for kernels older than 2.6.24 +ifeq (0,$(shell [ ${KVER_CODE} -lt $(call get_kvercode,2,6,24) ]; echo $$?)) +CCFLAGS_VAR := EXTRA_CFLAGS +else +CCFLAGS_VAR := ccflags-y +endif + +################# +# KBUILD_OUTPUT # +################# + +# Only set KBUILD_OUTPUT if the real paths of KOBJ and KSRC differ +ifneq ($(call readlink,${KSRC}),$(call readlink,${KOBJ})) +export KBUILD_OUTPUT ?= ${KOBJ} +endif + +############################ +# Module Install Directory # +############################ + +# Default to using updates/drivers/net/ethernet/intel/ path, since depmod since +# v3.1 defaults to checking updates folder first, and only checking kernels/ +# and extra afterwards. We use updates instead of kernel/* due to desire to +# prevent over-writing built-in modules files. +export INSTALL_MOD_DIR ?= updates/drivers/net/ethernet/intel/${DRIVER} + +###################### +# Kernel Build Macro # +###################### + +# kernel build function +# ${1} is the kernel build target +# ${2} may contain any extra rules to pass directly to the sub-make process +# +# This function is expected to be executed by +# @+$(call kernelbuild,,) +# from within a Makefile recipe. +# +# The following variables are expected to be defined for its use: +# GCC_I_SYS -- if set it will enable use of gcc-i-sys.sh wrapper to use -isystem +# CCFLAGS_VAR -- the CCFLAGS variable to set extra CFLAGS +# EXTRA_CFLAGS -- a set of extra CFLAGS to pass into the ccflags-y variable +# KSRC -- the location of the kernel source tree to build against +# DRIVER_UPPERCASE -- the uppercase name of the kernel module, set from DRIVER +# W -- if set, enables the W= kernel warnings options +# C -- if set, enables the C= kernel sparse build options +# +kernelbuild = $(call warn_signed_modules) \ + ${MAKE} $(if ${GCC_I_SYS},CC="${GCC_I_SYS}") \ + ${CCFLAGS_VAR}="${EXTRA_CFLAGS}" \ + -C "${KSRC}" \ + CONFIG_${DRIVER_UPPERCASE}=m \ + $(if ${DISABLE_MODULE_SIGNING},CONFIG_MODULE_SIG=n) \ + $(if ${DISABLE_MODULE_SIGNING},CONFIG_MODULE_SIG_ALL=) \ + M="${CURDIR}" \ + $(if ${W},W="${W}") \ + $(if ${C},C="${C}") \ + ${2} ${1} diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 1b804c22c..4f1852b8b 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -45,6 +45,7 @@ #include "iavf_type.h" #include "virtchnl.h" #include "iavf_txrx.h" +#include "iavf_ptp.h" #include #define DEFAULT_DEBUG_LEVEL_SHIFT 3 @@ -63,9 +64,10 @@ struct iavf_vsi { struct net_device *netdev; #ifdef HAVE_VLAN_RX_REGISTER struct vlan_group *vlgrp; -#else - unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; #endif + unsigned long active_cvlans[BITS_TO_LONGS(VLAN_N_VID)]; + unsigned long active_svlans[BITS_TO_LONGS(VLAN_N_VID)]; + /* dummy pointer - VF plans to add this functionality in the future */ struct iavf_ring **xdp_rings; u16 seid; @@ -74,7 +76,6 @@ struct iavf_vsi { int base_vector; u16 work_limit; u16 qs_handle; - void *priv; /* client driver data reference. */ }; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -98,10 +99,100 @@ struct iavf_vsi { (&(((struct iavf_tx_context_desc *)((R)->desc))[i])) #define IAVF_MAX_REQ_QUEUES 16 +#define IAVF_START_CHNL_TC 1 + #define IAVF_HKEY_ARRAY_SIZE ((IAVF_VFQF_HKEY_MAX_INDEX + 1) * 4) #define IAVF_HLUT_ARRAY_SIZE ((IAVF_VFQF_HLUT_MAX_INDEX + 1) * 4) #define IAVF_MBPS_DIVISOR 125000 /* divisor to convert to Mbps */ +#define IAVF_VIRTCHNL_VF_RESOURCE_SIZE (sizeof(struct virtchnl_vf_resource) + \ + (IAVF_MAX_VF_VSI * \ + sizeof(struct virtchnl_vsi_resource))) + +#define IAVF_NETIF_F_HW_VLAN_BITS +#ifdef NETIF_F_HW_VLAN_CTAG_RX +#define IAVF_NETIF_F_HW_VLAN_CTAG_RX NETIF_F_HW_VLAN_CTAG_RX +#else +#define IAVF_NETIF_F_HW_VLAN_CTAG_RX NETIF_F_HW_VLAN_RX +#endif + +#ifdef NETIF_F_HW_VLAN_CTAG_TX +#define IAVF_NETIF_F_HW_VLAN_CTAG_TX NETIF_F_HW_VLAN_CTAG_TX +#else +#define IAVF_NETIF_F_HW_VLAN_CTAG_TX NETIF_F_HW_VLAN_TX +#endif + +#ifdef NETIF_F_HW_VLAN_CTAG_FILTER +#define IAVF_NETIF_F_HW_VLAN_CTAG_FILTER NETIF_F_HW_VLAN_CTAG_FILTER +#else +#define IAVF_NETIF_F_HW_VLAN_CTAG_FILTER NETIF_F_HW_VLAN_FILTER +#endif + +enum iavf_chnl_vector_state { + IAVF_VEC_IN_BP, + IAVF_VEC_PREV_IN_BP, + IAVF_VEC_ONCE_IN_BP, + IAVF_VEC_PREV_DATA_PKT_RECV, + IAVF_VEC_NBITS, /* This must be last */ +}; + +struct iavf_channel_ex { + atomic_t fd_queue; + u32 fd_cnt_idx; + u16 num_rxq; + u16 base_q; + /* number of filter specific to this channel (aka ADQ TC) */ + u32 num_fltr; +}; + +struct iavf_q_vector_ch_stats { + /* following are used as part of managing driver internal + * state machine. Only to be used for perf debugging. + */ + u64 in_bp; + u64 in_intr; + u64 intr_to_bp; + u64 bp_to_intr; + u64 intr_to_intr; + u64 bp_to_bp; + + /* This counter is used to track real transition of vector from + * BUSY_POLL to INTERRUPT based on enhanced logic (using state + * machine and control packets). + */ + u64 unlikely_cb_to_bp; + /* Tracking "unlikely_cb_bp and once_in_bp is true" */ + u64 ucb_once_in_bp_true; + /* This is used to keep track of enabling interrupt from napi_poll + * when state machine condition indicated once_in_bp is false + */ + u64 intr_once_bp_false; + u64 bp_stop_need_resched; + u64 bp_stop_timeout; + + u64 cleaned_any_data_pkt; + /* busy_poll stop, need_resched is set and did not clean + * any data packet during this previous invocation of napi_poll + */ + u64 need_resched_no_data_pkt; + /* busy_poll stop, need_resched is not set: hence it is inferred as + * possible timeout and did not clean any data packet during this + * previous invocation of napi_poll + */ + u64 timeout_no_data_pkt; + u64 sw_intr_timeout; /* track SW INTR from napi_poll */ + u64 sw_intr_serv_task; /* track SW INTR from service_task */ + /* This keeps track of how many times, bailout when once_in_bp is set, + * unlikely_cb_to_bp is set, but pkt based interrupt optimization + * is OFF + */ + u64 no_sw_intr_opt_off; + /* tracking, how many times WB_ON_ITR is set */ + u64 wb_on_itr_set; + /* keeps track of SW triggered interrupt due to not clean_complete */ + u64 intr_en_not_clean_complete; +}; + /* MAX_MSIX_Q_VECTORS of these are allocated, * but we only use one per queue-specific vector. */ @@ -122,8 +213,151 @@ struct iavf_q_vector { cpumask_t affinity_mask; struct irq_affinity_notify affinity_notify; #endif + /* This tracks current state of vector, BUSY_POLL or INTR */ +#define IAVF_VECTOR_STATE_IN_BP BIT(IAVF_VEC_IN_BP) + /* This tracks prev state of vector, BUSY_POLL or INTR */ +#define IAVF_VECTOR_STATE_PREV_IN_BP BIT(IAVF_VEC_PREV_IN_BP) + /* This tracks state of vector, was the ever in BUSY_POLL. This + * state goes to INTT if interrupt are enabled or SW interrupts + * are triggered from either service_task or napi_poll + */ +#define IAVF_VECTOR_STATE_ONCE_IN_BP BIT(IAVF_VEC_ONCE_IN_BP) + + /* Tracks if previously - were there any data packets received + * on per channel enabled vector or not + */ +#define IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV BIT(IAVF_VEC_PREV_DATA_PKT_RECV) + /* it is used to keep track of various states as defined earlier + * and those states are used during ADQ performance optimization + */ + u8 state_flags; + +#define IAVF_VECTOR_CHNL_PERF_ENA BIT(0) + /* controls packet inspection based optimization is OFF/ON */ +#define IAVF_VECTOR_CHNL_PKT_OPT_ENA BIT(1) + u16 chnl_flags; + + /* Used in logic to determine if SW inter is needed or not. + * This is used only for channel enabled vector + */ + u64 jiffies; + + struct iavf_channel_ex *ch; + struct iavf_q_vector_ch_stats ch_stats; }; +static inline bool vector_pkt_inspect_opt_ena(struct iavf_q_vector *q_vector) +{ + return q_vector->chnl_flags & IAVF_VECTOR_CHNL_PKT_OPT_ENA; +} + +static inline bool vector_ch_ena(struct iavf_q_vector *qv) +{ + return !!qv->ch; +} + +static inline bool vector_ch_perf_ena(struct iavf_q_vector *qv) +{ + return qv->chnl_flags & IAVF_VECTOR_CHNL_PERF_ENA; +} + +/** + * vector_busypoll_intr + * @qv: pointer to q_vector + * + * This function returns true if vector is transitioning from BUSY_POLL + * to INTERRUPT based on current and previous state of vector + */ +static inline bool vector_busypoll_intr(struct iavf_q_vector *qv) +{ + return (qv->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) && + !(qv->state_flags & IAVF_VECTOR_STATE_IN_BP); +} + +/** + * vector_ever_in_busypoll + * @qv: pointer to q_vector + * + * This function returns true if vectors current OR previous state + * is BUSY_POLL + */ +static inline bool vector_ever_in_busypoll(struct iavf_q_vector *qv) +{ + return (qv->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) || + (qv->state_flags & IAVF_VECTOR_STATE_IN_BP); +} + +/** + * vector_state_curr_prev_intr + * @qv: pointer to q_vector + * + * This function returns true if vectors current AND previous state + * is INTERRUPT + */ +static inline bool vector_state_curr_prev_intr(struct iavf_q_vector *qv) +{ + return !(qv->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) && + !(qv->state_flags & IAVF_VECTOR_STATE_IN_BP); +} + +/** + * vector_intr_busypoll + * @qv: pointer to q_vector + * + * This function returns true if vector is transitioning from INTERRUPT + * to BUSY_POLL based on current and previous state of vector + */ +static inline bool vector_intr_busypoll(struct iavf_q_vector *qv) +{ + return !(qv->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) && + (qv->state_flags & IAVF_VECTOR_STATE_IN_BP); +} + +/** + * iavf_inc_napi_sw_intr_counter + * @q_vector: pointer to q_vector + * + * Track software interrupt from napi_poll codeflow. Caller of this + * expected to call iavf_force_wb to actually trigger SW intr. + */ +static inline void +iavf_inc_napi_sw_intr_counter(struct iavf_q_vector *q_vector) +{ + q_vector->ch_stats.sw_intr_timeout++; +} + +/** + * iavf_inc_serv_task_sw_intr_counter + * @q_vector: pointer to q_vector + * + * Track software interrupt from service_task codeflow. Caller of this + * expected to call iavf_force_wb to actually trigger SW intr. + */ +static inline void +iavf_inc_serv_task_sw_intr_counter(struct iavf_q_vector *q_vector) +{ + q_vector->ch_stats.sw_intr_serv_task++; +} + +/** + * iavf_set_wb_on_itr - trigger force write-back by setting WB_ON_ITR bit + * @hw: ptr to HW + * @qv: pointer to vector + * + * This function is used to force write-backs by setting WB_ON_ITR bit + * in DYN_CTLN register. WB_ON_ITR and INTENA are mutually exclusive bits. + * Seting WB_ON_ITR bits means TX and RX descriptors are written back based + * on ITR expiration irrespective of INTENA setting + */ +static inline void +iavf_set_wb_on_itr(struct iavf_hw *hw, struct iavf_q_vector *qv) +{ + qv->ch_stats.wb_on_itr_set++; + wr32(hw, IAVF_VFINT_DYN_CTLN1(qv->reg_idx), + IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | + IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK); +} + /* Helper macros to switch between ints/sec and what the register uses. * And yes, it's the same math going both ways. The lowest value * supported by all of the iavf hardware is 8. @@ -150,11 +384,18 @@ struct iavf_mac_filter { bool is_new_mac; /* filter is new, wait for PF decision */ bool remove; /* filter needs to be removed */ bool add; /* filter needs to be added */ + bool is_primary; /* filter is a default VF MAC */ +}; + +#define IAVF_VLAN(vid, tpid) ((struct iavf_vlan){ vid, tpid }) +struct iavf_vlan { + u16 vid; + u16 tpid; }; struct iavf_vlan_filter { struct list_head list; - u16 vlan; + struct iavf_vlan vlan; bool remove; /* filter needs to be removed */ bool add; /* filter needs to be added */ }; @@ -170,6 +411,7 @@ struct iavf_channel_config { struct virtchnl_channel_info ch_info[VIRTCHNL_MAX_ADQ_V2_CHANNELS]; enum iavf_tc_state_t state; u8 total_qps; + struct iavf_channel_ex ch_ex_info[VIRTCHNL_MAX_ADQ_V2_CHANNELS]; }; /* State of cloud filter */ @@ -186,6 +428,8 @@ enum iavf_state_t { __IAVF_REMOVE, /* driver is being unloaded */ __IAVF_INIT_VERSION_CHECK, /* aq msg sent, awaiting reply */ __IAVF_INIT_GET_RESOURCES, /* aq msg sent, awaiting reply */ + __IAVF_INIT_EXTENDED_CAPS, /* process extended caps which require aq msg exchange */ + __IAVF_INIT_CONFIG_ADAPTER, __IAVF_INIT_SW, /* got resources, setting up structs */ __IAVF_INIT_FAILED, /* init failed, restarting procedure */ __IAVF_RESETTING, /* in reset */ @@ -199,8 +443,8 @@ enum iavf_state_t { enum iavf_critical_section_t { __IAVF_IN_CRITICAL_TASK, /* cannot be interrupted */ - __IAVF_IN_CLIENT_TASK, __IAVF_IN_REMOVE_TASK, /* device being removed */ + __IAVF_TX_TSTAMP_IN_PROGRESS, /* PTP Tx timestamp request in progress */ }; #define IAVF_CLOUD_FIELD_OMAC 0x01 @@ -231,13 +475,17 @@ struct iavf_cloud_filter { unsigned long cookie; bool del; /* filter needs to be deleted */ bool add; /* filter needs to be added */ + struct iavf_channel_ex *ch; }; +#define IAVF_RESET_WAIT_MS 10 +#define IAVF_RESET_WAIT_DETECTED_COUNT 500 +#define IAVF_RESET_WAIT_COMPLETE_COUNT 2000 + /* board specific private data structure */ struct iavf_adapter { struct work_struct adminq_task; struct delayed_work watchdog_task; - struct delayed_work client_task; wait_queue_head_t down_waitqueue; struct iavf_q_vector *q_vectors; struct list_head vlan_filter_list; @@ -245,6 +493,7 @@ struct iavf_adapter { /* Lock to protect accesses to MAC and VLAN lists */ spinlock_t mac_vlan_list_lock; char misc_vector_name[IFNAMSIZ + 9]; + u8 rxdid; int num_active_queues; int num_req_queues; @@ -258,10 +507,6 @@ struct iavf_adapter { u64 hw_csum_rx_error; u32 rx_desc_count; int num_msix_vectors; - int num_iwarp_msix; - int iwarp_base_vector; - u32 client_pending; - struct iavf_client_instance *cinst; struct msix_entry *msix_entries; u32 flags; @@ -270,47 +515,99 @@ struct iavf_adapter { #define IAVF_FLAG_RESET_PENDING BIT(4) #define IAVF_FLAG_RESET_NEEDED BIT(5) #define IAVF_FLAG_WB_ON_ITR_CAPABLE BIT(6) -#define IAVF_FLAG_SERVICE_CLIENT_REQUESTED BIT(9) -#define IAVF_FLAG_CLIENT_NEEDS_OPEN BIT(10) -#define IAVF_FLAG_CLIENT_NEEDS_CLOSE BIT(11) -#define IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS BIT(12) -#define IAVF_FLAG_PROMISC_ON BIT(13) -#define IAVF_FLAG_ALLMULTI_ON BIT(14) #define IAVF_FLAG_LEGACY_RX BIT(15) #define IAVF_FLAG_REINIT_ITR_NEEDED BIT(16) #define IAVF_FLAG_QUEUES_ENABLED BIT(17) #define IAVF_FLAG_QUEUES_DISABLED BIT(18) #define IAVF_FLAG_REINIT_MSIX_NEEDED BIT(20) +#define IAVF_FLAG_REINIT_CHNL_NEEDED BIT(21) +#define IAVF_FLAG_RESET_DETECTED BIT(22) + + + u32 chnl_perf_flags; +#define IAVF_FLAG_CHNL_PKT_OPT_ENA BIT(0) + /* duplicates for common code */ #define IAVF_FLAG_DCB_ENABLED 0 /* flags for admin queue service task */ - u32 aq_required; -#define IAVF_FLAG_AQ_ENABLE_QUEUES BIT(0) -#define IAVF_FLAG_AQ_DISABLE_QUEUES BIT(1) -#define IAVF_FLAG_AQ_ADD_MAC_FILTER BIT(2) -#define IAVF_FLAG_AQ_ADD_VLAN_FILTER BIT(3) -#define IAVF_FLAG_AQ_DEL_MAC_FILTER BIT(4) -#define IAVF_FLAG_AQ_DEL_VLAN_FILTER BIT(5) -#define IAVF_FLAG_AQ_CONFIGURE_QUEUES BIT(6) -#define IAVF_FLAG_AQ_MAP_VECTORS BIT(7) -#define IAVF_FLAG_AQ_HANDLE_RESET BIT(8) -#define IAVF_FLAG_AQ_CONFIGURE_RSS BIT(9) /* direct AQ config */ -#define IAVF_FLAG_AQ_GET_CONFIG BIT(10) + u64 aq_required; +#define IAVF_FLAG_AQ_ENABLE_QUEUES BIT(0) +#define IAVF_FLAG_AQ_DISABLE_QUEUES BIT(1) +#define IAVF_FLAG_AQ_ADD_MAC_FILTER BIT(2) +#define IAVF_FLAG_AQ_ADD_VLAN_FILTER BIT(3) +#define IAVF_FLAG_AQ_DEL_MAC_FILTER BIT(4) +#define IAVF_FLAG_AQ_DEL_VLAN_FILTER BIT(5) +#define IAVF_FLAG_AQ_CONFIGURE_QUEUES BIT(6) +#define IAVF_FLAG_AQ_MAP_VECTORS BIT(7) +#define IAVF_FLAG_AQ_HANDLE_RESET BIT(8) +#define IAVF_FLAG_AQ_CONFIGURE_RSS BIT(9) /* direct AQ config */ +#define IAVF_FLAG_AQ_GET_CONFIG BIT(10) /* Newer style, RSS done by the PF so we can ignore hardware vagaries. */ -#define IAVF_FLAG_AQ_GET_HENA BIT(11) -#define IAVF_FLAG_AQ_SET_HENA BIT(12) -#define IAVF_FLAG_AQ_SET_RSS_KEY BIT(13) -#define IAVF_FLAG_AQ_SET_RSS_LUT BIT(14) -#define IAVF_FLAG_AQ_REQUEST_PROMISC BIT(15) -#define IAVF_FLAG_AQ_RELEASE_PROMISC BIT(16) -#define IAVF_FLAG_AQ_REQUEST_ALLMULTI BIT(17) -#define IAVF_FLAG_AQ_RELEASE_ALLMULTI BIT(18) -#define IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING BIT(19) -#define IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING BIT(20) -#define IAVF_FLAG_AQ_ENABLE_CHANNELS BIT(21) -#define IAVF_FLAG_AQ_DISABLE_CHANNELS BIT(22) -#define IAVF_FLAG_AQ_ADD_CLOUD_FILTER BIT(23) -#define IAVF_FLAG_AQ_DEL_CLOUD_FILTER BIT(24) +#define IAVF_FLAG_AQ_GET_HENA BIT(11) +#define IAVF_FLAG_AQ_SET_HENA BIT(12) +#define IAVF_FLAG_AQ_SET_RSS_KEY BIT(13) +#define IAVF_FLAG_AQ_SET_RSS_LUT BIT(14) +#define IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE BIT(15) +#define IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING BIT(19) +#define IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING BIT(20) +#define IAVF_FLAG_AQ_ENABLE_CHANNELS BIT(21) +#define IAVF_FLAG_AQ_DISABLE_CHANNELS BIT(22) +#define IAVF_FLAG_AQ_ADD_CLOUD_FILTER BIT(23) +#define IAVF_FLAG_AQ_DEL_CLOUD_FILTER BIT(24) +#define IAVF_FLAG_AQ_REQUEST_STATS BIT(25) +#define IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS BIT(26) +#define IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING BIT(27) +#define IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING BIT(28) +#define IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING BIT(29) +#define IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING BIT(30) +#define IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION BIT(31) +#define IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION BIT(32) +#define IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION BIT(33) +#define IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION BIT(34) +#define IAVF_FLAG_AQ_GET_SUPPORTED_RXDIDS BIT(35) +#define IAVF_FLAG_AQ_GET_PTP_CAPS BIT(36) +#define IAVF_FLAG_AQ_SEND_PTP_CMD BIT(37) + + /* AQ messages that must be sent after IAVF_FLAG_AQ_GET_CONFIG, in + * order to negotiated extended capabilities. + */ +#define IAVF_FLAG_AQ_EXTENDED_CAPS \ + (IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS | \ + IAVF_FLAG_AQ_GET_SUPPORTED_RXDIDS | \ + IAVF_FLAG_AQ_GET_PTP_CAPS) + + /* flags for processing extended capability messages during + * __IAVF_INIT_EXTENDED_CAPS. Each capability exchange requires + * both a SEND and a RECV step, which must be processed in sequence. + * + * During the __IAVF_INIT_EXTENDED_CAPS state, the driver will + * process one flag at a time during each state loop. + */ + u64 extended_caps; +#define IAVF_EXTENDED_CAP_SEND_VLAN_V2 BIT(0) +#define IAVF_EXTENDED_CAP_RECV_VLAN_V2 BIT(1) +#define IAVF_EXTENDED_CAP_SEND_RXDID BIT(2) +#define IAVF_EXTENDED_CAP_RECV_RXDID BIT(3) +#define IAVF_EXTENDED_CAP_SEND_PTP BIT(4) +#define IAVF_EXTENDED_CAP_RECV_PTP BIT(5) + +#define IAVF_EXTENDED_CAPS \ + (IAVF_EXTENDED_CAP_SEND_VLAN_V2 | \ + IAVF_EXTENDED_CAP_RECV_VLAN_V2 | \ + IAVF_EXTENDED_CAP_SEND_RXDID | \ + IAVF_EXTENDED_CAP_RECV_RXDID | \ + IAVF_EXTENDED_CAP_SEND_PTP | \ + IAVF_EXTENDED_CAP_RECV_PTP) + + /* Lock to prevent possible clobbering of + * current_netdev_promisc_flags + */ + spinlock_t current_netdev_promisc_flags_lock; +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT + u32 current_netdev_promisc_flags; +#else + netdev_features_t current_netdev_promisc_flags; +#endif /* HAVE_RHEL6_NET_DEVICE_OPS_EXT */ /* OS defined structs */ struct net_device *netdev; @@ -352,25 +649,34 @@ struct iavf_adapter { VIRTCHNL_VF_OFFLOAD_RSS_PF))) #define VLAN_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ VIRTCHNL_VF_OFFLOAD_VLAN) +#define VLAN_V2_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_OFFLOAD_VLAN_V2) +#define VLAN_V2_FILTERING_ALLOWED(_a) \ + (VLAN_V2_ALLOWED((_a)) && \ + ((_a)->vlan_v2_caps.filtering.filtering_support.outer || \ + (_a)->vlan_v2_caps.filtering.filtering_support.inner)) +#define VLAN_FILTERING_ALLOWED(_a) \ + (VLAN_ALLOWED((_a)) || VLAN_V2_FILTERING_ALLOWED((_a))) #ifdef VIRTCHNL_VF_CAP_ADV_LINK_SPEED #define ADV_LINK_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \ VIRTCHNL_VF_CAP_ADV_LINK_SPEED) -#ifdef SPEED_25000 -#define ALL_SPEEDS (SPEED_100000 | SPEED_50000 | SPEED_25000 | SPEED_10000 | \ - SPEED_5000 | SPEED_2500 | SPEED_1000 | SPEED_100 | SPEED_10) -#else -#define ALL_SPEEDS (SPEED_100000 | SPEED_50000 | SPEED_10000 | SPEED_5000 | \ - SPEED_2500 | SPEED_1000 | SPEED_100 | SPEED_10) -#endif -#define SUPPORTED_SPEED(_s) ((_s) & ALL_SPEEDS) #endif /* VIRTCHNL_VF_CAP_ADV_LINK_SPEED */ +#define ADQ_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_OFFLOAD_ADQ) #define ADQ_V2_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ VIRTCHNL_VF_OFFLOAD_ADQ_V2) +#define RXDID_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) +#define PTP_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_CAP_PTP) struct virtchnl_vf_resource *vf_res; /* incl. all VSIs */ struct virtchnl_vsi_resource *vsi_res; /* our LAN VSI */ struct virtchnl_version_info pf_version; #define PF_IS_V11(_a) (((_a)->pf_version.major == 1) && \ ((_a)->pf_version.minor == 1)) + struct virtchnl_vlan_caps vlan_v2_caps; + struct virtchnl_supported_rxdids supported_rxdids; + struct iavf_ptp ptp; u16 msg_enable; struct iavf_eth_stats current_stats; struct iavf_vsi vsi; @@ -387,7 +693,16 @@ struct iavf_adapter { struct list_head cloud_filter_list; /* lock to protect access to the cloud filter list */ spinlock_t cloud_filter_list_lock; + + /* max allowed ADQ filters */ +#define IAVF_MAX_CLOUD_ADQ_FILTERS 128 u16 num_cloud_filters; + /* snapshot of "num_active_queues" before setup_tc for qdisc add + * is invoked. This information is useful during qdisc del flow, + * to restore correct number of queues + */ + int orig_num_active_queues; + #ifdef IAVF_ADD_PROBES u64 tcp_segs; u64 udp_segs; @@ -396,11 +711,13 @@ struct iavf_adapter { u64 tx_sctp_cso; u64 tx_ip4_cso; u64 tx_vlano; + u64 tx_ad_vlano; u64 rx_tcp_cso; u64 rx_udp_cso; u64 rx_sctp_cso; u64 rx_ip4_cso; u64 rx_vlano; + u64 rx_ad_vlano; u64 rx_tcp_cso_err; u64 hw_csum_rx_vxlan; u64 hw_csum_rx_geneve; @@ -413,17 +730,51 @@ struct iavf_adapter { /* Ethtool Private Flags */ -/* lan device, used by client interface */ -struct iavf_device { - struct list_head list; - struct iavf_adapter *vf; -}; - /* needed by iavf_ethtool.c */ extern char iavf_driver_name[]; extern const char iavf_driver_version[]; extern struct workqueue_struct *iavf_wq; +/** + * iavf_is_adq_enabled - adq enabled or not + * @adapter: pointer to adapter + * + * This function returns true based on negotiated capability of ADQ, + * num_tc and channel config state and channel config state is _RUNNING and ADQ + * has been successfully configured + **/ +static inline bool iavf_is_adq_enabled(struct iavf_adapter *adapter) +{ + return (ADQ_ALLOWED(adapter) && + (adapter->num_tc >= IAVF_START_CHNL_TC) && + (adapter->ch_config.state == __IAVF_TC_RUNNING)); +} + +/** + * iavf_is_adq_v2_enabled - adq v2 enabled or not + * @adapter: pointer to adapter + * + * This function returns true based on negotiated capability ADQ_V2 + * if set and basic ADQ enabled + **/ +static inline bool iavf_is_adq_v2_enabled(struct iavf_adapter *adapter) +{ + return (iavf_is_adq_enabled(adapter) && ADQ_V2_ALLOWED(adapter)); +} + +/** + * iavf_chnl_filters_exist - channel filters exists + * @adapter: pointer to adapter + * + * This function returns true if adq_v2_enabled is true and if there + * are active filters otherwise false + **/ +static inline bool iavf_chnl_filters_exist(struct iavf_adapter *adapter) +{ + return (iavf_is_adq_v2_enabled(adapter) && + adapter->num_cloud_filters) ? true : false; +} + static inline void iavf_change_state(struct iavf_adapter *adapter, enum iavf_state_t state) { @@ -445,10 +796,35 @@ static inline bool iavf_is_reset(struct iavf_hw *hw) return !(rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK); } +/** + * iavf_force_wb - Issue SW Interrupt so HW does a wb + * @vsi: the VSI we care about + * @q_vector: the vector on which to force writeback + * + **/ +static inline void iavf_force_wb(struct iavf_vsi *vsi, + struct iavf_q_vector *q_vector) +{ + u32 val = IAVF_VFINT_DYN_CTLN1_INTENA_MASK | + IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */ + IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_MASK | + IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK + /* allow 00 to be written to the index */; + + if (vector_ch_ena(q_vector)) + q_vector->state_flags &= ~IAVF_VECTOR_STATE_ONCE_IN_BP; + + wr32(&vsi->back->hw, + IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx), + val); +} + int iavf_up(struct iavf_adapter *adapter); void iavf_down(struct iavf_adapter *adapter); int iavf_process_config(struct iavf_adapter *adapter); +int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter); void iavf_schedule_reset(struct iavf_adapter *adapter); +void iavf_schedule_request_stats(struct iavf_adapter *adapter); void iavf_reset(struct iavf_adapter *adapter); void iavf_set_ethtool_ops(struct net_device *netdev); void iavf_update_stats(struct iavf_adapter *adapter); @@ -465,6 +841,13 @@ int iavf_send_api_ver(struct iavf_adapter *adapter); int iavf_verify_api_ver(struct iavf_adapter *adapter); int iavf_send_vf_config_msg(struct iavf_adapter *adapter); int iavf_get_vf_config(struct iavf_adapter *adapter); +int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter); +int iavf_send_vf_offload_vlan_v2_msg(struct iavf_adapter *adapter); +int iavf_send_vf_supported_rxdids_msg(struct iavf_adapter *adapter); +int iavf_get_vf_supported_rxdids(struct iavf_adapter *adapter); +int iavf_send_vf_ptp_caps_msg(struct iavf_adapter *adapter); +int iavf_get_vf_ptp_caps(struct iavf_adapter *adapter); +void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter); void iavf_irq_enable(struct iavf_adapter *adapter, bool flush); void iavf_configure_queues(struct iavf_adapter *adapter); void iavf_deconfigure_queues(struct iavf_adapter *adapter); @@ -476,7 +859,8 @@ void iavf_add_ether_addrs(struct iavf_adapter *adapter); void iavf_del_ether_addrs(struct iavf_adapter *adapter); void iavf_add_vlans(struct iavf_adapter *adapter); void iavf_del_vlans(struct iavf_adapter *adapter); -void iavf_set_promiscuous(struct iavf_adapter *adapter, int flags); +void iavf_set_promiscuous(struct iavf_adapter *adapter); +bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter); void iavf_request_stats(struct iavf_adapter *adapter); int iavf_request_reset(struct iavf_adapter *adapter); void iavf_get_hena(struct iavf_adapter *adapter); @@ -493,13 +877,13 @@ void iavf_enable_channels(struct iavf_adapter *adapter); void iavf_disable_channels(struct iavf_adapter *adapter); void iavf_add_cloud_filter(struct iavf_adapter *adapter); void iavf_del_cloud_filter(struct iavf_adapter *adapter); -int iavf_lan_add_device(struct iavf_adapter *adapter); -int iavf_lan_del_device(struct iavf_adapter *adapter); -void iavf_client_subtask(struct iavf_adapter *adapter); -void iavf_notify_client_message(struct iavf_vsi *vsi, u8 *msg, u16 len); -void iavf_notify_client_l2_params(struct iavf_vsi *vsi); -void iavf_notify_client_open(struct iavf_vsi *vsi); -void iavf_notify_client_close(struct iavf_vsi *vsi, bool reset); +void iavf_enable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid); +void iavf_disable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid); +void iavf_enable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid); +void iavf_disable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid); +int iavf_replace_primary_mac(struct iavf_adapter *adapter, + const u8 *new_mac); +void iavf_setup_ch_info(struct iavf_adapter *adapter, u32 flags); #ifdef CONFIG_DEBUG_FS void iavf_dbg_vf_init(struct iavf_adapter *adapter); void iavf_dbg_vf_exit(struct iavf_adapter *adapter); diff --git a/drivers/net/ethernet/intel/iavf/iavf_adminq.h b/drivers/net/ethernet/intel/iavf/iavf_adminq.h index 1e88fbe0c..943d41e6e 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_adminq.h +++ b/drivers/net/ethernet/intel/iavf/iavf_adminq.h @@ -85,8 +85,8 @@ struct iavf_adminq_info { /** * iavf_aq_rc_to_posix - convert errors to user-land codes - * aq_ret: AdminQ handler error code can override aq_rc - * aq_rc: AdminQ firmware error code to convert + * @aq_ret: AdminQ handler error code can override aq_rc + * @aq_rc: AdminQ firmware error code to convert **/ static inline int iavf_aq_rc_to_posix(int aq_ret, int aq_rc) { diff --git a/drivers/net/ethernet/intel/iavf/iavf_client.c b/drivers/net/ethernet/intel/iavf/iavf_client.c deleted file mode 100644 index 4da8571e7..000000000 --- a/drivers/net/ethernet/intel/iavf/iavf_client.c +++ /dev/null @@ -1,585 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2013, Intel Corporation. */ - -#include -#include - -#include "iavf.h" -#include "iavf_prototype.h" -#include "iavf_client.h" - -static -const char iavf_client_interface_version_str[] = IAVF_CLIENT_VERSION_STR; -static struct iavf_client *vf_registered_client; -static LIST_HEAD(iavf_devices); -static DEFINE_MUTEX(iavf_device_mutex); - - -static u32 iavf_client_virtchnl_send(struct iavf_info *ldev, - struct iavf_client *client, - u8 *msg, u16 len); - -static int iavf_client_setup_qvlist(struct iavf_info *ldev, - struct iavf_client *client, - struct iavf_qvlist_info *qvlist_info); - -static struct iavf_ops iavf_lan_ops = { - .virtchnl_send = iavf_client_virtchnl_send, - .setup_qvlist = iavf_client_setup_qvlist, -}; - -/** - * iavf_client_get_params - retrieve relevant client parameters - * @vsi: VSI with parameters - * @params: client param struct - **/ -static -void iavf_client_get_params(struct iavf_vsi *vsi, struct iavf_params *params) -{ - int i; - - memset(params, 0, sizeof(struct iavf_params)); - params->mtu = vsi->netdev->mtu; - params->link_up = vsi->back->link_up; - - for (i = 0; i < IAVF_MAX_USER_PRIORITY; i++) { - params->qos.prio_qos[i].tc = 0; - params->qos.prio_qos[i].qs_handle = vsi->qs_handle; - } -} - -/** - * iavf_notify_client_message - call the client message receive callback - * @vsi: the VSI associated with this client - * @msg: message buffer - * @len: length of message - * - * If there is a client to this VSI, call the client - **/ -void iavf_notify_client_message(struct iavf_vsi *vsi, u8 *msg, u16 len) -{ - struct iavf_client_instance *cinst; - - cinst = vsi->back->cinst; - if (!cinst || !cinst->client || !cinst->client->ops || - !cinst->client->ops->virtchnl_receive) { - dev_dbg(&vsi->back->pdev->dev, - "Cannot locate client instance virtchnl_receive function\n"); - return; - } - cinst->client->ops->virtchnl_receive(&cinst->lan_info, cinst->client, - msg, len); -} - -/** - * iavf_notify_client_l2_params - call the client notify callback - * @vsi: the VSI with l2 param changes - * - * If there is a client to this VSI, call the client - **/ -void iavf_notify_client_l2_params(struct iavf_vsi *vsi) -{ - struct iavf_client_instance *cinst; - struct iavf_params params; - - cinst = vsi->back->cinst; - if (!cinst || !cinst->client || !cinst->client->ops || - !cinst->client->ops->l2_param_change) { - dev_dbg(&vsi->back->pdev->dev, - "Cannot locate client instance l2_param_change function\n"); - return; - } - - iavf_client_get_params(vsi, ¶ms); - cinst->lan_info.params = params; - cinst->client->ops->l2_param_change(&cinst->lan_info, cinst->client, - ¶ms); -} - -/** - * iavf_notify_client_open - call the client open callback - * @vsi: the VSI with netdev opened - * - * If there is a client to this netdev, call the client with open - **/ -void iavf_notify_client_open(struct iavf_vsi *vsi) -{ - struct iavf_adapter *adapter = vsi->back; - struct iavf_client_instance *cinst = adapter->cinst; - int ret; - - if (!cinst || !cinst->client || !cinst->client->ops || - !cinst->client->ops->open) { - dev_dbg(&vsi->back->pdev->dev, - "Cannot locate client instance open function\n"); - return; - } - if (!(test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state))) { - ret = cinst->client->ops->open(&cinst->lan_info, cinst->client); - if (!ret) - set_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state); - } -} - -/** - * iavf_client_release_qvlist - send a message to the PF to release iwarp qv map - * @ldev: pointer to L2 context. - * - * Return 0 on success or < 0 on error - **/ -static int iavf_client_release_qvlist(struct iavf_info *ldev) -{ - struct iavf_adapter *adapter = ldev->vf; - enum iavf_status err; - - if (adapter->aq_required) - return -EAGAIN; - - err = iavf_aq_send_msg_to_pf(&adapter->hw, - VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP, - IAVF_SUCCESS, NULL, 0, NULL); - - if (err) - dev_err(&adapter->pdev->dev, - "Unable to send iWarp vector release message to PF, error %d, aq status %d\n", - err, adapter->hw.aq.asq_last_status); - - return err; -} - -/** - * iavf_notify_client_close - call the client close callback - * @vsi: the VSI with netdev closed - * @reset: true when close called due to reset pending - * - * If there is a client to this netdev, call the client with close - **/ -void iavf_notify_client_close(struct iavf_vsi *vsi, bool reset) -{ - struct iavf_adapter *adapter = vsi->back; - struct iavf_client_instance *cinst = adapter->cinst; - - if (!cinst || !cinst->client || !cinst->client->ops || - !cinst->client->ops->close) { - dev_dbg(&vsi->back->pdev->dev, - "Cannot locate client instance close function\n"); - return; - } - cinst->client->ops->close(&cinst->lan_info, cinst->client, reset); - iavf_client_release_qvlist(&cinst->lan_info); - clear_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state); -} - -/** - * iavf_client_add_instance - add a client instance to the instance list - * @adapter: pointer to the board struct - * - * Returns cinst ptr on success, NULL on failure - **/ -static struct iavf_client_instance * -iavf_client_add_instance(struct iavf_adapter *adapter) -{ - struct iavf_client_instance *cinst = NULL; - struct iavf_vsi *vsi = &adapter->vsi; - struct netdev_hw_addr *mac = NULL; - struct iavf_params params; - - if (!vf_registered_client) - goto out; - - if (adapter->cinst) { - cinst = adapter->cinst; - goto out; - } - - cinst = kzalloc(sizeof(*cinst), GFP_KERNEL); - if (!cinst) - goto out; - - cinst->lan_info.vf = (void *)adapter; - cinst->lan_info.netdev = vsi->netdev; - cinst->lan_info.pcidev = adapter->pdev; - cinst->lan_info.fid = 0; - cinst->lan_info.ftype = IAVF_CLIENT_FTYPE_VF; - cinst->lan_info.hw_addr = adapter->hw.hw_addr; - cinst->lan_info.ops = &iavf_lan_ops; - cinst->lan_info.version.major = IAVF_CLIENT_VERSION_MAJOR; - cinst->lan_info.version.minor = IAVF_CLIENT_VERSION_MINOR; - cinst->lan_info.version.build = IAVF_CLIENT_VERSION_BUILD; - iavf_client_get_params(vsi, ¶ms); - cinst->lan_info.params = params; - set_bit(__IAVF_CLIENT_INSTANCE_NONE, &cinst->state); - - cinst->lan_info.msix_count = adapter->num_iwarp_msix; - cinst->lan_info.msix_entries = - &adapter->msix_entries[adapter->iwarp_base_vector]; - - mac = list_first_entry(&cinst->lan_info.netdev->dev_addrs.list, - struct netdev_hw_addr, list); - if (mac) - ether_addr_copy(cinst->lan_info.lanmac, mac->addr); - else - dev_err(&adapter->pdev->dev, "MAC address list is empty!\n"); - - cinst->client = vf_registered_client; - adapter->cinst = cinst; -out: - return cinst; -} - -/** - * iavf_client_del_instance - removes a client instance from the list - * @adapter: pointer to the board struct - * - **/ -static -void iavf_client_del_instance(struct iavf_adapter *adapter) -{ - kfree(adapter->cinst); - adapter->cinst = NULL; -} - -/** - * iavf_client_subtask - client maintenance work - * @adapter: board private structure - * Called under client lock - **/ -void iavf_client_subtask(struct iavf_adapter *adapter) -{ - struct iavf_client *client = vf_registered_client; - struct iavf_client_instance *cinst; - int ret = 0; - - - if (adapter->state < __IAVF_DOWN) - return; - - /* first check client is registered */ - if (!client) - return; - - /* Add the client instance to the instance list */ - cinst = adapter->cinst; - if (!cinst) - return; - - dev_info(&adapter->pdev->dev, "Added instance of Client %s\n", - client->name); - - if (!test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state)) { - /* Send an Open request to the client */ - - if (client->ops && client->ops->open) - ret = client->ops->open(&cinst->lan_info, client); - if (!ret) - set_bit(__IAVF_CLIENT_INSTANCE_OPENED, - &cinst->state); - else - /* remove client instance */ - iavf_client_del_instance(adapter); - } -} - -/** - * iavf_lan_add_device - add a lan device struct to the list of lan devices - * @adapter: pointer to the board struct - * - * Returns 0 on success or none 0 on error - **/ -int iavf_lan_add_device(struct iavf_adapter *adapter) -{ - struct iavf_device *ldev; - int ret = 0; - - mutex_lock(&iavf_device_mutex); - list_for_each_entry(ldev, &iavf_devices, list) { - if (ldev->vf == adapter) { - ret = -EEXIST; - goto out; - } - } - ldev = kzalloc(sizeof(*ldev), GFP_KERNEL); - if (!ldev) { - ret = -ENOMEM; - goto out; - } - ldev->vf = adapter; - INIT_LIST_HEAD(&ldev->list); - list_add(&ldev->list, &iavf_devices); - dev_info(&adapter->pdev->dev, "Added LAN device bus=0x%02x dev=0x%02x func=0x%02x\n", - adapter->hw.bus.bus_id, adapter->hw.bus.device, - adapter->hw.bus.func); - - /* Since in some cases register may have happened before a device gets - * added, we can schedule a subtask to go initiate the clients. - */ - adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; - -out: - mutex_unlock(&iavf_device_mutex); - return ret; -} - -/** - * iavf_lan_del_device - removes a lan device from the device list - * @adapter: pointer to the board struct - * - * Returns 0 on success or non-0 on error - **/ -int iavf_lan_del_device(struct iavf_adapter *adapter) -{ - struct iavf_device *ldev, *tmp; - int ret = -ENODEV; - - mutex_lock(&iavf_device_mutex); - list_for_each_entry_safe(ldev, tmp, &iavf_devices, list) { - if (ldev->vf == adapter) { - dev_info(&adapter->pdev->dev, - "Deleted LAN device bus=0x%02x dev=0x%02x func=0x%02x\n", - adapter->hw.bus.bus_id, adapter->hw.bus.device, - adapter->hw.bus.func); - list_del(&ldev->list); - kfree(ldev); - ret = 0; - break; - } - } - - mutex_unlock(&iavf_device_mutex); - return ret; -} - -/** - * iavf_client_release - release client specific resources - * @client: pointer to the registered client - * - **/ -static void iavf_client_release(struct iavf_client *client) -{ - struct iavf_client_instance *cinst; - struct iavf_device *ldev; - struct iavf_adapter *adapter; - - mutex_lock(&iavf_device_mutex); - list_for_each_entry(ldev, &iavf_devices, list) { - adapter = ldev->vf; - cinst = adapter->cinst; - if (!cinst) - continue; - adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | - IAVF_FLAG_CLIENT_NEEDS_CLOSE | - IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | - IAVF_FLAG_SERVICE_CLIENT_REQUESTED); - cancel_delayed_work_sync(&adapter->client_task); - while (test_and_set_bit(__IAVF_IN_CLIENT_TASK, - &adapter->crit_section)) - usleep_range(500, 1000); - if (test_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state)) { - if (client->ops && client->ops->close) - client->ops->close(&cinst->lan_info, client, - false); - iavf_client_release_qvlist(&cinst->lan_info); - clear_bit(__IAVF_CLIENT_INSTANCE_OPENED, &cinst->state); - - dev_warn(&adapter->pdev->dev, - "Client %s instance closed\n", client->name); - } - /* delete the client instance */ - iavf_client_del_instance(adapter); - dev_info(&adapter->pdev->dev, "Deleted client instance of Client %s\n", - client->name); - clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); - } - mutex_unlock(&iavf_device_mutex); -} -/** - * iavf_client_prepare - prepare client specific resources - * @client: pointer to the registered client - * - **/ -static void iavf_client_prepare(struct iavf_client *client) -{ - struct iavf_device *ldev; - struct iavf_adapter *adapter; - - mutex_lock(&iavf_device_mutex); - list_for_each_entry(ldev, &iavf_devices, list) { - adapter = ldev->vf; - iavf_client_add_instance(adapter); - /* Signal the watchdog to service the client */ - adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; - } - mutex_unlock(&iavf_device_mutex); -} - -/** - * iavf_client_virtchnl_send - send a message to the PF instance - * @ldev: pointer to L2 context. - * @client: Client pointer. - * @msg: pointer to message buffer - * @len: message length - * - * Return 0 on success or < 0 on error - **/ -static u32 iavf_client_virtchnl_send(struct iavf_info *ldev, - struct iavf_client *client, - u8 *msg, u16 len) -{ - struct iavf_adapter *adapter = ldev->vf; - enum iavf_status err; - - if (adapter->aq_required) - return -EAGAIN; - - err = iavf_aq_send_msg_to_pf(&adapter->hw, VIRTCHNL_OP_IWARP, - IAVF_SUCCESS, msg, len, NULL); - if (err) - dev_err(&adapter->pdev->dev, "Unable to send iWarp message to PF, error %d, aq status %d\n", - err, adapter->hw.aq.asq_last_status); - - return err; -} - -/** - * iavf_client_setup_qvlist - send a message to the PF to setup iwarp qv map - * @ldev: pointer to L2 context. - * @client: Client pointer. - * @qvlist_info: queue and vector list - * - * Return 0 on success or < 0 on error - **/ -static int iavf_client_setup_qvlist(struct iavf_info *ldev, - struct iavf_client *client, - struct iavf_qvlist_info *qvlist_info) -{ - struct virtchnl_iwarp_qvlist_info *v_qvlist_info; - struct iavf_adapter *adapter = ldev->vf; - struct iavf_qv_info *qv_info; - enum iavf_status err; - u32 v_idx, i; - u32 msg_size; - - if (adapter->aq_required) - return -EAGAIN; - - /* A quick check on whether the vectors belong to the client */ - for (i = 0; i < qvlist_info->num_vectors; i++) { - qv_info = &qvlist_info->qv_info[i]; - if (!qv_info) - continue; - v_idx = qv_info->v_idx; - if ((v_idx >= - (adapter->iwarp_base_vector + adapter->num_iwarp_msix)) || - (v_idx < adapter->iwarp_base_vector)) - return -EINVAL; - } - - v_qvlist_info = (struct virtchnl_iwarp_qvlist_info *)qvlist_info; - msg_size = sizeof(struct virtchnl_iwarp_qvlist_info) + - (sizeof(struct virtchnl_iwarp_qv_info) * - (v_qvlist_info->num_vectors - 1)); - - adapter->client_pending |= BIT(VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP); - err = iavf_aq_send_msg_to_pf(&adapter->hw, - VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP, IAVF_SUCCESS, - (u8 *)v_qvlist_info, msg_size, NULL); - - if (err) { - dev_err(&adapter->pdev->dev, - "Unable to send iWarp vector config message to PF, error %d, aq status %d\n", - err, adapter->hw.aq.asq_last_status); - goto out; - } - - err = -EBUSY; - for (i = 0; i < 5; i++) { - msleep(100); - if (!(adapter->client_pending & - BIT(VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP))) { - err = 0; - break; - } - } -out: - return err; -} - -/** - * iavf_register_client - Register a iavf client driver with the L2 driver - * @client: pointer to the iavf_client struct - * - * Returns 0 on success or non-0 on error - **/ -int iavf_register_client(struct iavf_client *client) -{ - int ret = 0; - - if (!client) { - ret = -EIO; - goto out; - } - - if (strlen(client->name) == 0) { - pr_info("iavf: Failed to register client with no name\n"); - ret = -EIO; - goto out; - } - - if (vf_registered_client) { - pr_info("iavf: Client %s has already been registered!\n", - client->name); - ret = -EEXIST; - goto out; - } - - if ((client->version.major != IAVF_CLIENT_VERSION_MAJOR) || - (client->version.minor != IAVF_CLIENT_VERSION_MINOR)) { - pr_info("iavf: Failed to register client %s due to mismatched client interface version\n", - client->name); - pr_info("Client is using version: %02d.%02d.%02d while LAN driver supports %s\n", - client->version.major, client->version.minor, - client->version.build, - iavf_client_interface_version_str); - ret = -EIO; - goto out; - } - - vf_registered_client = client; - - iavf_client_prepare(client); - - pr_info("iavf: Registered client %s with return code %d\n", - client->name, ret); -out: - return ret; -} -EXPORT_SYMBOL(iavf_register_client); - -/** - * iavf_unregister_client - Unregister a iavf client driver with the L2 driver - * @client: pointer to the iavf_client struct - * - * Returns 0 on success or non-0 on error - **/ -int iavf_unregister_client(struct iavf_client *client) -{ - int ret = 0; - - if (vf_registered_client != client) { - pr_info("iavf: Client %s has not been registered\n", - client->name); - ret = -ENODEV; - goto out; - } - vf_registered_client = NULL; - /* When a unregister request comes through we would have to send - * a close for each of the client instances that were opened. - * client_release function is called to handle this. - */ - iavf_client_release(client); - - pr_info("iavf: Unregistered client %s\n", client->name); -out: - return ret; -} -EXPORT_SYMBOL(iavf_unregister_client); diff --git a/drivers/net/ethernet/intel/iavf/iavf_client.h b/drivers/net/ethernet/intel/iavf/iavf_client.h deleted file mode 100644 index 95f5f5032..000000000 --- a/drivers/net/ethernet/intel/iavf/iavf_client.h +++ /dev/null @@ -1,169 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (c) 2013, Intel Corporation. */ - -#ifndef _IAVF_CLIENT_H_ -#define _IAVF_CLIENT_H_ - -#define IAVF_CLIENT_STR_LENGTH 10 - -/* Client interface version should be updated anytime there is a change in the - * existing APIs or data structures. - */ -#define IAVF_CLIENT_VERSION_MAJOR 0 -#define IAVF_CLIENT_VERSION_MINOR 01 -#define IAVF_CLIENT_VERSION_BUILD 00 -#define IAVF_CLIENT_VERSION_STR \ - __stringify(IAVF_CLIENT_VERSION_MAJOR) "." \ - __stringify(IAVF_CLIENT_VERSION_MINOR) "." \ - __stringify(IAVF_CLIENT_VERSION_BUILD) - -struct iavf_client_version { - u8 major; - u8 minor; - u8 build; - u8 rsvd; -}; - -enum iavf_client_state { - __IAVF_CLIENT_NULL, - __IAVF_CLIENT_REGISTERED -}; - -enum iavf_client_instance_state { - __IAVF_CLIENT_INSTANCE_NONE, - __IAVF_CLIENT_INSTANCE_OPENED, -}; - -struct iavf_ops; -struct iavf_client; - -/* HW does not define a type value for AEQ; only for RX/TX and CEQ. - * In order for us to keep the interface simple, SW will define a - * unique type value for AEQ. - */ -#define IAVF_QUEUE_TYPE_PE_AEQ 0x80 -#define IAVF_QUEUE_INVALID_IDX 0xFFFF - -struct iavf_qv_info { - u32 v_idx; /* msix_vector */ - u16 ceq_idx; - u16 aeq_idx; - u8 itr_idx; -}; - -struct iavf_qvlist_info { - u32 num_vectors; - struct iavf_qv_info qv_info[1]; -}; - -#define IAVF_CLIENT_MSIX_ALL 0xFFFFFFFF - -/* set of LAN parameters useful for clients managed by LAN */ - -/* Struct to hold per priority info */ -struct iavf_prio_qos_params { - u16 qs_handle; /* qs handle for prio */ - u8 tc; /* TC mapped to prio */ - u8 reserved; -}; - -#define IAVF_CLIENT_MAX_USER_PRIORITY 8 -/* Struct to hold Client QoS */ -struct iavf_qos_params { - struct iavf_prio_qos_params prio_qos[IAVF_CLIENT_MAX_USER_PRIORITY]; -}; - -struct iavf_params { - struct iavf_qos_params qos; - u16 mtu; - u16 link_up; /* boolean */ -}; - -/* Structure to hold LAN device info for a client device */ -struct iavf_info { - struct iavf_client_version version; - u8 lanmac[6]; - struct net_device *netdev; - struct pci_dev *pcidev; - u8 __iomem *hw_addr; - u8 fid; /* function id, PF id or VF id */ -#define IAVF_CLIENT_FTYPE_PF 0 -#define IAVF_CLIENT_FTYPE_VF 1 - u8 ftype; /* function type, PF or VF */ - void *vf; /* cast to iavf_adapter */ - - /* All L2 params that could change during the life span of the device - * and needs to be communicated to the client when they change - */ - struct iavf_params params; - struct iavf_ops *ops; - - u16 msix_count; /* number of msix vectors*/ - /* Array down below will be dynamically allocated based on msix_count */ - struct msix_entry *msix_entries; - u16 itr_index; /* Which ITR index the PE driver is suppose to use */ -}; - -struct iavf_ops { - /* setup_q_vector_list enables queues with a particular vector */ - int (*setup_qvlist)(struct iavf_info *ldev, struct iavf_client *client, - struct iavf_qvlist_info *qv_info); - - u32 (*virtchnl_send)(struct iavf_info *ldev, struct iavf_client *client, - u8 *msg, u16 len); - - /* If the PE Engine is unresponsive, RDMA driver can request a reset.*/ - void (*request_reset)(struct iavf_info *ldev, - struct iavf_client *client); -}; - -struct iavf_client_ops { - /* Should be called from register_client() or whenever the driver is - * ready to create a specific client instance. - */ - int (*open)(struct iavf_info *ldev, struct iavf_client *client); - - /* Should be closed when netdev is unavailable or when unregister - * call comes in. If the close happens due to a reset, set the reset - * bit to true. - */ - void (*close)(struct iavf_info *ldev, struct iavf_client *client, - bool reset); - - /* called when some l2 managed parameters changes - mss */ - void (*l2_param_change)(struct iavf_info *ldev, - struct iavf_client *client, - struct iavf_params *params); - - /* called when a message is received from the PF */ - int (*virtchnl_receive)(struct iavf_info *ldev, - struct iavf_client *client, - u8 *msg, u16 len); -}; - -/* Client device */ -struct iavf_client_instance { - struct list_head list; - struct iavf_info lan_info; - struct iavf_client *client; - unsigned long state; -}; - -struct iavf_client { - struct list_head list; /* list of registered clients */ - char name[IAVF_CLIENT_STR_LENGTH]; - struct iavf_client_version version; - unsigned long state; /* client state */ - atomic_t ref_cnt; /* Count of all the client devices of this kind */ - u32 flags; -#define IAVF_CLIENT_FLAGS_LAUNCH_ON_PROBE BIT(0) -#define IAVF_TX_FLAGS_NOTIFY_OTHER_EVENTS BIT(2) - u8 type; -#define IAVF_CLIENT_IWARP 0 - struct iavf_client_ops *ops; /* client ops provided by the client */ -}; - -/* used by clients */ -int iavf_register_client(struct iavf_client *client); -int iavf_unregister_client(struct iavf_client *client); -#endif /* _IAVF_CLIENT_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_common.c b/drivers/net/ethernet/intel/iavf/iavf_common.c index e2eed8083..e799b0360 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_common.c +++ b/drivers/net/ethernet/intel/iavf/iavf_common.c @@ -505,9 +505,9 @@ enum iavf_status iavf_aq_set_rss_key(struct iavf_hw *hw, return iavf_aq_get_set_rss_key(hw, vsi_id, key, true); } -/* The iavf_ptype_lookup table is used to convert from the 8-bit ptype in the - * hardware to a bit-field that can be used by SW to more easily determine the - * packet type. +/* The iavf_ptype_lookup table is used to convert from the 8-bit and 10-bit + * ptype in the hardware to a bit-field that can be used by SW to more easily + * determine the packet type. * * Macros are used to shorten the table lines and make this table human * readable. @@ -867,7 +867,852 @@ struct iavf_rx_ptype_decoded iavf_ptype_lookup[] = { IAVF_PTT_UNUSED_ENTRY(252), IAVF_PTT_UNUSED_ENTRY(253), IAVF_PTT_UNUSED_ENTRY(254), - IAVF_PTT_UNUSED_ENTRY(255) + IAVF_PTT_UNUSED_ENTRY(255), + IAVF_PTT_UNUSED_ENTRY(256), + IAVF_PTT_UNUSED_ENTRY(257), + IAVF_PTT_UNUSED_ENTRY(258), + IAVF_PTT_UNUSED_ENTRY(259), + + IAVF_PTT_UNUSED_ENTRY(260), + IAVF_PTT_UNUSED_ENTRY(261), + IAVF_PTT_UNUSED_ENTRY(262), + IAVF_PTT_UNUSED_ENTRY(263), + IAVF_PTT_UNUSED_ENTRY(264), + IAVF_PTT_UNUSED_ENTRY(265), + IAVF_PTT_UNUSED_ENTRY(266), + IAVF_PTT_UNUSED_ENTRY(267), + IAVF_PTT_UNUSED_ENTRY(268), + IAVF_PTT_UNUSED_ENTRY(269), + + IAVF_PTT_UNUSED_ENTRY(270), + IAVF_PTT_UNUSED_ENTRY(271), + IAVF_PTT_UNUSED_ENTRY(272), + IAVF_PTT_UNUSED_ENTRY(273), + IAVF_PTT_UNUSED_ENTRY(274), + IAVF_PTT_UNUSED_ENTRY(275), + IAVF_PTT_UNUSED_ENTRY(276), + IAVF_PTT_UNUSED_ENTRY(277), + IAVF_PTT_UNUSED_ENTRY(278), + IAVF_PTT_UNUSED_ENTRY(279), + + IAVF_PTT_UNUSED_ENTRY(280), + IAVF_PTT_UNUSED_ENTRY(281), + IAVF_PTT_UNUSED_ENTRY(282), + IAVF_PTT_UNUSED_ENTRY(283), + IAVF_PTT_UNUSED_ENTRY(284), + IAVF_PTT_UNUSED_ENTRY(285), + IAVF_PTT_UNUSED_ENTRY(286), + IAVF_PTT_UNUSED_ENTRY(287), + IAVF_PTT_UNUSED_ENTRY(288), + IAVF_PTT_UNUSED_ENTRY(289), + + IAVF_PTT_UNUSED_ENTRY(290), + IAVF_PTT_UNUSED_ENTRY(291), + IAVF_PTT_UNUSED_ENTRY(292), + IAVF_PTT_UNUSED_ENTRY(293), + IAVF_PTT_UNUSED_ENTRY(294), + IAVF_PTT_UNUSED_ENTRY(295), + IAVF_PTT_UNUSED_ENTRY(296), + IAVF_PTT_UNUSED_ENTRY(297), + IAVF_PTT_UNUSED_ENTRY(298), + IAVF_PTT_UNUSED_ENTRY(299), + + IAVF_PTT_UNUSED_ENTRY(300), + IAVF_PTT_UNUSED_ENTRY(301), + IAVF_PTT_UNUSED_ENTRY(302), + IAVF_PTT_UNUSED_ENTRY(303), + IAVF_PTT_UNUSED_ENTRY(304), + IAVF_PTT_UNUSED_ENTRY(305), + IAVF_PTT_UNUSED_ENTRY(306), + IAVF_PTT_UNUSED_ENTRY(307), + IAVF_PTT_UNUSED_ENTRY(308), + IAVF_PTT_UNUSED_ENTRY(309), + + IAVF_PTT_UNUSED_ENTRY(310), + IAVF_PTT_UNUSED_ENTRY(311), + IAVF_PTT_UNUSED_ENTRY(312), + IAVF_PTT_UNUSED_ENTRY(313), + IAVF_PTT_UNUSED_ENTRY(314), + IAVF_PTT_UNUSED_ENTRY(315), + IAVF_PTT_UNUSED_ENTRY(316), + IAVF_PTT_UNUSED_ENTRY(317), + IAVF_PTT_UNUSED_ENTRY(318), + IAVF_PTT_UNUSED_ENTRY(319), + + IAVF_PTT_UNUSED_ENTRY(320), + IAVF_PTT_UNUSED_ENTRY(321), + IAVF_PTT_UNUSED_ENTRY(322), + IAVF_PTT_UNUSED_ENTRY(323), + IAVF_PTT_UNUSED_ENTRY(324), + IAVF_PTT_UNUSED_ENTRY(325), + IAVF_PTT_UNUSED_ENTRY(326), + IAVF_PTT_UNUSED_ENTRY(327), + IAVF_PTT_UNUSED_ENTRY(328), + IAVF_PTT_UNUSED_ENTRY(329), + + IAVF_PTT_UNUSED_ENTRY(330), + IAVF_PTT_UNUSED_ENTRY(331), + IAVF_PTT_UNUSED_ENTRY(332), + IAVF_PTT_UNUSED_ENTRY(333), + IAVF_PTT_UNUSED_ENTRY(334), + IAVF_PTT_UNUSED_ENTRY(335), + IAVF_PTT_UNUSED_ENTRY(336), + IAVF_PTT_UNUSED_ENTRY(337), + IAVF_PTT_UNUSED_ENTRY(338), + IAVF_PTT_UNUSED_ENTRY(339), + + IAVF_PTT_UNUSED_ENTRY(340), + IAVF_PTT_UNUSED_ENTRY(341), + IAVF_PTT_UNUSED_ENTRY(342), + IAVF_PTT_UNUSED_ENTRY(343), + IAVF_PTT_UNUSED_ENTRY(344), + IAVF_PTT_UNUSED_ENTRY(345), + IAVF_PTT_UNUSED_ENTRY(346), + IAVF_PTT_UNUSED_ENTRY(347), + IAVF_PTT_UNUSED_ENTRY(348), + IAVF_PTT_UNUSED_ENTRY(349), + + IAVF_PTT_UNUSED_ENTRY(350), + IAVF_PTT_UNUSED_ENTRY(351), + IAVF_PTT_UNUSED_ENTRY(352), + IAVF_PTT_UNUSED_ENTRY(353), + IAVF_PTT_UNUSED_ENTRY(354), + IAVF_PTT_UNUSED_ENTRY(355), + IAVF_PTT_UNUSED_ENTRY(356), + IAVF_PTT_UNUSED_ENTRY(357), + IAVF_PTT_UNUSED_ENTRY(358), + IAVF_PTT_UNUSED_ENTRY(359), + + IAVF_PTT_UNUSED_ENTRY(360), + IAVF_PTT_UNUSED_ENTRY(361), + IAVF_PTT_UNUSED_ENTRY(362), + IAVF_PTT_UNUSED_ENTRY(363), + IAVF_PTT_UNUSED_ENTRY(364), + IAVF_PTT_UNUSED_ENTRY(365), + IAVF_PTT_UNUSED_ENTRY(366), + IAVF_PTT_UNUSED_ENTRY(367), + IAVF_PTT_UNUSED_ENTRY(368), + IAVF_PTT_UNUSED_ENTRY(369), + + IAVF_PTT_UNUSED_ENTRY(370), + IAVF_PTT_UNUSED_ENTRY(371), + IAVF_PTT_UNUSED_ENTRY(372), + IAVF_PTT_UNUSED_ENTRY(373), + IAVF_PTT_UNUSED_ENTRY(374), + IAVF_PTT_UNUSED_ENTRY(375), + IAVF_PTT_UNUSED_ENTRY(376), + IAVF_PTT_UNUSED_ENTRY(377), + IAVF_PTT_UNUSED_ENTRY(378), + IAVF_PTT_UNUSED_ENTRY(379), + + IAVF_PTT_UNUSED_ENTRY(380), + IAVF_PTT_UNUSED_ENTRY(381), + IAVF_PTT_UNUSED_ENTRY(382), + IAVF_PTT_UNUSED_ENTRY(383), + IAVF_PTT_UNUSED_ENTRY(384), + IAVF_PTT_UNUSED_ENTRY(385), + IAVF_PTT_UNUSED_ENTRY(386), + IAVF_PTT_UNUSED_ENTRY(387), + IAVF_PTT_UNUSED_ENTRY(388), + IAVF_PTT_UNUSED_ENTRY(389), + + IAVF_PTT_UNUSED_ENTRY(390), + IAVF_PTT_UNUSED_ENTRY(391), + IAVF_PTT_UNUSED_ENTRY(392), + IAVF_PTT_UNUSED_ENTRY(393), + IAVF_PTT_UNUSED_ENTRY(394), + IAVF_PTT_UNUSED_ENTRY(395), + IAVF_PTT_UNUSED_ENTRY(396), + IAVF_PTT_UNUSED_ENTRY(397), + IAVF_PTT_UNUSED_ENTRY(398), + IAVF_PTT_UNUSED_ENTRY(399), + + IAVF_PTT_UNUSED_ENTRY(400), + IAVF_PTT_UNUSED_ENTRY(401), + IAVF_PTT_UNUSED_ENTRY(402), + IAVF_PTT_UNUSED_ENTRY(403), + IAVF_PTT_UNUSED_ENTRY(404), + IAVF_PTT_UNUSED_ENTRY(405), + IAVF_PTT_UNUSED_ENTRY(406), + IAVF_PTT_UNUSED_ENTRY(407), + IAVF_PTT_UNUSED_ENTRY(408), + IAVF_PTT_UNUSED_ENTRY(409), + + IAVF_PTT_UNUSED_ENTRY(410), + IAVF_PTT_UNUSED_ENTRY(411), + IAVF_PTT_UNUSED_ENTRY(412), + IAVF_PTT_UNUSED_ENTRY(413), + IAVF_PTT_UNUSED_ENTRY(414), + IAVF_PTT_UNUSED_ENTRY(415), + IAVF_PTT_UNUSED_ENTRY(416), + IAVF_PTT_UNUSED_ENTRY(417), + IAVF_PTT_UNUSED_ENTRY(418), + IAVF_PTT_UNUSED_ENTRY(419), + + IAVF_PTT_UNUSED_ENTRY(420), + IAVF_PTT_UNUSED_ENTRY(421), + IAVF_PTT_UNUSED_ENTRY(422), + IAVF_PTT_UNUSED_ENTRY(423), + IAVF_PTT_UNUSED_ENTRY(424), + IAVF_PTT_UNUSED_ENTRY(425), + IAVF_PTT_UNUSED_ENTRY(426), + IAVF_PTT_UNUSED_ENTRY(427), + IAVF_PTT_UNUSED_ENTRY(428), + IAVF_PTT_UNUSED_ENTRY(429), + + IAVF_PTT_UNUSED_ENTRY(430), + IAVF_PTT_UNUSED_ENTRY(431), + IAVF_PTT_UNUSED_ENTRY(432), + IAVF_PTT_UNUSED_ENTRY(433), + IAVF_PTT_UNUSED_ENTRY(434), + IAVF_PTT_UNUSED_ENTRY(435), + IAVF_PTT_UNUSED_ENTRY(436), + IAVF_PTT_UNUSED_ENTRY(437), + IAVF_PTT_UNUSED_ENTRY(438), + IAVF_PTT_UNUSED_ENTRY(439), + + IAVF_PTT_UNUSED_ENTRY(440), + IAVF_PTT_UNUSED_ENTRY(441), + IAVF_PTT_UNUSED_ENTRY(442), + IAVF_PTT_UNUSED_ENTRY(443), + IAVF_PTT_UNUSED_ENTRY(444), + IAVF_PTT_UNUSED_ENTRY(445), + IAVF_PTT_UNUSED_ENTRY(446), + IAVF_PTT_UNUSED_ENTRY(447), + IAVF_PTT_UNUSED_ENTRY(448), + IAVF_PTT_UNUSED_ENTRY(449), + + IAVF_PTT_UNUSED_ENTRY(450), + IAVF_PTT_UNUSED_ENTRY(451), + IAVF_PTT_UNUSED_ENTRY(452), + IAVF_PTT_UNUSED_ENTRY(453), + IAVF_PTT_UNUSED_ENTRY(454), + IAVF_PTT_UNUSED_ENTRY(455), + IAVF_PTT_UNUSED_ENTRY(456), + IAVF_PTT_UNUSED_ENTRY(457), + IAVF_PTT_UNUSED_ENTRY(458), + IAVF_PTT_UNUSED_ENTRY(459), + + IAVF_PTT_UNUSED_ENTRY(460), + IAVF_PTT_UNUSED_ENTRY(461), + IAVF_PTT_UNUSED_ENTRY(462), + IAVF_PTT_UNUSED_ENTRY(463), + IAVF_PTT_UNUSED_ENTRY(464), + IAVF_PTT_UNUSED_ENTRY(465), + IAVF_PTT_UNUSED_ENTRY(466), + IAVF_PTT_UNUSED_ENTRY(467), + IAVF_PTT_UNUSED_ENTRY(468), + IAVF_PTT_UNUSED_ENTRY(469), + + IAVF_PTT_UNUSED_ENTRY(470), + IAVF_PTT_UNUSED_ENTRY(471), + IAVF_PTT_UNUSED_ENTRY(472), + IAVF_PTT_UNUSED_ENTRY(473), + IAVF_PTT_UNUSED_ENTRY(474), + IAVF_PTT_UNUSED_ENTRY(475), + IAVF_PTT_UNUSED_ENTRY(476), + IAVF_PTT_UNUSED_ENTRY(477), + IAVF_PTT_UNUSED_ENTRY(478), + IAVF_PTT_UNUSED_ENTRY(479), + + IAVF_PTT_UNUSED_ENTRY(480), + IAVF_PTT_UNUSED_ENTRY(481), + IAVF_PTT_UNUSED_ENTRY(482), + IAVF_PTT_UNUSED_ENTRY(483), + IAVF_PTT_UNUSED_ENTRY(484), + IAVF_PTT_UNUSED_ENTRY(485), + IAVF_PTT_UNUSED_ENTRY(486), + IAVF_PTT_UNUSED_ENTRY(487), + IAVF_PTT_UNUSED_ENTRY(488), + IAVF_PTT_UNUSED_ENTRY(489), + + IAVF_PTT_UNUSED_ENTRY(490), + IAVF_PTT_UNUSED_ENTRY(491), + IAVF_PTT_UNUSED_ENTRY(492), + IAVF_PTT_UNUSED_ENTRY(493), + IAVF_PTT_UNUSED_ENTRY(494), + IAVF_PTT_UNUSED_ENTRY(495), + IAVF_PTT_UNUSED_ENTRY(496), + IAVF_PTT_UNUSED_ENTRY(497), + IAVF_PTT_UNUSED_ENTRY(498), + IAVF_PTT_UNUSED_ENTRY(499), + + IAVF_PTT_UNUSED_ENTRY(500), + IAVF_PTT_UNUSED_ENTRY(501), + IAVF_PTT_UNUSED_ENTRY(502), + IAVF_PTT_UNUSED_ENTRY(503), + IAVF_PTT_UNUSED_ENTRY(504), + IAVF_PTT_UNUSED_ENTRY(505), + IAVF_PTT_UNUSED_ENTRY(506), + IAVF_PTT_UNUSED_ENTRY(507), + IAVF_PTT_UNUSED_ENTRY(508), + IAVF_PTT_UNUSED_ENTRY(509), + + IAVF_PTT_UNUSED_ENTRY(510), + IAVF_PTT_UNUSED_ENTRY(511), + IAVF_PTT_UNUSED_ENTRY(512), + IAVF_PTT_UNUSED_ENTRY(513), + IAVF_PTT_UNUSED_ENTRY(514), + IAVF_PTT_UNUSED_ENTRY(515), + IAVF_PTT_UNUSED_ENTRY(516), + IAVF_PTT_UNUSED_ENTRY(517), + IAVF_PTT_UNUSED_ENTRY(518), + IAVF_PTT_UNUSED_ENTRY(519), + + IAVF_PTT_UNUSED_ENTRY(520), + IAVF_PTT_UNUSED_ENTRY(521), + IAVF_PTT_UNUSED_ENTRY(522), + IAVF_PTT_UNUSED_ENTRY(523), + IAVF_PTT_UNUSED_ENTRY(524), + IAVF_PTT_UNUSED_ENTRY(525), + IAVF_PTT_UNUSED_ENTRY(526), + IAVF_PTT_UNUSED_ENTRY(527), + IAVF_PTT_UNUSED_ENTRY(528), + IAVF_PTT_UNUSED_ENTRY(529), + + IAVF_PTT_UNUSED_ENTRY(530), + IAVF_PTT_UNUSED_ENTRY(531), + IAVF_PTT_UNUSED_ENTRY(532), + IAVF_PTT_UNUSED_ENTRY(533), + IAVF_PTT_UNUSED_ENTRY(534), + IAVF_PTT_UNUSED_ENTRY(535), + IAVF_PTT_UNUSED_ENTRY(536), + IAVF_PTT_UNUSED_ENTRY(537), + IAVF_PTT_UNUSED_ENTRY(538), + IAVF_PTT_UNUSED_ENTRY(539), + + IAVF_PTT_UNUSED_ENTRY(540), + IAVF_PTT_UNUSED_ENTRY(541), + IAVF_PTT_UNUSED_ENTRY(542), + IAVF_PTT_UNUSED_ENTRY(543), + IAVF_PTT_UNUSED_ENTRY(544), + IAVF_PTT_UNUSED_ENTRY(545), + IAVF_PTT_UNUSED_ENTRY(546), + IAVF_PTT_UNUSED_ENTRY(547), + IAVF_PTT_UNUSED_ENTRY(548), + IAVF_PTT_UNUSED_ENTRY(549), + + IAVF_PTT_UNUSED_ENTRY(550), + IAVF_PTT_UNUSED_ENTRY(551), + IAVF_PTT_UNUSED_ENTRY(552), + IAVF_PTT_UNUSED_ENTRY(553), + IAVF_PTT_UNUSED_ENTRY(554), + IAVF_PTT_UNUSED_ENTRY(555), + IAVF_PTT_UNUSED_ENTRY(556), + IAVF_PTT_UNUSED_ENTRY(557), + IAVF_PTT_UNUSED_ENTRY(558), + IAVF_PTT_UNUSED_ENTRY(559), + + IAVF_PTT_UNUSED_ENTRY(560), + IAVF_PTT_UNUSED_ENTRY(561), + IAVF_PTT_UNUSED_ENTRY(562), + IAVF_PTT_UNUSED_ENTRY(563), + IAVF_PTT_UNUSED_ENTRY(564), + IAVF_PTT_UNUSED_ENTRY(565), + IAVF_PTT_UNUSED_ENTRY(566), + IAVF_PTT_UNUSED_ENTRY(567), + IAVF_PTT_UNUSED_ENTRY(568), + IAVF_PTT_UNUSED_ENTRY(569), + + IAVF_PTT_UNUSED_ENTRY(570), + IAVF_PTT_UNUSED_ENTRY(571), + IAVF_PTT_UNUSED_ENTRY(572), + IAVF_PTT_UNUSED_ENTRY(573), + IAVF_PTT_UNUSED_ENTRY(574), + IAVF_PTT_UNUSED_ENTRY(575), + IAVF_PTT_UNUSED_ENTRY(576), + IAVF_PTT_UNUSED_ENTRY(577), + IAVF_PTT_UNUSED_ENTRY(578), + IAVF_PTT_UNUSED_ENTRY(579), + + IAVF_PTT_UNUSED_ENTRY(580), + IAVF_PTT_UNUSED_ENTRY(581), + IAVF_PTT_UNUSED_ENTRY(582), + IAVF_PTT_UNUSED_ENTRY(583), + IAVF_PTT_UNUSED_ENTRY(584), + IAVF_PTT_UNUSED_ENTRY(585), + IAVF_PTT_UNUSED_ENTRY(586), + IAVF_PTT_UNUSED_ENTRY(587), + IAVF_PTT_UNUSED_ENTRY(588), + IAVF_PTT_UNUSED_ENTRY(589), + + IAVF_PTT_UNUSED_ENTRY(590), + IAVF_PTT_UNUSED_ENTRY(591), + IAVF_PTT_UNUSED_ENTRY(592), + IAVF_PTT_UNUSED_ENTRY(593), + IAVF_PTT_UNUSED_ENTRY(594), + IAVF_PTT_UNUSED_ENTRY(595), + IAVF_PTT_UNUSED_ENTRY(596), + IAVF_PTT_UNUSED_ENTRY(597), + IAVF_PTT_UNUSED_ENTRY(598), + IAVF_PTT_UNUSED_ENTRY(599), + + IAVF_PTT_UNUSED_ENTRY(600), + IAVF_PTT_UNUSED_ENTRY(601), + IAVF_PTT_UNUSED_ENTRY(602), + IAVF_PTT_UNUSED_ENTRY(603), + IAVF_PTT_UNUSED_ENTRY(604), + IAVF_PTT_UNUSED_ENTRY(605), + IAVF_PTT_UNUSED_ENTRY(606), + IAVF_PTT_UNUSED_ENTRY(607), + IAVF_PTT_UNUSED_ENTRY(608), + IAVF_PTT_UNUSED_ENTRY(609), + + IAVF_PTT_UNUSED_ENTRY(610), + IAVF_PTT_UNUSED_ENTRY(611), + IAVF_PTT_UNUSED_ENTRY(612), + IAVF_PTT_UNUSED_ENTRY(613), + IAVF_PTT_UNUSED_ENTRY(614), + IAVF_PTT_UNUSED_ENTRY(615), + IAVF_PTT_UNUSED_ENTRY(616), + IAVF_PTT_UNUSED_ENTRY(617), + IAVF_PTT_UNUSED_ENTRY(618), + IAVF_PTT_UNUSED_ENTRY(619), + + IAVF_PTT_UNUSED_ENTRY(620), + IAVF_PTT_UNUSED_ENTRY(621), + IAVF_PTT_UNUSED_ENTRY(622), + IAVF_PTT_UNUSED_ENTRY(623), + IAVF_PTT_UNUSED_ENTRY(624), + IAVF_PTT_UNUSED_ENTRY(625), + IAVF_PTT_UNUSED_ENTRY(626), + IAVF_PTT_UNUSED_ENTRY(627), + IAVF_PTT_UNUSED_ENTRY(628), + IAVF_PTT_UNUSED_ENTRY(629), + + IAVF_PTT_UNUSED_ENTRY(630), + IAVF_PTT_UNUSED_ENTRY(631), + IAVF_PTT_UNUSED_ENTRY(632), + IAVF_PTT_UNUSED_ENTRY(633), + IAVF_PTT_UNUSED_ENTRY(634), + IAVF_PTT_UNUSED_ENTRY(635), + IAVF_PTT_UNUSED_ENTRY(636), + IAVF_PTT_UNUSED_ENTRY(637), + IAVF_PTT_UNUSED_ENTRY(638), + IAVF_PTT_UNUSED_ENTRY(639), + + IAVF_PTT_UNUSED_ENTRY(640), + IAVF_PTT_UNUSED_ENTRY(641), + IAVF_PTT_UNUSED_ENTRY(642), + IAVF_PTT_UNUSED_ENTRY(643), + IAVF_PTT_UNUSED_ENTRY(644), + IAVF_PTT_UNUSED_ENTRY(645), + IAVF_PTT_UNUSED_ENTRY(646), + IAVF_PTT_UNUSED_ENTRY(647), + IAVF_PTT_UNUSED_ENTRY(648), + IAVF_PTT_UNUSED_ENTRY(649), + + IAVF_PTT_UNUSED_ENTRY(650), + IAVF_PTT_UNUSED_ENTRY(651), + IAVF_PTT_UNUSED_ENTRY(652), + IAVF_PTT_UNUSED_ENTRY(653), + IAVF_PTT_UNUSED_ENTRY(654), + IAVF_PTT_UNUSED_ENTRY(655), + IAVF_PTT_UNUSED_ENTRY(656), + IAVF_PTT_UNUSED_ENTRY(657), + IAVF_PTT_UNUSED_ENTRY(658), + IAVF_PTT_UNUSED_ENTRY(659), + + IAVF_PTT_UNUSED_ENTRY(660), + IAVF_PTT_UNUSED_ENTRY(661), + IAVF_PTT_UNUSED_ENTRY(662), + IAVF_PTT_UNUSED_ENTRY(663), + IAVF_PTT_UNUSED_ENTRY(664), + IAVF_PTT_UNUSED_ENTRY(665), + IAVF_PTT_UNUSED_ENTRY(666), + IAVF_PTT_UNUSED_ENTRY(667), + IAVF_PTT_UNUSED_ENTRY(668), + IAVF_PTT_UNUSED_ENTRY(669), + + IAVF_PTT_UNUSED_ENTRY(670), + IAVF_PTT_UNUSED_ENTRY(671), + IAVF_PTT_UNUSED_ENTRY(672), + IAVF_PTT_UNUSED_ENTRY(673), + IAVF_PTT_UNUSED_ENTRY(674), + IAVF_PTT_UNUSED_ENTRY(675), + IAVF_PTT_UNUSED_ENTRY(676), + IAVF_PTT_UNUSED_ENTRY(677), + IAVF_PTT_UNUSED_ENTRY(678), + IAVF_PTT_UNUSED_ENTRY(679), + + IAVF_PTT_UNUSED_ENTRY(680), + IAVF_PTT_UNUSED_ENTRY(681), + IAVF_PTT_UNUSED_ENTRY(682), + IAVF_PTT_UNUSED_ENTRY(683), + IAVF_PTT_UNUSED_ENTRY(684), + IAVF_PTT_UNUSED_ENTRY(685), + IAVF_PTT_UNUSED_ENTRY(686), + IAVF_PTT_UNUSED_ENTRY(687), + IAVF_PTT_UNUSED_ENTRY(688), + IAVF_PTT_UNUSED_ENTRY(689), + + IAVF_PTT_UNUSED_ENTRY(690), + IAVF_PTT_UNUSED_ENTRY(691), + IAVF_PTT_UNUSED_ENTRY(692), + IAVF_PTT_UNUSED_ENTRY(693), + IAVF_PTT_UNUSED_ENTRY(694), + IAVF_PTT_UNUSED_ENTRY(695), + IAVF_PTT_UNUSED_ENTRY(696), + IAVF_PTT_UNUSED_ENTRY(697), + IAVF_PTT_UNUSED_ENTRY(698), + IAVF_PTT_UNUSED_ENTRY(699), + + IAVF_PTT_UNUSED_ENTRY(700), + IAVF_PTT_UNUSED_ENTRY(701), + IAVF_PTT_UNUSED_ENTRY(702), + IAVF_PTT_UNUSED_ENTRY(703), + IAVF_PTT_UNUSED_ENTRY(704), + IAVF_PTT_UNUSED_ENTRY(705), + IAVF_PTT_UNUSED_ENTRY(706), + IAVF_PTT_UNUSED_ENTRY(707), + IAVF_PTT_UNUSED_ENTRY(708), + IAVF_PTT_UNUSED_ENTRY(709), + + IAVF_PTT_UNUSED_ENTRY(710), + IAVF_PTT_UNUSED_ENTRY(711), + IAVF_PTT_UNUSED_ENTRY(712), + IAVF_PTT_UNUSED_ENTRY(713), + IAVF_PTT_UNUSED_ENTRY(714), + IAVF_PTT_UNUSED_ENTRY(715), + IAVF_PTT_UNUSED_ENTRY(716), + IAVF_PTT_UNUSED_ENTRY(717), + IAVF_PTT_UNUSED_ENTRY(718), + IAVF_PTT_UNUSED_ENTRY(719), + + IAVF_PTT_UNUSED_ENTRY(720), + IAVF_PTT_UNUSED_ENTRY(721), + IAVF_PTT_UNUSED_ENTRY(722), + IAVF_PTT_UNUSED_ENTRY(723), + IAVF_PTT_UNUSED_ENTRY(724), + IAVF_PTT_UNUSED_ENTRY(725), + IAVF_PTT_UNUSED_ENTRY(726), + IAVF_PTT_UNUSED_ENTRY(727), + IAVF_PTT_UNUSED_ENTRY(728), + IAVF_PTT_UNUSED_ENTRY(729), + + IAVF_PTT_UNUSED_ENTRY(730), + IAVF_PTT_UNUSED_ENTRY(731), + IAVF_PTT_UNUSED_ENTRY(732), + IAVF_PTT_UNUSED_ENTRY(733), + IAVF_PTT_UNUSED_ENTRY(734), + IAVF_PTT_UNUSED_ENTRY(735), + IAVF_PTT_UNUSED_ENTRY(736), + IAVF_PTT_UNUSED_ENTRY(737), + IAVF_PTT_UNUSED_ENTRY(738), + IAVF_PTT_UNUSED_ENTRY(739), + + IAVF_PTT_UNUSED_ENTRY(740), + IAVF_PTT_UNUSED_ENTRY(741), + IAVF_PTT_UNUSED_ENTRY(742), + IAVF_PTT_UNUSED_ENTRY(743), + IAVF_PTT_UNUSED_ENTRY(744), + IAVF_PTT_UNUSED_ENTRY(745), + IAVF_PTT_UNUSED_ENTRY(746), + IAVF_PTT_UNUSED_ENTRY(747), + IAVF_PTT_UNUSED_ENTRY(748), + IAVF_PTT_UNUSED_ENTRY(749), + + IAVF_PTT_UNUSED_ENTRY(750), + IAVF_PTT_UNUSED_ENTRY(751), + IAVF_PTT_UNUSED_ENTRY(752), + IAVF_PTT_UNUSED_ENTRY(753), + IAVF_PTT_UNUSED_ENTRY(754), + IAVF_PTT_UNUSED_ENTRY(755), + IAVF_PTT_UNUSED_ENTRY(756), + IAVF_PTT_UNUSED_ENTRY(757), + IAVF_PTT_UNUSED_ENTRY(758), + IAVF_PTT_UNUSED_ENTRY(759), + + IAVF_PTT_UNUSED_ENTRY(760), + IAVF_PTT_UNUSED_ENTRY(761), + IAVF_PTT_UNUSED_ENTRY(762), + IAVF_PTT_UNUSED_ENTRY(763), + IAVF_PTT_UNUSED_ENTRY(764), + IAVF_PTT_UNUSED_ENTRY(765), + IAVF_PTT_UNUSED_ENTRY(766), + IAVF_PTT_UNUSED_ENTRY(767), + IAVF_PTT_UNUSED_ENTRY(768), + IAVF_PTT_UNUSED_ENTRY(769), + + IAVF_PTT_UNUSED_ENTRY(770), + IAVF_PTT_UNUSED_ENTRY(771), + IAVF_PTT_UNUSED_ENTRY(772), + IAVF_PTT_UNUSED_ENTRY(773), + IAVF_PTT_UNUSED_ENTRY(774), + IAVF_PTT_UNUSED_ENTRY(775), + IAVF_PTT_UNUSED_ENTRY(776), + IAVF_PTT_UNUSED_ENTRY(777), + IAVF_PTT_UNUSED_ENTRY(778), + IAVF_PTT_UNUSED_ENTRY(779), + + IAVF_PTT_UNUSED_ENTRY(780), + IAVF_PTT_UNUSED_ENTRY(781), + IAVF_PTT_UNUSED_ENTRY(782), + IAVF_PTT_UNUSED_ENTRY(783), + IAVF_PTT_UNUSED_ENTRY(784), + IAVF_PTT_UNUSED_ENTRY(785), + IAVF_PTT_UNUSED_ENTRY(786), + IAVF_PTT_UNUSED_ENTRY(787), + IAVF_PTT_UNUSED_ENTRY(788), + IAVF_PTT_UNUSED_ENTRY(789), + + IAVF_PTT_UNUSED_ENTRY(790), + IAVF_PTT_UNUSED_ENTRY(791), + IAVF_PTT_UNUSED_ENTRY(792), + IAVF_PTT_UNUSED_ENTRY(793), + IAVF_PTT_UNUSED_ENTRY(794), + IAVF_PTT_UNUSED_ENTRY(795), + IAVF_PTT_UNUSED_ENTRY(796), + IAVF_PTT_UNUSED_ENTRY(797), + IAVF_PTT_UNUSED_ENTRY(798), + IAVF_PTT_UNUSED_ENTRY(799), + + IAVF_PTT_UNUSED_ENTRY(800), + IAVF_PTT_UNUSED_ENTRY(801), + IAVF_PTT_UNUSED_ENTRY(802), + IAVF_PTT_UNUSED_ENTRY(803), + IAVF_PTT_UNUSED_ENTRY(804), + IAVF_PTT_UNUSED_ENTRY(805), + IAVF_PTT_UNUSED_ENTRY(806), + IAVF_PTT_UNUSED_ENTRY(807), + IAVF_PTT_UNUSED_ENTRY(808), + IAVF_PTT_UNUSED_ENTRY(809), + + IAVF_PTT_UNUSED_ENTRY(810), + IAVF_PTT_UNUSED_ENTRY(811), + IAVF_PTT_UNUSED_ENTRY(812), + IAVF_PTT_UNUSED_ENTRY(813), + IAVF_PTT_UNUSED_ENTRY(814), + IAVF_PTT_UNUSED_ENTRY(815), + IAVF_PTT_UNUSED_ENTRY(816), + IAVF_PTT_UNUSED_ENTRY(817), + IAVF_PTT_UNUSED_ENTRY(818), + IAVF_PTT_UNUSED_ENTRY(819), + + IAVF_PTT_UNUSED_ENTRY(820), + IAVF_PTT_UNUSED_ENTRY(821), + IAVF_PTT_UNUSED_ENTRY(822), + IAVF_PTT_UNUSED_ENTRY(823), + IAVF_PTT_UNUSED_ENTRY(824), + IAVF_PTT_UNUSED_ENTRY(825), + IAVF_PTT_UNUSED_ENTRY(826), + IAVF_PTT_UNUSED_ENTRY(827), + IAVF_PTT_UNUSED_ENTRY(828), + IAVF_PTT_UNUSED_ENTRY(829), + + IAVF_PTT_UNUSED_ENTRY(830), + IAVF_PTT_UNUSED_ENTRY(831), + IAVF_PTT_UNUSED_ENTRY(832), + IAVF_PTT_UNUSED_ENTRY(833), + IAVF_PTT_UNUSED_ENTRY(834), + IAVF_PTT_UNUSED_ENTRY(835), + IAVF_PTT_UNUSED_ENTRY(836), + IAVF_PTT_UNUSED_ENTRY(837), + IAVF_PTT_UNUSED_ENTRY(838), + IAVF_PTT_UNUSED_ENTRY(839), + + IAVF_PTT_UNUSED_ENTRY(840), + IAVF_PTT_UNUSED_ENTRY(841), + IAVF_PTT_UNUSED_ENTRY(842), + IAVF_PTT_UNUSED_ENTRY(843), + IAVF_PTT_UNUSED_ENTRY(844), + IAVF_PTT_UNUSED_ENTRY(845), + IAVF_PTT_UNUSED_ENTRY(846), + IAVF_PTT_UNUSED_ENTRY(847), + IAVF_PTT_UNUSED_ENTRY(848), + IAVF_PTT_UNUSED_ENTRY(849), + + IAVF_PTT_UNUSED_ENTRY(850), + IAVF_PTT_UNUSED_ENTRY(851), + IAVF_PTT_UNUSED_ENTRY(852), + IAVF_PTT_UNUSED_ENTRY(853), + IAVF_PTT_UNUSED_ENTRY(854), + IAVF_PTT_UNUSED_ENTRY(855), + IAVF_PTT_UNUSED_ENTRY(856), + IAVF_PTT_UNUSED_ENTRY(857), + IAVF_PTT_UNUSED_ENTRY(858), + IAVF_PTT_UNUSED_ENTRY(859), + + IAVF_PTT_UNUSED_ENTRY(860), + IAVF_PTT_UNUSED_ENTRY(861), + IAVF_PTT_UNUSED_ENTRY(862), + IAVF_PTT_UNUSED_ENTRY(863), + IAVF_PTT_UNUSED_ENTRY(864), + IAVF_PTT_UNUSED_ENTRY(865), + IAVF_PTT_UNUSED_ENTRY(866), + IAVF_PTT_UNUSED_ENTRY(867), + IAVF_PTT_UNUSED_ENTRY(868), + IAVF_PTT_UNUSED_ENTRY(869), + + IAVF_PTT_UNUSED_ENTRY(870), + IAVF_PTT_UNUSED_ENTRY(871), + IAVF_PTT_UNUSED_ENTRY(872), + IAVF_PTT_UNUSED_ENTRY(873), + IAVF_PTT_UNUSED_ENTRY(874), + IAVF_PTT_UNUSED_ENTRY(875), + IAVF_PTT_UNUSED_ENTRY(876), + IAVF_PTT_UNUSED_ENTRY(877), + IAVF_PTT_UNUSED_ENTRY(878), + IAVF_PTT_UNUSED_ENTRY(879), + + IAVF_PTT_UNUSED_ENTRY(880), + IAVF_PTT_UNUSED_ENTRY(881), + IAVF_PTT_UNUSED_ENTRY(882), + IAVF_PTT_UNUSED_ENTRY(883), + IAVF_PTT_UNUSED_ENTRY(884), + IAVF_PTT_UNUSED_ENTRY(885), + IAVF_PTT_UNUSED_ENTRY(886), + IAVF_PTT_UNUSED_ENTRY(887), + IAVF_PTT_UNUSED_ENTRY(888), + IAVF_PTT_UNUSED_ENTRY(889), + + IAVF_PTT_UNUSED_ENTRY(890), + IAVF_PTT_UNUSED_ENTRY(891), + IAVF_PTT_UNUSED_ENTRY(892), + IAVF_PTT_UNUSED_ENTRY(893), + IAVF_PTT_UNUSED_ENTRY(894), + IAVF_PTT_UNUSED_ENTRY(895), + IAVF_PTT_UNUSED_ENTRY(896), + IAVF_PTT_UNUSED_ENTRY(897), + IAVF_PTT_UNUSED_ENTRY(898), + IAVF_PTT_UNUSED_ENTRY(899), + + IAVF_PTT_UNUSED_ENTRY(900), + IAVF_PTT_UNUSED_ENTRY(901), + IAVF_PTT_UNUSED_ENTRY(902), + IAVF_PTT_UNUSED_ENTRY(903), + IAVF_PTT_UNUSED_ENTRY(904), + IAVF_PTT_UNUSED_ENTRY(905), + IAVF_PTT_UNUSED_ENTRY(906), + IAVF_PTT_UNUSED_ENTRY(907), + IAVF_PTT_UNUSED_ENTRY(908), + IAVF_PTT_UNUSED_ENTRY(909), + + IAVF_PTT_UNUSED_ENTRY(910), + IAVF_PTT_UNUSED_ENTRY(911), + IAVF_PTT_UNUSED_ENTRY(912), + IAVF_PTT_UNUSED_ENTRY(913), + IAVF_PTT_UNUSED_ENTRY(914), + IAVF_PTT_UNUSED_ENTRY(915), + IAVF_PTT_UNUSED_ENTRY(916), + IAVF_PTT_UNUSED_ENTRY(917), + IAVF_PTT_UNUSED_ENTRY(918), + IAVF_PTT_UNUSED_ENTRY(919), + + IAVF_PTT_UNUSED_ENTRY(920), + IAVF_PTT_UNUSED_ENTRY(921), + IAVF_PTT_UNUSED_ENTRY(922), + IAVF_PTT_UNUSED_ENTRY(923), + IAVF_PTT_UNUSED_ENTRY(924), + IAVF_PTT_UNUSED_ENTRY(925), + IAVF_PTT_UNUSED_ENTRY(926), + IAVF_PTT_UNUSED_ENTRY(927), + IAVF_PTT_UNUSED_ENTRY(928), + IAVF_PTT_UNUSED_ENTRY(929), + + IAVF_PTT_UNUSED_ENTRY(930), + IAVF_PTT_UNUSED_ENTRY(931), + IAVF_PTT_UNUSED_ENTRY(932), + IAVF_PTT_UNUSED_ENTRY(933), + IAVF_PTT_UNUSED_ENTRY(934), + IAVF_PTT_UNUSED_ENTRY(935), + IAVF_PTT_UNUSED_ENTRY(936), + IAVF_PTT_UNUSED_ENTRY(937), + IAVF_PTT_UNUSED_ENTRY(938), + IAVF_PTT_UNUSED_ENTRY(939), + + IAVF_PTT_UNUSED_ENTRY(940), + IAVF_PTT_UNUSED_ENTRY(941), + IAVF_PTT_UNUSED_ENTRY(942), + IAVF_PTT_UNUSED_ENTRY(943), + IAVF_PTT_UNUSED_ENTRY(944), + IAVF_PTT_UNUSED_ENTRY(945), + IAVF_PTT_UNUSED_ENTRY(946), + IAVF_PTT_UNUSED_ENTRY(947), + IAVF_PTT_UNUSED_ENTRY(948), + IAVF_PTT_UNUSED_ENTRY(949), + + IAVF_PTT_UNUSED_ENTRY(950), + IAVF_PTT_UNUSED_ENTRY(951), + IAVF_PTT_UNUSED_ENTRY(952), + IAVF_PTT_UNUSED_ENTRY(953), + IAVF_PTT_UNUSED_ENTRY(954), + IAVF_PTT_UNUSED_ENTRY(955), + IAVF_PTT_UNUSED_ENTRY(956), + IAVF_PTT_UNUSED_ENTRY(957), + IAVF_PTT_UNUSED_ENTRY(958), + IAVF_PTT_UNUSED_ENTRY(959), + + IAVF_PTT_UNUSED_ENTRY(960), + IAVF_PTT_UNUSED_ENTRY(961), + IAVF_PTT_UNUSED_ENTRY(962), + IAVF_PTT_UNUSED_ENTRY(963), + IAVF_PTT_UNUSED_ENTRY(964), + IAVF_PTT_UNUSED_ENTRY(965), + IAVF_PTT_UNUSED_ENTRY(966), + IAVF_PTT_UNUSED_ENTRY(967), + IAVF_PTT_UNUSED_ENTRY(968), + IAVF_PTT_UNUSED_ENTRY(969), + + IAVF_PTT_UNUSED_ENTRY(970), + IAVF_PTT_UNUSED_ENTRY(971), + IAVF_PTT_UNUSED_ENTRY(972), + IAVF_PTT_UNUSED_ENTRY(973), + IAVF_PTT_UNUSED_ENTRY(974), + IAVF_PTT_UNUSED_ENTRY(975), + IAVF_PTT_UNUSED_ENTRY(976), + IAVF_PTT_UNUSED_ENTRY(977), + IAVF_PTT_UNUSED_ENTRY(978), + IAVF_PTT_UNUSED_ENTRY(979), + + IAVF_PTT_UNUSED_ENTRY(980), + IAVF_PTT_UNUSED_ENTRY(981), + IAVF_PTT_UNUSED_ENTRY(982), + IAVF_PTT_UNUSED_ENTRY(983), + IAVF_PTT_UNUSED_ENTRY(984), + IAVF_PTT_UNUSED_ENTRY(985), + IAVF_PTT_UNUSED_ENTRY(986), + IAVF_PTT_UNUSED_ENTRY(987), + IAVF_PTT_UNUSED_ENTRY(988), + IAVF_PTT_UNUSED_ENTRY(989), + + IAVF_PTT_UNUSED_ENTRY(990), + IAVF_PTT_UNUSED_ENTRY(991), + IAVF_PTT_UNUSED_ENTRY(992), + IAVF_PTT_UNUSED_ENTRY(993), + IAVF_PTT_UNUSED_ENTRY(994), + IAVF_PTT_UNUSED_ENTRY(995), + IAVF_PTT_UNUSED_ENTRY(996), + IAVF_PTT_UNUSED_ENTRY(997), + IAVF_PTT_UNUSED_ENTRY(998), + IAVF_PTT_UNUSED_ENTRY(999), + + IAVF_PTT_UNUSED_ENTRY(1000), + IAVF_PTT_UNUSED_ENTRY(1001), + IAVF_PTT_UNUSED_ENTRY(1002), + IAVF_PTT_UNUSED_ENTRY(1003), + IAVF_PTT_UNUSED_ENTRY(1004), + IAVF_PTT_UNUSED_ENTRY(1005), + IAVF_PTT_UNUSED_ENTRY(1006), + IAVF_PTT_UNUSED_ENTRY(1007), + IAVF_PTT_UNUSED_ENTRY(1008), + IAVF_PTT_UNUSED_ENTRY(1009), + + IAVF_PTT_UNUSED_ENTRY(1010), + IAVF_PTT_UNUSED_ENTRY(1011), + IAVF_PTT_UNUSED_ENTRY(1012), + IAVF_PTT_UNUSED_ENTRY(1013), + IAVF_PTT_UNUSED_ENTRY(1014), + IAVF_PTT_UNUSED_ENTRY(1015), + IAVF_PTT_UNUSED_ENTRY(1016), + IAVF_PTT_UNUSED_ENTRY(1017), + IAVF_PTT_UNUSED_ENTRY(1018), + IAVF_PTT_UNUSED_ENTRY(1019), + + IAVF_PTT_UNUSED_ENTRY(1020), + IAVF_PTT_UNUSED_ENTRY(1021), + IAVF_PTT_UNUSED_ENTRY(1022), + IAVF_PTT_UNUSED_ENTRY(1023), }; /** @@ -885,7 +1730,7 @@ struct iavf_rx_ptype_decoded iavf_ptype_lookup[] = { **/ enum iavf_status iavf_aq_send_msg_to_pf(struct iavf_hw *hw, enum virtchnl_ops v_opcode, - enum iavf_status v_retval, + enum virtchnl_status_code v_retval, u8 *msg, u16 msglen, struct iavf_asq_cmd_details *cmd_details) { @@ -936,6 +1781,7 @@ void iavf_vf_parse_hw_config(struct iavf_hw *hw, hw->dev_caps.num_msix_vectors_vf = msg->max_vectors; hw->dev_caps.dcb = msg->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_L2; + hw->dev_caps.max_mtu = msg->max_mtu; for (i = 0; i < msg->num_vsis; i++) { if (vsi_res->vsi_type == VIRTCHNL_VSI_SRIOV) { ether_addr_copy(hw->mac.perm_addr, @@ -958,7 +1804,7 @@ void iavf_vf_parse_hw_config(struct iavf_hw *hw, enum iavf_status iavf_vf_reset(struct iavf_hw *hw) { return iavf_aq_send_msg_to_pf(hw, VIRTCHNL_OP_RESET_VF, - IAVF_SUCCESS, NULL, 0, NULL); + VIRTCHNL_STATUS_SUCCESS, NULL, 0, NULL); } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index b9cf53750..67075abc6 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -33,6 +33,8 @@ static const struct iavf_stats iavf_gstrings_stats[] = { VF_STAT("tx_broadcast", current_stats.tx_broadcast), VF_STAT("tx_discards", current_stats.tx_discards), VF_STAT("tx_errors", current_stats.tx_errors), + VF_STAT("tx_hwtstamp_skipped", ptp.tx_hwtstamp_skipped), + VF_STAT("tx_hwtstamp_timeouts", ptp.tx_hwtstamp_timeouts), #ifdef IAVF_ADD_PROBES VF_STAT("tx_tcp_segments", tcp_segs), VF_STAT("tx_udp_segments", udp_segs), @@ -41,11 +43,13 @@ static const struct iavf_stats iavf_gstrings_stats[] = { VF_STAT("tx_sctp_cso", tx_sctp_cso), VF_STAT("tx_ip4_cso", tx_ip4_cso), VF_STAT("tx_vlano", tx_vlano), + VF_STAT("tx_ad_vlano", tx_ad_vlano), VF_STAT("rx_tcp_cso", rx_tcp_cso), VF_STAT("rx_udp_cso", rx_udp_cso), VF_STAT("rx_sctp_cso", rx_sctp_cso), VF_STAT("rx_ip4_cso", rx_ip4_cso), VF_STAT("rx_vlano", rx_vlano), + VF_STAT("rx_ad_vlano", rx_ad_vlano), VF_STAT("rx_tcp_cso_error", rx_tcp_cso_err), VF_STAT("rx_udp_cso_error", rx_udp_cso_err), VF_STAT("rx_sctp_cso_error", rx_sctp_cso_err), @@ -55,7 +59,11 @@ static const struct iavf_stats iavf_gstrings_stats[] = { #define IAVF_STATS_LEN ARRAY_SIZE(iavf_gstrings_stats) -#define IAVF_QUEUE_STATS_LEN ARRAY_SIZE(iavf_gstrings_queue_stats) +#define IAVF_QUEUE_STATS_LEN (ARRAY_SIZE(iavf_gstrings_queue_stats) + \ + ARRAY_SIZE(iavf_gstrings_queue_stats_poll)) +#define IAVF_TX_QUEUE_STATS_LEN ARRAY_SIZE(iavf_gstrings_queue_stats_tx) +#define IAVF_RX_QUEUE_STATS_LEN ARRAY_SIZE(iavf_gstrings_queue_stats_rx) +#define IAVF_VECTOR_STATS_LEN ARRAY_SIZE(iavf_gstrings_queue_stats_vector) #ifdef HAVE_SWIOTLB_SKIP_CPU_SYNC /* For now we have one and only one private flag and it is only defined @@ -80,6 +88,13 @@ static const struct iavf_priv_flags iavf_gstrings_priv_flags[] = { }; #define IAVF_PRIV_FLAGS_STR_LEN ARRAY_SIZE(iavf_gstrings_priv_flags) + +static const struct iavf_priv_flags iavf_gstrings_chnl_priv_flags[] = { + IAVF_PRIV_FLAG("channel-pkt-inspect-optimize", + IAVF_FLAG_CHNL_PKT_OPT_ENA, 0), +}; +#define IAVF_CHNL_PRIV_FLAGS_STR_LEN ARRAY_SIZE(iavf_gstrings_chnl_priv_flags) + #endif /* HAVE_SWIOTLB_SKIP_CPU_SYNC */ /** @@ -100,15 +115,17 @@ static int iavf_get_link_ksettings(struct net_device *netdev, cmd->base.autoneg = AUTONEG_DISABLE; cmd->base.port = PORT_NONE; - /* Set speed and duplex */ + cmd->base.duplex = DUPLEX_FULL; + #ifdef VIRTCHNL_VF_CAP_ADV_LINK_SPEED if (ADV_LINK_SUPPORT(adapter)) { - if (SUPPORTED_SPEED(adapter->link_speed_mbps)) + if (adapter->link_speed_mbps && + adapter->link_speed_mbps < U32_MAX) cmd->base.speed = adapter->link_speed_mbps; else cmd->base.speed = SPEED_UNKNOWN; - goto set_duplex; + return 0; } #endif /* VIRTCHNL_VF_CAP_ADV_LINK_SPEED */ @@ -130,6 +147,12 @@ static int iavf_get_link_ksettings(struct net_device *netdev, case VIRTCHNL_LINK_SPEED_10GB: cmd->base.speed = SPEED_10000; break; + case VIRTCHNL_LINK_SPEED_5GB: + cmd->base.speed = SPEED_5000; + break; + case VIRTCHNL_LINK_SPEED_2_5GB: + cmd->base.speed = SPEED_2500; + break; case VIRTCHNL_LINK_SPEED_1GB: cmd->base.speed = SPEED_1000; break; @@ -141,11 +164,6 @@ static int iavf_get_link_ksettings(struct net_device *netdev, break; } -#ifdef VIRTCHNL_VF_CAP_ADV_LINK_SPEED -set_duplex: -#endif /* VIRTCHNL_VF_CAP_ADV_LINK_SPEED */ - cmd->base.duplex = DUPLEX_FULL; - return 0; } @@ -180,12 +198,20 @@ static int iavf_get_settings(struct net_device *netdev, **/ static int iavf_get_sset_count(struct net_device *netdev, int sset) { + /* Report the maximum number queues, even if not every queue is + * currently configured. Since allocation of queues is in pairs, + * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set + * at device creation and never changes. + */ if (sset == ETH_SS_STATS) return IAVF_STATS_LEN + - (IAVF_QUEUE_STATS_LEN * 2 * IAVF_MAX_REQ_QUEUES); + (IAVF_QUEUE_STATS_LEN * 2 * + netdev->real_num_tx_queues) + + ((IAVF_TX_QUEUE_STATS_LEN + IAVF_RX_QUEUE_STATS_LEN + + IAVF_VECTOR_STATS_LEN) * netdev->real_num_tx_queues); #ifdef HAVE_SWIOTLB_SKIP_CPU_SYNC else if (sset == ETH_SS_PRIV_FLAGS) - return IAVF_PRIV_FLAGS_STR_LEN; + return IAVF_PRIV_FLAGS_STR_LEN + IAVF_PRIV_FLAGS_STR_LEN; #endif else return -EINVAL; @@ -205,21 +231,30 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, struct iavf_adapter *adapter = netdev_priv(netdev); unsigned int i; + /* Explicitly request stats refresh */ + iavf_schedule_request_stats(adapter); + iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats); rcu_read_lock(); - for (i = 0; i < IAVF_MAX_REQ_QUEUES; i++) { + /* As num_active_queues describe both tx and rx queues, we can use + * it to iterate over rings' stats. + */ + for (i = 0; i < adapter->num_active_queues; i++) { struct iavf_ring *ring; - /* Avoid accessing un-allocated queues */ - ring = (i < adapter->num_active_queues ? - &adapter->tx_rings[i] : NULL); + /* Tx rings stats */ + ring = &adapter->tx_rings[i]; iavf_add_queue_stats(&data, ring); + iavf_add_queue_stats_chnl(&data, ring, IAVF_CHNL_STAT_POLL); + iavf_add_queue_stats_chnl(&data, ring, IAVF_CHNL_STAT_TX); - /* Avoid accessing un-allocated queues */ - ring = (i < adapter->num_active_queues ? - &adapter->rx_rings[i] : NULL); + /* Rx rings stats */ + ring = &adapter->rx_rings[i]; iavf_add_queue_stats(&data, ring); + iavf_add_queue_stats_chnl(&data, ring, IAVF_CHNL_STAT_POLL); + iavf_add_queue_stats_chnl(&data, ring, IAVF_CHNL_STAT_RX); + iavf_add_queue_stats_chnl(&data, ring, IAVF_CHNL_STAT_VECTOR); } rcu_read_unlock(); } @@ -241,6 +276,12 @@ static void iavf_get_priv_flag_strings(struct net_device *netdev, u8 *data) iavf_gstrings_priv_flags[i].flag_string); data += ETH_GSTRING_LEN; } + + for (i = 0; i < IAVF_CHNL_PRIV_FLAGS_STR_LEN; i++) { + snprintf(data, ETH_GSTRING_LEN, "%s", + iavf_gstrings_chnl_priv_flags[i].flag_string); + data += ETH_GSTRING_LEN; + } } #endif @@ -257,14 +298,24 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) iavf_add_stat_strings(&data, iavf_gstrings_stats); - /* Queues are always allocated in pairs, so we just use num_tx_queues - * for both Tx and Rx queues. + /* Queues are always allocated in pairs, so we just use + * real_num_tx_queues for both Tx and Rx queues. */ - for (i = 0; i < netdev->num_tx_queues; i++) { + for (i = 0; i < netdev->real_num_tx_queues; i++) { iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, "tx", i); + iavf_add_stat_strings(&data, iavf_gstrings_queue_stats_poll, + "tx", i); + iavf_add_stat_strings(&data, iavf_gstrings_queue_stats_tx, + "tx", i); iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, "rx", i); + iavf_add_stat_strings(&data, iavf_gstrings_queue_stats_poll, + "rx", i); + iavf_add_stat_strings(&data, iavf_gstrings_queue_stats_rx, + "rx", i); + iavf_add_stat_strings(&data, iavf_gstrings_queue_stats_vector, + "rx", i); } } @@ -306,40 +357,50 @@ static void iavf_get_strings(struct net_device *netdev, u32 sset, u8 *data) static u32 iavf_get_priv_flags(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); + const struct iavf_priv_flags *priv_flags; u32 i, ret_flags = 0; for (i = 0; i < IAVF_PRIV_FLAGS_STR_LEN; i++) { - const struct iavf_priv_flags *priv_flags; - priv_flags = &iavf_gstrings_priv_flags[i]; if (priv_flags->flag & adapter->flags) ret_flags |= BIT(i); } + for (i = 0; i < IAVF_CHNL_PRIV_FLAGS_STR_LEN; i++) { + priv_flags = &iavf_gstrings_chnl_priv_flags[i]; + + if (priv_flags->flag & adapter->chnl_perf_flags) + ret_flags |= BIT(i + IAVF_PRIV_FLAGS_STR_LEN); + } + return ret_flags; } /** - * iavf_set_priv_flags - set private flags - * @netdev: network interface device structure + * iavf_determine_priv_flag_change - detect any change in private flags + * @priv_flags: Ptr to private flags array + * @num: count of private flags + * @bit_offset: "offset" into unified view of bits * @flags: bit flags to be set + * @orig: Ptr to flags + * @changed_flags: bits changed (based on orig and new value) + * + * Detect any changes in priv flags and return those changed bits **/ -static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) +static int +iavf_determine_priv_flag_change(const struct iavf_priv_flags *priv_flags, + int num, int bit_offset, u32 flags, u32 *orig, + u32 *changed_flags) { - struct iavf_adapter *adapter = netdev_priv(netdev); - u32 orig_flags, new_flags, changed_flags; - u32 i; + u32 orig_flags = READ_ONCE(*orig); + u32 new_flags; + int i; - orig_flags = READ_ONCE(adapter->flags); new_flags = orig_flags; - for (i = 0; i < IAVF_PRIV_FLAGS_STR_LEN; i++) { - const struct iavf_priv_flags *priv_flags; - - priv_flags = &iavf_gstrings_priv_flags[i]; - - if (flags & BIT(i)) + for (i = 0; i < num; i++) { + if (flags & BIT(i + bit_offset)) new_flags |= priv_flags->flag; else new_flags &= ~(priv_flags->flag); @@ -347,6 +408,7 @@ static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) if (priv_flags->read_only && ((orig_flags ^ new_flags) & ~BIT(i))) return -EOPNOTSUPP; + priv_flags++; } /* Before we finalize any flag changes, any checks which we need to @@ -360,13 +422,47 @@ static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) * copied it. We'll just punt with an error and log something in the * message buffer. */ - if (cmpxchg(&adapter->flags, orig_flags, new_flags) != orig_flags) { - dev_warn(&adapter->pdev->dev, - "Unable to update adapter->flags as it was modified by another thread...\n"); + if (cmpxchg(orig, orig_flags, new_flags) != orig_flags) return -EAGAIN; + + *changed_flags = orig_flags ^ new_flags; + return 0; +} + +/** + * iavf_set_priv_flags - set private flags + * @netdev: network interface device structure + * @flags: bit flags to be set + **/ +static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) +{ + struct iavf_adapter *adapter = netdev_priv(netdev); + u32 changed_chnl_flags; + u32 changed_flags; + int ret; + + ret = iavf_determine_priv_flag_change(&iavf_gstrings_priv_flags[0], + IAVF_PRIV_FLAGS_STR_LEN, 0, + flags, &adapter->flags, + &changed_flags); + if (ret) { + if (ret == -EAGAIN) + dev_warn(&adapter->pdev->dev, + "Unable to update adapter->flags as it was modified by another thread...\n"); + return ret; } - changed_flags = orig_flags ^ new_flags; + ret = iavf_determine_priv_flag_change(&iavf_gstrings_chnl_priv_flags[0], + IAVF_CHNL_PRIV_FLAGS_STR_LEN, + IAVF_PRIV_FLAGS_STR_LEN, + flags, &adapter->chnl_perf_flags, + &changed_chnl_flags); + if (ret) { + if (ret == -EAGAIN) + dev_warn(&adapter->pdev->dev, + "Unable to update adapter->chnl_perf_flags as it was modified by another thread...\n"); + return ret; + } /* Process any additional changes needed as a result of flag changes. * The changed_flags value reflects the list of bits that were changed @@ -378,6 +474,10 @@ static int iavf_set_priv_flags(struct net_device *netdev, u32 flags) if (netif_running(netdev)) iavf_schedule_reset(adapter); } + /* Process any additional changes needed as a result of change + * in channel specific flag(s) + */ + iavf_setup_ch_info(adapter, changed_chnl_flags); return 0; } @@ -568,23 +668,44 @@ static int iavf_set_ringparam(struct net_device *netdev, if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) return -EINVAL; - new_tx_count = clamp_t(u32, ring->tx_pending, - IAVF_MIN_TXD, - IAVF_MAX_TXD); - new_tx_count = ALIGN(new_tx_count, IAVF_REQ_DESCRIPTOR_MULTIPLE); + if (ring->tx_pending > IAVF_MAX_TXD || + ring->tx_pending < IAVF_MIN_TXD || + ring->rx_pending > IAVF_MAX_RXD || + ring->rx_pending < IAVF_MIN_RXD) { + netdev_err(netdev, "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d] (increment %d)\n", + ring->tx_pending, ring->rx_pending, IAVF_MIN_TXD, + IAVF_MAX_RXD, IAVF_REQ_DESCRIPTOR_MULTIPLE); + return -EINVAL; + } - new_rx_count = clamp_t(u32, ring->rx_pending, - IAVF_MIN_RXD, - IAVF_MAX_RXD); - new_rx_count = ALIGN(new_rx_count, IAVF_REQ_DESCRIPTOR_MULTIPLE); + new_tx_count = ALIGN(ring->tx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE); + if (new_tx_count != ring->tx_pending) + netdev_info(netdev, "Requested Tx descriptor count rounded up to %d\n", + new_tx_count); + + new_rx_count = ALIGN(ring->rx_pending, IAVF_REQ_DESCRIPTOR_MULTIPLE); + if (new_rx_count != ring->rx_pending) + netdev_info(netdev, "Requested Rx descriptor count rounded up to %d\n", + new_rx_count); /* if nothing to do return success */ if ((new_tx_count == adapter->tx_desc_count) && - (new_rx_count == adapter->rx_desc_count)) + (new_rx_count == adapter->rx_desc_count)) { + netdev_dbg(netdev, "Nothing to change, descriptor count is same as requested\n"); return 0; + } - adapter->tx_desc_count = new_tx_count; - adapter->rx_desc_count = new_rx_count; + if (new_tx_count != adapter->tx_desc_count) { + netdev_info(netdev, "Changing Tx descriptor count from %d to %d\n", + adapter->tx_desc_count, new_tx_count); + adapter->tx_desc_count = new_tx_count; + } + + if (new_rx_count != adapter->rx_desc_count) { + netdev_info(netdev, "Changing Rx descriptor count from %d to %d\n", + adapter->rx_desc_count, new_rx_count); + adapter->rx_desc_count = new_rx_count; + } if (netif_running(netdev)) iavf_schedule_reset(adapter); @@ -882,12 +1003,12 @@ static int iavf_set_channels(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); u32 num_req = ch->combined_count; + int i; #ifdef __TC_MQPRIO_MODE_MAX - if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && - adapter->num_tc) { + if (iavf_is_adq_enabled(adapter)) { dev_info(&adapter->pdev->dev, "Cannot set channels since ADQ is enabled.\n"); - return -EINVAL; + return -EOPNOTSUPP; } #endif /* __TC_MQPRIO_MODE_MAX */ @@ -906,6 +1027,20 @@ static int iavf_set_channels(struct net_device *netdev, adapter->num_req_queues = num_req; adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; iavf_schedule_reset(adapter); + + /* wait for the reset is done */ + for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { + msleep(IAVF_RESET_WAIT_MS); + if (adapter->flags & IAVF_FLAG_RESET_PENDING) + continue; + break; + } + if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { + adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; + adapter->num_active_queues = num_req; + return -EOPNOTSUPP; + } + return 0; } @@ -999,6 +1134,14 @@ static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, struct iavf_adapter *adapter = netdev_priv(netdev); u16 i; +#ifdef __TC_MQPRIO_MODE_MAX + if (iavf_is_adq_enabled(adapter)) { + dev_info(&adapter->pdev->dev, + "Change in RSS params is not supported when ADQ is configured.\n"); + return -EOPNOTSUPP; + } +#endif /* __TC_MQPRIO_MODE_MAX */ + #ifdef HAVE_RXFH_HASHFUNC if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; @@ -1025,7 +1168,58 @@ static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, } #endif /* ETHTOOL_GRSSH && ETHTOOL_SRSSH */ +#ifdef HAVE_ETHTOOL_GET_TS_INFO +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) +/** + * iavf_get_ts_info - Report available timestamping capabilities + * @netdev: the netdevice to report for + * @info: structure to fill in + * + * Based on device features enabled, report the Tx and Rx timestamp + * capabilities, as well as the PTP hardware clock index to user space. + */ +static int iavf_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) +{ + struct iavf_adapter *adapter = netdev_priv(netdev); + + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE; + + if (iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_TX_TSTAMP)) { + info->so_timestamping |= SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + } + + /* Rx timestamps are only supported on the flexible descriptors. Do + * not report support unless we both have the capability and + * configured with the appropriate descriptor format + */ + if (iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP) && + adapter->rxdid == VIRTCHNL_RXDID_2_FLEX_SQ_NIC) { + info->so_timestamping |= SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | BIT(HWTSTAMP_FILTER_ALL); + } + + if (adapter->ptp.initialized) + info->phc_index = ptp_clock_index(adapter->ptp.clock); + else + info->phc_index = -1; + + return 0; +} +#endif /* CONFIG_PTP_1588_CLOCK */ +#endif /* HAVE_ETHTOOL_GET_TS_INFO */ + static const struct ethtool_ops iavf_ethtool_ops = { +#ifdef ETHTOOL_COALESCE_USECS + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_MAX_FRAMES_IRQ | + ETHTOOL_COALESCE_USE_ADAPTIVE, +#endif /* ETHTOOL_COALESCE_USECS */ .get_drvinfo = iavf_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = iavf_get_ringparam, @@ -1075,6 +1269,11 @@ static const struct ethtool_ops iavf_ethtool_ops = { #else .get_settings = iavf_get_settings, #endif /* ETHTOOL_GLINKSETTINGS */ +#ifdef HAVE_ETHTOOL_GET_TS_INFO +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) + .get_ts_info = iavf_get_ts_info, +#endif +#endif }; #ifdef HAVE_RHEL6_ETHTOOL_OPS_EXT_STRUCT diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool_stats.h b/drivers/net/ethernet/intel/iavf/iavf_ethtool_stats.h index 8ffb919ff..422367e76 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool_stats.h +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool_stats.h @@ -52,6 +52,80 @@ static const struct iavf_stats iavf_gstrings_queue_stats[] = { IAVF_QUEUE_STAT("%s-%u.bytes", stats.bytes), }; +#define IAVF_VECTOR_STAT(_name, _stat) \ + IAVF_STAT(struct iavf_q_vector, _name, _stat) + +/* Stats associated with a Tx or Rx ring */ +static struct iavf_stats iavf_gstrings_queue_stats_poll[] = { + IAVF_QUEUE_STAT("%s-%u.pkt_busy_poll", ch_q_stats.poll.pkt_busy_poll), + IAVF_QUEUE_STAT("%s-%u.pkt_not_busy_poll", + ch_q_stats.poll.pkt_not_busy_poll), +}; + +static struct iavf_stats iavf_gstrings_queue_stats_tx[] = { +}; + +static struct iavf_stats iavf_gstrings_queue_stats_rx[] = { + IAVF_QUEUE_STAT("%s-%u.tcp_ctrl_pkts", ch_q_stats.rx.tcp_ctrl_pkts), + IAVF_QUEUE_STAT("%s-%u.only_ctrl_pkts", ch_q_stats.rx.only_ctrl_pkts), + IAVF_QUEUE_STAT("%s-%u.tcp_fin_recv", ch_q_stats.rx.tcp_fin_recv), + IAVF_QUEUE_STAT("%s-%u.tcp_rst_recv", ch_q_stats.rx.tcp_rst_recv), + IAVF_QUEUE_STAT("%s-%u.tcp_syn_recv", ch_q_stats.rx.tcp_syn_recv), + IAVF_QUEUE_STAT("%s-%u.bp_no_data_pkt", ch_q_stats.rx.bp_no_data_pkt), +}; + +static struct iavf_stats iavf_gstrings_queue_stats_vector[] = { + /* tracking BP, INT, BP->INT, INT->BP */ + IAVF_VECTOR_STAT("%s-%u.in_bp", ch_stats.in_bp), + IAVF_VECTOR_STAT("%s-%u.intr_to_bp", ch_stats.intr_to_bp), + IAVF_VECTOR_STAT("%s-%u.bp_to_bp", ch_stats.bp_to_bp), + IAVF_VECTOR_STAT("%s-%u.in_intr", ch_stats.in_intr), + IAVF_VECTOR_STAT("%s-%u.bp_to_intr", ch_stats.bp_to_intr), + IAVF_VECTOR_STAT("%s-%u.intr_to_intr", ch_stats.intr_to_intr), + + /* unlikely comeback to busy_poll */ + IAVF_VECTOR_STAT("%s-%u.unlikely_cb_to_bp", ch_stats.unlikely_cb_to_bp), + /* unlikely comeback to busy_poll and once_in_bp is true */ + IAVF_VECTOR_STAT("%s-%u.ucb_once_in_bp_true", + ch_stats.ucb_once_in_bp_true), + /* once_in_bp is false */ + IAVF_VECTOR_STAT("%s-%u.intr_once_in_bp_false", + ch_stats.intr_once_bp_false), + /* busy_poll stop due to need_resched() */ + IAVF_VECTOR_STAT("%s-%u.bp_stop_need_resched", + ch_stats.bp_stop_need_resched), + /* busy_poll stop due to possible due to timeout */ + IAVF_VECTOR_STAT("%s-%u.bp_stop_timeout", ch_stats.bp_stop_timeout), + /* Transition: BP->INT: previously cleaned data packets */ + IAVF_VECTOR_STAT("%s-%u.cleaned_any_data_pkt", + ch_stats.cleaned_any_data_pkt), + /* need_resched(), but didn't clean any data packets */ + IAVF_VECTOR_STAT("%s-%u.need_resched_no_data_pkt", + ch_stats.need_resched_no_data_pkt), + /* possible timeout(), but didn't clean any data packets */ + IAVF_VECTOR_STAT("%s-%u.timeout_no_data_pkt", + ch_stats.timeout_no_data_pkt), + /* number of SW triggered interrupt from napi_poll due to + * possible timeout detected + */ + IAVF_VECTOR_STAT("%s-%u.sw_intr_timeout", ch_stats.sw_intr_timeout), + /* number of SW triggered interrupt from service_task */ + IAVF_VECTOR_STAT("%s-%u.sw_intr_service_task", + ch_stats.sw_intr_serv_task), + /* number of times, SW triggered interrupt is not triggered from + * napi_poll even when unlikely_cb_to_bp is set, once_in_bp is set + * but ethtool private featute flag is off (for interrupt optimization + * strategy + */ + IAVF_VECTOR_STAT("%s-%u.no_sw_intr_opt_off", + ch_stats.no_sw_intr_opt_off), + /* number of times WB_ON_ITR is set */ + IAVF_VECTOR_STAT("%s-%u.wb_on_itr_set", ch_stats.wb_on_itr_set), + + /* enable SW triggered interrupt due to not_clean_complete */ + IAVF_VECTOR_STAT("%s-%u.sw_intr_not_cc", + ch_stats.intr_en_not_clean_complete), +}; /** * iavf_add_one_ethtool_stat - copy the stat into the supplied buffer @@ -138,6 +212,90 @@ __iavf_add_ethtool_stats(u64 **data, void *pointer, #define iavf_add_ethtool_stats(data, pointer, stats) \ __iavf_add_ethtool_stats(data, pointer, stats, ARRAY_SIZE(stats)) +enum iavf_chnl_stat_type { + IAVF_CHNL_STAT_INVALID, + IAVF_CHNL_STAT_POLL, + IAVF_CHNL_STAT_TX, + IAVF_CHNL_STAT_RX, + IAVF_CHNL_STAT_VECTOR, + IAVF_CHNL_STAT_LAST, /* This must be last */_ +}; + +/** + * iavf_add_queue_stats_chnl - copy channel specific queue stats + * @data: ethtool stats buffer + * @ring: the ring to copy + * @stat_type: stat_type could be TX/TX/VECTOR + * + * Queue statistics must be copied while protected by + * u64_stats_fetch_begin_irq, so we can't directly use iavf_add_ethtool_stats. + * Assumes that queue stats are defined in iavf_gstrings_queue_stats. If the + * ring pointer is null, zero out the queue stat values and update the data + * pointer. Otherwise safely copy the stats from the ring into the supplied + * buffer and update the data pointer when finished. + * + * This function expects to be called while under rcu_read_lock(). + **/ +static void +iavf_add_queue_stats_chnl(u64 **data, struct iavf_ring *ring, + enum iavf_chnl_stat_type stat_type) +{ + struct iavf_stats *stats = NULL; +#ifdef HAVE_NDO_GET_STATS64 + unsigned int start; +#endif + unsigned int size; + unsigned int i; + + switch (stat_type) { + case IAVF_CHNL_STAT_POLL: + size = ARRAY_SIZE(iavf_gstrings_queue_stats_poll); + stats = iavf_gstrings_queue_stats_poll; + break; + case IAVF_CHNL_STAT_TX: + size = ARRAY_SIZE(iavf_gstrings_queue_stats_tx); + stats = iavf_gstrings_queue_stats_tx; + break; + case IAVF_CHNL_STAT_RX: + size = ARRAY_SIZE(iavf_gstrings_queue_stats_rx); + stats = iavf_gstrings_queue_stats_rx; + break; + case IAVF_CHNL_STAT_VECTOR: + size = ARRAY_SIZE(iavf_gstrings_queue_stats_vector); + stats = iavf_gstrings_queue_stats_vector; + break; + default: + break; /* unsupported stat type */ + } + + if (!stats) + return; + + /* To avoid invalid statistics values, ensure that we keep retrying + * the copy until we get a consistent value according to + * u64_stats_fetch_retry_irq. But first, make sure our ring is + * non-null before attempting to access its syncp. + */ +#ifdef HAVE_NDO_GET_STATS64 + do { + start = !ring ? 0 : u64_stats_fetch_begin_irq(&ring->syncp); +#endif + for (i = 0; i < size; i++) { + void *ptr = ring; + + if (stat_type == IAVF_CHNL_STAT_VECTOR) + ptr = ring ? ring->q_vector : NULL; + iavf_add_one_ethtool_stat(&(*data)[i], ptr, + &stats[i]); + } +#ifdef HAVE_NDO_GET_STATS64 + } while (ring && u64_stats_fetch_retry_irq(&ring->syncp, start)); +#endif + + /* Once we successfully copy the stats in, update the data pointer */ + *data += size; +} + /** * iavf_add_queue_stats - copy queue statistics into supplied buffer * @data: ethtool stats buffer diff --git a/drivers/net/ethernet/intel/iavf/iavf_helper.h b/drivers/net/ethernet/intel/iavf/iavf_helper.h index edfa21698..68206f3ff 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_helper.h +++ b/drivers/net/ethernet/intel/iavf/iavf_helper.h @@ -10,6 +10,7 @@ * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code * @hw: pointer to the HW structure * @mem: ptr to mem struct to fill out + * @mtype: unused parameter for documenting memory type of dma * @size: size of memory requested * @alignment: what to align the allocation to **/ diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 2b62d8ddd..b6f418ea6 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -4,7 +4,6 @@ #include "iavf.h" #include "iavf_helper.h" #include "iavf_prototype.h" -#include "iavf_client.h" /* All iavf tracepoints are defined by the include below, which must * be included exactly once across the whole kernel with * CREATE_TRACE_POINTS defined @@ -23,10 +22,10 @@ char iavf_driver_name[] = "iavf"; static const char iavf_driver_string[] = "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; -#define DRV_VERSION_MAJOR (3) -#define DRV_VERSION_MINOR (9) -#define DRV_VERSION_BUILD (3) -#define DRV_VERSION "3.9.3" +#define DRV_VERSION_MAJOR (4) +#define DRV_VERSION_MINOR (2) +#define DRV_VERSION_BUILD (7) +#define DRV_VERSION "4.2.7" const char iavf_driver_version[] = DRV_VERSION; static const char iavf_copyright[] = "Copyright (c) 2013, Intel Corporation."; @@ -65,6 +64,9 @@ struct workqueue_struct *iavf_wq; /** * iavf_schedule_reset - Set the flags and schedule a reset event * @adapter: board private structure + * + * Set IAVF_FLAG_RESET_NEEDED flag so iavf_watchdog_task() will change drivers + * state to __IAVF_RESETTING. **/ void iavf_schedule_reset(struct iavf_adapter *adapter) { @@ -72,11 +74,30 @@ void iavf_schedule_reset(struct iavf_adapter *adapter) mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); } +/** + * iavf_schedule_request_stats - Set the flags and schedule statistics request + * @adapter: board private structure + * + * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly + * request and refresh ethtool stats + **/ +void iavf_schedule_request_stats(struct iavf_adapter *adapter) +{ + adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS; + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); +} + /** * iavf_tx_timeout - Respond to a Tx Hang * @netdev: network interface device structure + * @txqueue: stuck queue **/ +#ifdef HAVE_TX_TIMEOUT_TXQUEUE +static void +iavf_tx_timeout(struct net_device *netdev, __always_unused unsigned int txqueue) +#else static void iavf_tx_timeout(struct net_device *netdev) +#endif { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -490,6 +511,46 @@ static void iavf_configure_tx(struct iavf_adapter *adapter) adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); } +/** + * iavf_select_rx_desc_format - Select Rx descriptor format + * @adapter: adapter private structure + * + * Select what Rx descriptor format based on availability and enabled + * features. + * + * Returns the desired RXDID to select for a given Rx queue, as defined by + * enum virtchnl_rxdid_format. + */ +static u8 iavf_select_rx_desc_format(struct iavf_adapter *adapter) +{ + u64 supported_rxdids = adapter->supported_rxdids.supported_rxdids; + + /* If we did not negotiate VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC, we must + * stick with the default value of the legacy 32 byte format. + */ + if (!RXDID_ALLOWED(adapter)) + return VIRTCHNL_RXDID_1_32B_BASE; + + /* Rx timestamping requires the use of flexible NIC descriptors */ + if (iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP)) { + if (supported_rxdids & BIT(VIRTCHNL_RXDID_2_FLEX_SQ_NIC)) + return VIRTCHNL_RXDID_2_FLEX_SQ_NIC; + + dev_dbg(&adapter->pdev->dev, "Unable to negotiate flexible descriptor format.\n"); + } + + /* Warn if the PF does not list support for the default legacy + * descriptor format. This shouldn't happen, as this is the format + * used if VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC is not supported. It is + * likely caused by a bug in the PF implementation failing to indicate + * support for the format. + */ + if (supported_rxdids & BIT(VIRTCHNL_RXDID_1_32B_BASE)) + dev_warn(&adapter->pdev->dev, "PF does not list support for default Rx descriptor format\n"); + + return VIRTCHNL_RXDID_1_32B_BASE; +} + /** * iavf_configure_rx - Configure Receive Unit after Reset * @adapter: board private structure @@ -502,6 +563,11 @@ static void iavf_configure_rx(struct iavf_adapter *adapter) struct iavf_hw *hw = &adapter->hw; int i; + adapter->rxdid = iavf_select_rx_desc_format(adapter); + + dev_dbg(&adapter->pdev->dev, "Configuring Rx using descriptor ID of %u\n", + adapter->rxdid); + /* Legacy Rx will always default to a 2048 buffer size. */ #if (PAGE_SIZE < 8192) if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) { @@ -526,6 +592,7 @@ static void iavf_configure_rx(struct iavf_adapter *adapter) for (i = 0; i < adapter->num_active_queues; i++) { adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); adapter->rx_rings[i].rx_buf_len = rx_buf_len; + adapter->rx_rings[i].rxdid = adapter->rxdid; if (adapter->flags & IAVF_FLAG_LEGACY_RX) clear_ring_build_skb_enabled(&adapter->rx_rings[i]); @@ -561,14 +628,17 @@ static void iavf_vlan_rx_register(struct net_device *netdev, * mac_vlan_list_lock. **/ static struct -iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan) +iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, + struct iavf_vlan vlan) { struct iavf_vlan_filter *f; list_for_each_entry(f, &adapter->vlan_filter_list, list) { - if (vlan == f->vlan) + if (f->vlan.vid == vlan.vid && + f->vlan.tpid == vlan.tpid) return f; } + return NULL; } @@ -580,7 +650,8 @@ iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, u16 vlan) * Returns ptr to the filter object or NULL when no memory available. **/ static struct -iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, u16 vlan) +iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, + struct iavf_vlan vlan) { struct iavf_vlan_filter *f = NULL; @@ -609,7 +680,7 @@ clearout: * @adapter: board private structure * @vlan: VLAN tag **/ -static void iavf_del_vlan(struct iavf_adapter *adapter, u16 vlan) +static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan) { struct iavf_vlan_filter *f; @@ -637,16 +708,63 @@ static void iavf_restore_filters(struct iavf_adapter *adapter) if (adapter->vsi.vlgrp) iavf_vlan_rx_register(adapter->netdev, adapter->vsi.vlgrp); #else /* HAVE_VLAN_RX_REGISTER */ - if (VLAN_ALLOWED(adapter)) { + if (VLAN_FILTERING_ALLOWED(adapter)) { u16 vid; - for_each_set_bit(vid, adapter->vsi.active_vlans, VLAN_N_VID) - iavf_add_vlan(adapter, vid); + for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID) + iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q)); + + for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID) + iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD)); } #endif /* HAVE_VLAN_RX_REGISTER */ } +/** + * iavf_get_num_vlans_added - get number of VLANs added + * @adapter: board private structure + */ +static u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) +{ + return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) + + bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID); +} + +/** + * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF + * @adapter: board private structure + * + * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN, + * do not impose a limit as that maintains current behavior and for + * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF. + **/ +static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter) +{ + /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has + * never been a limit on the VF driver side + */ + if (VLAN_ALLOWED(adapter)) + return VLAN_N_VID; + else if (VLAN_V2_ALLOWED(adapter)) + return adapter->vlan_v2_caps.filtering.max_filters; + + return 0; +} + +/** + * iavf_max_vlans_added - check if maximum VLANs allowed already exist + * @adapter: board private structure + **/ +static bool iavf_max_vlans_added(struct iavf_adapter *adapter) +{ + if (iavf_get_num_vlans_added(adapter) < + iavf_get_max_vlans_allowed(adapter)) + return false; + + return true; +} + /** * iavf_vlan_rx_add_vid - Add a VLAN filter to a device * @netdev: network device struct @@ -662,16 +780,29 @@ static int iavf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) #endif { struct iavf_adapter *adapter = netdev_priv(netdev); +#ifdef NETIF_F_HW_VLAN_CTAG_RX + u16 local_vlan_proto = be16_to_cpu(proto); +#else + u16 local_vlan_proto = ETH_P_8021Q; +#endif - if (!VLAN_ALLOWED(adapter)) + if (!VLAN_FILTERING_ALLOWED(adapter)) return -EIO; - if (iavf_add_vlan(adapter, vid) == NULL) + if (iavf_max_vlans_added(adapter)) { + netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n", + iavf_get_max_vlans_allowed(adapter)); + return -EIO; + } + + if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, local_vlan_proto))) return -ENOMEM; -#ifndef HAVE_VLAN_RX_REGISTER - set_bit(vid, adapter->vsi.active_vlans); -#endif /* HAVE_VLAN_RX_REGISTER */ + if (local_vlan_proto == ETH_P_8021Q) + set_bit(vid, adapter->vsi.active_cvlans); + else + set_bit(vid, adapter->vsi.active_svlans); + return 0; } #else @@ -679,13 +810,17 @@ static void iavf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) { struct iavf_adapter *adapter = netdev_priv(netdev); - if (!VLAN_ALLOWED(adapter)) + if (!VLAN_FILTERING_ALLOWED(adapter)) return; - iavf_add_vlan(adapter, vid); -#ifndef HAVE_VLAN_RX_REGISTER - set_bit(vid, adapter->vsi.active_vlans); -#endif /* HAVE_VLAN_RX_REGISTER */ + if (iavf_max_vlans_added(adapter)) { + netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n", + iavf_get_max_vlans_allowed(adapter)); + return; + } + + iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q)); + set_bit(vid, adapter->vsi.active_cvlans); } #endif @@ -704,14 +839,20 @@ static int iavf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) #endif { struct iavf_adapter *adapter = netdev_priv(netdev); +#ifdef NETIF_F_HW_VLAN_CTAG_RX + u16 local_vlan_proto = be16_to_cpu(proto); +#else + u16 local_vlan_proto = ETH_P_8021Q; +#endif - if (!VLAN_ALLOWED(adapter)) + if (!VLAN_FILTERING_ALLOWED(adapter)) return -EIO; - iavf_del_vlan(adapter, vid); -#ifndef HAVE_VLAN_RX_REGISTER - clear_bit(vid, adapter->vsi.active_vlans); -#endif /* HAVE_VLAN_RX_REGISTER */ + iavf_del_vlan(adapter, IAVF_VLAN(vid, local_vlan_proto)); + if (local_vlan_proto == ETH_P_8021Q) + clear_bit(vid, adapter->vsi.active_cvlans); + else + clear_bit(vid, adapter->vsi.active_svlans); return 0; } @@ -720,13 +861,11 @@ static void iavf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) { struct iavf_adapter *adapter = netdev_priv(netdev); - if (!VLAN_ALLOWED(adapter)) + if (!VLAN_FILTERING_ALLOWED(adapter)) return; - iavf_del_vlan(adapter, vid); -#ifndef HAVE_VLAN_RX_REGISTER - clear_bit(vid, adapter->vsi.active_vlans); -#endif /* HAVE_VLAN_RX_REGISTER */ + iavf_del_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q)); + clear_bit(vid, adapter->vsi.active_cvlans); } #endif @@ -781,6 +920,7 @@ iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, list_add_tail(&f->list, &adapter->mac_filter_list); f->add = true; f->is_new_mac = true; + f->is_primary = false; adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; } else { f->remove = false; @@ -789,6 +929,50 @@ iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, return f; } +/** + * iavf_replace_primary_mac - Replace current primary address + * @adapter: board private structure + * @new_mac: new mac address to be applied + * + * Replace current dev_addr and send request to PF for removal of previous + * primary mac address filter and addition of new primary mac filter. + * Return 0 for success, -ENOMEM for failure. + * + * Do not call this with mac_vlan_list_lock! + **/ +int iavf_replace_primary_mac(struct iavf_adapter *adapter, + const u8 *new_mac) +{ + struct net_device *netdev = adapter->netdev; + struct iavf_hw *hw = &adapter->hw; + struct iavf_mac_filter *f; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + + f = iavf_find_filter(adapter, hw->mac.addr); + if (f) { + f->remove = true; + f->is_primary = true; + adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; + } + + f = iavf_add_filter(adapter, new_mac); + + if (f) { + f->is_primary = true; + ether_addr_copy(netdev->dev_addr, new_mac); + } + + spin_unlock_bh(&adapter->mac_vlan_list_lock); + + /* schedule the watchdog task to immediately process the request */ + if (f) { + queue_work(iavf_wq, &adapter->watchdog_task.work); + return 0; + } + return -ENOMEM; +} + /** * iavf_set_mac - NDO callback to set port mac address * @netdev: network interface device structure @@ -799,33 +983,19 @@ iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, static int iavf_set_mac(struct net_device *netdev, void *p) { struct iavf_adapter *adapter = netdev_priv(netdev); - struct iavf_hw *hw = &adapter->hw; - struct iavf_mac_filter *f; struct sockaddr *addr = p; if (!is_valid_ether_addr(addr->sa_data)) return -EADDRNOTAVAIL; - if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) - return 0; - - spin_lock_bh(&adapter->mac_vlan_list_lock); - - f = iavf_find_filter(adapter, hw->mac.addr); - if (f) { - f->remove = true; - adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; + if (iavf_chnl_filters_exist(adapter)) { + netdev_err(netdev, + "unable to set mac address because device %s has tc-flower filters. Delete all of them and try again\n", + netdev->name); + return -EAGAIN; } - f = iavf_add_filter(adapter, addr->sa_data); - - spin_unlock_bh(&adapter->mac_vlan_list_lock); - - if (f) { - ether_addr_copy(hw->mac.addr, addr->sa_data); - } - - return (f == NULL) ? -ENOMEM : 0; + return iavf_replace_primary_mac(adapter, addr->sa_data); } /** @@ -876,6 +1046,16 @@ static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) return 0; } +/** + * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed + * @adapter: device specific adapter + */ +bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter) +{ + return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) + & (IFF_PROMISC | IFF_ALLMULTI); +} + /** * iavf_set_rx_mode - NDO callback to set the netdev filters * @netdev: network interface device structure @@ -891,19 +1071,11 @@ static void iavf_set_rx_mode(struct net_device *netdev) spin_unlock_bh(&adapter->mac_vlan_list_lock); - if (netdev->flags & IFF_PROMISC && - !(adapter->flags & IAVF_FLAG_PROMISC_ON)) - adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC; - else if (!(netdev->flags & IFF_PROMISC) && - adapter->flags & IAVF_FLAG_PROMISC_ON) - adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC; + spin_lock_bh(&adapter->current_netdev_promisc_flags_lock); - if (netdev->flags & IFF_ALLMULTI && - !(adapter->flags & IAVF_FLAG_ALLMULTI_ON)) - adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI; - else if (!(netdev->flags & IFF_ALLMULTI) && - adapter->flags & IAVF_FLAG_ALLMULTI_ON) - adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI; + if (iavf_promiscuous_mode_changed(adapter)) + adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE; + spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock); } /** @@ -982,8 +1154,6 @@ static void iavf_up_complete(struct iavf_adapter *adapter) iavf_napi_enable_all(adapter); adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES; - if (CLIENT_ENABLED(adapter)) - adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN; mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); } @@ -1091,13 +1261,7 @@ iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) return v_actual; } - if (CLIENT_ALLOWED(adapter)) { - adapter->num_iwarp_msix = (v_actual - 1) / 2; - adapter->num_msix_vectors = v_actual - adapter->num_iwarp_msix; - adapter->iwarp_base_vector = adapter->num_msix_vectors; - } else { - adapter->num_msix_vectors = v_actual; - } + adapter->num_msix_vectors = v_actual; return 0; } @@ -1119,6 +1283,86 @@ static void iavf_free_queues(struct iavf_adapter *adapter) adapter->rx_rings = NULL; } +/** + * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload + * @adapter: board private structure + * + * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or + * stripped in certain descriptor fields. Instead of checking the offload + * capability bits in the hot path, cache the location the ring specific + * flags. + */ +void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_active_queues; i++) { + struct iavf_ring *tx_ring = &adapter->tx_rings[i]; + struct iavf_ring *rx_ring = &adapter->rx_rings[i]; + + /* prevent multiple L2TAG bits being set after VFR */ + tx_ring->flags &= + ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | + IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2); + rx_ring->flags &= + ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | + IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2); + + if (VLAN_ALLOWED(adapter)) { + tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + } else if (VLAN_V2_ALLOWED(adapter)) { + struct virtchnl_vlan_supported_caps *stripping_support; + struct virtchnl_vlan_supported_caps *insertion_support; + + stripping_support = + &adapter->vlan_v2_caps.offloads.stripping_support; + insertion_support = + &adapter->vlan_v2_caps.offloads.insertion_support; + + if (stripping_support->outer) { + if (stripping_support->outer & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) + rx_ring->flags |= + IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + else if (stripping_support->outer & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) + rx_ring->flags |= + IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; + } else if (stripping_support->inner) { + if (stripping_support->inner & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) + rx_ring->flags |= + IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + else if (stripping_support->inner & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) + rx_ring->flags |= + IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; + } + + if (insertion_support->outer) { + if (insertion_support->outer & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) + tx_ring->flags |= + IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + else if (insertion_support->outer & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) + tx_ring->flags |= + IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; + } else if (insertion_support->inner) { + if (insertion_support->inner & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) + tx_ring->flags |= + IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; + else if (insertion_support->inner & + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) + tx_ring->flags |= + IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; + } + } + } +} + /** * iavf_alloc_queues - Allocate memory for all rings * @adapter: board private structure to initialize @@ -1139,10 +1383,11 @@ static int iavf_alloc_queues(struct iavf_adapter *adapter) if (adapter->num_req_queues) num_active_queues = adapter->num_req_queues; #ifdef __TC_MQPRIO_MODE_MAX - else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && - adapter->num_tc) + else if (iavf_is_adq_enabled(adapter)) num_active_queues = adapter->ch_config.total_qps; #endif /* __TC_MQPRIO_MODE_MAX */ + else if (adapter->orig_num_active_queues) + num_active_queues = adapter->orig_num_active_queues; else num_active_queues = min_t(int, adapter->vsi_res->num_queue_pairs, @@ -1183,6 +1428,8 @@ static int iavf_alloc_queues(struct iavf_adapter *adapter) adapter->num_active_queues = num_active_queues; + iavf_set_queue_vlan_tag_loc(adapter); + return 0; err_out: @@ -1353,8 +1600,7 @@ int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) * resources have been allocated in the reset path. * Now we can truly claim that ADQ is enabled. */ - if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && - adapter->num_tc) + if (iavf_is_adq_enabled(adapter)) dev_info(&adapter->pdev->dev, "ADQ Enabled, %u TCs created", adapter->num_tc); #endif /* __TC_MQPRIO_MODE_MAX */ @@ -1548,6 +1794,120 @@ err: return err; } +/** + * iavf_set_vlan_offload_features - set VLAN offload configuration + * @adapter: board private structure + * @prev_features: previous features used for comparison + * @features: updated features used for configuration + * + * Set the aq_required bit(s) based on the requested features passed in to + * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule + * the watchdog if any changes are requested to expedite the request via + * virtchnl. + */ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT +static void +iavf_set_vlan_offload_features(struct iavf_adapter *adapter, + u32 prev_features, + u32 features) +#else +static void +iavf_set_vlan_offload_features(struct iavf_adapter *adapter, + netdev_features_t prev_features, + netdev_features_t features) +#endif +{ + bool enable_stripping = true, enable_insertion = true; + u16 vlan_ethertype = 0; + u64 aq_required = 0; + +#ifdef NETIF_F_HW_VLAN_CTAG_RX + /* keep cases separate because one ethertype for offloads can be + * disabled at the same time as another is disabled, so check for an + * enabled ethertype first, then check for disabled. Default to + * ETH_P_8021Q so an ethertype is specified if disabling insertion and + * stripping. + */ + if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) + vlan_ethertype = ETH_P_8021AD; + else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) + vlan_ethertype = ETH_P_8021Q; + else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) + vlan_ethertype = ETH_P_8021AD; + else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) + vlan_ethertype = ETH_P_8021Q; + else + vlan_ethertype = ETH_P_8021Q; + + if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) + enable_stripping = false; + if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) + enable_insertion = false; +#else + if (features & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX)) + vlan_ethertype = ETH_P_8021Q; + if (prev_features & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX)) + vlan_ethertype = ETH_P_8021Q; + else + vlan_ethertype = ETH_P_8021Q; + + if (!(features & NETIF_F_HW_VLAN_RX)) + enable_stripping = false; + if (!(features & NETIF_F_HW_VLAN_TX)) + enable_insertion = false; +#endif + + if (VLAN_ALLOWED(adapter)) { + /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN + * stripping via virtchnl. VLAN insertion can be toggled on the + * netdev, but it doesn't require a virtchnl message + */ + if (enable_stripping) + aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; + else + aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; + + } else if (VLAN_V2_ALLOWED(adapter)) { + switch (vlan_ethertype) { + case ETH_P_8021Q: + if (enable_stripping) + aq_required |= + IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING; + else + aq_required |= + IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING; + + if (enable_insertion) + aq_required |= + IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION; + else + aq_required |= + IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION; + break; + case ETH_P_8021AD: + if (enable_stripping) + aq_required |= + IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING; + else + aq_required |= + IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING; + + if (enable_insertion) + aq_required |= + IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION; + else + aq_required |= + IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION; + break; + } + } + + if (aq_required) { + adapter->aq_required |= aq_required; + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); + } +} + /** * iavf_startup - first step of driver startup * @adapter: board private structure @@ -1650,6 +2010,59 @@ err: iavf_change_state(adapter, __IAVF_INIT_FAILED); } +/** + * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES + * @adapter: board private structure + */ +int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter) +{ + int i, num_req_queues = adapter->num_req_queues; + struct iavf_vsi *vsi = &adapter->vsi; + + for (i = 0; i < adapter->vf_res->num_vsis; i++) { + if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) + adapter->vsi_res = &adapter->vf_res->vsi_res[i]; + } + if (!adapter->vsi_res) { + dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); + return -ENODEV; + } + + if (num_req_queues && + num_req_queues > adapter->vsi_res->num_queue_pairs) { + /* Problem. The PF gave us fewer queues than what we had + * negotiated in our request. Need a reset to see if we can't + * get back to a working state. + */ + dev_err(&adapter->pdev->dev, + "Requested %d queues, but PF only gave us %d.\n", + num_req_queues, + adapter->vsi_res->num_queue_pairs); + adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED; + adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; + iavf_schedule_reset(adapter); + + return -EAGAIN; + } + adapter->num_req_queues = 0; + adapter->vsi.id = adapter->vsi_res->vsi_id; + + adapter->vsi.back = adapter; + adapter->vsi.base_vector = 1; + adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; + vsi->netdev = adapter->netdev; + vsi->qs_handle = adapter->vsi_res->qset_handle; + if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { + adapter->rss_key_size = adapter->vf_res->rss_key_size; + adapter->rss_lut_size = adapter->vf_res->rss_lut_size; + } else { + adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; + adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; + } + + return 0; +} + /** * iavf_init_get_resources - third step of driver startup * @adapter: board private structure @@ -1661,25 +2074,22 @@ err: **/ static void iavf_init_get_resources(struct iavf_adapter *adapter) { - struct net_device *netdev = adapter->netdev; struct iavf_hw *hw = &adapter->hw; struct pci_dev *pdev = adapter->pdev; - int ret, bufsz; + int ret; WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES); /* aq msg sent, awaiting reply */ if (!adapter->vf_res) { - bufsz = sizeof(struct virtchnl_vf_resource) + - (IAVF_MAX_VF_VSI * - sizeof(struct virtchnl_vsi_resource)); - adapter->vf_res = kzalloc(bufsz, GFP_KERNEL); + adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE, + GFP_KERNEL); if (!adapter->vf_res) goto err; } ret = iavf_get_vf_config(adapter); if (ret == IAVF_ERR_ADMIN_QUEUE_NO_WORK) { ret = iavf_send_vf_config_msg(adapter); - goto err; + goto err_alloc; } else if (ret == IAVF_ERR_PARAM) { /* We only get ERR_PARAM if the device is in a very bad * state or if we've been disabled for previous bad @@ -1694,8 +2104,267 @@ static void iavf_init_get_resources(struct iavf_adapter *adapter) goto err_alloc; } - if (iavf_process_config(adapter)) + ret = iavf_parse_vf_resource_msg(adapter); + if (ret) { + dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n", + ret); goto err_alloc; + } + + /* Some features require additional messages to negotiate extended + * capabilities. These are processed in sequence by the + * __IAVF_INIT_EXTENDED_CAPS driver state. + */ + adapter->extended_caps = IAVF_EXTENDED_CAPS; + + iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS); + return; + +err_alloc: + kfree(adapter->vf_res); + adapter->vf_res = NULL; +err: + iavf_change_state(adapter, __IAVF_INIT_FAILED); +} + +/** + * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps + * @adapter: board private structure + * + * Function processes send of the extended VLAN V2 capability message to the + * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent, + * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2. + */ +static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2)); + + ret = iavf_send_vf_offload_vlan_v2_msg(adapter); + if (ret && ret == -EOPNOTSUPP) { + /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case, + * we did not send the capability exchange message and do not + * expect a response. + */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; + } + + /* We sent the message, so move on to the next step */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2; +} + +/** + * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps + * @adapter: board private structure + * + * Function processes receipt of the extended VLAN V2 capability message from + * the PF. + **/ +static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2)); + + memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps)); + + ret = iavf_get_vf_vlan_v2_caps(adapter); + if (ret) + goto err; + + /* We've processed receipt of the VLAN V2 caps message */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; + return; +err: + /* We didn't receive a reply. Make sure we try sending again when + * __IAVF_INIT_FAILED attempts to recover. + */ + adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2; + iavf_change_state(adapter, __IAVF_INIT_FAILED); +} + +/** + * iavf_init_send_supported_rxdids - part of querying for supported RXDID formats + * @adapter: board private structure + * + * Function processes send of the request for supported RXDIDs to the PF. + * Must clear IAVF_EXTENDED_CAP_RECV_RXDID if the message is not sent, e.g. + * due to the PF not negotiating VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC. + */ +static void iavf_init_send_supported_rxdids(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_RXDID)); + + ret = iavf_send_vf_supported_rxdids_msg(adapter); + if (ret && ret == -EOPNOTSUPP) { + /* PF does not support VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC. In this + * case, we did not send the capability exchange message and + * do not expect a response. + */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_RXDID; + } + + /* We sent the message, so move on to the next step */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_RXDID; +} + +/** + * iavf_init_recv_supported_rxdids - part of querying for supported RXDID formats + * @adapter: board private structure + * + * Function processes receipt of the supported RXDIDs message from the PF. + **/ +static void iavf_init_recv_supported_rxdids(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_RXDID)); + + memset(&adapter->supported_rxdids, 0, sizeof(adapter->supported_rxdids)); + + ret = iavf_get_vf_supported_rxdids(adapter); + if (ret) + goto err; + + /* We've processed the PF response to the VIRTCHNL_OP_GET_SUPPORTED_RXDIDS + * message we sent previously. + */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_RXDID; + return; +err: + /* We didn't receive a reply. Make sure we try sending again when + * __IAVF_INIT_FAILED attempts to recover. + */ + adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_RXDID; + iavf_change_state(adapter, __IAVF_INIT_FAILED); +} + +/** + * iavf_init_send_ptp_caps - part of querying for extended PTP capabilities + * @adapter: board private structure + * + * Function processes send of the request for 1588 PTP capabilities to the PF. + * Must clear IAVF_EXTENDED_CAP_SEND_PTP if the message is not sent, e.g. + * due to the PF not negotiating VIRTCHNL_VF_PTP_CAP + */ +static void iavf_init_send_ptp_caps(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_PTP)); + + ret = iavf_send_vf_ptp_caps_msg(adapter); + if (ret && ret == -EOPNOTSUPP) { + /* PF does not support VIRTCHNL_VF_PTP_CAP. In this case, we + * did not send the capability exchange message and do not + * expect a response. + */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_PTP; + } + + /* We sent the message, so move on to the next step */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_PTP; +} + +/** + * iavf_init_recv_ptp_caps - part of querying for supported PTP capabilities + * @adapter: board private structure + * + * Function processes receipt of the PTP capabilities supported on this VF. + **/ +static void iavf_init_recv_ptp_caps(struct iavf_adapter *adapter) +{ + int ret; + + WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_PTP)); + + memset(&adapter->ptp.hw_caps, 0, sizeof(adapter->ptp.hw_caps)); + + ret = iavf_get_vf_ptp_caps(adapter); + if (ret) + goto err; + + /* We've processed the PF response to the VIRTCHNL_OP_1588_PTP_GET_CAPS + * message we sent previously. + */ + adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_PTP; + return; +err: + /* We didn't receive a reply. Make sure we try sending again when + * __IAVF_INIT_FAILED attempts to recover. + */ + adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_PTP; + iavf_change_state(adapter, __IAVF_INIT_FAILED); +} + +/** + * iavf_init_process_extended_caps - Part of driver startup + * @adapter: board private structure + * + * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state + * handles negotiating capabilities for features which require an additional + * message. + * + * Once all extended capabilities exchanges are finished, the driver will + * transition into __IAVF_INIT_CONFIG_ADAPTER. + */ +static void iavf_init_process_extended_caps(struct iavf_adapter *adapter) +{ + WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS); + + /* Process capability exchange for VLAN V2 */ + if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) { + iavf_init_send_offload_vlan_v2_caps(adapter); + return; + } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) { + iavf_init_recv_offload_vlan_v2_caps(adapter); + return; + } + + /* Process capability exchange for RXDID formats */ + if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_RXDID) { + iavf_init_send_supported_rxdids(adapter); + return; + } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_RXDID) { + iavf_init_recv_supported_rxdids(adapter); + return; + } + + /* Process capability exchange for PTP features */ + if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_PTP) { + iavf_init_send_ptp_caps(adapter); + return; + } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_PTP) { + iavf_init_recv_ptp_caps(adapter); + return; + } + + /* When we reach here, no further extended capabilities exchanges are + * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER + */ + iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER); +} + +/** + * iavf_init_config_adapter - last part of driver startup + * @adapter: board private structure + * + * After all the supported capabilities are negotiated, then the + * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization. + */ +static void iavf_init_config_adapter(struct iavf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + struct pci_dev *pdev = adapter->pdev; + int ret; + + WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER); + + if (iavf_process_config(adapter)) + goto err; adapter->current_op = VIRTCHNL_OP_UNKNOWN; adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; @@ -1758,12 +2427,6 @@ static void iavf_init_get_resources(struct iavf_adapter *adapter) adapter->netdev_registered = true; netif_tx_stop_all_queues(netdev); - if (CLIENT_ALLOWED(adapter)) { - ret = iavf_lan_add_device(adapter); - if (ret) - dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n", - ret); - } dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); if (netdev->features & NETIF_F_GRO) dev_info(&pdev->dev, "GRO is enabled\n"); @@ -1783,6 +2446,12 @@ static void iavf_init_get_resources(struct iavf_adapter *adapter) else iavf_init_rss(adapter); + /* request initial VLAN offload settings */ + iavf_set_vlan_offload_features(adapter, 0, netdev->features); + + /* Setup initial PTP configuration */ + iavf_ptp_init(adapter); + return; err_mem: iavf_free_rss(adapter); @@ -1790,11 +2459,9 @@ err_register: iavf_free_misc_irq(adapter); err_sw_init: iavf_reset_interrupt_capability(adapter); -err_alloc: - kfree(adapter->vf_res); - adapter->vf_res = NULL; err: iavf_change_state(adapter, __IAVF_INIT_FAILED); + } /** @@ -1810,6 +2477,12 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter) { if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) return iavf_send_vf_config_msg(adapter); + if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS) + return iavf_send_vf_offload_vlan_v2_msg(adapter); + if (adapter->aq_required & IAVF_FLAG_AQ_GET_SUPPORTED_RXDIDS) + return iavf_send_vf_supported_rxdids_msg(adapter); + if (adapter->aq_required & IAVF_FLAG_AQ_GET_PTP_CAPS) + return iavf_send_vf_ptp_caps_msg(adapter); if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { iavf_disable_queues(adapter); return 0; @@ -1874,18 +2547,8 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter) iavf_set_rss_lut(adapter); return 0; } - if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) { - iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC | - FLAG_VF_MULTICAST_PROMISC); - return 0; - } - if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) { - iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC); - return 0; - } - if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) || - (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) { - iavf_set_promiscuous(adapter, 0); + if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) { + iavf_set_promiscuous(adapter); return 0; } #ifdef __TC_MQPRIO_MODE_MAX @@ -1907,6 +2570,52 @@ static int iavf_process_aq_command(struct iavf_adapter *adapter) iavf_add_cloud_filter(adapter); return 0; } + if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) { + iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) { + iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) { + iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) { + iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) { + iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) { + iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) { + iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) { + iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD); + return 0; + } + if (adapter->aq_required & IAVF_FLAG_AQ_SEND_PTP_CMD) { + iavf_virtchnl_send_ptp_cmd(adapter); + return 0; + } + + /* since only one operation is processed at a time, always keep stats + * requests at the lowest priority so all other operations get processed + * first + */ + if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) { + iavf_request_stats(adapter); + return 0; + } + return -EAGAIN; } @@ -1937,6 +2646,21 @@ static void iavf_send_reset_request(struct iavf_adapter *adapter) adapter->flags |= IAVF_FLAG_RESET_PENDING; } +/** + * iavf_set_flags_reset_detected - set flags for handling reset + * @adapter: pointer to iavf_adapter + * + * Set IAVF_FLAG_RESET_DETECTED flag and IAVF_FLAG_RESET_PENDING flags to handle + * reset without sending reset request to PF in iavf_watchdog_task() via + * iavf_send_reset_request(). + **/ +static void iavf_set_flags_reset_detected(struct iavf_adapter *adapter) +{ + adapter->flags &= ~IAVF_FLAG_QUEUES_ENABLED; + adapter->flags |= IAVF_FLAG_RESET_DETECTED; + adapter->flags |= IAVF_FLAG_RESET_PENDING; +} + /** * iavf_watchdog_task - Periodic call-back task * @work: pointer to work_struct @@ -1949,29 +2673,25 @@ static void iavf_watchdog_task(struct work_struct *work) struct iavf_hw *hw = &adapter->hw; u32 reg_val; + /* If the driver is in the process of being removed then don't run or + * reschedule the watchdog task. + */ + if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) + return; + if (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section)) goto restart_watchdog; if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) iavf_change_state(adapter, __IAVF_COMM_FAILED); + /* IAVF_FLAG_RESET_NEEDED is set in iavf_schedule_reset() */ if (adapter->flags & IAVF_FLAG_RESET_NEEDED && adapter->state != __IAVF_RESETTING) { adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; iavf_change_state(adapter, __IAVF_RESETTING); adapter->aq_required = 0; adapter->current_op = VIRTCHNL_OP_UNKNOWN; - while (test_and_set_bit(__IAVF_IN_CLIENT_TASK, - &adapter->crit_section)) - usleep_range(500, 1000); - if (CLIENT_ENABLED(adapter)) { - adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | - IAVF_FLAG_CLIENT_NEEDS_CLOSE | - IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | - IAVF_FLAG_SERVICE_CLIENT_REQUESTED); - cancel_delayed_work_sync(&adapter->client_task); - iavf_notify_client_close(&adapter->vsi, true); - } } switch (adapter->state) { @@ -1993,6 +2713,18 @@ static void iavf_watchdog_task(struct work_struct *work) queue_delayed_work(iavf_wq, &adapter->watchdog_task, msecs_to_jiffies(1)); return; + case __IAVF_INIT_EXTENDED_CAPS: + iavf_init_process_extended_caps(adapter); + clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); + queue_delayed_work(iavf_wq, &adapter->watchdog_task, + msecs_to_jiffies(1)); + return; + case __IAVF_INIT_CONFIG_ADAPTER: + iavf_init_config_adapter(adapter); + clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); + queue_delayed_work(iavf_wq, &adapter->watchdog_task, + msecs_to_jiffies(1)); + return; case __IAVF_INIT_FAILED: if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { dev_err(&adapter->pdev->dev, @@ -2005,7 +2737,7 @@ static void iavf_watchdog_task(struct work_struct *work) &adapter->watchdog_task, (5 * HZ)); return; } - /* Try again from failed step*/ + /* Try again from failed step */ iavf_change_state(adapter, adapter->last_state); clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ); @@ -2034,6 +2766,10 @@ static void iavf_watchdog_task(struct work_struct *work) msecs_to_jiffies(10)); return; case __IAVF_RESETTING: + /* Proceed with handling reset if IAVF_FLAG_RESET_PENDING has + * been set in either iavf_send_reset_request() or + * iavf_set_flags_reset_detected(). + */ if (!(adapter->flags & IAVF_FLAG_RESET_PENDING)) iavf_send_reset_request(adapter); else @@ -2054,12 +2790,26 @@ static void iavf_watchdog_task(struct work_struct *work) iavf_send_api_ver(adapter); } } else { - if (iavf_process_aq_command(adapter) && + int ret = iavf_process_aq_command(adapter); + + /* An error will be returned if no commands were + * processed; use this opportunity to update stats + * if the error isn't -ENOTSUPP + */ + if (ret && ret != -EOPNOTSUPP && adapter->state == __IAVF_RUNNING) iavf_request_stats(adapter); } - if (adapter->state == __IAVF_RUNNING) + if (adapter->state == __IAVF_RUNNING) { iavf_detect_recover_hung(&adapter->vsi); + iavf_chnl_detect_recover(&adapter->vsi); + } +#ifndef HAVE_PTP_CLOCK_DO_AUX_WORK +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) + if (adapter->ptp.initialized) + iavf_ptp_do_aux_work(&adapter->ptp.info); +#endif +#endif break; case __IAVF_REMOVE: clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); @@ -2070,6 +2820,7 @@ static void iavf_watchdog_task(struct work_struct *work) /* check for hw reset */ if (iavf_is_reset(hw)) { + iavf_set_flags_reset_detected(adapter); iavf_schedule_reset(adapter); adapter->aq_required = 0; adapter->current_op = VIRTCHNL_OP_UNKNOWN; @@ -2079,8 +2830,6 @@ static void iavf_watchdog_task(struct work_struct *work) return; } - schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5)); - clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); restart_watchdog: queue_work(iavf_wq, &adapter->adminq_task); @@ -2152,22 +2901,45 @@ static void iavf_disable_vf(struct iavf_adapter *adapter) iavf_reset_interrupt_capability(adapter); iavf_free_q_vectors(adapter); iavf_free_queues(adapter); - kfree(adapter->vf_res); - adapter->vf_res = NULL; + memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE); iavf_shutdown_adminq(&adapter->hw); adapter->netdev->flags &= ~IFF_UP; adapter->flags &= ~IAVF_FLAG_RESET_PENDING; adapter->state = __IAVF_DOWN; clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); - clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); adapter->flags &= ~IAVF_FLAG_RESET_PENDING; iavf_change_state(adapter, __IAVF_DOWN); wake_up(&adapter->down_waitqueue); dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); } -#define IAVF_RESET_WAIT_MS 10 -#define IAVF_RESET_WAIT_COUNT 500 +/** + * iavf_is_reset_detected - check if reset has been detected + * @adapter: pointer to iavf_adapter + * + * IAVF_FLAG_RESET_DETECTED is set if a HW reset is detected in + * iavf_watchdog_task() and cleared here, else poll for reset. + */ +static bool iavf_is_reset_detected(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = &adapter->hw; + int i; + + if (adapter->flags & IAVF_FLAG_RESET_DETECTED) { + adapter->flags &= ~IAVF_FLAG_RESET_DETECTED; + return true; + } + + /* poll until we see the reset actually happen */ + for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) { + if (iavf_is_reset(hw)) + return true; + usleep_range(5000, 10000); + } + + return false; +} + /** * iavf_handle_reset - Handle hardware reset * @adapter: pointer to iavf_adapter @@ -2175,6 +2947,13 @@ static void iavf_disable_vf(struct iavf_adapter *adapter) * During reset we need to shut down and reinitialize the admin queue * before we can use it to communicate with the PF again. We also clear * and reinit the rings because that context is lost as well. + * + * This function is called in the __IAVF_RESETTING driver state. If a reset + * is detected and completes, the driver state changed to __IAVF_RUNNING or + * __IAVF_DOWN, else driver state will remain in __IAVF_RESETTING. + * + * The function is called with the IAVF_FLAG_RESET_PENDING flag set and it is + * cleared when a reset is detected and completes. **/ static void iavf_handle_reset(struct iavf_adapter *adapter) { @@ -2184,19 +2963,16 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) bool running; u32 reg_val; - /* poll until we see the reset actually happen */ - for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) { - if (iavf_is_reset(hw)) - break; - usleep_range(5000, 10000); - } - if (i == IAVF_RESET_WAIT_COUNT) { + if (!iavf_is_reset_detected(adapter)) { + /* Driver state remains __IAVF_RESETTING and flags are not + * cleared, so iavf_watchdog_task() will call again. + */ dev_info(&adapter->pdev->dev, "Never saw reset\n"); - return; /* bail out, I'll be back */ + return; } /* wait until the reset is complete and the PF is responding to us */ - for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) { + for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { /* sleep first to make sure a minimum wait time is met */ msleep(IAVF_RESET_WAIT_MS); @@ -2209,7 +2985,7 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) pci_set_master(adapter->pdev); pci_restore_msi_state(adapter->pdev); - if (i == IAVF_RESET_WAIT_COUNT) { + if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", reg_val); iavf_disable_vf(adapter); @@ -2261,6 +3037,7 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) adapter->aq_required = 0; if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || + (adapter->flags & IAVF_FLAG_REINIT_CHNL_NEEDED) || (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { err = iavf_reinit_interrupt_scheme(adapter); if (err) @@ -2277,6 +3054,16 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; + /* Certain capabilities require an extended negotiation process using + * extra messages that must be processed after getting the VF + * configuration. The related checks such as VLAN_V2_ALLOWED() are not + * reliable here, since the configuration has not yet been negotiated. + * + * Always set these flags, since them related VIRTCHNL messages won't + * be sent until after VIRTCHNL_OP_GET_VF_RESOURCES. + */ + adapter->aq_required |= IAVF_FLAG_AQ_EXTENDED_CAPS; + iavf_misc_irq_enable(adapter); /* We were running when the reset started, so we need to restore some @@ -2294,6 +3081,7 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) goto reset_err; if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || + (adapter->flags & IAVF_FLAG_REINIT_CHNL_NEEDED) || (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { err = iavf_request_traffic_irqs(adapter, netdev->name); if (err) @@ -2315,15 +3103,14 @@ static void iavf_handle_reset(struct iavf_adapter *adapter) } adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; + adapter->flags &= ~IAVF_FLAG_REINIT_CHNL_NEEDED; - clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); return; reset_err: if (running) { iavf_change_state(adapter, __IAVF_RUNNING); netdev->flags |= IFF_UP; } - clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); iavf_close(netdev); } @@ -2343,6 +3130,12 @@ static void iavf_adminq_task(struct work_struct *work) u32 val, oldval; u16 pending; + /* If the driver is in the process of being removed then return + * immediately and don't re-enable the Admin Queue interrupt. + */ + if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) + return; + if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) goto out; @@ -2418,48 +3211,6 @@ out: iavf_misc_irq_enable(adapter); } -/** - * iavf_client_task - worker thread to perform client work - * @work: pointer to work_struct containing our data - * - * This task handles client interactions. Because client calls can be - * reentrant, we can't handle them in the watchdog. - **/ -static void iavf_client_task(struct work_struct *work) -{ - struct iavf_adapter *adapter = - container_of(work, struct iavf_adapter, client_task.work); - - /* If we can't get the client bit, just give up. We'll be rescheduled - * later. - */ - - if (test_and_set_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section)) - return; - - if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) { - iavf_client_subtask(adapter); - adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED; - goto out; - } - if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) { - iavf_notify_client_l2_params(&adapter->vsi); - adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; - goto out; - } - if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) { - iavf_notify_client_close(&adapter->vsi, false); - adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE; - goto out; - } - if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) { - iavf_notify_client_open(&adapter->vsi); - adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN; - } -out: - clear_bit(__IAVF_IN_CLIENT_TASK, &adapter->crit_section); -} - /** * iavf_free_all_tx_resources - Free Tx Resources for All Queues * @adapter: board private structure @@ -2564,7 +3315,7 @@ static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, #ifdef VIRTCHNL_VF_CAP_ADV_LINK_SPEED if (ADV_LINK_SUPPORT(adapter)) { - if (SUPPORTED_SPEED(adapter->link_speed_mbps)) { + if (adapter->link_speed_mbps < U32_MAX) { speed = adapter->link_speed_mbps; goto validate_bw; } else { @@ -2576,22 +3327,28 @@ static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, #endif /* VIRTCHNL_VF_CAP_ADV_LINK_SPEED */ switch (adapter->link_speed) { case VIRTCHNL_LINK_SPEED_40GB: - speed = 40000; + speed = SPEED_40000; break; case VIRTCHNL_LINK_SPEED_25GB: - speed = 25000; + speed = SPEED_25000; break; case VIRTCHNL_LINK_SPEED_20GB: - speed = 20000; + speed = SPEED_20000; break; case VIRTCHNL_LINK_SPEED_10GB: - speed = 10000; + speed = SPEED_10000; + break; + case VIRTCHNL_LINK_SPEED_5GB: + speed = SPEED_5000; + break; + case VIRTCHNL_LINK_SPEED_2_5GB: + speed = SPEED_2500; break; case VIRTCHNL_LINK_SPEED_1GB: - speed = 1000; + speed = SPEED_1000; break; case VIRTCHNL_LINK_SPEED_100MB: - speed = 100; + speed = SPEED_100; break; default: break; @@ -2609,7 +3366,7 @@ validate_bw: } /** - * iavf_validate_channel_config - validate queue mapping info + * iavf_validate_ch_config - validate queue mapping info * @adapter: board private structure * @mqprio_qopt: queue parameters * @max_tc_allowed: MAX TC allowed, it could be 4 or 16 depends. @@ -2753,7 +3510,6 @@ static int __iavf_setup_tc(struct net_device *netdev, void *type_data) { struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; struct iavf_adapter *adapter = netdev_priv(netdev); - struct virtchnl_vf_resource *vfres = adapter->vf_res; u8 num_tc = 0, total_qps = 0; int ret = 0, netdev_tc = 0; u8 max_tc_allowed; @@ -2782,7 +3538,7 @@ static int __iavf_setup_tc(struct net_device *netdev, void *type_data) /* add queue channel */ if (mode == TC_MQPRIO_MODE_CHANNEL) { - if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { + if (!ADQ_ALLOWED(adapter)) { dev_err(&adapter->pdev->dev, "ADQ not supported\n"); return -EOPNOTSUPP; } @@ -2823,12 +3579,29 @@ static int __iavf_setup_tc(struct net_device *netdev, void *type_data) IAVF_MBPS_DIVISOR); adapter->ch_config.ch_info[i].max_tx_rate = max_tx_rate; + adapter->ch_config.ch_ex_info[i].num_rxq = + mqprio_qopt->qopt.count[i]; + adapter->ch_config.ch_ex_info[i].base_q = + mqprio_qopt->qopt.offset[i]; } else { adapter->ch_config.ch_info[i].count = 1; adapter->ch_config.ch_info[i].offset = 0; } } + + /* Take snapshot of original config such as "num_active_queues" + * It is used later when delete ADQ flow is exercised, so that + * once delete ADQ flow completes, VF shall go back to its + * original queue configuration + */ + + adapter->orig_num_active_queues = adapter->num_active_queues; + /* Store queue infor based on TC so that, VF gets configured + * with correct number of queues when VF completes ADQ config + * flow + */ adapter->ch_config.total_qps = total_qps; + netif_tx_stop_all_queues(netdev); netif_tx_disable(netdev); adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; @@ -2848,6 +3621,48 @@ exit: return ret; } +/** + * iavf_is_vlan_tc_filter_allowed - allowed to add tc-filter using VLAN + * @adapter: board private structure + * @vlan: VLAN to verify + * + * Using specified "vlan" ID, there must be active VLAN filter in VF's + * MAC-VLAN filter list. + */ +static bool +iavf_is_vlan_tc_filter_allowed(struct iavf_adapter *adapter, u16 vlan) +{ + struct iavf_vlan_filter *f; + bool allowed; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + f = iavf_find_vlan(adapter, IAVF_VLAN(vlan, ETH_P_8021Q)); + allowed = (f && !f->add && !f->remove); + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return allowed; +} + +/** + * iavf_is_mac_tc_filter_allowed - allowed to add tc-filter using MAC addr + * @adapter: board private structure + * @macaddr: MAC address + * + * Using specified MAC address, there must be active MAC filter in VF's + * MAC-VLAN filter list. + */ +static bool +iavf_is_mac_tc_filter_allowed(struct iavf_adapter *adapter, const u8 *macaddr) +{ + struct iavf_mac_filter *f; + bool allowed; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + f = iavf_find_filter(adapter, macaddr); + allowed = (f && !f->add && !f->is_new_mac && !f->remove); + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return allowed; +} + /** * iavf_parse_cls_flower - Parse tc flower filters provided by kernel * @adapter: board private structure @@ -2860,6 +3675,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, { struct flow_rule *rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = rule->match.dissector; + struct virtchnl_l4_spec *d_spec, *m_spec; struct virtchnl_filter *cf = &filter->f; enum virtchnl_flow_type flow_type; u16 n_proto_mask = 0; @@ -2897,6 +3713,16 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } #endif /* HAVE_TC_FLOWER_ENC */ + /* even though following code refers as "tcp_sec", it is not + * just for TCP but a generic struct representing + * L2, L3 + L4 fields if specified + */ + m_spec = &cf->mask.tcp_spec; + d_spec = &cf->data.tcp_spec; + + /* determine flow type, TCP/UDP_V4[6]_FLOW based on + * L2 proto (aka ETH proto) and L3 proto (aka IP_PROTO) + */ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { struct flow_match_basic match; @@ -2913,7 +3739,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) return -EINVAL; - if (ADQ_V2_ALLOWED(adapter)) { + if (iavf_is_adq_v2_enabled(adapter)) { if (match.key->ip_proto != IPPROTO_TCP && match.key->ip_proto != IPPROTO_UDP) { dev_err(&adapter->pdev->dev, @@ -2940,6 +3766,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, filter->f.flow_type = flow_type; } + /* process Ethernet header fields */ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { struct flow_match_eth_addrs match; @@ -2947,7 +3774,8 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, /* use is_broadcast and is_zero to check for all 0xf or 0 */ if (!is_zero_ether_addr(match.mask->dst)) { - if (is_broadcast_ether_addr(match.mask->dst)) { + if (ADQ_V2_ALLOWED(adapter) || + is_broadcast_ether_addr(match.mask->dst)) { field_flags |= IAVF_CLOUD_FIELD_OMAC; } else { dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", @@ -2957,7 +3785,8 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } if (!is_zero_ether_addr(match.mask->src)) { - if (is_broadcast_ether_addr(match.mask->src)) { + if (ADQ_V2_ALLOWED(adapter) || + is_broadcast_ether_addr(match.mask->src)) { field_flags |= IAVF_CLOUD_FIELD_IMAC; } else { dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", @@ -2966,43 +3795,66 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } } - if (!is_zero_ether_addr(match.key->dst)) + if (!is_zero_ether_addr(match.key->dst)) { + if (!iavf_is_mac_tc_filter_allowed(adapter, + match.key->dst)) { + dev_err(&adapter->pdev->dev, + "Dest MAC %pM doesn't belong to this VF\n", + match.key->dst); + return -EINVAL; + } + if (is_valid_ether_addr(match.key->dst) || is_multicast_ether_addr(match.key->dst)) { /* set the mask if a valid dst_mac address */ - for (i = 0; i < ETH_ALEN; i++) - cf->mask.tcp_spec.dst_mac[i] |= 0xff; - ether_addr_copy(cf->data.tcp_spec.dst_mac, + if (ADQ_V2_ALLOWED(adapter)) + ether_addr_copy(m_spec->dst_mac, + match.mask->dst); + else + eth_broadcast_addr(m_spec->dst_mac); + ether_addr_copy(d_spec->dst_mac, match.key->dst); } + } if (!is_zero_ether_addr(match.key->src)) if (is_valid_ether_addr(match.key->src) || is_multicast_ether_addr(match.key->src)) { /* set the mask if a valid src_mac address */ - for (i = 0; i < ETH_ALEN; i++) - cf->mask.tcp_spec.src_mac[i] |= 0xff; - ether_addr_copy(cf->data.tcp_spec.src_mac, + if (ADQ_V2_ALLOWED(adapter)) + ether_addr_copy(m_spec->src_mac, + match.mask->src); + else + eth_broadcast_addr(m_spec->src_mac); + ether_addr_copy(d_spec->src_mac, match.key->src); } } + /* process VLAN header for single VLAN (type could be S/C-tag) */ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { struct flow_match_vlan match; flow_rule_match_vlan(rule, &match); if (match.mask->vlan_id) { - if (match.mask->vlan_id == VLAN_VID_MASK) { - field_flags |= IAVF_CLOUD_FIELD_IVLAN; - } else { + u16 vlan = match.key->vlan_id & VLAN_VID_MASK; + + if (match.mask->vlan_id != VLAN_VID_MASK) { dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", match.mask->vlan_id); return -EINVAL; } + if (!iavf_is_vlan_tc_filter_allowed(adapter, vlan)) { + dev_err(&adapter->pdev->dev, + "VLAN %u doesn't belong to this VF\n", + vlan); + return -EINVAL; + } + field_flags |= IAVF_CLOUD_FIELD_IVLAN; + m_spec->vlan_id = cpu_to_be16(match.mask->vlan_id); + d_spec->vlan_id = cpu_to_be16(match.key->vlan_id); } - cf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); - cf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); } if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { @@ -3012,13 +3864,21 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, addr_type = match.key->addr_type; } + /* process IPv4 header */ if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { struct flow_match_ipv4_addrs match; flow_rule_match_ipv4_addrs(rule, &match); + if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { + dev_info(&adapter->pdev->dev, + "Tenant id not allowed for ip filter\n"); + return -EINVAL; + } + if (match.mask->dst) { - if (match.mask->dst == cpu_to_be32(0xffffffff)) { + if (ADQ_V2_ALLOWED(adapter) || + match.mask->dst == cpu_to_be32(0xffffffff)) { field_flags |= IAVF_CLOUD_FIELD_IIP; } else { dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", @@ -3028,7 +3888,8 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } if (match.mask->src) { - if (match.mask->src == cpu_to_be32(0xffffffff)) { + if (ADQ_V2_ALLOWED(adapter) || + match.mask->src == cpu_to_be32(0xffffffff)) { field_flags |= IAVF_CLOUD_FIELD_IIP; } else { dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", @@ -3037,20 +3898,23 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } } - if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { - dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); - return -EINVAL; - } if (match.key->dst) { - cf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); - cf->data.tcp_spec.dst_ip[0] = match.key->dst; + if (ADQ_V2_ALLOWED(adapter)) + m_spec->dst_ip[0] = match.mask->dst; + else + m_spec->dst_ip[0] = cpu_to_be32(0xffffffff); + d_spec->dst_ip[0] = match.key->dst; } if (match.key->src) { - cf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); - cf->data.tcp_spec.src_ip[0] = match.key->src; + if (ADQ_V2_ALLOWED(adapter)) + m_spec->src_ip[0] = match.mask->src; + else + m_spec->src_ip[0] = cpu_to_be32(0xffffffff); + d_spec->src_ip[0] = match.key->src; } } + /* process IPv6 header */ if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { struct flow_match_ipv6_addrs match; @@ -3076,25 +3940,40 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, !ipv6_addr_any(&match.mask->src)) field_flags |= IAVF_CLOUD_FIELD_IIP; - for (i = 0; i < 4; i++) - cf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); - memcpy(&cf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, - sizeof(cf->data.tcp_spec.dst_ip)); - for (i = 0; i < 4; i++) - cf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); - memcpy(&cf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, - sizeof(cf->data.tcp_spec.src_ip)); + /* copy dest IPv6 mask and address */ + if (ADQ_V2_ALLOWED(adapter)) { + memcpy(&m_spec->dst_ip, &match.mask->dst.s6_addr32, + sizeof(m_spec->dst_ip)); + } else { + for (i = 0; i < 4; i++) + m_spec->dst_ip[i] = cpu_to_be32(0xffffffff); + } + memcpy(&d_spec->dst_ip, &match.key->dst.s6_addr32, + sizeof(d_spec->dst_ip)); + + /* copy source IPv6 mask and address */ + if (ADQ_V2_ALLOWED(adapter)) { + memcpy(&m_spec->src_ip, &match.mask->src.s6_addr32, + sizeof(m_spec->src_ip)); + } else { + for (i = 0; i < 4; i++) + m_spec->src_ip[i] = cpu_to_be32(0xffffffff); + } + memcpy(&d_spec->src_ip, &match.key->src.s6_addr32, + sizeof(d_spec->src_ip)); } + + /* process L4 header, supported L4 protocols are TCP and UDP */ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { struct flow_match_ports match; flow_rule_match_ports(rule, &match); if (match.key->dst) { - if (match.mask->dst == cpu_to_be16(0xffff)) { - cf->mask.tcp_spec.dst_port |= - cpu_to_be16(0xffff); - cf->data.tcp_spec.dst_port = match.key->dst; + if (ADQ_V2_ALLOWED(adapter) || + match.mask->dst == cpu_to_be16(0xffff)) { + m_spec->dst_port = match.mask->dst; + d_spec->dst_port = match.key->dst; } else { dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", be16_to_cpu(match.mask->dst)); @@ -3103,10 +3982,10 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } if (match.key->src) { - if (match.mask->src == cpu_to_be16(0xffff)) { - cf->mask.tcp_spec.src_port |= - cpu_to_be16(0xffff); - cf->data.tcp_spec.src_port = match.key->src; + if (ADQ_V2_ALLOWED(adapter) || + match.mask->src == cpu_to_be16(0xffff)) { + m_spec->src_port = match.mask->src; + d_spec->src_port = match.key->src; } else { dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", be16_to_cpu(match.mask->src)); @@ -3127,12 +4006,13 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, * * Return 0 on success, negative on failure */ -static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, +static int iavf_handle_tclass(struct iavf_adapter *adapter, int tc, struct iavf_cloud_filter *filter) { - if (tc == 0) - return 0; - if (tc < adapter->num_tc && (!ADQ_V2_ALLOWED(adapter)) && + if (tc < 0) + return -EINVAL; + + if (tc < adapter->num_tc && (!iavf_is_adq_v2_enabled(adapter)) && !filter->f.data.tcp_spec.dst_port) { dev_err(&adapter->pdev->dev, "Specify destination port to redirect to traffic classother than TC0\n"); @@ -3183,6 +4063,22 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter, return -EINVAL; } + if (adapter->num_cloud_filters >= IAVF_MAX_CLOUD_ADQ_FILTERS) { + dev_err(&adapter->pdev->dev, + "Unable to add filter (action is forward to TC) because VF reached the limit of max allowed filters (%u)\n", + IAVF_MAX_CLOUD_ADQ_FILTERS); + return -ENOSPC; + } + + /* bail out here if filter already exists */ + spin_lock_bh(&adapter->cloud_filter_list_lock); + if (iavf_find_cf(adapter, &cls_flower->cookie)) { + dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n"); + spin_unlock_bh(&adapter->cloud_filter_list_lock); + return -EEXIST; + } + spin_unlock_bh(&adapter->cloud_filter_list_lock); + filter = kzalloc(sizeof(*filter), GFP_KERNEL); if (!filter) return -ENOMEM; @@ -3197,15 +4093,6 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter, } filter->cookie = cls_flower->cookie; - /* bail out here if filter already exists */ - spin_lock_bh(&adapter->cloud_filter_list_lock); - if (iavf_find_cf(adapter, &cls_flower->cookie)) { - dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n"); - err = -EEXIST; - goto spin_unlock; - } - spin_unlock_bh(&adapter->cloud_filter_list_lock); - /* set the mask to all zeroes to begin with */ memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); /* start out with flow type and eth type IPv4 to begin with */ @@ -3218,16 +4105,33 @@ static int iavf_configure_clsflower(struct iavf_adapter *adapter, if (err) goto err; + /* store "channel" as back ptr to filter and it is applicable + * only if filter is for ADQ TC, where "hw_tc " + */ + if (tc >= IAVF_START_CHNL_TC && + tc < ARRAY_SIZE(adapter->ch_config.ch_ex_info)) + filter->ch = &adapter->ch_config.ch_ex_info[tc]; + else + filter->ch = NULL; + /* add filter to the list */ spin_lock_bh(&adapter->cloud_filter_list_lock); list_add_tail(&filter->list, &adapter->cloud_filter_list); adapter->num_cloud_filters++; filter->add = true; adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; -spin_unlock: spin_unlock_bh(&adapter->cloud_filter_list_lock); + + /* instead of waiting for the timer to expire (which could be as long as + * 1 sec), trigger the watchdog_task so that filter add command can be + * sent immediately. This will also reduce the time lag between when + * the filter add user command 'completes' and when the filter is + * actually added in HW. + */ + if (filter && filter->add) + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); err: - if (err) + if (err && filter) kfree(filter); clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); return err; @@ -3249,11 +4153,33 @@ static int iavf_delete_clsflower(struct iavf_adapter *adapter, if (filter) { filter->del = true; adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; - } else { + } else if (adapter->num_cloud_filters) { + /* "num_cloud_filters" can become zero if egress qdisc is + * detached as per design, driver deletes related filters + * when qdisc is detached to avoid stale filters, hence + * num_cloud_filters can become zero. But since netdev + * layer doesn't know that filters are deleted by driver + * implictly when egress qdisc is deleted, it sees filters + * being present and "in_hw". User can request delete + * of specific filter of detach ingress qdisc - in either of + * those operation, filter(s) won't be found in driver cache, + * hence instead if returning, let this function return SUCCESS + * Returning of err as -EINVAL is only applicable when + * unable to find filter and num_cloud_filters is non-zero + */ err = -EINVAL; } spin_unlock_bh(&adapter->cloud_filter_list_lock); + /* instead of waiting for the timer to expire (which could be as long as + * 1 sec), trigger the watchdog_task so that filter delete command can + * be sent immediately. This will also reduce the time lag between when + * the filter delete user command 'completes' and when the filter is + * actually deleted from HW. + */ + if (filter && filter->del) + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); + return err; } @@ -3265,6 +4191,9 @@ static int iavf_delete_clsflower(struct iavf_adapter *adapter, static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, struct flow_cls_offload *cls_flower) { + if (cls_flower->common.chain_index) + return -EOPNOTSUPP; + switch (cls_flower->command) { case FLOW_CLS_REPLACE: return iavf_configure_clsflower(adapter, cls_flower); @@ -3288,11 +4217,6 @@ static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) { - struct iavf_adapter *adapter = cb_priv; - - if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) - return -EOPNOTSUPP; - switch (type) { case TC_SETUP_CLSFLOWER: return iavf_setup_tc_cls_flower(cb_priv, type_data); @@ -3440,8 +4364,6 @@ static int iavf_close(struct net_device *netdev) } set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); - if (CLIENT_ENABLED(adapter)) - adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; iavf_down(adapter); iavf_change_state(adapter, __IAVF_DOWN_PENDING); @@ -3449,6 +4371,13 @@ static int iavf_close(struct net_device *netdev) clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); + /* If we're closing the interface as part of driver removal then don't + * wait. The VF resources will be reinitialized when the hardware is + * reset. + */ + if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) + return 0; + /* We explicitly don't free resources here because the hardware is * still active and can DMA into memory. Resources are cleared in * iavf_virtchnl_completion() after we get confirmation from the PF @@ -3527,6 +4456,16 @@ static int iavf_change_mtu(struct net_device *netdev, int new_mtu) return 0; } +#ifdef NETIF_F_HW_VLAN_CTAG_RX +#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ + NETIF_F_HW_VLAN_CTAG_TX | \ + NETIF_F_HW_VLAN_STAG_RX | \ + NETIF_F_HW_VLAN_STAG_TX) +#else +#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_RX | \ + NETIF_F_HW_VLAN_TX) +#endif + /** * iavf_set_features - set the netdev feature flags * @netdev: ptr to the netdev being adjusted @@ -3542,29 +4481,11 @@ static int iavf_set_features(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); - /* Don't allow changing VLAN_RX flag when adapter is not capable - * of VLAN offload - */ - if (!VLAN_ALLOWED(adapter)) { -#ifdef NETIF_F_HW_VLAN_CTAG_RX - if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) -#else - if ((netdev->features ^ features) & NETIF_F_HW_VLAN_RX) -#endif - return -EINVAL; -#ifdef NETIF_F_HW_VLAN_CTAG_RX - } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) { - if (features & NETIF_F_HW_VLAN_CTAG_RX) -#else - } else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_RX) { - if (features & NETIF_F_HW_VLAN_RX) -#endif - adapter->aq_required |= - IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; - else - adapter->aq_required |= - IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; - } + /* trigger update on any VLAN feature change */ + if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^ + (features & NETIF_VLAN_OFFLOAD_FEATURES)) + iavf_set_vlan_offload_features(adapter, netdev->features, + features); return 0; } @@ -3629,7 +4550,258 @@ out_err: } #endif /* HAVE_NDO_FEATURES_CHECK */ + +/** + * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features + * @adapter: board private structure + * + * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 + * were negotiated determine the VLAN features that can be toggled on and off. + **/ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT +static u32 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter) +#else +static netdev_features_t +iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter) +#endif +{ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT + u32 hw_features = 0; +#else + netdev_features_t hw_features = 0; +#endif /* HAVE_RHEL6_NET_DEVICE_OPS_EXT */ + + if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) + return hw_features; + + /* Enable VLAN features if supported */ + if (VLAN_ALLOWED(adapter)) { + hw_features |= (IAVF_NETIF_F_HW_VLAN_CTAG_TX | + IAVF_NETIF_F_HW_VLAN_CTAG_RX); + } else if (VLAN_V2_ALLOWED(adapter)) { + struct virtchnl_vlan_caps *vlan_v2_caps = + &adapter->vlan_v2_caps; + struct virtchnl_vlan_supported_caps *stripping_support = + &vlan_v2_caps->offloads.stripping_support; + struct virtchnl_vlan_supported_caps *insertion_support = + &vlan_v2_caps->offloads.insertion_support; + + if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && + stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) { + if (stripping_support->outer & VIRTCHNL_VLAN_ETHERTYPE_8100) + hw_features |= IAVF_NETIF_F_HW_VLAN_CTAG_RX; +#ifdef NETIF_F_HW_VLAN_STAG_RX + if (stripping_support->outer & VIRTCHNL_VLAN_ETHERTYPE_88A8) + hw_features |= NETIF_F_HW_VLAN_STAG_RX; +#endif /* NETIF_F_HW_VLAN_STAG_RX */ + } else if (stripping_support->inner != VIRTCHNL_VLAN_UNSUPPORTED && + stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) { + if (stripping_support->inner & VIRTCHNL_VLAN_ETHERTYPE_8100) + hw_features |= IAVF_NETIF_F_HW_VLAN_CTAG_RX; + } + + if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && + insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) { + if (insertion_support->outer & VIRTCHNL_VLAN_ETHERTYPE_8100) + hw_features |= IAVF_NETIF_F_HW_VLAN_CTAG_TX; +#ifdef NETIF_F_HW_VLAN_STAG_TX + if (insertion_support->outer & VIRTCHNL_VLAN_ETHERTYPE_88A8) + hw_features |= NETIF_F_HW_VLAN_STAG_TX; +#endif /* NETIF_F_HW_VLAN_STAG_TX */ + } else if (insertion_support->inner && + insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) { + if (insertion_support->inner & VIRTCHNL_VLAN_ETHERTYPE_8100) + hw_features |= IAVF_NETIF_F_HW_VLAN_CTAG_TX; + } + } + + return hw_features; +} + +/** + * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures + * @adapter: board private structure + * + * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 + * were negotiated determine the VLAN features that are enabled by default. + **/ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT +static u32 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter) +#else +static netdev_features_t +iavf_get_netdev_vlan_features(struct iavf_adapter *adapter) +#endif +{ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT + u32 features = 0; +#else + netdev_features_t features = 0; +#endif + + if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) + return features; + + if (VLAN_ALLOWED(adapter)) { + features |= IAVF_NETIF_F_HW_VLAN_CTAG_FILTER | + IAVF_NETIF_F_HW_VLAN_CTAG_RX | + IAVF_NETIF_F_HW_VLAN_CTAG_TX; + } else if (VLAN_V2_ALLOWED(adapter)) { + struct virtchnl_vlan_caps *vlan_v2_caps = + &adapter->vlan_v2_caps; + struct virtchnl_vlan_supported_caps *filtering_support = + &vlan_v2_caps->filtering.filtering_support; + struct virtchnl_vlan_supported_caps *stripping_support = + &vlan_v2_caps->offloads.stripping_support; + struct virtchnl_vlan_supported_caps *insertion_support = + &vlan_v2_caps->offloads.insertion_support; + u32 ethertype_init; + + /* give priority to outer stripping and don't support both outer + * and inner stripping + */ + ethertype_init = vlan_v2_caps->offloads.ethertype_init; + if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { + if (stripping_support->outer & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_RX; +#ifdef NETIF_F_HW_VLAN_STAG_RX + else if (stripping_support->outer & VIRTCHNL_VLAN_ETHERTYPE_88A8 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) + features |= NETIF_F_HW_VLAN_STAG_RX; +#endif /* NETIF_F_HW_VLAN_STAG_RX */ + } else if (stripping_support->inner != VIRTCHNL_VLAN_UNSUPPORTED) { + if (stripping_support->inner & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_RX; + } + + /* give priority to outer insertion and don't support both outer + * and inner insertion + */ + if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { + if (insertion_support->outer & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_TX; +#ifdef NETIF_F_HW_VLAN_STAG_TX + else if (insertion_support->outer & VIRTCHNL_VLAN_ETHERTYPE_88A8 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) + features |= NETIF_F_HW_VLAN_STAG_TX; +#endif /* NETIF_F_HW_VLAN_STAG_TX */ + } else if (insertion_support->inner != VIRTCHNL_VLAN_UNSUPPORTED) { + if (insertion_support->inner & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_TX; + } + + /* give priority to outer filtering and don't bother if both + * outer and inner filtering are enabled + */ + ethertype_init = vlan_v2_caps->filtering.ethertype_init; + if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { + if (filtering_support->outer & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_FILTER; +#ifdef NETIF_F_HW_VLAN_STAG_FILTER + if (filtering_support->outer & VIRTCHNL_VLAN_ETHERTYPE_88A8 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) + features |= NETIF_F_HW_VLAN_STAG_FILTER; +#endif /* NETIF_F_HW_VLAN_STAG_FILTER */ + } else if (filtering_support->inner != VIRTCHNL_VLAN_UNSUPPORTED) { + if (filtering_support->inner & VIRTCHNL_VLAN_ETHERTYPE_8100 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) + features |= IAVF_NETIF_F_HW_VLAN_CTAG_FILTER; +#ifdef NETIF_F_HW_VLAN_STAG_FILTER + if (filtering_support->inner & VIRTCHNL_VLAN_ETHERTYPE_88A8 && + ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) + features |= NETIF_F_HW_VLAN_STAG_FILTER; +#endif /* NETIF_F_HW_VLAN_STAG_FILTER */ + } + } + + return features; +} + #ifdef HAVE_NDO_SET_FEATURES + +#define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \ + (!(((requested) & (feature_bit)) && \ + !((allowed) & (feature_bit)))) + +/** + * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support + * @adapter: board private structure + * @requested_features: stack requested NETDEV features + **/ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT +static u32 +iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter, + u32 requested_features) +#else +static netdev_features_t +iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter, + netdev_features_t requested_features) +#endif +{ +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT + u32 allowed_features; +#else + netdev_features_t allowed_features; +#endif + + allowed_features = iavf_get_netdev_vlan_hw_features(adapter) | + iavf_get_netdev_vlan_features(adapter); + + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + IAVF_NETIF_F_HW_VLAN_CTAG_TX)) + requested_features &= ~IAVF_NETIF_F_HW_VLAN_CTAG_TX; + + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + IAVF_NETIF_F_HW_VLAN_CTAG_RX)) + requested_features &= ~IAVF_NETIF_F_HW_VLAN_CTAG_RX; + +#ifdef NETIF_F_HW_VLAN_STAG_TX + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + NETIF_F_HW_VLAN_STAG_TX)) + requested_features &= ~NETIF_F_HW_VLAN_STAG_TX; +#endif /* NETIF_F_HW_VLAN_STAG_TX */ +#ifdef NETIF_F_HW_VLAN_STAG_RX + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + NETIF_F_HW_VLAN_STAG_RX)) + requested_features &= ~NETIF_F_HW_VLAN_STAG_RX; +#endif /* NETIF_F_HW_VLAN_STAG_RX */ + + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + IAVF_NETIF_F_HW_VLAN_CTAG_FILTER)) + requested_features &= ~IAVF_NETIF_F_HW_VLAN_CTAG_FILTER; + +#ifdef NETIF_F_HW_VLAN_STAG_FILTER + if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, + allowed_features, + NETIF_F_HW_VLAN_STAG_FILTER)) + requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER; +#endif /* NETIF_F_HW_VLAN_STAG_FILTER */ + +#if defined(NETIF_F_HW_VLAN_STAG_RX) && defined(NETIF_F_HW_VLAN_STAG_TX) + if ((requested_features & + (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && + (requested_features & + (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) && + (adapter->vlan_v2_caps.offloads.ethertype_match == + VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION)) { + netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); + requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX | + NETIF_F_HW_VLAN_STAG_TX); + } +#endif /* NETIF_F_HW_VLAN_STAG_RX && NETIF_F_HW_VLAN_STAG_TX */ + + return requested_features; +} + /** * iavf_fix_features - fix up the netdev feature bits * @netdev: our net device @@ -3646,22 +4818,37 @@ static netdev_features_t iavf_fix_features(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); - if (adapter->vf_res && - !(adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) -#ifdef NETIF_F_HW_VLAN_CTAG_RX - features &= ~(NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_FILTER); -#else - features &= ~(NETIF_F_HW_VLAN_TX | - NETIF_F_HW_VLAN_RX | - NETIF_F_HW_VLAN_FILTER); -#endif - - return features; + return iavf_fix_netdev_vlan_features(adapter, features); } #endif /* HAVE_NDO_SET_FEATURES */ + +/** + * iavf_do_ioctl - Handle network device specific ioctls + * @netdev: network interface device structure + * @ifr: interface request data + * @cmd: ioctl command + * + * Callback to handle the networking device specific ioctls. Used to handle + * the SIOCGHWTSTAMP and SIOCSHWTSTAMP ioctl requests that configure Tx and Rx + * timstamping support. + */ +static int iavf_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + struct iavf_adapter *adapter = netdev_priv(netdev); + + switch (cmd) { +#ifdef SIOCGHWTSTAMP + case SIOCGHWTSTAMP: + return iavf_ptp_get_ts_config(adapter, ifr); +#endif + case SIOCSHWTSTAMP: + return iavf_ptp_set_ts_config(adapter, ifr); + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops iavf_netdev_ops = { #ifdef HAVE_RHEL7_NET_DEVICE_OPS_EXT /* RHEL7 requires this to be defined to enable extended ops. RHEL7 uses the @@ -3678,6 +4865,7 @@ static const struct net_device_ops iavf_netdev_ops = { .ndo_set_rx_mode = iavf_set_rx_mode, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = iavf_set_mac, + .ndo_do_ioctl = iavf_do_ioctl, #ifdef HAVE_RHEL7_EXTENDED_MIN_MAX_MTU .extended.ndo_change_mtu = iavf_change_mtu, #else @@ -3727,7 +4915,7 @@ static int iavf_check_reset_complete(struct iavf_hw *hw) u32 rstat; int i; - for (i = 0; i < IAVF_RESET_WAIT_COUNT; i++) { + for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { rstat = rd32(hw, IAVF_VFGEN_RSTAT) & IAVF_VFGEN_RSTAT_VFR_STATE_MASK; if ((rstat == VIRTCHNL_VFR_VFACTIVE) || @@ -3749,38 +4937,15 @@ static int iavf_check_reset_complete(struct iavf_hw *hw) int iavf_process_config(struct iavf_adapter *adapter) { struct virtchnl_vf_resource *vfres = adapter->vf_res; - int i, num_req_queues = adapter->num_req_queues; +#ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT + u32 hw_vlan_features, vlan_features; +#else + netdev_features_t hw_vlan_features, vlan_features; +#endif struct net_device *netdev = adapter->netdev; - struct iavf_vsi *vsi = &adapter->vsi; netdev_features_t hw_enc_features; netdev_features_t hw_features; - /* got VF config message back from PF, now we can parse it */ - for (i = 0; i < vfres->num_vsis; i++) { - if (vfres->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) - adapter->vsi_res = &vfres->vsi_res[i]; - } - if (!adapter->vsi_res) { - dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); - return -ENODEV; - } - - if (num_req_queues && - num_req_queues > adapter->vsi_res->num_queue_pairs) { - /* Problem. The PF gave us fewer queues than what we had - * negotiated in our request. Need a reset to see if we can't - * get back to a working state. - */ - dev_err(&adapter->pdev->dev, - "Requested %d queues, but PF only gave us %d.\n", - num_req_queues, - adapter->vsi_res->num_queue_pairs); - adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED; - adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; - iavf_schedule_reset(adapter); - return -ENODEV; - } - adapter->num_req_queues = 0; hw_enc_features = NETIF_F_SG | NETIF_F_IP_CSUM | @@ -3857,24 +5022,17 @@ int iavf_process_config(struct iavf_adapter *adapter) netdev->vlan_features |= hw_enc_features; #endif #endif - /* Write features and hw_features separately to avoid polluting * with, or dropping, features that are set when we registered. */ hw_features = hw_enc_features; - /* Enable VLAN features if supported */ - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) -#ifdef NETIF_F_HW_VLAN_CTAG_RX - hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_CTAG_RX); -#else - hw_features |= (NETIF_F_HW_VLAN_TX | - NETIF_F_HW_VLAN_RX); -#endif + /* get HW VLAN features that can be toggled */ + hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); + #ifdef NETIF_F_HW_TC /* Enable cloud filter if ADQ is supported */ - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) + if (ADQ_ALLOWED(adapter)) hw_features |= NETIF_F_HW_TC; #endif #ifdef NETIF_F_GSO_UDP_L4 @@ -3885,20 +5043,16 @@ int iavf_process_config(struct iavf_adapter *adapter) #ifdef HAVE_NDO_SET_FEATURES #ifdef HAVE_RHEL6_NET_DEVICE_OPS_EXT hw_features |= get_netdev_hw_features(netdev); - set_netdev_hw_features(netdev, hw_features); + set_netdev_hw_features(netdev, hw_features | hw_vlan_features); #else - netdev->hw_features |= hw_features; + netdev->hw_features |= hw_features | hw_vlan_features; #endif #endif /* HAVE_NDO_SET_FEATURES */ - netdev->features |= hw_features; + /* get VLAN features that cannot be toggled */ + vlan_features = iavf_get_netdev_vlan_features(adapter); - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) -#ifdef NETIF_F_HW_VLAN_CTAG_RX - netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; -#else - netdev->features |= NETIF_F_HW_VLAN_FILTER; -#endif + netdev->features |= hw_features | vlan_features; #ifdef IFF_UNICAST_FLT netdev->priv_flags |= IFF_UNICAST_FLT; @@ -3939,21 +5093,6 @@ int iavf_process_config(struct iavf_adapter *adapter) #endif /* HAVE_RHEL6_NET_DEVICE_OPS_EXT */ } - adapter->vsi.id = adapter->vsi_res->vsi_id; - - adapter->vsi.back = adapter; - adapter->vsi.base_vector = 1; - adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; - vsi->netdev = adapter->netdev; - vsi->qs_handle = adapter->vsi_res->qset_handle; - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { - adapter->rss_key_size = vfres->rss_key_size; - adapter->rss_lut_size = vfres->rss_lut_size; - } else { - adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; - adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; - } - return 0; } @@ -4077,6 +5216,7 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) spin_lock_init(&adapter->mac_vlan_list_lock); spin_lock_init(&adapter->cloud_filter_list_lock); + spin_lock_init(&adapter->current_netdev_promisc_flags_lock); INIT_LIST_HEAD(&adapter->mac_filter_list); INIT_LIST_HEAD(&adapter->vlan_filter_list); @@ -4084,12 +5224,22 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) INIT_WORK(&adapter->adminq_task, iavf_adminq_task); INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task); - INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task); queue_delayed_work(iavf_wq, &adapter->watchdog_task, msecs_to_jiffies(5 * (pdev->devfn & 0x07))); /* Setup the wait queue for indicating transition to down status */ init_waitqueue_head(&adapter->down_waitqueue); + INIT_LIST_HEAD(&adapter->ptp.aq_cmds); + init_waitqueue_head(&adapter->ptp.phc_time_waitqueue); + spin_lock_init(&adapter->ptp.aq_cmd_lock); + + /* By default, start the value of priv flags + * "channel-pkt-inspect-optimize" as ON. It's not in effect, + * unless ADQ type of filter is added and ADQ_V2 capability + * is negotiated + */ + adapter->flags |= IAVF_FLAG_CHNL_PKT_OPT_ENA; + return 0; err_ioremap: @@ -4105,12 +5255,77 @@ err_dma: #ifdef CONFIG_PM /** * iavf_suspend - Power management suspend routine + * @dev_d: device information struct + * + * Called when the system (VM) is entering sleep/suspend. + **/ +static int iavf_suspend(struct device *dev_d) +{ + struct net_device *netdev = dev_get_drvdata(dev_d); + struct iavf_adapter *adapter = netdev_priv(netdev); + + netif_device_detach(netdev); + + while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, + &adapter->crit_section)) + usleep_range(500, 1000); + + if (netif_running(netdev)) { + rtnl_lock(); + iavf_down(adapter); + rtnl_unlock(); + } + iavf_free_misc_irq(adapter); + iavf_reset_interrupt_capability(adapter); + + clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); + + return 0; +} + +/** + * iavf_resume - Power management resume routine + * @dev_d: device information struct + * + * Called when the system (VM) is resumed from sleep/suspend. + **/ +static int iavf_resume(struct device *dev_d) +{ + struct pci_dev *pdev = to_pci_dev(dev_d); + struct net_device *netdev = pci_get_drvdata(pdev); + struct iavf_adapter *adapter = netdev_priv(netdev); + u32 err; + + pci_set_master(pdev); + + rtnl_lock(); + err = iavf_set_interrupt_capability(adapter); + if (err) { + dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); + return err; + } + err = iavf_request_misc_irq(adapter); + rtnl_unlock(); + if (err) { + dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); + return err; + } + + iavf_schedule_reset(adapter); + netif_device_attach(netdev); + + return err; +} + +#ifdef USE_LEGACY_PM_SUPPORT +/** + * iavf_suspend_legacy - Power management suspend routine * @pdev: PCI device information struct * @state: unused * * Called when the system (VM) is entering sleep/suspend. **/ -static int iavf_suspend(struct pci_dev *pdev, pm_message_t state) +static int iavf_suspend_legacy(struct pci_dev *pdev, pm_message_t state) { struct net_device *netdev = pci_get_drvdata(pdev); struct iavf_adapter *adapter = netdev_priv(netdev); @@ -4142,15 +5357,15 @@ static int iavf_suspend(struct pci_dev *pdev, pm_message_t state) } /** - * iavf_resume - Power management resume routine + * iavf_resume_legacy - Power management resume routine * @pdev: PCI device information struct * * Called when the system (VM) is resumed from sleep/suspend. **/ -static int iavf_resume(struct pci_dev *pdev) +static int iavf_resume_legacy(struct pci_dev *pdev) { - struct iavf_adapter *adapter = pci_get_drvdata(pdev); - struct net_device *netdev = adapter->netdev; + struct net_device *netdev = pci_get_drvdata(pdev); + struct iavf_adapter *adapter = netdev_priv(netdev); u32 err; pci_set_power_state(pdev, PCI_D0); @@ -4185,8 +5400,9 @@ static int iavf_resume(struct pci_dev *pdev) return err; } - +#endif /* USE_LEGACY_PM_SUPPORT */ #endif /* CONFIG_PM */ + /** * iavf_remove - Device Removal Routine * @pdev: PCI device information struct @@ -4208,28 +5424,30 @@ static void iavf_remove(struct pci_dev *pdev) struct iavf_cloud_filter *cf, *cftmp; struct iavf_mac_filter *f, *ftmp; struct iavf_hw *hw = &adapter->hw; - int err; - /* Indicate we are in remove and not to run reset_task */ + /* Indicate we are in remove and not to run/schedule any driver tasks */ set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section); - cancel_delayed_work_sync(&adapter->client_task); + cancel_work_sync(&adapter->adminq_task); + cancel_delayed_work_sync(&adapter->watchdog_task); + + iavf_misc_irq_disable(adapter); + if (adapter->netdev_registered) { - /* will call iavf_close if the device was open previously */ + /* This will call iavf_close if the device was open previously. + * The Admin Queue and watchdog tasks have already been shut + * down at this point so the driver will rely on + * iavf_request_reset below to disable the queues and handle + * any other Admin Queue-based cleanup normally done as part of + * iavf_close. + */ unregister_netdev(netdev); adapter->netdev_registered = false; - /* wait for device down */ - wait_event_timeout(adapter->down_waitqueue, - adapter->state == __IAVF_DOWN, - msecs_to_jiffies(200)); } - if (CLIENT_ALLOWED(adapter)) { - err = iavf_lan_del_device(adapter); - if (err) - dev_warn(&pdev->dev, "Failed to delete client device: %d\n", - err); - } dev_info(&adapter->pdev->dev, "Removing device\n"); + + iavf_ptp_release(adapter); + /* Shut down all the garbage mashers on the detention level */ iavf_change_state(adapter, __IAVF_REMOVE); adapter->aq_required = 0; @@ -4242,11 +5460,6 @@ static void iavf_remove(struct pci_dev *pdev) msleep(50); } - iavf_misc_irq_disable(adapter); - cancel_work_sync(&adapter->adminq_task); - - cancel_delayed_work_sync(&adapter->watchdog_task); - iavf_free_all_tx_resources(adapter); iavf_free_all_rx_resources(adapter); iavf_free_misc_irq(adapter); @@ -4299,6 +5512,10 @@ static void iavf_remove(struct pci_dev *pdev) pci_disable_device(pdev); } +#if defined(CONFIG_PM) && !defined(USE_LEGACY_PM_SUPPORT) +static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume); +#endif /* CONFIG_PM && !USE_LEGACY_PM_SUPPORT */ + static struct pci_driver iavf_driver = { .name = iavf_driver_name, .id_table = iavf_pci_tbl, @@ -4309,9 +5526,13 @@ static struct pci_driver iavf_driver = { .remove = iavf_remove, #endif #ifdef CONFIG_PM - .suspend = iavf_suspend, - .resume = iavf_resume, -#endif +#ifdef USE_LEGACY_PM_SUPPORT + .suspend = iavf_suspend_legacy, + .resume = iavf_resume_legacy, +#else + .driver.pm = &iavf_pm_ops, +#endif /* USE_LEGACY_PM_SUPPORT */ +#endif /* CONFIG_PM */ .shutdown = iavf_shutdown, }; diff --git a/drivers/net/ethernet/intel/iavf/iavf_prototype.h b/drivers/net/ethernet/intel/iavf/iavf_prototype.h index d0be03842..a180e83ad 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_prototype.h +++ b/drivers/net/ethernet/intel/iavf/iavf_prototype.h @@ -54,7 +54,7 @@ enum iavf_status iavf_set_mac_type(struct iavf_hw *hw); extern struct iavf_rx_ptype_decoded iavf_ptype_lookup[]; -static inline struct iavf_rx_ptype_decoded decode_rx_desc_ptype(u8 ptype) +static inline struct iavf_rx_ptype_decoded decode_rx_desc_ptype(u16 ptype) { return iavf_ptype_lookup[ptype]; } @@ -70,7 +70,7 @@ void iavf_vf_parse_hw_config(struct iavf_hw *hw, enum iavf_status iavf_vf_reset(struct iavf_hw *hw); enum iavf_status iavf_aq_send_msg_to_pf(struct iavf_hw *hw, enum virtchnl_ops v_opcode, - enum iavf_status v_retval, + enum virtchnl_status_code v_retval, u8 *msg, u16 msglen, struct iavf_asq_cmd_details *cmd_details); enum iavf_status iavf_aq_debug_dump(struct iavf_hw *hw, u8 cluster_id, diff --git a/drivers/net/ethernet/intel/iavf/iavf_ptp.c b/drivers/net/ethernet/intel/iavf/iavf_ptp.c new file mode 100644 index 000000000..3017a5b5f --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/iavf_ptp.c @@ -0,0 +1,1041 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2013, Intel Corporation. */ + +#include "iavf.h" + +/** + * iavf_ptp_disable_tx_tstamp - Disable timestamping in Tx rings + * @adapter: private adapter structure + * + * Disable timestamp capture for all Tx rings + */ +static void iavf_ptp_disable_tx_tstamp(struct iavf_adapter *adapter) +{ + unsigned int i; + + for (i = 0; i < adapter->num_active_queues; i++) + adapter->tx_rings[i].flags &= ~IAVF_TXRX_FLAGS_HW_TSTAMP; +} + +/** + * iavf_ptp_enable_tx_tstamp - Enable timestamping in Tx rings + * @adapter: private adapter structure + * + * Enable timestamp capture for all Tx rings + */ +static void iavf_ptp_enable_tx_tstamp(struct iavf_adapter *adapter) +{ + unsigned int i; + + for (i = 0; i < adapter->num_active_queues; i++) + adapter->tx_rings[i].flags |= IAVF_TXRX_FLAGS_HW_TSTAMP; +} + +/** + * iavf_ptp_disable_rx_tstamp - Disable timestamping in Rx rings + * @adapter: private adapter structure + * + * Disable timestamp reporting for all Rx rings. + */ +static void iavf_ptp_disable_rx_tstamp(struct iavf_adapter *adapter) +{ + unsigned int i; + + for (i = 0; i < adapter->num_active_queues; i++) + adapter->rx_rings[i].flags &= ~IAVF_TXRX_FLAGS_HW_TSTAMP; +} + +/** + * iavf_ptp_enable_rx_tstamp - Enable timestamping in Rx rings + * @adapter: private adapter structure + * + * Enable timestamp reporting for all Rx rings. + */ +static void iavf_ptp_enable_rx_tstamp(struct iavf_adapter *adapter) +{ + unsigned int i; + + for (i = 0; i < adapter->num_active_queues; i++) + adapter->rx_rings[i].flags |= IAVF_TXRX_FLAGS_HW_TSTAMP; +} + +/** + * iavf_ptp_set_timestamp_mode - Set device timestamping mode + * @adapter: private adapter structure + * @config: timestamping configuration request + * + * Set the timestamping mode requested from the SIOCSHWTSTAMP ioctl. + * + * Note: this function always translates Rx timestamp requests for any packet + * category into HWTSTAMP_FILTER_ALL. + */ +static int +iavf_ptp_set_timestamp_mode(struct iavf_adapter *adapter, struct hwtstamp_config *config) +{ + /* Reserved for future extensions. */ + if (config->flags) + return -EINVAL; + + switch (config->tx_type) { + case HWTSTAMP_TX_OFF: + iavf_ptp_disable_tx_tstamp(adapter); + break; + case HWTSTAMP_TX_ON: + if (!(iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_TX_TSTAMP))) + return -EOPNOTSUPP; + iavf_ptp_enable_tx_tstamp(adapter); + break; + default: + return -ERANGE; + } + + switch (config->rx_filter) { + case HWTSTAMP_FILTER_NONE: + iavf_ptp_disable_rx_tstamp(adapter); + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: +#ifdef HAVE_HWTSTAMP_FILTER_NTP_ALL + case HWTSTAMP_FILTER_NTP_ALL: +#endif /* HAVE_HWTSTAMP_FILTER_NTP_ALL */ + case HWTSTAMP_FILTER_ALL: + if (!(iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP))) + return -EOPNOTSUPP; + config->rx_filter = HWTSTAMP_FILTER_ALL; + iavf_ptp_enable_rx_tstamp(adapter); + break; + default: + return -ERANGE; + } + + return 0; +} + +/** + * iavf_ptp_get_ts_config - Get timestamping configuration for SIOCGHWTSTAMP + * @adapter: private adapter structure + * @ifr: the ioctl request structure + * + * Copy the current hardware timestamping configuration back to userspace. + * Called in response to the SIOCGHWTSTAMP ioctl that queries a device's + * current timestamp settings. + */ +int iavf_ptp_get_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr) +{ + struct hwtstamp_config *config = &adapter->ptp.hwtstamp_config; + + return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ? -EFAULT : 0; +} + +/** + * iavf_ptp_set_ts_config - Set timestamping configuration from SIOCSHWTSTAMP + * @adapter: private adapter structure + * @ifr: the ioctl request structure + * + * Program the requested timestamping configuration from SIOCSHWTSTAMP ioctl + * to the device. + */ +int iavf_ptp_set_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr) +{ + struct hwtstamp_config config; + int err; + + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + err = iavf_ptp_set_timestamp_mode(adapter, &config); + if (err) + return err; + + /* Save successful settings for future reference */ + adapter->ptp.hwtstamp_config = config; + + return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0; +} + +/** + * clock_to_adapter - Convert clock info pointer to adapter pointer + * @ptp_info: PTP info structure + * + * Use container_of in order to extract a pointer to the iAVF adapter private + * structure. + */ +static struct iavf_adapter *clock_to_adapter(struct ptp_clock_info *ptp_info) +{ + struct iavf_ptp *ptp_priv; + + ptp_priv = container_of(ptp_info, struct iavf_ptp, info); + return container_of(ptp_priv, struct iavf_adapter, ptp); +} + +/** + * iavf_ptp_cap_supported - Check if a PTP capability is supported + * @adapter: private adapter structure + * @cap: the capability bitmask to check + * + * Return true if every capability set in cap is also set in the enabled + * capabilities reported by the PF. + */ +bool iavf_ptp_cap_supported(struct iavf_adapter *adapter, u32 cap) +{ + if (!PTP_ALLOWED(adapter)) + return false; + + /* Only return true if every bit in cap is set in hw_caps.caps */ + return (adapter->ptp.hw_caps.caps & cap) == cap; +} + +/** + * iavf_allocate_ptp_cmd - Allocate a PTP command message structure + * @v_opcode: the virtchnl opcode + * @msglen: length in bytes of the associated virtchnl structure + * + * Allocates a PTP command message and pre-fills it with the provided message + * length and opcode. + */ +static struct iavf_ptp_aq_cmd *iavf_allocate_ptp_cmd(enum virtchnl_ops v_opcode, u16 msglen) +{ + struct iavf_ptp_aq_cmd *cmd; + + cmd = kzalloc(struct_size(cmd, msg, msglen), GFP_KERNEL); + if (!cmd) + return NULL; + + cmd->v_opcode = v_opcode; + cmd->msglen = msglen; + + return cmd; +} + +/** + * iavf_queue_ptp_cmd - Queue PTP command for sending over virtchnl + * @adapter: private adapter structure + * @cmd: the command structure to send + * + * Queue the given command structure into the PTP virtchnl command queue tos + * end to the PF. + */ +static void iavf_queue_ptp_cmd(struct iavf_adapter *adapter, struct iavf_ptp_aq_cmd *cmd) +{ + spin_lock(&adapter->ptp.aq_cmd_lock); + list_add_tail(&cmd->list, &adapter->ptp.aq_cmds); + spin_unlock(&adapter->ptp.aq_cmd_lock); + + adapter->aq_required |= IAVF_FLAG_AQ_SEND_PTP_CMD; + mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); +} + +/** + * iavf_send_phc_read - Send request to read PHC time + * @adapter: private adapter structure + * + * Send a request to obtain the PTP hardware clock time. This allocates the + * VIRTCHNL_OP_1588_PTP_GET_TIME message and queues it up to send to + * indirectly read the PHC time. + * + * This function does not wait for the reply from the PF. + */ +static int iavf_send_phc_read(struct iavf_adapter *adapter) +{ + struct iavf_ptp_aq_cmd *cmd; + + if (!adapter->ptp.initialized) + return -EOPNOTSUPP; + + cmd = iavf_allocate_ptp_cmd(VIRTCHNL_OP_1588_PTP_GET_TIME, + sizeof(struct virtchnl_phc_time)); + if (!cmd) + return -ENOMEM; + + iavf_queue_ptp_cmd(adapter, cmd); + + return 0; +} + +/** + * iavf_read_phc_indirect - Indirectly read the PHC time via virtchnl + * @adapter: private adapter structure + * @ts: storage for the timestamp value + * @sts: system timestamp values before and after the read + * + * Used when the device does not have direct register access to the PHC time. + * Indirectly reads the time via the VIRTCHNL_OP_1588_PTP_GET_TIME, and waits + * for the reply from the PF. + * + * Based on some simple measurements using ftrace and phc2sys, this clock + * access method has about a ~110 usec latency even when the system is not + * under load. In order to achieve acceptable results when using phc2sys with + * the indirect clock access method, it is recommended to use more + * conservative proportional and integration constants with the P/I servo. + */ +static int iavf_read_phc_indirect(struct iavf_adapter *adapter, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + long ret; + int err; + + adapter->ptp.phc_time_ready = false; + ptp_read_system_prets(sts); + + err = iavf_send_phc_read(adapter); + if (err) + return err; + + ret = wait_event_interruptible_timeout(adapter->ptp.phc_time_waitqueue, + adapter->ptp.phc_time_ready, + HZ); + if (ret < 0) + return ret; + else if (!ret) + return -EBUSY; + + *ts = ns_to_timespec64(adapter->ptp.cached_phc_time); + + ptp_read_system_postts(sts); + + return 0; +} + +/** + * iavf_read_phc_ns - Read PHC time from registers and convert to nanoseconds + * @adapter: private adapter structure + * @sts: system timestamp values before and after the read + * + * Capture the PHC time from the registers and convert it to nanoseconds. + * Capture the system time before and after reading the lower clock register, + * to allow more precise comparison between the PHC time and CLOCK_REALTIME. + * + * This requires direct access to the PHC registers, which may not be + * available on all devices. + * + * If this method is available, it has a significantly reduced latency of + * about 2 microseconds. It is preferred whenever available. + */ +static u64 iavf_read_phc_ns(struct iavf_adapter *adapter, struct ptp_system_timestamp *sts) +{ + u8 __iomem *phc_addr, *clock_lo, *clock_hi; + u32 hi, hi2, lo; + + phc_addr = READ_ONCE(adapter->ptp.phc_addr); + if (WARN_ON(!phc_addr)) + return 0; + + clock_lo = phc_addr + adapter->ptp.hw_caps.phc_regs.clock_lo; + clock_hi = phc_addr + adapter->ptp.hw_caps.phc_regs.clock_hi; + + hi = readl(clock_hi); + ptp_read_system_prets(sts); + lo = readl(clock_lo); + ptp_read_system_postts(sts); + hi2 = readl(clock_hi); + + if (hi != hi2) { + /* clock_lo might have rolled over, so recapture it */ + ptp_read_system_prets(sts); + lo = readl(clock_lo); + ptp_read_system_postts(sts); + hi = hi2; + } + + return ((u64)hi << 32) | lo; +} + +/** + * iavf_read_phc_direct - Directly read PHC time from the registers + * @adapter: private adapter structure + * @ts: storage for the PHC time + * @sts: system timestamp values before and after the read + * + * Read the PHC time from the registers, and convert it to a timespec64. + */ +static int iavf_read_phc_direct(struct iavf_adapter *adapter, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + u64 time = iavf_read_phc_ns(adapter, sts); + + *ts = ns_to_timespec64(time); + + return 0; +} + +/** + * iavf_ptp_gettimex64 - Get current PTP clock time + * @ptp: PTP clock info structure + * @ts: storage for the current time + * @sts: system timestamps before and after time captured + * + * Read the current PTP clock time, and return it in the ts structure. Capture + * the system time before and after the PTP clock time in sts. Note that + * ptp_read_sytsem_prets and ptp_read_system_postts are NULL-aware and will do + * nothing if sts is NULL. + */ +static int iavf_ptp_gettimex64(struct ptp_clock_info *ptp, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct iavf_adapter *adapter = clock_to_adapter(ptp); + + if (!adapter->ptp.initialized) + return -ENODEV; + + if (adapter->ptp.phc_addr) + return iavf_read_phc_direct(adapter, ts, sts); + else + return iavf_read_phc_indirect(adapter, ts, sts); +} + +#ifndef HAVE_PTP_CLOCK_INFO_GETTIMEX64 +/** + * iavf_ptp_gettime64 - wrapper in case ptp_caps doesn't have .gettimex64 + * @ptp: PTP clock info structure + * @ts: storage for the current time + * + * Implement .gettime64 for the PTP clock. Wrapper that just calls + * iavf_ptp_gettimex64 with a NULL sts pointer. + */ +static int iavf_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts) +{ + return iavf_ptp_gettimex64(ptp, ts, NULL); +} + +#ifndef HAVE_PTP_CLOCK_INFO_GETTIME64 +/** + * iavf_ptp_gettime32 - wrapper in case ptp_caps doesn't have .gettime64 + * @ptp: PTP clock info structure + * @ts: storage for the current time + * + * Implement .gettime for the PTP clock. Wrapper that just calls + * iavf_ptp_gettime64 and converts the timespec back to a 32bit timespec + * before returning. + */ +static int iavf_ptp_gettime32(struct ptp_clock_info *ptp, struct timespec *ts) +{ + struct timespec64 ts64; + int err; + + err = iavf_ptp_gettime64(ptp, &ts64); + if (err) + return err; + + *ts = timespec64_to_timespec(ts64); + return 0; +} +#endif /* !HAVE_PTP_CLOCK_INFO_GETTIME64 */ +#endif /* !HAVE_PTP_CLOCK_INFO_GETTIMEX64 */ + +/** + * iavf_ptp_settime64 - Set PTP clock time + * @ptp: PTP clock info structure + * @ts: the time to set the clock to + * + * Set the PTP clock time to the requested value. + */ +static int iavf_ptp_settime64(struct ptp_clock_info *ptp, const struct timespec64 *ts) +{ + struct iavf_adapter *adapter = clock_to_adapter(ptp); + struct virtchnl_phc_time *msg; + struct iavf_ptp_aq_cmd *cmd; + + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_WRITE_PHC)) + return -EACCES; + + if (!adapter->ptp.initialized) + return -ENODEV; + + cmd = iavf_allocate_ptp_cmd(VIRTCHNL_OP_1588_PTP_SET_TIME, sizeof(*msg)); + if (!cmd) + return -ENOMEM; + + msg = (typeof(msg))cmd->msg; + msg->time = timespec64_to_ns(ts); + + iavf_queue_ptp_cmd(adapter, cmd); + + return 0; +} + +#ifndef HAVE_PTP_CLOCK_INFO_GETTIME64 +/** + * iavf_ptp_settime32 - wrapper in case ptp_caps doesn't have .settime64 + * @ptp: PTP clock info structure + * @ts: 32bit timespec with requested time + * + * Implement .settime for the PTP clock. Wrapper that just calls + * iavf_ptp_settime64 after converting the 32bit timespec to a 64bit timespec. + */ +static int iavf_ptp_settime32(struct ptp_clock_info *ptp, const struct timespec *ts) +{ + struct timespec64 ts64 = timespec_to_timespec64(*ts); + + return iavf_ptp_settime64(ptp, &ts64); +} +#endif + +/** + * iavf_ptp_adjtime - Adjust PTP clock time by requested amount + * @ptp: PTP clock info structure + * @delta: Offset in nanoseconds to adjust the clock time by + * + * Adjust the PTP clock time by the provided delta. + */ +static int iavf_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct iavf_adapter *adapter = clock_to_adapter(ptp); + struct virtchnl_phc_adj_time *msg; + struct iavf_ptp_aq_cmd *cmd; + + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_WRITE_PHC)) + return -EACCES; + + if (!adapter->ptp.initialized) + return -ENODEV; + + cmd = iavf_allocate_ptp_cmd(VIRTCHNL_OP_1588_PTP_ADJ_TIME, sizeof(*msg)); + if (!cmd) + return -ENOMEM; + + msg = (typeof(msg))cmd->msg; + msg->delta = delta; + + iavf_queue_ptp_cmd(adapter, cmd); + + return 0; +} + +/** + * iavf_ptp_adjfine - Adjust PTP clock time by scaled parts per million + * @ptp: PTP clock info structure + * @scaled_ppm: scaled parts per million adjustment + * + * Perform a frequency adjustment by the provided scaled parts per million + * value. + */ +static int iavf_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct iavf_adapter *adapter = clock_to_adapter(ptp); + struct virtchnl_phc_adj_freq *msg; + struct iavf_ptp_aq_cmd *cmd; + + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_WRITE_PHC)) + return -EACCES; + + if (!adapter->ptp.initialized) + return -ENODEV; + + cmd = iavf_allocate_ptp_cmd(VIRTCHNL_OP_1588_PTP_ADJ_FREQ, sizeof(*msg)); + if (!cmd) + return -ENOMEM; + + msg = (typeof(msg))cmd->msg; + msg->scaled_ppm = (s64)scaled_ppm; + + iavf_queue_ptp_cmd(adapter, cmd); + + return 0; +} + +#ifndef HAVE_PTP_CLOCK_INFO_ADJFINE +/** + * ppb_to_scaled_ppm - Convert parts per billion to scaled parts per million + * @ppb: parts per billion value + * + * Older versions of the kernel stack request frequency adjustments in parts + * per billion. Newer kernels can request adjustment using the full 'freq' + * field from the 'struct timex'. This is represented as parts per million, + * but with a 16 bit binary fractional field, i.e. parts per 1 million * 2^16. + * + * In essence, this is adjustments in parts per 65,536,000,000, which we call + * scaled_ppm. + * + * The following equation shows the relationship between ppb and scaled_ppm: + * + * ppb = scaled_ppm * 1000 / 2^16 + * + * i.e. + * + * scaled_ppm = (ppb / 1000) * 2^16 + * + * We can further simplify this to: + * + * scaled_ppm = ( ppb / 125 ) * 2^13 + * + * For reference, here is the approximate conversion between scaled_ppm and ppb: + * + * 1 scaled_ppm ~= 0.015 ppb + * 1 ppb ~= 65.5 scaled_ppm + */ +static long ppb_to_scaled_ppm(s32 ppb) +{ + long scaled_ppm; + + scaled_ppm = (s64)ppb << 13; + scaled_ppm /= 125; + + return scaled_ppm; +} + +/** + * iavf_ptp_adjfreq - wrapper in case ptp_caps doesn't have .adjfine + * @ptp: PTP clock info structure + * @ppb: parts per billion frequency adjustment + * + * Implement .adjfreq for the PTP clock. Wrapper that converts ppb to + * scaled_ppm and then calls iavf_ptp_adjfine. + */ +static int iavf_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +{ + return iavf_ptp_adjfine(ptp, ppb_to_scaled_ppm(ppb)); +} +#endif + +/** + * iavf_ptp_tx_hang - Detect when Tx timestamp has taken too long + * @adapter: private adapter structure + * + * Detect when a Tx timestamp event has been outstanding for more than one + * second. If this occurs, discard the waiting SKB and clear the flag. + * + * This is important for two reasons. First, if a timestamp event is missed + * and we do nothing, the driver could prevent all future timestamp requests + * indefinitely. Second, if a timestamp event is late, the timestamp extension + * algorithm might incorrectly calculate the wrong timestamp. + */ +static void iavf_ptp_tx_hang(struct iavf_adapter *adapter) +{ + if (!test_bit(__IAVF_TX_TSTAMP_IN_PROGRESS, &adapter->crit_section)) + return; + + if (time_is_before_jiffies(adapter->ptp.tx_start + HZ)) { + struct sk_buff *skb = adapter->ptp.tx_skb; + + adapter->ptp.tx_skb = NULL; + clear_bit_unlock(__IAVF_TX_TSTAMP_IN_PROGRESS, &adapter->crit_section); + + /* Free the SKB after we've cleared the bitlock */ + dev_kfree_skb_any(skb); + adapter->ptp.tx_hwtstamp_timeouts++; + } +} + +/** + * iavf_ptp_cache_phc_time - Cache PHC time for performing timestamp extension + * @adapter: private adapter structure + * + * Periodically cache the PHC time in order to allow for timestamp extension. + * This is required because the Tx and Rx timestamps only contain 32bits of + * nanoseconds. Timestamp extension allows calculating the corrected 64bit + * timestamp. This algorithm relies on the cached time being within ~1 second + * of the timestamp. + */ +static void iavf_ptp_cache_phc_time(struct iavf_adapter *adapter) +{ + if (time_is_before_jiffies(adapter->ptp.cached_phc_updated + HZ)) { + if (adapter->ptp.phc_addr) { + adapter->ptp.cached_phc_time = iavf_read_phc_ns(adapter, NULL); + adapter->ptp.cached_phc_updated = jiffies; + } else { + /* The response from virtchnl will store the time into cached_phc_time */ + iavf_send_phc_read(adapter); + } + } +} + +/** + * iavf_ptp_do_aux_work - Perform periodic work required for PTP support + * @ptp: PTP clock info structure + * + * Handler to take care of periodic work required for PTP operation. This + * includes the following tasks: + * + * 1) updating cached_phc_time + * + * cached_phc_time is used by the Tx and Rx timestamp flows in order to + * perform timestamp extension, by carefully comparing the timestamp + * 32bit nanosecond timestamps and determining the corrected 64bit + * timestamp value to report to userspace. This algorithm only works if + * the cached_phc_time is within ~1 second of the Tx or Rx timestamp + * event. This task periodically reads the PHC time and stores it, to + * ensure that timestamp extension operates correctly. + * + * 2) canceling outstanding Tx timestamp events + * + * Tx timestamps require waiting to receive a timestamp event indication + * from hardware. In some rare cases, the packet might have been dropped + * without a timestamp. If this occurs, the Tx timestamp event will never + * complete. To avoid this, we check if a timestamp event has taken too + * long, and discard it if so. + * + * Returns: time in jiffies until the periodic task should be re-scheduled. + */ +long iavf_ptp_do_aux_work(struct ptp_clock_info *ptp) +{ + struct iavf_adapter *adapter = clock_to_adapter(ptp); + + iavf_ptp_cache_phc_time(adapter); + iavf_ptp_tx_hang(adapter); + + /* Check work about twice a second */ + return msecs_to_jiffies(500); +} + +/** + * iavf_ptp_register_clock - Register a new PTP for userspace + * @adapter: private adapter structure + * + * Allocate and register a new PTP clock device if necessary. + */ +static int iavf_ptp_register_clock(struct iavf_adapter *adapter) +{ + struct ptp_clock_info *ptp_info = &adapter->ptp.info; + struct device *dev = &adapter->pdev->dev; + + memset(ptp_info, 0, sizeof(*ptp_info)); + + snprintf(ptp_info->name, sizeof(ptp_info->name) - 1, "%s-%s-clk", dev_driver_string(dev), + netdev_name(adapter->netdev)); + ptp_info->owner = THIS_MODULE; + ptp_info->max_adj = adapter->ptp.hw_caps.max_adj; + +#if defined(HAVE_PTP_CLOCK_INFO_GETTIMEX64) + ptp_info->gettimex64 = iavf_ptp_gettimex64; +#elif defined(HAVE_PTP_CLOCK_INFO_GETTIME64) + ptp_info->gettime64 = iavf_ptp_gettime64; +#else + ptp_info->gettime = iavf_ptp_gettime32; +#endif +#ifdef HAVE_PTP_CLOCK_INFO_GETTIME64 + ptp_info->settime64 = iavf_ptp_settime64; +#else + ptp_info->settime = iavf_ptp_settime32; +#endif + ptp_info->adjtime = iavf_ptp_adjtime; +#ifdef HAVE_PTP_CLOCK_INFO_ADJFINE + ptp_info->adjfine = iavf_ptp_adjfine; +#else + ptp_info->adjfreq = iavf_ptp_adjfreq; +#endif +#ifdef HAVE_PTP_CLOCK_DO_AUX_WORK + ptp_info->do_aux_work = iavf_ptp_do_aux_work; +#endif + + dev_info(&adapter->pdev->dev, "registering PTP clock %s\n", adapter->ptp.info.name); + + adapter->ptp.clock = ptp_clock_register(ptp_info, dev); + if (IS_ERR(adapter->ptp.clock)) + return PTR_ERR(adapter->ptp.clock); + + return 0; +} + +/** + * iavf_ptp_map_phc_addr - Map PHC clock register region + * @adapter: private adapter structure + * + * Map the PCI region that contains the PTP hardware clock registers for + * directly accessing the device time. + */ +static void iavf_ptp_map_phc_addr(struct iavf_adapter *adapter) +{ + struct virtchnl_ptp_caps *hw_caps = &adapter->ptp.hw_caps; + struct device *dev = &adapter->pdev->dev; + resource_size_t region_size; + void __iomem *phc_addr; + + WARN(adapter->ptp.phc_addr, "PHC clock register address already mapped"); + + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_PHC_REGS)) { + dev_dbg(dev, "Device does not have direct clock register access. Falling back to indirect clock access\n"); + return; + } + + region_size = pci_resource_len(adapter->pdev, hw_caps->phc_regs.pcie_region); + + if (hw_caps->phc_regs.clock_lo > region_size) { + dev_warn(dev, "Low clock register outside of PHC bar area. Falling back to indirect clock access\n"); + return; + } + + if (hw_caps->phc_regs.clock_hi > region_size) { + dev_warn(dev, "High clock register outside of PHC bar area. Falling back to indirect clock access\n"); + return; + } + + phc_addr = pci_ioremap_bar(adapter->pdev, hw_caps->phc_regs.pcie_region); + if (!phc_addr) { + dev_warn(dev, "Unable to map PHC registers for clock access. Falling back to indirect clock access\n"); + return; + } + + adapter->ptp.phc_addr = phc_addr; +} + +/** + * iavf_ptp_unmap_phc_addr - Unmap the PHC clock register region + * @adapter: private adapter structure + * + * Unmap and release the PHC clock register region. + */ +static void iavf_ptp_unmap_phc_addr(struct iavf_adapter *adapter) +{ + if (adapter->ptp.phc_addr) { + iounmap(adapter->ptp.phc_addr); + adapter->ptp.phc_addr = NULL; + } +} + +/** + * iavf_validate_tx_tstamp_format - Check if driver knows timestamp format + * @adapter: private adapter structure + * + * Check that the driver understands the timestamp format that the PF + * indicated. If we do not understand the format, then we must disable Tx + * timestamps. Otherwise we might process timestamps from + * VIRTCHNL_OP_1588_PTP_TX_TSTAMP incorrectly. + */ +static void iavf_validate_tx_tstamp_format(struct iavf_adapter *adapter) +{ + struct device *dev = &adapter->pdev->dev; + + switch (adapter->ptp.hw_caps.tx_tstamp_format) { + case VIRTCHNL_1588_PTP_TSTAMP_40BIT: + case VIRTCHNL_1588_PTP_TSTAMP_64BIT_NS: + dev_dbg(dev, "%s: got Tx timestamp format %u\n", + __func__, adapter->ptp.hw_caps.tx_tstamp_format); + break; + default: + dev_warn(dev, "Disabling Tx timestamps due to unexpected Tx timestamp format %u\n", + adapter->ptp.hw_caps.tx_tstamp_format); + adapter->ptp.hw_caps.caps &= ~VIRTCHNL_1588_PTP_CAP_TX_TSTAMP; + break; + } +} + +/** + * iavf_ptp_init - Initialize PTP support if capability was negotiated + * @adapter: private adapter structure + * + * Initialize PTP functionality, based on the capabilities that the PF has + * enabled for this VF. + */ +void iavf_ptp_init(struct iavf_adapter *adapter) +{ + struct device *dev = &adapter->pdev->dev; + int err; + + if (WARN_ON(adapter->ptp.initialized)) { + dev_err(dev, "PTP functionality was already initialized!\n"); + return; + } + + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_READ_PHC)) { + dev_dbg(dev, "Device does not have PTP clock support\n"); + return; + } + + err = iavf_ptp_register_clock(adapter); + if (err) { + dev_warn(dev, "Failed to register PTP clock device\n"); + return; + } + +#ifdef HAVE_PTP_CLOCK_DO_AUX_WORK + ptp_schedule_worker(adapter->ptp.clock, 0); +#endif + + iavf_ptp_map_phc_addr(adapter); + + iavf_validate_tx_tstamp_format(adapter); + + adapter->ptp.initialized = true; +} + +/** + * iavf_ptp_release - Disable PTP support + * @adapter: private adapter structure + * + * Release all PTP resources that were previously initialized. + */ +void iavf_ptp_release(struct iavf_adapter *adapter) +{ + struct iavf_ptp_aq_cmd *cmd, *tmp; + + if (!IS_ERR_OR_NULL(adapter->ptp.clock)) { + dev_info(&adapter->pdev->dev, "removing PTP clock %s\n", adapter->ptp.info.name); + ptp_clock_unregister(adapter->ptp.clock); + adapter->ptp.clock = NULL; + } + + /* Cancel any remaining uncompleted PTP clock commands */ + spin_lock(&adapter->ptp.aq_cmd_lock); + list_for_each_entry_safe(cmd, tmp, &adapter->ptp.aq_cmds, list) { + list_del(&cmd->list); + kfree(cmd); + } + adapter->aq_required &= ~IAVF_FLAG_AQ_SEND_PTP_CMD; + spin_unlock(&adapter->ptp.aq_cmd_lock); + + iavf_ptp_unmap_phc_addr(adapter); + + adapter->ptp.hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; + iavf_ptp_disable_tx_tstamp(adapter); + + adapter->ptp.hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; + iavf_ptp_disable_rx_tstamp(adapter); + + adapter->ptp.initialized = false; +} + +/** + * iavf_ptp_process_caps - Handle change in PTP capabilities + * @adapter: private adapter structure + * + * Handle any state changes necessary due to change in PTP capabilities, such + * as after a device reset or change in configuration from the PF. + */ +void iavf_ptp_process_caps(struct iavf_adapter *adapter) +{ + struct device *dev = &adapter->pdev->dev; + + dev_dbg(dev, "PTP capabilities changed at runtime\n"); + + /* Check if we lost PTP capability after loading */ + if (adapter->ptp.initialized && + !iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_READ_PHC)) { + iavf_ptp_release(adapter); + return; + } + + /* Check if we gained PTP capability after loading */ + if (!adapter->ptp.initialized && + iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_READ_PHC)) { + iavf_ptp_init(adapter); + return; + } + + /* The following checks are only necessary if we still have PTP clock + * capability. These handle if one of the extended capabilities is + * changed. + */ + + if (adapter->ptp.phc_addr && + !(iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_PHC_REGS))) + iavf_ptp_unmap_phc_addr(adapter); + else if (!adapter->ptp.phc_addr && + (iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_PHC_REGS))) + iavf_ptp_map_phc_addr(adapter); + + iavf_validate_tx_tstamp_format(adapter); + + /* Check if the device lost access to Tx timestamp outgoing packets */ + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_TX_TSTAMP)) { + adapter->ptp.hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; + iavf_ptp_disable_tx_tstamp(adapter); + } + + /* Check if the device lost access to Rx timestamp incoming packets */ + if (!iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP)) { + adapter->ptp.hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; + iavf_ptp_disable_rx_tstamp(adapter); + } +} + +/** + * iavf_ptp_extend_32b_timestamp - Convert a 32b nanoseconds timestamp to 64b nanoseconds + * @cached_phc_time: recently cached copy of PHC time + * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value + * + * Hardware captures timestamps which contain only 32 bits of nominal + * nanoseconds, as opposed to the 64bit timestamps that the stack expects. + * + * Extend the 32bit nanosecond timestamp using the following algorithm and + * assumptions: + * + * 1) have a recently cached copy of the PHC time + * 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1 + * seconds) before or after the PHC time was captured. + * 3) calculate the delta between the cached time and the timestamp + * 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was + * captured after the PHC time. In this case, the full timestamp is just + * the cached PHC time plus the delta. + * 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the + * timestamp was captured *before* the PHC time, i.e. because the PHC + * cache was updated after the timestamp was captured by hardware. In this + * case, the full timestamp is the cached time minus the inverse delta. + * + * This algorithm works even if the PHC time was updated after a Tx timestamp + * was requested, but before the Tx timestamp event was reported from + * hardware. + * + * This calculation primarily relies on keeping the cached PHC time up to + * date. If the timestamp was captured more than 2^31 nanoseconds after the + * PHC time, it is possible that the lower 32bits of PHC time have + * overflowed more than once, and we might generate an incorrect timestamp. + * + * This is prevented by (a) periodically updating the cached PHC time once + * a second, and (b) discarding any Tx timestamp packet if it has waited for + * a timestamp for more than one second. + */ +u64 iavf_ptp_extend_32b_timestamp(u64 cached_phc_time, u32 in_tstamp) +{ + const u64 mask = GENMASK_ULL(31, 0); + u32 delta; + u64 ns; + + /* Calculate the delta between the lower 32bits of the cached PHC + * time and the in_tstamp value + */ + delta = (in_tstamp - (u32)(cached_phc_time & mask)); + + /* Do not assume that the in_tstamp is always more recent than the + * cached PHC time. If the delta is large, it indicates that the + * in_tstamp was taken in the past, and should be converted + * forward. + */ + if (delta > (mask / 2)) { + /* reverse the delta calculation here */ + delta = ((u32)(cached_phc_time & mask) - in_tstamp); + ns = cached_phc_time - delta; + } else { + ns = cached_phc_time + delta; + } + + return ns; +} + +/** + * iavf_ptp_extend_40b_timestamp - Convert a 40b timestamp to 64b nanoseconds + * @cached_phc_time: recently cached copy of PHC time + * @in_tstamp: Ingress/egress 40b timestamp value + * + * For some devices, the Tx and Rx timestamps use a 40bit timestamp: + * + * *--------------------------------------------------------------* + * | 32 bits of nanoseconds | 7 high bits of sub ns underflow | v | + * *--------------------------------------------------------------* + * + * The low bit is an indicator of whether the timestamp is valid. The next + * 7 bits are a capture of the upper 7 bits of the sub-nanosecond underflow, + * and the remaining 32 bits are the lower 32 bits of the PHC timer. + * + * It is assumed that the caller verifies the timestamp is valid prior to + * calling this function. + * + * Extract the 32bit nominal nanoseconds and extend them. See + * iavf_ptp_extend_32b_timestamp for a detailed explanation of the extension + * algorithm. + */ +u64 iavf_ptp_extend_40b_timestamp(u64 cached_phc_time, u64 in_tstamp) +{ + const u64 mask = GENMASK_ULL(31, 0); + + return iavf_ptp_extend_32b_timestamp(cached_phc_time, (in_tstamp >> 8) & mask); +} diff --git a/drivers/net/ethernet/intel/iavf/iavf_ptp.h b/drivers/net/ethernet/intel/iavf/iavf_ptp.h new file mode 100644 index 000000000..98a6fb458 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/iavf_ptp.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _IAVF_PTP_H_ +#define _IAVF_PTP_H_ + +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) +#include +#include +#include +#endif /* CONFIG_PTP_1588_CLOCK */ + +struct iavf_adapter; + +/* bit indicating whether a 40bit timestamp is valid */ +#define IAVF_PTP_40B_TSTAMP_VALID BIT(0) + +/* structure used to queue PTP commands for processing */ +struct iavf_ptp_aq_cmd { + struct list_head list; + enum virtchnl_ops v_opcode; + u16 msglen; + u8 msg[]; +}; + +/* fields used for PTP support */ +struct iavf_ptp { + wait_queue_head_t phc_time_waitqueue; + struct virtchnl_ptp_caps hw_caps; + struct list_head aq_cmds; + /* Lock protecting access to the AQ command list */ + spinlock_t aq_cmd_lock; + struct hwtstamp_config hwtstamp_config; + u64 cached_phc_time; + unsigned long cached_phc_updated; + u64 tx_hwtstamp_skipped; + u64 tx_hwtstamp_timeouts; + struct sk_buff *tx_skb; /* protected by __IAVF_TX_TSTAMP_IN_PROGRESS */ + unsigned long tx_start; /* protected by __IAVF_TX_TSTAMP_IN_PROGRESS */ + u8 __iomem *phc_addr; /* PHC register mapping */ + bool initialized; + bool phc_time_ready; +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) + struct ptp_clock_info info; + struct ptp_clock *clock; +#endif +}; + +void iavf_virtchnl_send_ptp_cmd(struct iavf_adapter *adapter); + +#if IS_ENABLED(CONFIG_PTP_1588_CLOCK) +void iavf_ptp_init(struct iavf_adapter *adapter); +void iavf_ptp_release(struct iavf_adapter *adapter); +void iavf_ptp_process_caps(struct iavf_adapter *adapter); +long iavf_ptp_do_aux_work(struct ptp_clock_info *ptp); +bool iavf_ptp_cap_supported(struct iavf_adapter *adapter, u32 cap); +u64 iavf_ptp_extend_32b_timestamp(u64 cached_phc_time, u32 in_tstamp); +u64 iavf_ptp_extend_40b_timestamp(u64 cached_phc_time, u64 in_tstamp); +int iavf_ptp_get_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr); +int iavf_ptp_set_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr); +#else +static inline void iavf_ptp_init(struct iavf_adapter *adapter) {} +static inline void iavf_ptp_release(struct iavf_adapter *adapter) {} +static inline void iavf_ptp_process_caps(struct iavf_adapter *adapter) {} +static inline bool iavf_ptp_cap_supported(struct iavf_adapter *adapter, u32 cap) { return false; } +static inline u64 iavf_ptp_extend_32b_timestamp(u64 cached_phc_time, u32 in_tstamp) { return 0; } +static inline u64 iavf_ptp_extend_40b_timestamp(u64 cached_phc_time, u64 in_tstamp) { return 0; } + +static inline int iavf_ptp_get_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr) +{ + return -EOPNOTSUPP; +} + +static inline int iavf_ptp_set_ts_config(struct iavf_adapter *adapter, struct ifreq *ifr) +{ + return -EOPNOTSUPP; +} +#endif + +#endif /* _IAVF_PTP_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_trace.h b/drivers/net/ethernet/intel/iavf/iavf_trace.h index 816c5266e..b35626408 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_trace.h +++ b/drivers/net/ethernet/intel/iavf/iavf_trace.h @@ -37,7 +37,7 @@ #include -/** +/* * iavf_trace() macro enables shared code to refer to trace points * like: * diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index 026d17344..2a2195640 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -6,8 +6,7 @@ #include "iavf_trace.h" #include "iavf_prototype.h" -static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, - u32 td_tag) +static __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, u32 td_tag) { return cpu_to_le64(IAVF_TX_DESC_DTYPE_DATA | ((u64)td_cmd << IAVF_TXD_QW1_CMD_SHIFT) | @@ -123,6 +122,67 @@ u32 iavf_get_tx_pending(struct iavf_ring *ring, bool in_sw) return 0; } +/** + * iavf_chnl_detect_recover - logic to revive ADQ enabled vectors + * @vsi: ptr to VSI + * + * This function implements "jiffy" based logic to revive ADQ enabled + * vectors by triggering software interrupt. It is invoked from + * "service_task" which typically runs once every second. + **/ +void iavf_chnl_detect_recover(struct iavf_vsi *vsi) +{ + struct iavf_ring *tx_ring = NULL; + struct net_device *netdev; + unsigned long end; + unsigned int i; + + if (!vsi) + return; + + if (test_bit(__IAVF_VSI_DOWN, vsi->state)) + return; + + netdev = vsi->netdev; + if (!netdev) + return; + + if (!netif_carrier_ok(netdev)) + return; + + for (i = 0; i < vsi->back->num_active_queues; i++) { + u8 qv_state_flags; + + tx_ring = &vsi->back->tx_rings[i]; + if (!(tx_ring && tx_ring->desc)) + continue; + if (!tx_ring->q_vector) + continue; + if (!vector_ch_ena(tx_ring->q_vector) || + !vector_ch_perf_ena(tx_ring->q_vector)) + continue; + + end = tx_ring->q_vector->jiffies; + if (!end) + continue; + + qv_state_flags = tx_ring->q_vector->state_flags; + + /* trigger software interrupt (to revive queue processing) if + * vector is channel enabled and only if current jiffies is at + * least 1 sec (worth of jiffies, hence multiplying by HZ) more + * than old_jiffies + */ +#define IAVF_CH_JIFFY_DELTA_IN_SEC (1 * HZ) + end += IAVF_CH_JIFFY_DELTA_IN_SEC; + if (time_is_before_jiffies(end) && + (qv_state_flags & IAVF_VECTOR_STATE_ONCE_IN_BP)) { + iavf_inc_serv_task_sw_intr_counter(tx_ring->q_vector); + iavf_force_wb(vsi, tx_ring->q_vector); + } + } +} + /** * iavf_detect_recover_hung - Function to detect and recover hung_queues * @vsi: pointer to vsi struct with tx queues @@ -132,7 +192,6 @@ u32 iavf_get_tx_pending(struct iavf_ring *ring, bool in_sw) **/ void iavf_detect_recover_hung(struct iavf_vsi *vsi) { - struct iavf_ring *tx_ring = NULL; struct net_device *netdev; unsigned int i; int packets; @@ -151,8 +210,13 @@ void iavf_detect_recover_hung(struct iavf_vsi *vsi) return; for (i = 0; i < vsi->back->num_active_queues; i++) { - tx_ring = &vsi->back->tx_rings[i]; - if (tx_ring && tx_ring->desc) { + struct iavf_ring *tx_ring = &vsi->back->tx_rings[i]; + + if (!tx_ring || !tx_ring->q_vector) + continue; + if (vector_ch_ena(tx_ring->q_vector)) + continue; + if (tx_ring->desc) { /* If packet counter has not changed the queue is * likely stalled, so force an interrupt for this * queue. @@ -176,6 +240,23 @@ void iavf_detect_recover_hung(struct iavf_vsi *vsi) } } +static void iavf_chnl_queue_stats(struct iavf_ring *ring, u64 pkts) +{ + u64_stats_update_begin(&ring->syncp); + /* separate accounting of packets (either from busy_poll or + * napi_poll depending upon state of vector specific + * flag 'in_bp', 'prev_in_bp' + */ + if (ring->q_vector->state_flags & IAVF_VECTOR_STATE_IN_BP) { + ring->ch_q_stats.poll.pkt_busy_poll += pkts; + } else { + if (ring->q_vector->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) + ring->ch_q_stats.poll.pkt_busy_poll += pkts; + else + ring->ch_q_stats.poll.pkt_not_busy_poll += pkts; + } + u64_stats_update_end(&ring->syncp); +} #define WB_STRIDE 4 /** @@ -207,7 +288,7 @@ static bool iavf_clean_tx_irq(struct iavf_vsi *vsi, break; /* prevent any other reads prior to eop_desc */ - read_barrier_depends(); + smp_rmb(); iavf_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); /* if the descriptor isn't done, no work yet to do */ @@ -283,6 +364,7 @@ static bool iavf_clean_tx_irq(struct iavf_vsi *vsi, u64_stats_update_end(&tx_ring->syncp); tx_ring->q_vector->tx.total_bytes += total_bytes; tx_ring->q_vector->tx.total_packets += total_packets; + iavf_chnl_queue_stats(tx_ring, total_packets); if (tx_ring->flags & IAVF_TXR_FLAGS_WB_ON_ITR) { /* check to see if there are < 4 descriptors @@ -351,54 +433,65 @@ static void iavf_enable_wb_on_itr(struct iavf_vsi *vsi, q_vector->arm_wb_state = true; } -/** - * iavf_force_wb - Issue SW Interrupt so HW does a wb - * @vsi: the VSI we care about - * @q_vector: the vector on which to force writeback - * - **/ -void iavf_force_wb(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector) -{ - u32 val = IAVF_VFINT_DYN_CTLN1_INTENA_MASK | - IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */ - IAVF_VFINT_DYN_CTLN1_SWINT_TRIG_MASK | - IAVF_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK - /* allow 00 to be written to the index */; - - wr32(&vsi->back->hw, - IAVF_VFINT_DYN_CTLN1(q_vector->reg_idx), - val); -} - -static inline bool iavf_container_is_rx(struct iavf_q_vector *q_vector, - struct iavf_ring_container *rc) +static bool iavf_container_is_rx(struct iavf_q_vector *q_vector, struct iavf_ring_container *rc) { return &q_vector->rx == rc; } -static inline unsigned int iavf_itr_divisor(struct iavf_q_vector *q_vector) -{ - unsigned int divisor; +#define IAVF_AIM_MULTIPLIER_100G 2560 +#define IAVF_AIM_MULTIPLIER_50G 1280 +#define IAVF_AIM_MULTIPLIER_40G 1024 +#define IAVF_AIM_MULTIPLIER_20G 512 +#define IAVF_AIM_MULTIPLIER_10G 256 +#define IAVF_AIM_MULTIPLIER_1G 32 - switch (q_vector->adapter->link_speed) { +static unsigned int iavf_mbps_itr_multiplier(u32 speed_mbps) +{ + switch (speed_mbps) { + case SPEED_100000: + return IAVF_AIM_MULTIPLIER_100G; + case SPEED_50000: + return IAVF_AIM_MULTIPLIER_50G; + case SPEED_40000: + return IAVF_AIM_MULTIPLIER_40G; + case SPEED_25000: + case SPEED_20000: + return IAVF_AIM_MULTIPLIER_20G; + case SPEED_10000: + default: + return IAVF_AIM_MULTIPLIER_10G; + case SPEED_1000: + case SPEED_100: + return IAVF_AIM_MULTIPLIER_1G; + } +} + +static unsigned int +iavf_virtchnl_itr_multiplier(enum virtchnl_link_speed speed_virtchnl) +{ + switch (speed_virtchnl) { case VIRTCHNL_LINK_SPEED_40GB: - divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 1024; - break; + return IAVF_AIM_MULTIPLIER_40G; case VIRTCHNL_LINK_SPEED_25GB: case VIRTCHNL_LINK_SPEED_20GB: - divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 512; - break; - default: + return IAVF_AIM_MULTIPLIER_20G; case VIRTCHNL_LINK_SPEED_10GB: - divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 256; - break; + default: + return IAVF_AIM_MULTIPLIER_10G; case VIRTCHNL_LINK_SPEED_1GB: case VIRTCHNL_LINK_SPEED_100MB: - divisor = IAVF_ITR_ADAPTIVE_MIN_INC * 32; - break; + return IAVF_AIM_MULTIPLIER_1G; } +} - return divisor; +static unsigned int iavf_itr_divisor(struct iavf_adapter *adapter) +{ + if (ADV_LINK_SUPPORT(adapter)) + return IAVF_ITR_ADAPTIVE_MIN_INC * + iavf_mbps_itr_multiplier(adapter->link_speed_mbps); + else + return IAVF_ITR_ADAPTIVE_MIN_INC * + iavf_virtchnl_itr_multiplier(adapter->link_speed); } /** @@ -588,8 +681,9 @@ adjust_by_size: * Use addition as we have already recorded the new latency flag * for the ITR value. */ - itr += DIV_ROUND_UP(avg_wire_size, iavf_itr_divisor(q_vector)) * - IAVF_ITR_ADAPTIVE_MIN_INC; + itr += DIV_ROUND_UP(avg_wire_size, + iavf_itr_divisor(q_vector->adapter)) * + IAVF_ITR_ADAPTIVE_MIN_INC; if ((itr & IAVF_ITR_MASK) > IAVF_ITR_ADAPTIVE_MAX_USECS) { itr &= IAVF_ITR_ADAPTIVE_LATENCY; @@ -803,7 +897,7 @@ err: * @rx_ring: ring to bump * @val: new head index **/ -static inline void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val) +static void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val) { rx_ring->next_to_use = val; @@ -825,7 +919,7 @@ static inline void iavf_release_rx_desc(struct iavf_ring *rx_ring, u32 val) * * Returns the offset value for ring into the data buffer. */ -static inline unsigned int iavf_rx_offset(struct iavf_ring *rx_ring) +static unsigned int iavf_rx_offset(struct iavf_ring *rx_ring) { return ring_uses_build_skb(rx_ring) ? IAVF_SKB_PAD : 0; } @@ -903,28 +997,25 @@ static void iavf_receive_skb(struct iavf_ring *rx_ring, else vlan_gro_receive(&q_vector->napi, vsi->vlgrp, vlan_tag, skb); - } else { - napi_gro_receive(&q_vector->napi, skb); } #else /* HAVE_VLAN_RX_REGISTER */ -#ifdef NETIF_F_HW_VLAN_CTAG_RX - if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && - (vlan_tag & VLAN_VID_MASK)) -#else - if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_RX) && - (vlan_tag & VLAN_VID_MASK)) -#endif /* NETIF_F_HW_VLAN_CTAG_RX */ - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); + if (vlan_tag & VLAN_VID_MASK) { + if (rx_ring->netdev->features & IAVF_NETIF_F_HW_VLAN_CTAG_RX) { + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + vlan_tag); #ifdef IAVF_ADD_PROBES -#ifdef NETIF_F_HW_VLAN_CTAG_RX - if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && - (vlan_tag & VLAN_VID_MASK)) -#else - if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_RX) && - (vlan_tag & VLAN_VID_MASK)) -#endif /* NETIF_F_HW_VLAN_CTAG_RX */ - rx_ring->vsi->back->rx_vlano++; + rx_ring->vsi->back->rx_vlano++; #endif /* IAVF_ADD_PROBES */ +#ifdef NETIF_F_HW_VLAN_STAG_RX + } else if (rx_ring->netdev->features & NETIF_F_HW_VLAN_STAG_RX) { + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD), + vlan_tag); +#ifdef IAVF_ADD_PROBES + rx_ring->vsi->back->rx_ad_vlano++; +#endif /* IAVF_ADD_PROBES */ +#endif /* NETIF_F_HW_VLAN_STAG_RX */ + } + } napi_gro_receive(&q_vector->napi, skb); #endif /* HAVE_VLAN_RX_REGISTER */ @@ -995,37 +1086,52 @@ no_buffers: return true; } +/* + * iavf_rx_csum_decoded + * + * Checksum offload bits decoded from the receive descriptor. + */ +struct iavf_rx_csum_decoded { + u8 l3l4p : 1; + u8 ipe : 1; + u8 eipe : 1; + u8 eudpe : 1; + u8 ipv6exadd : 1; + u8 l4e : 1; + u8 pprs : 1; + u8 nat : 1; +}; + #ifdef IAVF_ADD_PROBES -static void iavf_rx_extra_counters(struct iavf_vsi *vsi, u32 rx_error, - const struct iavf_rx_ptype_decoded decoded) +static void iavf_rx_extra_counters(struct iavf_vsi *vsi, + struct iavf_rx_csum_decoded *csum_bits, + struct iavf_rx_ptype_decoded *decoded) { bool ipv4; - ipv4 = (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP) && - (decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4); + ipv4 = (decoded->outer_ip == IAVF_RX_PTYPE_OUTER_IP) && + (decoded->outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4); - if (ipv4 && - (rx_error & (BIT(IAVF_RX_DESC_ERROR_IPE_SHIFT) | - BIT(IAVF_RX_DESC_ERROR_EIPE_SHIFT)))) + if (ipv4 && (csum_bits->ipe | csum_bits->eipe)) vsi->back->rx_ip4_cso_err++; - if (rx_error & BIT(IAVF_RX_DESC_ERROR_L4E_SHIFT)) { - if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_TCP) + if (csum_bits->l4e) { + if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_TCP) vsi->back->rx_tcp_cso_err++; - else if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_UDP) + else if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_UDP) vsi->back->rx_udp_cso_err++; - else if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_SCTP) + else if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_SCTP) vsi->back->rx_sctp_cso_err++; } - if (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP && - decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4) + if (decoded->outer_ip == IAVF_RX_PTYPE_OUTER_IP && + decoded->outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4) vsi->back->rx_ip4_cso++; - if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_TCP) + if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_TCP) vsi->back->rx_tcp_cso++; - else if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_UDP) + else if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_UDP) vsi->back->rx_udp_cso++; - else if (decoded.inner_prot == IAVF_RX_PTYPE_INNER_PROT_SCTP) + else if (decoded->inner_prot == IAVF_RX_PTYPE_INNER_PROT_SCTP) vsi->back->rx_sctp_cso++; } @@ -1033,32 +1139,22 @@ static void iavf_rx_extra_counters(struct iavf_vsi *vsi, u32 rx_error, #if defined(HAVE_VXLAN_RX_OFFLOAD) || defined(HAVE_GENEVE_RX_OFFLOAD) || defined(HAVE_UDP_ENC_RX_OFFLOAD) #define IAVF_TUNNEL_SUPPORT #endif + /** - * iavf_rx_checksum - Indicate in skb if hw indicated a good cksum + * iavf_rx_csum - Indicate in skb if hw indicated a good cksum * @vsi: the VSI we care about * @skb: skb currently being received and modified - * @rx_desc: the receive descriptor + * @ptype: decoded ptype information + * @csum_bits: decoded Rx descriptor information **/ -static inline void iavf_rx_checksum(struct iavf_vsi *vsi, - struct sk_buff *skb, - union iavf_rx_desc *rx_desc) +static void +iavf_rx_csum(struct iavf_vsi *vsi, struct sk_buff *skb, + struct iavf_rx_ptype_decoded *ptype, + struct iavf_rx_csum_decoded *csum_bits) { - struct iavf_rx_ptype_decoded decoded; - u32 rx_error, rx_status; bool ipv4, ipv6; - u8 ptype; - u64 qword; - - qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); - ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT; - rx_error = (qword & IAVF_RXD_QW1_ERROR_MASK) >> - IAVF_RXD_QW1_ERROR_SHIFT; - rx_status = (qword & IAVF_RXD_QW1_STATUS_MASK) >> - IAVF_RXD_QW1_STATUS_SHIFT; - decoded = decode_rx_desc_ptype(ptype); skb->ip_summed = CHECKSUM_NONE; - skb_checksum_none_assert(skb); /* Rx csum enabled and ip headers found? */ @@ -1071,49 +1167,59 @@ static inline void iavf_rx_checksum(struct iavf_vsi *vsi, #endif /* did the hardware decode the packet and checksum? */ - if (!(rx_status & BIT(IAVF_RX_DESC_STATUS_L3L4P_SHIFT))) + if (!csum_bits->l3l4p) return; /* both known and outer_ip must be set for the below code to work */ - if (!(decoded.known && decoded.outer_ip)) + if (!(ptype->known && ptype->outer_ip)) return; #ifdef IAVF_ADD_PROBES vsi->back->hw_csum_rx_outer++; #endif - ipv4 = (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP) && - (decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4); - ipv6 = (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP) && - (decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV6); + ipv4 = (ptype->outer_ip == IAVF_RX_PTYPE_OUTER_IP) && + (ptype->outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV4); + ipv6 = (ptype->outer_ip == IAVF_RX_PTYPE_OUTER_IP) && + (ptype->outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV6); #ifdef IAVF_ADD_PROBES - iavf_rx_extra_counters(vsi, rx_error, decoded); - + iavf_rx_extra_counters(vsi, csum_bits, ptype); #endif /* IAVF_ADD_PROBES */ - if (ipv4 && - (rx_error & (BIT(IAVF_RX_DESC_ERROR_IPE_SHIFT) | - BIT(IAVF_RX_DESC_ERROR_EIPE_SHIFT)))) + if (ipv4 && (csum_bits->ipe || csum_bits->eipe)) goto checksum_fail; /* likely incorrect csum if alternate IP extension headers found */ - if (ipv6 && - rx_status & BIT(IAVF_RX_DESC_STATUS_IPV6EXADD_SHIFT)) + if (ipv6 && csum_bits->ipv6exadd) /* don't increment checksum err here, non-fatal err */ return; /* there was some L4 error, count error and punt packet to the stack */ - if (rx_error & BIT(IAVF_RX_DESC_ERROR_L4E_SHIFT)) + if (csum_bits->l4e) + goto checksum_fail; + + if (csum_bits->nat && csum_bits->eudpe) goto checksum_fail; /* handle packets that were not able to be checksummed due * to arrival speed, in this case the stack can compute * the csum. */ - if (rx_error & BIT(IAVF_RX_DESC_ERROR_PPRS_SHIFT)) + if (csum_bits->pprs) return; + /* If there is an outer header present that might contain a checksum + * we need to bump the checksum level by 1 to reflect the fact that + * we are indicating we validated the inner checksum. + */ + if (ptype->tunnel_type >= IAVF_RX_PTYPE_TUNNEL_IP_GRENAT) +#ifdef HAVE_SKBUFF_CSUM_LEVEL + skb->csum_level = 1; +#else + skb->encapsulation = 1; +#endif + /* Only report checksum unnecessary for TCP, UDP, or SCTP */ - switch (decoded.inner_prot) { + switch (ptype->inner_prot) { case IAVF_RX_PTYPE_INNER_PROT_TCP: case IAVF_RX_PTYPE_INNER_PROT_UDP: case IAVF_RX_PTYPE_INNER_PROT_SCTP: @@ -1128,13 +1234,85 @@ checksum_fail: vsi->back->hw_csum_rx_error++; } +/** + * iavf_legacy_rx_csum - Indicate in skb if hw indicated a good cksum + * @vsi: the VSI we care about + * @skb: skb currently being received and modified + * @rx_desc: the receive descriptor + * + * This function only operates on the VIRTCHNL_RXDID_1_32B_BASE legacy 32byte + * descriptor writeback format. + **/ +static void +iavf_legacy_rx_csum(struct iavf_vsi *vsi, struct sk_buff *skb, union iavf_rx_desc *rx_desc) +{ + struct iavf_rx_csum_decoded csum_bits; + struct iavf_rx_ptype_decoded decoded; + u32 rx_error, rx_status; + u64 qword; + u16 ptype; + + qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); + ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT; + rx_error = (qword & IAVF_RXD_QW1_ERROR_MASK) >> + IAVF_RXD_QW1_ERROR_SHIFT; + rx_status = (qword & IAVF_RXD_QW1_STATUS_MASK) >> + IAVF_RXD_QW1_STATUS_SHIFT; + decoded = decode_rx_desc_ptype(ptype); + + csum_bits.ipe = rx_error & BIT(IAVF_RX_DESC_ERROR_IPE_SHIFT); + csum_bits.eipe = rx_error & BIT(IAVF_RX_DESC_ERROR_EIPE_SHIFT); + csum_bits.l4e = rx_error & BIT(IAVF_RX_DESC_ERROR_L4E_SHIFT); + csum_bits.pprs = rx_error & BIT(IAVF_RX_DESC_ERROR_PPRS_SHIFT); + csum_bits.l3l4p = rx_status & BIT(IAVF_RX_DESC_STATUS_L3L4P_SHIFT); + csum_bits.ipv6exadd = rx_status & BIT(IAVF_RX_DESC_STATUS_IPV6EXADD_SHIFT); + csum_bits.nat = 0; + csum_bits.eudpe = 0; + + iavf_rx_csum(vsi, skb, &decoded, &csum_bits); +} + +/** + * iavf_flex_rx_csum - Indicate in skb if hw indicated a good cksum + * @vsi: the VSI we care about + * @skb: skb currently being received and modified + * @rx_desc: the receive descriptor + * + * This function only operates on the VIRTCHNL_RXDID_2_FLEX_SQ_NIC flexible + * descriptor writeback format. + **/ +static void +iavf_flex_rx_csum(struct iavf_vsi *vsi, struct sk_buff *skb, union iavf_rx_desc *rx_desc) +{ + struct iavf_rx_csum_decoded csum_bits; + struct iavf_rx_ptype_decoded decoded; + u16 rx_status0, rx_status1, ptype; + + rx_status0 = le16_to_cpu(rx_desc->flex_wb.status_error0); + rx_status1 = le16_to_cpu(rx_desc->flex_wb.status_error1); + ptype = le16_to_cpu(rx_desc->flex_wb.ptype_flexi_flags0) & + IAVF_RX_FLEX_DESC_PTYPE_M; + decoded = decode_rx_desc_ptype(ptype); + + csum_bits.ipe = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S); + csum_bits.eipe = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S); + csum_bits.l4e = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S); + csum_bits.eudpe = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S); + csum_bits.pprs = 0; + csum_bits.l3l4p = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S); + csum_bits.ipv6exadd = rx_status0 & BIT(IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S); + csum_bits.nat = rx_status1 & BIT(IAVF_RX_FLEX_DESC_STATUS1_NAT_S); + + iavf_rx_csum(vsi, skb, &decoded, &csum_bits); +} + /** * iavf_ptype_to_htype - get a hash type * @ptype: the ptype value from the descriptor * * Returns a hash type to be used by skb_set_hash **/ -static inline enum pkt_hash_types iavf_ptype_to_htype(u8 ptype) +static enum pkt_hash_types iavf_ptype_to_htype(u16 ptype) { struct iavf_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); @@ -1152,16 +1330,18 @@ static inline enum pkt_hash_types iavf_ptype_to_htype(u8 ptype) } /** - * iavf_rx_hash - set the hash value in the skb + * iavf_legacy_rx_hash - set the hash value in the skb * @ring: descriptor ring * @rx_desc: specific descriptor * @skb: skb currently being received and modified * @rx_ptype: Rx packet type + * + * This function only operates on the VIRTCHNL_RXDID_1_32B_BASE legacy 32byte + * descriptor writeback format. **/ -static inline void iavf_rx_hash(struct iavf_ring *ring, - union iavf_rx_desc *rx_desc, - struct sk_buff *skb, - u8 rx_ptype) +static void +iavf_legacy_rx_hash(struct iavf_ring *ring, union iavf_rx_desc *rx_desc, struct sk_buff *skb, + u16 rx_ptype) { #ifdef NETIF_F_RXHASH u32 hash; @@ -1179,6 +1359,75 @@ static inline void iavf_rx_hash(struct iavf_ring *ring, #endif /* NETIF_F_RXHASH */ } +/** + * iavf_flex_rx_hash - set the hash value in the skb + * @ring: descriptor ring + * @rx_desc: specific descriptor + * @skb: skb currently being received and modified + * @rx_ptype: Rx packet type + * + * This function only operates on the VIRTCHNL_RXDID_2_FLEX_SQ_NIC flexible + * descriptor writeback format. + **/ +static void +iavf_flex_rx_hash(struct iavf_ring *ring, union iavf_rx_desc *rx_desc, struct sk_buff *skb, + u16 rx_ptype) +{ +#ifdef NETIF_F_RXHASH + __le16 status0; + + if (!(ring->netdev->features & NETIF_F_RXHASH)) + return; + + status0 = rx_desc->flex_wb.status_error0; + if (status0 & cpu_to_le16(BIT(IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S))) { + u32 hash = le32_to_cpu(rx_desc->flex_wb.rss_hash); + + skb_set_hash(skb, hash, iavf_ptype_to_htype(rx_ptype)); + } +#endif /* NETIF_F_RXHASH */ +} + +/** + * iavf_flex_rx_tstamp - Capture Rx timestamp from the descriptor + * @rx_ring: descriptor ring + * @rx_desc: specific descriptor + * @skb: skb currently being received + * + * Read the Rx timestamp value from the descriptor and pass it to the stack. + * + * This function only operates on the VIRTCHNL_RXDID_2_FLEX_SQ_NIC flexible + * descriptor writeback format. + */ +static void +iavf_flex_rx_tstamp(struct iavf_ring *rx_ring, union iavf_rx_desc *rx_desc, struct sk_buff *skb) +{ + struct skb_shared_hwtstamps *skb_tstamps; + struct iavf_adapter *adapter; + u32 tstamp; + u64 ns; + + /* Skip processing if timestamps aren't enabled */ + if (!(rx_ring->flags & IAVF_TXRX_FLAGS_HW_TSTAMP)) + return; + + /* Check if this Rx descriptor has a valid timestamp */ + if (!(rx_desc->flex_wb.ts_low & IAVF_PTP_40B_TSTAMP_VALID)) + return; + + adapter = netdev_priv(rx_ring->netdev); + + /* the ts_low field only contains the valid bit and sub-nanosecond + * precision, so we don't need to extract it. + */ + tstamp = le32_to_cpu(rx_desc->flex_wb.flex_ts.ts_high); + ns = iavf_ptp_extend_32b_timestamp(adapter->ptp.cached_phc_time, tstamp); + + skb_tstamps = skb_hwtstamps(skb); + memset(skb_tstamps, 0, sizeof(*skb_tstamps)); + skb_tstamps->hwtstamp = ns_to_ktime(ns); +} + /** * iavf_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: rx descriptor ring packet is being transacted on @@ -1190,14 +1439,21 @@ static inline void iavf_rx_hash(struct iavf_ring *ring, * order to populate the hash, checksum, VLAN, protocol, and * other fields within the skb. **/ -static inline -void iavf_process_skb_fields(struct iavf_ring *rx_ring, - union iavf_rx_desc *rx_desc, struct sk_buff *skb, - u8 rx_ptype) +static void +iavf_process_skb_fields(struct iavf_ring *rx_ring, union iavf_rx_desc *rx_desc, + struct sk_buff *skb, u16 rx_ptype) { - iavf_rx_hash(rx_ring, rx_desc, skb, rx_ptype); + if (rx_ring->rxdid == VIRTCHNL_RXDID_1_32B_BASE) { + iavf_legacy_rx_hash(rx_ring, rx_desc, skb, rx_ptype); - iavf_rx_checksum(rx_ring->vsi, skb, rx_desc); + iavf_legacy_rx_csum(rx_ring->vsi, skb, rx_desc); + } else { + iavf_flex_rx_hash(rx_ring, rx_desc, skb, rx_ptype); + + iavf_flex_rx_csum(rx_ring->vsi, skb, rx_desc); + + iavf_flex_rx_tstamp(rx_ring, rx_desc, skb); + } skb_record_rx_queue(skb, rx_ring->queue_index); @@ -1251,7 +1507,7 @@ static bool iavf_cleanup_headers(struct iavf_ring *rx_ring, struct sk_buff *skb, * A page is not reusable if it was allocated under low memory * conditions, or it's not in the same NUMA node as this CPU. */ -static inline bool iavf_page_is_reusable(struct page *page) +static bool iavf_page_is_reusable(struct page *page) { return (page_to_nid(page) == numa_mem_id()) && !page_is_pfmemalloc(page); @@ -1620,7 +1876,7 @@ static void iavf_rx_buffer_flip(struct iavf_ring *rx_ring, #endif } -static inline void iavf_xdp_ring_update_tail(struct iavf_ring *xdp_ring) +static void iavf_xdp_ring_update_tail(struct iavf_ring *xdp_ring) { /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. @@ -1629,6 +1885,204 @@ static inline void iavf_xdp_ring_update_tail(struct iavf_ring *xdp_ring) writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); } +/** + * iavf_is_ctrl_pkt - check if packet is a TCP control packet or data packet + * @skb: receive buffer + * @rx_ring: ptr to Rx ring + * + * Returns true for all unsupported protocol/configuration. Supported protocol + * is TCP/IPv4[6]. For TCP/IPv6, this function returns true if packet contains + * nested header. + * Logic to determine control packet: + * - packets is control packet if it contains flags like SYN, SYN+ACK, FIN, RST + * + * Returns true if packet is classified as control packet and for all unhandled + * condition otherwise false if packet is classified as data packet + */ +static bool iavf_is_ctrl_pkt(struct sk_buff *skb, struct iavf_ring *rx_ring) + +{ + union { + unsigned char *network; + struct ipv6hdr *ipv6; + struct iphdr *ipv4; + } hdr; + struct tcphdr *th; + + /* at this point, skb->data points to network header since + * ethernet_header was pulled inline due to eth_type_trans + */ + hdr.network = skb->data; + + /* only support IPv4/IPv6, all other protocol being treated like + * control packets + */ + if (skb->protocol == htons(ETH_P_IP)) { + unsigned int hlen; + + /* access ihl as u8 to avoid unaligned access on ia64 */ + hlen = (hdr.network[0] & 0x0F) << 2; + + /* for now, assume all non TCP packets are ctrl packets, so that + * they don't get counted to evaluate the likelihood of being + * called back for polling + */ + if (hdr.ipv4->protocol != IPPROTO_TCP) + return true; + + th = (struct tcphdr *)(hdr.network + hlen); + } else if (skb->protocol == htons(ETH_P_IPV6)) { + /* for now, if next_hdr is not TCP, means it contains nested + * header. IPv6 packets which contains nested header: + * treat them like control packet, so that interrupts gets + * enabled normally, otherwise driver need to duplicate the + * code to parse nested IPv6 header. + */ + if (hdr.ipv6->nexthdr != IPPROTO_TCP) + return true; + + th = (struct tcphdr *)(hdr.network + sizeof(struct ipv6hdr)); + } else { + return true; /* if any other than IPv4[6], ctrl packet */ + } + + /* definition of control packet is, if packet is TCP/IPv4[6] and + * TCP flags are either SYN | FIN | RST. + * If neither of those flags (SYN|FIN|RST) are set, then it is + * data packet + */ + if (!th->fin && !th->rst && !th->syn) + return false; /* data packet */ + + u64_stats_update_begin(&rx_ring->syncp); + rx_ring->ch_q_stats.rx.tcp_ctrl_pkts++; + if (th->fin) + rx_ring->ch_q_stats.rx.tcp_fin_recv++; + else if (th->rst) + rx_ring->ch_q_stats.rx.tcp_rst_recv++; + else if (th->syn) + rx_ring->ch_q_stats.rx.tcp_syn_recv++; + u64_stats_update_end(&rx_ring->syncp); + + /* at this point based on L4 header (if it is TCP/IPv4[6]:flags, + * packet is detected as control packets + */ + return true; +} + +static void iavf_chnl_rx_stats(struct iavf_ring *rx_ring, u64 pkts) +{ + iavf_chnl_queue_stats(rx_ring, pkts); + + /* if vector is transitioning from BP->INT (due to busy_poll_stop()) and + * we find no packets, in that case: to avoid entering into INTR mode + * (which happens from napi_poll - enabling interrupt if + * unlikely_comeback_to_bp getting set), make "prev_data_pkt_recv" to be + * non-zero, so that interrupts won't be enabled. This is to address the + * issue where num_force_wb on some queues is 2 to 3 times higher than + * other queues and those queues also sees lot of interrupts + */ + if (vector_ch_ena(rx_ring->q_vector) && + vector_ch_perf_ena(rx_ring->q_vector) && + vector_busypoll_intr(rx_ring->q_vector)) { + if (!pkts) + rx_ring->q_vector->state_flags |= + IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV; + } else if (vector_ch_ena(rx_ring->q_vector)) { + struct iavf_q_vector *q_vector = rx_ring->q_vector; + u8 qv_flags = q_vector->state_flags; + + u64_stats_update_begin(&rx_ring->syncp); + if (pkts && + !(qv_flags & IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV)) + rx_ring->ch_q_stats.rx.only_ctrl_pkts++; + if (qv_flags & IAVF_VECTOR_STATE_IN_BP && + !(qv_flags & IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV)) + rx_ring->ch_q_stats.rx.bp_no_data_pkt++; + u64_stats_update_end(&rx_ring->syncp); + } +} + +struct iavf_rx_extracted { + unsigned int size; + u16 vlan_tag; + u16 rx_ptype; +}; + +/** + * iavf_extract_legacy_rx_fields - Extract fields from the Rx descriptor + * @rx_ring: rx descriptor ring + * @rx_desc: the descriptor to process + * @fields: storage for extracted values + * + * Decode the Rx descriptor and extract relevant information including the + * size, VLAN tag, and Rx packet type. + * + * This function only operates on the VIRTCHNL_RXDID_1_32B_BASE legacy 32byte + * descriptor writeback format. + */ +static void +iavf_extract_legacy_rx_fields(struct iavf_ring *rx_ring, union iavf_rx_desc *rx_desc, + struct iavf_rx_extracted *fields) +{ + u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); + + fields->size = (qword & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> IAVF_RXD_QW1_LENGTH_PBUF_SHIFT; + fields->rx_ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT; + + if (qword & BIT(IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT) && + rx_ring->flags & IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1) + fields->vlan_tag = le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1); + + if (rx_desc->wb.qword2.ext_status & + cpu_to_le16(BIT(IAVF_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) && + rx_ring->flags & IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2) + fields->vlan_tag = le16_to_cpu(rx_desc->wb.qword2.l2tag2_2); +} + +/** + * iavf_extract_flex_rx_fields - Extract fields from the Rx descriptor + * @rx_ring: rx descriptor ring + * @rx_desc: the descriptor to process + * @fields: storage for extracted values + * + * Decode the Rx descriptor and extract relevant information including the + * size, VLAN tag, and Rx packet type. + * + * This function only operates on the VIRTCHNL_RXDID_2_FLEX_SQ_NIC flexible + * descriptor writeback format. + */ +static void +iavf_extract_flex_rx_fields(struct iavf_ring *rx_ring, union iavf_rx_desc *rx_desc, + struct iavf_rx_extracted *fields) +{ + __le16 status0, status1; + + fields->size = le16_to_cpu(rx_desc->flex_wb.pkt_len) & IAVF_RX_FLEX_DESC_PKT_LEN_M; + fields->rx_ptype = le16_to_cpu(rx_desc->flex_wb.ptype_flexi_flags0) & + IAVF_RX_FLEX_DESC_PTYPE_M; + + status0 = rx_desc->flex_wb.status_error0; + if (status0 & cpu_to_le16(BIT(IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S)) && + rx_ring->flags & IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1) + fields->vlan_tag = le16_to_cpu(rx_desc->flex_wb.l2tag1); + + status1 = rx_desc->flex_wb.status_error1; + if (status1 & cpu_to_le16(BIT(IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S)) && + rx_ring->flags & IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2) + fields->vlan_tag = le16_to_cpu(rx_desc->flex_wb.l2tag2_2nd); +} + +static void +iavf_extract_rx_fields(struct iavf_ring *rx_ring, union iavf_rx_desc *rx_desc, + struct iavf_rx_extracted *fields) +{ + if (rx_ring->rxdid == VIRTCHNL_RXDID_1_32B_BASE) + iavf_extract_legacy_rx_fields(rx_ring, rx_desc, fields); + else + iavf_extract_flex_rx_fields(rx_ring, rx_desc, fields); +} + /** * iavf_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @rx_ring: rx descriptor ring to transact packets on @@ -1655,12 +2109,9 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) #endif while (likely(total_rx_packets < (unsigned int)budget)) { + struct iavf_rx_extracted fields = {}; struct iavf_rx_buffer *rx_buffer; union iavf_rx_desc *rx_desc; - unsigned int size; - u16 vlan_tag; - u8 rx_ptype; - u64 qword; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IAVF_RX_BUFFER_WRITE) { @@ -1671,13 +2122,6 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) rx_desc = IAVF_RX_DESC(rx_ring, rx_ring->next_to_clean); - /* status_error_len will always be zero for unused descriptors - * because it's cleared in cleanup, and overlaps with hdr_addr - * which is always zero because packet split isn't used, if the - * hardware wrote DD then the length will be non-zero - */ - qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); - /* This memory barrier is needed to keep us from reading * any other fields out of the rx_desc until we have * verified the descriptor has been written back. @@ -1687,11 +2131,10 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) if (!iavf_test_staterr(rx_desc, IAVF_RXD_DD)) break; - size = (qword & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> - IAVF_RXD_QW1_LENGTH_PBUF_SHIFT; + iavf_extract_rx_fields(rx_ring, rx_desc, &fields); iavf_trace(clean_rx_irq, rx_ring, rx_desc, skb); - rx_buffer = iavf_get_rx_buffer(rx_ring, size); + rx_buffer = iavf_get_rx_buffer(rx_ring, fields.size); /* retrieve a buffer from the ring */ if (!skb) { @@ -1700,7 +2143,7 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) rx_buffer->page_offset; xdp.data_hard_start = (void *)((u8 *)xdp.data - iavf_rx_offset(rx_ring)); - xdp.data_end = (void *)((u8 *)xdp.data + size); + xdp.data_end = (void *)((u8 *)xdp.data + fields.size); skb = iavf_run_xdp(rx_ring, &xdp); } else { break; @@ -1712,15 +2155,15 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) if (xdp_res & (IAVF_XDP_TX | IAVF_XDP_REDIR)) { xdp_xmit |= xdp_res; - iavf_rx_buffer_flip(rx_ring, rx_buffer, size); + iavf_rx_buffer_flip(rx_ring, rx_buffer, fields.size); } else { if (rx_buffer) rx_buffer->pagecnt_bias++; } - total_rx_bytes += size; + total_rx_bytes += fields.size; total_rx_packets++; } else if (skb) { - iavf_add_rx_frag(rx_ring, rx_buffer, skb, size); + iavf_add_rx_frag(rx_ring, rx_buffer, skb, fields.size); #ifdef HAVE_SWIOTLB_SKIP_CPU_SYNC } else if (ring_uses_build_skb(rx_ring)) { skb = iavf_build_skb(rx_ring, rx_buffer, &xdp); @@ -1751,18 +2194,18 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; - qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); - rx_ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> - IAVF_RXD_QW1_PTYPE_SHIFT; - /* populate checksum, VLAN, and protocol */ - iavf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype); + iavf_process_skb_fields(rx_ring, rx_desc, skb, fields.rx_ptype); - vlan_tag = (qword & BIT(IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) ? - le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0; + if (vector_ch_ena(rx_ring->q_vector) && + vector_ch_perf_ena(rx_ring->q_vector)) { + if (!iavf_is_ctrl_pkt(skb, rx_ring)) + rx_ring->q_vector->state_flags |= + IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV; + } iavf_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb); - iavf_receive_skb(rx_ring, skb, vlan_tag); + iavf_receive_skb(rx_ring, skb, fields.vlan_tag); skb = NULL; /* update budget accounting */ @@ -1784,6 +2227,7 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) rx_ring->stats.packets += total_rx_packets; rx_ring->stats.bytes += total_rx_bytes; u64_stats_update_end(&rx_ring->syncp); + iavf_chnl_rx_stats(rx_ring, total_rx_packets); rx_ring->q_vector->rx.total_packets += total_rx_packets; rx_ring->q_vector->rx.total_bytes += total_rx_bytes; @@ -1791,7 +2235,7 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) return failure ? budget : (int)total_rx_packets; } -static inline u32 iavf_buildreg_itr(const int type, u16 itr) +static u32 iavf_buildreg_itr(const int type, u16 itr) { u32 val; @@ -1837,12 +2281,21 @@ static inline u32 iavf_buildreg_itr(const int type, u16 itr) * @q_vector: q_vector for which itr is being updated and interrupt enabled * **/ -static inline void iavf_update_enable_itr(struct iavf_vsi *vsi, - struct iavf_q_vector *q_vector) +static void iavf_update_enable_itr(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector) { struct iavf_hw *hw = &vsi->back->hw; u32 intval; + /* if vector is channel enabled, it doesn't use ITR countdown + * or pseudo-lazy update for ITR update + */ + if (vector_ch_ena(q_vector) && + vector_ch_perf_ena(q_vector)) { + /* No ITR update */ + intval = iavf_buildreg_itr(IAVF_ITR_NONE, 0); + goto do_write; + } + /* These will do nothing if dynamic updates are not enabled */ iavf_update_itr(q_vector, &q_vector->tx); iavf_update_itr(q_vector, &q_vector->rx); @@ -1883,11 +2336,123 @@ static inline void iavf_update_enable_itr(struct iavf_vsi *vsi, if (q_vector->itr_countdown) q_vector->itr_countdown--; } - +do_write: if (!test_bit(__IAVF_VSI_DOWN, vsi->state)) wr32(hw, INTREG(q_vector->reg_idx), intval); } +/** + * iavf_refresh_bp_state - refresh state machine + * @napi: ptr to NAPI struct + * + * Update ADQ state machine, and depending on whether this was called from + * busy poll, enable interrupts and update ITR + */ +static void iavf_refresh_bp_state(struct napi_struct *napi) +{ + struct iavf_q_vector *q_vector = + container_of(napi, struct iavf_q_vector, napi); + + /* cache previous state of vector */ + if (q_vector->state_flags & IAVF_VECTOR_STATE_IN_BP) + q_vector->state_flags |= IAVF_VECTOR_STATE_PREV_IN_BP; + else + q_vector->state_flags &= ~IAVF_VECTOR_STATE_PREV_IN_BP; + +#ifdef HAVE_NAPI_STATE_IN_BUSY_POLL + /* update current state of vector */ + if (test_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state)) + q_vector->state_flags |= IAVF_VECTOR_STATE_IN_BP; + else + q_vector->state_flags &= ~IAVF_VECTOR_STATE_IN_BP; +#endif /* HAVE_STATE_IN_BUSY_POLL */ + + if (q_vector->state_flags & IAVF_VECTOR_STATE_IN_BP) { + q_vector->jiffies = jiffies; + /* trigger force_wb by setting WB_ON_ITR only when + * - vector is transitioning from INTR->BUSY_POLL + * - once_in_bp is false, this is to prevent from doing it + * every time whenever vector state is changing from + * INTR->BUSY_POLL because that could be due to legit + * busy_poll stop + */ + if (!(q_vector->state_flags & IAVF_VECTOR_STATE_ONCE_IN_BP) && + vector_intr_busypoll(q_vector)) + iavf_set_wb_on_itr(&q_vector->vsi->back->hw, q_vector); + + q_vector->state_flags |= IAVF_VECTOR_STATE_ONCE_IN_BP; + q_vector->ch_stats.in_bp++; + /* state transition : INTERRUPT --> BUSY_POLL */ + if (!(q_vector->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP)) + q_vector->ch_stats.intr_to_bp++; + else + q_vector->ch_stats.bp_to_bp++; + } else { + q_vector->ch_stats.in_intr++; + /* state transition : BUSY_POLL --> INTERRUPT */ + if (q_vector->state_flags & IAVF_VECTOR_STATE_PREV_IN_BP) + q_vector->ch_stats.bp_to_intr++; + else + q_vector->ch_stats.intr_to_intr++; + } +} + +/* + * iavf_handle_chnl_vector - handle channel enabled vector + * @vsi: ptr to VSI + * @q_vector: ptr to q_vector + * @unlikely_cb_bp: will comeback to busy_poll or not + * + * This function eithers triggers software interrupt (when unlikely_cb_bp is + * true) or enable interrupt normally. unlikely_cb_bp gets determined based + * on state machine and packet parsing logic. + */ +static void +iavf_handle_chnl_vector(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector, + bool unlikely_cb_bp) +{ + struct iavf_q_vector_ch_stats *stats = &q_vector->ch_stats; + + /* caller of this function deteremines next occurrence/execution context + * of napi_poll (means next time whether napi_poll will be invoked from + * busy_poll or SOFT IRQ context). Please refer to the caller of this + * function to see logic for "unlikely_cb_bp" (aka, re-occurrence to + * busy_poll or not). + * If logic determines that, next occurrence of napi_poll will not be + * from busy_poll context, trigger software initiated interrupt on + * channel enabled vector to revive queue(s) processing, otherwise if + * in true interrupt state - just enable interrupt. + */ + if (unlikely_cb_bp) { + stats->unlikely_cb_to_bp++; + /* if once_in_bp is set and pkt inspection based optimization + * is off, do not trigger SW interrupt (simply bailout). + * No change in logic from service_task based software + * triggred interrupt - to revive the queue based on jiffy logic + */ + if (q_vector->state_flags & IAVF_VECTOR_STATE_ONCE_IN_BP) { + stats->ucb_once_in_bp_true++; + if (!vector_pkt_inspect_opt_ena(q_vector)) { + stats->no_sw_intr_opt_off++; + return; + } + } + + /* Since this real BP -> INT transition, reset jiffy snapshot */ + q_vector->jiffies = 0; + + /* Likewise for real BP -> INT, trigger + * SW interrupt, so that vector is put back + * in sane state, trigger sw interrupt to revive the queue + */ + iavf_inc_napi_sw_intr_counter(q_vector); + iavf_force_wb(vsi, q_vector); + } else if (!(q_vector->state_flags & IAVF_VECTOR_STATE_ONCE_IN_BP)) { + stats->intr_once_bp_false++; + iavf_update_enable_itr(vsi, q_vector); + } +} + /** * iavf_napi_poll - NAPI polling Rx/Tx cleanup routine * @napi: napi struct with our devices info in it @@ -1902,9 +2467,13 @@ int iavf_napi_poll(struct napi_struct *napi, int budget) struct iavf_q_vector *q_vector = container_of(napi, struct iavf_q_vector, napi); struct iavf_vsi *vsi = q_vector->vsi; - struct iavf_ring *ring; + bool cleaned_any_data_pkt = false; u64 flags = vsi->back->flags; + bool unlikely_cb_bp = false; bool clean_complete = true; + bool ch_enabled = false; + bool wb_on_itr_enabled; + struct iavf_ring *ring; bool arm_wb = false; int budget_per_ring; int work_done = 0; @@ -1914,6 +2483,55 @@ int iavf_napi_poll(struct napi_struct *napi, int budget) return 0; } + /* determine if WB_ON_ITR is enabled on not, if not - not need to + * apply any performance optimization + */ + wb_on_itr_enabled = true; + iavf_for_each_ring(ring, q_vector->tx) { + if (!(ring->flags & IAVF_TXR_FLAGS_WB_ON_ITR)) { + wb_on_itr_enabled &= false; + break; + } + } + + /* determine once if vector needs to be processed differently */ + ch_enabled = wb_on_itr_enabled && vector_ch_ena(q_vector) && + vector_ch_perf_ena(q_vector); + if (ch_enabled) { + u8 qv_flags; + + /* Refresh state machine */ + iavf_refresh_bp_state(napi); + + /* check during previous run of napi_poll whether at least one + * data packets is processed or not. If processed at least one + * data packet, set the local flag 'cleaned_any_data_pkt' + * which is used later in this function to determine if + * interrupt should be enabled or deferred (this is applicable + * only in case when busy_poll stop is invoked, means previous + * state of vector is in busy_poll and current state is not + * (aka BUSY_POLL -> INTR)) + */ + qv_flags = q_vector->state_flags; + if (qv_flags & IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV) { + q_vector->state_flags &= + ~IAVF_VECTOR_STATE_PREV_DATA_PKT_RECV; + /* It is important to check and cache correct\ + * information (cleaned any data packets or not) in + * local variable before napi_complete_done is finished. + * Once napi_complete_done is returned, napi_poll + * can get invoked again (means re-entrant) which can + * potentially results to incorrect decision making + * w.r.t. whether interrupt should be enabled or + * deferred) + */ + if (vector_busypoll_intr(q_vector)) { + cleaned_any_data_pkt = true; + q_vector->ch_stats.cleaned_any_data_pkt++; + } + } + } + /* Since the actual Tx work is minimal, we can give the Tx a larger * budget and be more aggressive about cleaning up the Tx descriptors. */ @@ -1930,6 +2548,20 @@ int iavf_napi_poll(struct napi_struct *napi, int budget) if (budget <= 0) goto tx_only; + /* state transitioning from BUSY_POLL --> INTERRUPT. This can happen + * due to several reason when stack calls busy_poll_stop + * 1. during last execution of napi_poll returned non-zero packets + * 2. busy_loop ended + * 3. need re-sched set + * driver keeps track of packets were cleaned during last run and if + * that is zero, means most likely napi_poll won't be invoked from + * busy_poll context; in that situation bypass processing of Rx queues + * and enable interrupt and let subsequent run of napi_poll from + * interrupt path handle cleanup of Rx queues + */ + if (ch_enabled && vector_busypoll_intr(q_vector)) + goto bypass; + /* We attempt to distribute budget to each Rx queue fairly, but don't * allow the budget to go below 1 because that would exit polling early. */ @@ -1950,6 +2582,10 @@ int iavf_napi_poll(struct napi_struct *napi, int budget) clean_complete = true; #endif + /* if this vector ever was/is in BUSY_POLL, skip processing */ + if (ch_enabled && vector_ever_in_busypoll(q_vector)) + goto bypass; + /* If work not completed, return budget and polling will return */ if (!clean_complete) { #ifdef HAVE_IRQ_AFFINITY_NOTIFY @@ -1965,7 +2601,7 @@ int iavf_napi_poll(struct napi_struct *napi, int budget) if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) { /* Tell napi that we are done polling */ napi_complete_done(napi, work_done); - + q_vector->ch_stats.intr_en_not_clean_complete++; /* Force an interrupt */ iavf_force_wb(vsi, q_vector); @@ -1984,10 +2620,85 @@ tx_only: if (flags & IAVF_TXR_FLAGS_WB_ON_ITR) q_vector->arm_wb_state = false; - /* Work is done so exit the polling mode and re-enable the interrupt */ - napi_complete_done(napi, work_done); +bypass: + /* Following block is only for stats, hence guarded by "debug_mask" */ + if (ch_enabled && vector_busypoll_intr(q_vector)) { + struct iavf_q_vector_ch_stats *stats; - iavf_update_enable_itr(vsi, q_vector); + stats = &q_vector->ch_stats; + if (unlikely(need_resched())) { + stats->bp_stop_need_resched++; + if (!cleaned_any_data_pkt) + stats->need_resched_no_data_pkt++; + } else { + /* here , means actually because of 2 reason + * - busy_poll timeout expired + * - last time, cleaned data packets, hence + * stack asked to stop busy_poll so that packet + * can be processed by consumer + */ + stats->bp_stop_timeout++; + if (!cleaned_any_data_pkt) + stats->timeout_no_data_pkt++; + } + } + /* if state transition from busy_poll to interrupt and during + * last run: did not cleanup TCP data packets - + * then application unlikely to comeback to busy_poll + */ + if (ch_enabled && vector_busypoll_intr(q_vector) && + !cleaned_any_data_pkt) { + /* for now, if need_resched is true (it can be either + * due to voluntary/in-voluntary context switches), + * do not trigger SW interrupt. + * if need_resched is not set, safely assuming, it is due + * to possible timeout and unlikely that application/context + * will return to busy_poll, hence set 'unlikely_cb_bp' to + * true which will cause software triggered interrupt + * to reviev the queue/vector + */ + if (unlikely(need_resched())) + unlikely_cb_bp = false; + else + unlikely_cb_bp = true; + } + + /* Work is done so exit the polling mode and re-enable the interrupt */ + if (likely(napi_complete_done(napi, work_done))) { + /* napi_ret : false (means vector is still in POLLING mode + * true (means out of POLLING) + * NOTE: Generally if napi_ret is TRUE, enable device interrupt + * but there are condition/optimization, where it can be + * optimized. Bascially, if napi_complete_done returns true buti + * last time Rx packets were cleaned, then most likely, consumer + * thread will come back to do busy_polling where cleaning of + * Tx/Rx queue will happen normally. Hence no reason to arm the + * interrupt. + * + * If for some reason, consumer thread/context doesn't comeback + * to busy_poll:napi_poll, there is bail-out mechanism to kick + * start the state machine thru' SW triggered interrupt from + * service task. + */ + if (ch_enabled) { + /* current state of NAPI is INTERRUPT */ + iavf_handle_chnl_vector(vsi, q_vector, unlikely_cb_bp); + } else { + iavf_update_enable_itr(vsi, q_vector); + } + } else { + /* if code makes it here, means busy_poll is still ON. + * if vector is channel enabled, setting WB_ON_ITR is handled + * from iavf_refresh_bp_state function. + * otherwise set WB_ON_ITR (if supported) + */ + if (!ch_enabled) { + if (wb_on_itr_enabled) + iavf_enable_wb_on_itr(vsi, q_vector); + else + iavf_update_enable_itr(vsi, q_vector); + } + } return min_t(int, work_done, budget - 1); } @@ -2004,51 +2715,35 @@ tx_only: * Returns error code indicate the frame should be dropped upon error and the * otherwise returns 0 to indicate the flags has been set properly. **/ -static inline int iavf_tx_prepare_vlan_flags(struct sk_buff *skb, - struct iavf_ring *tx_ring, - u32 *flags) +static void iavf_tx_prepare_vlan_flags(struct sk_buff *skb, + struct iavf_ring *tx_ring, + u32 *flags) { - __be16 protocol = skb->protocol; u32 tx_flags = 0; -#ifdef NETIF_F_HW_VLAN_CTAG_RX - if (protocol == htons(ETH_P_8021Q) && - !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { -#else - if (protocol == htons(ETH_P_8021Q) && - !(tx_ring->netdev->features & NETIF_F_HW_VLAN_TX)) { -#endif - /* When HW VLAN acceleration is turned off by the user the - * stack sets the protocol to 8021q so that the driver - * can take any steps required to support the SW only - * VLAN handling. In our case the driver doesn't need - * to take any further steps so just set the protocol - * to the encapsulated ethertype. - */ - skb->protocol = vlan_get_protocol(skb); - goto out; - } + /* stack will only request hardware VLAN insertion offload for protocols + * that the driver supports and has enabled + */ + if (!skb_vlan_tag_present(skb)) + return; - /* if we have a HW VLAN tag being added, default to the HW one */ - if (skb_vlan_tag_present(skb)) { - tx_flags |= skb_vlan_tag_get(skb) << IAVF_TX_FLAGS_VLAN_SHIFT; + tx_flags |= skb_vlan_tag_get(skb) << IAVF_TX_FLAGS_VLAN_SHIFT; + if (tx_ring->flags & IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2) { + tx_flags |= IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN; + } else if (tx_ring->flags & IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1) { tx_flags |= IAVF_TX_FLAGS_HW_VLAN; - /* else if it is a SW VLAN, check the next protocol and store the tag */ - } else if (protocol == htons(ETH_P_8021Q)) { - struct vlan_hdr *vhdr, _vhdr; - - vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); - if (!vhdr) - return -EINVAL; - - protocol = vhdr->h_vlan_encapsulated_proto; - tx_flags |= ntohs(vhdr->h_vlan_TCI) << IAVF_TX_FLAGS_VLAN_SHIFT; - tx_flags |= IAVF_TX_FLAGS_SW_VLAN; + } else { + dev_dbg(tx_ring->dev, "Unsupported Tx VLAN tag location requested\n"); + return; } +#ifdef IAVF_ADD_PROBES + if (tx_ring->netdev->features & IAVF_NETIF_F_HW_VLAN_CTAG_TX) + tx_ring->vsi->back->tx_vlano++; + else + tx_ring->vsi->back->tx_ad_vlano++; +#endif -out: *flags = tx_flags; - return 0; } #ifdef IAVF_ADD_PROBES @@ -2181,17 +2876,22 @@ static int iavf_tso(struct iavf_tx_buffer *first, u8 *hdr_len, /* remove payload length from inner checksum */ paylen = skb->len - l4_offset; - if (skb->csum_offset == offsetof(struct tcphdr, check)) { - csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(paylen)); - /* compute length of TCP segmentation header */ - *hdr_len = (l4.tcp->doff * 4) + l4_offset; - } else { +#ifdef NETIF_F_GSO_UDP_L4 + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { csum_replace_by_diff(&l4.udp->check, (__force __wsum)htonl(paylen)); /* compute length of UDP segmentation header */ - *hdr_len = sizeof(l4.udp) + l4_offset; + *hdr_len = (u8)sizeof(l4.udp) + l4_offset; + } else { + csum_replace_by_diff(&l4.tcp->check, + (__force __wsum)htonl(paylen)); + /* compute length of TCP segmentation header */ + *hdr_len = (u8)((l4.tcp->doff * 4) + l4_offset); } +#else + csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); + *hdr_len = (u8)((l4.tcp->doff * 4) + l4_offset); +#endif /* NETIF_F_GSO_UDP_L4 */ /* pull values out of skb_shinfo */ gso_size = skb_shinfo(skb)->gso_size; @@ -2425,7 +3125,57 @@ static int iavf_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, } /** - * iavf_create_tx_ctx Build the Tx context descriptor + * iavf_tstamp - setup context descriptor for timestamping + * @tx_ring: ring to send buffer on + * @skb: send buffer + * @tx_flags: collected send information + * @cd_type_cmd_tso_mss: Quad Word 1 + * + * Setup timestamp request for an outbound packet. The request will only be + * made if the user has requested it, and if we're not already waiting for + * timestamp completion of a previous packet. + * + * Return 1 if a Tx timestamp will happen, 0 otherwise. + */ +static int +iavf_tstamp(struct iavf_ring *tx_ring, struct sk_buff *skb, u32 tx_flags, u64 *cd_type_cmd_tso_mss) +{ + struct iavf_adapter *adapter; + struct iavf_ptp *ptp; + u64 ts_idx; + + /* Timestamping is not enabled */ + if (!(tx_ring->flags & IAVF_TXRX_FLAGS_HW_TSTAMP)) + return 0; + + if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) + return 0; + + /* Hardware cannot sample a timestamp when doing TSO */ + if (tx_flags & IAVF_TX_FLAGS_TSO) + return 0; + + adapter = netdev_priv(tx_ring->netdev); + ptp = &adapter->ptp; + + if (test_and_set_bit_lock(__IAVF_TX_TSTAMP_IN_PROGRESS, &adapter->crit_section)) { + ptp->tx_hwtstamp_skipped++; + return 0; + } + + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + ptp->tx_start = jiffies; + ptp->tx_skb = skb_get(skb); + + ts_idx = ptp->hw_caps.tx_tstamp_idx; + *cd_type_cmd_tso_mss |= (IAVF_TX_CTX_DESC_TSYN << IAVF_TXD_CTX_QW1_CMD_SHIFT) | + (ts_idx << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT); + + return 1; +} + +/** + * iavf_create_tx_ctx - Build the Tx context descriptor * @tx_ring: ring to create the descriptor on * @cd_type_cmd_tso_mss: Quad Word 1 * @cd_tunneling: Quad Word 0 - bits 0-31 @@ -2574,9 +3324,9 @@ bool __iavf_chk_linearize(struct sk_buff *skb) * * Returns 0 on success, negative error code on DMA failure. **/ -static inline int iavf_tx_map(struct iavf_ring *tx_ring, struct sk_buff *skb, - struct iavf_tx_buffer *first, u32 tx_flags, - const u8 hdr_len, u32 td_cmd, u32 td_offset) +static int iavf_tx_map(struct iavf_ring *tx_ring, struct sk_buff *skb, + struct iavf_tx_buffer *first, u32 tx_flags, + const u8 hdr_len, u32 td_cmd, u32 td_offset) { unsigned int data_len = skb->data_len; unsigned int size = skb_headlen(skb); @@ -2591,9 +3341,6 @@ static inline int iavf_tx_map(struct iavf_ring *tx_ring, struct sk_buff *skb, td_cmd |= IAVF_TX_DESC_CMD_IL2TAG1; td_tag = (tx_flags & IAVF_TX_FLAGS_VLAN_MASK) >> IAVF_TX_FLAGS_VLAN_SHIFT; -#ifdef IAVF_ADD_PROBES - tx_ring->vsi->back->tx_vlano++; -#endif } first->tx_flags = tx_flags; @@ -2760,12 +3507,12 @@ static netdev_tx_t iavf_xmit_frame_ring(struct sk_buff *skb, u64 cd_type_cmd_tso_mss = IAVF_TX_DESC_DTYPE_CONTEXT; u32 cd_tunneling = 0, cd_l2tag2 = 0; struct iavf_tx_buffer *first; + int tso, tstamp, count; u32 td_offset = 0; u32 tx_flags = 0; __be16 protocol; u32 td_cmd = 0; u8 hdr_len = 0; - int tso, count; /* prefetch the data, we'll need it later */ prefetch(skb->data); @@ -2800,8 +3547,13 @@ static netdev_tx_t iavf_xmit_frame_ring(struct sk_buff *skb, first->gso_segs = 1; /* prepare the xmit flags */ - if (iavf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) - goto out_drop; + iavf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags); + if (tx_flags & IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN) { + cd_type_cmd_tso_mss |= IAVF_TX_CTX_DESC_IL2TAG2 << + IAVF_TXD_CTX_QW1_CMD_SHIFT; + cd_l2tag2 = (tx_flags & IAVF_TX_FLAGS_VLAN_MASK) >> + IAVF_TX_FLAGS_VLAN_SHIFT; + } /* obtain protocol of skb */ protocol = vlan_get_protocol(skb); @@ -2825,14 +3577,19 @@ static netdev_tx_t iavf_xmit_frame_ring(struct sk_buff *skb, if (tso < 0) goto out_drop; + tstamp = iavf_tstamp(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss); + if (tstamp) + tx_flags |= IAVF_TX_FLAGS_TSTAMP; + /* always enable CRC insertion offload */ td_cmd |= IAVF_TX_DESC_CMD_ICRC; iavf_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, cd_tunneling, cd_l2tag2); - iavf_tx_map(tx_ring, skb, first, tx_flags, hdr_len, - td_cmd, td_offset); + if (iavf_tx_map(tx_ring, skb, first, tx_flags, hdr_len, + td_cmd, td_offset)) + goto cleanup_tx_tstamp; #ifndef HAVE_TRANS_START_IN_QUEUE tx_ring->netdev->trans_start = jiffies; @@ -2843,6 +3600,15 @@ out_drop: iavf_trace(xmit_frame_ring_drop, first->skb, tx_ring); dev_kfree_skb_any(first->skb); first->skb = NULL; +cleanup_tx_tstamp: + if (unlikely(tx_flags & IAVF_TX_FLAGS_TSTAMP)) { + struct iavf_adapter *adapter = netdev_priv(tx_ring->netdev); + + dev_kfree_skb_any(adapter->ptp.tx_skb); + adapter->ptp.tx_skb = NULL; + clear_bit_unlock(__IAVF_TX_TSTAMP_IN_PROGRESS, &adapter->crit_section); + } + return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.h b/drivers/net/ethernet/intel/iavf/iavf_txrx.h index d1232452f..a2a6a22a1 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.h +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.h @@ -262,19 +262,21 @@ static inline unsigned int iavf_txd_use_count(unsigned int size) #define DESC_NEEDED (MAX_SKB_FRAGS + 6) #define IAVF_MIN_DESC_PENDING 4 -#define IAVF_TX_FLAGS_HW_VLAN BIT(1) -#define IAVF_TX_FLAGS_SW_VLAN BIT(2) -#define IAVF_TX_FLAGS_TSO BIT(3) -#define IAVF_TX_FLAGS_IPV4 BIT(4) -#define IAVF_TX_FLAGS_IPV6 BIT(5) -#define IAVF_TX_FLAGS_FCCRC BIT(6) -#define IAVF_TX_FLAGS_FSO BIT(7) -#define IAVF_TX_FLAGS_FD_SB BIT(9) -#define IAVF_TX_FLAGS_TUNNEL BIT(10) -#define IAVF_TX_FLAGS_VLAN_MASK 0xffff0000 -#define IAVF_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000 -#define IAVF_TX_FLAGS_VLAN_PRIO_SHIFT 29 -#define IAVF_TX_FLAGS_VLAN_SHIFT 16 +#define IAVF_TX_FLAGS_HW_VLAN BIT(1) +#define IAVF_TX_FLAGS_SW_VLAN BIT(2) +#define IAVF_TX_FLAGS_TSO BIT(3) +#define IAVF_TX_FLAGS_IPV4 BIT(4) +#define IAVF_TX_FLAGS_IPV6 BIT(5) +#define IAVF_TX_FLAGS_FCCRC BIT(6) +#define IAVF_TX_FLAGS_FSO BIT(7) +#define IAVF_TX_FLAGS_FD_SB BIT(9) +#define IAVF_TX_FLAGS_TUNNEL BIT(10) +#define IAVF_TX_FLAGS_HW_OUTER_SINGLE_VLAN BIT(11) +#define IAVF_TX_FLAGS_TSTAMP BIT(12) +#define IAVF_TX_FLAGS_VLAN_MASK 0xffff0000 +#define IAVF_TX_FLAGS_VLAN_PRIO_MASK 0xe0000000 +#define IAVF_TX_FLAGS_VLAN_PRIO_SHIFT 29 +#define IAVF_TX_FLAGS_VLAN_SHIFT 16 struct iavf_tx_buffer { struct iavf_tx_desc *next_to_watch; @@ -324,6 +326,32 @@ struct iavf_rx_queue_stats { u64 realloc_count; }; +struct iavf_ch_tx_q_stats { +}; + +struct iavf_ch_rx_q_stats { + u64 tcp_ctrl_pkts; + u64 only_ctrl_pkts; + u64 bp_no_data_pkt; + u64 tcp_fin_recv; + u64 tcp_rst_recv; + u64 tcp_syn_recv; +}; + +struct iavf_ch_q_poll_stats { + /* general packet counters for busy_poll versus napi_poll */ + u64 pkt_busy_poll; + u64 pkt_not_busy_poll; +}; + +struct iavf_ch_q_stats { + struct iavf_ch_q_poll_stats poll; + union { + struct iavf_ch_tx_q_stats tx; + struct iavf_ch_rx_q_stats rx; + }; +}; + enum iavf_ring_state_t { __IAVF_TX_FDIR_INIT_DONE, __IAVF_TX_XPS_INIT_DONE, @@ -379,10 +407,16 @@ struct iavf_ring { bool arm_wb; /* do something to arm write back */ u8 packet_stride; + u8 rxdid; /* Rx descriptor format */ + u16 flags; #define IAVF_TXR_FLAGS_WB_ON_ITR BIT(0) #define IAVF_RXR_FLAGS_BUILD_SKB_ENABLED BIT(1) #define IAVF_TXR_FLAGS_XDP BIT(2) +#define IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 BIT(3) +#define IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2 BIT(4) +#define IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2 BIT(5) +#define IAVF_TXRX_FLAGS_HW_TSTAMP BIT(6) /* stats structs */ struct iavf_queue_stats stats; @@ -394,6 +428,8 @@ struct iavf_ring { struct iavf_rx_queue_stats rx_stats; }; + struct iavf_ch_q_stats ch_q_stats; + unsigned int size; /* length of descriptor ring in bytes */ dma_addr_t dma; /* physical address of ring */ @@ -411,12 +447,26 @@ struct iavf_ring { * for this ring. */ - struct iavf_channel *ch; + u16 chnl_flags; +#define IAVF_RING_CHNL_PERF_ENA BIT(0) + + struct iavf_channel_ex *ch; + #ifdef HAVE_XDP_BUFF_RXQ struct xdp_rxq_info xdp_rxq; #endif } ____cacheline_internodealigned_in_smp; +static inline bool ring_ch_ena(struct iavf_ring *ring) +{ + return !!ring->ch; +} + +static inline bool ring_ch_perf_ena(struct iavf_ring *ring) +{ + return ring->chnl_flags & IAVF_RING_CHNL_PERF_ENA; +} + static inline bool ring_uses_build_skb(struct iavf_ring *ring) { return !!(ring->flags & IAVF_RXR_FLAGS_BUILD_SKB_ENABLED); @@ -483,9 +533,9 @@ int iavf_setup_rx_descriptors(struct iavf_ring *rx_ring); void iavf_free_tx_resources(struct iavf_ring *tx_ring); void iavf_free_rx_resources(struct iavf_ring *rx_ring); int iavf_napi_poll(struct napi_struct *napi, int budget); -void iavf_force_wb(struct iavf_vsi *vsi, struct iavf_q_vector *q_vector); u32 iavf_get_tx_pending(struct iavf_ring *ring, bool in_sw); void iavf_detect_recover_hung(struct iavf_vsi *vsi); +void iavf_chnl_detect_recover(struct iavf_vsi *vsi); int __iavf_maybe_stop_tx(struct iavf_ring *tx_ring, int size); bool __iavf_chk_linearize(struct sk_buff *skb); #ifdef HAVE_XDP_FRAME_STRUCT @@ -500,7 +550,6 @@ void iavf_xdp_flush(struct net_device *dev); /** * iavf_xmit_descriptor_count - calculate number of Tx descriptors needed * @skb: send buffer - * @tx_ring: ring to send buffer on * * Returns number of data descriptors needed for this skb. Returns 0 to indicate * there is not enough descriptors available in this ring since we need at least diff --git a/drivers/net/ethernet/intel/iavf/iavf_type.h b/drivers/net/ethernet/intel/iavf/iavf_type.h index ea5008524..77f0efb0a 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_type.h +++ b/drivers/net/ethernet/intel/iavf/iavf_type.h @@ -158,6 +158,7 @@ struct iavf_hw_capabilities { u32 num_tx_qp; u32 base_queue; u32 num_msix_vectors_vf; + u32 max_mtu; bool apm_wol_support; enum iavf_acpi_programming_method acpi_prog_method; bool proxy_support; @@ -328,6 +329,45 @@ union iavf_16byte_rx_desc { } wb; /* writeback */ }; +/* Rx Flex Descriptor NIC Profile + * RxDID Profile ID 2 + * Flex-field 0: RSS hash lower 16-bits + * Flex-field 1: RSS hash upper 16-bits + * Flex-field 2: Flow ID lower 16-bits + * Flex-field 3: Flow ID higher 16-bits + * Flex-field 4: reserved, VLAN ID taken from L2Tag + */ +struct iavf_32byte_rx_flex_wb { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 rss_hash; + + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 flow_id; + union { + struct { + __le16 rsvd; + __le16 flow_id_ipv6; + } flex; + __le32 ts_high; + } flex_ts; +}; + union iavf_32byte_rx_desc { struct { __le64 pkt_addr; /* Packet buffer address */ @@ -375,6 +415,7 @@ union iavf_32byte_rx_desc { } hi_dword; } qword3; } wb; /* writeback */ + struct iavf_32byte_rx_flex_wb flex_wb; }; enum iavf_rx_desc_status_bits { @@ -443,6 +484,47 @@ enum iavf_rx_desc_error_l3l4e_fcoe_masks { #define IAVF_RXD_QW1_PTYPE_SHIFT 30 #define IAVF_RXD_QW1_PTYPE_MASK (0xFFULL << IAVF_RXD_QW1_PTYPE_SHIFT) +/* for iavf_32byte_rx_flex_wb.ptype_flexi_flags0 member */ +#define IAVF_RX_FLEX_DESC_PTYPE_M (0x3FF) /* 10-bits */ + +/* for iavf_32byte_rx_flex_wb.pkt_length member */ +#define IAVF_RX_FLEX_DESC_PKT_LEN_M (0x3FFF) /* 14-bits */ + +enum iavf_rx_flex_desc_status_error_0_bits { + /* Note: These are predefined bit offsets */ + IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0, + IAVF_RX_FLEX_DESC_STATUS0_EOF_S, + IAVF_RX_FLEX_DESC_STATUS0_HBO_S, + IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S, + IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S, + IAVF_RX_FLEX_DESC_STATUS0_LPBK_S, + IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S, + IAVF_RX_FLEX_DESC_STATUS0_RXE_S, + IAVF_RX_FLEX_DESC_STATUS0_CRCP_S, + IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S, + IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S, + IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +enum iavf_rx_flex_desc_status_error_1_bits { + /* Note: These are predefined bit offsets */ + IAVF_RX_FLEX_DESC_STATUS1_CPM_S = 0, /* 4 bits */ + IAVF_RX_FLEX_DESC_STATUS1_NAT_S = 4, + IAVF_RX_FLEX_DESC_STATUS1_CRYPTO_S = 5, + /* [10:6] reserved */ + IAVF_RX_FLEX_DESC_STATUS1_L2TAG2P_S = 11, + IAVF_RX_FLEX_DESC_STATUS1_XTRMD2_VALID_S = 12, + IAVF_RX_FLEX_DESC_STATUS1_XTRMD3_VALID_S = 13, + IAVF_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S = 14, + IAVF_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S = 15, + IAVF_RX_FLEX_DESC_STATUS1_LAST /* this entry must be last!!! */ +}; + /* Packet type non-ip values */ enum iavf_rx_l2_ptype { IAVF_RX_PTYPE_L2_RESERVED = 0, @@ -470,11 +552,12 @@ enum iavf_rx_l2_ptype { IAVF_RX_PTYPE_GRENAT4_MAC_PAY3 = 58, IAVF_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4 = 87, IAVF_RX_PTYPE_GRENAT6_MAC_PAY3 = 124, - IAVF_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4 = 153 + IAVF_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4 = 153, + IAVF_RX_PTYPE_PARSER_ABORTED = 255 }; struct iavf_rx_ptype_decoded { - u32 ptype:8; + u32 ptype:10; u32 known:1; u32 outer_ip:1; u32 outer_ip_ver:1; diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 8311e0658..7cd673730 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -3,7 +3,6 @@ #include "iavf.h" #include "iavf_prototype.h" -#include "iavf_client.h" /* busy wait delay in msec */ #define IAVF_BUSY_WAIT_DELAY 10 @@ -123,7 +122,6 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter) u32 caps; caps = VIRTCHNL_VF_OFFLOAD_L2 | - VIRTCHNL_VF_OFFLOAD_IWARP | VIRTCHNL_VF_OFFLOAD_RSS_PF | VIRTCHNL_VF_OFFLOAD_RSS_AQ | VIRTCHNL_VF_OFFLOAD_RSS_REG | @@ -131,7 +129,10 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter) VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 | VIRTCHNL_VF_OFFLOAD_ENCAP | + VIRTCHNL_VF_OFFLOAD_VLAN_V2 | + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC | VIRTCHNL_VF_OFFLOAD_REQ_QUEUES | + VIRTCHNL_VF_CAP_PTP | #ifdef __TC_MQPRIO_MODE_MAX VIRTCHNL_VF_OFFLOAD_ADQ | VIRTCHNL_VF_OFFLOAD_ADQ_V2 | @@ -156,6 +157,71 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter) NULL, 0); } +int iavf_send_vf_offload_vlan_v2_msg(struct iavf_adapter *adapter) +{ + adapter->aq_required &= ~IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS; + + if (!VLAN_V2_ALLOWED(adapter)) + return -EOPNOTSUPP; + + adapter->current_op = VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS; + + return iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS, + NULL, 0); +} + +int iavf_send_vf_supported_rxdids_msg(struct iavf_adapter *adapter) +{ + adapter->aq_required &= ~IAVF_FLAG_AQ_GET_SUPPORTED_RXDIDS; + + if (!RXDID_ALLOWED(adapter)) + return -EOPNOTSUPP; + + adapter->current_op = VIRTCHNL_OP_GET_SUPPORTED_RXDIDS; + + return iavf_send_pf_msg(adapter, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, + NULL, 0); +} + +/** + * iavf_send_vf_ptp_caps_msg - Send request for PTP capabilities + * @adapter: private adapter structure + * + * Send the VIRTCHNL_OP_1588_PTP_GET_CAPS command to the PF to request the PTP + * capabilities available to this device. This includes the following + * potential access: + * + * * READ_PHC - access to read the PTP hardware clock time + * * WRITE_PHC - access to control the PHC time via adjustments + * * TX_TSTAMP - access to request up to one transmit timestamp at a time + * * RX_TSTAMP - access to request Rx timestamps on all received packets + * * PHC_REGS - direct access to the clock time registers for reading PHC + * + * The PF will reply with the same opcode a filled out copy of the + * virtchnl_ptp_caps structure which defines the specifics of which features + * are accessible to this device. + */ +int iavf_send_vf_ptp_caps_msg(struct iavf_adapter *adapter) +{ + struct virtchnl_ptp_caps hw_caps = {}; + + adapter->aq_required &= ~IAVF_FLAG_AQ_GET_PTP_CAPS; + + if (!PTP_ALLOWED(adapter)) + return -EOPNOTSUPP; + + hw_caps.caps = (VIRTCHNL_1588_PTP_CAP_READ_PHC | + VIRTCHNL_1588_PTP_CAP_WRITE_PHC | + VIRTCHNL_1588_PTP_CAP_TX_TSTAMP | + VIRTCHNL_1588_PTP_CAP_RX_TSTAMP | + VIRTCHNL_1588_PTP_CAP_PHC_REGS); + + adapter->current_op = VIRTCHNL_OP_1588_PTP_GET_CAPS; + + return iavf_send_pf_msg(adapter, VIRTCHNL_OP_1588_PTP_GET_CAPS, + (u8 *)&hw_caps, sizeof(hw_caps)); +} + /** * iavf_validate_num_queues * @adapter: adapter structure @@ -169,8 +235,7 @@ static void iavf_validate_num_queues(struct iavf_adapter *adapter) * the specified number of queues it's been requested for (as per TC * info). So this check should be skipped when ADQ is enabled. */ - if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && - adapter->num_tc) + if (iavf_is_adq_enabled(adapter)) return; if (adapter->vf_res->num_queue_pairs > IAVF_MAX_REQ_QUEUES) { @@ -244,6 +309,126 @@ out: return err; } +int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = &adapter->hw; + struct iavf_arq_event_info event; + enum virtchnl_ops op; + enum iavf_status err; + u16 len; + + len = sizeof(struct virtchnl_vlan_caps); + event.buf_len = len; + event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); + if (!event.msg_buf) { + err = -ENOMEM; + goto out; + } + + while (1) { + /* When the AQ is empty, iavf_clean_arq_element will return + * nonzero and this loop will terminate. + */ + err = iavf_clean_arq_element(hw, &event, NULL); + if (err) + goto out_alloc; + op = + (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); + if (op == VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS) + break; + } + + err = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); + if (err) + goto out_alloc; + + memcpy(&adapter->vlan_v2_caps, event.msg_buf, min(event.msg_len, len)); +out_alloc: + kfree(event.msg_buf); +out: + return err; +} + +int iavf_get_vf_supported_rxdids(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = &adapter->hw; + struct iavf_arq_event_info event; + enum virtchnl_ops op; + enum iavf_status err; + u16 len; + + len = sizeof(struct virtchnl_supported_rxdids); + event.buf_len = len; + event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); + if (!event.msg_buf) { + err = -ENOMEM; + goto out; + } + + while (1) { + /* When the AQ is empty, iavf_clean_arq_element will return + * nonzero and this loop will terminate. + */ + err = iavf_clean_arq_element(hw, &event, NULL); + if (err) + goto out_alloc; + op = + (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); + if (op == VIRTCHNL_OP_GET_SUPPORTED_RXDIDS) + break; + } + + err = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); + if (err) + goto out_alloc; + + memcpy(&adapter->supported_rxdids, event.msg_buf, min(event.msg_len, len)); +out_alloc: + kfree(event.msg_buf); +out: + return err; +} + +int iavf_get_vf_ptp_caps(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = &adapter->hw; + struct iavf_arq_event_info event; + enum virtchnl_ops op; + enum iavf_status err; + u16 len; + + len = sizeof(struct virtchnl_ptp_caps); + event.buf_len = len; + event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); + if (!event.msg_buf) { + err = -ENOMEM; + goto out; + } + + while (1) { + /* When the AQ is empty, iavf_clean_arq_element will return + * nonzero and this loop will terminate. + */ + err = iavf_clean_arq_element(hw, &event, NULL); + if (err) + goto out_alloc; + op = + (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); + if (op == VIRTCHNL_OP_1588_PTP_GET_CAPS) + break; + } + + err = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); + if (err) + goto out_alloc; + + memcpy(&adapter->ptp.hw_caps, event.msg_buf, min(event.msg_len, len)); +out_alloc: + kfree(event.msg_buf); +out: + return err; +} + /** * iavf_configure_queues * @adapter: adapter structure @@ -290,6 +475,8 @@ void iavf_configure_queues(struct iavf_adapter *adapter) vqpi->rxq.databuffer_size = ALIGN(adapter->rx_rings[i].rx_buf_len, BIT_ULL(IAVF_RXQ_CTX_DBUFF_SHIFT)); + if (RXDID_ALLOWED(adapter)) + vqpi->rxq.rxdid = adapter->rxdid; vqpi++; } @@ -433,6 +620,22 @@ int iavf_request_queues(struct iavf_adapter *adapter, int num) (u8 *)&vfres, sizeof(vfres)); } +/** + * iavf_set_mac_addr_type + * @virtchnl_ether_addr: pointer to request list element + * @filter: pointer filter being requested + * + * Set the correct request type. + **/ +static void +iavf_set_mac_addr_type(struct virtchnl_ether_addr *virtchnl_ether_addr, + struct iavf_mac_filter *filter) +{ + virtchnl_ether_addr->type = filter->is_primary ? + VIRTCHNL_ETHER_ADDR_PRIMARY : + VIRTCHNL_ETHER_ADDR_EXTRA; +} + /** * iavf_add_ether_addrs * @adapter: adapter structure @@ -489,6 +692,7 @@ void iavf_add_ether_addrs(struct iavf_adapter *adapter) list_for_each_entry(f, &adapter->mac_filter_list, list) { if (f->add) { ether_addr_copy(veal->list[i].addr, f->macaddr); + iavf_set_mac_addr_type(&veal->list[i], f); i++; f->add = false; if (i == count) @@ -560,6 +764,7 @@ void iavf_del_ether_addrs(struct iavf_adapter *adapter) list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { if (f->remove) { ether_addr_copy(veal->list[i].addr, f->macaddr); + iavf_set_mac_addr_type(&veal->list[i], f); i++; list_del(&f->list); kfree(f); @@ -578,7 +783,7 @@ void iavf_del_ether_addrs(struct iavf_adapter *adapter) } /** - * iavf_is_mac_add_ok + * iavf_mac_add_ok * @adapter: adapter structure * * Submit list of filters based on PF response. @@ -595,7 +800,7 @@ static void iavf_mac_add_ok(struct iavf_adapter *adapter) } /** - * iavf_is_mac_add_reject + * iavf_mac_add_reject * @adapter: adapter structure * * Remove filters from list based on PF response. @@ -626,7 +831,6 @@ static void iavf_mac_add_reject(struct iavf_adapter *adapter) **/ void iavf_add_vlans(struct iavf_adapter *adapter) { - struct virtchnl_vlan_filter_list *vvfl; int len, i = 0, count = 0; struct iavf_vlan_filter *f; bool more = false; @@ -649,43 +853,100 @@ void iavf_add_vlans(struct iavf_adapter *adapter) spin_unlock_bh(&adapter->mac_vlan_list_lock); return; } - adapter->current_op = VIRTCHNL_OP_ADD_VLAN; - len = sizeof(struct virtchnl_vlan_filter_list) + - (count * sizeof(u16)); - if (len > IAVF_MAX_AQ_BUF_SIZE) { - dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n"); - count = (IAVF_MAX_AQ_BUF_SIZE - - sizeof(struct virtchnl_vlan_filter_list)) / - sizeof(u16); - len = sizeof(struct virtchnl_vlan_filter_list) + - (count * sizeof(u16)); - more = true; - } - vvfl = kzalloc(len, GFP_ATOMIC); - if (!vvfl) { - spin_unlock_bh(&adapter->mac_vlan_list_lock); - return; - } + if (VLAN_ALLOWED(adapter)) { + struct virtchnl_vlan_filter_list *vvfl; - vvfl->vsi_id = adapter->vsi_res->vsi_id; - vvfl->num_elements = count; - list_for_each_entry(f, &adapter->vlan_filter_list, list) { - if (f->add) { - vvfl->vlan_id[i] = f->vlan; - i++; - f->add = false; - if (i == count) - break; + adapter->current_op = VIRTCHNL_OP_ADD_VLAN; + + len = sizeof(*vvfl) + (count * sizeof(u16)); + if (len > IAVF_MAX_AQ_BUF_SIZE) { + dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n"); + count = (IAVF_MAX_AQ_BUF_SIZE - sizeof(*vvfl)) / + sizeof(u16); + len = sizeof(*vvfl) + (count * sizeof(u16)); + more = true; } + vvfl = kzalloc(len, GFP_ATOMIC); + if (!vvfl) { + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return; + } + + vvfl->vsi_id = adapter->vsi_res->vsi_id; + vvfl->num_elements = count; + list_for_each_entry(f, &adapter->vlan_filter_list, list) { + if (f->add) { + vvfl->vlan_id[i] = f->vlan.vid; + i++; + f->add = false; + if (i == count) + break; + } + } + if (!more) + adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER; + + spin_unlock_bh(&adapter->mac_vlan_list_lock); + + iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN, (u8 *)vvfl, len); + kfree(vvfl); + } else if (VLAN_V2_ALLOWED(adapter)) { + struct virtchnl_vlan_filter_list_v2 *vvfl_v2; + + adapter->current_op = VIRTCHNL_OP_ADD_VLAN_V2; + + len = sizeof(*vvfl_v2) + ((count - 1) * + sizeof(struct virtchnl_vlan_filter)); + if (len > IAVF_MAX_AQ_BUF_SIZE) { + dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n"); + count = (IAVF_MAX_AQ_BUF_SIZE - sizeof(*vvfl_v2)) / + sizeof(struct virtchnl_vlan_filter); + len = sizeof(*vvfl_v2) + + ((count - 1) * + sizeof(struct virtchnl_vlan_filter)); + more = true; + } + + vvfl_v2 = kzalloc(len, GFP_ATOMIC); + if (!vvfl_v2) { + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return; + } + + vvfl_v2->vport_id = adapter->vsi_res->vsi_id; + vvfl_v2->num_elements = count; + list_for_each_entry(f, &adapter->vlan_filter_list, list) { + if (f->add) { + struct virtchnl_vlan_supported_caps *filtering_support = + &adapter->vlan_v2_caps.filtering.filtering_support; + struct virtchnl_vlan *vlan; + + /* give priority over outer if it's enabled */ + if (filtering_support->outer) + vlan = &vvfl_v2->filters[i].outer; + else + vlan = &vvfl_v2->filters[i].inner; + + vlan->tci = f->vlan.vid; + vlan->tpid = f->vlan.tpid; + + i++; + f->add = false; + if (i == count) + break; + } + } + + if (!more) + adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER; + + spin_unlock_bh(&adapter->mac_vlan_list_lock); + + iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN_V2, + (u8 *)vvfl_v2, len); + kfree(vvfl_v2); } - if (!more) - adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_VLAN_FILTER; - - spin_unlock_bh(&adapter->mac_vlan_list_lock); - - iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN, (u8 *)vvfl, len); - kfree(vvfl); } /** @@ -696,7 +957,6 @@ void iavf_add_vlans(struct iavf_adapter *adapter) **/ void iavf_del_vlans(struct iavf_adapter *adapter) { - struct virtchnl_vlan_filter_list *vvfl; struct iavf_vlan_filter *f, *ftmp; int len, i = 0, count = 0; bool more = false; @@ -719,96 +979,177 @@ void iavf_del_vlans(struct iavf_adapter *adapter) spin_unlock_bh(&adapter->mac_vlan_list_lock); return; } - adapter->current_op = VIRTCHNL_OP_DEL_VLAN; - len = sizeof(struct virtchnl_vlan_filter_list) + - (count * sizeof(u16)); - if (len > IAVF_MAX_AQ_BUF_SIZE) { - dev_warn(&adapter->pdev->dev, "Too many delete VLAN changes in one request\n"); - count = (IAVF_MAX_AQ_BUF_SIZE - - sizeof(struct virtchnl_vlan_filter_list)) / - sizeof(u16); - len = sizeof(struct virtchnl_vlan_filter_list) + - (count * sizeof(u16)); - more = true; - } - vvfl = kzalloc(len, GFP_ATOMIC); - if (!vvfl) { - spin_unlock_bh(&adapter->mac_vlan_list_lock); - return; - } + if (VLAN_ALLOWED(adapter)) { + struct virtchnl_vlan_filter_list *vvfl; - vvfl->vsi_id = adapter->vsi_res->vsi_id; - vvfl->num_elements = count; - list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { - if (f->remove) { - vvfl->vlan_id[i] = f->vlan; - i++; - list_del(&f->list); - kfree(f); - if (i == count) - break; + adapter->current_op = VIRTCHNL_OP_DEL_VLAN; + + len = sizeof(*vvfl) + (count * sizeof(u16)); + if (len > IAVF_MAX_AQ_BUF_SIZE) { + dev_warn(&adapter->pdev->dev, "Too many delete VLAN changes in one request\n"); + count = (IAVF_MAX_AQ_BUF_SIZE - sizeof(*vvfl)) / + sizeof(u16); + len = sizeof(*vvfl) + (count * sizeof(u16)); + more = true; } + vvfl = kzalloc(len, GFP_ATOMIC); + if (!vvfl) { + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return; + } + + vvfl->vsi_id = adapter->vsi_res->vsi_id; + vvfl->num_elements = count; + list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { + if (f->remove) { + vvfl->vlan_id[i] = f->vlan.vid; + i++; + list_del(&f->list); + kfree(f); + if (i == count) + break; + } + } + + if (!more) + adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER; + + spin_unlock_bh(&adapter->mac_vlan_list_lock); + + iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN, (u8 *)vvfl, len); + kfree(vvfl); + } else if (VLAN_V2_ALLOWED(adapter)) { + struct virtchnl_vlan_filter_list_v2 *vvfl_v2; + + adapter->current_op = VIRTCHNL_OP_DEL_VLAN_V2; + + len = sizeof(*vvfl_v2) + + ((count - 1) * sizeof(struct virtchnl_vlan_filter)); + if (len > IAVF_MAX_AQ_BUF_SIZE) { + dev_warn(&adapter->pdev->dev, "Too many add VLAN changes in one request\n"); + count = (IAVF_MAX_AQ_BUF_SIZE - + sizeof(*vvfl_v2)) / + sizeof(struct virtchnl_vlan_filter); + len = sizeof(*vvfl_v2) + + ((count - 1) * + sizeof(struct virtchnl_vlan_filter)); + more = true; + } + + vvfl_v2 = kzalloc(len, GFP_ATOMIC); + if (!vvfl_v2) { + spin_unlock_bh(&adapter->mac_vlan_list_lock); + return; + } + + vvfl_v2->vport_id = adapter->vsi_res->vsi_id; + vvfl_v2->num_elements = count; + list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { + if (f->remove) { + struct virtchnl_vlan_supported_caps *filtering_support = + &adapter->vlan_v2_caps.filtering.filtering_support; + struct virtchnl_vlan *vlan; + + /* give priority over outer if it's enabled */ + if (filtering_support->outer) + vlan = &vvfl_v2->filters[i].outer; + else + vlan = &vvfl_v2->filters[i].inner; + + vlan->tci = f->vlan.vid; + vlan->tpid = f->vlan.tpid; + + list_del(&f->list); + kfree(f); + i++; + if (i == count) + break; + } + } + + if (!more) + adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER; + + spin_unlock_bh(&adapter->mac_vlan_list_lock); + + iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN_V2, + (u8 *)vvfl_v2, len); + kfree(vvfl_v2); } - if (!more) - adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_VLAN_FILTER; - - spin_unlock_bh(&adapter->mac_vlan_list_lock); - - iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN, (u8 *)vvfl, len); - kfree(vvfl); } /** * iavf_set_promiscuous * @adapter: adapter structure - * @flags: bitmask to control unicast/multicast promiscuous. * * Request that the PF enable promiscuous mode for our VSI. **/ -void iavf_set_promiscuous(struct iavf_adapter *adapter, int flags) +void iavf_set_promiscuous(struct iavf_adapter *adapter) { + struct net_device *netdev = adapter->netdev; struct virtchnl_promisc_info vpi; - int promisc_all; + unsigned int flags; if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { /* bail because we already have a command pending */ - dev_err(&adapter->pdev->dev, "Cannot set promiscuous mode, command %d pending\n", + dev_err(&adapter->pdev->dev, + "Cannot set promiscuous mode, command %d pending\n", adapter->current_op); return; } - promisc_all = FLAG_VF_UNICAST_PROMISC | - FLAG_VF_MULTICAST_PROMISC; - if ((flags & promisc_all) == promisc_all) { - adapter->flags |= IAVF_FLAG_PROMISC_ON; - adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_PROMISC; + /* prevent changes to promiscuous flags */ + spin_lock_bh(&adapter->current_netdev_promisc_flags_lock); + + /* sanity check to prevent duplicate AQ calls */ + if (!iavf_promiscuous_mode_changed(adapter)) { + adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE; + dev_dbg(&adapter->pdev->dev, "No change in promiscuous mode\n"); + /* allow changes to promiscuous flags */ + spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock); + return; + } + + /* there are 2 bits, but only 3 states */ + if (!(netdev->flags & IFF_PROMISC) && + netdev->flags & IFF_ALLMULTI) { + /* State 1 - only multicast promiscuous mode enabled + * - !IFF_PROMISC && IFF_ALLMULTI + */ + flags = FLAG_VF_MULTICAST_PROMISC; + adapter->current_netdev_promisc_flags |= IFF_ALLMULTI; + adapter->current_netdev_promisc_flags &= ~IFF_PROMISC; + dev_info(&adapter->pdev->dev, + "Entering multicast promiscuous mode\n"); + } else if (!(netdev->flags & IFF_PROMISC) && + !(netdev->flags & IFF_ALLMULTI)) { + /* State 2 - unicast/multicast promiscuous mode disabled + * - !IFF_PROMISC && !IFF_ALLMULTI + */ + flags = 0; + adapter->current_netdev_promisc_flags &= + ~(IFF_PROMISC | IFF_ALLMULTI); + dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n"); + } else { + /* State 3 - unicast/multicast promiscuous mode enabled + * - IFF_PROMISC && IFF_ALLMULTI + * - IFF_PROMISC && !IFF_ALLMULTI + */ + flags = FLAG_VF_UNICAST_PROMISC | FLAG_VF_MULTICAST_PROMISC; + adapter->current_netdev_promisc_flags |= IFF_PROMISC; + if (netdev->flags & IFF_ALLMULTI) + adapter->current_netdev_promisc_flags |= IFF_ALLMULTI; + else + adapter->current_netdev_promisc_flags &= ~IFF_ALLMULTI; + dev_info(&adapter->pdev->dev, "Entering promiscuous mode\n"); } - if (flags & FLAG_VF_MULTICAST_PROMISC) { - adapter->flags |= IAVF_FLAG_ALLMULTI_ON; - adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_ALLMULTI; - dev_info(&adapter->pdev->dev, - "%s is entering multicast promiscuous mode\n", - adapter->netdev->name); - } + adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE; - if (!flags) { - if (adapter->flags & IAVF_FLAG_PROMISC_ON) { - adapter->flags &= ~IAVF_FLAG_PROMISC_ON; - adapter->aq_required &= ~IAVF_FLAG_AQ_RELEASE_PROMISC; - dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n"); - } - - if (adapter->flags & IAVF_FLAG_ALLMULTI_ON) { - adapter->flags &= ~IAVF_FLAG_ALLMULTI_ON; - adapter->aq_required &= ~IAVF_FLAG_AQ_RELEASE_ALLMULTI; - dev_info(&adapter->pdev->dev, - "%s is leaving multicast promiscuous mode\n", - adapter->netdev->name); - } - } + /* allow changes to promiscuous flags */ + spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock); adapter->current_op = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; vpi.vsi_id = adapter->vsi_res->vsi_id; @@ -831,6 +1172,8 @@ void iavf_request_stats(struct iavf_adapter *adapter) /* no error message, this isn't crucial */ return; } + + adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_STATS; adapter->current_op = VIRTCHNL_OP_GET_STATS; vqs.vsi_id = adapter->vsi_res->vsi_id; /* queue maps are ignored for this message - only the vsi is used */ @@ -983,6 +1326,266 @@ void iavf_disable_vlan_stripping(struct iavf_adapter *adapter) iavf_send_pf_msg(adapter, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, NULL, 0); } +/** + * iavf_tpid_to_vc_ethertype - transform from VLAN TPID to virtchnl ethertype + * @tpid: VLAN TPID (i.e. 0x8100, 0x88a8, etc.) + */ +static u32 iavf_tpid_to_vc_ethertype(u16 tpid) +{ + switch (tpid) { + case ETH_P_8021Q: + return VIRTCHNL_VLAN_ETHERTYPE_8100; + case ETH_P_8021AD: + return VIRTCHNL_VLAN_ETHERTYPE_88A8; + } + + return 0; +} + +/** + * iavf_set_vc_offload_ethertype - set virtchnl ethertype for offload message + * @adapter: adapter structure + * @msg: message structure used for updating offloads over virtchnl to update + * @tpid: VLAN TPID (i.e. 0x8100, 0x88a8, etc.) + * @offload_op: opcode used to determine which support structure to check + */ +static int +iavf_set_vc_offload_ethertype(struct iavf_adapter *adapter, + struct virtchnl_vlan_setting *msg, u16 tpid, + enum virtchnl_ops offload_op) +{ + struct virtchnl_vlan_supported_caps *offload_support; + u32 vc_ethertype = iavf_tpid_to_vc_ethertype(tpid); + + /* reference the correct offload support structure */ + switch (offload_op) { + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2: + /* fall-through */ + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2: + offload_support = + &adapter->vlan_v2_caps.offloads.stripping_support; + break; + case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2: + /* fall-through */ + case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2: + offload_support = + &adapter->vlan_v2_caps.offloads.insertion_support; + break; + default: + dev_err(&adapter->pdev->dev, "Invalid opcode %d for setting virtchnl ethertype to enable/disable VLAN offloads\n", + offload_op); + return -EINVAL; + } + + /* make sure ethertype is supported */ + if ((offload_support->outer & vc_ethertype) && + (offload_support->outer & VIRTCHNL_VLAN_TOGGLE)) { + msg->outer_ethertype_setting = vc_ethertype; + } else if ((offload_support->inner & vc_ethertype) && + (offload_support->inner & VIRTCHNL_VLAN_TOGGLE)) { + msg->inner_ethertype_setting = vc_ethertype; + } else { + dev_dbg(&adapter->pdev->dev, "opcode %d unsupported for VLAN TPID 0x%04x\n", + offload_op, tpid); + return -EINVAL; + } + + return 0; +} + +/** + * iavf_clear_offload_v2_aq_required - clear AQ required bit for offload request + * @adapter: adapter structure + * @tpid: VLAN TPID + * @offload_op: opcode used to determine which AQ required bit to clear + */ +static void +iavf_clear_offload_v2_aq_required(struct iavf_adapter *adapter, u16 tpid, + enum virtchnl_ops offload_op) +{ + switch (offload_op) { + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2: + if (tpid == ETH_P_8021Q) + adapter->aq_required &= + ~IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING; + else if (tpid == ETH_P_8021AD) + adapter->aq_required &= + ~IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING; + break; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2: + if (tpid == ETH_P_8021Q) + adapter->aq_required &= + ~IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING; + else if (tpid == ETH_P_8021AD) + adapter->aq_required &= + ~IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING; + break; + case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2: + if (tpid == ETH_P_8021Q) + adapter->aq_required &= + ~IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION; + else if (tpid == ETH_P_8021AD) + adapter->aq_required &= + ~IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION; + break; + case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2: + if (tpid == ETH_P_8021Q) + adapter->aq_required &= + ~IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION; + else if (tpid == ETH_P_8021AD) + adapter->aq_required &= + ~IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION; + break; + default: + dev_err(&adapter->pdev->dev, "Unsupported opcode %d specified for clearing aq_required bits for VIRTCHNL_VF_OFFLOAD_VLAN_V2 offload request\n", + offload_op); + } +} + +/** + * iavf_send_vlan_offload_v2 - send offload enable/disable over virtchnl + * @adapter: adapter structure + * @tpid: VLAN TPID used for the command (i.e. 0x8100 or 0x88a8) + * @offload_op: offload_op used to make the request over virtchnl + */ +static void +iavf_send_vlan_offload_v2(struct iavf_adapter *adapter, u16 tpid, + enum virtchnl_ops offload_op) +{ + struct virtchnl_vlan_setting *msg; + int len = sizeof(*msg); + + if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { + /* bail because we already have a command pending */ + dev_err(&adapter->pdev->dev, "Cannot send %d, command %d pending\n", + offload_op, adapter->current_op); + return; + } + + adapter->current_op = offload_op; + + msg = kzalloc(len, GFP_KERNEL); + if (!msg) + return; + + msg->vport_id = adapter->vsi_res->vsi_id; + + /* always clear to prevent unsupported and endless requests */ + iavf_clear_offload_v2_aq_required(adapter, tpid, offload_op); + + /* only send valid offload requests */ + if (!iavf_set_vc_offload_ethertype(adapter, msg, tpid, offload_op)) + iavf_send_pf_msg(adapter, offload_op, (u8 *)msg, len); + else + /* since the current_op assigned in this function was never sent + * there will never be a completion to clear it, so do that now + * to allow other opcodes + */ + adapter->current_op = VIRTCHNL_OP_UNKNOWN; + + kfree(msg); +} + +/** + * iavf_enable_vlan_stripping_v2 - enable VLAN stripping + * @adapter: adapter structure + * @tpid: VLAN TPID used to enable VLAN stripping + */ +void iavf_enable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid) +{ + iavf_send_vlan_offload_v2(adapter, tpid, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2); +} + +/** + * iavf_disable_vlan_stripping_v2 - disable VLAN stripping + * @adapter: adapter structure + * @tpid: VLAN TPID used to disable VLAN stripping + */ +void iavf_disable_vlan_stripping_v2(struct iavf_adapter *adapter, u16 tpid) +{ + iavf_send_vlan_offload_v2(adapter, tpid, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2); +} + +/** + * iavf_enable_vlan_insertion_v2 - enable VLAN insertion + * @adapter: adapter structure + * @tpid: VLAN TPID used to enable VLAN insertion + */ +void iavf_enable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid) +{ + iavf_send_vlan_offload_v2(adapter, tpid, + VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2); +} + +/** + * iavf_disable_vlan_insertion_v2 - disable VLAN insertion + * @adapter: adapter structure + * @tpid: VLAN TPID used to disable VLAN insertion + */ +void iavf_disable_vlan_insertion_v2(struct iavf_adapter *adapter, u16 tpid) +{ + iavf_send_vlan_offload_v2(adapter, tpid, + VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2); +} + +/** + * iavf_virtchnl_send_ptp_cmd - Send one queued PTP command + * @adapter: adapter private structure + * + * De-queue one PTP command request and send the command message to the PF. + * Clear IAVF_FLAG_AQ_SEND_PTP_CMD if no more messages are left to send. + */ +void iavf_virtchnl_send_ptp_cmd(struct iavf_adapter *adapter) +{ + struct device *dev = &adapter->pdev->dev; + struct iavf_ptp_aq_cmd *cmd; + int err; + + if (WARN_ON(!adapter->ptp.initialized)) { + /* This shouldn't be possible to hit, since no messages should + * be queued if PTP is not initialized. + */ + adapter->aq_required &= ~IAVF_FLAG_AQ_SEND_PTP_CMD; + return; + } + + spin_lock(&adapter->ptp.aq_cmd_lock); + cmd = list_first_entry_or_null(&adapter->ptp.aq_cmds, struct iavf_ptp_aq_cmd, list); + if (!cmd) { + /* no further PTP messages to send */ + adapter->aq_required &= ~IAVF_FLAG_AQ_SEND_PTP_CMD; + goto out_unlock; + } + + if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { + /* bail because we already have a command pending */ + dev_err(dev, "Cannot send PTP command %d, command %d pending\n", + cmd->v_opcode, adapter->current_op); + goto out_unlock; + } + + err = iavf_send_pf_msg(adapter, cmd->v_opcode, cmd->msg, cmd->msglen); + if (!err) { + /* Command was sent without errors, so we can remove it from + * the list and discard it. + */ + list_del(&cmd->list); + kfree(cmd); + } else { + /* We failed to send the command, try again next cycle */ + dev_warn(dev, "Failed to send PTP command %d\n", cmd->v_opcode); + } + + if (list_empty(&adapter->ptp.aq_cmds)) + /* no further PTP messages to send */ + adapter->aq_required &= ~IAVF_FLAG_AQ_SEND_PTP_CMD; + +out_unlock: + spin_unlock(&adapter->ptp.aq_cmd_lock); +} + #define IAVF_MAX_SPEED_STRLEN 13 /** @@ -1104,7 +1707,7 @@ iavf_set_adapter_link_speed_from_vpe(struct iavf_adapter *adapter, #endif /* VIRTCHNL_VF_CAP_ADV_LINK_SPEED */ /** - * iavf_enable_channel + * iavf_enable_channels * @adapter: adapter structure * * Request that the PF enable channels as specified by @@ -1147,7 +1750,7 @@ void iavf_enable_channels(struct iavf_adapter *adapter) } /** - * iavf_disable_channel + * iavf_disable_channels * @adapter: adapter structure * * Request that the PF disable channels that are configured @@ -1233,7 +1836,8 @@ void iavf_add_cloud_filter(struct iavf_adapter *adapter) { struct iavf_cloud_filter *cf; struct virtchnl_filter *f; - int len = 0, count = 0; + bool process_fltr = false; + int len = 0; if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { /* bail because we already have a command pending */ @@ -1241,32 +1845,46 @@ void iavf_add_cloud_filter(struct iavf_adapter *adapter) adapter->current_op); return; } - list_for_each_entry(cf, &adapter->cloud_filter_list, list) { - if (cf->add) { - count++; - break; - } - } - if (!count) { - adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_CLOUD_FILTER; - return; - } - adapter->current_op = VIRTCHNL_OP_ADD_CLOUD_FILTER; len = sizeof(struct virtchnl_filter); f = kzalloc(len, GFP_KERNEL); if (!f) return; + /* Only add a single cloud filter per call to iavf_add_cloud_filter(), + * the aq_required IAVF_FLAG_AQ_ADD_CLOUD_FILTER bit will be set until + * no filters are left to add + */ + spin_lock_bh(&adapter->cloud_filter_list_lock); list_for_each_entry(cf, &adapter->cloud_filter_list, list) { if (cf->add) { - *f = cf->f; + process_fltr = true; cf->add = false; cf->state = __IAVF_CF_ADD_PENDING; - iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_CLOUD_FILTER, - (u8 *)f, len); + *f = cf->f; + /* must to store channel ptr in cloud filter if action + * is TC_REDIRECT since it is used later + */ + if (f->action == VIRTCHNL_ACTION_TC_REDIRECT) { + u32 tc = f->action_meta; + + cf->ch = &adapter->ch_config.ch_ex_info[tc]; + } + break; } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); + + if (!process_fltr) { + /* prevent iavf_add_cloud_filter() from being called when there + * are no filters to add + */ + adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_CLOUD_FILTER; + kfree(f); + return; + } + adapter->current_op = VIRTCHNL_OP_ADD_CLOUD_FILTER; + iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_CLOUD_FILTER, (u8 *)f, len); kfree(f); } @@ -1279,9 +1897,10 @@ void iavf_add_cloud_filter(struct iavf_adapter *adapter) **/ void iavf_del_cloud_filter(struct iavf_adapter *adapter) { - struct iavf_cloud_filter *cf, *cftmp; + struct iavf_cloud_filter *cf; struct virtchnl_filter *f; - int len = 0, count = 0; + bool process_fltr = false; + int len = 0; if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { /* bail because we already have a command pending */ @@ -1289,32 +1908,37 @@ void iavf_del_cloud_filter(struct iavf_adapter *adapter) adapter->current_op); return; } - list_for_each_entry(cf, &adapter->cloud_filter_list, list) { - if (cf->del) { - count++; - break; - } - } - if (!count) { - adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_CLOUD_FILTER; - return; - } - adapter->current_op = VIRTCHNL_OP_DEL_CLOUD_FILTER; - len = sizeof(struct virtchnl_filter); f = kzalloc(len, GFP_KERNEL); if (!f) return; - list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { + /* Only delete a single cloud filter per call to iavf_del_cloud_filter() + * the aq_required IAVF_FLAG_AQ_DEL_CLOUD_FILTER bit will be set until + * no filters are left to delete + */ + spin_lock_bh(&adapter->cloud_filter_list_lock); + list_for_each_entry(cf, &adapter->cloud_filter_list, list) { if (cf->del) { + process_fltr = true; *f = cf->f; cf->del = false; cf->state = __IAVF_CF_DEL_PENDING; - iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_CLOUD_FILTER, - (u8 *)f, len); + break; } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); + + if (!process_fltr) { + /* prevent iavf_del_cloud_filter() from being called when there + * are no filters to delete + */ + adapter->aq_required &= ~IAVF_FLAG_AQ_DEL_CLOUD_FILTER; + kfree(f); + return; + } + adapter->current_op = VIRTCHNL_OP_DEL_CLOUD_FILTER; + iavf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_CLOUD_FILTER, (u8 *)f, len); kfree(f); } @@ -1333,6 +1957,337 @@ int iavf_request_reset(struct iavf_adapter *adapter) return status; } +/** + * iavf_clear_chnl_ring_attr - clears rings attributes specific to channel + * @adapter: adapter structure + * @ring: Pointer to ring (Tx/Rx) + * @tx: TRUE means Tx and FALSE means Rx + * + * This function clears up ring attributes such as feature flag (optimization + * enabled or not, also resets vector feature flags associated with queue) + **/ +static void iavf_clear_chnl_ring_attr(struct iavf_adapter *adapter, + struct iavf_ring *ring, + bool tx) +{ + struct iavf_q_vector *qv = ring->q_vector; + + ring->ch = NULL; + ring->chnl_flags &= ~IAVF_RING_CHNL_PERF_ENA; + dev_dbg(&adapter->pdev->dev, + "%s_ring %u, ch_ena: %u, perf_ena: %u\n", + tx ? "Tx" : "Rx", ring->queue_index, ring_ch_ena(ring), + ring_ch_perf_ena(ring)); + + if (!qv) + return; + + qv->ch = NULL; + + /* revive the vector from ADQ state machine + * by triggering SW interrupt + */ + iavf_force_wb(&adapter->vsi, qv); + qv->chnl_flags &= ~IAVF_VECTOR_CHNL_PERF_ENA; + dev_dbg(&adapter->pdev->dev, + "vector(idx: %u): ch_ena: %u, perf_ena: %u\n", + qv->v_idx, vector_ch_ena(qv), vector_ch_perf_ena(qv)); +} + +/** + * iavf_clear_ch_info - clears channel specific information and flags_ + * @adapter: adapter structure + * + * This function clears channel specific configurations, flags for + * Tx, Rx queues, related vectors and triggers software interrupt + * to revive the ADQ specific vectors, so that vector is put back in + * interrupt state + **/ +static void iavf_clear_ch_info(struct iavf_adapter *adapter) +{ + int tc, q; + + /* to avoid running iAVF on older HW, do not want to support + * ADQ related performance bits, hence checking the ADQ_V2 as + * run-time type and prevent if ADQ_V2 is not set. + */ + if (!iavf_is_adq_v2_enabled(adapter)) + return; + + for (tc = 0; tc < VIRTCHNL_MAX_ADQ_V2_CHANNELS; tc++) { + struct iavf_channel_ex *ch; + int num_rxq; + + ch = &adapter->ch_config.ch_ex_info[tc]; + if (!ch) + continue; + + /* unlikely but make sure to have non-zero "num_rxq" for + * channel otherwise skip.. + */ + num_rxq = ch->num_rxq; + if (!num_rxq) + continue; + + /* proceed only when there is no active filter + * for given channel + */ + if (ch->num_fltr) + continue; + + /* do not proceed unless we have vectors >= num_active_queues. + * In future, this is subject to change if interrupt to queue + * assignment policy changesm but for now - expect as many + * vectors as data_queues + */ + if (adapter->num_msix_vectors <= adapter->num_active_queues) + continue; + + for (q = 0; q < num_rxq; q++) { + struct iavf_ring *tx_ring, *rx_ring; + + tx_ring = &adapter->tx_rings[ch->base_q + q]; + rx_ring = &adapter->rx_rings[ch->base_q + q]; + if (tx_ring) + iavf_clear_chnl_ring_attr(adapter, tx_ring, + true); + if (rx_ring) + iavf_clear_chnl_ring_attr(adapter, rx_ring, + false); + } + } +} + +/** + * iavf_set_chnl_ring_attr - sets rings attributes specific to channel + * @adapter: adapter structure + * @flags: adapter specific flags (various feature bits) + * @ring: Pointer to ring (Tx/Rx) + * @ch: Pointer to channel + * @tx: TRUE means Tx and FALSE means Rx + * + * This function sets up ring attributes such as feature flag (optimization + * enabled or not, also sets up vector feature flags associated with queue) + **/ +static void iavf_set_chnl_ring_attr(struct iavf_adapter *adapter, u32 flags, + struct iavf_ring *ring, + struct iavf_channel_ex *ch, + bool tx) +{ + struct iavf_q_vector *qv = ring->q_vector; + + ring->ch = ch; + ring->chnl_flags |= IAVF_RING_CHNL_PERF_ENA; + dev_dbg(&adapter->pdev->dev, "%s_ring %u, ch_ena: %u, perf_ena: %u\n", + tx ? "Tx" : "Rx", ring->queue_index, ring_ch_ena(ring), + ring_ch_perf_ena(ring)); + + if (!qv) + return; + + qv->ch = ch; + qv->chnl_flags |= IAVF_VECTOR_CHNL_PERF_ENA; + if (flags & IAVF_FLAG_CHNL_PKT_OPT_ENA) + qv->chnl_flags |= IAVF_VECTOR_CHNL_PKT_OPT_ENA; + else + qv->chnl_flags &= ~IAVF_VECTOR_CHNL_PKT_OPT_ENA; + dev_dbg(&adapter->pdev->dev, + "vector(idx %u): ch_ena: %u, perf_ena: %u\n", + qv->v_idx, vector_ch_ena(qv), vector_ch_perf_ena(qv)); +} + +/** + * iavf_setup_ch_info - sets channel specific information and flags + * @adapter: adapter structure + * @flags: adapter specific flags (various feature bits) + * + * This function sets up queues (Tx and Rx) and vector specific flags + * as appliable for ADQ. This function is invoked as soon as filters + * were added successfully, so that queues and vectors are setup to engage + * for optimized packets processing using ADQ state machine based logic. + **/ +void iavf_setup_ch_info(struct iavf_adapter *adapter, u32 flags) +{ + int tc; + + /* to avoid running iAVF on older HW, do not want to support + * ADQ related performance bits, hence checking the ADQ_V2 as + * run-time type and prevent if ADQ_V2 is not set. + */ + if (!iavf_is_adq_v2_enabled(adapter)) + return; + + for (tc = 0; tc < VIRTCHNL_MAX_ADQ_V2_CHANNELS; tc++) { + struct iavf_channel_ex *ch; + int num_rxq, q; + + ch = &adapter->ch_config.ch_ex_info[tc]; + if (!ch) + continue; + + /* unlikely but make sure to have non-zero "num_rxq" for + * channel otherwise skip.. + */ + num_rxq = ch->num_rxq; + if (!num_rxq) + continue; + + /* do not proceed unless there is at least one filter + * for given channel + */ + if (!ch->num_fltr) + continue; + + /* do not proceed unless we have vectors >= num_active_queues. + * In future, this is subject to change if interrupt to queue + * assignment policy changesm but for now - expect as many + * vectors as data_queues + */ + if (adapter->num_msix_vectors <= adapter->num_active_queues) + continue; + + for (q = 0; q < num_rxq; q++) { + struct iavf_ring *tx_ring, *rx_ring; + + tx_ring = &adapter->tx_rings[ch->base_q + q]; + rx_ring = &adapter->rx_rings[ch->base_q + q]; + if (tx_ring) + iavf_set_chnl_ring_attr(adapter, flags, + tx_ring, ch, true); + if (rx_ring) + iavf_set_chnl_ring_attr(adapter, flags, + rx_ring, ch, false); + } + } +} + +/** + * iavf_netdev_features_vlan_strip_set + * @netdev: ptr to netdev being adjusted + * @enable: enable or disable vlan strip + * + * Helper function to change vlan strip status in netdev->features. + **/ +static void iavf_netdev_features_vlan_strip_set(struct net_device *netdev, + const bool enable) +{ + if (enable) +#ifdef NETIF_F_HW_VLAN_CTAG_RX + netdev->features |= NETIF_F_HW_VLAN_CTAG_RX; +#else + netdev->features |= NETIF_F_HW_VLAN_RX; +#endif /* NETIF_F_HW_VLAN_CTAG_RX */ + else +#ifdef NETIF_F_HW_VLAN_CTAG_RX + netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; +#else + netdev->features &= ~NETIF_F_HW_VLAN_RX; +#endif /* NETIF_F_HW_VLAN_CTAG_RX */ +} + +/** + * iavf_virtchnl_ptp_get_time - Respond to VIRTCHNL_OP_1588_PTP_GET_TIME + * @adapter: private adapter structure + * @data: the message from the PF + * @len: length of the message from the PF + * + * Handle the VIRTCHNL_OP_1588_PTP_GET_TIME message from the PF. This message + * is sent by the PF in response to the same op as a request from the VF. + * Extract the 64bit nanoseconds time from the message and store it in + * cached_phc_time. Then, notify any thread that is waiting for the update via + * the wait queue. + */ +static void iavf_virtchnl_ptp_get_time(struct iavf_adapter *adapter, void *data, u16 len) +{ + struct virtchnl_phc_time *msg; + + if (len == sizeof(*msg)) { + msg = (struct virtchnl_phc_time *)data; + } else { + dev_err_once(&adapter->pdev->dev, "Invalid VIRTCHNL_OP_1588_PTP_GET_TIME from PF. Got size %u, expected %lu\n", + len, sizeof(*msg)); + return; + } + + adapter->ptp.cached_phc_time = msg->time; + adapter->ptp.cached_phc_updated = jiffies; + adapter->ptp.phc_time_ready = true; + + wake_up(&adapter->ptp.phc_time_waitqueue); +} + +/** + * iavf_virtchnl_ptp_tx_timestamp - Handle Tx timestamp events from the PF + * @adapter: private adapter structure + * @data: message contents from PF + * @len: length of the message from the PF + * + * Handle the VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP op from the PF. This is sent + * whenever the PF has detected a transmit timestamp associated with this VF. + * + * First, check if there is a pending skb that needs a transmit timestamp. If + * so, extract the time value from the message and report it to the stack. + * Note that 40bit timestamp values must first be extended using + * iavf_ptp_extend_40b_timestamp(). + */ +static void iavf_virtchnl_ptp_tx_timestamp(struct iavf_adapter *adapter, void *data, u16 len) +{ + struct skb_shared_hwtstamps skb_tstamps = {}; + struct device *dev = &adapter->pdev->dev; + struct virtchnl_phc_tx_tstamp *msg; + struct sk_buff *skb; + u64 ns; + + if (len == sizeof(*msg)) { + msg = (struct virtchnl_phc_tx_tstamp *)data; + } else { + dev_err_once(dev, "Invalid VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP from PF. Got size %u, expected %lu\n", + len, sizeof(*msg)); + return; + } + + /* No need to process the event if timestamping isn't on */ + if (adapter->ptp.hwtstamp_config.tx_type != HWTSTAMP_TX_ON) + return; + + /* don't attempt to timestamp if we don't have a pending skb */ + skb = adapter->ptp.tx_skb; + if (!skb) + return; + + /* Since we only request one outstanding timestamp at once, we assume + * this event must belong to the saved SKB. Clear the bit lock and the + * skb now prior to notifying the stack via skb_tstamp_tx(). + */ + adapter->ptp.tx_skb = NULL; + clear_bit_unlock(__IAVF_TX_TSTAMP_IN_PROGRESS, &adapter->crit_section); + + switch (adapter->ptp.hw_caps.tx_tstamp_format) { + case VIRTCHNL_1588_PTP_TSTAMP_40BIT: + if (!(msg->tstamp & IAVF_PTP_40B_TSTAMP_VALID)) { + dev_warn(dev, "Got a VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP message with an invalid timestamp\n"); + goto out_free_skb; + } + ns = iavf_ptp_extend_40b_timestamp(adapter->ptp.cached_phc_time, msg->tstamp); + break; + case VIRTCHNL_1588_PTP_TSTAMP_64BIT_NS: + ns = msg->tstamp; + break; + default: + /* This shouldn't happen since we won't enable Tx timestamps + * if we don't know the timestamp format. + */ + dev_dbg(dev, "Got a VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP event, when timestamp format is unknown\n"); + goto out_free_skb; + } + + skb_tstamps.hwtstamp = ns_to_ktime(ns); + skb_tstamp_tx(skb, &skb_tstamps); + +out_free_skb: + dev_kfree_skb_any(skb); +} + /** * iavf_virtchnl_completion * @adapter: adapter structure @@ -1401,12 +2356,15 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, netif_tx_start_all_queues(netdev); netif_carrier_on(netdev); } + if (!ether_addr_equal(netdev->dev_addr, + adapter->hw.mac.addr)) + iavf_replace_primary_mac + (adapter, netdev->dev_addr); } else { netif_tx_stop_all_queues(netdev); netif_carrier_off(netdev); } iavf_print_link_message(adapter); - adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; break; case VIRTCHNL_EVENT_RESET_IMPENDING: dev_info(&adapter->pdev->dev, "Reset indication received from the PF\n"); @@ -1430,12 +2388,17 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, if (ADQ_V2_ALLOWED(adapter) && !v_retval && (v_opcode == VIRTCHNL_OP_ENABLE_CHANNELS || v_opcode == VIRTCHNL_OP_DISABLE_CHANNELS)) { - struct virtchnl_tc_info *ch_info = - (struct virtchnl_tc_info *)msg; - if (ch_info->num_tc) { - dev_info(&adapter->pdev->dev, "Scheduling reset\n"); - iavf_schedule_reset(adapter); - } + adapter->flags |= IAVF_FLAG_REINIT_CHNL_NEEDED; + dev_info(&adapter->pdev->dev, + "Scheduling reset due to %s retval %d\n", + v_opcode == VIRTCHNL_OP_ENABLE_CHANNELS ? + "VIRTCHNL_OP_ENABLE_CHANNELS" : + "VIRTCHNL_OP_DISABLE_CHANNELS", v_retval); + /* schedule reset always if processing ENABLE/DISABLE_CHANNEL + * ops so that as part of reset handling, appropriate steps are + * taken such as num_tc, per TC queue_map, etc... + */ + iavf_schedule_reset(adapter); } if (v_retval) { @@ -1449,7 +2412,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, iavf_stat_str(&adapter->hw, v_retval)); iavf_mac_add_reject(adapter); /* restore administratively set mac address */ - ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); + ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); break; case VIRTCHNL_OP_DEL_VLAN: dev_err(&adapter->pdev->dev, "Failed to delete VLAN filter, error %s\n", @@ -1477,6 +2440,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, case VIRTCHNL_OP_ADD_CLOUD_FILTER: { struct iavf_cloud_filter *cf, *cftmp; + spin_lock_bh(&adapter->cloud_filter_list_lock); list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { @@ -1495,11 +2459,13 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, adapter->num_cloud_filters--; } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); } break; case VIRTCHNL_OP_DEL_CLOUD_FILTER: { struct iavf_cloud_filter *cf; + spin_lock_bh(&adapter->cloud_filter_list_lock); list_for_each_entry(cf, &adapter->cloud_filter_list, list) { if (cf->state == __IAVF_CF_DEL_PENDING) { @@ -1511,12 +2477,52 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, &cf->f); } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); } break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + dev_warn(&adapter->pdev->dev, + "Changing VLAN Stripping is not allowed when Port VLAN is configured\n"); + /* + * Vlan stripping could not be enabled by ethtool. + * Disable it in netdev->features. + */ + iavf_netdev_features_vlan_strip_set(netdev, false); + break; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + dev_warn(&adapter->pdev->dev, + "Changing VLAN Stripping is not allowed when Port VLAN is configured\n"); + /* + * Vlan stripping could not be disabled by ethtool. + * Enable it in netdev->features. + */ + iavf_netdev_features_vlan_strip_set(netdev, true); + break; default: dev_err(&adapter->pdev->dev, "PF returned error %d (%s) to our request %d\n", v_retval, iavf_stat_str(&adapter->hw, v_retval), v_opcode); + + /* Assume that the ADQ configuration caused one of the + * v_opcodes in this if statement to fail. Set the + * flag so the reset path can return to the pre-ADQ + * configuration and traffic can resume + */ + if (iavf_is_adq_enabled(adapter) && + (v_opcode == VIRTCHNL_OP_ENABLE_QUEUES || + v_opcode == VIRTCHNL_OP_CONFIG_IRQ_MAP || + v_opcode == VIRTCHNL_OP_CONFIG_VSI_QUEUES)) { + dev_err(&adapter->pdev->dev, + "ADQ is enabled and opcode %d failed (%d)\n", + v_opcode, v_retval); + adapter->ch_config.state = __IAVF_TC_INVALID; + adapter->num_tc = 0; + netdev_reset_tc(netdev); + adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; + iavf_schedule_reset(adapter); + adapter->current_op = VIRTCHNL_OP_UNKNOWN; + return; + } } } switch (v_opcode) { @@ -1524,7 +2530,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, if (!v_retval) iavf_mac_add_ok(adapter); if (!ether_addr_equal(netdev->dev_addr, adapter->hw.mac.addr)) - ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); + ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); break; case VIRTCHNL_OP_GET_STATS: { struct iavf_eth_stats *stats = @@ -1544,10 +2550,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, } break; case VIRTCHNL_OP_GET_VF_RESOURCES: { - struct iavf_vlan_filter *vlf; - struct iavf_cloud_filter *cf; - struct iavf_mac_filter *f; - bool was_mac_changed; u16 len = sizeof(struct virtchnl_vf_resource) + IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource); @@ -1563,8 +2565,49 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); } + + iavf_parse_vf_resource_msg(adapter); + + /* negotiated VIRTCHNL_VF_OFFLOAD_VLAN_V2, so wait for the + * response to VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS to finish + * configuration + */ + if (VLAN_V2_ALLOWED(adapter)) + break; + /* fall-through and finish config if VIRTCHNL_VF_OFFLOAD_VLAN_V2 + * wasn't successfully negotiated with the PF + */ + } + /* fall-through */ + case VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS: { + struct iavf_mac_filter *f; + bool was_mac_changed; + + if (v_opcode == VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS) + memcpy(&adapter->vlan_v2_caps, msg, + min_t(u16, msglen, + sizeof(adapter->vlan_v2_caps))); + iavf_process_config(adapter); + /* Clear 'critical task' bit before acquiring rtnl_lock + * as other process holding rtnl_lock could be waiting + * for the same bit resulting in deadlock + */ + clear_bit(__IAVF_IN_CRITICAL_TASK, &adapter->crit_section); + /* VLAN capabilities can change during VFR, so make sure to + * update the netdev features with the new capabilities + */ + rtnl_lock(); + netdev_update_features(netdev); + rtnl_unlock(); + /* Set 'critical task' bit again */ + while (test_and_set_bit(__IAVF_IN_CRITICAL_TASK, + &adapter->crit_section)) + usleep_range(500, 1000); + + iavf_set_queue_vlan_tag_loc(adapter); + was_mac_changed = !ether_addr_equal(netdev->dev_addr, adapter->hw.mac.addr); @@ -1577,35 +2620,66 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, ether_addr_copy(f->macaddr, adapter->hw.mac.addr); + f->is_new_mac = true; f->add = true; f->remove = false; } /* re-add all VLAN filters */ - list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { - vlf->add = true; + if (VLAN_FILTERING_ALLOWED(adapter)) { + struct iavf_vlan_filter *vlf; + + list_for_each_entry(vlf, &adapter->vlan_filter_list, + list) + vlf->add = true; } spin_unlock_bh(&adapter->mac_vlan_list_lock); - /* check if TCs are running and re-add all cloud filters */ - spin_lock_bh(&adapter->cloud_filter_list_lock); - if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && - adapter->num_tc) { - list_for_each_entry(cf, &adapter->cloud_filter_list, - list) { - cf->add = true; + /* check if TCs are running and re-add all cloud filters + * Set ADD_CLOUD_FILTER only if list is not empty so that + * re-add of filters can happen correctly + */ + if (iavf_is_adq_enabled(adapter) || + iavf_is_adq_v2_enabled(adapter)) { + struct iavf_cloud_filter *cf; + + spin_lock_bh(&adapter->cloud_filter_list_lock); + if (!list_empty(&adapter->cloud_filter_list)) { + list_for_each_entry(cf, + &adapter->cloud_filter_list, + list) { + cf->add = true; + } + adapter->aq_required |= + IAVF_FLAG_AQ_ADD_CLOUD_FILTER; } + spin_unlock_bh(&adapter->cloud_filter_list_lock); } - spin_unlock_bh(&adapter->cloud_filter_list_lock); ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; - adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; } break; + case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS: + memcpy(&adapter->supported_rxdids, msg, + min_t(u16, msglen, + sizeof(adapter->supported_rxdids))); + break; + case VIRTCHNL_OP_1588_PTP_GET_CAPS: + memcpy(&adapter->ptp.hw_caps, msg, + min_t(u16, msglen, sizeof(adapter->ptp.hw_caps))); + /* process any state change needed due to new capabilities */ + iavf_ptp_process_caps(adapter); + break; + case VIRTCHNL_OP_1588_PTP_GET_TIME: + iavf_virtchnl_ptp_get_time(adapter, msg, msglen); + break; + case VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP: + iavf_virtchnl_ptp_tx_timestamp(adapter, msg, msglen); + break; case VIRTCHNL_OP_ENABLE_QUEUES: /* enable transmits */ if (adapter->state == __IAVF_RUNNING) { @@ -1640,18 +2714,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, if (v_opcode != adapter->current_op) return; break; - case VIRTCHNL_OP_IWARP: - /* Gobble zero-length replies from the PF. They indicate that - * a previous message was received OK, and the client doesn't - * care about that. - */ - if (msglen && CLIENT_ENABLED(adapter)) - iavf_notify_client_message(&adapter->vsi, msg, msglen); - break; - case VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP: - adapter->client_pending &= - ~(BIT(VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP)); - break; case VIRTCHNL_OP_GET_RSS_HENA_CAPS: { struct virtchnl_rss_hena *vrh = (struct virtchnl_rss_hena *)msg; if (msglen == sizeof(*vrh)) @@ -1677,32 +2739,66 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, case VIRTCHNL_OP_ADD_CLOUD_FILTER: { struct iavf_cloud_filter *cf; + spin_lock_bh(&adapter->cloud_filter_list_lock); list_for_each_entry(cf, &adapter->cloud_filter_list, list) { - if (cf->state == __IAVF_CF_ADD_PENDING) + if (cf->state == __IAVF_CF_ADD_PENDING) { cf->state = __IAVF_CF_ACTIVE; + if (cf->ch) + cf->ch->num_fltr++; + } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); if (!v_retval) dev_info(&adapter->pdev->dev, - "Cloud filters have been added\n"); + "Cloud filters are added\n"); + /* if not done, set channel specific attribute + * such as "is it ADQ enabled", "queues are ADD ena", + * "vectors are ADQ ena" or not + */ + iavf_setup_ch_info(adapter, adapter->flags); } break; case VIRTCHNL_OP_DEL_CLOUD_FILTER: { struct iavf_cloud_filter *cf, *cftmp; + spin_lock_bh(&adapter->cloud_filter_list_lock); list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { if (cf->state == __IAVF_CF_DEL_PENDING) { cf->state = __IAVF_CF_INVALID; list_del(&cf->list); + if (cf->ch) + cf->ch->num_fltr--; kfree(cf); adapter->num_cloud_filters--; } } + spin_unlock_bh(&adapter->cloud_filter_list_lock); if (!v_retval) dev_info(&adapter->pdev->dev, - "Cloud filters have been deleted\n"); + "Cloud filters are deleted\n"); + /* if active ADQ filters for channels reached zero, + * put the rings, vectors back in non-ADQ state + */ + iavf_clear_ch_info(adapter); } break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + /* + * Got information that PF enabled vlan strip on this VF. + * Update netdev->features if needed to be in sync with ethtool. + */ + if (!v_retval) + iavf_netdev_features_vlan_strip_set(netdev, true); + break; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + /* + * Got information that PF disabled vlan strip on this VF. + * Update netdev->features if needed to be in sync with ethtool. + */ + if (!v_retval) + iavf_netdev_features_vlan_strip_set(netdev, false); + break; default: if (adapter->current_op && (v_opcode != adapter->current_op)) dev_dbg(&adapter->pdev->dev, "Expected response %d from PF, received %d\n", diff --git a/drivers/net/ethernet/intel/iavf/kcompat.c b/drivers/net/ethernet/intel/iavf/kcompat.c index c116eec4b..96e5282f4 100644 --- a/drivers/net/ethernet/intel/iavf/kcompat.c +++ b/drivers/net/ethernet/intel/iavf/kcompat.c @@ -1475,7 +1475,6 @@ static const unsigned char __maybe_unused pcie_link_speed[] = { int __kc_pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed, enum pcie_link_width *width) { - int ret; *speed = PCI_SPEED_UNKNOWN; *width = PCIE_LNK_WIDTH_UNKNOWN; @@ -1484,8 +1483,8 @@ int __kc_pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed, u16 lnksta; enum pci_bus_speed next_speed; enum pcie_link_width next_width; + int ret = pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); - ret = pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); if (ret) return ret; @@ -1541,6 +1540,50 @@ int __kc_dma_set_mask_and_coherent(struct device *dev, u64 mask) err = dma_set_coherent_mask(dev, mask); return err; } + +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,0)) +static bool _kc_pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, + u32 *l, int crs_timeout) +{ + int delay = 1; + + if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l)) + return false; + + /* some broken boards return 0 or ~0 if a slot is empty: */ + if (*l == 0xffffffff || *l == 0x00000000 || + *l == 0x0000ffff || *l == 0xffff0000) + return false; + + /* Configuration request Retry Status */ + while (*l == 0xffff0001) { + if (!crs_timeout) + return false; + + msleep(delay); + delay *= 2; + if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l)) + return false; + /* Card hasn't responded in 60 seconds? Must be stuck. */ + if (delay > crs_timeout) { + printk(KERN_WARNING "pci %04x:%02x:%02x.%d: not " + "responding\n", pci_domain_nr(bus), + bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn)); + return false; + } + } + + return true; +} + +bool _kc_pci_device_is_present(struct pci_dev *pdev) +{ + u32 v; + + return _kc_pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); +} +#endif /* nexthdr; - unsigned int len; bool found; #define __KC_IP6_FH_F_FRAG BIT(0) @@ -1574,7 +1616,6 @@ int __kc_ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, start = *offset + sizeof(struct ipv6hdr); nexthdr = ip6->nexthdr; } - len = skb->len - start; do { struct ipv6_opt_hdr _hdr, *hp; @@ -1639,7 +1680,6 @@ int __kc_ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, if (!found) { nexthdr = hp->nexthdr; - len -= hdrlen; start += hdrlen; } } while (!found); @@ -2111,6 +2151,100 @@ unsigned int _kc_cpumask_local_spread(unsigned int i, int node) #endif #endif +/******************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0)) +#if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,4)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,2,0))) +/** + * _kc_skb_flow_dissect_flow_keys - parse SKB to fill _kc_flow_keys + * @skb: SKB used to fille _kc_flow_keys + * @flow: _kc_flow_keys to set with SKB fields + * @flags: currently unused flags + * + * The purpose of using kcompat for this function is so the caller doesn't have + * to care about which kernel version they are on, which prevents a larger than + * normal #ifdef mess created by using a HAVE_* flag for this case. This is also + * done for 4.2 kernels to simplify calling skb_flow_dissect_flow_keys() + * because in 4.2 kernels skb_flow_dissect_flow_keys() exists, but only has 2 + * arguments. Recent kernels have skb_flow_dissect_flow_keys() that has 3 + * arguments. + * + * The caller needs to understand that this function was only implemented as a + * bare-minimum replacement for recent versions of skb_flow_dissect_flow_keys() + * and this function is in no way similar to skb_flow_dissect_flow_keys(). An + * example use can be found in the ice driver, specifically ice_arfs.c. + * + * This function is treated as a whitelist of supported fields the SKB can + * parse. If new functionality is added make sure to keep this format (i.e. only + * check for fields that are explicity wanted). + * + * Current whitelist: + * + * TCPv4, TCPv6, UDPv4, UDPv6 + * + * If any unexpected protocol or other field is found this function memsets the + * flow passed in back to 0 and returns false. Otherwise the flow is populated + * and returns true. + */ +bool +_kc_skb_flow_dissect_flow_keys(const struct sk_buff *skb, + struct _kc_flow_keys *flow, + unsigned int __always_unused flags) +{ + memset(flow, 0, sizeof(*flow)); + + flow->basic.n_proto = skb->protocol; + switch (flow->basic.n_proto) { + case htons(ETH_P_IP): + flow->basic.ip_proto = ip_hdr(skb)->protocol; + flow->addrs.v4addrs.src = ip_hdr(skb)->saddr; + flow->addrs.v4addrs.dst = ip_hdr(skb)->daddr; + break; + case htons(ETH_P_IPV6): + flow->basic.ip_proto = ipv6_hdr(skb)->nexthdr; + memcpy(&flow->addrs.v6addrs.src, &ipv6_hdr(skb)->saddr, + sizeof(struct in6_addr)); + memcpy(&flow->addrs.v6addrs.dst, &ipv6_hdr(skb)->daddr, + sizeof(struct in6_addr)); + break; + default: + netdev_dbg(skb->dev, "%s: Unsupported/unimplemented layer 3 protocol %04x\n", __func__, htons(flow->basic.n_proto)); + goto unsupported; + } + + switch (flow->basic.ip_proto) { + case IPPROTO_TCP: + { + struct tcphdr *tcph; + + tcph = tcp_hdr(skb); + flow->ports.src = tcph->source; + flow->ports.dst = tcph->dest; + break; + } + case IPPROTO_UDP: + { + struct udphdr *udph; + + udph = udp_hdr(skb); + flow->ports.src = udph->source; + flow->ports.dst = udph->dest; + break; + } + default: + netdev_dbg(skb->dev, "%s: Unsupported/unimplemented layer 4 protocol %02x\n", __func__, flow->basic.ip_proto); + return false; + } + + return true; + +unsupported: + memset(flow, 0, sizeof(*flow)); + return false; +} +#endif /* ! >= RHEL7.4 && ! >= SLES12.2 */ +#endif /* 4.3.0 */ + /******************************************************************************/ #if ( LINUX_VERSION_CODE < KERNEL_VERSION(4,5,0) ) #if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,3))) @@ -2220,6 +2354,38 @@ void _kc_ethtool_intersect_link_masks(struct ethtool_link_ksettings *dst, } #endif /* 4.15.0 */ +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0)) +#if !(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,5,0) && \ + SLE_VERSION_CODE < SLE_VERSION(15,0,0) || \ + SLE_VERSION_CODE >= SLE_VERSION(15,1,0)) +#if BITS_PER_LONG == 64 +/** + * bitmap_from_arr32 - copy the contents of u32 array of bits to bitmap + * @bitmap: array of unsigned longs, the destination bitmap + * @buf: array of u32 (in host byte order), the source bitmap + * @nbits: number of bits in @bitmap + */ +void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits) +{ + unsigned int i, halfwords; + + halfwords = DIV_ROUND_UP(nbits, 32); + for (i = 0; i < halfwords; i++) { + bitmap[i/2] = (unsigned long) buf[i]; + if (++i < halfwords) + bitmap[i/2] |= ((unsigned long) buf[i]) << 32; + } + + /* Clear tail bits in last word beyond nbits. */ + if (nbits % BITS_PER_LONG) + bitmap[(halfwords - 1) / 2] &= BITMAP_LAST_WORD_MASK(nbits); +} +#endif /* BITS_PER_LONG == 64 */ +#endif /* !(RHEL >= 8.0) && !(SLES >= 12.5 && SLES < 15.0 || SLES >= 15.1) */ +#endif /* 4.16.0 */ + /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,17,0)) /* PCIe link information */ @@ -2369,10 +2535,7 @@ void _kc_pcie_print_link_status(struct pci_dev *dev) { #endif /* 4.17.0 */ /*****************************************************************************/ -#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,1,0)) -#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,1))) -#define HAVE_NDO_FDB_ADD_EXTACK -#else /* !RHEL || RHEL < 8.1 */ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,1,0)) || (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,1))) #ifdef HAVE_TC_SETUP_CLSFLOWER #define FLOW_DISSECTOR_MATCH(__rule, __type, __out) \ const struct flow_match *__m = &(__rule)->match; \ @@ -2457,8 +2620,7 @@ void flow_rule_match_ports(const struct flow_rule *rule, FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_PORTS, out); } #endif /* HAVE_TC_SETUP_CLSFLOWER */ -#endif /* !RHEL || RHEL < 8.1 */ -#endif /* 5.1.0 */ +#endif /* 5.1.0 || (RHEL && RHEL < 8.1) */ /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0)) @@ -2493,3 +2655,30 @@ int _kc_flow_block_cb_setup_simple(struct flow_block_offload *f, #endif /* HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO */ #endif /* !RHEL >= 8.2 */ #endif /* 5.3.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,7,0)) +u64 _kc_pci_get_dsn(struct pci_dev *dev) +{ + u32 dword; + u64 dsn; + int pos; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); + if (!pos) + return 0; + + /* + * The Device Serial Number is two dwords offset 4 bytes from the + * capability position. The specification says that the first dword is + * the lower half, and the second dword is the upper half. + */ + pos += 4; + pci_read_config_dword(dev, pos, &dword); + dsn = (u64)dword; + pci_read_config_dword(dev, pos + 4, &dword); + dsn |= ((u64)dword) << 32; + + return dsn; +} +#endif /* 5.7.0 */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat.h b/drivers/net/ethernet/intel/iavf/kcompat.h index b37a3f3ac..25d4f0f35 100644 --- a/drivers/net/ethernet/intel/iavf/kcompat.h +++ b/drivers/net/ethernet/intel/iavf/kcompat.h @@ -244,7 +244,7 @@ struct msix_entry { #endif #ifndef WARN_ON -#define WARN_ON(x) +#define WARN_ON(x) ({0;}) #endif #ifndef PCI_DEVICE @@ -257,10 +257,6 @@ struct msix_entry { #define node_online(node) ((node) == 0) #endif -#ifndef cpu_online -#define cpu_online(cpuid) test_bit((cpuid), &cpu_online_map) -#endif - #ifndef _LINUX_RANDOM_H #include #endif @@ -315,6 +311,9 @@ struct msix_entry { #define __ARG_PLACEHOLDER_1 0, #define config_enabled(cfg) _config_enabled(cfg) +#ifdef __CHECKER__ +/* cppcheck-suppress preprocessorErrorDirective */ +#endif /* __CHECKER__ */ #define _config_enabled(value) __config_enabled(__ARG_PLACEHOLDER_##value) #define __config_enabled(arg1_or_junk) ___config_enabled(arg1_or_junk 1, 0) #define ___config_enabled(__ignored, val, ...) val @@ -802,13 +801,16 @@ struct _kc_ethtool_pauseparam { * ignore differences in sublevel which are not important since we have the * ABI value. Otherwise, it becomes impossible to correlate ABI to version for * ordering checks. + * + * This also lets us store an ABI value up to 65535, since it can take the + * space that would use the lower byte of the Linux version code. */ #define UBUNTU_VERSION_CODE (((~0xFF & LINUX_VERSION_CODE) << 8) + \ UTS_UBUNTU_RELEASE_ABI) -#if UTS_UBUNTU_RELEASE_ABI > 255 -#error UTS_UBUNTU_RELEASE_ABI is too large... -#endif /* UTS_UBUNTU_RELEASE_ABI > 255 */ +#if UTS_UBUNTU_RELEASE_ABI > 65535 +#error UTS_UBUNTU_RELEASE_ABI is larger than 65535... +#endif /* UTS_UBUNTU_RELEASE_ABI > 65535 */ #if ( LINUX_VERSION_CODE <= KERNEL_VERSION(3,0,0) ) /* Our version code scheme does not make sense for non 3.x or newer kernels, @@ -817,22 +819,46 @@ struct _kc_ethtool_pauseparam { */ #define UTS_UBUNTU_RELEASE_ABI 0 #define UBUNTU_VERSION_CODE 0 -#endif +#endif /* <= 3.0.0 */ +#endif /* !UTS_UBUNTU_RELEASE_ABI */ -#endif - -/* Note that the 3rd digit is always zero, and will be ignored. This is - * because Ubuntu kernels are based on x.y.0-ABI values, and while their linux - * version codes are 3 digit, this 3rd digit is superseded by the ABI value. +/* We ignore the 3rd digit since we want to give precedence to the additional + * ABI value provided by Ubuntu. */ -#define UBUNTU_VERSION(a,b,c,d) ((KERNEL_VERSION(a,b,0) << 8) + (d)) +#define UBUNTU_VERSION(a,b,c,d) (((a) << 24) + ((b) << 16) + (d)) + +/* SLE_VERSION is used to generate a 3-digit encoding that can order SLE + * kernels based on their major release, service pack, and a possible + * maintenance release. + */ +#define SLE_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c)) + +/* The SLE_LOCALVERSION_CODE comes from a 3-digit code added as part of the + * Linux kernel version. It is extracted by the driver Makefile. This macro is + * used to generate codes for making comparisons below. + */ +#define SLE_LOCALVERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c)) -/* SuSE version macros are the same as Linux kernel version macro */ -#ifndef SLE_VERSION -#define SLE_VERSION(a,b,c) KERNEL_VERSION(a,b,c) -#endif -#define SLE_LOCALVERSION(a,b,c) KERNEL_VERSION(a,b,c) #ifdef CONFIG_SUSE_KERNEL +/* Starting since at least SLE 12sp4 and SLE 15, the SUSE kernels have + * provided CONFIG_SUSE_VERSION, CONFIG_SUSE_PATCHLEVEL and + * CONFIG_SUSE_AUXRELEASE. Use these to generate SLE_VERSION if available. + * Only fall back to the manual table otherwise. We expect all future versions + * of SLE kernels to include these values, so the table will remain only for + * the older releases. + */ +#ifdef CONFIG_SUSE_VERSION +#ifndef CONFIG_SUSE_PATCHLEVEL +#error "CONFIG_SUSE_VERSION exists but CONFIG_SUSE_PATCHLEVEL is missing" +#endif +#ifndef CONFIG_SUSE_AUXRELEASE +#error "CONFIG_SUSE_VERSION exists but CONFIG_SUSE_AUXRELEASE is missing" +#endif +#define SLE_VERSION_CODE SLE_VERSION(CONFIG_SUSE_VERSION, CONFIG_SUSE_PATCHLEVEL, CONFIG_SUSE_AUXRELEASE) +#else +/* If we do not have the CONFIG_SUSE_VERSION configuration values, fall back + * to the following table for older releases. + */ #if ( LINUX_VERSION_CODE == KERNEL_VERSION(2,6,27) ) /* SLES11 GA is 2.6.27 based */ #define SLE_VERSION_CODE SLE_VERSION(11,0,0) @@ -891,38 +917,10 @@ struct _kc_ethtool_pauseparam { * - 4.4.103-6.33.1, 4.4.103-6.38.1 * - 4.4.{114,120}-94.nn.y */ #define SLE_VERSION_CODE SLE_VERSION(12,3,0) -#elif (LINUX_VERSION_CODE == KERNEL_VERSION(4,12,14) && \ - (SLE_LOCALVERSION_CODE == KERNEL_VERSION(94,41,0) || \ - (SLE_LOCALVERSION_CODE >= KERNEL_VERSION(95,0,0) && \ - SLE_LOCALVERSION_CODE < KERNEL_VERSION(96,0,0)))) -/* SLES12 SP4 GM is 4.12.14-94.41 and update kernel is 4.12.14-95.x. */ -#define SLE_VERSION_CODE SLE_VERSION(12,4,0) -#elif (LINUX_VERSION_CODE == KERNEL_VERSION(4,12,14) && \ - (SLE_LOCALVERSION_CODE == KERNEL_VERSION(23,0,0) || \ - SLE_LOCALVERSION_CODE == KERNEL_VERSION(2,0,0) || \ - SLE_LOCALVERSION_CODE == KERNEL_VERSION(136,0,0) || \ - (SLE_LOCALVERSION_CODE >= KERNEL_VERSION(25,0,0) && \ - SLE_LOCALVERSION_CODE < KERNEL_VERSION(26,0,0)) || \ - (SLE_LOCALVERSION_CODE >= KERNEL_VERSION(150,0,0) && \ - SLE_LOCALVERSION_CODE < KERNEL_VERSION(151,0,0)))) -/* SLES15 Beta1 is 4.12.14-2 - * SLES15 GM is 4.12.14-23 and update kernel is 4.12.14-{25,136}, - * and 4.12.14-150.14. - */ -#define SLE_VERSION_CODE SLE_VERSION(15,0,0) -#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,14) && \ - SLE_LOCALVERSION_CODE >= KERNEL_VERSION(25,23,0)) -/* SLES15 SP1 Beta1 is 4.12.14-25.23 */ -#define SLE_VERSION_CODE SLE_VERSION(15,1,0) -#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(5,3,13)) -/* SLES15 SP2 Beta1 is 5.3.13 */ -#define SLE_VERSION_CODE SLE_VERSION(15,2,0) - -/* new SLES kernels must be added here with >= based on kernel - * the idea is to order from newest to oldest and just catch all - * of them using the >= - */ +#else +#error "This looks like a SUSE kernel, but it has an unrecognized local version code." #endif /* LINUX_VERSION_CODE == KERNEL_VERSION(x,y,z) */ +#endif /* !CONFIG_SUSE_VERSION */ #endif /* CONFIG_SUSE_KERNEL */ #ifndef SLE_VERSION_CODE #define SLE_VERSION_CODE 0 @@ -931,6 +929,40 @@ struct _kc_ethtool_pauseparam { #define SLE_LOCALVERSION_CODE 0 #endif /* SLE_LOCALVERSION_CODE */ +/* + * Include the definitions file for HAVE/NEED flags for the standard upstream + * kernels. + * + * Then, based on the distribution we detect, load the distribution specific + * definitions file that customizes the definitions for the target + * distribution. + */ +#include "kcompat_std_defs.h" + +#ifdef CONFIG_SUSE_KERNEL +#include "kcompat_sles_defs.h" +#elif UBUNTU_VERSION_CODE +#include "kcompat_ubuntu_defs.h" +#elif RHEL_RELEASE_CODE +#include "kcompat_rhel_defs.h" +#endif + +/* + * ADQ depends on __TC_MQPRIO_MODE_MAX and related kernel code + * added around 4.15. Some distributions (e.g. Oracle Linux 7.7) + * have done a partial back-port of that to their kernels based + * on older mainline kernels that did not include all the necessary + * kernel enablement to support ADQ. + * Undefine __TC_MQPRIO_MODE_MAX for all OSV distributions with + * kernels based on mainline kernels older than 4.15 except for + * RHEL, SLES and Ubuntu which are known to have good back-ports. + */ +#if (!RHEL_RELEASE_CODE && !SLE_VERSION_CODE && !UBUNTU_VERSION_CODE) + #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) + #undef __TC_MQPRIO_MODE_MAX + #endif /* LINUX_VERSION_CODE == KERNEL_VERSION(4,15,0) */ +#endif /* if (NOT RHEL && NOT SLES && NOT UBUNTU) */ + #ifdef __KLOCWORK__ /* The following are not compiled into the binary driver; they are here * only to tune Klocwork scans to workaround false-positive issues. @@ -993,6 +1025,41 @@ static inline int _kc_test_and_set_bit(int nr, volatile unsigned long *addr) #undef uninitialized_var #define uninitialized_var(x) x = *(&(x)) #endif + +#ifdef WRITE_ONCE +#undef WRITE_ONCE +#define WRITE_ONCE(x, val) ((x) = (val)) +#endif /* WRITE_ONCE */ + +#ifdef wait_event_interruptible_timeout +#undef wait_event_interruptible_timeout +#define wait_event_interruptible_timeout(wq_head, condition, timeout) ({ \ + long ret; \ + if ((condition)) \ + ret = timeout; \ + else \ + ret = 0; \ + ret; \ +}) +#endif /* wait_event_interruptible_timeout */ + +#ifdef max_t +#undef max_t +#define max_t(type, x, y) ({ \ +type __x = (x); \ +type __y = (y); \ +__x > __y ? __x : __y; \ +}) +#endif /* max_t */ + +#ifdef min_t +#undef min_t +#define min_t(type, x, y) ({ \ +type __x = (x); \ +type __y = (y); \ +__x < __y ? __x : __y; \ +}) +#endif /* min_t */ #endif /* __KLOCWORK__ */ /* Older versions of GCC will trigger -Wformat-nonliteral warnings for const @@ -1018,7 +1085,9 @@ static inline int _kc_test_and_set_bit(int nr, volatile unsigned long *addr) #endif /* GCC_VERSION < 5.1.0 */ /* Newer kernels removed */ -#if ( LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) ) +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)) && \ + (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3)) && \ + !(SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(15,3,0))))) #define HAVE_PCI_ASPM_H #endif @@ -4218,7 +4287,7 @@ of_get_mac_address(struct device_node __always_unused *np) #define eth_broadcast_addr _kc_eth_broadcast_addr static inline void _kc_eth_broadcast_addr(u8 *addr) { - eth_broadcast_addr(addr); + memset(addr, 0xff, ETH_ALEN); } #endif @@ -4831,8 +4900,8 @@ of_get_mac_address(struct device_node __always_unused *np) #define HAVE_SKB_INNER_NETWORK_HEADER #if (RHEL_RELEASE_CODE && \ - (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,0)) && \ - (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0))) + (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,0))) +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0)) #define HAVE_RHEL7_PCI_DRIVER_RH #if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,2)) #define HAVE_RHEL7_PCI_RESET_NOTIFY @@ -4840,12 +4909,12 @@ of_get_mac_address(struct device_node __always_unused *np) #if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,3)) #if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,5)) #define HAVE_GENEVE_RX_OFFLOAD -#endif /* RHEL >=7.3 && RHEL < 7.5 */ +#endif /* RHEL < 7.5 */ #define HAVE_ETHTOOL_FLOW_UNION_IP6_SPEC #define HAVE_RHEL7_NET_DEVICE_OPS_EXT #if !defined(HAVE_UDP_ENC_TUNNEL) && IS_ENABLED(CONFIG_GENEVE) #define HAVE_UDP_ENC_TUNNEL -#endif +#endif /* !HAVE_UDP_ENC_TUNNEL && CONFIG_GENEVE */ #endif /* RHEL >= 7.3 */ /* new hooks added to net_device_ops_extended in RHEL7.4 */ @@ -4854,12 +4923,12 @@ of_get_mac_address(struct device_node __always_unused *np) #define HAVE_RHEL7_NETDEV_OPS_EXT_NDO_UDP_TUNNEL #define HAVE_UDP_ENC_RX_OFFLOAD #endif /* RHEL >= 7.4 */ -#endif /* RHEL >= 7.0 && RHEL < 8.0 */ - -#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0)) +#else /* RHEL >= 8.0 */ #define HAVE_TCF_BLOCK_CB_REGISTER_EXTACK #define NO_NETDEV_BPF_PROG_ATTACHED +#define HAVE_NDO_SELECT_QUEUE_SB_DEV #endif /* RHEL >= 8.0 */ +#endif /* RHEL >= 7.0 */ #endif /* >= 3.10.0 */ /*****************************************************************************/ @@ -4947,13 +5016,17 @@ static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev) #if ( LINUX_VERSION_CODE > KERNEL_VERSION(2,6,20) ) #define devm_kcalloc(dev, cnt, size, flags) \ - devm_kzalloc(dev, cnt * size, flags) + devm_kzalloc(dev, (cnt) * (size), flags) #endif /* > 2.6.20 */ #if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,2))) #define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) #endif +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,0)) +bool _kc_pci_device_is_present(struct pci_dev *pdev); +#define pci_device_is_present _kc_pci_device_is_present +#endif /* = 3.13.0 */ #define HAVE_VXLAN_CHECKS #if (UBUNTU_VERSION_CODE && UBUNTU_VERSION_CODE >= UBUNTU_VERSION(3,13,0,24)) @@ -4975,6 +5048,10 @@ static inline struct pci_dev *pci_upstream_bridge(struct pci_dev *dev) #define U32_MAX ((u32)~0U) #endif +#ifndef U64_MAX +#define U64_MAX ((u64)~0ULL) +#endif + #if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,2))) #define dev_consume_skb_any(x) dev_kfree_skb_any(x) #define dev_consume_skb_irq(x) dev_kfree_skb_irq(x) @@ -5015,6 +5092,20 @@ static inline void __kc_skb_set_hash(struct sk_buff __maybe_unused *skb, #else /* RHEL_RELEASE_CODE >= 7.0 || SLE_VERSION_CODE >= 12.0 */ +#if ((RHEL_RELEASE_CODE && RHEL_RELEASE_CODE <= RHEL_RELEASE_VERSION(7,0)) ||\ + (SLE_VERSION_CODE && SLE_VERSION_CODE <= SLE_VERSION(12,1,0))) +/* GPLv2 code taken from 5.10-rc2 kernel source include/linux/pci.h, Copyright + * original authors. + */ +static inline int pci_enable_msix_exact(struct pci_dev *dev, + struct msix_entry *entries, int nvec) +{ + int rc = pci_enable_msix_range(dev, entries, nvec, nvec); + if (rc < 0) + return rc; + return 0; +} +#endif /* <=EL7.0 || <=SLES 12.1 */ #if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,5))) #ifndef HAVE_VXLAN_RX_OFFLOAD #define HAVE_VXLAN_RX_OFFLOAD @@ -5098,6 +5189,9 @@ static inline __u32 skb_get_hash_raw(const struct sk_buff *skb) /*****************************************************************************/ #if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,15,0) ) +#if ( LINUX_VERSION_CODE < KERNEL_VERSION(2,6,35) ) +#define HAVE_SKBUFF_RXHASH +#endif /* >= 2.6.35 */ #if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(7,1)) && \ !(UBUNTU_VERSION_CODE && UBUNTU_VERSION_CODE >= UBUNTU_VERSION(3,13,0,30))) #define u64_stats_fetch_begin_irq u64_stats_fetch_begin_bh @@ -5107,7 +5201,7 @@ static inline __u32 skb_get_hash_raw(const struct sk_buff *skb) char *_kc_devm_kstrdup(struct device *dev, const char *s, gfp_t gfp); #define devm_kstrdup(dev, s, gfp) _kc_devm_kstrdup(dev, s, gfp) -#else +#else /* >= 3.15.0 */ #define HAVE_NET_GET_RANDOM_ONCE #define HAVE_PTP_1588_CLOCK_PINS #define HAVE_NETDEV_PORT @@ -5275,7 +5369,9 @@ static inline void ktime_get_ts64(struct timespec64 *ts) #if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,5)) #endif /* RHEL_RELEASE_CODE < RHEL7.5 */ -#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,3)) +#if RHEL_RELEASE_CODE && \ + RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(6,3) && \ + RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,3) static inline u64 ktime_get_ns(void) { return ktime_to_ns(ktime_get()); @@ -5295,6 +5391,7 @@ static inline u64 ktime_get_boot_ns(void) #else #define HAVE_DCBNL_OPS_SETAPP_RETURN_INT #include +#define HAVE_RHASHTABLE #endif /* 3.17.0 */ /*****************************************************************************/ @@ -5537,6 +5634,8 @@ of_find_net_device_by_node(struct device_node __always_unused *np) #endif #if RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(7,2)) #define HAVE_NDO_BRIDGE_GETLINK_NLFLAGS +#define HAVE_RHEL7_EXTENDED_NDO_SET_TX_MAXRATE +#define HAVE_NDO_SET_TX_MAXRATE #endif #if !((RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(6,8) && RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,0)) && \ (RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(7,2)) && \ @@ -5544,7 +5643,43 @@ of_find_net_device_by_node(struct device_node __always_unused *np) unsigned int _kc_cpumask_local_spread(unsigned int i, int node); #define cpumask_local_spread _kc_cpumask_local_spread #endif +#ifdef HAVE_RHASHTABLE +#define rhashtable_loopup_fast(ht, key, params) \ + do { \ + (void)params; \ + rhashtable_lookup((ht), (key)); \ + } while (0) + +#if ( LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0) ) +#define rhashtable_insert_fast(ht, obj, params) \ + do { \ + (void)params; \ + rhashtable_insert((ht), (obj), GFP_KERNEL); \ + } while (0) + +#define rhashtable_remove_fast(ht, obj, params) \ + do { \ + (void)params; \ + rhashtable_remove((ht), (obj), GFP_KERNEL); \ + } while (0) + +#else /* >= 3,19,0 */ +#define rhashtable_insert_fast(ht, obj, params) \ + do { \ + (void)params; \ + rhashtable_insert((ht), (obj)); \ + } while (0) + +#define rhashtable_remove_fast(ht, obj, params) \ + do { \ + (void)params; \ + rhashtable_remove((ht), (obj)); \ + } while (0) + +#endif /* 3,19,0 */ +#endif /* HAVE_RHASHTABLE */ #else /* >= 4,1,0 */ +#define HAVE_NDO_GET_PHYS_PORT_NAME #define HAVE_PTP_CLOCK_INFO_GETTIME64 #define HAVE_NDO_BRIDGE_GETLINK_NLFLAGS #define HAVE_PASSTHRU_FEATURES_CHECK @@ -5603,6 +5738,9 @@ static inline bool pci_ari_enabled(struct pci_bus *bus) { return bus->self && bus->self->ari_enabled; } +#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,2)) +#define HAVE_VF_STATS +#endif /* (RHEL7.2+) */ #endif /* !(RHEL6.8+ || RHEL7.2+) */ #else static inline bool pci_ari_enabled(struct pci_bus *bus) @@ -5615,6 +5753,112 @@ static inline bool pci_ari_enabled(struct pci_bus *bus) #define HAVE_VF_STATS #endif /* 4.2.0 */ +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,3,0)) +#if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,4)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,2,0))) +/** + * _kc_flow_dissector_key_ipv4_addrs: + * @src: source ip address + * @dst: destination ip address + */ +struct _kc_flow_dissector_key_ipv4_addrs { + __be32 src; + __be32 dst; +}; + +/** + * _kc_flow_dissector_key_ipv6_addrs: + * @src: source ip address + * @dst: destination ip address + */ +struct _kc_flow_dissector_key_ipv6_addrs { + struct in6_addr src; + struct in6_addr dst; +}; + +/** + * _kc_flow_dissector_key_addrs: + * @v4addrs: IPv4 addresses + * @v6addrs: IPv6 addresses + */ +struct _kc_flow_dissector_key_addrs { + union { + struct _kc_flow_dissector_key_ipv4_addrs v4addrs; + struct _kc_flow_dissector_key_ipv6_addrs v6addrs; + }; +}; + +/** + * _kc_flow_dissector_key_tp_ports: + * @ports: port numbers of Transport header + * src: source port number + * dst: destination port number + */ +struct _kc_flow_dissector_key_ports { + union { + __be32 ports; + struct { + __be16 src; + __be16 dst; + }; + }; +}; + +/** + * _kc_flow_dissector_key_basic: + * @n_proto: Network header protocol (eg. IPv4/IPv6) + * @ip_proto: Transport header protocol (eg. TCP/UDP) + * @padding: padding for alignment + */ +struct _kc_flow_dissector_key_basic { + __be16 n_proto; + u8 ip_proto; + u8 padding; +}; + +struct _kc_flow_keys { + struct _kc_flow_dissector_key_basic basic; + struct _kc_flow_dissector_key_ports ports; + struct _kc_flow_dissector_key_addrs addrs; +}; + +/* These are all the include files for kernels inside this #ifdef block that + * have any reference to the in kernel definition of struct flow_keys. The + * reason for putting them here is to make 100% sure that these files do not get + * included after re-defining flow_keys to _kc_flow_keys. This is done to + * prevent any possible ABI issues that this structure re-definition could case. + */ +#if ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,3,0) && \ + LINUX_VERSION_CODE < KERNEL_VERSION(4,2,0)) || \ + RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(6,7) || \ + SLE_VERSION_CODE >= SLE_VERSION(11,4,0)) +#include +#endif /* (>= 3.3.0 && < 4.2.0) || >= RHEL 6.7 || >= SLE 11.4 */ +#if (LINUX_VERSION_CODE == KERNEL_VERSION(4,2,0)) +#include +#endif /* 4.2.0 */ +#include +#include +#include +#include + +#define flow_keys _kc_flow_keys +bool +_kc_skb_flow_dissect_flow_keys(const struct sk_buff *skb, + struct flow_keys *flow, + unsigned int __always_unused flags); +#define skb_flow_dissect_flow_keys _kc_skb_flow_dissect_flow_keys +#endif /* ! >= RHEL 7.4 && ! >= SLES 12.2 */ + +#if ((RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,3)) || \ + (SLE_VERSION_CODE >= SLE_VERSION(12,2,0))) +#include +#endif /* >= RHEL7.3 || >= SLE12sp2 */ +#else /* >= 4.3.0 */ +#include +#endif /* 4.3.0 */ + /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,4,0)) #if (RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,3)) @@ -5645,7 +5889,7 @@ static inline __u64 readq(const volatile void __iomem *addr) static inline void writeq(__u64 val, volatile void __iomem *addr) { writel(val, addr); - writel(val >> 32, addr + 4); + writel(val >> 32, (u8 *)addr + 4); } #define writeq writeq #endif @@ -5681,6 +5925,7 @@ int _kc_eth_platform_get_mac_address(struct device *dev __maybe_unused, #endif #endif /* < 4.8.0 */ #define HAVE_NETIF_NAPI_ADD_CALLS_NAPI_HASH_ADD +#define HAVE_NETDEV_UPPER_INFO #endif /* 4.5.0 */ /*****************************************************************************/ @@ -5756,11 +6001,6 @@ static inline void page_ref_inc(struct page *page) (RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,4))) #define HAVE_NETIF_TRANS_UPDATE #endif /* SLES12sp3+ || RHEL7.4+ */ -#if ((UBUNTU_VERSION_CODE >= UBUNTU_VERSION(4,4,0,21)) || \ - (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,4)) || \ - (SLE_VERSION_CODE >= SLE_VERSION(12,3,0))) -#define HAVE_DEVLINK_SUPPORT -#endif /* UBUNTU 4,4,0,21, RHEL 7.4, SLES12 SP3 */ #if ((RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,3)) ||\ (SLE_VERSION_CODE >= SLE_VERSION(12,3,0))) #define HAVE_ETHTOOL_25G_BITS @@ -5768,7 +6008,6 @@ static inline void page_ref_inc(struct page *page) #define HAVE_ETHTOOL_100G_BITS #endif /* RHEL7.3+ || SLES12sp3+ */ #else /* 4.7.0 */ -#define HAVE_DEVLINK_SUPPORT #define HAVE_NETIF_TRANS_UPDATE #define HAVE_ETHTOOL_CONVERT_U32_AND_LINK_MODE #define HAVE_ETHTOOL_25G_BITS @@ -5791,10 +6030,6 @@ struct udp_tunnel_info { }; #endif -#if (RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,5)) -#define HAVE_TCF_EXTS_TO_LIST -#endif - #if (UBUNTU_VERSION_CODE && UBUNTU_VERSION_CODE < UBUNTU_VERSION(4,8,0,0)) #define tc_no_actions(_exts) true #define tc_for_each_action(_a, _exts) while (0) @@ -5843,7 +6078,6 @@ pci_release_mem_regions(struct pci_dev *pdev) #endif /* RHEL7.4+ || SLES12sp3+ */ #else #define HAVE_UDP_ENC_RX_OFFLOAD -#define HAVE_TCF_EXTS_TO_LIST #define HAVE_ETHTOOL_NEW_50G_BITS #endif /* 4.8.0 */ @@ -5873,6 +6107,22 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask) dst[1] = mask >> 32; } #endif /* = RHEL_RELEASE_VERSION(7,4)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,3,0)) && \ + !(UBUNTU_VERSION_CODE >= UBUNTU_VERSION(4,13,0,16))) +static inline bool eth_type_vlan(__be16 ethertype) +{ + switch (ethertype) { + case htons(ETH_P_8021Q): +#ifdef ETH_P_8021AD + case htons(ETH_P_8021AD): +#endif + return true; + default: + return false; + } +} +#endif /* Linux < 4.9 || RHEL < 7.4 || SLES < 12.3 || Ubuntu < 4.3.0-16 */ #else /* >=4.9 */ #define HAVE_FLOW_DISSECTOR_KEY_VLAN_PRIO #define HAVE_ETHTOOL_NEW_1G_BITS @@ -5974,6 +6224,14 @@ static inline void __page_frag_cache_drain(struct page *page, #ifndef ETH_MIN_MTU #define ETH_MIN_MTU 68 #endif /* ETH_MIN_MTU */ + +/* If kernel is older than 4.10 but distro is RHEL >= 7.5 || SLES > 12SP4, + * it does have support for NAPI_STATE + */ +#if ((RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,5))) ||\ + (SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(12,4,0)))) +#define HAVE_NAPI_STATE_IN_BUSY_POLL +#endif /* RHEL >= 7.5 || SLES >=12.4 */ #else /* >= 4.10 */ #define HAVE_TC_FLOWER_ENC #define HAVE_NETDEVICE_MIN_MAX_MTU @@ -5989,6 +6247,7 @@ static inline void __page_frag_cache_drain(struct page *page, */ #define HAVE_NAPI_STATE_IN_BUSY_POLL #define HAVE_TCF_MIRRED_EGRESS_REDIRECT +#define HAVE_PTP_CLOCK_INFO_ADJFINE #endif /* 4.10.0 */ /*****************************************************************************/ @@ -6046,6 +6305,32 @@ static inline void _kc_dev_consume_skb_any(struct sk_buff *skb) #define HAVE_VM_OPS_FAULT_NO_VMA #endif /* 4.11.0 */ +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0)) +#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,7) && \ + RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0)) +/* The RHEL 7.7+ NL_SET_ERR_MSG_MOD triggers unused parameter warnings */ +#undef NL_SET_ERR_MSG_MOD +#endif +/* If kernel is older than 4.12 but distro is RHEL >= 7.5 || SLES > 12SP4, + * it does have support for MIN_NAPI_ID + */ +#if ((RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,5))) || \ + (SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(12,4,0)))) +#define HAVE_MIN_NAPI_ID +#endif /* RHEL >= 7.5 || SLES >= 12.4 */ +#ifndef NL_SET_ERR_MSG_MOD +#define NL_SET_ERR_MSG_MOD(extack, msg) \ + do { \ + uninitialized_var(extack); \ + pr_err(KBUILD_MODNAME ": " msg); \ + } while (0) +#endif /* !NL_SET_ERR_MSG_MOD */ +#else /* >= 4.12 */ +#define HAVE_NAPI_BUSY_LOOP +#define HAVE_MIN_NAPI_ID +#endif /* 4.12 */ + /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,13,0)) #if ((SLE_VERSION_CODE && (SLE_VERSION_CODE > SLE_VERSION(12,3,0))) || \ @@ -6056,7 +6341,28 @@ static inline void _kc_dev_consume_skb_any(struct sk_buff *skb) #if (SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(12,4,0))) #define HAVE_PCI_ERROR_HANDLER_RESET_PREPARE #endif /* SLES >= 12sp4 */ +#if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,5)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,4,0))) +#define UUID_SIZE 16 +typedef struct { + __u8 b[UUID_SIZE]; +} uuid_t; +#define UUID_INIT(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7) \ +((uuid_t) \ +{{ ((a) >> 24) & 0xff, ((a) >> 16) & 0xff, ((a) >> 8) & 0xff, (a) & 0xff, \ + ((b) >> 8) & 0xff, (b) & 0xff, \ + ((c) >> 8) & 0xff, (c) & 0xff, \ + (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }}) + +static inline bool uuid_equal(const uuid_t *u1, const uuid_t *u2) +{ + return memcmp(u1, u2, sizeof(uuid_t)) == 0; +} +#else +#define HAVE_METADATA_PORT_INFO +#endif /* !(RHEL >= 7.5) && !(SLES >= 12.4) */ #else /* > 4.13 */ +#define HAVE_METADATA_PORT_INFO #define HAVE_HWTSTAMP_FILTER_NTP_ALL #define HAVE_NDO_SETUP_TC_CHAIN_INDEX #define HAVE_PCI_ERROR_HANDLER_RESET_PREPARE @@ -6231,7 +6537,6 @@ const char *_kc_phy_speed_to_str(int speed); #define HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO #define HAVE_TCF_BLOCK #else /* RHEL >= 7.6 || SLES >= 15.1 */ -#define TC_SETUP_QDISC_MQPRIO TC_SETUP_MQPRIO #endif /* !(RHEL >= 7.6) && !(SLES >= 15.1) */ void _kc_ethtool_intersect_link_masks(struct ethtool_link_ksettings *dst, struct ethtool_link_ksettings *src); @@ -6245,6 +6550,19 @@ void _kc_ethtool_intersect_link_masks(struct ethtool_link_ksettings *dst, /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0)) +#if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,7)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,4,0) && \ + SLE_VERSION_CODE < SLE_VERSION(15,0,0)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(15,1,0))) +/* The return value of the strscpy() and strlcpy() functions is different. + * This could be potentially hazard for the future. + * To avoid this the void result is forced. + * So it is not possible use this function with the return value. + * Return value is required in kernel 4.3 through 4.15 + */ +#define strscpy(...) (void)(strlcpy(__VA_ARGS__)) +#endif /* !RHEL >= 7.7 && !SLES12sp4+ && !SLES15sp1+ */ + #define pci_printk(level, pdev, fmt, arg...) \ dev_printk(level, &(pdev)->dev, fmt, ##arg) #define pci_emerg(pdev, fmt, arg...) dev_emerg(&(pdev)->dev, fmt, ##arg) @@ -6281,26 +6599,36 @@ static inline unsigned long _kc_array_index_mask_nospec(unsigned long index, (typeof(_i)) (_i & _mask); \ }) #endif /* array_index_nospec */ -#if (!(RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,6))) && \ - !(SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(15,1,0)))) -#ifdef HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO -#include -static inline bool -tc_cls_can_offload_and_chain0(const struct net_device *dev, - struct tc_cls_common_offload *common) -{ - if (!tc_can_offload(dev)) - return false; - if (common->chain_index) - return false; - - return true; -} -#endif /* HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO */ -#endif /* !(RHEL >= 7.6) && !(SLES >= 15.1) */ #ifndef sizeof_field #define sizeof_field(TYPE, MEMBER) (sizeof((((TYPE *)0)->MEMBER))) #endif /* sizeof_field */ +#if !(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0)) && \ + !(SLE_VERSION_CODE >= SLE_VERSION(12,5,0) && \ + SLE_VERSION_CODE < SLE_VERSION(15,0,0) || \ + SLE_VERSION_CODE >= SLE_VERSION(15,1,0)) +/* + * Copy bitmap and clear tail bits in last word. + */ +static inline void +bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits) +{ + bitmap_copy(dst, src, nbits); + if (nbits % BITS_PER_LONG) + dst[nbits / BITS_PER_LONG] &= BITMAP_LAST_WORD_MASK(nbits); +} + +/* + * On 32-bit systems bitmaps are represented as u32 arrays internally, and + * therefore conversion is not needed when copying data from/to arrays of u32. + */ +#if BITS_PER_LONG == 64 +void bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits); +#else +#define bitmap_from_arr32(bitmap, buf, nbits) \ + bitmap_copy_clear_tail((unsigned long *) (bitmap), \ + (const unsigned long *) (buf), (nbits)) +#endif /* BITS_PER_LONG == 64 */ +#endif /* !(RHEL >= 8.0) && !(SLES >= 12.5 && SLES < 15.0 || SLES >= 15.1) */ #else /* >= 4.16 */ #include #define HAVE_XDP_BUFF_RXQ @@ -6325,48 +6653,11 @@ void _kc_pcie_print_link_status(struct pci_dev *dev); /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,18,0)) -#ifdef NETIF_F_HW_L2FW_DOFFLOAD -#include -#ifndef macvlan_supports_dest_filter -#define macvlan_supports_dest_filter _kc_macvlan_supports_dest_filter -static inline bool _kc_macvlan_supports_dest_filter(struct net_device *dev) -{ - struct macvlan_dev *macvlan = netdev_priv(dev); - - return macvlan->mode == MACVLAN_MODE_PRIVATE || - macvlan->mode == MACVLAN_MODE_VEPA || - macvlan->mode == MACVLAN_MODE_BRIDGE; -} -#endif - -#if (!SLE_VERSION_CODE || (SLE_VERSION_CODE < SLE_VERSION(15,1,0))) -#ifndef macvlan_accel_priv -#define macvlan_accel_priv _kc_macvlan_accel_priv -static inline void *_kc_macvlan_accel_priv(struct net_device *dev) -{ - struct macvlan_dev *macvlan = netdev_priv(dev); - - return macvlan->fwd_priv; -} -#endif - -#ifndef macvlan_release_l2fw_offload -#define macvlan_release_l2fw_offload _kc_macvlan_release_l2fw_offload -static inline int _kc_macvlan_release_l2fw_offload(struct net_device *dev) -{ - struct macvlan_dev *macvlan = netdev_priv(dev); - - macvlan->fwd_priv = NULL; - return dev_uc_add(macvlan->lowerdev, dev->dev_addr); -} -#endif -#endif /* !SLES || SLES < 15.1 */ -#endif /* NETIF_F_HW_L2FW_DOFFLOAD */ #include "kcompat_overflow.h" #if (SLE_VERSION_CODE < SLE_VERSION(15,1,0)) #define firmware_request_nowarn request_firmware_direct -#endif /* !SLES || SLES < 15.1 */ +#endif /* SLES < 15.1 */ #else #include @@ -6376,26 +6667,14 @@ static inline int _kc_macvlan_release_l2fw_offload(struct net_device *dev) #define HAVE_NDO_XDP_XMIT_BULK_AND_FLAGS #define NO_NDO_XDP_FLUSH #define HAVE_AF_XDP_SUPPORT -#ifndef xdp_umem_get_data -static inline char *__kc_xdp_umem_get_data(struct xdp_umem *umem, u64 addr) -{ - return umem->pages[addr >> PAGE_SHIFT].addr + (addr & (PAGE_SIZE - 1)); -} - -#define xdp_umem_get_data __kc_xdp_umem_get_data -#endif /* !xdp_umem_get_data */ -#ifndef xdp_umem_get_dma -static inline dma_addr_t __kc_xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) -{ - return umem->pages[addr >> PAGE_SHIFT].dma + (addr & (PAGE_SIZE - 1)); -} - -#define xdp_umem_get_dma __kc_xdp_umem_get_dma -#endif /* !xdp_umem_get_dma */ #endif /* 4.18.0 */ /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)) +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0)) && \ + (RHEL_RELEASE_CODE <= RHEL_RELEASE_VERSION(8,2))) +#define HAVE_DEVLINK_REGIONS +#endif /* RHEL >= 8.0 && RHEL <= 8.2 */ #define bitmap_alloc(nbits, flags) \ kmalloc_array(BITS_TO_LONGS(nbits), sizeof(unsigned long), flags) #define bitmap_zalloc(nbits, flags) bitmap_alloc(nbits, ((flags) | __GFP_ZERO)) @@ -6417,19 +6696,22 @@ static inline dma_addr_t __kc_xdp_umem_get_dma(struct xdp_umem *umem, u64 addr) #define HAVE_NDO_SELECT_QUEUE_SB_DEV #define HAVE_TCF_BLOCK_CB_REGISTER_EXTACK #endif -#if ((RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0)) ||\ - (SLE_VERSION_CODE >= SLE_VERSION(15,1,0))) -#define HAVE_TCF_EXTS_FOR_EACH_ACTION -#undef HAVE_TCF_EXTS_TO_LIST -#endif /* RHEL8.0+ */ + +static inline void __kc_metadata_dst_free(void *md_dst) +{ + kfree(md_dst); +} + +#define metadata_dst_free(md_dst) __kc_metadata_dst_free(md_dst) #else /* >= 4.19.0 */ #define HAVE_TCF_BLOCK_CB_REGISTER_EXTACK #define NO_NETDEV_BPF_PROG_ATTACHED #define HAVE_NDO_SELECT_QUEUE_SB_DEV #define HAVE_NETDEV_SB_DEV -#undef HAVE_TCF_EXTS_TO_LIST -#define HAVE_TCF_EXTS_FOR_EACH_ACTION #define HAVE_TCF_VLAN_TPID +#define HAVE_RHASHTABLE_TYPES +#define HAVE_DEVLINK_REGIONS +#define HAVE_DEVLINK_PARAMS #endif /* 4.19.0 */ /*****************************************************************************/ @@ -6458,9 +6740,19 @@ static inline bool __kc_napi_if_scheduled_mark_missed(struct napi_struct *n) #define napi_if_scheduled_mark_missed __kc_napi_if_scheduled_mark_missed #endif /* !napi_if_scheduled_mark_missed */ #endif /* HAVE_AF_XDP_SUPPORT */ +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,0))) +#define HAVE_DEVLINK_ESWITCH_OPS_EXTACK +#endif /* RHEL >= 8.0 */ +#if ((SLE_VERSION_CODE >= SLE_VERSION(12,5,0) && \ + SLE_VERSION_CODE < SLE_VERSION(15,0,0)) || \ + (SLE_VERSION_CODE >= SLE_VERSION(15,1,0))) +#define HAVE_DEVLINK_ESWITCH_OPS_EXTACK +#endif /* SLE == 12sp5 || SLE >= 15sp1 */ #else /* >= 4.20.0 */ +#define HAVE_DEVLINK_ESWITCH_OPS_EXTACK #define HAVE_AF_XDP_ZC_SUPPORT #define HAVE_VXLAN_TYPE +#define HAVE_ETF_SUPPORT /* Earliest TxTime First */ #endif /* 4.20.0 */ /*****************************************************************************/ @@ -6483,12 +6775,22 @@ static inline int _kc_dev_open(struct net_device *netdev, } #define dev_open _kc_dev_open + +static inline int +_kc_dev_change_flags(struct net_device *netdev, unsigned int flags, + struct netlink_ext_ack __always_unused *extack) +{ + return dev_change_flags(netdev, flags); +} + +#define dev_change_flags _kc_dev_change_flags #endif /* !(RHEL_RELEASE_CODE && RHEL > RHEL(8,0)) */ #if (RHEL_RELEASE_CODE && \ (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,7) && \ RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0)) || \ (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,1))) #define HAVE_PTP_SYS_OFFSET_EXTENDED_IOCTL +#define HAVE_PTP_CLOCK_INFO_GETTIMEX64 #else /* RHEL >= 7.7 && RHEL < 8.0 || RHEL >= 8.1 */ struct ptp_system_timestamp { struct timespec64 pre_ts; @@ -6513,12 +6815,15 @@ ptp_read_system_postts(struct ptp_system_timestamp __always_unused *sts) #if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,2)) #define HAVE_TC_INDIR_BLOCK #endif /* RHEL 8.2 */ +#define INDIRECT_CALLABLE_DECLARE(x) x #else /* >= 5.0.0 */ #define HAVE_PTP_SYS_OFFSET_EXTENDED_IOCTL +#define HAVE_PTP_CLOCK_INFO_GETTIMEX64 #define HAVE_NDO_BRIDGE_SETLINK_EXTACK #define HAVE_DMA_ALLOC_COHERENT_ZEROES_MEM #define HAVE_GENEVE_TYPE #define HAVE_TC_INDIR_BLOCK +#define HAVE_INDIRECT_CALL_WRAPPER_HEADER #endif /* 5.0.0 */ /*****************************************************************************/ @@ -6526,6 +6831,8 @@ ptp_read_system_postts(struct ptp_system_timestamp __always_unused *sts) #if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,1))) #define HAVE_TC_FLOW_RULE_INFRASTRUCTURE #define HAVE_NDO_FDB_ADD_EXTACK +#define HAVE_DEVLINK_INFO_GET +#define HAVE_DEVLINK_FLASH_UPDATE #else /* RHEL < 8.1 */ #ifdef HAVE_TC_SETUP_CLSFLOWER #include @@ -6619,11 +6926,21 @@ static inline bool flow_rule_match_key(const struct flow_rule *rule, #endif /* HAVE_TC_SETUP_CLSFLOWER */ #endif /* RHEL < 8.1 */ + +#if (!(RHEL_RELEASE_CODE && RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,1))) +#define devlink_params_publish(devlink) do { } while (0) +#define devlink_params_unpublish(devlink) do { } while (0) +#endif + #else /* >= 5.1.0 */ #define HAVE_NDO_FDB_ADD_EXTACK #define NO_XDP_QUERY_XSK_UMEM +#define HAVE_AF_XDP_NETDEV_UMEM #define HAVE_TC_FLOW_RULE_INFRASTRUCTURE #define HAVE_TC_FLOWER_ENC_IP +#define HAVE_DEVLINK_INFO_GET +#define HAVE_DEVLINK_FLASH_UPDATE +#define HAVE_DEVLINK_PORT_PARAMS #endif /* 5.1.0 */ /*****************************************************************************/ @@ -6656,9 +6973,14 @@ __kc_eth_get_headlen(const struct net_device __always_unused *dev, void *data, #endif #endif /* mmiowb */ +#if (RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(8,1)) +#define HAVE_NDO_GET_DEVLINK_PORT +#endif /* RHEL > 8.1 */ + #else /* >= 5.2.0 */ #define HAVE_NDO_SELECT_QUEUE_FALLBACK_REMOVED #define SPIN_UNLOCK_IMPLIES_MMIOWB +#define HAVE_NDO_GET_DEVLINK_PORT #endif /* 5.2.0 */ /*****************************************************************************/ @@ -6696,28 +7018,29 @@ int _kc_flow_block_cb_setup_simple(struct flow_block_offload *f, #endif /* HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO */ #else /* RHEL >= 8.2 */ #define HAVE_FLOW_BLOCK_API +#define HAVE_DEVLINK_PORT_ATTR_PCI_VF #endif /* RHEL >= 8.2 */ + +#ifndef ETH_P_LLDP +#define ETH_P_LLDP 0x88CC +#endif /* !ETH_P_LLDP */ + #else /* >= 5.3.0 */ #define XSK_UMEM_RETURNS_XDP_DESC +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0)) +#if !(SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(15,3,0)) +#define HAVE_XSK_UMEM_HAS_ADDRS +#endif /* SLE < 15.3 */ +#endif /* < 5.8.0*/ #define HAVE_FLOW_BLOCK_API +#define HAVE_DEVLINK_PORT_ATTR_PCI_VF +#if IS_ENABLED(CONFIG_DIMLIB) +#define HAVE_CONFIG_DIMLIB +#endif #endif /* 5.3.0 */ /*****************************************************************************/ #if (LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)) -#if (!(RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,2)) && \ - !(SLE_VERSION_CODE >= SLE_VERSION(15,2,0))) -static inline unsigned int skb_frag_off(const skb_frag_t *frag) -{ - return frag->page_offset; -} - -static inline void skb_frag_off_add(skb_frag_t *frag, int delta) -{ - frag->page_offset += delta; -} -#define __flow_indr_block_cb_register __tc_indr_block_cb_register -#define __flow_indr_block_cb_unregister __tc_indr_block_cb_unregister -#endif /* !(RHEL >= 8.2) && !(SLES >= 15sp2) */ #if (SLE_VERSION_CODE >= SLE_VERSION(15,2,0)) #define HAVE_NDO_XSK_WAKEUP #endif /* SLES15sp2 */ @@ -6725,4 +7048,216 @@ static inline void skb_frag_off_add(skb_frag_t *frag, int delta) #define HAVE_NDO_XSK_WAKEUP #endif /* 5.4.0 */ +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,5,0)) +static inline unsigned long _kc_bitmap_get_value8(const unsigned long *map, + unsigned long start) +{ + const size_t index = BIT_WORD(start); + const unsigned long offset = start % BITS_PER_LONG; + + return (map[index] >> offset) & 0xFF; +} +#define bitmap_get_value8 _kc_bitmap_get_value8 + +static inline void _kc_bitmap_set_value8(unsigned long *map, + unsigned long value, + unsigned long start) +{ + const size_t index = BIT_WORD(start); + const unsigned long offset = start % BITS_PER_LONG; + + map[index] &= ~(0xFFUL << offset); + map[index] |= value << offset; +} +#define bitmap_set_value8 _kc_bitmap_set_value8 + +#endif /* 5.5.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,6,0)) +#ifdef HAVE_AF_XDP_SUPPORT +#define xsk_umem_release_addr xsk_umem_discard_addr +#define xsk_umem_release_addr_rq xsk_umem_discard_addr_rq +#endif /* HAVE_AF_XDP_SUPPORT */ +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3)) || \ + (SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(15,3,0))) +#define HAVE_TX_TIMEOUT_TXQUEUE +#endif +#else /* >= 5.6.0 */ +#define HAVE_TX_TIMEOUT_TXQUEUE +#endif /* 5.6.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,7,0)) +u64 _kc_pci_get_dsn(struct pci_dev *dev); +#define pci_get_dsn(dev) _kc_pci_get_dsn(dev) +#if !(SLE_VERSION_CODE > SLE_VERSION(15,2,0)) && \ + !((LINUX_VERSION_CODE == KERNEL_VERSION(5,3,18)) && \ + (SLE_LOCALVERSION_CODE >= KERNEL_VERSION(14,0,0))) && \ + !(RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3))) +#define pci_aer_clear_nonfatal_status pci_cleanup_aer_uncorrect_error_status +#endif + +#define cpu_latency_qos_update_request pm_qos_update_request +#define cpu_latency_qos_add_request(arg1, arg2) pm_qos_add_request(arg1, PM_QOS_CPU_DMA_LATENCY, arg2) +#define cpu_latency_qos_remove_request pm_qos_remove_request + +#ifndef DEVLINK_INFO_VERSION_GENERIC_FW_BUNDLE_ID +#define DEVLINK_INFO_VERSION_GENERIC_FW_BUNDLE_ID "fw.bundle_id" +#endif +#else /* >= 5.7.0 */ +#define HAVE_DEVLINK_REGION_OPS_SNAPSHOT +#define HAVE_ETHTOOL_COALESCE_PARAMS_SUPPORT +#endif /* 5.7.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0)) +#if !(RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,4))) && \ + !(SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(15,3,0)) +#define xdp_convert_buff_to_frame convert_to_xdp_frame +#endif /* (RHEL < 8.4) || (SLE < 15.3) */ +#define flex_array_size(p, member, count) \ + array_size(count, sizeof(*(p)->member) + __must_be_array((p)->member)) +#if (!(SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(15,3,0))) +#ifdef HAVE_AF_XDP_ZC_SUPPORT +#ifndef xsk_umem_get_rx_frame_size +static inline u32 _xsk_umem_get_rx_frame_size(struct xdp_umem *umem) +{ + return umem->chunk_size_nohr - XDP_PACKET_HEADROOM; +} + +#define xsk_umem_get_rx_frame_size _xsk_umem_get_rx_frame_size +#endif /* xsk_umem_get_rx_frame_size */ +#endif /* HAVE_AF_XDP_ZC_SUPPORT */ +#else /* SLE >= 15.3 */ +#define HAVE_XDP_BUFF_FRAME_SZ +#define HAVE_MEM_TYPE_XSK_BUFF_POOL +#endif /* SLE >= 15.3 */ +#else /* >= 5.8.0 */ +#define HAVE_TC_FLOW_INDIR_DEV +#define HAVE_TC_FLOW_INDIR_BLOCK_CLEANUP +#define HAVE_XDP_BUFF_FRAME_SZ +#define HAVE_MEM_TYPE_XSK_BUFF_POOL +#endif /* 5.8.0 */ +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3))) +#define HAVE_TC_FLOW_INDIR_DEV +#endif +#if (SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(15,3,0))) +#define HAVE_TC_FLOW_INDIR_DEV +#endif /* SLE_VERSION_CODE && SLE_VERSION_CODE >= SLES15SP3 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,4))) +#define HAVE_TC_FLOW_INDIR_BLOCK_CLEANUP +#endif /* (RHEL >= 8.4) */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,9,0)) +#else /* >= 5.9.0 */ +#define HAVE_FLOW_INDIR_BLOCK_QDISC +#define HAVE_UDP_TUNNEL_NIC_INFO +#endif /* 5.9.0 */ +#if (RHEL_RELEASE_CODE && (RHEL_RELEASE_CODE > RHEL_RELEASE_VERSION(8,3))) +#define HAVE_FLOW_INDIR_BLOCK_QDISC +#endif +#if (SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(15,3,0))) +#define HAVE_FLOW_INDIR_BLOCK_QDISC +#endif /* SLE_VERSION_CODE && SLE_VERSION_CODE >= SLES15SP3 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)) +#if (SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(15,3,0)) +#define HAVE_DEVLINK_REGION_OPS_SNAPSHOT_OPS +#define HAVE_DEVLINK_FLASH_UPDATE_PARAMS +#else /* SLE >= 15.3 */ +struct devlink_flash_update_params { + const char *file_name; + const char *component; + u32 overwrite_mask; +}; + +#ifndef DEVLINK_FLASH_OVERWRITE_SETTINGS +#define DEVLINK_FLASH_OVERWRITE_SETTINGS BIT(0) +#endif + +#ifndef DEVLINK_FLASH_OVERWRITE_IDENTIFIERS +#define DEVLINK_FLASH_OVERWRITE_IDENTIFIERS BIT(1) +#endif +#endif /* !(SLE >= 15.3) */ + +#if (!(SLE_VERSION_CODE && (SLE_VERSION_CODE >= SLE_VERSION(15,3,0)))) +#define XDP_SETUP_XSK_POOL XDP_SETUP_XSK_UMEM +#define xsk_get_pool_from_qid xdp_get_umem_from_qid +#define xsk_pool_get_rx_frame_size xsk_umem_get_rx_frame_size +#define xsk_pool_set_rxq_info xsk_buff_set_rxq_info +#define xsk_pool_dma_unmap xsk_buff_dma_unmap +#define xsk_pool_dma_map xsk_buff_dma_map +#define xsk_tx_peek_desc xsk_umem_consume_tx +#define xsk_tx_release xsk_umem_consume_tx_done +#define xsk_tx_completed xsk_umem_complete_tx +#define xsk_uses_need_wakeup xsk_umem_uses_need_wakeup +#ifdef HAVE_MEM_TYPE_XSK_BUFF_POOL +#include +static inline void +_kc_xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp, + void __always_unused *pool) +{ + xsk_buff_dma_sync_for_cpu(xdp); +} + +#define xsk_buff_dma_sync_for_cpu(xdp, pool) \ + _kc_xsk_buff_dma_sync_for_cpu(xdp, pool) +#endif /* HAVE_MEM_TYPE_XSK_BUFF_POOL */ +#else /* SLE >= 15.3 */ +#define HAVE_NETDEV_BPF_XSK_POOL +#endif /* SLE >= 15.3 */ +#else /* >= 5.10.0 */ +#define HAVE_DEVLINK_REGION_OPS_SNAPSHOT_OPS +#define HAVE_DEVLINK_FLASH_UPDATE_PARAMS +#define HAVE_NETDEV_BPF_XSK_POOL +#endif /* 5.10.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,11,0)) +#ifdef HAVE_XDP_BUFF_RXQ +#include +static inline int +_kc_xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, struct net_device *dev, + u32 queue_index, unsigned int __always_unused napi_id) +{ + return xdp_rxq_info_reg(xdp_rxq, dev, queue_index); +} + +#define xdp_rxq_info_reg(xdp_rxq, dev, queue_index, napi_id) \ + _kc_xdp_rxq_info_reg(xdp_rxq, dev, queue_index, napi_id) +#endif /* HAVE_XDP_BUFF_RXQ */ +#ifdef HAVE_NAPI_BUSY_LOOP +#ifdef CONFIG_NET_RX_BUSY_POLL +#include +static inline void +_kc_napi_busy_loop(unsigned int napi_id, + bool (*loop_end)(void *, unsigned long), void *loop_end_arg, + bool __always_unused prefer_busy_poll, + u16 __always_unused budget) +{ + napi_busy_loop(napi_id, loop_end, loop_end_arg); +} + +#define napi_busy_loop(napi_id, loop_end, loop_end_arg, prefer_busy_poll, budget) \ + _kc_napi_busy_loop(napi_id, loop_end, loop_end_arg, prefer_busy_poll, budget) +#endif /* CONFIG_NET_RX_BUSY_POLL */ +#endif /* HAVE_NAPI_BUSY_LOOP */ +#define HAVE_DEVLINK_FLASH_UPDATE_BEGIN_END_NOTIFY +#else /* >= 5.11.0 */ +#define HAVE_DEVLINK_FLASH_UPDATE_PARAMS_FW +#endif /* 5.11.0 */ + +/* + * Load the implementations file which actually defines kcompat backports. + * Legacy backports still exist in this file, but all new backports must be + * implemented using kcompat_*defs.h and kcompat_impl.h + */ +#include "kcompat_impl.h" + #endif /* _KCOMPAT_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_impl.h b/drivers/net/ethernet/intel/iavf/kcompat_impl.h new file mode 100644 index 000000000..f22449b5e --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/kcompat_impl.h @@ -0,0 +1,411 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _KCOMPAT_IMPL_H_ +#define _KCOMPAT_IMPL_H_ + +/* This file contains implementations of backports from various kernels. It + * must rely only on NEED_ and HAVE_ checks. It must not make any + * checks to determine the kernel version when deciding whether to include an + * implementation. + * + * All new implementations must go in this file, and legacy implementations + * should be migrated to the new format over time. + */ + +/* + * generic network stack functions + */ + +/* NEED_NET_PREFETCH + * + * net_prefetch was introduced by commit f468f21b7af0 ("net: Take common + * prefetch code structure into a function") + * + * This function is trivial to re-implement in full. + */ +#ifdef NEED_NET_PREFETCH +static inline void net_prefetch(void *p) +{ + prefetch(p); +#if L1_CACHE_BYTES < 128 + prefetch((u8 *)p + L1_CACHE_BYTES); +#endif +} +#endif /* NEED_NET_PREFETCH */ + +/* NEED_SKB_FRAG_OFF_ACCESSORS + * + * skb_frag_off and skb_frag_off_add were added in upstream commit + * 7240b60c98d6 ("linux: Add skb_frag_t page_offset accessors") + * + * Implementing the wrappers directly for older kernels which still have the + * old implementation of skb_frag_t is trivial. + */ +#ifdef NEED_SKB_FRAG_OFF_ACCESSORS +static inline unsigned int skb_frag_off(const skb_frag_t *frag) +{ + return frag->page_offset; +} + +static inline void skb_frag_off_add(skb_frag_t *frag, int delta) +{ + frag->page_offset += delta; +} +#endif + +/* + * NETIF_F_HW_L2FW_DOFFLOAD related functions + * + * Support for NETIF_F_HW_L2FW_DOFFLOAD was first introduced upstream by + * commit a6cc0cfa72e0 ("net: Add layer 2 hardware acceleration operations for + * macvlan devices") + */ +#ifdef NETIF_F_HW_L2FW_DOFFLOAD + +#include + +/* NEED_MACVLAN_ACCEL_PRIV + * + * macvlan_accel_priv is an accessor function that replaced direct access to + * the macvlan->fwd_priv variable. It was introduced in commit 7d775f63470c + * ("macvlan: Rename fwd_priv to accel_priv and add accessor function") + * + * Implement the new wrapper name by simply accessing the older + * macvlan->fwd_priv name. + */ +#ifdef NEED_MACVLAN_ACCEL_PRIV +static inline void *macvlan_accel_priv(struct net_device *dev) +{ + struct macvlan_dev *macvlan = netdev_priv(dev); + + return macvlan->fwd_priv; +} +#endif /* NEED_MACVLAN_ACCEL_PRIV */ + +/* NEED_MACVLAN_RELEASE_L2FW_OFFLOAD + * + * macvlan_release_l2fw_offload was introduced upstream by commit 53cd4d8e4dfb + * ("macvlan: Provide function for interfaces to release HW offload") + * + * Implementing this is straight forward, but we must be careful to use + * fwd_priv instead of accel_priv. Note that both the change to accel_priv and + * introduction of this function happened in the same release. + */ +#ifdef NEED_MACVLAN_RELEASE_L2FW_OFFLOAD +static inline int macvlan_release_l2fw_offload(struct net_device *dev) +{ + struct macvlan_dev *macvlan = netdev_priv(dev); + + macvlan->fwd_priv = NULL; + return dev_uc_add(macvlan->lowerdev, dev->dev_addr); +} +#endif /* NEED_MACVLAN_RELEASE_L2FW_OFFLOAD */ + +/* NEED_MACVLAN_SUPPORTS_DEST_FILTER + * + * macvlan_supports_dest_filter was introduced upstream by commit 6cb1937d4eff + * ("macvlan: Add function to test for destination filtering support") + * + * The implementation doesn't rely on anything new and is trivial to backport + * for kernels that have NETIF_F_HW_L2FW_DOFFLOAD support. + */ +#ifdef NEED_MACVLAN_SUPPORTS_DEST_FILTER +static inline bool macvlan_supports_dest_filter(struct net_device *dev) +{ + struct macvlan_dev *macvlan = netdev_priv(dev); + + return macvlan->mode == MACVLAN_MODE_PRIVATE || + macvlan->mode == MACVLAN_MODE_VEPA || + macvlan->mode == MACVLAN_MODE_BRIDGE; +} +#endif /* NEED_MACVLAN_SUPPORTS_DEST_FILTER */ + +#endif /* NETIF_F_HW_L2FW_DOFFLOAD */ + +/* + * tc functions + */ + +/* NEED_FLOW_INDR_BLOCK_CB_REGISTER + * + * __flow_indr_block_cb_register and __flow_indr_block_cb_unregister were + * added in upstream commit 4e481908c51b ("flow_offload: move tc indirect + * block to flow offload") + * + * This was a simple rename so we can just translate from the old + * naming scheme with a macro. + */ +#ifdef NEED_FLOW_INDR_BLOCK_CB_REGISTER +#define __flow_indr_block_cb_register __tc_indr_block_cb_register +#define __flow_indr_block_cb_unregister __tc_indr_block_cb_unregister +#endif + +/* + * devlink support + */ +#if IS_ENABLED(CONFIG_NET_DEVLINK) + +#include + +#ifdef HAVE_DEVLINK_REGIONS +/* NEED_DEVLINK_REGION_CREATE_OPS + * + * The ops parameter to devlink_region_create was added by commit e8937681797c + * ("devlink: prepare to support region operations") + * + * For older kernels, define _kc_devlink_region_create that takes an ops + * parameter, and calls the old implementation function by extracting the name + * from the structure. + */ +#ifdef NEED_DEVLINK_REGION_CREATE_OPS +struct devlink_region_ops { + const char *name; + void (*destructor)(const void *data); +}; + +static inline struct devlink_region * +_kc_devlink_region_create(struct devlink *devlink, + const struct devlink_region_ops *ops, + u32 region_max_snapshots, u64 region_size) +{ + return devlink_region_create(devlink, ops->name, region_max_snapshots, + region_size); +} + +#define devlink_region_create _kc_devlink_region_create +#endif /* NEED_DEVLINK_REGION_CREATE_OPS */ +#endif /* HAVE_DEVLINK_REGIONS */ + +/* NEED_DEVLINK_FLASH_UPDATE_STATUS_NOTIFY + * + * devlink_flash_update_status_notify, _begin_notify, and _end_notify were + * added by upstream commit 191ed2024de9 ("devlink: allow driver to update + * progress of flash update") + * + * For older kernels that lack the netlink messages, convert the functions + * into no-ops. + */ +#ifdef NEED_DEVLINK_FLASH_UPDATE_STATUS_NOTIFY +static inline void +devlink_flash_update_begin_notify(struct devlink __always_unused *devlink) +{ +} + +static inline void +devlink_flash_update_end_notify(struct devlink __always_unused *devlink) +{ +} + +static inline void +devlink_flash_update_status_notify(struct devlink __always_unused *devlink, + const char __always_unused *status_msg, + const char __always_unused *component, + unsigned long __always_unused done, + unsigned long __always_unused total) +{ +} +#endif /* NEED_DEVLINK_FLASH_UPDATE_STATUS_NOTIFY */ + +/* NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY + * + * devlink_flash_update_timeout_notify was added by upstream commit + * f92970c694b3 ("devlink: add timeout information to status_notify"). + * + * For older kernels, just convert timeout notifications into regular status + * notification messages without timeout information. + */ +#ifdef NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY +static inline void +devlink_flash_update_timeout_notify(struct devlink *devlink, + const char *status_msg, + const char *component, + unsigned long __always_unused timeout) +{ + devlink_flash_update_status_notify(devlink, status_msg, component, 0, 0); +} +#endif /* NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY */ + +/* + * NEED_DEVLINK_PORT_ATTRS_SET_STRUCT + * + * HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR + * HAVE_DEVLINK_PORT_ATTRS_SET_SWITCH_ID + * + * devlink_port_attrs_set was introduced by commit b9ffcbaf56d3 ("devlink: + * introduce devlink_port_attrs_set") + * + * It's function signature has changed multiple times over several kernel + * releases: + * + * commit 5ec1380a21bb ("devlink: extend attrs_set for setting port + * flavours") added the ability to set port flavour. (Note that there is no + * official kernel release with devlink_port_attrs_set without the flavour + * argument, as they were introduced in the same series.) + * + * commit bec5267cded2 ("net: devlink: extend port attrs for switch ID") added + * the ability to set the switch ID (HAVE_DEVLINK_PORT_ATTRS_SET_SWITCH_ID) + * + * Finally commit 71ad8d55f8e5 ("devlink: Replace devlink_port_attrs_set + * parameters with a struct") refactored to pass devlink_port_attrs struct + * instead of individual parameters. (!NEED_DEVLINK_PORT_ATTRS_SET_STRUCT) + * + * We want core drivers to just use the latest form that takes + * a devlink_port_attrs structure. Note that this structure did exist as part + * of but was never used directly by driver code prior to the + * function parameter change. For this reason, the implementation always + * relies on _kc_devlink_port_attrs instead of what was defined in the kernel. + */ +#ifdef NEED_DEVLINK_PORT_ATTRS_SET_STRUCT + +#ifndef HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR +enum devlink_port_flavour { + DEVLINK_PORT_FLAVOUR_PHYSICAL, + DEVLINK_PORT_FLAVOUR_CPU, + DEVLINK_PORT_FLAVOUR_DSA, + DEVLINK_PORT_FLAVOUR_PCI_PF, + DEVLINK_PORT_FLAVOUR_PCI_VF, +}; +#endif + +struct _kc_devlink_port_phys_attrs { + u32 port_number; + u32 split_subport_number; +}; + +struct _kc_devlink_port_pci_pf_attrs { + u16 pf; +}; + +struct _kc_devlink_port_pci_vf_attrs { + u16 pf; + u16 vf; +}; + +struct _kc_devlink_port_attrs { + u8 split:1, + splittable:1; + u32 lanes; + enum devlink_port_flavour flavour; + struct netdev_phys_item_id switch_id; + union { + struct _kc_devlink_port_phys_attrs phys; + struct _kc_devlink_port_pci_pf_attrs pci_pf; + struct _kc_devlink_port_pci_vf_attrs pci_vf; + }; +}; + +#define devlink_port_attrs _kc_devlink_port_attrs + +static inline void +_kc_devlink_port_attrs_set(struct devlink_port *devlink_port, + struct _kc_devlink_port_attrs *attrs) +{ +#if defined(HAVE_DEVLINK_PORT_ATTRS_SET_SWITCH_ID) + devlink_port_attrs_set(devlink_port, attrs->flavour, attrs->phys.port_number, + attrs->split, attrs->phys.split_subport_number, + attrs->switch_id.id, attrs->switch_id.id_len); +#elif defined(HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR) + devlink_port_attrs_set(devlink_port, attrs->flavour, attrs->phys.port_number, + attrs->split, attrs->phys.split_subport_number); +#else + if (attrs->split) + devlink_port_split_set(devlink_port, attrs->phys.port_number); +#endif +} + +#define devlink_port_attrs_set _kc_devlink_port_attrs_set + +#endif /* NEED_DEVLINK_PORT_ATTRS_SET_STRUCT */ + +#endif /* CONFIG_NET_DEVLINK */ + +/* + * dev_printk implementations + */ + +/* NEED_DEV_PRINTK_ONCE + * + * The dev_*_once family of printk functions was introduced by commit + * e135303bd5be ("device: Add dev__once variants") + * + * The implementation is very straight forward so we will just implement them + * as-is here. + */ +#ifdef NEED_DEV_PRINTK_ONCE +#ifdef CONFIG_PRINTK +#define dev_level_once(dev_level, dev, fmt, ...) \ +do { \ + static bool __print_once __read_mostly; \ + \ + if (!__print_once) { \ + __print_once = true; \ + dev_level(dev, fmt, ##__VA_ARGS__); \ + } \ +} while (0) +#else +#define dev_level_once(dev_level, dev, fmt, ...) \ +do { \ + if (0) \ + dev_level(dev, fmt, ##__VA_ARGS__); \ +} while (0) +#endif + +#define dev_emerg_once(dev, fmt, ...) \ + dev_level_once(dev_emerg, dev, fmt, ##__VA_ARGS__) +#define dev_alert_once(dev, fmt, ...) \ + dev_level_once(dev_alert, dev, fmt, ##__VA_ARGS__) +#define dev_crit_once(dev, fmt, ...) \ + dev_level_once(dev_crit, dev, fmt, ##__VA_ARGS__) +#define dev_err_once(dev, fmt, ...) \ + dev_level_once(dev_err, dev, fmt, ##__VA_ARGS__) +#define dev_warn_once(dev, fmt, ...) \ + dev_level_once(dev_warn, dev, fmt, ##__VA_ARGS__) +#define dev_notice_once(dev, fmt, ...) \ + dev_level_once(dev_notice, dev, fmt, ##__VA_ARGS__) +#define dev_info_once(dev, fmt, ...) \ + dev_level_once(dev_info, dev, fmt, ##__VA_ARGS__) +#define dev_dbg_once(dev, fmt, ...) \ + dev_level_once(dev_dbg, dev, fmt, ##__VA_ARGS__) +#endif /* NEED_DEV_PRINTK_ONCE */ + +#ifdef HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO + +/* NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 + * + * tc_cls_can_offload_and_chain0 was added by upstream commit + * 878db9f0f26d ("pkt_cls: add new tc cls helper to check offload flag and + * chain index"). + * + * This patch backports this function for older kernels by calling + * tc_can_offload() directly. + */ +#ifdef NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 +#include +static inline bool +tc_cls_can_offload_and_chain0(const struct net_device *dev, + struct tc_cls_common_offload *common) +{ + if (!tc_can_offload(dev)) + return false; + if (common->chain_index) + return false; + + return true; +} +#endif /* NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 */ +#endif /* HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO */ + +/* NEED_TC_SETUP_QDISC_MQPRIO + * + * TC_SETUP_QDISC_MQPRIO was added by upstream commit + * 575ed7d39e2f ("net_sch: mqprio: Change TC_SETUP_MQPRIO to + * TC_SETUP_QDISC_MQPRIO"). + * + * For older kernels which are using TC_SETUP_MQPRIO + */ +#ifdef NEED_TC_SETUP_QDISC_MQPRIO +#define TC_SETUP_QDISC_MQPRIO TC_SETUP_MQPRIO +#endif /* NEED_TC_SETUP_QDISC_MQPRIO */ + +#endif /* _KCOMPAT_IMPL_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_rhel_defs.h b/drivers/net/ethernet/intel/iavf/kcompat_rhel_defs.h new file mode 100644 index 000000000..4bdf5b040 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/kcompat_rhel_defs.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _KCOMPAT_RHEL_DEFS_H_ +#define _KCOMPAT_RHEL_DEFS_H_ + +/* This is the RedHat Enterprise Linux distribution specific definitions file. + * It defines what features need backports for a given version of the RHEL + * kernel. + * + * It checks the RHEL_RELEASE_CODE and RHEL_RELEASE_VERSION macros to decide + * what support the target kernel has. + * + * It assumes that kcompat_std_defs.h has already been processed, and will + * #define or #undef any flags that have changed based on backports done by + * RHEL. + */ + +#if !RHEL_RELEASE_CODE +#error "RHEL_RELEASE_CODE is 0 or undefined" +#endif + +#ifndef RHEL_RELEASE_VERSION +#error "RHEL_RELEASE_VERSION is undefined" +#endif + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,3)) +#else /* >= 7.3 */ +#undef NEED_DEV_PRINTK_ONCE +#endif /* 7.3 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,5)) +#else /* >= 7.5 */ +#define HAVE_TCF_EXTS_TO_LIST +#endif /* 7.5 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,6)) +#else /* >= 7.6 */ +#undef NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 +#undef NEED_TC_SETUP_QDISC_MQPRIO +#endif /* 7.6 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(7,7)) +#else /* >= 7.7 */ +#define HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR +#endif /* 7.7 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0)) +#else /* >= 8.0 */ +#undef HAVE_TCF_EXTS_TO_LIST +#define HAVE_TCF_EXTS_FOR_EACH_ACTION +#endif /* 7.5 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,2)) +#else /* >= 8.2 */ +#undef NEED_DEVLINK_FLASH_UPDATE_STATUS_NOTIFY +#undef NEED_SKB_FRAG_OFF_ACCESSORS +#undef NEED_FLOW_INDR_BLOCK_CB_REGISTER +#define HAVE_DEVLINK_PORT_ATTRS_SET_SWITCH_ID +#endif /* 8.2 */ + +/*****************************************************************************/ +#if (RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,4)) +#else /* >= 8.4 */ +#undef NEED_DEVLINK_PORT_ATTRS_SET_STRUCT +#undef NEED_NET_PREFETCH +#undef NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY +#undef HAVE_XDP_QUERY_PROG +#endif /* 8.4 */ + +#endif /* _KCOMPAT_RHEL_DEFS_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_sles_defs.h b/drivers/net/ethernet/intel/iavf/kcompat_sles_defs.h new file mode 100644 index 000000000..fe4e2f1be --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/kcompat_sles_defs.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _KCOMPAT_SLES_DEFS_H_ +#define _KCOMPAT_SLES_DEFS_H_ + +/* This is the SUSE Linux Enterprise distribution specific definitions file. + * It defines what features need backports for a given version of the SUSE + * Linux Enterprise kernel. + * + * It checks a combination of the LINUX_VERSION code and the + * SLE_LOCALVERSION_CODE to determine what support the kernel has. + * + * It assumes that kcompat_std_defs.h has already been processed, and will + * #define or #undef any flags that have changed based on backports done by + * SUSE. + */ + +#ifndef LINUX_VERSION_CODE +#error "LINUX_VERSION_CODE is undefined" +#endif + +#ifndef KERNEL_VERSION +#error "KERNEL_VERSION is undefined" +#endif + +#if !SLE_KERNEL_REVISION +#error "SLE_KERNEL_REVISION is 0 or undefined" +#endif + +#if SLE_KERNEL_REVISION > 65535 +#error "SLE_KERNEL_REVISION is unexpectedly large" +#endif + +/* SLE kernel versions are a combination of the LINUX_VERSION_CODE along with + * an extra digit that indicates the SUSE specific revision of that kernel. + * This value is found in the CONFIG_LOCALVERSION of the SUSE kernel, which is + * extracted by common.mk and placed into SLE_KERNEL_REVISION_CODE. + * + * We combine the value of SLE_KERNEL_REVISION along with the LINUX_VERSION code + * to generate the useful value that determines what specific kernel we're + * dealing with. + * + * Just in case the SLE_KERNEL_REVISION ever goes above 255, we reserve 16 bits + * instead of 8 for this value. + */ +#define SLE_KERNEL_CODE ((LINUX_VERSION_CODE << 16) + SLE_KERNEL_REVISION) +#define SLE_KERNEL_VERSION(a,b,c,d) ((KERNEL_VERSION(a,b,c) << 16) + (d)) + +/* Unlike RHEL, SUSE kernels are not always tied to a single service pack. For + * example, 4.12.14 was used as the base for SLE 15 SP1, SLE 12 SP4, and SLE 12 + * SP5. + * + * You can find the patches that SUSE applied to the kernel tree at + * https://github.com/SUSE/kernel-source. + * + * You can find the correct kernel version for a check by using steps similar + * to the following + * + * 1) download the kernel-source repo + * 2) checkout the relevant branch, i.e SLE15-SP3 + * 3) find the relevant backport you're interested in the patches.suse + * directory + * 4) git log to locate the commit that introduced the backport + * 5) git describe --contains to find the relevant tag that includes that + * commit, i.e. rpm-5.3.18-37 + * 6) those digits represent the SLE kernel that introduced that backport. + * + * Try to keep the checks in SLE_KERNEL_CODE order and condense where + * possible. + */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE > SLE_KERNEL_VERSION(4,12,14,23) && \ + SLE_KERNEL_CODE < SLE_KERNEL_VERSION(4,12,14,94)) +/* + * 4.12.14 is used as the base for SLE 12 SP4, SLE 12 SP5, SLE 15, and SLE 15 + * SP1. Unfortunately the revision codes do not line up cleanly. SLE 15 + * launched with 4.12.14-23. It appears that SLE 12 SP4 and SLE 15 SP1 both + * diverged from this point, with SLE 12 SP4 kernels starting around + * 4.12.14-94. A few backports for SLE 15 SP1 landed in some alpha and beta + * kernels tagged between 4.12.14-25 up to 4.12.14-32. These changes did not + * make it into SLE 12 SP4. This was cleaned up with SLE 12 SP5 by an apparent + * merge in 4.12.14-111. The official launch of SLE 15 SP1 ended up with + * version 4.12.14-195. + * + * Because of this inconsistency and because all of these kernels appear to be + * alpha or beta kernel releases for SLE 15 SP1, we do not rely on version + * checks between this range. Issue a warning to indicate that we do not + * support these. + */ +#warning "SLE kernel versions between 4.12.14-23 and 4.12.14-94 are not supported" +#endif + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(4,12,14,100)) +#else /* >= 4.12.14-100 */ +#undef HAVE_TCF_EXTS_TO_LIST +#define HAVE_TCF_EXTS_FOR_EACH_ACTION +#endif /* 4.12.14-100 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(4,12,14,111)) +#else /* >= 4.12.14-111 */ +#define HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR +#undef NEED_MACVLAN_ACCEL_PRIV +#undef NEED_MACVLAN_RELEASE_L2FW_OFFLOAD +#undef NEED_MACVLAN_SUPPORTS_DEST_FILTER +#endif /* 4.12.14-111 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(4,12,14,120)) +#else /* >= 4.12.14-120 */ +#define HAVE_NDO_SELECT_QUEUE_SB_DEV +#define HAVE_TCF_MIRRED_DEV +#define HAVE_TCF_BLOCK +#define HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO +#define HAVE_TCF_BLOCK_CB_REGISTER_EXTACK +#undef NEED_TC_SETUP_QDISC_MQPRIO +#undef NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 +#endif /* 4.12.14-120 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(5,3,8,2)) +#else /* >= 5.3.8-2 */ +#undef NEED_FLOW_INDR_BLOCK_CB_REGISTER +#undef NEED_SKB_FRAG_OFF_ACCESSORS +#endif /* 5.3.8-2 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(5,3,18,34)) +#else /* >= 5.3.18-34 */ +#undef NEED_DEVLINK_REGION_CREATE_OPS +#undef NEED_DEVLINK_PORT_ATTRS_SET_STRUCT +#endif /* 5.3.18-34 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(5,3,18,37)) +#else /* >= 5.3.18-37 */ +#undef NEED_NET_PREFETCH +#endif /* 5.3.18-37 */ + +/*****************************************************************************/ +#if (SLE_KERNEL_CODE < SLE_KERNEL_VERSION(5,3,18,38)) +#else /* >= 5.3.18-38 */ +#undef NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY +#endif /* 5.3.18-38 */ + +#endif /* _KCOMPAT_SLES_DEFS_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_std_defs.h b/drivers/net/ethernet/intel/iavf/kcompat_std_defs.h new file mode 100644 index 000000000..03fdb5853 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/kcompat_std_defs.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _KCOMPAT_STD_DEFS_H_ +#define _KCOMPAT_STD_DEFS_H_ + +/* This file contains the definitions for what kernel features need backports + * for a given kernel. It targets only the standard stable kernel releases. + * It must check only LINUX_VERSION_CODE and assume the kernel is a standard + * release, and not a custom distribution. + * + * It must define HAVE_ and NEED_ for features. It must not + * implement any backports, instead leaving the implementation to the + * kcompat_impl.h header. + * + * If a feature can be easily implemented as a replacement macro or fully + * backported, use a NEED_ to indicate that the feature needs + * a backport. (If NEED_ is undefined, then no backport for that feature + * is needed). + * + * If a feature cannot be easily implemented in kcompat directly, but + * requires drivers to make specific changes such as stripping out an entire + * feature or modifying a function pointer prototype, use a HAVE_. + */ + +#ifndef LINUX_VERSION_CODE +#error "LINUX_VERSION_CODE is undefined" +#endif + +#ifndef KERNEL_VERSION +#error "KERNEL_VERSION is undefined" +#endif + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0)) +#define NEED_DEV_PRINTK_ONCE +#else /* >= 3,19,0 */ +#endif /* 3,19,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,8,0)) +#else /* >= 4,8,0 */ +#define HAVE_TCF_EXTS_TO_LIST +#endif /* 4,8,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,15,0)) +#define NEED_TC_SETUP_QDISC_MQPRIO +#else /* >= 4,15,0 */ +#define HAVE_TC_CB_AND_SETUP_QDISC_MQPRIO +#endif /* 4,15,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,16,0)) +#define NEED_TC_CLS_CAN_OFFLOAD_AND_CHAIN0 +#else /* >= 4,16,0 */ +#endif /* 4,16,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,18,0)) +#define NEED_MACVLAN_ACCEL_PRIV +#define NEED_MACVLAN_RELEASE_L2FW_OFFLOAD +#define NEED_MACVLAN_SUPPORTS_DEST_FILTER +#else /* >= 4,18,0 */ +#define HAVE_DEVLINK_PORT_ATTRS_SET_PORT_FLAVOUR +#endif /* 4,18,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4,19,0)) +#else /* >= 4,19,0 */ +#undef HAVE_TCF_EXTS_TO_LIST +#define HAVE_TCF_EXTS_FOR_EACH_ACTION +#endif /* 4,19,0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,2,0)) +#else /* >= 5.2.0 */ +#define HAVE_DEVLINK_PORT_ATTRS_SET_SWITCH_ID +#endif /* 5.2.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0)) +#define NEED_DEVLINK_FLASH_UPDATE_STATUS_NOTIFY +#else /* >= 5.3.0 */ +#endif /* 5.3.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,7,0)) +#define NEED_DEVLINK_REGION_CREATE_OPS +#else /* >= 5.7.0 */ +#endif /* 5.7.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)) +#define NEED_SKB_FRAG_OFF_ACCESSORS +#define NEED_FLOW_INDR_BLOCK_CB_REGISTER +#else /* >= 5.4.0 */ +#endif /* 5.4.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,9,0)) +#define NEED_DEVLINK_PORT_ATTRS_SET_STRUCT +#define HAVE_XDP_QUERY_PROG +#else /* >= 5.9.0 */ +#endif /* 5.9.0 */ + +/*****************************************************************************/ +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)) +#define NEED_NET_PREFETCH +#define NEED_DEVLINK_FLASH_UPDATE_TIMEOUT_NOTIFY +#else /* >= 5.10.0 */ +#endif /* 5.10.0 */ + +#endif /* _KCOMPAT_STD_DEFS_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_ubuntu_defs.h b/drivers/net/ethernet/intel/iavf/kcompat_ubuntu_defs.h new file mode 100644 index 000000000..43e00dee1 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/kcompat_ubuntu_defs.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +#ifndef _KCOMPAT_UBUNTU_DEFS_H_ +#define _KCOMPAT_UBUNTU_DEFS_H_ + +/* This file contains the definitions for the Ubuntu specific distribution of + * the Linux kernel. + * + * It checks the UBUNTU_VERSION_CODE to decide which features are available in + * the target kernel. It assumes that kcompat_std_defs.h has already been + * processed, and will #define or #undef the relevant flags based on what + * features were backported by Ubuntu. + */ + +#if !UTS_UBUNTU_RELEASE_ABI +#error "UTS_UBUNTU_RELEASE_ABI is 0 or undefined" +#endif + +#if !UBUNTU_VERSION_CODE +#error "UBUNTU_VERSION_CODE is 0 or undefined" +#endif + +#ifndef UBUNTU_VERSION +#error "UBUNTU_VERSION is undefined" +#endif + +#endif /* _KCOMPAT_UBUNTU_DEFS_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/kcompat_vfd.c b/drivers/net/ethernet/intel/iavf/kcompat_vfd.c index 03c2c5f8b..9c62c76e9 100644 --- a/drivers/net/ethernet/intel/iavf/kcompat_vfd.c +++ b/drivers/net/ethernet/intel/iavf/kcompat_vfd.c @@ -55,6 +55,70 @@ static int __get_pdev_and_vfid(struct kobject *kobj, struct pci_dev **pdev, return 0; } +/** + * __get_tc - helper function to get the pdev and the vf id + * @pdev: PCI device information struct + * @tc_kobj: kobject passed + * @tc: number of extracted TC + */ +static int __get_tc(struct pci_dev *pdev, struct kobject *tc_kobj, int *tc) +{ + if (kstrtoint(tc_kobj->name, 10, tc) != 0) { + dev_err(&pdev->dev, "Failed to convert %s to tc\n", + tc_kobj->name); + return -EINVAL; + } + + return 0; +} + +/** + * __get_vf_tc_pdev - helper function to get the pdev and the vf id + * @kobj: kobject passed + * @pdev: PCI device information struct + * @vf_id: VF id of the VF under consideration + * @tc: number of extracted TC + */ +static int __get_vf_tc_pdev(struct kobject *kobj, struct pci_dev **pdev, + int *vf_id, int *tc) +{ + int ret; + + if (!kobj->parent->parent) + return -EINVAL; + + ret = __get_pdev_and_vfid(kobj->parent->parent, pdev, vf_id); + if (ret) + goto err; + + ret = __get_tc(*pdev, kobj, tc); +err: + return ret; +} + +/** + * __get_pdev_tc - helper function to get the pdev and the vf id + * @kobj: kobject passed + * @pdev: PCI device information struct + * @tc: number of extracted TC + */ +static int __get_pdev_tc(struct kobject *kobj, struct pci_dev **pdev, int *tc) +{ + int ret; + + /* check for pci_dev kobject */ + if (!kobj->parent->parent->parent) + return -EINVAL; + + ret = __get_pf_pdev(kobj->parent->parent, pdev); + if (ret) + goto err; + + ret = __get_tc(*pdev, kobj, tc); +err: + return ret; +} + /** * __parse_bool_data - helper function to parse boolean data * @pdev: PCI device information struct @@ -577,7 +641,7 @@ static ssize_t vfd_mac_anti_spoof_show(struct kobject *kobj, char *buff) { struct pci_dev *pdev; - int vf_id, ret = 0; + int vf_id, ret; bool data; if (!vfd_ops->get_mac_anti_spoof) @@ -613,9 +677,9 @@ static ssize_t vfd_mac_anti_spoof_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buff, size_t count) { - struct pci_dev *pdev; - int vf_id, ret = 0; bool data_new, data_old; + struct pci_dev *pdev; + int vf_id, ret; if (!vfd_ops->set_mac_anti_spoof || !vfd_ops->get_mac_anti_spoof) return -EOPNOTSUPP; @@ -649,7 +713,7 @@ static ssize_t vfd_vlan_anti_spoof_show(struct kobject *kobj, char *buff) { struct pci_dev *pdev; - int vf_id, ret = 0; + int vf_id, ret; bool data; if (!vfd_ops->get_vlan_anti_spoof) @@ -685,9 +749,9 @@ static ssize_t vfd_vlan_anti_spoof_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buff, size_t count) { - struct pci_dev *pdev; - int vf_id, ret = 0; bool data_new, data_old; + struct pci_dev *pdev; + int vf_id, ret; if (!vfd_ops->set_vlan_anti_spoof || !vfd_ops->get_vlan_anti_spoof) return -EOPNOTSUPP; @@ -1520,38 +1584,6 @@ static ssize_t vfd_min_tx_rate_store(struct kobject *kobj, return vfd_ops->set_min_tx_rate(kobj, attr, buff, count); } -/** - * vfd_spoofcheck_show - handler for spoofcheck show function - * @kobj: kobject being called - * @attr: struct kobj_attribute - * @buff: buffer for data - **/ -static ssize_t vfd_spoofcheck_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buff) -{ - if (!vfd_ops->get_spoofcheck) - return -EOPNOTSUPP; - - return vfd_ops->get_spoofcheck(kobj, attr, buff); -} - -/** - * vfd_spoofcheck_store - handler for spoofcheck store function - * @kobj: kobject being called - * @attr: struct kobj_attribute - * @buff: buffer with input data - * @count: size of buff - **/ -static ssize_t vfd_spoofcheck_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buff, size_t count) -{ - if (!vfd_ops->set_spoofcheck) - return -EOPNOTSUPP; - - return vfd_ops->set_spoofcheck(kobj, attr, buff, count); -} - /** * vfd_trust_show - handler for trust show function * @kobj: kobject being called @@ -1573,7 +1605,7 @@ static ssize_t vfd_trust_show(struct kobject *kobj, return ret; ret = vfd_ops->get_trust_state(pdev, vf_id, &data); - if (ret < 0) + if (ret) return ret; if (data) @@ -1607,7 +1639,7 @@ static ssize_t vfd_trust_store(struct kobject *kobj, return ret; ret = vfd_ops->get_trust_state(pdev, vf_id, &data_old); - if (ret < 0) + if (ret) return ret; ret = __parse_bool_data(pdev, buff, "trust", &data_new); @@ -1893,7 +1925,7 @@ static ssize_t vfd_tx_errors_show(struct kobject *kobj, * @buff: buffer for data **/ static ssize_t qos_share_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buff) + struct kobj_attribute *attr, char *buff) { struct pci_dev *pdev; int vf_id, ret; @@ -1902,7 +1934,7 @@ static ssize_t qos_share_show(struct kobject *kobj, if (!vfd_ops->get_vf_bw_share) return -EOPNOTSUPP; - ret = __get_pdev_and_vfid(kobj, &pdev, &vf_id); + ret = __get_pdev_and_vfid(kobj->parent, &pdev, &vf_id); if (ret) return ret; @@ -1935,7 +1967,7 @@ static ssize_t qos_share_store(struct kobject *kobj, if (!vfd_ops->set_vf_bw_share) return -EOPNOTSUPP; - ret = __get_pdev_and_vfid(kobj, &pdev, &vf_id); + ret = __get_pdev_and_vfid(kobj->parent, &pdev, &vf_id); if (ret) return ret; @@ -1972,7 +2004,7 @@ static ssize_t pf_qos_apply_store(struct kobject *kobj, if (!vfd_ops->set_pf_qos_apply) return -EOPNOTSUPP; - ret = __get_pf_pdev(kobj, &pdev); + ret = __get_pf_pdev(kobj->parent, &pdev); if (ret) return ret; @@ -2030,8 +2062,8 @@ static ssize_t pf_ingress_mirror_show(struct kobject *kobj, * @count: size of buff **/ static ssize_t pf_ingress_mirror_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buff, size_t count) + struct kobj_attribute *attr, + const char *buff, size_t count) { int data_new, data_old; struct pci_dev *pdev; @@ -2261,6 +2293,512 @@ static ssize_t vfd_num_queues_store(struct kobject *kobj, return ret ? ret : count; } +/** + * vfd_queue_type_show - handler for queue_type show function + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t vfd_queue_type_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct pci_dev *pdev; + int vf_id, ret = 0; + u8 data; + + if (!vfd_ops->get_queue_type) + return -EOPNOTSUPP; + + ret = __get_pdev_and_vfid(kobj, &pdev, &vf_id); + if (ret) + return ret; + + ret = vfd_ops->get_queue_type(pdev, vf_id, &data); + if (ret) + return ret; + + ret = scnprintf(buff, PAGE_SIZE, "%d\n", data); + + return ret; +} + +/** + * vfd_queue_type_store - handler for queue_type store function + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + **/ +static ssize_t vfd_queue_type_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + // the setting will be updated via different sysfs + return -EOPNOTSUPP; +} + +/** + * vfd_allow_bcast_show - handler for VF allow broadcast show function + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t vfd_allow_bcast_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buff) +{ + struct pci_dev *pdev; + int vf_id, ret; + bool data; + + if (!vfd_ops->get_allow_bcast) + return -EOPNOTSUPP; + + ret = __get_pdev_and_vfid(kobj, &pdev, &vf_id); + if (ret) + return ret; + + ret = vfd_ops->get_allow_bcast(pdev, vf_id, &data); + if (ret < 0) + return ret; + + if (data) + ret = scnprintf(buff, PAGE_SIZE, "on\n"); + else + ret = scnprintf(buff, PAGE_SIZE, "off\n"); + + return ret; +} + +/** + * vfd_allow_bcast_store - handler for VF allow broadcast store function + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t vfd_allow_bcast_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + bool data_new, data_old; + struct pci_dev *pdev; + int vf_id, ret; + + if (!vfd_ops->set_allow_bcast || !vfd_ops->get_allow_bcast) + return -EOPNOTSUPP; + + ret = __get_pdev_and_vfid(kobj, &pdev, &vf_id); + if (ret) + return ret; + + ret = vfd_ops->get_allow_bcast(pdev, vf_id, &data_old); + if (ret < 0) + return ret; + + ret = __parse_bool_data(pdev, buff, "allow_bcast", &data_new); + if (ret) + return ret; + + if (data_new != data_old) + ret = vfd_ops->set_allow_bcast(pdev, vf_id, data_new); + + return ret ? ret : count; +} + +/** + * round_nearest_quanta - helper function for calculating quanta + * @num: Number to be rounded + * + * Calculates nearest multiple of 50, which is quanta accepted by FW. + * For 0 it returns 0, which means unlimitied bandwidth + **/ +static int round_nearest_quanta(int num) +{ + static const int base = 50; + + if (!(num % base) || !num) + return num; + else + return num + base - (num % base); +} + +/** + * pf_qos_tc_priority_show - handler for PF's priority for given TC show + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t pf_qos_tc_priority_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct pci_dev *pdev; + int i, tc, ret; + char *written; + u8 prio; + + /* check if option is implemented in vfd_ops*/ + if (!vfd_ops->set_pf_qos_tc_priority || + !vfd_ops->get_pf_qos_tc_priority) + return -EOPNOTSUPP; + + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + return ret; + + ret = vfd_ops->get_pf_qos_tc_priority(pdev, tc, &prio); + + if (!prio) + return ret; + + written = buff; + /* iterate over prio bits */ + for (i = 0; i < 8; i++) { + if (BIT(i) & prio) { + ret += scnprintf(written, PAGE_SIZE, "%d,", i); + written += 2; + } + } + ret += scnprintf(written, PAGE_SIZE, "\n"); + return ret; +} + +/** + * pf_qos_tc_priority_store - handler for PF's priority for given TC store + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t pf_qos_tc_priority_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct pci_dev *pdev; + int tc, tmp, ret; + char *tok, *str; + u8 prio = 0; + + /* check if option is implemented in vfd_ops*/ + if (!vfd_ops->set_pf_qos_tc_priority || + !vfd_ops->get_pf_qos_tc_priority) + return -EOPNOTSUPP; + + str = kzalloc(sizeof(*str) * count + 1, GFP_KERNEL); + if (!str) + return -ENOMEM; + + strncpy(str, buff, count); + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + goto err; + + while ((tok = strsep(&str, ",")) != NULL) { + tok = strim(tok); + + ret = kstrtoint(tok, 10, &tmp); + if (ret) { + dev_err(&pdev->dev, "Invalid input\n"); + goto err; + } + + if (tmp < 0 || tmp >= VFD_NUM_TC) { + dev_err(&pdev->dev, "Only numbers 0-7 are allowed.\n"); + ret = -EINVAL; + goto err; + } + prio |= BIT(tmp); + } + vfd_ops->set_pf_qos_tc_priority(pdev, tc, prio); + + kfree(str); + return count; + +err: + kfree(str); + return ret; +} + +/** + * pf_qos_tc_lsp_show - handler for PF's link strict priority for given TC show + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t pf_qos_tc_lsp_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct pci_dev *pdev; + int tc, ret; + bool lsp; + + /* check if option is implemented in vfd_ops*/ + if (!vfd_ops->set_pf_qos_tc_lsp || !vfd_ops->get_pf_qos_tc_lsp) + return -EOPNOTSUPP; + + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + return ret; + + ret = vfd_ops->get_pf_qos_tc_lsp(pdev, tc, &lsp); + if (ret) + return ret; + + ret = scnprintf(buff, PAGE_SIZE, lsp ? "on\n" : "off\n"); + + return ret; +} + +/** + * pf_qos_tc_lsp_store - handler for PF link strict priority for given TC store + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t pf_qos_tc_lsp_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct pci_dev *pdev; + int tc, ret; + bool lsp; + + /* check if option is implemented in vfd_ops*/ + if (!vfd_ops->set_pf_qos_tc_lsp || !vfd_ops->get_pf_qos_tc_lsp) + return -EOPNOTSUPP; + + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + return ret; + + __parse_bool_data(pdev, buff, "lsp", &lsp); + + ret = vfd_ops->set_pf_qos_tc_lsp(pdev, tc, lsp); + if (ret) { + dev_err(&pdev->dev, "Failed to store PF QoS lsp value.\n"); + return ret; + } + + return count; +} + +/** + * pf_qos_tc_max_bw_show - handler for PF's max bandwidth for given TC show + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t pf_qos_tc_max_bw_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct pci_dev *pdev; + int tc, ret; + u16 max_bw; + + if (!vfd_ops->set_pf_qos_tc_max_bw || !vfd_ops->get_pf_qos_tc_max_bw) + return -EOPNOTSUPP; + + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + return ret; + + ret = vfd_ops->get_pf_qos_tc_max_bw(pdev, tc, &max_bw); + if (ret) + return ret; + + ret = scnprintf(buff, PAGE_SIZE, "%d\n", max_bw); + + return ret; +} + +/** + * pf_qos_tc_max_bw_store - handler for PF's max bandwidth for given TC store + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t pf_qos_tc_max_bw_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct pci_dev *pdev; + int tc, ret; + u16 bw; + + if (!vfd_ops->set_pf_qos_tc_max_bw || !vfd_ops->get_pf_qos_tc_max_bw) + return -EOPNOTSUPP; + + ret = __get_pdev_tc(kobj, &pdev, &tc); + if (ret) + return ret; + + ret = kstrtou16(buff, 10, &bw); + if (ret) { + dev_err(&pdev->dev, "Invalid input\n"); + return ret; + } + + ret = vfd_ops->set_pf_qos_tc_max_bw(pdev, tc, + round_nearest_quanta(bw)); + if (ret) + return ret; + + return count; +} + +/** + * vf_max_tc_tx_rate_show - handler for VF's max per TC tx rate show + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t vf_max_tc_tx_rate_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + int tc, vf_id, tc_tx_rate, ret; + struct pci_dev *pdev; + + if (!vfd_ops->set_vf_max_tc_tx_rate || !vfd_ops->get_vf_max_tc_tx_rate) + return -EOPNOTSUPP; + + ret = __get_vf_tc_pdev(kobj, &pdev, &vf_id, &tc); + if (ret) + return ret; + + ret = vfd_ops->get_vf_max_tc_tx_rate(pdev, vf_id, tc, &tc_tx_rate); + if (ret) + return ret; + + ret = scnprintf(buff, PAGE_SIZE, "%d\n", tc_tx_rate); + return ret; +} + +/** + * vf_max_tc_tx_rate_store - handler for VF's max per TC tx rate store + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t vf_max_tc_tx_rate_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + int tc, vf_id, tc_tx_rate, ret; + struct pci_dev *pdev; + + if (!vfd_ops->set_vf_max_tc_tx_rate || !vfd_ops->get_vf_max_tc_tx_rate) + return -EOPNOTSUPP; + + ret = __get_vf_tc_pdev(kobj, &pdev, &vf_id, &tc); + if (ret) + return ret; + ret = kstrtoint(buff, 10, &tc_tx_rate); + if (ret) { + dev_err(&pdev->dev, + "Invalid input, provide bandwidth as number.\n"); + return ret; + } + + ret = vfd_ops->set_vf_max_tc_tx_rate(pdev, vf_id, tc, + round_nearest_quanta(tc_tx_rate)); + if (ret) { + dev_err(&pdev->dev, + "Failed to assign max TC tx rate.\n"); + return ret; + } + + return count; +} + +/** + * vf_qos_tc_share_show - handler for VF bandwidth share per TC show + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer for data + **/ +static ssize_t vf_qos_tc_share_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + struct pci_dev *pdev; + int tc, vf_id, ret; + u8 share; + + if (!vfd_ops->set_vf_qos_tc_share || !vfd_ops->get_vf_qos_tc_share) + return -EOPNOTSUPP; + + ret = __get_vf_tc_pdev(kobj, &pdev, &vf_id, &tc); + if (ret) + return ret; + + ret = vfd_ops->get_vf_qos_tc_share(pdev, vf_id, tc, &share); + if (ret) + return ret; + + ret = scnprintf(buff, PAGE_SIZE, "%d\n", share); + return ret; +} + +/** + * vf_qos_tc_share_store - handler for VF bandwidth share per TC store + * @kobj: kobject being called + * @attr: struct kobj_attribute + * @buff: buffer with input data + * @count: size of buff + * + * On success return count, indicating that we used the whole buffer. On + * failure return a negative error condition. + **/ +static ssize_t vf_qos_tc_share_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buff, size_t count) +{ + struct pci_dev *pdev; + int tc, vf_id, ret; + u8 share; + + if (!vfd_ops->set_vf_qos_tc_share || !vfd_ops->get_vf_qos_tc_share) + return -EOPNOTSUPP; + + ret = __get_vf_tc_pdev(kobj, &pdev, &vf_id, &tc); + if (ret) + return ret; + + ret = kstrtou8(buff, 10, &share); + if (ret) { + dev_err(&pdev->dev, "Invalid input\n"); + return ret; + } + + if (share > 100) { + dev_err(&pdev->dev, "Share must be in range 0-100.\n"); + return -EINVAL; + } + + ret = vfd_ops->set_vf_qos_tc_share(pdev, vf_id, tc, share); + if (ret) + return ret; + + return count; +} + static struct kobj_attribute trunk_attribute = __ATTR(trunk, 0644, vfd_trunk_show, vfd_trunk_store); static struct kobj_attribute vlan_mirror_attribute = @@ -2296,8 +2834,6 @@ static struct kobj_attribute max_tx_rate_attribute = __ATTR(max_tx_rate, 0644, vfd_max_tx_rate_show, vfd_max_tx_rate_store); static struct kobj_attribute min_tx_rate_attribute = __ATTR(min_tx_rate, 0644, vfd_min_tx_rate_show, vfd_min_tx_rate_store); -static struct kobj_attribute spoofcheck_attribute = - __ATTR(spoofcheck, 0644, vfd_spoofcheck_show, vfd_spoofcheck_store); static struct kobj_attribute trust_attribute = __ATTR(trust, 0644, vfd_trust_show, vfd_trust_store); static struct kobj_attribute reset_stats_attribute = @@ -2306,6 +2842,10 @@ static struct kobj_attribute enable_attribute = __ATTR(enable, 0644, vfd_enable_show, vfd_enable_store); static struct kobj_attribute num_queues_attribute = __ATTR(num_queues, 0644, vfd_num_queues_show, vfd_num_queues_store); +static struct kobj_attribute queue_type_attribute = + __ATTR(queue_type, 0644, vfd_queue_type_show, vfd_queue_type_store); +static struct kobj_attribute allow_bcast_attribute = + __ATTR(allow_bcast, 0644, vfd_allow_bcast_show, vfd_allow_bcast_store); static struct attribute *s_attrs[] = { &trunk_attribute.attr, @@ -2323,11 +2863,12 @@ static struct attribute *s_attrs[] = { &link_state_attribute.attr, &max_tx_rate_attribute.attr, &min_tx_rate_attribute.attr, - &spoofcheck_attribute.attr, &trust_attribute.attr, &reset_stats_attribute.attr, &enable_attribute.attr, &num_queues_attribute.attr, + &queue_type_attribute.attr, + &allow_bcast_attribute.attr, NULL, }; @@ -2378,7 +2919,6 @@ static struct attribute *qos_attrs[] = { }; static struct attribute_group qos_group = { - .name = "qos", .attrs = qos_attrs, }; @@ -2391,7 +2931,6 @@ static struct attribute *pf_qos_attrs[] = { }; static struct attribute_group pf_qos_group = { - .name = "qos", .attrs = pf_qos_attrs, }; @@ -2413,16 +2952,110 @@ static struct attribute_group pf_attr_group = { .attrs = pf_attrs, }; +static struct kobj_attribute vf_qos_tc_max_tc_tx_rate_attribute = + __ATTR(max_tc_tx_rate, 0644, vf_max_tc_tx_rate_show, + vf_max_tc_tx_rate_store); +static struct kobj_attribute vf_qos_tc_share_attribute = + __ATTR(share, 0644, vf_qos_tc_share_show, + vf_qos_tc_share_store); + +static struct attribute *vf_qos_tc_attrs[] = { + &vf_qos_tc_max_tc_tx_rate_attribute.attr, + &vf_qos_tc_share_attribute.attr, + NULL, +}; + +static struct attribute_group vf_qos_tc_group = { + .attrs = vf_qos_tc_attrs, +}; + +static struct kobj_attribute pf_qos_tc_priority_attribute = + __ATTR(priority, 0644, pf_qos_tc_priority_show, + pf_qos_tc_priority_store); +static struct kobj_attribute pf_qos_tc_lsp_attribute = + __ATTR(lsp, 0644, pf_qos_tc_lsp_show, pf_qos_tc_lsp_store); +static struct kobj_attribute pf_qos_tc_max_bw_attribute = + __ATTR(max_bw, 0644, pf_qos_tc_max_bw_show, pf_qos_tc_max_bw_store); + +static struct attribute *pf_qos_tc_attrs[] = { + &pf_qos_tc_priority_attribute.attr, + &pf_qos_tc_lsp_attribute.attr, + &pf_qos_tc_max_bw_attribute.attr, + NULL, +}; + +static struct attribute_group pf_qos_tc_group = { + .attrs = pf_qos_tc_attrs, +}; + +/** + * create_qos_tc_sysfs - create sysfs hierarchy for PF QOS traffic classes. + * @pdev: PCI device information struct + * @tc: Pointer to preallocated array of 8 kobjects. + * @parent: QOS parent + * @attr_group: attribute group to assign to tc kobject + * + * Creates a kobject for PF QOS traffic classes and assigns attributes to it. + * Assumes mem is preallocated + **/ +static int create_qos_tc_sysfs(struct pci_dev *pdev, struct kobject **tc, + struct kobject *parent, + struct attribute_group *attr_group) +{ + struct kobject *pf_qos_tc; + char kname[2]; + int ret, i; + + for (i = 0; i < VFD_NUM_TC; i++) { + int length = snprintf(kname, sizeof(kname), "%d", i); + + if (length >= sizeof(kname)) { + dev_err(&pdev->dev, + "cannot request %d tcs, try again with smaller number of vfs\n", + i); + --i; + ret = -EINVAL; + goto err_qos_tc_sysfs; + } + pf_qos_tc = kobject_create_and_add(kname, parent); + + if (!pf_qos_tc) { + dev_err(&pdev->dev, + "failed to create VF kobj: %s\n", kname); + i--; + ret = -ENOMEM; + goto err_qos_tc_sysfs; + } + dev_info(&pdev->dev, "created VF %s sysfs", parent->name); + tc[i] = pf_qos_tc; + + /* create VF sys attr */ + ret = sysfs_create_group(tc[i], attr_group); + if (ret) { + dev_err(&pdev->dev, "failed to create PF QOS TC attributes: %d", + i); + goto err_qos_tc_sysfs; + } + } + + return 0; +err_qos_tc_sysfs: + for (; i >= 0; i--) + kobject_put(tc[i]); + return ret; +} + /** * create_vfs_sysfs - create sysfs hierarchy for VF - * @pdev: PCI device information struct - * @vfd_obj: VF-d kobjects information struct + * @pdev: PCI device information struct + * @vfd_obj: VF-d kobjects information struct * * Creates a kobject for Virtual Function and assigns attributes to it. **/ static int create_vfs_sysfs(struct pci_dev *pdev, struct vfd_objects *vfd_obj) { struct kobject *vf_kobj; + struct vfd_vf_obj *vfs; char kname[4]; int ret, i; @@ -2438,6 +3071,8 @@ static int create_vfs_sysfs(struct pci_dev *pdev, struct vfd_objects *vfd_obj) goto err_vfs_sysfs; } + vfs = &vfd_obj->vfs[i]; + vf_kobj = kobject_create_and_add(kname, vfd_obj->sriov_kobj); if (!vf_kobj) { dev_err(&pdev->dev, @@ -2447,26 +3082,32 @@ static int create_vfs_sysfs(struct pci_dev *pdev, struct vfd_objects *vfd_obj) goto err_vfs_sysfs; } dev_info(&pdev->dev, "created VF %s sysfs", vf_kobj->name); - vfd_obj->vf_kobj[i] = vf_kobj; + vfs->vf_kobj = vf_kobj; + + vfs->vf_qos_kobj = kobject_create_and_add("qos", vfs->vf_kobj); + create_qos_tc_sysfs(pdev, vfs->vf_tc_kobjs, vfs->vf_qos_kobj, + &vf_qos_tc_group); /* create VF sys attr */ - ret = sysfs_create_group(vfd_obj->vf_kobj[i], &vfd_group); + ret = sysfs_create_group(vfs->vf_kobj, &vfd_group); if (ret) { - dev_err(&pdev->dev, "failed to create VF sys attribute: %d", i); + dev_err(&pdev->dev, "failed to create VF sys attribute: %d", + i); goto err_vfs_sysfs; } - /* create VF stats sys attr */ - ret = sysfs_create_group(vfd_obj->vf_kobj[i], &stats_group); + ret = sysfs_create_group(vfs->vf_kobj, &stats_group); if (ret) { - dev_err(&pdev->dev, "failed to create VF stats attribute: %d", i); + dev_err(&pdev->dev, "failed to create VF stats attribute: %d", + i); goto err_vfs_sysfs; } /* create VF qos sys attr */ - ret = sysfs_create_group(vfd_obj->vf_kobj[i], &qos_group); + ret = sysfs_create_group(vfs->vf_qos_kobj, &qos_group); if (ret) { - dev_err(&pdev->dev, "failed to create VF qos attribute: %d", i); + dev_err(&pdev->dev, "failed to create VF qos attribute: %d", + i); goto err_vfs_sysfs; } } @@ -2475,76 +3116,124 @@ static int create_vfs_sysfs(struct pci_dev *pdev, struct vfd_objects *vfd_obj) err_vfs_sysfs: for (; i >= 0; i--) - kobject_put(vfd_obj->vf_kobj[i]); + kobject_put(vfd_obj->vfs[i].vf_kobj); return ret; } /** * create_vfd_sysfs - create sysfs hierarchy used by VF-d - * @pdev: PCI device information struct - * @num_alloc_vfs: number of VFs to allocate + * @pdev: PCI device information struct + * @num_alloc_vfs: number of VFs to allocate * * If the kobjects were not able to be created, NULL will be returned. **/ struct vfd_objects *create_vfd_sysfs(struct pci_dev *pdev, int num_alloc_vfs) { + struct vfd_qos_objects *qos_objs; struct vfd_objects *vfd_obj; int ret; - vfd_obj = kzalloc(sizeof(*vfd_obj) + - sizeof(struct kobject *)*num_alloc_vfs, GFP_KERNEL); + vfd_obj = kzalloc(sizeof(*vfd_obj), GFP_KERNEL); if (!vfd_obj) return NULL; - vfd_obj->num_vfs = num_alloc_vfs; + qos_objs = kzalloc(sizeof(*qos_objs), GFP_KERNEL); + if (!qos_objs) + goto err_qos; + vfd_obj->vfs = kcalloc(num_alloc_vfs, sizeof(*vfd_obj->vfs), + GFP_KERNEL); + if (!vfd_obj->vfs) + goto err_vfs; + + vfd_obj->qos = qos_objs; + vfd_obj->num_vfs = num_alloc_vfs; vfd_obj->sriov_kobj = kobject_create_and_add("sriov", &pdev->dev.kobj); if (!vfd_obj->sriov_kobj) goto err_sysfs; - dev_info(&pdev->dev, "created %s sysfs", vfd_obj->sriov_kobj->name); + qos_objs->qos_kobj = kobject_create_and_add("qos", + vfd_obj->sriov_kobj); + if (!qos_objs->qos_kobj) { + dev_err(&pdev->dev, "failed to create VF qos pf kobject"); + goto err_pf_qos; + } + ret = create_vfs_sysfs(pdev, vfd_obj); if (ret) - goto err_sysfs; + goto err_pf_qos; + create_qos_tc_sysfs(pdev, qos_objs->pf_qos_kobjs, qos_objs->qos_kobj, + &pf_qos_tc_group); /* create PF qos sys attr */ - ret = sysfs_create_group(vfd_obj->sriov_kobj, &pf_qos_group); + ret = sysfs_create_group(qos_objs->qos_kobj, &pf_qos_group); if (ret) { dev_err(&pdev->dev, "failed to create PF qos sys attribute"); - goto err_sysfs; + goto err_pf_qos; } /* create PF attrs */ ret = sysfs_create_group(vfd_obj->sriov_kobj, &pf_attr_group); if (ret) { dev_err(&pdev->dev, "failed to create PF attr sys attribute"); - goto err_sysfs; + goto err_pf_qos; } + return vfd_obj; -err_sysfs: +err_pf_qos: kobject_put(vfd_obj->sriov_kobj); +err_sysfs: + kfree(vfd_obj->vfs); +err_vfs: + kfree(qos_objs); +err_qos: kfree(vfd_obj); return NULL; } +static void free_vfd_vf(struct pci_dev *pdev, struct vfd_vf_obj *vf) +{ + int i; + + for (i = 0; i < VFD_NUM_TC; i++) { + dev_dbg(&pdev->dev, "deleting VF %s tc", + vf->vf_tc_kobjs[i]->name); + kobject_put(vf->vf_tc_kobjs[i]); + } + + dev_info(&pdev->dev, "deleting VF %s sysfs", vf->vf_qos_kobj->name); + kobject_put(vf->vf_qos_kobj); + dev_info(&pdev->dev, "deleting VF %s sysfs", vf->vf_kobj->name); + kobject_put(vf->vf_kobj); +} + /** * destroy_vfd_sysfs - destroy sysfs hierarchy used by VF-d * @pdev: PCI device information struct - * @vfd_obj: VF-d kobjects information struct + * @vfd_obj: VF-d kobjects information struct **/ void destroy_vfd_sysfs(struct pci_dev *pdev, struct vfd_objects *vfd_obj) { int i; - for (i = 0; i < vfd_obj->num_vfs; i++) { - dev_info(&pdev->dev, "deleting VF %s sysfs", - vfd_obj->vf_kobj[i]->name); - kobject_put(vfd_obj->vf_kobj[i]); + for (i = 0; i < vfd_obj->num_vfs; i++) + free_vfd_vf(pdev, &vfd_obj->vfs[i]); + + for (i = 0; i < VFD_NUM_TC; i++) { + dev_info(&pdev->dev, "deleting sriov qos %s sysfs", + vfd_obj->qos->pf_qos_kobjs[i]->name); + kobject_put(vfd_obj->qos->pf_qos_kobjs[i]); } + dev_info(&pdev->dev, "deleting %s sysfs", + vfd_obj->qos->qos_kobj->name); + kobject_put(vfd_obj->qos->qos_kobj); + dev_info(&pdev->dev, "deleting %s sysfs", vfd_obj->sriov_kobj->name); kobject_put(vfd_obj->sriov_kobj); + kfree(vfd_obj->qos); + kfree(vfd_obj->vfs); kfree(vfd_obj); } diff --git a/drivers/net/ethernet/intel/iavf/kcompat_vfd.h b/drivers/net/ethernet/intel/iavf/kcompat_vfd.h index e5f391864..53de8238d 100644 --- a/drivers/net/ethernet/intel/iavf/kcompat_vfd.h +++ b/drivers/net/ethernet/intel/iavf/kcompat_vfd.h @@ -15,6 +15,11 @@ #define VFD_EGRESS_MIRROR_OFF -1 #define VFD_INGRESS_MIRROR_OFF -1 +#define VFD_QUEUE_TYPE_RSS 0x00 +#define VFD_QUEUE_TYPE_QOS 0x01 + +#define VFD_NUM_TC 0x8 + /** * struct vfd_objects - VF-d kobjects information struct * @num_vfs: number of VFs allocated @@ -24,7 +29,30 @@ struct vfd_objects { int num_vfs; struct kobject *sriov_kobj; - struct kobject *vf_kobj[0]; + struct vfd_vf_obj *vfs; + struct vfd_qos_objects *qos; +}; + +/** + * struct vfd_vf_obj - VF-d VF kobjects information struct + * @vf_kobj: pointer to VF qos kobject + * @vf_qos_kobj: pointer to VF kobject + * @vf_tc_kobj: pointer to VF TC kobjects + */ +struct vfd_vf_obj { + struct kobject *vf_qos_kobj; + struct kobject *vf_kobj; + struct kobject *vf_tc_kobjs[VFD_NUM_TC]; +}; + +/** + * struct vfd_qos_objects - VF-d qos kobjects information struct + * @qos_kobj: pointer to PF qos kobject + * @pf_qos_kobj: pointer to PF TC kobjects + */ +struct vfd_qos_objects { + struct kobject *qos_kobj; + struct kobject *pf_qos_kobjs[VFD_NUM_TC]; }; struct vfd_macaddr { @@ -134,6 +162,26 @@ struct vfd_ops { int (*set_num_queues)(struct pci_dev *pdev, int vf_id, const int num_queues); int (*get_trust_state)(struct pci_dev *pdev, int vf_id, bool *data); int (*set_trust_state)(struct pci_dev *pdev, int vf_id, bool data); + int (*get_queue_type)(struct pci_dev *pdev, int vf_id, u8 *data); + int (*set_queue_type)(struct pci_dev *pdev, int vf_id, const u8 data); + int (*get_allow_bcast)(struct pci_dev *pdev, int vf_id, bool *data); + int (*set_allow_bcast)(struct pci_dev *pdev, int vf_id, const bool data); + int (*get_pf_qos_tc_max_bw)(struct pci_dev *pdev, int tc, u16 *req_bw); + int (*set_pf_qos_tc_max_bw)(struct pci_dev *pdev, int tc, u16 req_bw); + int (*get_pf_qos_tc_lsp)(struct pci_dev *pdev, int tc, bool *on); + int (*set_pf_qos_tc_lsp)(struct pci_dev *pdev, int tc, bool on); + int (*get_pf_qos_tc_priority)(struct pci_dev *pdev, int tc, + char *tc_bitmap); + int (*set_pf_qos_tc_priority)(struct pci_dev *pdev, int tc, + char tc_bitmap); + int (*get_vf_qos_tc_share)(struct pci_dev *pdev, int vf_id, int tc, + u8 *share); + int (*set_vf_qos_tc_share)(struct pci_dev *pdev, int vf_id, int tc, + u8 share); + int (*get_vf_max_tc_tx_rate)(struct pci_dev *pdev, int vf_id, int tc, + int *rate); + int (*set_vf_max_tc_tx_rate)(struct pci_dev *pdev, int vf_id, int tc, + int rate); }; extern const struct vfd_ops *vfd_ops; diff --git a/drivers/net/ethernet/intel/iavf/virtchnl.h b/drivers/net/ethernet/intel/iavf/virtchnl.h index 217798188..38347ac53 100644 --- a/drivers/net/ethernet/intel/iavf/virtchnl.h +++ b/drivers/net/ethernet/intel/iavf/virtchnl.h @@ -5,8 +5,9 @@ #define _VIRTCHNL_H_ /* Description: - * This header file describes the VF-PF communication protocol used - * by the drivers for all devices starting from our 40G product line + * This header file describes the Virtual Function (VF) - Physical Function + * (PF) communication protocol used by the drivers for all devices starting + * from our 40G product line * * Admin queue buffer usage: * desc->opcode is always aqc_opc_send_msg_to_pf @@ -20,8 +21,8 @@ * have a maximum of sixteen queues for all of its VSIs. * * The PF is required to return a status code in v_retval for all messages - * except RESET_VF, which does not require any response. The return value - * is of status_code type, defined in the shared type.h. + * except RESET_VF, which does not require any response. The returned value + * is of virtchnl_status_code type, defined in the shared type.h. * * In general, VF driver initialization should roughly follow the order of * these opcodes. The VF driver must first validate the API version of the @@ -114,9 +115,7 @@ enum virtchnl_ops { VIRTCHNL_OP_RSVD = 16, VIRTCHNL_OP_EVENT = 17, /* must ALWAYS be 17 */ /* opcode 19 is reserved */ - VIRTCHNL_OP_IWARP = 20, /* advanced opcode */ - VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP = 21, /* advanced opcode */ - VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP = 22, /* advanced opcode */ + /* opcodes 20, 21, and 22 are reserved */ VIRTCHNL_OP_CONFIG_RSS_KEY = 23, VIRTCHNL_OP_CONFIG_RSS_LUT = 24, VIRTCHNL_OP_GET_RSS_HENA_CAPS = 25, @@ -128,9 +127,141 @@ enum virtchnl_ops { VIRTCHNL_OP_DISABLE_CHANNELS = 31, VIRTCHNL_OP_ADD_CLOUD_FILTER = 32, VIRTCHNL_OP_DEL_CLOUD_FILTER = 33, - /* opcodes 34, 35, 36, 37 and 38 are reserved */ + /* opcode 34 is reserved */ + /* opcodes 38, 39, 40, 41, 42 and 43 are reserved */ + VIRTCHNL_OP_GET_SUPPORTED_RXDIDS = 44, + /* opcode 45, 46, 47, 48 and 49 are reserved */ + VIRTCHNL_OP_GET_MAX_RSS_QREGION = 50, + VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS = 51, + VIRTCHNL_OP_ADD_VLAN_V2 = 52, + VIRTCHNL_OP_DEL_VLAN_V2 = 53, + VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 = 54, + VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 = 55, + VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 = 56, + VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2 = 57, + VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2 = 58, + VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2 = 59, + VIRTCHNL_OP_1588_PTP_GET_CAPS = 60, + VIRTCHNL_OP_1588_PTP_GET_TIME = 61, + VIRTCHNL_OP_1588_PTP_SET_TIME = 62, + VIRTCHNL_OP_1588_PTP_ADJ_TIME = 63, + VIRTCHNL_OP_1588_PTP_ADJ_FREQ = 64, + VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP = 65, + /* opcode 66, 67, 68, and 69 are reserved */ + VIRTCHNL_OP_ENABLE_QUEUES_V2 = 107, + VIRTCHNL_OP_DISABLE_QUEUES_V2 = 108, + VIRTCHNL_OP_MAP_QUEUE_VECTOR = 111, + VIRTCHNL_OP_MAX, }; +static inline const char *virtchnl_op_str(enum virtchnl_ops v_opcode) +{ + switch (v_opcode) { + case VIRTCHNL_OP_UNKNOWN: + return "VIRTCHNL_OP_UNKNOWN"; + case VIRTCHNL_OP_VERSION: + return "VIRTCHNL_OP_VERSION"; + case VIRTCHNL_OP_RESET_VF: + return "VIRTCHNL_OP_RESET_VF"; + case VIRTCHNL_OP_GET_VF_RESOURCES: + return "VIRTCHNL_OP_GET_VF_RESOURCES"; + case VIRTCHNL_OP_CONFIG_TX_QUEUE: + return "VIRTCHNL_OP_CONFIG_TX_QUEUE"; + case VIRTCHNL_OP_CONFIG_RX_QUEUE: + return "VIRTCHNL_OP_CONFIG_RX_QUEUE"; + case VIRTCHNL_OP_CONFIG_VSI_QUEUES: + return "VIRTCHNL_OP_CONFIG_VSI_QUEUES"; + case VIRTCHNL_OP_CONFIG_IRQ_MAP: + return "VIRTCHNL_OP_CONFIG_IRQ_MAP"; + case VIRTCHNL_OP_ENABLE_QUEUES: + return "VIRTCHNL_OP_ENABLE_QUEUES"; + case VIRTCHNL_OP_DISABLE_QUEUES: + return "VIRTCHNL_OP_DISABLE_QUEUES"; + case VIRTCHNL_OP_ADD_ETH_ADDR: + return "VIRTCHNL_OP_ADD_ETH_ADDR"; + case VIRTCHNL_OP_DEL_ETH_ADDR: + return "VIRTCHNL_OP_DEL_ETH_ADDR"; + case VIRTCHNL_OP_ADD_VLAN: + return "VIRTCHNL_OP_ADD_VLAN"; + case VIRTCHNL_OP_DEL_VLAN: + return "VIRTCHNL_OP_DEL_VLAN"; + case VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: + return "VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE"; + case VIRTCHNL_OP_GET_STATS: + return "VIRTCHNL_OP_GET_STATS"; + case VIRTCHNL_OP_RSVD: + return "VIRTCHNL_OP_RSVD"; + case VIRTCHNL_OP_EVENT: + return "VIRTCHNL_OP_EVENT"; + case VIRTCHNL_OP_CONFIG_RSS_KEY: + return "VIRTCHNL_OP_CONFIG_RSS_KEY"; + case VIRTCHNL_OP_CONFIG_RSS_LUT: + return "VIRTCHNL_OP_CONFIG_RSS_LUT"; + case VIRTCHNL_OP_GET_RSS_HENA_CAPS: + return "VIRTCHNL_OP_GET_RSS_HENA_CAPS"; + case VIRTCHNL_OP_SET_RSS_HENA: + return "VIRTCHNL_OP_SET_RSS_HENA"; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: + return "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING"; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING: + return "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING"; + case VIRTCHNL_OP_REQUEST_QUEUES: + return "VIRTCHNL_OP_REQUEST_QUEUES"; + case VIRTCHNL_OP_ENABLE_CHANNELS: + return "VIRTCHNL_OP_ENABLE_CHANNELS"; + case VIRTCHNL_OP_DISABLE_CHANNELS: + return "VIRTCHNL_OP_DISABLE_CHANNELS"; + case VIRTCHNL_OP_ADD_CLOUD_FILTER: + return "VIRTCHNL_OP_ADD_CLOUD_FILTER"; + case VIRTCHNL_OP_DEL_CLOUD_FILTER: + return "VIRTCHNL_OP_DEL_CLOUD_FILTER"; + case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS: + return "VIRTCHNL_OP_GET_SUPPORTED_RXDIDS"; + case VIRTCHNL_OP_GET_MAX_RSS_QREGION: + return "VIRTCHNL_OP_GET_MAX_RSS_QREGION"; + case VIRTCHNL_OP_ENABLE_QUEUES_V2: + return "VIRTCHNL_OP_ENABLE_QUEUES_V2"; + case VIRTCHNL_OP_DISABLE_QUEUES_V2: + return "VIRTCHNL_OP_DISABLE_QUEUES_V2"; + case VIRTCHNL_OP_MAP_QUEUE_VECTOR: + return "VIRTCHNL_OP_MAP_QUEUE_VECTOR"; + case VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS: + return "VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS"; + case VIRTCHNL_OP_ADD_VLAN_V2: + return "VIRTCHNL_OP_ADD_VLAN_V2"; + case VIRTCHNL_OP_DEL_VLAN_V2: + return "VIRTCHNL_OP_DEL_VLAN_V2"; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2: + return "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2"; + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2: + return "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2"; + case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2: + return "VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2"; + case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2: + return "VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2"; + case VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2: + return "VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2"; + case VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2: + return "VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2"; + case VIRTCHNL_OP_1588_PTP_GET_CAPS: + return "VIRTCHNL_OP_1588_PTP_GET_CAPS"; + case VIRTCHNL_OP_1588_PTP_GET_TIME: + return "VIRTCHNL_OP_1588_PTP_GET_TIME"; + case VIRTCHNL_OP_1588_PTP_SET_TIME: + return "VIRTCHNL_OP_1588_PTP_SET_TIME"; + case VIRTCHNL_OP_1588_PTP_ADJ_TIME: + return "VIRTCHNL_OP_1588_PTP_ADJ_TIME"; + case VIRTCHNL_OP_1588_PTP_ADJ_FREQ: + return "VIRTCHNL_OP_1588_PTP_ADJ_FREQ"; + case VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP: + return "VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP"; + case VIRTCHNL_OP_MAX: + return "VIRTCHNL_OP_MAX"; + default: + return "Unsupported (update virtchnl.h)"; + } +} + /* These macros are used to generate compilation errors if a structure/union * is not exactly the correct length. It gives a divide by zero error if the * structure/union is not of the correct size, otherwise it creates an enum @@ -141,19 +272,6 @@ enum virtchnl_ops { #define VIRTCHNL_CHECK_UNION_LEN(n, X) enum virtchnl_static_asset_enum_##X \ { virtchnl_static_assert_##X = (n)/((sizeof(union X) == (n)) ? 1 : 0) } -/* Virtual channel message descriptor. This overlays the admin queue - * descriptor. All other data is passed in external buffers. - */ - -struct virtchnl_msg { - u8 pad[8]; /* AQ flags/opcode/len/retval fields */ - enum virtchnl_ops v_opcode; /* avoid confusion with desc->opcode */ - enum virtchnl_status_code v_retval; /* ditto for desc->retval */ - u32 vfid; /* used by PF when sending to VF */ -}; - -VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_msg); - /* Message descriptions and data structures. */ /* VIRTCHNL_OP_VERSION @@ -170,6 +288,8 @@ VIRTCHNL_CHECK_STRUCT_LEN(20, virtchnl_msg); */ #define VIRTCHNL_VERSION_MAJOR 1 #define VIRTCHNL_VERSION_MINOR 1 +#define VIRTCHNL_VERSION_MAJOR_2 2 +#define VIRTCHNL_VERSION_MINOR_0 0 #define VIRTCHNL_VERSION_MINOR_NO_VF_CAPS 0 struct virtchnl_version_info { @@ -214,7 +334,9 @@ enum virtchnl_vsi_type { struct virtchnl_vsi_resource { u16 vsi_id; u16 num_queue_pairs; - enum virtchnl_vsi_type vsi_type; + + /* see enum virtchnl_vsi_type */ + s32 vsi_type; u16 qset_handle; u8 default_mac_addr[ETH_ALEN]; }; @@ -227,12 +349,18 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); */ #define VIRTCHNL_VF_OFFLOAD_L2 0x00000001 #define VIRTCHNL_VF_OFFLOAD_IWARP 0x00000002 +#define VIRTCHNL_VF_CAP_RDMA VIRTCHNL_VF_OFFLOAD_IWARP #define VIRTCHNL_VF_OFFLOAD_RSVD 0x00000004 #define VIRTCHNL_VF_OFFLOAD_RSS_AQ 0x00000008 #define VIRTCHNL_VF_OFFLOAD_RSS_REG 0x00000010 #define VIRTCHNL_VF_OFFLOAD_WB_ON_ITR 0x00000020 #define VIRTCHNL_VF_OFFLOAD_REQ_QUEUES 0x00000040 -#define VIRTCHNL_VF_OFFLOAD_CRC 0x00000080 +/* used to negotiate communicating link speeds in Mbps */ +#define VIRTCHNL_VF_CAP_ADV_LINK_SPEED 0x00000080 + /* 0X00000100 is reserved */ +#define VIRTCHNL_VF_LARGE_NUM_QPAIRS 0x00000200 +#define VIRTCHNL_VF_OFFLOAD_CRC 0x00000400 +#define VIRTCHNL_VF_OFFLOAD_VLAN_V2 0x00008000 #define VIRTCHNL_VF_OFFLOAD_VLAN 0x00010000 #define VIRTCHNL_VF_OFFLOAD_RX_POLLING 0x00020000 #define VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 0x00040000 @@ -243,10 +371,12 @@ VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vsi_resource); #define VIRTCHNL_VF_OFFLOAD_ADQ 0X00800000 #define VIRTCHNL_VF_OFFLOAD_ADQ_V2 0X01000000 #define VIRTCHNL_VF_OFFLOAD_USO 0X02000000 - /* 0X80000000 is reserved */ +#define VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC 0X04000000 + /* 0X08000000 and 0X10000000 are reserved */ + /* 0X20000000 is reserved */ + /* 0X40000000 is reserved */ +#define VIRTCHNL_VF_CAP_PTP 0X80000000 -/* Define below the capability flags that are not offloads */ -#define VIRTCHNL_VF_CAP_ADV_LINK_SPEED 0x00000080 #define VF_BASE_MODE_OFFLOADS (VIRTCHNL_VF_OFFLOAD_L2 | \ VIRTCHNL_VF_OFFLOAD_VLAN | \ VIRTCHNL_VF_OFFLOAD_RSS_PF) @@ -284,6 +414,46 @@ struct virtchnl_txq_info { VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_txq_info); +/* RX descriptor IDs (range from 0 to 63) */ +enum virtchnl_rx_desc_ids { + VIRTCHNL_RXDID_0_16B_BASE = 0, + VIRTCHNL_RXDID_1_32B_BASE = 1, + VIRTCHNL_RXDID_2_FLEX_SQ_NIC = 2, + VIRTCHNL_RXDID_3_FLEX_SQ_SW = 3, + VIRTCHNL_RXDID_4_FLEX_SQ_NIC_VEB = 4, + VIRTCHNL_RXDID_5_FLEX_SQ_NIC_ACL = 5, + VIRTCHNL_RXDID_6_FLEX_SQ_NIC_2 = 6, + VIRTCHNL_RXDID_7_HW_RSVD = 7, + /* 8 through 15 are reserved */ + VIRTCHNL_RXDID_16_COMMS_GENERIC = 16, + VIRTCHNL_RXDID_17_COMMS_AUX_VLAN = 17, + VIRTCHNL_RXDID_18_COMMS_AUX_IPV4 = 18, + VIRTCHNL_RXDID_19_COMMS_AUX_IPV6 = 19, + VIRTCHNL_RXDID_20_COMMS_AUX_FLOW = 20, + VIRTCHNL_RXDID_21_COMMS_AUX_TCP = 21, + /* 22 through 63 are reserved */ +}; + +/* RX descriptor ID bitmasks */ +enum virtchnl_rx_desc_id_bitmasks { + VIRTCHNL_RXDID_0_16B_BASE_M = BIT(VIRTCHNL_RXDID_0_16B_BASE), + VIRTCHNL_RXDID_1_32B_BASE_M = BIT(VIRTCHNL_RXDID_1_32B_BASE), + VIRTCHNL_RXDID_2_FLEX_SQ_NIC_M = BIT(VIRTCHNL_RXDID_2_FLEX_SQ_NIC), + VIRTCHNL_RXDID_3_FLEX_SQ_SW_M = BIT(VIRTCHNL_RXDID_3_FLEX_SQ_SW), + VIRTCHNL_RXDID_4_FLEX_SQ_NIC_VEB_M = BIT(VIRTCHNL_RXDID_4_FLEX_SQ_NIC_VEB), + VIRTCHNL_RXDID_5_FLEX_SQ_NIC_ACL_M = BIT(VIRTCHNL_RXDID_5_FLEX_SQ_NIC_ACL), + VIRTCHNL_RXDID_6_FLEX_SQ_NIC_2_M = BIT(VIRTCHNL_RXDID_6_FLEX_SQ_NIC_2), + VIRTCHNL_RXDID_7_HW_RSVD_M = BIT(VIRTCHNL_RXDID_7_HW_RSVD), + /* 8 through 15 are reserved */ + VIRTCHNL_RXDID_16_COMMS_GENERIC_M = BIT(VIRTCHNL_RXDID_16_COMMS_GENERIC), + VIRTCHNL_RXDID_17_COMMS_AUX_VLAN_M = BIT(VIRTCHNL_RXDID_17_COMMS_AUX_VLAN), + VIRTCHNL_RXDID_18_COMMS_AUX_IPV4_M = BIT(VIRTCHNL_RXDID_18_COMMS_AUX_IPV4), + VIRTCHNL_RXDID_19_COMMS_AUX_IPV6_M = BIT(VIRTCHNL_RXDID_19_COMMS_AUX_IPV6), + VIRTCHNL_RXDID_20_COMMS_AUX_FLOW_M = BIT(VIRTCHNL_RXDID_20_COMMS_AUX_FLOW), + VIRTCHNL_RXDID_21_COMMS_AUX_TCP_M = BIT(VIRTCHNL_RXDID_21_COMMS_AUX_TCP), + /* 22 through 63 are reserved */ +}; + /* VIRTCHNL_OP_CONFIG_RX_QUEUE * VF sends this message to set up parameters for one RX queue. * External data buffer contains one instance of virtchnl_rxq_info. @@ -306,9 +476,17 @@ struct virtchnl_rxq_info { u32 databuffer_size; u32 max_pkt_size; u8 crc_disable; - u8 pad1[3]; + /* see enum virtchnl_rx_desc_ids; + * only used when VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC is supported. Note + * that when the offload is not supported, the descriptor format aligns + * with VIRTCHNL_RXDID_1_32B_BASE. + */ + u8 rxdid; + u8 pad1[2]; u64 dma_ring_addr; - enum virtchnl_rx_hsplit rx_split_pos; /* deprecated with AVF 1.0 */ + + /* see enum virtchnl_rx_hsplit; deprecated with AVF 1.0 */ + s32 rx_split_pos; u32 pad2; }; @@ -406,6 +584,35 @@ struct virtchnl_queue_select { VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_select); +/* VIRTCHNL_OP_GET_MAX_RSS_QREGION + * + * if VIRTCHNL_VF_LARGE_NUM_QPAIRS was negotiated in VIRTCHNL_OP_GET_VF_RESOURCES + * then this op must be supported. + * + * VF sends this message in order to query the max RSS queue region + * size supported by PF, when VIRTCHNL_VF_LARGE_NUM_QPAIRS is enabled. + * This information should be used when configuring the RSS LUT and/or + * configuring queue region based filters. + * + * The maximum RSS queue region is 2^qregion_width. So, a qregion_width + * of 6 would inform the VF that the PF supports a maximum RSS queue region + * of 64. + * + * A queue region represents a range of queues that can be used to configure + * a RSS LUT. For example, if a VF is given 64 queues, but only a max queue + * region size of 16 (i.e. 2^qregion_width = 16) then it will only be able + * to configure the RSS LUT with queue indices from 0 to 15. However, other + * filters can be used to direct packets to queues >15 via specifying a queue + * base/offset and queue region width. + */ +struct virtchnl_max_rss_qregion { + u16 vport_id; + u16 qregion_width; + u8 pad[4]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_max_rss_qregion); + /* VIRTCHNL_OP_ADD_ETH_ADDR * VF sends this message in order to add one or more unicast or multicast * address filters for the specified VSI. @@ -418,9 +625,36 @@ VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_select); * PF removes the filters and returns status. */ +/* VIRTCHNL_ETHER_ADDR_LEGACY + * Prior to adding the @type member to virtchnl_ether_addr, there were 2 pad + * bytes. Moving forward all VF drivers should not set type to + * VIRTCHNL_ETHER_ADDR_LEGACY. This is only here to not break previous/legacy + * behavior. The control plane function (i.e. PF) can use a best effort method + * of tracking the primary/device unicast in this case, but there is no + * guarantee and functionality depends on the implementation of the PF. + */ + +/* VIRTCHNL_ETHER_ADDR_PRIMARY + * All VF drivers should set @type to VIRTCHNL_ETHER_ADDR_PRIMARY for the + * primary/device unicast MAC address filter for VIRTCHNL_OP_ADD_ETH_ADDR and + * VIRTCHNL_OP_DEL_ETH_ADDR. This allows for the underlying control plane + * function (i.e. PF) to accurately track and use this MAC address for + * displaying on the host and for VM/function reset. + */ + +/* VIRTCHNL_ETHER_ADDR_EXTRA + * All VF drivers should set @type to VIRTCHNL_ETHER_ADDR_EXTRA for any extra + * unicast and/or multicast filters that are being added/deleted via + * VIRTCHNL_OP_DEL_ETH_ADDR/VIRTCHNL_OP_ADD_ETH_ADDR respectively. + */ struct virtchnl_ether_addr { u8 addr[ETH_ALEN]; - u8 pad[2]; + u8 type; +#define VIRTCHNL_ETHER_ADDR_LEGACY 0 +#define VIRTCHNL_ETHER_ADDR_PRIMARY 1 +#define VIRTCHNL_ETHER_ADDR_EXTRA 2 +#define VIRTCHNL_ETHER_ADDR_TYPE_MASK 3 /* first two bits of type are valid */ + u8 pad; }; VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_ether_addr); @@ -455,6 +689,388 @@ struct virtchnl_vlan_filter_list { VIRTCHNL_CHECK_STRUCT_LEN(6, virtchnl_vlan_filter_list); +/* This enum is used for all of the VIRTCHNL_VF_OFFLOAD_VLAN_V2_CAPS related + * structures and opcodes. + * + * VIRTCHNL_VLAN_UNSUPPORTED - This field is not supported and if a VF driver + * populates it the PF should return VIRTCHNL_STATUS_ERR_NOT_SUPPORTED. + * + * VIRTCHNL_VLAN_ETHERTYPE_8100 - This field supports 0x8100 ethertype. + * VIRTCHNL_VLAN_ETHERTYPE_88A8 - This field supports 0x88A8 ethertype. + * VIRTCHNL_VLAN_ETHERTYPE_9100 - This field supports 0x9100 ethertype. + * + * VIRTCHNL_VLAN_ETHERTYPE_AND - Used when multiple ethertypes can be supported + * by the PF concurrently. For example, if the PF can support + * VIRTCHNL_VLAN_ETHERTYPE_8100 AND VIRTCHNL_VLAN_ETHERTYPE_88A8 filters it + * would OR the following bits: + * + * VIRTHCNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * The VF would interpret this as VLAN filtering can be supported on both 0x8100 + * and 0x88A8 VLAN ethertypes. + * + * VIRTCHNL_ETHERTYPE_XOR - Used when only a single ethertype can be supported + * by the PF concurrently. For example if the PF can support + * VIRTCHNL_VLAN_ETHERTYPE_8100 XOR VIRTCHNL_VLAN_ETHERTYPE_88A8 stripping + * offload it would OR the following bits: + * + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * The VF would interpret this as VLAN stripping can be supported on either + * 0x8100 or 0x88a8 VLAN ethertypes. So when requesting VLAN stripping via + * VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 the specified ethertype will override + * the previously set value. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1 - Used to tell the VF to insert and/or + * strip the VLAN tag using the L2TAG1 field of the Tx/Rx descriptors. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 - Used to tell the VF to insert hardware + * offloaded VLAN tags using the L2TAG2 field of the Tx descriptor. + * + * VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 - Used to tell the VF to strip hardware + * offloaded VLAN tags using the L2TAG2_2 field of the Rx descriptor. + * + * VIRTCHNL_VLAN_PRIO - This field supports VLAN priority bits. This is used for + * VLAN filtering if the underlying PF supports it. + * + * VIRTCHNL_VLAN_TOGGLE_ALLOWED - This field is used to say whether a + * certain VLAN capability can be toggled. For example if the underlying PF/CP + * allows the VF to toggle VLAN filtering, stripping, and/or insertion it should + * set this bit along with the supported ethertypes. + */ +enum virtchnl_vlan_support { + VIRTCHNL_VLAN_UNSUPPORTED = 0, + VIRTCHNL_VLAN_ETHERTYPE_8100 = 0x00000001, + VIRTCHNL_VLAN_ETHERTYPE_88A8 = 0x00000002, + VIRTCHNL_VLAN_ETHERTYPE_9100 = 0x00000004, + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1 = 0x00000100, + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2 = 0x00000200, + VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2 = 0x00000400, + VIRTCHNL_VLAN_PRIO = 0x01000000, + VIRTCHNL_VLAN_FILTER_MASK = 0x10000000, + VIRTCHNL_VLAN_ETHERTYPE_AND = 0x20000000, + VIRTCHNL_VLAN_ETHERTYPE_XOR = 0x40000000, + VIRTCHNL_VLAN_TOGGLE = 0x80000000 +}; + +/* This structure is used as part of the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS + * for filtering, insertion, and stripping capabilities. + * + * If only outer capabilities are supported (for filtering, insertion, and/or + * stripping) then this refers to the outer most or single VLAN from the VF's + * perspective. + * + * If only inner capabilities are supported (for filtering, insertion, and/or + * stripping) then this refers to the outer most or single VLAN from the VF's + * perspective. Functionally this is the same as if only outer capabilities are + * supported. The VF driver is just forced to use the inner fields when + * adding/deleting filters and enabling/disabling offloads (if supported). + * + * If both outer and inner capabilities are supported (for filtering, insertion, + * and/or stripping) then outer refers to the outer most or single VLAN and + * inner refers to the second VLAN, if it exists, in the packet. + * + * There is no support for tunneled VLAN offloads, so outer or inner are never + * referring to a tunneled packet from the VF's perspective. + */ +struct virtchnl_vlan_supported_caps { + u32 outer; + u32 inner; +}; + +/* The PF populates these fields based on the supported VLAN filtering. If a + * field is VIRTCHNL_VLAN_UNSUPPORTED then it's not supported and the PF will + * reject any VIRTCHNL_OP_ADD_VLAN_V2 or VIRTCHNL_OP_DEL_VLAN_V2 messages using + * the unsupported fields. + * + * Also, a VF is only allowed to toggle its VLAN filtering setting if the + * VIRTCHNL_VLAN_TOGGLE bit is set. + * + * The ethertype(s) specified in the ethertype_init field are the ethertypes + * enabled for VLAN filtering. VLAN filtering in this case refers to the outer + * most VLAN from the VF's perspective. If both inner and outer filtering are + * allowed then ethertype_init only refers to the outer most VLAN as only + * VLAN ethertype supported for inner VLAN filtering is + * VIRTCHNL_VLAN_ETHERTYPE_8100. By default, inner VLAN filtering is disabled + * when both inner and outer filtering are allowed. + * + * The max_filters field tells the VF how many VLAN filters it's allowed to have + * at any one time. If it exceeds this amount and tries to add another filter, + * then the request will be rejected by the PF. To prevent failures, the VF + * should keep track of how many VLAN filters it has added and not attempt to + * add more than max_filters. + */ +struct virtchnl_vlan_filtering_caps { + struct virtchnl_vlan_supported_caps filtering_support; + u32 ethertype_init; + u16 max_filters; + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vlan_filtering_caps); + +/* This enum is used for the virtchnl_vlan_offload_caps structure to specify + * if the PF supports a different ethertype for stripping and insertion. + * + * VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION - The ethertype(s) specified + * for stripping affect the ethertype(s) specified for insertion and visa versa + * as well. If the VF tries to configure VLAN stripping via + * VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 with VIRTCHNL_VLAN_ETHERTYPE_8100 then + * that will be the ethertype for both stripping and insertion. + * + * VIRTCHNL_ETHERTYPE_MATCH_NOT_REQUIRED - The ethertype(s) specified for + * stripping do not affect the ethertype(s) specified for insertion and visa + * versa. + */ +enum virtchnl_vlan_ethertype_match { + VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION = 0, + VIRTCHNL_ETHERTYPE_MATCH_NOT_REQUIRED = 1, +}; + +/* The PF populates these fields based on the supported VLAN offloads. If a + * field is VIRTCHNL_VLAN_UNSUPPORTED then it's not supported and the PF will + * reject any VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 or + * VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 messages using the unsupported fields. + * + * Also, a VF is only allowed to toggle its VLAN offload setting if the + * VIRTCHNL_VLAN_TOGGLE_ALLOWED bit is set. + * + * The VF driver needs to be aware of how the tags are stripped by hardware and + * inserted by the VF driver based on the level of offload support. The PF will + * populate these fields based on where the VLAN tags are expected to be + * offloaded via the VIRTHCNL_VLAN_TAG_LOCATION_* bits. The VF will need to + * interpret these fields. See the definition of the + * VIRTCHNL_VLAN_TAG_LOCATION_* bits above the virtchnl_vlan_support + * enumeration. + */ +struct virtchnl_vlan_offload_caps { + struct virtchnl_vlan_supported_caps stripping_support; + struct virtchnl_vlan_supported_caps insertion_support; + u32 ethertype_init; + u8 ethertype_match; + u8 pad[3]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_vlan_offload_caps); + +/* VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS + * VF sends this message to determine its VLAN capabilities. + * + * PF will mark which capabilities it supports based on hardware support and + * current configuration. For example, if a port VLAN is configured the PF will + * not allow outer VLAN filtering, stripping, or insertion to be configured so + * it will block these features from the VF. + * + * The VF will need to cross reference its capabilities with the PFs + * capabilities in the response message from the PF to determine the VLAN + * support. + */ +struct virtchnl_vlan_caps { + struct virtchnl_vlan_filtering_caps filtering; + struct virtchnl_vlan_offload_caps offloads; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_vlan_caps); + +struct virtchnl_vlan { + u16 tci; /* tci[15:13] = PCP and tci[11:0] = VID */ + u16 tci_mask; /* only valid if VIRTCHNL_VLAN_FILTER_MASK set in + * filtering caps + */ + u16 tpid; /* 0x8100, 0x88a8, etc. and only type(s) set in + * filtering caps. Note that tpid here does not refer to + * VIRTCHNL_VLAN_ETHERTYPE_*, but it refers to the + * actual 2-byte VLAN TPID + */ + u8 pad[2]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_vlan); + +struct virtchnl_vlan_filter { + struct virtchnl_vlan inner; + struct virtchnl_vlan outer; + u8 pad[16]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(32, virtchnl_vlan_filter); + +/* VIRTCHNL_OP_ADD_VLAN_V2 + * VIRTCHNL_OP_DEL_VLAN_V2 + * + * VF sends these messages to add/del one or more VLAN tag filters for Rx + * traffic. + * + * The PF attempts to add the filters and returns status. + * + * The VF should only ever attempt to add/del virtchnl_vlan_filter(s) using the + * supported fields negotiated via VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS. + */ +struct virtchnl_vlan_filter_list_v2 { + u16 vport_id; + u16 num_elements; + u8 pad[4]; + struct virtchnl_vlan_filter filters[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(40, virtchnl_vlan_filter_list_v2); + +/* VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 + * VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2 + * VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 + * VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2 + * + * VF sends this message to enable or disable VLAN stripping or insertion. It + * also needs to specify an ethertype. The VF knows which VLAN ethertypes are + * allowed and whether or not it's allowed to enable/disable the specific + * offload via the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS message. The VF needs to + * parse the virtchnl_vlan_caps.offloads fields to determine which offload + * messages are allowed. + * + * For example, if the PF populates the virtchnl_vlan_caps.offloads in the + * following manner the VF will be allowed to enable and/or disable 0x8100 inner + * VLAN insertion and/or stripping via the opcodes listed above. Inner in this + * case means the outer most or single VLAN from the VF's perspective. This is + * because no outer offloads are supported. See the comments above the + * virtchnl_vlan_supported_caps structure for more details. + * + * virtchnl_vlan_caps.offloads.stripping_support.inner = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * virtchnl_vlan_caps.offloads.insertion_support.inner = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * In order to enable inner (again note that in this case inner is the outer + * most or single VLAN from the VF's perspective) VLAN stripping for 0x8100 + * VLANs, the VF would populate the virtchnl_vlan_setting structure in the + * following manner and send the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 message. + * + * virtchnl_vlan_setting.inner_ethertype_setting = + * VIRTCHNL_VLAN_ETHERTYPE_8100; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + * + * The reason that VLAN TPID(s) are not being used for the + * outer_ethertype_setting and inner_ethertype_setting fields is because it's + * possible a device could support VLAN insertion and/or stripping offload on + * multiple ethertypes concurrently, so this method allows a VF to request + * multiple ethertypes in one message using the virtchnl_vlan_support + * enumeration. + * + * For example, if the PF populates the virtchnl_vlan_caps.offloads in the + * following manner the VF will be allowed to enable 0x8100 and 0x88a8 outer + * VLAN insertion and stripping simultaneously. The + * virtchnl_vlan_caps.offloads.ethertype_match field will also have to be + * populated based on what the PF can support. + * + * virtchnl_vlan_caps.offloads.stripping_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * virtchnl_vlan_caps.offloads.insertion_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * In order to enable outer VLAN stripping for 0x8100 and 0x88a8 VLANs, the VF + * would populate the virthcnl_vlan_offload_structure in the following manner + * and send the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2 message. + * + * virtchnl_vlan_setting.outer_ethertype_setting = + * VIRTHCNL_VLAN_ETHERTYPE_8100 | + * VIRTHCNL_VLAN_ETHERTYPE_88A8; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + * + * There is also the case where a PF and the underlying hardware can support + * VLAN offloads on multiple ethertypes, but not concurrently. For example, if + * the PF populates the virtchnl_vlan_caps.offloads in the following manner the + * VF will be allowed to enable and/or disable 0x8100 XOR 0x88a8 outer VLAN + * offloads. The ethertypes must match for stripping and insertion. + * + * virtchnl_vlan_caps.offloads.stripping_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * virtchnl_vlan_caps.offloads.insertion_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_XOR; + * + * virtchnl_vlan_caps.offloads.ethertype_match = + * VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION; + * + * In order to enable outer VLAN stripping for 0x88a8 VLANs, the VF would + * populate the virtchnl_vlan_setting structure in the following manner and send + * the VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2. Also, this will change the + * ethertype for VLAN insertion if it's enabled. So, for completeness, a + * VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2 with the same ethertype should be sent. + * + * virtchnl_vlan_setting.outer_ethertype_setting = VIRTHCNL_VLAN_ETHERTYPE_88A8; + * + * virtchnl_vlan_setting.vport_id = vport_id or vsi_id assigned to the VF on + * initialization. + * + * VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2 + * VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2 + * + * VF sends this message to enable or disable VLAN filtering. It also needs to + * specify an ethertype. The VF knows which VLAN ethertypes are allowed and + * whether or not it's allowed to enable/disable filtering via the + * VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS message. The VF needs to + * parse the virtchnl_vlan_caps.filtering fields to determine which, if any, + * filtering messages are allowed. + * + * For example, if the PF populates the virtchnl_vlan_caps.filtering in the + * following manner the VF will be allowed to enable/disable 0x8100 and 0x88a8 + * outer VLAN filtering together. Note, that the VIRTCHNL_VLAN_ETHERTYPE_AND + * means that all filtering ethertypes will to be enabled and disabled together + * regardless of the request from the VF. This means that the underlying + * hardware only supports VLAN filtering for all VLAN the specified ethertypes + * or none of them. + * + * virtchnl_vlan_caps.filtering.filtering_support.outer = + * VIRTCHNL_VLAN_TOGGLE | + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTHCNL_VLAN_ETHERTYPE_88A8 | + * VIRTCHNL_VLAN_ETHERTYPE_9100 | + * VIRTCHNL_VLAN_ETHERTYPE_AND; + * + * In order to enable outer VLAN filtering for 0x88a8 and 0x8100 VLANs (0x9100 + * VLANs aren't supported by the VF driver), the VF would populate the + * virtchnl_vlan_setting structure in the following manner and send the + * VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2. The same message format would be used + * to disable outer VLAN filtering for 0x88a8 and 0x8100 VLANs, but the + * VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2 opcode is used. + * + * virtchnl_vlan_setting.outer_ethertype_setting = + * VIRTCHNL_VLAN_ETHERTYPE_8100 | + * VIRTCHNL_VLAN_ETHERTYPE_88A8; + * + */ +struct virtchnl_vlan_setting { + u32 outer_ethertype_setting; + u32 inner_ethertype_setting; + u16 vport_id; + u8 pad[6]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_vlan_setting); + /* VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE * VF sends VSI id and flags. * PF returns status code in retval. @@ -599,6 +1215,11 @@ enum virtchnl_action { /* action types */ VIRTCHNL_ACTION_DROP = 0, VIRTCHNL_ACTION_TC_REDIRECT, + VIRTCHNL_ACTION_PASSTHRU, + VIRTCHNL_ACTION_QUEUE, + VIRTCHNL_ACTION_Q_REGION, + VIRTCHNL_ACTION_MARK, + VIRTCHNL_ACTION_COUNT, }; enum virtchnl_flow_type { @@ -612,14 +1233,25 @@ enum virtchnl_flow_type { struct virtchnl_filter { union virtchnl_flow_spec data; union virtchnl_flow_spec mask; - enum virtchnl_flow_type flow_type; - enum virtchnl_action action; + + /* see enum virtchnl_flow_type */ + s32 flow_type; + + /* see enum virtchnl_action */ + s32 action; u32 action_meta; u8 field_flags; }; VIRTCHNL_CHECK_STRUCT_LEN(272, virtchnl_filter); +struct virtchnl_supported_rxdids { + /* see enum virtchnl_rx_desc_id_bitmasks */ + u64 supported_rxdids; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_supported_rxdids); + /* VIRTCHNL_OP_EVENT * PF sends this message to inform the VF driver of events that may affect it. * No direct response is expected from the VF, though it may generate other @@ -636,7 +1268,8 @@ enum virtchnl_event_codes { #define PF_EVENT_SEVERITY_CERTAIN_DOOM 255 struct virtchnl_pf_event { - enum virtchnl_event_codes event; + /* see enum virtchnl_event_codes */ + s32 event; union { /* If the PF driver does not support the new speed reporting * capabilities then use link_event else use link_event_adv to @@ -648,46 +1281,422 @@ struct virtchnl_pf_event { */ struct { enum virtchnl_link_speed link_speed; - u8 link_status; + bool link_status; + u8 pad[3]; } link_event; struct { /* link_speed provided in Mbps */ u32 link_speed; u8 link_status; + u8 pad[3]; } link_event_adv; + struct { + /* link_speed provided in Mbps */ + u32 link_speed; + u16 vport_id; + u8 link_status; + u8 pad; + } link_event_adv_vport; } event_data; - int severity; + s32 severity; }; VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_pf_event); -/* VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP - * VF uses this message to request PF to map IWARP vectors to IWARP queues. - * The request for this originates from the VF IWARP driver through - * a client interface between VF LAN and VF IWARP driver. - * A vector could have an AEQ and CEQ attached to it although - * there is a single AEQ per VF IWARP instance in which case - * most vectors will have an INVALID_IDX for aeq and valid idx for ceq. - * There will never be a case where there will be multiple CEQs attached - * to a single vector. - * PF configures interrupt mapping and returns status. +/* VF reset states - these are written into the RSTAT register: + * VFGEN_RSTAT on the VF + * When the PF initiates a reset, it writes 0 + * When the reset is complete, it writes 1 + * When the PF detects that the VF has recovered, it writes 2 + * VF checks this register periodically to determine if a reset has occurred, + * then polls it to know when the reset is complete. + * If either the PF or VF reads the register while the hardware + * is in a reset state, it will return DEADBEEF, which, when masked + * will result in 3. */ -struct virtchnl_iwarp_qv_info { - u32 v_idx; /* msix_vector */ - u16 ceq_idx; - u16 aeq_idx; - u8 itr_idx; +enum virtchnl_vfr_states { + VIRTCHNL_VFR_INPROGRESS = 0, + VIRTCHNL_VFR_COMPLETED, + VIRTCHNL_VFR_VFACTIVE, }; -VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_iwarp_qv_info); - -struct virtchnl_iwarp_qvlist_info { - u32 num_vectors; - struct virtchnl_iwarp_qv_info qv_info[1]; +/* TX and RX queue types are valid in legacy as well as split queue models. + * With Split Queue model, 2 additional types are introduced - TX_COMPLETION + * and RX_BUFFER. In split queue model, RX corresponds to the queue where HW + * posts completions. + */ +enum virtchnl_queue_type { + VIRTCHNL_QUEUE_TYPE_TX = 0, + VIRTCHNL_QUEUE_TYPE_RX = 1, + VIRTCHNL_QUEUE_TYPE_TX_COMPLETION = 2, + VIRTCHNL_QUEUE_TYPE_RX_BUFFER = 3, + VIRTCHNL_QUEUE_TYPE_CONFIG_TX = 4, + VIRTCHNL_QUEUE_TYPE_CONFIG_RX = 5 }; -VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_iwarp_qvlist_info); +/* structure to specify a chunk of contiguous queues */ +struct virtchnl_queue_chunk { + /* see enum virtchnl_queue_type */ + s32 type; + u16 start_queue_id; + u16 num_queues; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(8, virtchnl_queue_chunk); + +/* structure to specify several chunks of contiguous queues */ +struct virtchnl_queue_chunks { + u16 num_chunks; + u16 rsvd; + struct virtchnl_queue_chunk chunks[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(12, virtchnl_queue_chunks); + +/* VIRTCHNL_OP_ENABLE_QUEUES_V2 + * VIRTCHNL_OP_DISABLE_QUEUES_V2 + * VIRTCHNL_OP_DEL_QUEUES + * + * If VIRTCHNL version was negotiated in VIRTCHNL_OP_VERSION as 2.0 + * then all of these ops are available. + * + * If VIRTCHNL_VF_LARGE_NUM_QPAIRS was negotiated in VIRTCHNL_OP_GET_VF_RESOURCES + * then VIRTCHNL_OP_ENABLE_QUEUES_V2 and VIRTCHNL_OP_DISABLE_QUEUES_V2 are + * available. + * + * PF sends these messages to enable, disable or delete queues specified in + * chunks. PF sends virtchnl_del_ena_dis_queues struct to specify the queues + * to be enabled/disabled/deleted. Also applicable to single queue RX or + * TX. CP performs requested action and returns status. + */ +struct virtchnl_del_ena_dis_queues { + u16 vport_id; + u16 pad; + struct virtchnl_queue_chunks chunks; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_del_ena_dis_queues); + +/* Virtchannel interrupt throttling rate index */ +enum virtchnl_itr_idx { + VIRTCHNL_ITR_IDX_0 = 0, + VIRTCHNL_ITR_IDX_1 = 1, + VIRTCHNL_ITR_IDX_NO_ITR = 3, +}; + +/* Queue to vector mapping */ +struct virtchnl_queue_vector { + u16 queue_id; + u16 vector_id; + u8 pad[4]; + + /* see enum virtchnl_itr_idx */ + s32 itr_idx; + + /* see enum virtchnl_queue_type */ + s32 queue_type; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_queue_vector); + +/* VIRTCHNL_OP_MAP_QUEUE_VECTOR + * + * If VIRTCHNL_VF_LARGE_NUM_QPAIRS was negotiated in VIRTCHNL_OP_GET_VF_RESOURCES + * then only VIRTCHNL_OP_MAP_QUEUE_VECTOR is available. + * + * PF sends this message to map or unmap queues to vectors and ITR index + * registers. External data buffer contains virtchnl_queue_vector_maps structure + * that contains num_qv_maps of virtchnl_queue_vector structures. + * CP maps the requested queue vector maps after validating the queue and vector + * ids and returns a status code. + */ +struct virtchnl_queue_vector_maps { + u16 vport_id; + u16 num_qv_maps; + u8 pad[4]; + struct virtchnl_queue_vector qv_maps[1]; +}; + +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_queue_vector_maps); + +/* VIRTCHNL_VF_CAP_PTP + * VIRTCHNL_OP_1588_PTP_GET_CAPS + * VIRTCHNL_OP_1588_PTP_GET_TIME + * VIRTCHNL_OP_1588_PTP_SET_TIME + * VIRTCHNL_OP_1588_PTP_ADJ_TIME + * VIRTCHNL_OP_1588_PTP_ADJ_FREQ + * VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP + * + * Support for offloading control of the device PTP hardware clock (PHC) is enabled + * by VIRTCHNL_VF_CAP_PTP. This capability allows a VF to request that PF + * enable Tx and Rx timestamps, and request access to read and/or write the + * PHC on the device, as well as query if the VF has direct access to the PHC + * time registers. + * + * The VF must set VIRTCHNL_VF_CAP_PTP in its capabilities when requesting + * resources. If the capability is set in reply, the VF must then send + * a VIRTCHNL_OP_1588_PTP_GET_CAPS request during initialization. The VF indicates + * what extended capabilities it wants by setting the appropriate flags in the + * caps field. The PF reply will indicate what features are enabled for + * that VF. + */ +#define VIRTCHNL_1588_PTP_CAP_TX_TSTAMP 0X00000001 +#define VIRTCHNL_1588_PTP_CAP_RX_TSTAMP 0X00000002 +#define VIRTCHNL_1588_PTP_CAP_READ_PHC 0X00000004 +#define VIRTCHNL_1588_PTP_CAP_WRITE_PHC 0X00000008 +#define VIRTCHNL_1588_PTP_CAP_PHC_REGS 0X00000010 + +/** + * virtchnl_phc_regs + * + * Structure defines how the VF should access PHC related registers. The VF + * must request VIRTCHNL_1588_PTP_CAP_PHC_REGS. If the VF has access to PHC + * registers, the PF will reply with the capability flag set, and with this + * structure detailing what PCIe region and what offsets to use. If direct + * access is not available, this entire structure is reserved and the fields + * will be zero. + * + * If necessary in a future extension, a separate capability mutually + * exclusive with VIRTCHNL_1588_PTP_CAP_PHC_REGS might be used to change the + * entire format of this structure within virtchnl_ptp_caps. + * + * @clock_hi: Register offset of the high 32 bits of clock time + * @clock_lo: Register offset of the low 32 bits of clock time + * @pcie_region: The PCIe region the registers are located in. + * @rsvd: Reserved bits for future extension + */ +struct virtchnl_phc_regs { + u32 clock_hi; + u32 clock_lo; + u8 pcie_region; + u8 rsvd[15]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(24, virtchnl_phc_regs); + +/* timestamp format enumeration + * + * VIRTCHNL_1588_PTP_TSTAMP_40BIT + * + * This format indicates a timestamp that uses the 40bit format from the + * flexible Rx descriptors. It is also the default Tx timestamp format used + * today. + * + * Such a timestamp has the following 40bit format: + * + * *--------------------------------*-------------------------------*-----------* + * | 32 bits of time in nanoseconds | 7 bits of sub-nanosecond time | valid bit | + * *--------------------------------*-------------------------------*-----------* + * + * The timestamp is passed in a u64, with the upper 24bits of the field + * reserved as zero. + * + * With this format, in order to report a full 64bit timestamp to userspace + * applications, the VF is responsible for performing timestamp extension by + * carefully comparing the timestamp with the PHC time. This can correctly + * be achieved with a recent cached copy of the PHC time by doing delta + * comparison between the 32bits of nanoseconds in the timestamp with the + * lower 32 bits of the clock time. For this to work, the cached PHC time + * must be from within 2^31 nanoseconds (~2.1 seconds) of when the timestamp + * was captured. + * + * VIRTCHNL_1588_PTP_TSTAMP_64BIT_NS + * + * This format indicates a timestamp that is 64 bits of nanoseconds. + */ +enum virtchnl_ptp_tstamp_format { + VIRTCHNL_1588_PTP_TSTAMP_40BIT = 0, + VIRTCHNL_1588_PTP_TSTAMP_64BIT_NS = 1, +}; + +/** + * virtchnl_ptp_caps + * + * Structure that defines the PTP capabilities available to the VF. The VF + * sends VIRTCHNL_OP_1588_PTP_GET_CAPS, and must fill in the ptp_caps field + * indicating what capabilities it is requesting. The PF will respond with the + * same message with the virtchnl_ptp_caps structure indicating what is + * enabled for the VF. + * + * @phc_regs: If VIRTCHNL_1588_PTP_CAP_PHC_REGS is set, contains information + * on the PHC related registers available to the VF. + * @caps: On send, VF sets what capabilities it requests. On reply, PF + * indicates what has been enabled for this VF. The PF shall not set + * bits which were not requested by the VF. + * @max_adj: The maximum adjustment capable of being requested by + * VIRTCHNL_OP_1588_PTP_ADJ_FREQ, in parts per billion. Note that 1 ppb + * is approximately 65.5 scaled_ppm. The PF shall clamp any + * frequency adjustment in VIRTCHNL_op_1588_ADJ_FREQ to +/- max_adj. + * Use of ppb in this field allows fitting the value into 4 bytes + * instead of potentially requiring 8 if scaled_ppm units were used. + * @tx_tstamp_idx: The Tx timestamp index to set in the transmit descriptor + * when requesting a timestamp for an outgoing packet. + * Reserved if VIRTCHNL_1588_PTP_CAP_TX_TSTAMP is not enabled. + * @n_ext_ts: Reserved. May be used in a future extension to indicate the + * number of programmable external timestamp event functions are + * available to the VF. + * @n_per_out: Reserved. May be used in a future extension to indicate number + * programmable output functions are available to the VF. + * @n_pins: Reserved. May be used in a future extension to indicate the number + * of programmable SDPs are available to the VF. + * @tx_tstamp_format: Format of the Tx timestamps. Valid formats are defined + * by the virtchnl_ptp_tstamp enumeration. Note that Rx + * timestamps are tied to the descriptor format, and do not + * have a separate format field. + * @rsvd: Reserved bits for future extension. + * + * PTP capabilities + * + * VIRTCHNL_1588_PTP_CAP_TX_TSTAMP indicates that the VF can request transmit + * timestamps for packets in its transmit descriptors. If this is unset, + * transmit timestamp requests are ignored. Note that only one outstanding Tx + * timestamp request will be honored at a time. The PF shall handle receipt of + * the timestamp from the hardware, and will forward this to the VF by sending + * a VIRTCHNL_OP_1588_TX_TIMESTAMP message. + * + * VIRTCHNL_1588_PTP_CAP_RX_TSTAMP indicates that the VF receive queues have + * receive timestamps enabled in the flexible descriptors. Note that this + * requires a VF to also negotiate to enable advanced flexible descriptors in + * the receive path instead of the default legacy descriptor format. + * + * For a detailed description of the current Tx and Rx timestamp format, see + * the section on virtchnl_phc_tx_tstamp. Future extensions may indicate + * timestamp format in the capability structure. + * + * VIRTCHNL_1588_PTP_CAP_READ_PHC indicates that the VF may read the PHC time + * via the VIRTCHNL_OP_1588_PTP_GET_TIME command, or by directly reading PHC + * registers if VIRTCHNL_1588_PTP_CAP_PHC_REGS is also set. + * + * VIRTCHNL_1588_PTP_CAP_WRITE_PHC indicates that the VF may request updates + * to the PHC time via VIRTCHNL_OP_1588_PTP_SET_TIME, + * VIRTCHNL_OP_1588_PTP_ADJ_TIME, and VIRTCHNL_OP_1588_PTP_ADJ_FREQ. + * + * VIRTCHNL_1588_PTP_CAP_PHC_REGS indicates that the VF has direct access to + * certain PHC related registers, primarily for lower latency access to the + * PHC time. If this is set, the VF shall read the virtchnl_phc_regs section + * of the capabilities to determine the location of the clock registers. If + * this capability is not set, the entire 24 bytes of virtchnl_phc_regs is + * reserved as zero. Future extensions define alternative formats for this + * data, in which case they will be mutually exclusive with this capability. + * + * Note that in the future, additional capability flags may be added such as +* to indicate support for SDP configuration, or to indicate Rx timestamp +* format if a descriptor could support multiple formats. All fields marked as +* reserved in this header will be set to zero, and implementations should +* verify this. + */ +struct virtchnl_ptp_caps { + struct virtchnl_phc_regs phc_regs; + u32 caps; + s32 max_adj; + u8 tx_tstamp_idx; + u8 n_ext_ts; + u8 n_per_out; + u8 n_pins; + /* see enum virtchnl_ptp_tstamp_format */ + u8 tx_tstamp_format; + u8 rsvd[11]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(48, virtchnl_ptp_caps); + +/** + * virtchnl_phc_time + * @time: PHC time in nanoseconds + * @rsvd: Reserved for future extension + * + * Structure sent with VIRTCHNL_OP_1588_PTP_SET_TIME and received with + * VIRTCHNL_OP_1588_PTP_GET_TIME. Contains the 64bits of PHC clock time in + * nanoseconds. + * + * VIRTCHNL_OP_1588_PTP_SET_TIME may be sent by the VF if + * VIRTCHNL_1588_PTP_CAP_WRITE_PHC is set. This will request that the PHC time + * be set to the requested value. This operation is non-atomic and thus does + * not adjust for the delay between request and completion. It is recommended + * that the VF use VIRTCHNL_OP_1588_PTP_ADJ_TIME and + * VIRTCHNL_OP_1588_PTP_ADJ_FREQ when possible to steer the PHC clock. + * + * VIRTCHNL_OP_1588_PTP_GET_TIME may be sent to request the current time of + * the PHC. This op is available in case direct access via the PHC registers + * is not available. + */ +struct virtchnl_phc_time { + u64 time; + u8 rsvd[8]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_phc_time); + +/** + * virtchnl_phc_adj_time + * @delta: offset requested to adjust clock by + * @rsvd: reserved for future extension + * + * Sent with VIRTCHNL_OP_1588_PTP_ADJ_TIME. Used to request an adjustment of + * the clock time by the provided delta, with negative values representing + * subtraction. VIRTCHNL_OP_1588_PTP_ADJ_TIME may not be sent unless + * VIRTCHNL_1588_PTP_CAP_WRITE_PHC is set. + * + * The atomicity of this operation is not guaranteed. The PF should perform an + * atomic update using appropriate mechanisms if possible. However, this is + * not guaranteed. + */ +struct virtchnl_phc_adj_time { + s64 delta; + u8 rsvd[8]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_phc_adj_time); + +/** + * virtchnl_phc_adj_freq + * @scaled_ppm: frequency adjustment represented in scaled parts per million + * @rsvd: Reserved for future extension + * + * Sent with the VIRTCHNL_OP_1588_PTP_ADJ_FREQ to request an adjustment to the + * clock frequency. The adjustment is in scaled_ppm, which is parts per + * million with a 16bit binary fractional portion. 1 part per billion is + * approximately 65.5 scaled_ppm. + * + * ppm = scaled_ppm / 2^16 + * + * ppb = scaled_ppm * 1000 / 2^16 or + * + * ppb = scaled_ppm * 125 / 2^13 + * + * The PF shall clamp any adjustment request to plus or minus the specified + * max_adj in the PTP capabilities. + * + * Requests for adjustment are always based off of nominal clock frequency and + * not compounding. To reset clock frequency, send a request with a scaled_ppm + * of 0. + */ +struct virtchnl_phc_adj_freq { + s64 scaled_ppm; + u8 rsvd[8]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_phc_adj_freq); + +/** + * virtchnl_phc_tx_stamp + * @tstamp: timestamp value + * @rsvd: Reserved for future extension + * + * Sent along with VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP from the PF when a Tx + * timestamp for the index associated with this VF in the tx_tstamp_idx field + * is captured by hardware. + * + * If VIRTCHNL_1588_PTP_CAP_TX_TSTAMP is set, the VF may request a timestamp + * for a packet in its transmit context descriptor by setting the appropriate + * flag and setting the timestamp index provided by the PF. On transmission, + * the timestamp will be captured and sent to the PF. The PF will forward this + * timestamp to the VF via the VIRTCHNL_1588_PTP_CAP_TX_TSTAMP op. + * + * The timestamp format is defined by the tx_tstamp_format field of the + * virtchnl_ptp_caps structure. + */ +struct virtchnl_phc_tx_tstamp { + u64 tstamp; + u8 rsvd[8]; +}; +VIRTCHNL_CHECK_STRUCT_LEN(16, virtchnl_phc_tx_tstamp); /* Since VF messages are limited by u16 size, precalculate the maximum possible * values of nested elements in virtchnl structures that virtual channel can @@ -710,30 +1719,21 @@ enum virtchnl_vector_limits { ((u16)(~0) - sizeof(struct virtchnl_vlan_filter_list)) / sizeof(u16), - VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP_MAX = - ((u16)(~0) - sizeof(struct virtchnl_iwarp_qvlist_info)) / - sizeof(struct virtchnl_iwarp_qv_info), - VIRTCHNL_OP_ENABLE_CHANNELS_MAX = ((u16)(~0) - sizeof(struct virtchnl_tc_info)) / sizeof(struct virtchnl_channel_info), -}; -/* VF reset states - these are written into the RSTAT register: - * VFGEN_RSTAT on the VF - * When the PF initiates a reset, it writes 0 - * When the reset is complete, it writes 1 - * When the PF detects that the VF has recovered, it writes 2 - * VF checks this register periodically to determine if a reset has occurred, - * then polls it to know when the reset is complete. - * If either the PF or VF reads the register while the hardware - * is in a reset state, it will return DEADBEEF, which, when masked - * will result in 3. - */ -enum virtchnl_vfr_states { - VIRTCHNL_VFR_INPROGRESS = 0, - VIRTCHNL_VFR_COMPLETED, - VIRTCHNL_VFR_VFACTIVE, + VIRTCHNL_OP_ENABLE_DISABLE_DEL_QUEUES_V2_MAX = + ((u16)(~0) - sizeof(struct virtchnl_del_ena_dis_queues)) / + sizeof(struct virtchnl_queue_chunk), + + VIRTCHNL_OP_MAP_UNMAP_QUEUE_VECTOR_MAX = + ((u16)(~0) - sizeof(struct virtchnl_queue_vector_maps)) / + sizeof(struct virtchnl_queue_vector), + + VIRTCHNL_OP_ADD_DEL_VLAN_V2_MAX = + ((u16)(~0) - sizeof(struct virtchnl_vlan_filter_list_v2)) / + sizeof(struct virtchnl_vlan_filter), }; /** @@ -750,7 +1750,7 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, u8 *msg, u16 msglen) { bool err_msg_format = false; - int valid_len = 0; + u32 valid_len = 0; /* Validate message length. */ switch (v_opcode) { @@ -806,6 +1806,8 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, case VIRTCHNL_OP_DISABLE_QUEUES: valid_len = sizeof(struct virtchnl_queue_select); break; + case VIRTCHNL_OP_GET_MAX_RSS_QREGION: + break; case VIRTCHNL_OP_ADD_ETH_ADDR: case VIRTCHNL_OP_DEL_ETH_ADDR: valid_len = sizeof(struct virtchnl_ether_addr_list); @@ -845,34 +1847,6 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, case VIRTCHNL_OP_GET_STATS: valid_len = sizeof(struct virtchnl_queue_select); break; - case VIRTCHNL_OP_IWARP: - /* These messages are opaque to us and will be validated in - * the RDMA client code. We just need to check for nonzero - * length. The firmware will enforce max length restrictions. - */ - if (msglen) - valid_len = msglen; - else - err_msg_format = true; - break; - case VIRTCHNL_OP_RELEASE_IWARP_IRQ_MAP: - break; - case VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP: - valid_len = sizeof(struct virtchnl_iwarp_qvlist_info); - if (msglen >= valid_len) { - struct virtchnl_iwarp_qvlist_info *qv = - (struct virtchnl_iwarp_qvlist_info *)msg; - - if (qv->num_vectors == 0 || qv->num_vectors > - VIRTCHNL_OP_CONFIG_IWARP_IRQ_MAP_MAX) { - err_msg_format = true; - break; - } - - valid_len += ((qv->num_vectors - 1) * - sizeof(struct virtchnl_iwarp_qv_info)); - } - break; case VIRTCHNL_OP_CONFIG_RSS_KEY: valid_len = sizeof(struct virtchnl_rss_key); if (msglen >= valid_len) { @@ -934,6 +1908,80 @@ virtchnl_vc_validate_vf_msg(struct virtchnl_version_info *ver, u32 v_opcode, case VIRTCHNL_OP_DEL_CLOUD_FILTER: valid_len = sizeof(struct virtchnl_filter); break; + case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS: + break; + case VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS: + break; + case VIRTCHNL_OP_ADD_VLAN_V2: + case VIRTCHNL_OP_DEL_VLAN_V2: + valid_len = sizeof(struct virtchnl_vlan_filter_list_v2); + if (msglen >= valid_len) { + struct virtchnl_vlan_filter_list_v2 *vfl = + (struct virtchnl_vlan_filter_list_v2 *)msg; + + if (vfl->num_elements == 0 || vfl->num_elements > + VIRTCHNL_OP_ADD_DEL_VLAN_V2_MAX) { + err_msg_format = true; + break; + } + + valid_len += (vfl->num_elements - 1) * + sizeof(struct virtchnl_vlan_filter); + } + break; + case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING_V2: + case VIRTCHNL_OP_DISABLE_VLAN_STRIPPING_V2: + case VIRTCHNL_OP_ENABLE_VLAN_INSERTION_V2: + case VIRTCHNL_OP_DISABLE_VLAN_INSERTION_V2: + case VIRTCHNL_OP_ENABLE_VLAN_FILTERING_V2: + case VIRTCHNL_OP_DISABLE_VLAN_FILTERING_V2: + valid_len = sizeof(struct virtchnl_vlan_setting); + break; + case VIRTCHNL_OP_1588_PTP_GET_CAPS: + valid_len = sizeof(struct virtchnl_ptp_caps); + break; + case VIRTCHNL_OP_1588_PTP_GET_TIME: + case VIRTCHNL_OP_1588_PTP_SET_TIME: + valid_len = sizeof(struct virtchnl_phc_time); + break; + case VIRTCHNL_OP_1588_PTP_ADJ_TIME: + valid_len = sizeof(struct virtchnl_phc_adj_time); + break; + case VIRTCHNL_OP_1588_PTP_ADJ_FREQ: + valid_len = sizeof(struct virtchnl_phc_adj_freq); + break; + case VIRTCHNL_OP_1588_PTP_TX_TIMESTAMP: + valid_len = sizeof(struct virtchnl_phc_tx_tstamp); + break; + case VIRTCHNL_OP_ENABLE_QUEUES_V2: + case VIRTCHNL_OP_DISABLE_QUEUES_V2: + valid_len = sizeof(struct virtchnl_del_ena_dis_queues); + if (msglen >= valid_len) { + struct virtchnl_del_ena_dis_queues *qs = + (struct virtchnl_del_ena_dis_queues *)msg; + if (qs->chunks.num_chunks == 0 || + qs->chunks.num_chunks > VIRTCHNL_OP_ENABLE_DISABLE_DEL_QUEUES_V2_MAX) { + err_msg_format = true; + break; + } + valid_len += (qs->chunks.num_chunks - 1) * + sizeof(struct virtchnl_queue_chunk); + } + break; + case VIRTCHNL_OP_MAP_QUEUE_VECTOR: + valid_len = sizeof(struct virtchnl_queue_vector_maps); + if (msglen >= valid_len) { + struct virtchnl_queue_vector_maps *v_qp = + (struct virtchnl_queue_vector_maps *)msg; + if (v_qp->num_qv_maps == 0 || + v_qp->num_qv_maps > VIRTCHNL_OP_MAP_UNMAP_QUEUE_VECTOR_MAX) { + err_msg_format = true; + break; + } + valid_len += (v_qp->num_qv_maps - 1) * + sizeof(struct virtchnl_queue_vector); + } + break; /* These are always errors coming from the VF. */ case VIRTCHNL_OP_EVENT: case VIRTCHNL_OP_UNKNOWN: diff --git a/drivers/net/ethernet/intel/iavf/virtchnl_inline_ipsec.h b/drivers/net/ethernet/intel/iavf/virtchnl_inline_ipsec.h index 93880652d..987109c9a 100644 --- a/drivers/net/ethernet/intel/iavf/virtchnl_inline_ipsec.h +++ b/drivers/net/ethernet/intel/iavf/virtchnl_inline_ipsec.h @@ -10,8 +10,11 @@ #define VIRTCHNL_IPSEC_MAX_CRYPTO_ITEM_NUMBER 2 #define VIRTCHNL_IPSEC_MAX_KEY_LEN 128 #define VIRTCHNL_IPSEC_MAX_SA_DESTROY_NUM 8 -#define VIRTCHNL_IPSEC_SELECTED_SA_DESTROY 0 -#define VIRTCHNL_IPSEC_ALL_SA_DESTROY 1 +#define VIRTCHNL_IPSEC_SA_DESTROY 0 +#define VIRTCHNL_IPSEC_BROADCAST_VFID 0xFFFFFFFF +#define VIRTCHNL_IPSEC_INVALID_REQ_ID 0xFFFF +#define VIRTCHNL_IPSEC_INVALID_SA_CFG_RESP 0xFFFFFFFF +#define VIRTCHNL_IPSEC_INVALID_SP_CFG_RESP 0xFFFFFFFF /* crypto type */ #define VIRTCHNL_AUTH 1 @@ -20,7 +23,7 @@ /* algorithm type */ /* Hash Algorithm */ -#define VIRTCHNL_NO_ALG 0 /* NULL algorithm */ +#define VIRTCHNL_HASH_NO_ALG 0 /* NULL algorithm */ #define VIRTCHNL_AES_CBC_MAC 1 /* AES-CBC-MAC algorithm */ #define VIRTCHNL_AES_CMAC 2 /* AES CMAC algorithm */ #define VIRTCHNL_AES_GMAC 3 /* AES GMAC algorithm */ @@ -36,13 +39,14 @@ #define VIRTCHNL_SHA3_384_HMAC 13 /* HMAC using 384 bit SHA3 algorithm */ #define VIRTCHNL_SHA3_512_HMAC 14 /* HMAC using 512 bit SHA3 algorithm */ /* Cipher Algorithm */ -#define VIRTCHNL_3DES_CBC 15 /* Triple DES algorithm in CBC mode */ -#define VIRTCHNL_AES_CBC 16 /* AES algorithm in CBC mode */ -#define VIRTCHNL_AES_CTR 17 /* AES algorithm in Counter mode */ +#define VIRTCHNL_CIPHER_NO_ALG 15 /* NULL algorithm */ +#define VIRTCHNL_3DES_CBC 16 /* Triple DES algorithm in CBC mode */ +#define VIRTCHNL_AES_CBC 17 /* AES algorithm in CBC mode */ +#define VIRTCHNL_AES_CTR 18 /* AES algorithm in Counter mode */ /* AEAD Algorithm */ -#define VIRTCHNL_AES_CCM 18 /* AES algorithm in CCM mode */ -#define VIRTCHNL_AES_GCM 19 /* AES algorithm in GCM mode */ -#define VIRTCHNL_CHACHA20_POLY1305 20 /* algorithm of ChaCha20-Poly1305 */ +#define VIRTCHNL_AES_CCM 19 /* AES algorithm in CCM mode */ +#define VIRTCHNL_AES_GCM 20 /* AES algorithm in GCM mode */ +#define VIRTCHNL_CHACHA20_POLY1305 21 /* algorithm of ChaCha20-Poly1305 */ /* protocol type */ #define VIRTCHNL_PROTO_ESP 1 @@ -68,6 +72,31 @@ #define VIRTCHNL_IPV4 1 #define VIRTCHNL_IPV6 2 +/* for virtchnl_ipsec_resp */ +enum inline_ipsec_resp { + INLINE_IPSEC_SUCCESS = 0, + INLINE_IPSEC_FAIL = -1, + INLINE_IPSEC_ERR_FIFO_FULL = -2, + INLINE_IPSEC_ERR_NOT_READY = -3, + INLINE_IPSEC_ERR_VF_DOWN = -4, + INLINE_IPSEC_ERR_INVALID_PARAMS = -5, + INLINE_IPSEC_ERR_NO_MEM = -6, +}; + +/* Detailed opcodes for DPDK and IPsec use */ +enum inline_ipsec_ops { + INLINE_IPSEC_OP_GET_CAP = 0, + INLINE_IPSEC_OP_GET_STATUS = 1, + INLINE_IPSEC_OP_SA_CREATE = 2, + INLINE_IPSEC_OP_SA_UPDATE = 3, + INLINE_IPSEC_OP_SA_DESTROY = 4, + INLINE_IPSEC_OP_SP_CREATE = 5, + INLINE_IPSEC_OP_SP_DESTROY = 6, + INLINE_IPSEC_OP_SA_READ = 7, + INLINE_IPSEC_OP_EVENT = 8, + INLINE_IPSEC_OP_RESP = 9, +}; + /* Not all valid, if certain field is invalid, set 1 for all bits */ struct virtchnl_algo_cap { u32 algo_type; @@ -89,7 +118,7 @@ struct virtchnl_algo_cap { u16 min_aad_size; u16 max_aad_size; u16 inc_aad_size; -}; +} __packed; /* vf record the capability of crypto from the virtchnl */ struct virtchnl_sym_crypto_cap { @@ -149,17 +178,6 @@ struct virtchnl_ipsec_cap { struct virtchnl_sym_crypto_cap cap[VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM]; }; -/* using desc_id to record the format of rx descriptor */ -struct virtchnl_rx_desc_fmt { - u16 desc_id; -}; - -/* using desc_id to record the format of tx descriptor */ -struct virtchnl_tx_desc_fmt { - u8 desc_num; - u16 desc_ids[VIRTCHNL_IPSEC_MAX_TX_DESC_NUM]; -}; - /* configuration of crypto function */ struct virtchnl_ipsec_crypto_cfg_item { u8 crypto_type; @@ -172,12 +190,15 @@ struct virtchnl_ipsec_crypto_cfg_item { /* Length of digest */ u16 digest_len; + /* SA salt */ + u32 salt; + /* The length of the symmetric key */ u16 key_len; /* key data buffer */ u8 key_data[VIRTCHNL_IPSEC_MAX_KEY_LEN]; -}; +} __packed; struct virtchnl_ipsec_sym_crypto_cfg { struct virtchnl_ipsec_crypto_cfg_item @@ -220,9 +241,6 @@ struct virtchnl_ipsec_sa_cfg { /* outer dst ip address */ u8 dst_addr[16]; - /* SA salt */ - u32 salt; - /* SPD reference. Used to link an SA with its policy. * PF drivers may ignore this field. */ @@ -281,7 +299,7 @@ struct virtchnl_ipsec_sa_cfg { /* crypto configuration */ struct virtchnl_ipsec_sym_crypto_cfg crypto_cfg; -}; +} __packed; /* VIRTCHNL_OP_IPSEC_SA_UPDATE * VF send configuration of index of SA to PF @@ -300,17 +318,15 @@ struct virtchnl_ipsec_sa_update { * be destroyed */ struct virtchnl_ipsec_sa_destroy { - /* VIRTCHNL_SELECTED_SA_DESTROY: selected SA will be destroyed. - * VIRTCHNL_ALL_SA_DESTROY: all SA will be destroyed. + /* All zero bitmap indicates all SA will be destroyed. + * Non-zero bitmap indicates the selected SA in + * array sa_index will be destroyed. */ u8 flag; - u8 pad1; /* pading */ - u16 pad2; /* pading */ - /* selected SA index */ u32 sa_index[VIRTCHNL_IPSEC_MAX_SA_DESTROY_NUM]; -}; +} __packed; /* VIRTCHNL_OP_IPSEC_SA_READ * VF send this SA configuration to PF using virtchnl; @@ -415,6 +431,125 @@ struct virtchnl_ipsec_sa_read { /* crypto configuration. Salt and keys are set to 0 */ struct virtchnl_ipsec_sym_crypto_cfg crypto_cfg; +} __packed; + +/* Add allowlist entry in IES */ +struct virtchnl_ipsec_sp_cfg { + u32 spi; + u32 dip[4]; + + /* Drop frame if true or redirect to QAT if false. */ + u8 drop; + + /* Congestion domain. For future use. */ + u8 cgd; + + /* 0 for IPv4 table, 1 for IPv6 table. */ + u8 table_id; + + /* Set TC (congestion domain) if true. For future use. */ + u8 set_tc; }; +/* Delete allowlist entry in IES */ +struct virtchnl_ipsec_sp_destroy { + /* 0 for IPv4 table, 1 for IPv6 table. */ + u8 table_id; + u32 rule_id; +} __packed; + +/* Response from IES to allowlist operations */ +struct virtchnl_ipsec_sp_cfg_resp { + u32 rule_id; +}; + +struct virtchnl_ipsec_sa_cfg_resp { + u32 sa_handle; +}; + +#define INLINE_IPSEC_EVENT_RESET 0x1 +#define INLINE_IPSEC_EVENT_CRYPTO_ON 0x2 +#define INLINE_IPSEC_EVENT_CRYPTO_OFF 0x4 + +struct virtchnl_ipsec_event { + u32 ipsec_event_data; +}; + +#define INLINE_IPSEC_STATUS_AVAILABLE 0x1 +#define INLINE_IPSEC_STATUS_UNAVAILABLE 0x2 + +struct virtchnl_ipsec_status { + u32 status; +}; + +struct virtchnl_ipsec_resp { + u32 resp; +}; + +/* Internal message descriptor for VF <-> IPsec communication */ +struct inline_ipsec_msg { + u16 ipsec_opcode; + u16 req_id; + + union { + /* IPsec request */ + struct virtchnl_ipsec_sa_cfg sa_cfg[0]; + struct virtchnl_ipsec_sp_cfg sp_cfg[0]; + struct virtchnl_ipsec_sa_update sa_update[0]; + struct virtchnl_ipsec_sa_destroy sa_destroy[0]; + struct virtchnl_ipsec_sp_destroy sp_destroy[0]; + + /* IPsec response */ + struct virtchnl_ipsec_sa_cfg_resp sa_cfg_resp[0]; + struct virtchnl_ipsec_sp_cfg_resp sp_cfg_resp[0]; + struct virtchnl_ipsec_cap ipsec_cap[0]; + struct virtchnl_ipsec_status ipsec_status[0]; + /* response to del_sa, del_sp, update_sa */ + struct virtchnl_ipsec_resp ipsec_resp[0]; + + /* IPsec event (no req_id is required) */ + struct virtchnl_ipsec_event event[0]; + + /* Reserved */ + struct virtchnl_ipsec_sa_read sa_read[0]; + } ipsec_data; +}; + +static inline u16 virtchnl_inline_ipsec_val_msg_len(u16 opcode) +{ + u16 valid_len = sizeof(struct inline_ipsec_msg); + + switch (opcode) { + case INLINE_IPSEC_OP_GET_CAP: + case INLINE_IPSEC_OP_GET_STATUS: + break; + case INLINE_IPSEC_OP_SA_CREATE: + valid_len += sizeof(struct virtchnl_ipsec_sa_cfg); + break; + case INLINE_IPSEC_OP_SP_CREATE: + valid_len += sizeof(struct virtchnl_ipsec_sp_cfg); + break; + case INLINE_IPSEC_OP_SA_UPDATE: + valid_len += sizeof(struct virtchnl_ipsec_sa_update); + break; + case INLINE_IPSEC_OP_SA_DESTROY: + valid_len += sizeof(struct virtchnl_ipsec_sa_destroy); + break; + case INLINE_IPSEC_OP_SP_DESTROY: + valid_len += sizeof(struct virtchnl_ipsec_sp_destroy); + break; + /* Only for msg length caculation of response to VF in case of + * inline ipsec failure. + */ + case INLINE_IPSEC_OP_RESP: + valid_len += sizeof(struct virtchnl_ipsec_resp); + break; + default: + valid_len = 0; + break; + } + + return valid_len; +} + #endif /* _VIRTCHNL_INLINE_IPSEC_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/virtchnl_lan_desc.h b/drivers/net/ethernet/intel/iavf/virtchnl_lan_desc.h new file mode 100644 index 000000000..7db691566 --- /dev/null +++ b/drivers/net/ethernet/intel/iavf/virtchnl_lan_desc.h @@ -0,0 +1,525 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2013, Intel Corporation. */ + +/* + * Copyright (C) 2019 Intel Corporation + * + * For licensing information, see the file 'LICENSE' in the root folder + */ +#ifndef _VIRTCHNL_LAN_DESC_H_ +#define _VIRTCHNL_LAN_DESC_H_ + +/* Rx */ +/* For virtchnl_splitq_base_rx_flex desc members */ +#define VIRTCHNL_RXD_FLEX_PTYPE_S 0 +#define VIRTCHNL_RXD_FLEX_PTYPE_M \ + MAKEMASK(0x3FFUL, VIRTCHNL_RXD_FLEX_PTYPE_S) +#define VIRTCHNL_RXD_FLEX_UMBCAST_S 10 +#define VIRTCHNL_RXD_FLEX_UMBCAST_M \ + MAKEMASK(0x3UL, VIRTCHNL_RXD_FLEX_UMBCAST_S) +#define VIRTCHNL_RXD_FLEX_FF0_S 12 +#define VIRTCHNL_RXD_FLEX_FF0_M MAKEMASK(0xFUL, VIRTCHNL_RXD_FLEX_FF0_S) +#define VIRTCHNL_RXD_FLEX_LEN_PBUF_S 0 +#define VIRTCHNL_RXD_FLEX_LEN_PBUF_M \ + MAKEMASK(0x3FFFUL, VIRTCHNL_RXD_FLEX_LEN_PBUF_S) +#define VIRTCHNL_RXD_FLEX_GEN_S 14 +#define VIRTCHNL_RXD_FLEX_GEN_M BIT_ULL(VIRTCHNL_RXD_FLEX_GEN_S) +#define VIRTCHNL_RXD_FLEX_BUFQ_ID_S 15 +#define VIRTCHNL_RXD_FLEX_BUFQ_ID_M \ + BIT_ULL(VIRTCHNL_RXD_FLEX_BUFQ_ID_S) +#define VIRTCHNL_RXD_FLEX_LEN_HDR_S 0 +#define VIRTCHNL_RXD_FLEX_LEN_HDR_M \ + MAKEMASK(0x3FFUL, VIRTCHNL_RXD_FLEX_LEN_HDR_S) +#define VIRTCHNL_RXD_FLEX_RSC_S 10 +#define VIRTCHNL_RXD_FLEX_RSC_M BIT_ULL(VIRTCHNL_RXD_FLEX_RSC_S) +#define VIRTCHNL_RXD_FLEX_SPH_S 11 +#define VIRTCHNL_RXD_FLEX_SPH_M BIT_ULL(VIRTCHNL_RXD_FLEX_SPH_S) +#define VIRTCHNL_RXD_FLEX_MISS_S 12 +#define VIRTCHNL_RXD_FLEX_MISS_M \ + BIT_ULL(VIRTCHNL_RXD_FLEX_MISS_S) +#define VIRTCHNL_RXD_FLEX_FF1_S 13 +#define VIRTCHNL_RXD_FLEX_FF1_M MAKEMASK(0x7UL, VIRTCHNL_RXD_FLEX_FF1_M) + +/* For virtchnl_singleq_base_rx_legacy desc members */ +#define VIRTCHNL_RXD_QW1_LEN_SPH_S 63 +#define VIRTCHNL_RXD_QW1_LEN_SPH_M BIT_ULL(VIRTCHNL_RXD_QW1_LEN_SPH_S) +#define VIRTCHNL_RXD_QW1_LEN_HBUF_S 52 +#define VIRTCHNL_RXD_QW1_LEN_HBUF_M \ + MAKEMASK(0x7FFULL, VIRTCHNL_RXD_QW1_LEN_HBUF_S) +#define VIRTCHNL_RXD_QW1_LEN_PBUF_S 38 +#define VIRTCHNL_RXD_QW1_LEN_PBUF_M \ + MAKEMASK(0x3FFFULL, VIRTCHNL_RXD_QW1_LEN_PBUF_S) +#define VIRTCHNL_RXD_QW1_PTYPE_S 30 +#define VIRTCHNL_RXD_QW1_PTYPE_M \ + MAKEMASK(0xFFULL, VIRTCHNL_RXD_QW1_PTYPE_S) +#define VIRTCHNL_RXD_QW1_ERROR_S 19 +#define VIRTCHNL_RXD_QW1_ERROR_M \ + MAKEMASK(0xFFUL, VIRTCHNL_RXD_QW1_ERROR_S) +#define VIRTCHNL_RXD_QW1_STATUS_S 0 +#define VIRTCHNL_RXD_QW1_STATUS_M \ + MAKEMASK(0x7FFFFUL, VIRTCHNL_RXD_QW1_STATUS_S) + +enum virtchnl_rx_flex_desc_status_error_0_qw1_bits { + /* Note: These are predefined bit offsets */ + VIRTCHNL_RX_FLEX_DESC_STATUS0_DD_S = 0, + VIRTCHNL_RX_FLEX_DESC_STATUS0_EOF_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_HBO_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_L3L4P_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XSUM_IPE_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XSUM_L4E_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S, +}; + +enum virtchnl_rx_flex_desc_status_error_0_qw0_bits { + VIRTCHNL_RX_FLEX_DESC_STATUS0_LPBK_S = 0, + VIRTCHNL_RX_FLEX_DESC_STATUS0_IPV6EXADD_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_RXE_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_CRCP_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_RSS_VALID_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_L2TAG1P_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S, + VIRTCHNL_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */ +}; + +enum virtchnl_rx_flex_desc_status_error_1_bits { + /* Note: These are predefined bit offsets */ + VIRTCHNL_RX_FLEX_DESC_STATUS1_RSVD_S = 0, /* 2 bits */ + VIRTCHNL_RX_FLEX_DESC_STATUS1_ATRAEFAIL_S = 2, + VIRTCHNL_RX_FLEX_DESC_STATUS1_L2TAG2P_S = 3, + VIRTCHNL_RX_FLEX_DESC_STATUS1_XTRMD2_VALID_S = 4, + VIRTCHNL_RX_FLEX_DESC_STATUS1_XTRMD3_VALID_S = 5, + VIRTCHNL_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_S = 6, + VIRTCHNL_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_S = 7, + VIRTCHNL_RX_FLEX_DESC_STATUS1_LAST /* this entry must be last!!! */ +}; + +enum virtchnl_rx_base_desc_status_bits { + /* Note: These are predefined bit offsets */ + VIRTCHNL_RX_BASE_DESC_STATUS_DD_S = 0, + VIRTCHNL_RX_BASE_DESC_STATUS_EOF_S = 1, + VIRTCHNL_RX_BASE_DESC_STATUS_L2TAG1P_S = 2, + VIRTCHNL_RX_BASE_DESC_STATUS_L3L4P_S = 3, + VIRTCHNL_RX_BASE_DESC_STATUS_CRCP_S = 4, + VIRTCHNL_RX_BASE_DESC_STATUS_RSVD_S = 5, /* 3 BITS */ + VIRTCHNL_RX_BASE_DESC_STATUS_EXT_UDP_0_S = 8, + VIRTCHNL_RX_BASE_DESC_STATUS_UMBCAST_S = 9, /* 2 BITS */ + VIRTCHNL_RX_BASE_DESC_STATUS_FLM_S = 11, + VIRTCHNL_RX_BASE_DESC_STATUS_FLTSTAT_S = 12, /* 2 BITS */ + VIRTCHNL_RX_BASE_DESC_STATUS_LPBK_S = 14, + VIRTCHNL_RX_BASE_DESC_STATUS_IPV6EXADD_S = 15, + VIRTCHNL_RX_BASE_DESC_STATUS_RSVD1_S = 16, /* 2 BITS */ + VIRTCHNL_RX_BASE_DESC_STATUS_INT_UDP_0_S = 18, + VIRTCHNL_RX_BASE_DESC_STATUS_LAST /* this entry must be last!!! */ +}; + +enum virtchnl_rx_desc_fltstat_values { + VIRTCHNL_RX_DESC_FLTSTAT_NO_DATA = 0, + VIRTCHNL_RX_DESC_FLTSTAT_RSV_FD_ID = 1, /* 16byte desc? FD_ID : RSV */ + VIRTCHNL_RX_DESC_FLTSTAT_RSV = 2, + VIRTCHNL_RX_DESC_FLTSTAT_RSS_HASH = 3, +}; + +enum virtchnl_rx_base_desc_error_bits { + /* Note: These are predefined bit offsets */ + VIRTCHNL_RX_BASE_DESC_ERROR_RXE_S = 0, + VIRTCHNL_RX_BASE_DESC_ERROR_ATRAEFAIL_S = 1, + VIRTCHNL_RX_BASE_DESC_ERROR_HBO_S = 2, + VIRTCHNL_RX_BASE_DESC_ERROR_L3L4E_S = 3, /* 3 BITS */ + VIRTCHNL_RX_BASE_DESC_ERROR_IPE_S = 3, + VIRTCHNL_RX_BASE_DESC_ERROR_L4E_S = 4, + VIRTCHNL_RX_BASE_DESC_ERROR_EIPE_S = 5, + VIRTCHNL_RX_BASE_DESC_ERROR_OVERSIZE_S = 6, + VIRTCHNL_RX_BASE_DESC_ERROR_RSVD_S = 7 +}; + +/* Receive Descriptors */ +/* splitq buf + | 16| 0| + ---------------------------------------------------------------- + | RSV | Buffer ID | + ---------------------------------------------------------------- + | Rx packet buffer adresss | + ---------------------------------------------------------------- + | Rx header buffer adresss | + ---------------------------------------------------------------- + | RSV | + ---------------------------------------------------------------- + | 0| + */ +struct virtchnl_splitq_rx_buf_desc { + struct { + __le16 buf_id; /* Buffer Identifier */ + __le16 rsvd0; + __le32 rsvd1; + } qword0; + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + __le64 rsvd2; +}; /* read used with buffer queues*/ + +/* singleq buf + | 0| + ---------------------------------------------------------------- + | Rx packet buffer adresss | + ---------------------------------------------------------------- + | Rx header buffer adresss | + ---------------------------------------------------------------- + | RSV | + ---------------------------------------------------------------- + | RSV | + ---------------------------------------------------------------- + | 0| + */ +struct virtchnl_singleq_rx_buf_desc { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + __le64 rsvd1; + __le64 rsvd2; +}; /* read used with buffer queues*/ + +union virtchnl_rx_buf_desc { + struct virtchnl_singleq_rx_buf_desc read; + struct virtchnl_splitq_rx_buf_desc split_rd; +}; + +/* (0x00) singleq wb(compl) */ +struct virtchnl_singleq_base_rx_desc { + struct { + struct { + __le16 mirroring_status; + __le16 l2tag1; + } lo_dword; + union { + __le32 rss; /* RSS Hash */ + __le32 fd_id; /* Flow Director filter id */ + } hi_dword; + } qword0; + struct { + /* status/error/PTYPE/length */ + __le64 status_error_ptype_len; + } qword1; + struct { + __le16 ext_status; /* extended status */ + __le16 rsvd; + __le16 l2tag2_1; + __le16 l2tag2_2; + } qword2; + struct { + __le32 reserved; + __le32 fd_id; + } qword3; +}; /* writeback */ + +/* (0x01) singleq flex compl */ +struct virtchnl_rx_flex_desc { + /* Qword 0 */ + u8 rxdid; /* descriptor builder profile id */ + u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */ + __le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */ + __le16 pkt_len; /* [15:14] are reserved */ + __le16 hdr_len_sph_flex_flags1; /* header=[10:0] */ + /* sph=[11:11] */ + /* ff1/ext=[15:12] */ + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 flex_meta0; + __le16 flex_meta1; + + /* Qword 2 */ + __le16 status_error1; + u8 flex_flags2; + u8 time_stamp_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le16 flex_meta2; + __le16 flex_meta3; + union { + struct { + __le16 flex_meta4; + __le16 flex_meta5; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/* (0x02) */ +struct virtchnl_rx_flex_desc_nic { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 rss_hash; + + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 flow_id; + union { + struct { + __le16 rsvd; + __le16 flow_id_ipv6; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor Switch Profile + * RxDID Profile Id 3 + * Flex-field 0: Source Vsi + */ +struct virtchnl_rx_flex_desc_sw { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 src_vsi; /* [10:15] are reserved */ + __le16 flex_md1_rsvd; + + /* Qword 2 */ + __le16 status_error1; + u8 flex_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 rsvd; /* flex words 2-3 are reserved */ + __le32 ts_high; +}; + +/* Rx Flex Descriptor NIC VEB Profile + * RxDID Profile Id 4 + * Flex-field 0: Destination Vsi + */ +struct virtchnl_rx_flex_desc_nic_veb_dbg { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 dst_vsi; /* [0:12]: destination vsi */ + /* 13: vsi valid bit */ + /* [14:15] are reserved */ + __le16 flex_field_1; + + /* Qword 2 */ + __le16 status_error1; + u8 flex_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le32 rsvd; /* flex words 2-3 are reserved */ + __le32 ts_high; +}; + +/* Rx Flex Descriptor NIC ACL Profile + * RxDID Profile Id 5 + * Flex-field 0: ACL Counter 0 + * Flex-field 1: ACL Counter 1 + * Flex-field 2: ACL Counter 2 + */ +struct virtchnl_rx_flex_desc_nic_acl_dbg { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le16 acl_ctr0; + __le16 acl_ctr1; + + /* Qword 2 */ + __le16 status_error1; + u8 flex_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le16 acl_ctr2; + __le16 rsvd; /* flex words 2-3 are reserved */ + __le32 ts_high; +}; + +/* Rx Flex Descriptor NIC Profile + * RxDID Profile Id 6 + * Flex-field 0: RSS hash lower 16-bits + * Flex-field 1: RSS hash upper 16-bits + * Flex-field 2: Flow Id lower 16-bits + * Flex-field 3: Source Vsi + * Flex-field 4: reserved, Vlan id taken from L2Tag + */ +struct virtchnl_rx_flex_desc_nic_2 { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flexi_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 rss_hash; + + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + + /* Qword 3 */ + __le16 flow_id; + __le16 src_vsi; + union { + struct { + __le16 rsvd; + __le16 flow_id_ipv6; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/* Rx Flex Descriptor Advanced (Split Queue Model) + * RxDID Profile Id 7 + */ +struct virtchnl_rx_flex_desc_adv { + /* Qword 0 */ + u8 rxdid_ucast; /* profile_id=[3:0] */ + /* rsvd=[5:4] */ + /* ucast=[7:6] */ + u8 status_err0_qw0; + __le16 ptype_err_fflags0; /* ptype=[9:0] */ + /* ip_hdr_err=[10:10] */ + /* udp_len_err=[11:11] */ + /* ff0=[15:12] */ + __le16 pktlen_gen_bufq_id; /* plen=[13:0] */ + /* gen=[14:14] only in splitq */ + /* bufq_id=[15:15] only in splitq */ + __le16 hdrlen_flags; /* header=[9:0] */ + /* rsc=[10:10] only in splitq */ + /* sph=[11:11] only in splitq */ + /* ext_udp_0=[12:12] */ + /* int_udp_0=[13:13] */ + /* trunc_mirr=[14:14] */ + /* miss_prepend=[15:15] */ + /* Qword 1 */ + u8 status_err0_qw1; + u8 status_err1; + u8 fflags1; + u8 ts_low; + __le16 fmd0; + __le16 fmd1; + /* Qword 2 */ + __le16 fmd2; + u8 fflags2; + u8 hash3; + __le16 fmd3; + __le16 fmd4; + /* Qword 3 */ + __le16 fmd5; + __le16 fmd6; + __le16 fmd7_0; + __le16 fmd7_1; +}; /* writeback */ + +/* Rx Flex Descriptor Advanced (Split Queue Model) NIC Profile + * RxDID Profile Id 8 + * Flex-field 0: BufferID + * Flex-field 1: Raw checksum/L2TAG1/RSC Seg Len (determined by HW) + * Flex-field 2: Hash[15:0] + * Flex-flags 2: Hash[23:16] + * Flex-field 3: L2TAG2 + * Flex-field 5: L2TAG1 + * Flex-field 7: Timestamp (upper 32 bits) + */ +struct virtchnl_rx_flex_desc_adv_nic_3 { + /* Qword 0 */ + u8 rxdid_ucast; /* profile_id=[3:0] */ + /* rsvd=[5:4] */ + /* ucast=[7:6] */ + u8 status_err0_qw0; + __le16 ptype_err_fflags0; /* ptype=[9:0] */ + /* ip_hdr_err=[10:10] */ + /* udp_len_err=[11:11] */ + /* ff0=[15:12] */ + __le16 pktlen_gen_bufq_id; /* plen=[13:0] */ + /* gen=[14:14] only in splitq */ + /* bufq_id=[15:15] only in splitq */ + __le16 hdrlen_flags; /* header=[9:0] */ + /* rsc=[10:10] only in splitq */ + /* sph=[11:11] only in splitq */ + /* ext_udp_0=[12:12] */ + /* int_udp_0=[13:13] */ + /* trunc_mirr=[14:14] */ + /* miss_prepend=[15:15] */ + /* Qword 1 */ + u8 status_err0_qw1; + u8 status_err1; + u8 fflags1; + u8 ts_low; + __le16 buf_id; /* only in splitq */ + union { + __le16 raw_cs; + __le16 l2tag1; + __le16 rscseglen; + } misc; + /* Qword 2 */ + __le16 hash1; + union { + u8 fflags2; + u8 mirrorid; + u8 hash2; + } ff2_mirrid_hash2; + u8 hash3; + __le16 l2tag2; + __le16 fmd4; + /* Qword 3 */ + __le16 l2tag1; + __le16 fmd6; + __le32 ts_high; +}; /* writeback */ + +union virtchnl_rx_desc { + struct virtchnl_singleq_rx_buf_desc read; + struct virtchnl_singleq_base_rx_desc base_wb; + struct virtchnl_rx_flex_desc flex_wb; + struct virtchnl_rx_flex_desc_adv flex_wb_adv; +}; + +#endif /* _VIRTCHNL_LAN_DESC_H_ */