diff --git a/QDMA/DPDK/RELEASE.txt b/QDMA/DPDK/RELEASE.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4f45da0ec2413736f7164447a3112293fc0c95cb
--- /dev/null
+++ b/QDMA/DPDK/RELEASE.txt
@@ -0,0 +1,42 @@
+Release: 2018.3
+===============
+
+This release is based on DPDK v17.11.1 containing QDMA poll mode driver and
+QDMA test application.
+
+This release includes a patch file for dpdk-pktgen v3.4.7 that extends
+dpdk-pktgen application to handle packets with packet sizes more than 1518 bytes
+and it disables the packet size classification logic in dpdk-pktgen to remove
+application overhead in performance measurement.
+This patch is used for performance testing with dpdk-pktgen application.
+
+The driver is validated against dpdk-pktgen and testpmd applications for API compliance.
+2018.3 driver is not backward compatible with 2018.2 and earlier QDMA IP.
+
+SUPPORTED FEATURES:
+===================
+- Support for both the AXI4 Memory Mapped(MM) and AXI4 Streaming(ST) Interfaces
+- 2048 Queue Sets
+ - 2048 H2C Descriptor Rings
+ - 2048 C2H Descriptor Rings
+ - 2048 C2H Completion Rings
+- Supports Polling Mode
+- Supports SR-IOV with 4 Physical Functions(PF) and 252 Virtual Functions(VF)
+- Allows Only Privileged/Physical functions to program the contexts and registers
+- Function Level Reset(FLR) Support
+- Mailbox Support
+- Supports Descriptor Prefetch
+- ST H2C to C2H and C2H to H2C loopback support
+- Zero-byte transfer support
+
+2018.3 Features
+- Descriptor (8, 16, 32, 64 bytes) bypass support
+- Support for Completion queue descriptors of 64 bytes size
+- Support flexible BAR mapping for QDMA configuration register space
+- Support disabling overflow check in completion ring
+- Indirect programming of FMAP registers
+
+
+KNOWN ISSUES:
+============
+- Occasionally, C2H completions are not observed when payload and immediate data transfers are combined
\ No newline at end of file
diff --git a/QDMA/DPDK/docs/DPDK_qdma_driver_user_guide.pdf b/QDMA/DPDK/docs/DPDK_qdma_driver_user_guide.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..c7951790fbc35f0e30f34fa61c8622772dd6b76c
Binary files /dev/null and b/QDMA/DPDK/docs/DPDK_qdma_driver_user_guide.pdf differ
diff --git a/QDMA/DPDK/drivers/net/qdma/Makefile b/QDMA/DPDK/drivers/net/qdma/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..ccfd9f48e04ffa486b7c133d212f6264422ae597
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/Makefile
@@ -0,0 +1,78 @@
+# BSD LICENSE
+#
+# Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+## if modified this BRAM_SIZE, then the same should be modified in the testapp Makefile also
+## default set to 512KB
+BRAM_SIZE ?= 524288
+
+#
+# library name
+#
+LIB = librte_pmd_qdma.a
+
+# library version
+ LIBABIVER := 1
+
+# versioning export map
+EXPORT_MAP := rte_pmd_qdma_version.map
+
+CFLAGS += -O3 -DDMA_BRAM_SIZE=$(BRAM_SIZE)
+#CFLAGS += -g
+CFLAGS += $(WERROR_FLAGS)
+
+ifeq ($(TEST_64B_DESC_BYPASS),1)
+ CFLAGS += -DTEST_64B_DESC_BYPASS
+endif
+
+# this lib depends upon:
+LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
+LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs
+LDLIBS += -lrte_bus_pci
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_vf_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_devops.c
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_common.c
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_rxtx.c
+SRCS-$(CONFIG_RTE_LIBRTE_QDMA_PMD) += qdma_xdebug.c
+
+#
+## Export include files
+#
+SYMLINK-y-include += qdma_export.h
+
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma.h b/QDMA/DPDK/drivers/net/qdma/qdma.h
new file mode 100644
index 0000000000000000000000000000000000000000..07e52db468486725c568f7015e96879cd464f5da
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma.h
@@ -0,0 +1,488 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __QDMA_H__
+#define __QDMA_H__
+
+#include <rte_dev.h>
+#include <rte_ethdev.h>
+#include <rte_spinlock.h>
+#include <rte_log.h>
+#include <rte_byteorder.h>
+#include <rte_memzone.h>
+#include <linux/pci.h>
+#include "qdma_user.h"
+
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+#define PMD_DRV_LOG(level, fmt, args...) \
+ RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#else
+#define PMD_DRV_LOG(level, fmt, args...) do { } while (0)
+#endif
+
+#define FMAP_CNTXT (1)
+
+#define NUM_BARS (6)
+#define DEFAULT_PF_CONFIG_BAR (0)
+#define DEFAULT_VF_CONFIG_BAR (0)
+#define BAR_ID_INVALID -1
+
+#define QDMA_QUEUES_NUM_MAX (2048)
+#define QDMA_MAX_BURST_SIZE (256)
+#define QDMA_MIN_RXBUFF_SIZE (256)
+
+#define RX_STATUS_EOP (1)
+
+#define QDMA_BUF_ALIGN (4096)
+/* Descriptor Rings aligned to 4KB boundaries - only supported value */
+#define QDMA_ALIGN (4096)
+
+/* If this RX_BUF_SIZE, modified then the same value need to be modified
+ * in the testapp commands.h file "MAX_BUF_SIZE" macro.
+ * This is maintained equal to the descriptor Bufffer size
+ */
+#define RX_BUF_SIZE 4096
+
+/* QDMA IP absolute maximum */
+#define QDMA_PF_MAX 4 /* 4 PFs */
+#define QDMA_VF_MAX 252
+#define QDMA_FUNC_ID_INVALID (QDMA_PF_MAX + QDMA_VF_MAX)
+
+/** MBOX OPCODE **/
+#define QDMA_MBOX_OPCD_HI (1)
+#define QDMA_MBOX_OPCD_BYE (2)
+#define QDMA_MBOX_OPCD_QADD (3)
+#define QDMA_MBOX_OPCD_QDEL (4)
+
+#define MBOX_POLL_FRQ 500000
+
+#define DEFAULT_QUEUE_BASE (0)
+
+/*Corresponding to 100ns (1tick = 4ns for 250MHz user clock)*/
+#define DEFAULT_C2H_INTR_TIMER_TICK (25)
+#define DEFAULT_TIMER_CNT_TRIG_MODE_TIMER (5)
+#define DEFAULT_TIMER_CNT_TRIG_MODE_COUNT_TIMER (30)
+#define DEFAULT_MAX_DESC_FETCH (3) /* Max nuber of descriptors to fetch in one
+ * request is 8 * 2^val; Max value is 6;
+ * Keep same as PCIe MRRS
+ */
+#define DEFAULT_WB_ACC_INT (4) // Calculated as 4 * 2^val; Max value is 7
+#define MAX_WB_ACC_INT (7) // Calculated as 4 * 2^val; Max value is 7
+
+#define DEFAULT_H2C_THROTTLE (0x14000) // Setting throttle to 16K buffer size
+
+#define MIN_RX_PIDX_UPDATE_THRESHOLD (1)
+#define MIN_TX_PIDX_UPDATE_THRESHOLD (1)
+#define QDMA_TXQ_PIDX_UPDATE_INTERVAL (1000) //100 uSec
+
+#define DEFAULT_PFCH_STOP_THRESH (256)
+#define DEFAULT_PFCH_NUM_ENTRIES_PER_Q (8)
+#define DEFAULT_PFCH_MAX_Q_CNT (16)
+#define DEFAULT_PFCH_EVICTION_Q_CNT (14)
+
+#define DEFAULT_CMPT_COAL_TIMER_CNT (5)
+#define DEFAULT_CMPT_COAL_TIMER_TICK (25)
+#define DEFAULT_CMPT_COAL_MAX_BUF_SZ (32)
+
+/** Delays **/
+#define CONTEXT_PROG_DELAY (1)
+#define MAILBOX_PF_MSG_DELAY (20)
+#define PF_RESET_DELAY (1)
+#define PF_START_DELAY (1)
+#define MAILBOX_VF_MSG_DELAY (10)
+#define VF_RESET_DELAY (200)
+#define CONTEXT_PROG_POLL_COUNT (10)
+#define MAILBOX_PROG_POLL_COUNT (1250)
+
+#define WB_TIMEOUT (UINT16_MAX * 4)
+
+#ifdef spin_lock_init
+#undef spin_lock_init
+#endif
+#define spin_lock_init(sl) rte_spinlock_init(sl)
+
+enum cmpt_desc_len {
+ CMPT_DESC_LEN_8B = 8,
+ CMPT_DESC_LEN_16B = 16,
+ CMPT_DESC_LEN_32B = 32,
+ CMPT_DESC_LEN_64B = 64
+};
+
+enum bypass_desc_len {
+ BYPASS_DESC_LEN_8B = 8,
+ BYPASS_DESC_LEN_16B = 16,
+ BYPASS_DESC_LEN_32B = 32,
+ BYPASS_DESC_LEN_64B = 64
+};
+
+enum c2h_bypass_mode {
+ C2H_BYPASS_MODE_NONE = 0,
+ C2H_BYPASS_MODE_CACHE = 1,
+ C2H_BYPASS_MODE_SIMPLE = 2,
+ C2H_BYPASS_MODE_MAX = C2H_BYPASS_MODE_SIMPLE,
+};
+
+#define DEFAULT_QDMA_CMPT_DESC_LEN (CMPT_DESC_LEN_8B)
+
+struct qdma_pkt_stats {
+ uint64_t pkts;
+ uint64_t bytes;
+};
+
+/**
+ * Structure associated with each RX queue.
+ */
+struct qdma_rx_queue {
+ struct rte_mempool *mb_pool; /**< mbuf pool to populate RX ring. */
+ void *rx_ring; /**< RX ring virtual address */
+ struct c2h_cmpt_ring *cmpt_ring;
+ struct wb_status *wb_status;
+ struct rte_mbuf **sw_ring; /**< address of RX software ring. */
+ volatile uint32_t *rx_pidx;
+ volatile uint32_t *rx_cidx;
+
+ uint16_t rx_tail;
+ uint16_t c2h_pidx;
+ uint16_t rx_cmpt_tail;
+ uint16_t cmpt_desc_len;
+ uint16_t rx_buff_size;
+ uint16_t nb_rx_desc; /**< number of RX descriptors. */
+ uint16_t nb_rx_cmpt_desc;
+ uint32_t queue_id; /**< RX queue index. */
+
+ struct qdma_pkt_stats stats;
+
+ uint32_t ep_addr;
+
+ uint8_t st_mode:1; /**< dma-mode: MM or ST */
+ uint8_t rx_deferred_start:1;
+ uint8_t en_prefetch:1;
+ uint8_t en_bypass:1;
+ uint8_t dump_immediate_data:1;
+ uint8_t en_bypass_prefetch:1;
+ uint8_t dis_overflow_check:1;
+ uint8_t status:1;
+ enum bypass_desc_len bypass_desc_sz;
+
+ uint16_t port_id; /**< Device port identifier. */
+ uint16_t func_id; /**< RX queue index. */
+
+
+ int8_t ringszidx;
+ int8_t cmpt_ringszidx;
+ int8_t buffszidx;
+ int8_t threshidx;
+ int8_t timeridx;
+ int8_t triggermode;
+
+ const struct rte_memzone *rx_mz;
+ /* C2H stream mode, completion descriptor result */
+ const struct rte_memzone *rx_cmpt_mz;
+};
+
+/**
+ * Structure associated with each TX queue.
+ */
+struct qdma_tx_queue {
+ void *tx_ring; /* TX ring virtual address*/
+ struct wb_status *wb_status;
+ struct rte_mbuf **sw_ring;/* SW ring virtual address*/
+ volatile uint32_t *tx_pidx;
+ uint16_t tx_tail;
+ uint16_t tx_fl_tail;
+ uint16_t tx_desc_pend;
+ uint16_t nb_tx_desc; /* No of TX descriptors.*/
+ rte_spinlock_t pidx_update_lock;
+
+ uint8_t st_mode:1;/* dma-mode: MM or ST */
+ uint8_t tx_deferred_start:1;
+ uint8_t en_bypass:1;
+ uint8_t status:1;
+ enum bypass_desc_len bypass_desc_sz;
+ uint16_t func_id; /* RX queue index. */
+ uint16_t port_id; /* Device port identifier. */
+ int8_t ringszidx;
+
+ struct qdma_pkt_stats stats;
+
+ uint32_t ep_addr;
+ uint32_t queue_id; /* TX queue index. */
+ uint32_t num_queues; /* TX queue index. */
+ const struct rte_memzone *tx_mz;
+};
+
+struct qdma_vf_queue_info {
+ uint32_t mode;
+};
+
+struct qdma_vf_info {
+ uint32_t qbase;
+ uint32_t qmax;
+ uint16_t func_id;
+ struct qdma_vf_queue_info *vfqinfo;
+};
+
+struct queue_info {
+ uint32_t queue_mode:1;
+ uint32_t rx_bypass_mode:2;
+ uint32_t tx_bypass_mode:1;
+ uint32_t cmpt_desc_sz:7;
+ enum bypass_desc_len rx_bypass_desc_sz:7;
+ enum bypass_desc_len tx_bypass_desc_sz:7;
+};
+
+struct qdma_pci_dev {
+ struct pci_dev *pdev; /* PCI device pointer */
+ unsigned long magic;
+ int config_bar_idx;
+ int user_bar_idx;
+ int bypass_bar_idx;
+ struct rte_eth_dev *eth_dev;
+
+ /* Driver Attributes */
+ uint32_t qsets_max; /* max. of queue pairs */
+ uint32_t qsets_en; /* no. of queue pairs enabled */
+ uint8_t st_mode_en; /* Streaming mode enabled? */
+ uint8_t mm_mode_en; /* Memory mapped mode enabled? */
+
+ uint16_t pf;
+ uint8_t is_vf;
+ uint8_t en_desc_prefetch;
+ uint8_t cmpt_desc_len;
+ uint8_t c2h_bypass_mode;
+ uint8_t h2c_bypass_mode;
+
+ void *bar_addr[NUM_BARS]; /* memory mapped I/O addr for BARs */
+ unsigned long bar_len[NUM_BARS]; /* length of BAR regions */
+ unsigned int flags;
+
+ volatile uint32_t *c2h_mm_control;
+ volatile uint32_t *h2c_mm_control;
+ unsigned int queue_base;
+ uint8_t wb_acc_int;
+ uint8_t trigger_mode;
+ uint16_t timer_count;
+
+ /* Hardware version info*/
+ uint32_t everest_ip:1;
+ uint32_t vivado_rel:4;
+ uint32_t rtl_version:8;
+ struct qdma_vf_info *vfinfo;
+ struct queue_info *q_info;
+};
+
+struct __attribute__ ((packed)) qdma_mbox_data {
+ uint8_t opcode;
+ uint16_t debug_funid:12; /* Used for debug information only*/
+ uint16_t filler:4;
+ uint8_t err;
+ uint32_t data[31];
+};
+
+struct __attribute__ ((packed)) qadd_msg {
+ uint32_t qid;
+ uint32_t st:1;
+ uint32_t c2h:1;
+ uint32_t prefetch:1;
+ uint32_t en_bypass:1;
+ uint32_t en_bypass_prefetch:1;
+ uint32_t dis_overflow_check:1;
+ uint32_t bypass_desc_sz_idx:4;
+ uint32_t ringsz;
+ uint32_t bufsz;
+ uint32_t thresidx;
+ uint32_t timeridx;
+ uint32_t triggermode;
+ uint64_t ring_bs_addr;
+ uint64_t cmpt_ring_bs_addr;
+ uint32_t cmpt_ringszidx;
+ uint8_t cmpt_desc_fmt;
+};
+
+enum get_param_type {
+ CONFIG_BAR = 0,
+ USER_BAR,
+ BYPASS_BAR
+};
+enum get_queue_param_type {
+ CHECK_DUMP_IMMEDIATE_DATA = 0x10
+};
+
+enum dev_param_type {
+ QUEUE_BASE = 0,
+ TIMER_COUNT,
+ TRIGGER_MODE,
+};
+
+enum queue_param_type {
+ QUEUE_MODE = 0x10,
+ DESC_PREFETCH,
+ TX_DST_ADDRESS,
+ RX_SRC_ADDRESS,
+ DIS_OVERFLOW_CHECK,
+ DUMP_IMMEDIATE_DATA,
+ RX_BYPASS_MODE,
+ RX_BYPASS_DESC_SIZE,
+ TX_BYPASS_MODE,
+ TX_BYPASS_DESC_SIZE,
+ CMPT_DESC_SIZE
+};
+
+/*
+ * queue_mode_t - queue modes
+ */
+enum queue_mode_t {
+ MEMORY_MAPPED_MODE,
+ STREAMING_MODE
+};
+
+/*
+ * tigger_mode_t - trigger modes
+ */
+enum tigger_mode_t {
+ TRIG_MODE_DISABLE,
+ TRIG_MODE_EVERY,
+ TRIG_MODE_USER_COUNT,
+ TRIG_MODE_USER,
+ TRIG_MODE_USER_TIMER,
+ TRIG_MODE_USER_TIMER_COUNT,
+ TRIG_MODE_MAX = TRIG_MODE_USER_TIMER_COUNT,
+};
+
+enum dma_data_direction {
+ DMA_BIDIRECTIONAL = 0,
+ DMA_TO_DEVICE = 1,
+ DMA_FROM_DEVICE = 2,
+ DMA_NONE = 3,
+};
+
+void qdma_dev_ops_init(struct rte_eth_dev *dev);
+uint32_t qdma_read_reg(uint64_t addr);
+void qdma_write_reg(uint64_t addr, uint32_t val);
+void qdma_txq_pidx_update(void *arg);
+
+int qdma_queue_ctxt_tx_prog(struct rte_eth_dev *dev, uint32_t qid,
+ uint32_t mode, uint32_t ringszidx, uint16_t func_id, uint64_t phys_addr,
+ uint8_t en_bypass, uint8_t bypass_desc_sz_idx);
+
+int qdma_queue_ctxt_rx_prog(struct rte_eth_dev *dev, uint32_t qid,
+ uint32_t mode, uint8_t prefetch,
+ uint32_t ringszidx, uint32_t cmpt_ringszidx,
+ uint32_t bufszidx, uint32_t threshidx,
+ uint32_t timeridx, uint32_t triggermode,
+ uint16_t func_id, uint64_t phys_addr,
+ uint64_t cmpt_phys_addr, uint8_t cmpt_desc_fmt,
+ uint8_t en_bypass, uint8_t en_bypass_prefetch,
+ uint8_t bypass_desc_sz_idx,
+ uint8_t dis_overflow_check);
+
+int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+ uint16_t nb_rx_desc, unsigned int socket_id,
+ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mb_pool);
+
+int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+ uint16_t nb_tx_desc, unsigned int socket_id,
+ const struct rte_eth_txconf *tx_conf);
+void qdma_dev_rx_queue_release(void *rqueue);
+void qdma_dev_tx_queue_release(void *tqueue);
+uint8_t qmda_get_desc_sz_idx(enum bypass_desc_len);
+int qdma_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int qdma_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int qdma_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int qdma_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int qdma_vf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int qdma_vf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int qdma_vf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int qdma_vf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+
+int qdma_start_rx_queue(struct qdma_rx_queue *rxq);
+void qdma_start_tx_queue(struct qdma_tx_queue *txq);
+void qdma_reset_tx_queue(struct qdma_tx_queue *txq);
+void qdma_reset_rx_queue(struct qdma_rx_queue *rxq);
+
+void qdma_reset_rx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode);
+void qdma_inv_rx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode);
+void qdma_reset_tx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode);
+void qdma_inv_tx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode);
+int qdma_set_fmap(struct qdma_pci_dev *qdma_dev, uint16_t devfn,
+ uint32_t q_base, uint32_t q_count);
+int qdma_identify_bars(struct rte_eth_dev *dev);
+int qdma_get_hw_version(struct rte_eth_dev *dev);
+/* implemented in rxtx.c */
+uint16_t qdma_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t qdma_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+int qdma_dev_rx_descriptor_done(void *rx_queue, uint16_t exp_count);
+
+uint16_t qdma_recv_pkts_st(struct qdma_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+uint16_t qdma_recv_pkts_mm(struct qdma_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
+uint16_t qdma_xmit_pkts_st(struct qdma_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+uint16_t qdma_xmit_pkts_mm(struct qdma_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
+uint32_t qdma_pci_read_reg(struct rte_eth_dev *dev, uint32_t bar, uint32_t reg);
+void qdma_pci_write_reg(struct rte_eth_dev *dev, uint32_t bar,
+ uint32_t reg, uint32_t val);
+void qdma_desc_dump(struct rte_eth_dev *dev, uint32_t qid);
+
+int index_of_array(uint32_t *arr, uint32_t n, uint32_t element);
+int get_param(struct rte_eth_dev *dev, enum get_param_type param, void *value);
+int get_queue_param(struct rte_eth_dev *dev, uint32_t qid,
+ enum get_queue_param_type param, void *value);
+int update_param(struct rte_eth_dev *dev, enum dev_param_type param,
+ uint16_t value);
+int update_queue_param(struct rte_eth_dev *dev, uint32_t qid,
+ enum queue_param_type param, uint32_t value);
+int qdma_check_kvargs(struct rte_devargs *devargs,
+ struct qdma_pci_dev *qdma_dev);
+
+static inline const
+struct rte_memzone *qdma_zone_reserve(struct rte_eth_dev *dev,
+ const char *ring_name,
+ uint32_t queue_id,
+ uint32_t ring_size,
+ int socket_id)
+{
+ char z_name[RTE_MEMZONE_NAMESIZE];
+ snprintf(z_name, sizeof(z_name), "%s%s%d_%d",
+ dev->device->driver->name, ring_name,
+ dev->data->port_id, queue_id);
+ return rte_memzone_reserve_aligned(z_name, (uint64_t)ring_size,
+ socket_id, 0, QDMA_ALIGN);
+}
+#endif /* ifndef __QDMA_H__ */
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_common.c b/QDMA/DPDK/drivers/net/qdma/qdma_common.c
new file mode 100644
index 0000000000000000000000000000000000000000..fb95d0a014b8f201185d5849ba33db3f36b90ee4
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_common.c
@@ -0,0 +1,1380 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <rte_malloc.h>
+#include <rte_common.h>
+#include <rte_ethdev_pci.h>
+#include <rte_cycles.h>
+#include <rte_kvargs.h>
+#include "qdma.h"
+#include "qdma_regs.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+/* Read register */
+uint32_t qdma_read_reg(uint64_t reg_addr)
+{
+ uint32_t val;
+
+ val = *((volatile uint32_t *)(reg_addr));
+ return val;
+}
+
+/* Write register */
+void qdma_write_reg(uint64_t reg_addr, uint32_t val)
+{
+ *((volatile uint32_t *)(reg_addr)) = val;
+}
+
+uint32_t qdma_pci_read_reg(struct rte_eth_dev *dev, uint32_t bar, uint32_t reg)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t baseaddr;
+ uint32_t val;
+
+ if (bar >= (NUM_BARS - 1)) {
+ printf("Error: PCI BAR number:%d not supported\n"
+ "Please enter valid BAR number\n", bar);
+ return -1;
+ }
+
+ baseaddr = (uint64_t)qdma_dev->bar_addr[bar];
+ if (!baseaddr) {
+ printf("Error: PCI BAR number:%d not mapped\n", bar);
+ return -1;
+ }
+ val = *((volatile uint32_t *)(baseaddr + reg));
+
+ return val;
+}
+
+void qdma_pci_write_reg(struct rte_eth_dev *dev, uint32_t bar,
+ uint32_t reg, uint32_t val)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t baseaddr;
+
+ if (bar >= (NUM_BARS - 1)) {
+ printf("Error: PCI BAR index:%d not supported\n"
+ "Please enter valid BAR index\n", bar);
+ return;
+ }
+
+ baseaddr = (uint64_t)qdma_dev->bar_addr[bar];
+ if (!baseaddr) {
+ printf("Error: PCI BAR number:%d not mapped\n", bar);
+ return;
+ }
+ *((volatile uint32_t *)(baseaddr + reg)) = val;
+}
+
+void qdma_desc_dump(struct rte_eth_dev *dev, uint32_t qid)
+{
+ struct qdma_rx_queue *rxq;
+ struct qdma_tx_queue *txq;
+ uint32_t i, k, bypass_desc_sz_idx;
+ struct qdma_c2h_desc *rx_ring_st;
+ struct qdma_mm_desc *ring_mm;
+ struct qdma_h2c_desc *tx_ring_st;
+ uint8_t *rx_ring_bypass = NULL;
+ uint8_t *tx_ring_bypass = NULL;
+
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+
+ if (rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA) {
+ rx_ring_bypass = (uint8_t *)rxq->rx_ring;
+ for (i = 0; i < rxq->nb_rx_desc; ++i) {
+ for (k = 0; k < rxq->bypass_desc_sz; k++)
+ printf("\trxq descriptor-data[%d]: %x\n", i,
+ rx_ring_bypass[i * (rxq->bypass_desc_sz)
+ + k]);
+ }
+ } else if (rxq->st_mode) {
+ rx_ring_st = (struct qdma_c2h_desc *)rxq->rx_ring;
+ printf("\nST-mode Rx descriptor dump on Queue-id:%d:\n", qid);
+ for (i = 0; i < rxq->nb_rx_desc; ++i) {
+ printf("\t desc-index:%d: dest-addr:%lx\n", i,
+ rx_ring_st[i].dst_addr);
+ }
+ } else {
+ ring_mm = (struct qdma_mm_desc *)rxq->rx_ring;
+ printf("\nMM-mode Rx descriptor dump on Queue-id:%d:\n", qid);
+ for (i = 0; i < rxq->nb_rx_desc; ++i) {
+ printf("\t desc-index:%d: dest-addr:%lx, src-addr:%lx,"
+ " valid:%d, eop:%d, sop:%d,"
+ "length:%d\n", i,
+ ring_mm[i].dst_addr,
+ ring_mm[i].src_addr,
+ ring_mm[i].dv,
+ ring_mm[i].eop,
+ ring_mm[i].sop,
+ ring_mm[i].len);
+ }
+ }
+
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+
+ if (txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA) {
+ tx_ring_bypass = (uint8_t *)txq->tx_ring;
+ for (i = 0; i < txq->nb_tx_desc; ++i) {
+ for (k = 0; k < txq->bypass_desc_sz; k++)
+ printf("\t txq descriptor-data[%d]: %x\n", i,
+ tx_ring_bypass[i *
+ (txq->bypass_desc_sz) + k]);
+ }
+ } else if (txq->st_mode) {
+ tx_ring_st = (struct qdma_h2c_desc *)txq->tx_ring;
+ printf("\nST-mode Tx descriptor dump on Queue-id:%d:\n", qid);
+ for (i = 0; i < txq->nb_tx_desc; ++i) {
+ printf("\t desc-index:%d: src-addr:%lx,length:%d\n", i,
+ tx_ring_st[i].src_addr,
+ tx_ring_st[i].len);
+ }
+
+ } else {
+ ring_mm = (struct qdma_mm_desc *)txq->tx_ring;
+ printf("\nMM-mode Tx descriptor dump on Queue-id:%d:\n", qid);
+ for (i = 0; i < txq->nb_tx_desc; ++i) {
+ printf("\t desc-index:%d: dest-addr:%lx, src-addr:%lx,"
+ " valid:%d, eop:%d,sop:%d, length:%d\n",
+ i, ring_mm[i].dst_addr,
+ ring_mm[i].src_addr,
+ ring_mm[i].dv,
+ ring_mm[i].eop,
+ ring_mm[i].sop,
+ ring_mm[i].len);
+ }
+ }
+}
+
+void qdma_reset_rx_queue(struct qdma_rx_queue *rxq)
+{
+ uint32_t i;
+ uint32_t sz;
+
+ rxq->rx_tail = 0;
+ rxq->c2h_pidx = 0;
+ rxq->rx_cmpt_tail = 0;
+
+ /* Zero out HW ring memory, For MM Descriptor */
+ if (rxq->st_mode) { /** if ST-mode **/
+ sz = rxq->cmpt_desc_len;
+ for (i = 0; i < (sz * rxq->nb_rx_cmpt_desc); i++)
+ ((volatile char *)rxq->cmpt_ring)[i] = 0;
+
+ sz = sizeof(struct qdma_c2h_desc);
+ for (i = 0; i < (sz * rxq->nb_rx_desc); i++)
+ ((volatile char *)rxq->rx_ring)[i] = 0;
+
+ } else {
+ sz = sizeof(struct qdma_mm_desc);
+ for (i = 0; i < (sz * rxq->nb_rx_desc); i++)
+ ((volatile char *)rxq->rx_ring)[i] = 0;
+ }
+
+ /* Initialize SW ring entries */
+ for (i = 0; i < rxq->nb_rx_desc; i++)
+ rxq->sw_ring[i] = NULL;
+}
+
+void qdma_inv_rx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+ uint64_t bar_addr;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ /** To clear the Queues **/
+ /** To clear the SW C2H Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "SW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ /** To clear the HW C2H Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (mode) { /** ST-mode **/
+ /** To clear the C2H prefetch context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+
+ /** To clear the C2H Completion Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+
+ /** To clear the C2H credit context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_CR_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "C2H creditsClear cmd for qid:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ }
+}
+/**
+ * reset the Rx device queues.
+ *
+ * reset the device by clearing the Rx descriptors for all
+ * queues and device registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * Nothing.
+ */
+void qdma_reset_rx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+ uint64_t bar_addr;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ /** To clear the Queues **/
+ /** To clear the SW C2H Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "SW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ /** To clear the HW C2H Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (mode) { /** ST-mode **/
+ /** To clear the C2H prefetch context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+
+ /** To clear the C2H Completion Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW C2H Clear cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+
+ /** To clear the C2H credit context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_CR_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "C2H creditsClear cmd for qid:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ }
+}
+
+int qdma_start_rx_queue(struct qdma_rx_queue *rxq)
+{
+ struct rte_mbuf *mb;
+ void *obj = NULL;
+ uint64_t phys_addr;
+ uint16_t i;
+ struct qdma_c2h_desc *rx_ring_st = NULL;
+
+ /* allocate new buffers for the Rx descriptor ring */
+ if (rxq->st_mode) { /** ST-mode **/
+ rx_ring_st = (struct qdma_c2h_desc *)rxq->rx_ring;
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: queue id %d, mbuf_avail_count =%d,"
+ "mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ for (i = 0; i < (rxq->nb_rx_desc - 1); i++) {
+ if (rte_mempool_get(rxq->mb_pool, &obj) != 0) {
+ PMD_DRV_LOG(ERR, "qdma-start-rx-queue(): "
+ "rte_mempool_get: failed");
+ goto fail;
+ }
+
+ mb = obj;
+ phys_addr = (uint64_t)mb->buf_physaddr +
+ RTE_PKTMBUF_HEADROOM;
+
+ mb->data_off = RTE_PKTMBUF_HEADROOM;
+ rxq->sw_ring[i] = mb;
+ rx_ring_st[i].dst_addr = phys_addr;
+ }
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: qid %d, mbuf_avail_count = %d,"
+ "mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ }
+
+ /* initialize tail */
+ rxq->rx_tail = 0;
+ rxq->c2h_pidx = 0;
+ rxq->rx_cmpt_tail = 0;
+
+ return 0;
+fail:
+ return -ENOMEM;
+}
+
+/**
+ * Start the Rx device queues.
+ *
+ * Start the device by configuring the Rx descriptors for all
+ * the queues and device registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+
+int qdma_queue_ctxt_rx_prog(struct rte_eth_dev *dev, uint32_t qid,
+ uint32_t mode, uint8_t en_prefetch,
+ uint32_t ringszidx, uint32_t cmpt_ringszidx,
+ uint32_t bufszidx, uint32_t threshidx,
+ uint32_t timeridx, uint32_t triggermode,
+ uint16_t func_id, uint64_t phys_addr,
+ uint64_t cmpt_phys_addr, uint8_t cmpt_desc_fmt,
+ uint8_t en_bypass, uint8_t en_bypass_prefetch,
+ uint8_t bypass_desc_sz_idx,
+ uint8_t dis_overflow_check)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint64_t bar_addr;
+ uint32_t reg_val, ctxt_sel;
+ uint16_t flag, i;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ /* C2H Queue initialization */
+ reg_val = (func_id << SW_DESC_CNTXT_FUNC_ID_SHIFT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[0], reg_val);
+ reg_val = (SW_DESC_CNTXT_WB_EN << SW_DESC_CNTXT_WB_EN_SHIFT_B) |
+ (ringszidx << SW_DESC_CNTXT_RING_SIZE_ID_SHIFT) |
+ (SW_DESC_CNTXT_QUEUE_ENABLE <<
+ SW_DESC_CNTXT_QUEUE_EN_SHIFT);
+
+ if (mode) { /** ST-mode **/
+ if (en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ reg_val = reg_val | (bypass_desc_sz_idx <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT);
+ else
+ reg_val = reg_val | (SW_DESC_CNTXT_C2H_STREAM_DMA <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT);
+
+ reg_val = reg_val | (en_bypass << SW_DESC_CNTXT_BYPASS_SHIFT) |
+ (SW_DESC_CNTXT_FETCH_CREDIT_EN <<
+ SW_DESC_CNTXT_FETCH_CREDIT_EN_SHIFT);
+ } else {
+ if (en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ reg_val = reg_val | (bypass_desc_sz_idx <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT) |
+ (SW_DESC_CNTXT_IS_MM <<
+ SW_DESC_CNTXT_IS_MM_SHIFT);
+ else
+ reg_val = reg_val | (SW_DESC_CNTXT_MEMORY_MAP_DMA <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT) |
+ (SW_DESC_CNTXT_IS_MM <<
+ SW_DESC_CNTXT_IS_MM_SHIFT);
+
+ reg_val = reg_val | (en_bypass << SW_DESC_CNTXT_BYPASS_SHIFT) |
+ (SW_DESC_CNTXT_WBI_INTERVAL <<
+ SW_DESC_CNTXT_WBI_INTERVAL_SHIFT) |
+ (SW_DESC_CNTXT_WBI_CHK <<
+ SW_DESC_CNTXT_WBI_CHK_SHIFT);
+ }
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[1], reg_val);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[2],
+ rte_cpu_to_le_32(phys_addr & MASK_32BIT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[3],
+ rte_cpu_to_le_32(phys_addr >> 32));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[4], 0);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[2], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[3], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[4], MASK_32BIT);
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_WR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for queue-id:%d: reg_val:%x\n",
+ qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ goto fail;
+ }
+
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ /** To read the Queue **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ for (i = 0; i < 5; i++) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_data[i]);
+ PMD_DRV_LOG(INFO, " Read context-data on queue-id:%d,"
+ " idx:%d:reg_val:%x\n", qid, i, reg_val);
+ /* TO-DO: Need to verify the Queue ctxt-data and if Queue has
+ * any errors, then return -1. **/
+ }
+#endif //RTE_LIBRTE_QDMA_DEBUG_DRIVER
+
+ /* C2H Prefetch & Completion context initialization */
+ if (mode) { /** ST-mode **/
+ /** C2H Prefetch Context **/
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[0],
+ (en_prefetch << PREFETCH_EN_SHIFT) |
+ (bufszidx << BUFF_SIZE_INDEX_SHIFT) |
+ (en_bypass_prefetch << PREFETCH_BYPASS_SHIFT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[1],
+ (VALID_CNTXT << VALID_CNTXT_SHIFT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[2], 0);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[3], 0);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[2], MASK_0BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[3], MASK_0BIT);
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_WR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ i = CONTEXT_PROG_POLL_COUNT;
+ flag = 1;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for queue-id:%d:"
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d:"
+ " C2H Prefetch initailization with cmd"
+ " reg_val:%x\n", qid, reg_val);
+ goto fail;
+ }
+
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ /** To read the Queue **/
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ for (i = 0; i < NUM_CONTEXT_REGS; i++) {
+ reg_val =
+ qdma_read_reg((uint64_t)&q_regs->ctxt_data[i]);
+ PMD_DRV_LOG(INFO, " Read context-data C2H Prefetch on "
+ "queue-id:%d, idx:%d:reg_val:%x\n",
+ qid, i, reg_val);
+ }
+#endif //RTE_LIBRTE_QDMA_DEBUG_DRIVER
+
+ /** C2H Completion Context **/
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[0],
+ (CMPT_CNTXT_EN_STAT_DESC <<
+ CMPT_CNTXT_EN_STAT_DESC_SHIFT) |
+ (CMPT_CNTXT_COLOR_BIT <<
+ CMPT_CNTXT_COLOR_BIT_SHIFT) |
+ ((uint32_t)func_id <<
+ CMPT_CNTXT_FUNC_ID_SHIFT) |
+ ((uint64_t)threshidx <<
+ CMPT_CNTXT_THRESHOLD_MODE_SHIFT) |
+ ((uint64_t)timeridx <<
+ CMPT_CNTXT_TIMER_INDEX_SHIFT) |
+ (triggermode <<
+ CMPT_CNTXT_TRIGGER_MODE_SHIFT) |
+ ((uint64_t)cmpt_ringszidx <<
+ CMPT_CNTXT_RING_SIZE_INDEX_SHIFT));
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[1],
+ (rte_cpu_to_le_32(cmpt_phys_addr >> 6) &
+ MASK_32BIT));
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[2],
+ (rte_cpu_to_le_32(cmpt_phys_addr >> 32) >> 6) |
+ ((uint32_t)cmpt_desc_fmt <<
+ CMPT_CNTXT_DESC_SIZE_SHIFT));
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[3],
+ (CMPT_CNTXT_CTXT_VALID <<
+ CMPT_CNTXT_CTXT_VALID_SHIFT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[4],
+ (dis_overflow_check <<
+ CMPT_CNTXT_OVF_CHK_DIS_SHIFT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[2], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[3], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[4], MASK_32BIT);
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_WR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ i = CONTEXT_PROG_POLL_COUNT;
+ flag = 1;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for queue-id:%d:"
+ " reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: C2H CMPT"
+ " initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ goto fail;
+ }
+
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ /** To read the Queue **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ for (i = 0; i < NUM_CONTEXT_REGS; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ PMD_DRV_LOG(INFO, " Read context-data C2H-CMPT on "
+ "queue-id:%d, idx:%d:reg_val:%x\n",
+ qid, i, reg_val);
+ }
+#endif //RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ }
+
+ return 0;
+fail:
+ return -1;
+}
+
+/*
+ * Tx queue reset
+ */
+void qdma_reset_tx_queue(struct qdma_tx_queue *txq)
+{
+ uint32_t i;
+ uint32_t sz;
+
+ txq->tx_tail = 0;
+ txq->tx_fl_tail = 0;
+ if (txq->st_mode) { /** ST-mode **/
+ sz = sizeof(struct qdma_h2c_desc);
+ /* Zero out HW ring memory */
+ for (i = 0; i < (sz * (txq->nb_tx_desc)); i++)
+ ((volatile char *)txq->tx_ring)[i] = 0;
+ } else {
+ sz = sizeof(struct qdma_mm_desc);
+ /* Zero out HW ring memory */
+ for (i = 0; i < (sz * (txq->nb_tx_desc)); i++)
+ ((volatile char *)txq->tx_ring)[i] = 0;
+ }
+
+ /* Initialize SW ring entries */
+ for (i = 0; i < txq->nb_tx_desc; i++)
+ txq->sw_ring[i] = NULL;
+}
+
+void qdma_inv_tx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint64_t bar_addr;
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ /** To clear the SW H2C Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "SW H2C Clear cmd for queue-id:%d:"
+ " reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ /** To clear the HW H2C Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW H2C Clear cmd for queue-id:%d:"
+ " reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (mode) { /** ST-mode **/
+ /** To clear the C2H credit context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_CR_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_INV << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "H2C Credits Clear cmd for queue-id:"
+ "%d: reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ }
+}
+
+/**
+ * reset the Tx device queues.
+ *
+ * reset the device by clearing the Tx descriptors for
+ * all queues and device registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * Nothing.
+ */
+void qdma_reset_tx_queues(struct rte_eth_dev *dev, uint32_t qid, uint32_t mode)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint64_t bar_addr;
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ /** To clear the SW H2C Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "SW H2C Clear cmd for queue-id:%d:"
+ " reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ /** To clear the HW H2C Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "HW H2C Clear cmd for queue-id:%d:"
+ " reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (mode) { /** ST-mode **/
+ /** To clear the C2H credit context Queues **/
+ ctxt_sel = (QDMA_CTXT_SEL_CR_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, "H2C Credits Clear cmd for queue-id:"
+ "%d: reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ }
+}
+
+void qdma_start_tx_queue(struct qdma_tx_queue *txq)
+{
+ /* initialize tail */
+ txq->tx_tail = 0;
+ *txq->tx_pidx = 0;
+}
+
+/**
+ * Start the tx device queues.
+ *
+ * Start the device by configuring the Tx descriptors for
+ * all the queues and device registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * Nothing.
+ */
+
+int qdma_queue_ctxt_tx_prog(struct rte_eth_dev *dev, uint32_t qid,
+ uint32_t mode, uint32_t ringszidx,
+ uint16_t func_id, uint64_t phys_addr,
+ uint8_t en_bypass, uint8_t bypass_desc_sz_idx)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint32_t ctxt_sel, reg_val;
+ uint64_t bar_addr;
+ uint16_t flag, i;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ reg_val = (func_id << SW_DESC_CNTXT_FUNC_ID_SHIFT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[0], reg_val);
+
+ reg_val = (en_bypass << SW_DESC_CNTXT_BYPASS_SHIFT) |
+ (SW_DESC_CNTXT_WB_EN <<
+ SW_DESC_CNTXT_WB_EN_SHIFT_B) |
+ (ringszidx <<
+ SW_DESC_CNTXT_RING_SIZE_ID_SHIFT) |
+ (SW_DESC_CNTXT_WBI_INTERVAL <<
+ SW_DESC_CNTXT_WBI_INTERVAL_SHIFT) |
+ (SW_DESC_CNTXT_WBI_CHK <<
+ SW_DESC_CNTXT_WBI_CHK_SHIFT) |
+ (SW_DESC_CNTXT_QUEUE_ENABLE <<
+ SW_DESC_CNTXT_QUEUE_EN_SHIFT);
+
+ if (en_bypass && bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ reg_val = reg_val | (bypass_desc_sz_idx <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT);
+ else if (mode) /** ST-mode **/
+ reg_val = reg_val | (SW_DESC_CNTXT_H2C_STREAM_DMA <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT);
+ else /* MM- mode*/
+ reg_val = reg_val | (SW_DESC_CNTXT_MEMORY_MAP_DMA <<
+ SW_DESC_CNTXT_DESC_SZ_SHIFT) |
+ (SW_DESC_CNTXT_IS_MM <<
+ SW_DESC_CNTXT_IS_MM_SHIFT);
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[1], reg_val);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[2],
+ rte_cpu_to_le_32(phys_addr &
+ MASK_32BIT));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[3],
+ rte_cpu_to_le_32(phys_addr >> 32));
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[4], 0);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[2], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[3], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[4], MASK_32BIT);
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_WR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for queue-id:%d: reg_val:%x\n",
+ qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "H2C Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ goto fail;
+ }
+
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ /** To read the Queue **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ for (i = 0; i < NUM_CONTEXT_REGS; i++) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_data[i]);
+ PMD_DRV_LOG(INFO, " Read context-data on queue-id:%d,"
+ "idx:%d:reg_val:%x\n", qid, i, reg_val);
+ }
+#endif //RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ return 0;
+fail:
+ return -1;
+}
+
+/* Utility function to find index of an element in an array */
+int index_of_array(uint32_t *arr, uint32_t n, uint32_t element)
+{
+ int index = 0;
+
+ for (index = 0; (uint32_t)index < n; index++) {
+ if (*(arr + index) == element)
+ return index;
+ }
+ return -1;
+}
+
+static int qbase_check_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs queue base is: %s\n", value);
+ qdma_dev->queue_base = (unsigned int)strtoul(value, &end, 10);
+ if (qdma_dev->queue_base >= QDMA_QUEUES_NUM_MAX) {
+ PMD_DRV_LOG(INFO, "QDMA devargs queue-base > max allowed\n");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int pfetch_check_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs desc_prefetch is: %s\n", value);
+ qdma_dev->en_desc_prefetch = (uint8_t)strtoul(value, &end, 10);
+ if (qdma_dev->en_desc_prefetch > 1) {
+ PMD_DRV_LOG(INFO, "QDMA devargs prefetch should be 1 or 0,"
+ " setting to 1.\n");
+ qdma_dev->en_desc_prefetch = 1;
+ }
+ return 0;
+}
+
+static int cmpt_desc_len_check_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs cmpt_desc_len is: %s\n", value);
+ qdma_dev->cmpt_desc_len = (uint8_t)strtoul(value, &end, 10);
+ if (qdma_dev->cmpt_desc_len != CMPT_DESC_LEN_8B &&
+ qdma_dev->cmpt_desc_len != CMPT_DESC_LEN_16B &&
+ qdma_dev->cmpt_desc_len != CMPT_DESC_LEN_32B &&
+ qdma_dev->cmpt_desc_len != CMPT_DESC_LEN_64B) {
+ PMD_DRV_LOG(INFO, "QDMA devargs incorrect cmpt_desc_len = %d "
+ "specified\n",
+ qdma_dev->cmpt_desc_len);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int trigger_mode_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs trigger mode: %s\n", value);
+ qdma_dev->trigger_mode = (uint8_t)strtoul(value, &end, 10);
+
+ if (qdma_dev->trigger_mode > TRIG_MODE_MAX) {
+ qdma_dev->trigger_mode = TRIG_MODE_MAX;
+ PMD_DRV_LOG(INFO, "QDMA devargs trigger mode invalid,"
+ "reset to default: %d\n",
+ qdma_dev->trigger_mode);
+ }
+ return 0;
+}
+
+static int wb_accumulation_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs trigger mode: %s\n", value);
+ qdma_dev->wb_acc_int = (uint8_t)strtoul(value, &end, 10);
+
+ if (qdma_dev->wb_acc_int > MAX_WB_ACC_INT) {
+ qdma_dev->wb_acc_int = DEFAULT_WB_ACC_INT;
+ PMD_DRV_LOG(INFO, "QDMA devargs write-back-accumulation "
+ "invalid, reset to default: %d\n",
+ qdma_dev->wb_acc_int);
+ }
+ return 0;
+}
+
+static int config_bar_idx_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs trigger mode: %s\n", value);
+ qdma_dev->config_bar_idx = (int)strtoul(value, &end, 10);
+
+ if (qdma_dev->config_bar_idx >= QDMA_BAR_NUM ||
+ qdma_dev->config_bar_idx < 0) {
+ PMD_DRV_LOG(INFO, "QDMA devargs config bar idx invalid: %d\n",
+ qdma_dev->config_bar_idx);
+ return -1;
+ }
+ return 0;
+}
+
+static int c2h_byp_mode_check_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs c2h_byp_mode is: %s\n", value);
+ qdma_dev->c2h_bypass_mode = (uint8_t)strtoul(value, &end, 10);
+
+ if (qdma_dev->c2h_bypass_mode > C2H_BYPASS_MODE_MAX) {
+ PMD_DRV_LOG(INFO, "QDMA devargs incorrect "
+ "c2h_byp_mode= %d specified\n",
+ qdma_dev->c2h_bypass_mode);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int h2c_byp_mode_check_handler(__rte_unused const char *key,
+ const char *value, void *opaque)
+{
+ struct qdma_pci_dev *qdma_dev = (struct qdma_pci_dev *)opaque;
+ char *end = NULL;
+
+ PMD_DRV_LOG(INFO, "QDMA devargs h2c_byp_mode is: %s\n", value);
+ qdma_dev->h2c_bypass_mode = (uint8_t)strtoul(value, &end, 10);
+
+ if (qdma_dev->h2c_bypass_mode > 1) {
+ PMD_DRV_LOG(INFO, "QDMA devargs incorrect"
+ " h2c_byp_mode =%d specified\n",
+ qdma_dev->h2c_bypass_mode);
+ return -1;
+ }
+
+ return 0;
+}
+
+/* Process the all devargs */
+int qdma_check_kvargs(struct rte_devargs *devargs,
+ struct qdma_pci_dev *qdma_dev)
+{
+ struct rte_kvargs *kvlist;
+ const char *qbase_key = "queue_base";
+ const char *pfetch_key = "desc_prefetch";
+ const char *cmpt_desc_len_key = "cmpt_desc_len";
+ const char *trigger_mode_key = "trigger_mode";
+ const char *wb_acc_int_key = "wb_acc_int";
+ const char *config_bar_key = "config_bar";
+ const char *c2h_byp_mode_key = "c2h_byp_mode";
+ const char *h2c_byp_mode_key = "h2c_byp_mode";
+ int ret = 0;
+
+ if (!devargs)
+ return 0;
+
+ kvlist = rte_kvargs_parse(devargs->args, NULL);
+ if (!kvlist)
+ return 0;
+
+ /* process the queue_base*/
+ if (rte_kvargs_count(kvlist, qbase_key)) {
+ ret = rte_kvargs_process(kvlist, qbase_key,
+ qbase_check_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process the desc_prefetch*/
+ if (rte_kvargs_count(kvlist, pfetch_key)) {
+ ret = rte_kvargs_process(kvlist, pfetch_key,
+ pfetch_check_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process the cmpt_desc_len*/
+ if (rte_kvargs_count(kvlist, cmpt_desc_len_key)) {
+ ret = rte_kvargs_process(kvlist, cmpt_desc_len_key,
+ cmpt_desc_len_check_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process the trigger_mode*/
+ if (rte_kvargs_count(kvlist, trigger_mode_key)) {
+ ret = rte_kvargs_process(kvlist, trigger_mode_key,
+ trigger_mode_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process the writeback accumalation*/
+ if (rte_kvargs_count(kvlist, wb_acc_int_key)) {
+ ret = rte_kvargs_process(kvlist, wb_acc_int_key,
+ wb_accumulation_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process the config bar*/
+ if (rte_kvargs_count(kvlist, config_bar_key)) {
+ ret = rte_kvargs_process(kvlist, config_bar_key,
+ config_bar_idx_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process c2h_byp_mode*/
+ if (rte_kvargs_count(kvlist, c2h_byp_mode_key)) {
+ ret = rte_kvargs_process(kvlist, c2h_byp_mode_key,
+ c2h_byp_mode_check_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ /* process h2c_byp_mode*/
+ if (rte_kvargs_count(kvlist, h2c_byp_mode_key)) {
+ ret = rte_kvargs_process(kvlist, h2c_byp_mode_key,
+ h2c_byp_mode_check_handler, qdma_dev);
+ if (ret) {
+ rte_kvargs_free(kvlist);
+ return ret;
+ }
+ }
+
+ rte_kvargs_free(kvlist);
+ return ret;
+}
+
+int qdma_identify_bars(struct rte_eth_dev *dev)
+{
+ uint64_t baseaddr;
+ int bar_len, i, j;
+ uint32_t xlnx_id_mask = 0xffff0000;
+ uint64_t reg_val;
+ uint32_t mask = 1;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_pci_dev *dma_priv;
+
+ dma_priv = (struct qdma_pci_dev *)dev->data->dev_private;
+
+ /* Config bar */
+ bar_len = pci_dev->mem_resource[dma_priv->config_bar_idx].len;
+ if (bar_len) {
+ baseaddr = (uint64_t)
+ pci_dev->mem_resource[dma_priv->config_bar_idx].addr;
+
+ if (dma_priv->is_vf)
+ reg_val = qdma_read_reg((uint64_t)baseaddr +
+ QDMA_VF_RTL_VER);
+ else
+ reg_val = qdma_read_reg((uint64_t)baseaddr);
+
+ if ((reg_val & xlnx_id_mask) != QDMA_CONFIG_BLOCK_ID) {
+ PMD_DRV_LOG(INFO, "QDMA config BAR index :%d invalid",
+ dma_priv->config_bar_idx);
+ return -1;
+ }
+ } else {
+ PMD_DRV_LOG(INFO, "QDMA config BAR index :%d is not enabled",
+ dma_priv->config_bar_idx);
+ return -1;
+ }
+
+ /* Find user bar*/
+ if (dma_priv->is_vf) {
+ reg_val = qdma_read_reg((uint64_t)baseaddr +
+ QDMA_GLBL2_VF_BARLITE_EXT);
+ reg_val = reg_val & 0x3F;
+
+ } else {
+ reg_val = qdma_read_reg((uint64_t)baseaddr +
+ QDMA_GLBL2_PF_BARLITE_EXT);
+ reg_val = (reg_val >> (6 * PCI_FUNC(dma_priv->pf))) & 0x3F;
+ }
+
+ for (j = 0 ; j < QDMA_BAR_NUM; j++) {
+ if (reg_val & mask) {
+ if (pci_dev->mem_resource[j].len) {
+ dma_priv->user_bar_idx = j;
+ break;
+ }
+ }
+ mask <<= 1;
+ }
+
+ /* Find bypass bar*/
+ for (i = 0; i < QDMA_BAR_NUM; i++) {
+ bar_len = pci_dev->mem_resource[i].len;
+ if (!bar_len) /* Bar not enabled ? */
+ continue;
+ if (dma_priv->user_bar_idx != i &&
+ dma_priv->config_bar_idx != i) {
+ dma_priv->bypass_bar_idx = i;
+ break;
+ }
+ }
+
+ PMD_DRV_LOG(INFO, "QDMA config bar idx :%d\n",
+ dma_priv->config_bar_idx);
+ PMD_DRV_LOG(INFO, "QDMA user bar idx :%d\n", dma_priv->user_bar_idx);
+ PMD_DRV_LOG(INFO, "QDMA bypass bar idx :%d\n",
+ dma_priv->bypass_bar_idx);
+
+ return 0;
+}
+int qdma_get_hw_version(struct rte_eth_dev *dev)
+{
+ uint64_t baseaddr, reg_val;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_pci_dev *dma_priv;
+
+ dma_priv = (struct qdma_pci_dev *)dev->data->dev_private;
+ baseaddr = ((uint64_t)
+ pci_dev->mem_resource[dma_priv->config_bar_idx].addr);
+
+ if (dma_priv->is_vf) {
+ reg_val = qdma_read_reg((uint64_t)baseaddr + QDMA_VF_RTL_VER);
+ dma_priv->rtl_version = (reg_val & VF_RTL_VERSION_MASK) >>
+ VF_RTL_VERSION_SHIFT;
+ dma_priv->vivado_rel = (reg_val & VF_VIVADO_RELEASE_ID_MASK) >>
+ VF_VIVADO_RELEASE_ID_SHIFT;
+ dma_priv->everest_ip = (reg_val & VF_EVEREST_IP_MASK) >>
+ VF_EVEREST_IP_SHIFT;
+
+ } else {
+ reg_val = qdma_read_reg((uint64_t)baseaddr + QDMA_PF_RTL_VER);
+ dma_priv->rtl_version = (reg_val & PF_RTL_VERSION_MASK) >>
+ PF_RTL_VERSION_SHIFT;
+ dma_priv->vivado_rel = (reg_val & PF_VIVADO_RELEASE_ID_MASK) >>
+ PF_VIVADO_RELEASE_ID_SHIFT;
+ dma_priv->everest_ip = (reg_val & PF_EVEREST_IP_MASK) >>
+ PF_EVEREST_IP_SHIFT;
+ }
+
+ if (dma_priv->rtl_version == 0)
+ PMD_DRV_LOG(INFO, "QDMA RTL VERSION : RTL-1\n");
+ else if (dma_priv->rtl_version == 1)
+ PMD_DRV_LOG(INFO, "QDMA RTL VERSION : RTL-2\n");
+
+ if (dma_priv->vivado_rel == 0)
+ PMD_DRV_LOG(INFO, "QDMA VIVADO RELEASE ID : 2018.3\n");
+
+ if (dma_priv->everest_ip == 0)
+ PMD_DRV_LOG(INFO, "QDMA IP : SOFT-IP\n");
+ else
+ PMD_DRV_LOG(INFO, "QDMA IP : HARD-IP\n");
+ return 0;
+}
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_devops.c b/QDMA/DPDK/drivers/net/qdma/qdma_devops.c
new file mode 100644
index 0000000000000000000000000000000000000000..371fbc2f90739085f86c784a30aaa231e501f67f
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_devops.c
@@ -0,0 +1,1447 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
+#include <rte_ethdev_pci.h>
+#include <rte_malloc.h>
+#include <rte_dev.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_alarm.h>
+#include <rte_cycles.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "qdma.h"
+#include "qdma_regs.h"
+
+uint32_t g_ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+uint32_t g_c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ];
+uint32_t g_c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+uint32_t g_c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ];
+
+int get_param(struct rte_eth_dev *dev, enum get_param_type param,
+ void *ret_param)
+{
+ struct qdma_pci_dev *dma_priv = dev->data->dev_private;
+
+ if (!ret_param)
+ return -1;
+
+ switch (param) {
+ case CONFIG_BAR:
+ *((int *)ret_param) = dma_priv->config_bar_idx;
+ break;
+ case USER_BAR:
+ *((int *)ret_param) = dma_priv->user_bar_idx;
+ break;
+ case BYPASS_BAR:
+ *((int *)ret_param) = dma_priv->bypass_bar_idx;
+ break;
+ }
+ return 0;
+}
+
+
+int get_queue_param(struct rte_eth_dev *dev, uint32_t qid,
+ enum get_queue_param_type param, void *ret_param)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+
+ if (qid >= qdma_dev->qsets_en) {
+ PMD_DRV_LOG(ERR, "Invalid Q-id passed qid %d max en_qid %d\n",
+ qid, qdma_dev->qsets_en);
+ return -1;
+ }
+
+ if (!ret_param) {
+ PMD_DRV_LOG(ERR, "Invalid ret_param for qid %d\n", qid);
+ return -1;
+ }
+ switch (param) {
+ /* Use this param after queue setup is done, and before queue start*/
+ case CHECK_DUMP_IMMEDIATE_DATA:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "DUMP_IMMEDIATE_DATA\n");
+ return -1;
+ }
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ *((int *)ret_param) = rxq->dump_immediate_data;
+ break;
+ }
+ return 0;
+}
+
+int update_param(struct rte_eth_dev *dev, enum dev_param_type param,
+ uint16_t value)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+ switch (param) {
+ case QUEUE_BASE:
+ if (value > qdma_dev->qsets_max) {
+ PMD_DRV_LOG(ERR, "Invalid Queue base passed\n");
+ return -1;
+ }
+ qdma_dev->queue_base = value;
+ break;
+ case TIMER_COUNT:
+ qdma_dev->timer_count = value;
+ break;
+ case TRIGGER_MODE:
+ if (value > TRIG_MODE_MAX) {
+ PMD_DRV_LOG(ERR, "Invalid Trigger mode passed\n");
+ return -1;
+ }
+ qdma_dev->trigger_mode = value;
+ break;
+ default:
+ PMD_DRV_LOG(ERR, "Invalid param %x specified\n", param);
+ break;
+ }
+ return 0;
+}
+
+int update_queue_param(struct rte_eth_dev *dev, uint32_t qid,
+ enum queue_param_type param, uint32_t value)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ struct qdma_tx_queue *txq;
+
+ if (qid >= qdma_dev->qsets_en) {
+ PMD_DRV_LOG(ERR, "Invalid Queue id passed\n");
+ return -1;
+ }
+
+ switch (param) {
+ case QUEUE_MODE:
+ if (value > STREAMING_MODE) {
+ PMD_DRV_LOG(ERR, "Invalid Queue mode passed\n");
+ return -1;
+ }
+ qdma_dev->q_info[qid].queue_mode = value;
+ break;
+
+ case DIS_OVERFLOW_CHECK: /* Update this param after queue-setup issued*/
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "DIS_OVERFLOW_CHECK\n");
+ return -1;
+ }
+ if (value > 1)
+ return -1;
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ rxq->dis_overflow_check = value;
+ break;
+
+ case DUMP_IMMEDIATE_DATA: /*Update this param after queue-setup issued*/
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "DUMP_IMMEDIATE_DATA\n");
+ return -1;
+ }
+ if (value > 1)
+ return -1;
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ rxq->dump_immediate_data = value;
+ break;
+
+ case DESC_PREFETCH:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "DESC_PREFETCH\n");
+ return -1;
+ }
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ rxq->en_prefetch = (value > 0) ? 1 : 0;
+ break;
+
+ case RX_BYPASS_MODE:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "RX_BYPASS_MODE\n");
+ return -1;
+ }
+ if (value > C2H_BYPASS_MODE_MAX) {
+ PMD_DRV_LOG(ERR, "Invalid Rx Bypass mode : %d\n",
+ value);
+ return -1;
+ }
+ qdma_dev->q_info[qid].rx_bypass_mode = value;
+ break;
+
+ case TX_BYPASS_MODE:
+ if (qid >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid TX Queue id passed for "
+ "TX_BYPASS_MODE\n");
+ return -1;
+ }
+ if (value > 1) {
+ PMD_DRV_LOG(ERR, "Invalid Tx Bypass mode : %d\n",
+ value);
+ return -1;
+ }
+ qdma_dev->q_info[qid].tx_bypass_mode = value;
+ break;
+
+ case RX_BYPASS_DESC_SIZE:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed for "
+ "RX_BYPASS_DESC_SIZE\n");
+ return -1;
+ }
+ /* Only 64byte descriptor size supported in 2018.3 Example Design*/
+ if (value != 64)
+ return -1;
+ qdma_dev->q_info[qid].rx_bypass_desc_sz =
+ (enum bypass_desc_len)value;
+ break;
+
+ case TX_BYPASS_DESC_SIZE:
+ if (qid >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid TX Queue id passed "
+ "for TX_BYPASS_DESC_SIZE\n");
+ return -1;
+ }
+ /* Only 64byte descriptor size supported in 2018.3 Example Design*/
+ if (value != 64)
+ return -1;
+ qdma_dev->q_info[qid].tx_bypass_desc_sz =
+ (enum bypass_desc_len)value;
+ break;
+
+ case TX_DST_ADDRESS:
+ if (qid >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid TX Queue id passed for "
+ "TX_DST_ADDRESS\n");
+ return -1;
+ }
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+ txq->ep_addr = value;
+ break;
+
+ case RX_SRC_ADDRESS:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed "
+ "for RX_SRC_ADDRESS\n");
+ return -1;
+ }
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ rxq->ep_addr = value;
+ break;
+
+ case CMPT_DESC_SIZE:
+ if (qid >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX Queue id passed "
+ "for CMPT_DESC_SIZE\n");
+ return -1;
+ }
+ if (value != CMPT_DESC_LEN_8B &&
+ value != CMPT_DESC_LEN_16B &&
+ value != CMPT_DESC_LEN_32B &&
+ value != CMPT_DESC_LEN_64B)
+ return -1;
+ qdma_dev->q_info[qid].cmpt_desc_sz = value;
+ break;
+
+ default:
+ PMD_DRV_LOG(ERR, "Invalid param %x specified\n", param);
+ break;
+ }
+
+ return 0;
+}
+
+int qdma_set_fmap(struct qdma_pci_dev *qdma_dev, uint16_t devfn,
+ uint32_t q_base, uint32_t q_count)
+{
+ uint64_t bar_addr;
+#if FMAP_CNTXT
+ struct queue_ind_prg *q_regs;
+ uint32_t ctxt_sel;
+ uint16_t flag, i;
+#else
+ uint64_t fmap;
+#endif
+#if defined(RTE_LIBRTE_QDMA_DEBUG_DRIVER) || FMAP_CNTXT
+ uint64_t reg_val;
+#endif
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+#if FMAP_CNTXT
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[0], q_base);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_data[1], q_count);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], MASK_32BIT);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], MASK_32BIT);
+ ctxt_sel = (QDMA_CTXT_SEL_FMAP << CTXT_SEL_SHIFT_B) |
+ (devfn << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_WR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for FMAP CTXT for device:%d: "
+ "reg_val:%lx\n", devfn, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on device:%d: FMAP "
+ "initailization with cmd reg_val:%lx\n",
+ devfn, reg_val);
+ goto fail;
+ }
+#else
+ fmap = (uint64_t)(bar_addr + QDMA_TRQ_SEL_FMAP + devfn * 4);
+ qdma_write_reg((uint64_t)fmap, (q_count << QID_MAX_SHIFT_B) |
+ (q_base & QID_BASE_MSK));
+#endif
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+#if FMAP_CNTXT
+ ctxt_sel = (QDMA_CTXT_SEL_FMAP << CTXT_SEL_SHIFT_B) |
+ (devfn << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+ for (i = 0; i < 2; i++) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_data[i]);
+ PMD_DRV_LOG(INFO, " FMAP CTXT for device :%d, idx:%d:"
+ "reg_val:%lx\n", devfn, i, reg_val);
+ /* TO-DO: Need to verify the Queue ctxt-data and
+ * if Queue has any errors, then return -1. **/
+ }
+#else
+ reg_val = qdma_read_reg((uint64_t)fmap);
+ PMD_DRV_LOG(INFO, "Fmap for function id %d reg_val:0x%lx,"
+ " qid_max:%lu\n", devfn, reg_val,
+ ((reg_val & QID_MAX_MSK) >> QID_MAX_SHIFT_B));
+#endif
+#endif
+ return 0;
+
+#if FMAP_CNTXT
+fail:
+ return -1;
+#endif
+}
+uint8_t qmda_get_desc_sz_idx(enum bypass_desc_len size)
+{
+ uint8_t ret;
+ switch (size) {
+ case BYPASS_DESC_LEN_8B:
+ ret = 0;
+ break;
+ case BYPASS_DESC_LEN_16B:
+ ret = 1;
+ break;
+ case BYPASS_DESC_LEN_32B:
+ ret = 2;
+ break;
+ case BYPASS_DESC_LEN_64B:
+ ret = 3;
+ break;
+ default:
+ /* Suppress compiler warnings*/
+ ret = 0;
+ }
+ return ret;
+}
+
+/**
+ * DPDK callback to configure a RX queue.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param rx_queue_id
+ * RX queue index.
+ * @param nb_rx_desc
+ * Number of descriptors to configure in queue.
+ * @param socket_id
+ * NUMA socket on which memory must be allocated.
+ * @param[in] rx_conf
+ * Thresholds parameters.
+ * @param mp_pool
+ * Memory pool for buffer allocations.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+ uint16_t nb_rx_desc, unsigned int socket_id,
+ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mb_pool)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ struct qdma_mm_desc *rx_ring_mm;
+ uint64_t dma_bar;
+ uint32_t sz;
+ uint8_t *rx_ring_bypass;
+ int bypass_desc_sz_idx;
+
+ PMD_DRV_LOG(INFO, "Configuring Rx queue id:%d\n", rx_queue_id);
+
+ if (nb_rx_desc == 0) {
+ PMD_DRV_LOG(ERR, "Invalid descriptor ring size %d\n",
+ nb_rx_desc);
+ return -EINVAL;
+ }
+
+ /* allocate rx queue data structure */
+ rxq = rte_zmalloc("QDMA_RxQ", sizeof(struct qdma_rx_queue),
+ RTE_CACHE_LINE_SIZE);
+ if (!rxq) {
+ PMD_DRV_LOG(ERR, "Unable to allocate structure rxq of "
+ "size %d\n",
+ (int)(sizeof(struct qdma_rx_queue)));
+ return (-ENOMEM);
+ }
+
+ dma_bar = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ rxq->st_mode = qdma_dev->q_info[rx_queue_id].queue_mode;
+ rxq->nb_rx_desc = nb_rx_desc;
+ rxq->bypass_desc_sz = qdma_dev->q_info[rx_queue_id].rx_bypass_desc_sz;
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+
+ if (qdma_dev->q_info[rx_queue_id].rx_bypass_mode ==
+ C2H_BYPASS_MODE_CACHE ||
+ qdma_dev->q_info[rx_queue_id].rx_bypass_mode ==
+ C2H_BYPASS_MODE_SIMPLE)
+ rxq->en_bypass = 1;
+ if (qdma_dev->q_info[rx_queue_id].rx_bypass_mode ==
+ C2H_BYPASS_MODE_SIMPLE)
+ rxq->en_bypass_prefetch = 1;
+
+ /* <= 2018.2 IP
+ * double the cmpl ring size to avoid run out of cmpl entry while
+ * desc. ring still have free entries
+ */
+ rxq->nb_rx_cmpt_desc = (nb_rx_desc * 2);
+ rxq->en_prefetch = qdma_dev->en_desc_prefetch;
+ rxq->cmpt_desc_len = qdma_dev->q_info[rx_queue_id].cmpt_desc_sz;
+
+ /* Disable the cmpt over flow check by default
+ * Applcation can test enable/disable via update param before
+ * queue_start is issued
+ */
+ rxq->dis_overflow_check = 0;
+
+ /* allocate memory for Rx descriptor ring */
+ if (rxq->st_mode) {
+ if (!qdma_dev->st_mode_en) {
+ PMD_DRV_LOG(ERR, "Streaming mode not enabled "
+ "in the hardware\n");
+ rte_free(rxq);
+ return -EINVAL;
+ }
+
+ if (rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ sz = (rxq->nb_rx_desc) * (rxq->bypass_desc_sz);
+ else
+ sz = (rxq->nb_rx_desc) * sizeof(struct qdma_c2h_desc);
+
+ rxq->rx_mz = qdma_zone_reserve(dev, "RxHwRn", rx_queue_id,
+ sz, socket_id);
+ if (!rxq->rx_mz) {
+ PMD_DRV_LOG(ERR, "Unable to allocate rxq->rx_mz "
+ "of size %d\n", sz);
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+ rxq->rx_ring = rxq->rx_mz->addr;
+ } else {
+ if (!qdma_dev->mm_mode_en) {
+ PMD_DRV_LOG(ERR, "Memory mapped mode not enabled "
+ "in the hardware\n");
+ rte_free(rxq);
+ return -EINVAL;
+ }
+ if (rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ sz = (rxq->nb_rx_desc) * (rxq->bypass_desc_sz);
+ else
+ sz = (rxq->nb_rx_desc) * sizeof(struct qdma_mm_desc);
+ rxq->rx_mz = qdma_zone_reserve(dev, "RxHwRn",
+ rx_queue_id, sz, socket_id);
+ if (!rxq->rx_mz) {
+ PMD_DRV_LOG(ERR, "Unable to allocate rxq->rx_mz "
+ "of size %d\n", sz);
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+ rxq->rx_ring = rxq->rx_mz->addr;
+ rx_ring_mm = (struct qdma_mm_desc *)rxq->rx_mz->addr;
+ rx_ring_bypass = (uint8_t *)rxq->rx_mz->addr;
+ if (rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ rxq->wb_status = (struct wb_status *)&
+ (rx_ring_bypass[(rxq->nb_rx_desc - 1) *
+ (rxq->bypass_desc_sz)]);
+ else
+ rxq->wb_status = (struct wb_status *)&
+ (rx_ring_mm[rxq->nb_rx_desc - 1]);
+ }
+
+ if (rxq->st_mode) {
+ /* allocate memory for Rx completion(cpmt) descriptor ring */
+ sz = (rxq->nb_rx_cmpt_desc) * rxq->cmpt_desc_len;
+ rxq->rx_cmpt_mz = qdma_zone_reserve(dev, "RxHwCmptRn",
+ rx_queue_id, sz, socket_id);
+ if (!rxq->rx_cmpt_mz) {
+ PMD_DRV_LOG(ERR, "Unable to allocate rxq->rx_cmpt_mz "
+ "of size %d\n", sz);
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+ rxq->cmpt_ring = (struct c2h_cmpt_ring *)rxq->rx_cmpt_mz->addr;
+
+ /* Write-back status structure */
+ rxq->wb_status = (struct wb_status *)((uint64_t)rxq->cmpt_ring +
+ (((uint64_t)rxq->nb_rx_cmpt_desc - 1) *
+ rxq->cmpt_desc_len));
+
+ if (!qdma_dev->is_vf) {
+ rxq->rx_cidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_PF +
+ QDMA_SEL_CMPT_CIDX_Q_OFF +
+ (rx_queue_id * QDMA_CIDX_STEP));
+ } else {
+ rxq->rx_cidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_VF +
+ QDMA_SEL_CMPT_CIDX_Q_OFF +
+ (rx_queue_id * QDMA_CIDX_STEP));
+ }
+ }
+
+ /* allocate memory for RX software ring */
+ sz = rxq->nb_rx_desc * sizeof(struct rte_mbuf *);
+ rxq->sw_ring = rte_zmalloc("RxSwRn", sz, RTE_CACHE_LINE_SIZE);
+ if (!rxq->sw_ring) {
+ PMD_DRV_LOG(ERR, "Unable to allocate rxq->sw_ring of size %d\n",
+ sz);
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+
+ rxq->queue_id = rx_queue_id;
+
+ rxq->port_id = dev->data->port_id;
+ rxq->func_id = qdma_dev->pf;
+ rxq->mb_pool = mb_pool;
+
+ /* Calculate the ring index, completion queue ring size,
+ * buffer index and threshold index.
+ * If index is not found , by default use the index as 0
+ */
+
+ /* Find C2H queue ring size index */
+ rxq->ringszidx = index_of_array(g_ring_sz, QDMA_GLOBAL_CSR_ARRAY_SZ,
+ rxq->nb_rx_desc);
+ if (rxq->ringszidx < 0) {
+ PMD_DRV_LOG(ERR, "Expected Ring size %d not found\n",
+ rxq->nb_rx_desc);
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ return -EINVAL;
+ }
+
+ /* Find completion ring size index */
+ rxq->cmpt_ringszidx = index_of_array(g_ring_sz,
+ QDMA_GLOBAL_CSR_ARRAY_SZ,
+ rxq->nb_rx_cmpt_desc);
+ if (rxq->cmpt_ringszidx < 0) {
+ PMD_DRV_LOG(ERR, "Expected completion ring size %d not found\n",
+ rxq->nb_rx_cmpt_desc);
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ return -EINVAL;
+ }
+
+ /* Find Threshold index */
+ rxq->threshidx = index_of_array(g_c2h_cnt_th, QDMA_GLOBAL_CSR_ARRAY_SZ,
+ rx_conf->rx_thresh.wthresh);
+ if (rxq->threshidx < 0) {
+ PMD_DRV_LOG(WARNING, "Expected Threshold %d not found,"
+ " using the value %d at index 0\n",
+ rx_conf->rx_thresh.wthresh, g_c2h_cnt_th[0]);
+ rxq->threshidx = 0;
+ }
+
+ /* Find Timer index */
+ rxq->timeridx = index_of_array(g_c2h_timer_cnt,
+ QDMA_GLOBAL_CSR_ARRAY_SZ,
+ qdma_dev->timer_count);
+ if (rxq->timeridx < 0) {
+ PMD_DRV_LOG(WARNING, "Expected timer %d not found, "
+ "using the value %d at index 1\n",
+ qdma_dev->timer_count, g_c2h_timer_cnt[1]);
+ rxq->timeridx = 1;
+ }
+
+ /* Find Buffer size index */
+ rxq->rx_buff_size = (uint16_t)
+ (rte_pktmbuf_data_room_size(rxq->mb_pool) -
+ RTE_PKTMBUF_HEADROOM);
+ rxq->buffszidx = index_of_array(g_c2h_buf_sz, QDMA_GLOBAL_CSR_ARRAY_SZ,
+ rxq->rx_buff_size);
+ if (rxq->buffszidx < 0) {
+ PMD_DRV_LOG(ERR, "Expected buffer size %d not found\n",
+ rxq->rx_buff_size);
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ return -EINVAL;
+ }
+
+ rxq->triggermode = qdma_dev->trigger_mode;
+ rxq->rx_deferred_start = rx_conf->rx_deferred_start;
+
+ if (!qdma_dev->is_vf) {
+ rxq->rx_pidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_PF +
+ QDMA_C2H_PIDX_Q_OFF +
+ (rx_queue_id *
+ QDMA_PIDX_STEP));
+ } else {
+ rxq->rx_pidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_VF +
+ QDMA_C2H_PIDX_Q_OFF +
+ (rx_queue_id *
+ QDMA_PIDX_STEP));
+ }
+
+ /* store rx_pkt_burst function pointer */
+ dev->rx_pkt_burst = qdma_recv_pkts;
+
+ dev->data->rx_queues[rx_queue_id] = rxq;
+
+ return 0;
+}
+
+/**
+ * DPDK callback to configure a TX queue.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param tx_queue_id
+ * TX queue index.
+ * @param nb_tx_desc
+ * Number of descriptors to configure in queue.
+ * @param socket_id
+ * NUMA socket on which memory must be allocated.
+ * @param[in] tx_conf
+ * Thresholds parameters.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+ uint16_t nb_tx_desc, unsigned int socket_id,
+ const struct rte_eth_txconf *tx_conf)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_mm_desc *tx_ring_mm;
+ struct qdma_h2c_desc *tx_ring_st;
+ uint8_t *tx_ring_bypass;
+ uint64_t dma_bar;
+ uint32_t sz;
+ int bypass_desc_sz_idx;
+
+ PMD_DRV_LOG(INFO, "Configuring Tx queue id:%d\n", tx_queue_id);
+
+ /* allocate rx queue data structure */
+ txq = rte_zmalloc("QDMA_TxQ", sizeof(struct qdma_tx_queue),
+ RTE_CACHE_LINE_SIZE);
+ if (!txq) {
+ PMD_DRV_LOG(ERR, "Memory allocation failed for "
+ "Tx queue SW structure\n");
+ return (-ENOMEM);
+ }
+
+ dma_bar = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ txq->st_mode = qdma_dev->q_info[tx_queue_id].queue_mode;
+ txq->en_bypass = (qdma_dev->q_info[tx_queue_id].tx_bypass_mode) ? 1 : 0;
+ txq->bypass_desc_sz = qdma_dev->q_info[tx_queue_id].tx_bypass_desc_sz;
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+
+ /* allocate memory for TX descriptor ring */
+ if (txq->st_mode) {
+ if (!qdma_dev->st_mode_en) {
+ PMD_DRV_LOG(ERR, "Streaming mode not enabled "
+ "in the hardware\n");
+ rte_free(txq);
+ return -EINVAL;
+ }
+ if (txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ sz = (nb_tx_desc) * (txq->bypass_desc_sz);
+ else
+ sz = (nb_tx_desc) * sizeof(struct qdma_h2c_desc);
+ txq->tx_mz = qdma_zone_reserve(dev, "TxHwRn", tx_queue_id, sz,
+ socket_id);
+ if (!txq->tx_mz) {
+ PMD_DRV_LOG(ERR, "Couldn't reserve memory for "
+ "ST H2C ring of size %d\n", sz);
+ rte_free(txq);
+ return -ENOMEM;
+ }
+
+ txq->tx_ring = txq->tx_mz->addr;
+ tx_ring_st = (struct qdma_h2c_desc *)txq->tx_ring;
+ tx_ring_bypass = (uint8_t *)txq->tx_ring;
+ /* Write-back status structure */
+ if (txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ txq->wb_status = (struct wb_status *)&
+ tx_ring_bypass[(nb_tx_desc - 1) *
+ (txq->bypass_desc_sz)];
+ else
+ txq->wb_status = (struct wb_status *)&
+ tx_ring_st[nb_tx_desc - 1];
+ } else {
+ if (!qdma_dev->mm_mode_en) {
+ PMD_DRV_LOG(ERR, "Memory mapped mode not "
+ "enabled in the hardware\n");
+ rte_free(txq);
+ return -EINVAL;
+ }
+
+ if (txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ sz = (nb_tx_desc) * (txq->bypass_desc_sz);
+ else
+ sz = (nb_tx_desc) * sizeof(struct qdma_mm_desc);
+ txq->tx_mz = qdma_zone_reserve(dev, "TxHwRn", tx_queue_id,
+ sz, socket_id);
+ if (!txq->tx_mz) {
+ PMD_DRV_LOG(ERR, "Couldn't reserve memory for "
+ "MM H2C ring of size %d\n", sz);
+ rte_free(txq);
+ return -ENOMEM;
+ }
+
+ txq->tx_ring = txq->tx_mz->addr;
+ tx_ring_mm = (struct qdma_mm_desc *)txq->tx_ring;
+ tx_ring_bypass = (uint8_t *)txq->tx_ring;
+
+ /* Write-back status structure */
+ if (txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)
+ txq->wb_status = (struct wb_status *)&
+ tx_ring_bypass[(nb_tx_desc - 1) *
+ (txq->bypass_desc_sz)];
+ else
+ txq->wb_status = (struct wb_status *)&
+ tx_ring_mm[nb_tx_desc - 1];
+ }
+
+ PMD_DRV_LOG(INFO, "Tx ring phys addr: 0x%lX, Tx Ring virt addr: 0x%lX",
+ (uint64_t)txq->tx_mz->phys_addr, (uint64_t)txq->tx_ring);
+
+ /* allocate memory for TX software ring */
+ sz = nb_tx_desc * sizeof(struct rte_mbuf *);
+ txq->sw_ring = rte_zmalloc("TxSwRn", sz, RTE_CACHE_LINE_SIZE);
+ if (!txq->sw_ring) {
+ PMD_DRV_LOG(ERR, "Memory allocation failed for Tx queue SW ring\n");
+ rte_memzone_free(txq->tx_mz);
+ rte_free(txq);
+ return -ENOMEM;
+ }
+
+ txq->nb_tx_desc = nb_tx_desc;
+ txq->queue_id = tx_queue_id;
+
+ txq->port_id = dev->data->port_id;
+ txq->func_id = qdma_dev->pf;
+ txq->num_queues = dev->data->nb_tx_queues;
+
+ txq->ringszidx = index_of_array(g_ring_sz, QDMA_GLOBAL_CSR_ARRAY_SZ,
+ txq->nb_tx_desc);
+
+ if (txq->ringszidx < 0) {
+ PMD_DRV_LOG(ERR, "Expected Ring size %d not found\n",
+ txq->nb_tx_desc);
+ rte_memzone_free(txq->tx_mz);
+ rte_free(txq->sw_ring);
+ rte_free(txq);
+ return -EINVAL;
+ }
+
+ txq->tx_deferred_start = tx_conf->tx_deferred_start;
+
+ if (!qdma_dev->is_vf) {
+ txq->tx_pidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_PF +
+ QDMA_H2C_PIDX_Q_OFF +
+ (tx_queue_id * 0x10));
+ } else {
+ txq->tx_pidx = (volatile uint32_t *)(dma_bar +
+ QDMA_TRQ_SEL_QUEUE_VF +
+ QDMA_H2C_PIDX_Q_OFF +
+ (tx_queue_id * 0x10));
+ }
+ rte_spinlock_init(&txq->pidx_update_lock);
+ /* store tx_pkt_burst function pointer */
+ dev->tx_pkt_burst = qdma_xmit_pkts;
+
+ dev->data->tx_queues[tx_queue_id] = txq;
+
+ return 0;
+}
+
+#if (MIN_TX_PIDX_UPDATE_THRESHOLD > 1)
+void qdma_txq_pidx_update(void *arg)
+{
+ struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+ struct qdma_tx_queue *txq;
+ uint32_t qid;
+
+ for (qid = 0; qid < dev->data->nb_tx_queues; qid++) {
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+ if (txq->tx_desc_pend) {
+ rte_spinlock_lock(&txq->pidx_update_lock);
+ if (txq->tx_desc_pend) {
+ *txq->tx_pidx = txq->tx_tail;
+ txq->tx_desc_pend = 0;
+ }
+ rte_spinlock_unlock(&txq->pidx_update_lock);
+ }
+ }
+ rte_eal_alarm_set(QDMA_TXQ_PIDX_UPDATE_INTERVAL,
+ qdma_txq_pidx_update, (void *)arg);
+}
+#endif
+
+
+
+void qdma_dev_tx_queue_release(void *tqueue)
+{
+ struct qdma_tx_queue *txq = (struct qdma_tx_queue *)tqueue;
+
+ if (txq) {
+ PMD_DRV_LOG(INFO, "Remove H2C queue: %d", txq->queue_id);
+
+ if (txq->sw_ring)
+ rte_free(txq->sw_ring);
+ if (txq->tx_mz)
+ rte_memzone_free(txq->tx_mz);
+ rte_free(txq);
+ PMD_DRV_LOG(INFO, "H2C queue %d removed", txq->queue_id);
+ }
+}
+
+void qdma_dev_rx_queue_release(void *rqueue)
+{
+ struct qdma_rx_queue *rxq = (struct qdma_rx_queue *)rqueue;
+
+ if (rxq) {
+ PMD_DRV_LOG(INFO, "Remove C2H queue: %d", rxq->queue_id);
+
+ if (rxq->sw_ring)
+ rte_free(rxq->sw_ring);
+ if (rxq->st_mode) { /** if ST-mode **/
+ if (rxq->rx_cmpt_mz)
+ rte_memzone_free(rxq->rx_cmpt_mz);
+ }
+ if (rxq->rx_mz)
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq);
+ PMD_DRV_LOG(INFO, "C2H queue %d removed", rxq->queue_id);
+ }
+}
+
+/**
+ * DPDK callback to start the device.
+ *
+ * Start the device by configuring the Rx/Tx descriptor and device registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int qdma_dev_start(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_rx_queue *rxq;
+ uint32_t qid;
+ int err;
+
+ PMD_DRV_LOG(INFO, "qdma-dev-start: Starting\n");
+
+ /* prepare descriptor rings for operation */
+ for (qid = 0; qid < dev->data->nb_tx_queues; qid++) {
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+
+ /*Deferred Queues should not start with dev_start*/
+ if (!txq->tx_deferred_start) {
+ err = qdma_dev_tx_queue_start(dev, qid);
+ if (err != 0)
+ return err;
+ }
+ }
+
+ for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+
+ /*Deferred Queues should not start with dev_start*/
+ if (!rxq->rx_deferred_start) {
+ err = qdma_dev_rx_queue_start(dev, qid);
+ if (err != 0)
+ return err;
+ }
+ }
+
+ /* Start Tx h2c engine */
+ *qdma_dev->h2c_mm_control = QDMA_MM_CTRL_START;
+
+ /* Start Rx c2h engine */
+ *qdma_dev->c2h_mm_control = QDMA_MM_CTRL_START;
+#if (MIN_TX_PIDX_UPDATE_THRESHOLD > 1)
+ rte_eal_alarm_set(QDMA_TXQ_PIDX_UPDATE_INTERVAL,
+ qdma_txq_pidx_update, (void *)dev);
+#endif
+ return 0;
+}
+
+/**
+ * DPDK callback to retrieve the physical link information.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param wait_to_complete
+ * wait_to_complete field is ignored.
+ */
+static int qdma_dev_link_update(struct rte_eth_dev *dev,
+ __rte_unused int wait_to_complete)
+{
+ dev->data->dev_link.link_status = ETH_LINK_UP;
+ dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
+ dev->data->dev_link.link_speed = ETH_SPEED_NUM_100G;
+ PMD_DRV_LOG(INFO, "Link update done\n");
+ return 0;
+}
+
+
+/**
+ * DPDK callback to get information about the device.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param[out] dev_info
+ * Device information structure output buffer.
+ */
+static void qdma_dev_infos_get(__rte_unused struct rte_eth_dev *dev,
+ struct rte_eth_dev_info *dev_info)
+{
+ //struct qdma_pci_dev *xdev = dev->data->dev_private;
+
+// xdev->magic = MAGIC_DEVICE;
+ dev_info->max_rx_queues = QDMA_QUEUES_NUM_MAX;
+ dev_info->max_tx_queues = QDMA_QUEUES_NUM_MAX;
+ dev_info->min_rx_bufsize = QDMA_MIN_RXBUFF_SIZE;
+ dev_info->max_rx_pktlen = DMA_BRAM_SIZE;
+ dev_info->max_mac_addrs = 1;
+}
+/**
+ * DPDK callback to stop the device.
+ *
+ * Stop the device by clearing all configured Rx/Tx queue
+ * descriptors and registers.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ */
+static void qdma_dev_stop(struct rte_eth_dev *dev)
+{
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+#endif
+ uint32_t qid;
+
+ /* reset driver's internal queue structures to default values */
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) Stop H2C & C2H queues", qdma_dev->pf);
+ for (qid = 0; qid < dev->data->nb_tx_queues; qid++)
+ qdma_dev_tx_queue_stop(dev, qid);
+ for (qid = 0; qid < dev->data->nb_rx_queues; qid++)
+ qdma_dev_rx_queue_stop(dev, qid);
+}
+
+/**
+ * DPDK callback to close the device.
+ *
+ * Destroy all queues and objects, free memory.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ */
+static void qdma_dev_close(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_rx_queue *rxq;
+ uint64_t bar_addr;
+#if FMAP_CNTXT
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+ struct queue_ind_prg *q_regs;
+#else
+ uint64_t reg_addr;
+#endif
+ uint32_t qid;
+
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) DEV Close\n", qdma_dev->pf);
+
+#if (MIN_TX_PIDX_UPDATE_THRESHOLD > 1)
+ /* Cancel pending PIDX updates */
+ rte_eal_alarm_cancel(qdma_txq_pidx_update, (void *)dev);
+#endif
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+#if FMAP_CNTXT
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+ ctxt_sel = (QDMA_CTXT_SEL_FMAP << CTXT_SEL_SHIFT_B) | (qdma_dev->pf <<
+ QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for device:%d: reg_val:%x\n",
+ qdma_dev->pf, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+#else
+ reg_addr = (uint64_t)(bar_addr + QDMA_TRQ_SEL_FMAP + qdma_dev->pf * 4);
+ qdma_write_reg((uint64_t)reg_addr, 0x0);
+#endif
+ /* iterate over rx queues */
+ for (qid = 0; qid < dev->data->nb_rx_queues; ++qid) {
+ rxq = dev->data->rx_queues[qid];
+ if (rxq) {
+ PMD_DRV_LOG(INFO, "Remove C2H queue: %d", qid);
+
+ if (rxq->sw_ring)
+ rte_free(rxq->sw_ring);
+ if (rxq->st_mode) { /** if ST-mode **/
+ if (rxq->rx_cmpt_mz)
+ rte_memzone_free(rxq->rx_cmpt_mz);
+ }
+ if (rxq->rx_mz)
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq);
+ PMD_DRV_LOG(INFO, "C2H queue %d removed", qid);
+ }
+ }
+ /* iterate over tx queues */
+ for (qid = 0; qid < dev->data->nb_tx_queues; ++qid) {
+ txq = dev->data->tx_queues[qid];
+ if (txq) {
+ PMD_DRV_LOG(INFO, "Remove H2C queue: %d", qid);
+
+ if (txq->sw_ring)
+ rte_free(txq->sw_ring);
+ if (txq->tx_mz)
+ rte_memzone_free(txq->tx_mz);
+ rte_free(txq);
+ PMD_DRV_LOG(INFO, "H2C queue %d removed", qid);
+ }
+ }
+}
+
+
+/**
+ * DPDK callback for Ethernet device configuration.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int qdma_dev_configure(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint16_t qid = 0;
+ int ret = 0;
+
+ PMD_DRV_LOG(INFO, "Configure the qdma engines\n");
+
+ qdma_dev->qsets_en = RTE_MAX(dev->data->nb_rx_queues,
+ dev->data->nb_tx_queues);
+ if ((qdma_dev->queue_base + qdma_dev->qsets_en) > qdma_dev->qsets_max) {
+ PMD_DRV_LOG(ERR, "PF-%d(DEVFN) Error: Number of Queues to be"
+ " configured are greater than the queues"
+ " supported by the hardware\n", qdma_dev->pf);
+ qdma_dev->qsets_en = 0;
+ return -1;
+ }
+
+ qdma_dev->q_info = rte_zmalloc("qinfo", sizeof(struct queue_info) *
+ (qdma_dev->qsets_en), 0);
+ if (!qdma_dev->q_info) {
+ PMD_DRV_LOG(ERR, "PF-%d(DEVFN) Cannot allocate "
+ "memory for queue info\n", qdma_dev->pf);
+ return (-ENOMEM);
+ }
+
+ /* Initialize queue_modes to all 1's ( i.e. Streaming) */
+ for (qid = 0 ; qid < qdma_dev->qsets_en; qid++)
+ qdma_dev->q_info[qid].queue_mode = STREAMING_MODE;
+
+ for (qid = 0 ; qid < dev->data->nb_rx_queues; qid++) {
+ qdma_dev->q_info[qid].cmpt_desc_sz = qdma_dev->cmpt_desc_len;
+ qdma_dev->q_info[qid].rx_bypass_mode =
+ qdma_dev->c2h_bypass_mode;
+ }
+
+ for (qid = 0 ; qid < dev->data->nb_tx_queues; qid++)
+ qdma_dev->q_info[qid].tx_bypass_mode =
+ qdma_dev->h2c_bypass_mode;
+
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) qdma-dev-configure:\n", qdma_dev->pf);
+
+ /** FMAP configuration **/
+ ret = qdma_set_fmap(qdma_dev, qdma_dev->pf, qdma_dev->queue_base,
+ qdma_dev->qsets_en);
+ if (ret < 0) {
+ rte_free(qdma_dev->q_info);
+ return -1;
+ }
+
+ /* Start Tx h2c engine */
+ *qdma_dev->h2c_mm_control = QDMA_MM_CTRL_START;
+ /* Start Rx c2h engine */
+ *qdma_dev->c2h_mm_control = QDMA_MM_CTRL_START;
+
+ return 0;
+}
+
+int qdma_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ uint64_t phys_addr;
+ uint32_t queue_base = qdma_dev->queue_base;
+ int err, bypass_desc_sz_idx;
+
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+
+ qdma_reset_tx_queue(txq);
+ qdma_reset_tx_queues(dev, (qid + queue_base), txq->st_mode);
+
+ rte_delay_ms(PF_START_DELAY);
+
+ phys_addr = (uint64_t)txq->tx_mz->phys_addr;
+
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) Phys-addr for H2C queue-id:%d: "
+ "is Lo:0x%lx, Hi:0x%lx\n", qdma_dev->pf, qid,
+ rte_cpu_to_le_32(phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(phys_addr >> 32));
+
+ err = qdma_queue_ctxt_tx_prog(dev, (qid + queue_base), txq->st_mode,
+ txq->ringszidx, txq->func_id, phys_addr,
+ txq->en_bypass, bypass_desc_sz_idx);
+ if (err != 0)
+ return err;
+ qdma_start_tx_queue(txq);
+
+ dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
+ txq->status = RTE_ETH_QUEUE_STATE_STARTED;
+ return 0;
+}
+
+
+int qdma_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ uint64_t phys_addr, cmpt_phys_addr;
+ uint32_t queue_base = qdma_dev->queue_base;
+ uint8_t en_pfetch;
+ uint8_t cmpt_desc_fmt;
+ int err, bypass_desc_sz_idx;
+
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ qdma_reset_rx_queue(rxq);
+ qdma_reset_rx_queues(dev, (qid + queue_base), rxq->st_mode);
+ bypass_desc_sz_idx = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+
+ rte_delay_ms(1);
+ en_pfetch = (rxq->en_prefetch) ? 1 : 0;
+ phys_addr = (uint64_t)rxq->rx_mz->phys_addr;
+
+ switch (rxq->cmpt_desc_len) {
+ case CMPT_DESC_LEN_8B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_8B;
+ break;
+ case CMPT_DESC_LEN_16B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_16B;
+ break;
+ case CMPT_DESC_LEN_32B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_32B;
+ break;
+ case CMPT_DESC_LEN_64B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_64B;
+ break;
+ default:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_8B;
+ break;
+ }
+
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) Phys-addr for C2H queue-id:%d: "
+ "is Lo:0x%lx, Hi:0x%lx\n",
+ qdma_dev->pf, qid,
+ rte_cpu_to_le_32(phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(phys_addr >> 32));
+
+ if (rxq->st_mode)
+ cmpt_phys_addr = (uint64_t)rxq->rx_cmpt_mz->phys_addr;
+ else
+ cmpt_phys_addr = 0;
+
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) C2H Completion Phys-addr for "
+ "queue-id:%d: is Lo:0x%lx, Hi:0x%lx\n",
+ qdma_dev->pf, qid,
+ rte_cpu_to_le_32(cmpt_phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(cmpt_phys_addr >> 32));
+
+ err = qdma_start_rx_queue(rxq);
+ if (err != 0)
+ return err;
+ err = qdma_queue_ctxt_rx_prog(dev, (qid + queue_base), rxq->st_mode,
+ en_pfetch, rxq->ringszidx,
+ rxq->cmpt_ringszidx, rxq->buffszidx,
+ rxq->threshidx, rxq->timeridx,
+ rxq->triggermode, rxq->func_id,
+ phys_addr, cmpt_phys_addr,
+ cmpt_desc_fmt, rxq->en_bypass,
+ rxq->en_bypass_prefetch,
+ bypass_desc_sz_idx,
+ rxq->dis_overflow_check);
+ if (err != 0)
+ return err;
+
+ if (rxq->st_mode) {
+ rte_wmb();
+ /* enable status desc , loading the triggermode,
+ * thresidx and timeridx passed from the user
+ */
+ *rxq->rx_cidx = (CMPT_STATUS_DESC_EN <<
+ CMPT_STATUS_DESC_EN_SHIFT) |
+ (rxq->triggermode << CMPT_TRIGGER_MODE_SHIFT) |
+ (rxq->threshidx << CMPT_THREHOLD_CNT_SHIFT) |
+ (rxq->timeridx << CMPT_TIMER_CNT_SHIFT);
+ *rxq->rx_pidx = (rxq->nb_rx_desc - 2);
+ }
+
+ dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
+ rxq->status = RTE_ETH_QUEUE_STATE_STARTED;
+ return 0;
+}
+
+int qdma_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ uint32_t queue_base = qdma_dev->queue_base;
+ int i = 0;
+ int cnt = 0;
+
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+
+ rxq->status = RTE_ETH_QUEUE_STATE_STOPPED;
+
+ /* Wait for queue to recv all packets. */
+ if (rxq->st_mode) { /** ST-mode **/
+ while (rxq->wb_status->pidx != rxq->rx_cmpt_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+ } else { /* MM mode */
+ while (rxq->wb_status->cidx != rxq->rx_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+ }
+
+ qdma_inv_rx_queues(dev, (qid + queue_base), rxq->st_mode);
+ qdma_reset_rx_queues(dev, (qid + queue_base), rxq->st_mode);
+
+ if (rxq->st_mode) { /** ST-mode **/
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: queue id %d,"
+ "mbuf_avail_count = %d, mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ for (i = 0; i < rxq->nb_rx_desc - 1; i++) {
+ rte_pktmbuf_free(rxq->sw_ring[i]);
+ rxq->sw_ring[i] = NULL;
+ }
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: queue id %d,"
+ "mbuf_avail_count = %d, mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ }
+
+ qdma_reset_rx_queue(rxq);
+
+ dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+ return 0;
+}
+
+int qdma_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint32_t queue_base = qdma_dev->queue_base;
+ struct qdma_tx_queue *txq;
+ int cnt = 0;
+ uint16_t count;
+
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+
+ txq->status = RTE_ETH_QUEUE_STATE_STOPPED;
+ /* Wait for TXQ to send out all packets. */
+ while (txq->wb_status->cidx != txq->tx_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+
+ qdma_inv_tx_queues(dev, (qid + queue_base), txq->st_mode);
+ qdma_reset_tx_queues(dev, (qid + queue_base), txq->st_mode);
+
+ /* Relinquish pending mbufs */
+ for (count = 0; count < txq->nb_tx_desc - 1; count++) {
+ rte_pktmbuf_free(txq->sw_ring[count]);
+ txq->sw_ring[count] = NULL;
+ }
+ qdma_reset_tx_queue(txq);
+
+ dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
+
+ return 0;
+}
+
+static int qdma_dev_stats_get(struct rte_eth_dev *dev,
+ struct rte_eth_stats *eth_stats)
+{
+ unsigned int i;
+
+ eth_stats->opackets = 0;
+ eth_stats->obytes = 0;
+ eth_stats->ipackets = 0;
+ eth_stats->ibytes = 0;
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ struct qdma_rx_queue *rxq =
+ (struct qdma_rx_queue *)dev->data->rx_queues[i];
+
+ eth_stats->q_ipackets[i] = rxq->stats.pkts;
+ eth_stats->q_ibytes[i] = rxq->stats.bytes;
+ eth_stats->ipackets += eth_stats->q_ipackets[i];
+ eth_stats->ibytes += eth_stats->q_ibytes[i];
+ }
+
+ for (i = 0; i < dev->data->nb_tx_queues; i++) {
+ struct qdma_tx_queue *txq =
+ (struct qdma_tx_queue *)dev->data->tx_queues[i];
+
+ eth_stats->q_opackets[i] = txq->stats.pkts;
+ eth_stats->q_obytes[i] = txq->stats.bytes;
+ eth_stats->opackets += eth_stats->q_opackets[i];
+ eth_stats->obytes += eth_stats->q_obytes[i];
+ }
+ return 0;
+}
+
+static struct eth_dev_ops qdma_eth_dev_ops = {
+ .dev_configure = qdma_dev_configure,
+ .dev_infos_get = qdma_dev_infos_get,
+ .dev_start = qdma_dev_start,
+ .dev_stop = qdma_dev_stop,
+ .dev_close = qdma_dev_close,
+ .link_update = qdma_dev_link_update,
+ .rx_queue_setup = qdma_dev_rx_queue_setup,
+ .tx_queue_setup = qdma_dev_tx_queue_setup,
+ .rx_queue_release = qdma_dev_rx_queue_release,
+ .tx_queue_release = qdma_dev_tx_queue_release,
+ .rx_queue_start = qdma_dev_rx_queue_start,
+ .rx_queue_stop = qdma_dev_rx_queue_stop,
+ .tx_queue_start = qdma_dev_tx_queue_start,
+ .tx_queue_stop = qdma_dev_tx_queue_stop,
+ .stats_get = qdma_dev_stats_get,
+};
+
+void qdma_dev_ops_init(struct rte_eth_dev *dev)
+{
+ dev->dev_ops = &qdma_eth_dev_ops;
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ dev->rx_pkt_burst = &qdma_recv_pkts;
+ dev->tx_pkt_burst = &qdma_xmit_pkts;
+ }
+}
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_ethdev.c b/QDMA/DPDK/drivers/net/qdma/qdma_ethdev.c
new file mode 100644
index 0000000000000000000000000000000000000000..2991194f6957726133bc09a8690722dfccc77836
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_ethdev.c
@@ -0,0 +1,966 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
+#include <rte_ethdev_pci.h>
+#include <rte_malloc.h>
+#include <rte_dev.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_alarm.h>
+#include <rte_cycles.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "qdma.h"
+#include "qdma_regs.h"
+#include "version.h"
+
+static void qdma_device_attributes_get(struct qdma_pci_dev *qdma_dev);
+
+uint32_t g_ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {
+1024, 256, 512, 768, 1280, 1536, 2048, 2560,
+3072, 4096, 5120, 6144, 8192, 1024, 1024, 1024
+};
+
+uint32_t g_c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ] = {
+64, 2, 4, 8, 12, 16, 24, 32, 40, 48, 96, 128, 192, 256, 384, 512
+};
+
+uint32_t g_c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {
+4096, 256, 512, 1024, 2048, 8192, 9018, 16384, 3968, 4096, 4096,
+4096, 4096, 4096, 4096, 4096
+};
+
+uint32_t g_c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ] = {
+0, 1, 2, 3, 4, 5, 7, 8, 10, 12, 15, 20, 30, 50, 100, 200
+};
+
+static int qdma_vf_hi(struct rte_eth_dev *dev, uint16_t vf_funid,
+ struct qdma_mbox_data *recv_msg)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint32_t qbase, qmax, i;
+ struct rte_pci_device *pci_dev;
+ int ret = 0;
+ uint64_t sz;
+
+ PMD_DRV_LOG(INFO, "PF-%d received HI msg from VF global function-id %d",
+ PCI_FUNC(qdma_dev->pf), vf_funid);
+ pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ /* Mapping internal function id index to external function id */
+ for (i = 0; i < pci_dev->max_vfs; i++) {
+ if (qdma_dev->vfinfo[i].func_id == QDMA_FUNC_ID_INVALID) {
+ qdma_dev->vfinfo[i].func_id = vf_funid;
+ break;
+ }
+ }
+
+ if (i == pci_dev->max_vfs) {
+ PMD_DRV_LOG(INFO, "PF-%d failed to create function "
+ "id mapping VF- func_id%d",
+ PCI_FUNC(qdma_dev->pf), vf_funid);
+ return -1;
+ }
+
+ PMD_DRV_LOG(INFO, "PF-%d received HI msg from VF internal "
+ "id %d global %d", PCI_FUNC(qdma_dev->pf), i, vf_funid);
+
+ qbase = recv_msg->data[0];
+ qdma_dev->vfinfo[i].qbase = recv_msg->data[0];
+ qmax = recv_msg->data[1];
+ qdma_dev->vfinfo[i].qmax = recv_msg->data[1];
+ qdma_dev->vfinfo[i].func_id = vf_funid;
+
+ sz = sizeof(struct qdma_vf_queue_info) * qmax;
+ qdma_dev->vfinfo[i].vfqinfo = rte_zmalloc("vfqinfo", sz, 0);
+ if (qdma_dev->vfinfo[i].vfqinfo == NULL) {
+ PMD_DRV_LOG(INFO, "cannot allocate memory for VF queue info");
+ return -1;
+ }
+
+ ret = qdma_set_fmap(qdma_dev, vf_funid, qbase, qmax);
+ if (ret < 0) {
+ rte_free(qdma_dev->vfinfo[i].vfqinfo);
+ return -1;
+ }
+ return 0;
+}
+
+static int qdma_vf_bye(struct rte_eth_dev *dev, uint16_t vf, uint16_t vf_funid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t bar_addr;
+#if FMAP_CNTXT
+ uint32_t ctxt_sel, reg_val;
+ uint32_t i, flag;
+ struct queue_ind_prg *q_regs;
+#else
+ uint64_t fmap;
+#endif
+
+ PMD_DRV_LOG(INFO, "PF-%d received BYE msg from VF internal id %d, "
+ "global id %d", PCI_FUNC(qdma_dev->pf), vf, vf_funid);
+
+ if (qdma_dev->vfinfo[vf].vfqinfo != NULL)
+ rte_free(qdma_dev->vfinfo[vf].vfqinfo);
+
+ qdma_dev->vfinfo[vf].qbase = 0;
+ qdma_dev->vfinfo[vf].qmax = 0;
+ qdma_dev->vfinfo[vf].func_id = QDMA_FUNC_ID_INVALID;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+#if FMAP_CNTXT
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ ctxt_sel = (QDMA_CTXT_SEL_FMAP << CTXT_SEL_SHIFT_B) |
+ (vf_funid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_CLR << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(INFO, " Read cmd for device:%d: reg_val:%x\n",
+ vf_funid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+#else
+
+ fmap = (uint64_t)(bar_addr + QDMA_TRQ_SEL_FMAP + (vf_funid * 4));
+ qdma_write_reg((uint64_t)fmap, 0);
+#endif
+
+ return 0;
+}
+
+/* Perform queue context programming */
+
+static int qdma_vf_qadd(struct rte_eth_dev *dev, uint16_t vf, uint8_t vf_funid,
+ struct qdma_mbox_data *recv_msg)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qadd_msg msg;
+ int err, qbase;
+ uint64_t addr;
+
+ PMD_DRV_LOG(INFO, "PF-%d received QADD msg from VF internal id %d, "
+ "global id %d", PCI_FUNC(qdma_dev->pf), vf, vf_funid);
+
+ msg.qid = recv_msg->data[0];
+ msg.st = recv_msg->data[1];
+ msg.c2h = recv_msg->data[2];
+ msg.ringsz = recv_msg->data[3];
+ msg.bufsz = recv_msg->data[4];
+ msg.thresidx = recv_msg->data[5];
+ msg.timeridx = recv_msg->data[6];
+ msg.triggermode = recv_msg->data[7];
+ msg.cmpt_ringszidx = recv_msg->data[12];
+ msg.prefetch = recv_msg->data[13];
+ msg.cmpt_desc_fmt = recv_msg->data[14];
+ msg.en_bypass = recv_msg->data[15];
+ msg.bypass_desc_sz_idx = recv_msg->data[16];
+ msg.en_bypass_prefetch = recv_msg->data[17];
+ msg.dis_overflow_check = recv_msg->data[18];
+
+ addr = rte_le_to_cpu_64(((uint64_t)recv_msg->data[9] << 32) |
+ recv_msg->data[8]);
+ msg.ring_bs_addr = addr;
+ addr = rte_le_to_cpu_64(((uint64_t)recv_msg->data[11] << 32) |
+ recv_msg->data[10]);
+ msg.cmpt_ring_bs_addr = addr;
+
+ qbase = qdma_dev->vfinfo[vf].qbase;
+
+ qdma_dev->vfinfo[vf].vfqinfo[msg.qid - qbase].mode = msg.st;
+
+ if (msg.c2h) {
+ qdma_reset_rx_queues(dev, msg.qid, msg.st);
+ err = qdma_queue_ctxt_rx_prog(dev, msg.qid, msg.st,
+ msg.prefetch, msg.ringsz,
+ msg.cmpt_ringszidx, msg.bufsz,
+ msg.thresidx, msg.timeridx,
+ msg.triggermode, vf_funid,
+ msg.ring_bs_addr,
+ msg.cmpt_ring_bs_addr,
+ msg.cmpt_desc_fmt,
+ msg.en_bypass,
+ msg.en_bypass_prefetch,
+ msg.bypass_desc_sz_idx,
+ msg.dis_overflow_check);
+ if (err != 0)
+ return err;
+ } else {
+ qdma_reset_tx_queues(dev, msg.qid, msg.st);
+ err = qdma_queue_ctxt_tx_prog(dev, msg.qid, msg.st, msg.ringsz,
+ vf_funid, msg.ring_bs_addr,
+ msg.en_bypass,
+ msg.bypass_desc_sz_idx);
+ if (err != 0)
+ return err;
+ }
+
+ *qdma_dev->h2c_mm_control = QDMA_MM_CTRL_START; /* Start Tx h2c engine*/
+ *qdma_dev->c2h_mm_control = QDMA_MM_CTRL_START; /* Start Rx c2h engine*/
+
+ return 0;
+}
+
+static int qdma_vf_qdel(struct rte_eth_dev *dev, uint8_t vf,
+ struct qdma_mbox_data *recv_msg)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ int qbase = 0;
+ uint32_t qid, dir;
+
+ PMD_DRV_LOG(INFO, "PF-%d received QDEL msg from VF-%d",
+ PCI_FUNC(qdma_dev->pf), vf);
+
+ qid = recv_msg->data[0];
+ dir = recv_msg->data[1];
+ qbase = qdma_dev->vfinfo[vf].qbase;
+
+ if (dir == DMA_FROM_DEVICE)
+ qdma_reset_rx_queues(dev, qid,
+ qdma_dev->vfinfo[vf].vfqinfo[qid - qbase].mode);
+ else
+ qdma_reset_tx_queues(dev, qid,
+ qdma_dev->vfinfo[vf].vfqinfo[qid - qbase].mode);
+
+ return 0;
+}
+
+/*
+ * Calculate vf internal funciton of PF from global function id
+ */
+static int qdma_get_dev_internal_vfid(struct rte_eth_dev *dev, uint8_t devfn)
+{
+ uint16_t i;
+ struct rte_pci_device *pci_dev;
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+ pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+
+ for (i = 0; i < pci_dev->max_vfs; i++) {
+ if (qdma_dev->vfinfo[i].func_id == devfn)
+ return i;
+ }
+
+ return QDMA_FUNC_ID_INVALID;
+}
+
+static void qdma_read_vf_msg(struct rte_eth_dev *dev, uint8_t devfn)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t bar_addr;
+ struct qdma_mbox_data recv_msg;
+ struct mbox_imsg *in_msg;
+ int32_t retval = -1;
+ uint32_t reg_val;
+ uint16_t vf;
+ int i;
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+
+ /* write function ID into target function register to get the
+ * msg in incoming message register
+ */
+
+ qdma_write_reg((uint64_t)(bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_TRGT_FN),
+ devfn);
+ in_msg = (struct mbox_imsg *)(bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_IMSG);
+ reg_val = qdma_read_reg((uint64_t)&in_msg->imsg[0]);
+
+ recv_msg.opcode = reg_val & 0xff;
+ recv_msg.debug_funid = (reg_val >> 8) & 0xfff;
+ recv_msg.filler = (reg_val >> 20) & 0xf;
+ recv_msg.err = (reg_val >> 24) & 0xff;
+
+ for (i = 1; i < 32; i++)
+ recv_msg.data[i - 1] = qdma_read_reg((uint64_t)&
+ in_msg->imsg[i]);
+
+ if (recv_msg.opcode == QDMA_MBOX_OPCD_HI) {
+ retval = qdma_vf_hi(dev, devfn, &recv_msg);
+ goto ack_ok;
+ }
+
+ vf = qdma_get_dev_internal_vfid(dev, devfn);
+
+ if (vf == QDMA_FUNC_ID_INVALID)
+ return;
+
+ switch (recv_msg.opcode) {
+ case QDMA_MBOX_OPCD_BYE:
+ retval = qdma_vf_bye(dev, vf, devfn);
+ break;
+ case QDMA_MBOX_OPCD_QADD:
+ retval = qdma_vf_qadd(dev, vf, devfn, &recv_msg);
+ break;
+ case QDMA_MBOX_OPCD_QDEL:
+ retval = qdma_vf_qdel(dev, vf, &recv_msg);
+ break;
+ }
+
+ack_ok:
+ /* once the msg is handled ACK it */
+ if (!retval)
+ qdma_write_reg((bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_FCMD),
+ QDMA_MBOX_CMD_RECV);
+}
+
+static void qdma_check_vf_msg(void *arg)
+{
+ struct qdma_pci_dev *qdma_dev = NULL;
+ uint64_t bar_addr, reg_addr;
+ struct mbox_fsr *mbx_stat;
+
+ qdma_dev = ((struct rte_eth_dev *)arg)->data->dev_private;
+ if (!qdma_dev)
+ return;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ reg_addr = bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_FSR;
+ mbx_stat = (struct mbox_fsr *)(reg_addr);
+
+
+ if (mbx_stat->imsg_stat == 1)
+ qdma_read_vf_msg((struct rte_eth_dev *)arg,
+ mbx_stat->curr_src_fn);
+
+ rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_check_vf_msg, arg);
+}
+
+/*
+ * The set of PCI devices this driver supports
+ */
+static struct rte_pci_id qdma_pci_id_tbl[] = {
+#define RTE_PCI_DEV_ID_DECL_XNIC(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
+#ifndef PCI_VENDOR_ID_XILINX
+#define PCI_VENDOR_ID_XILINX 0x10ee
+#endif
+
+ /** Gen 1 PF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9011) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9111) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9211) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9311) /** PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9014) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9114) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9214) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9314) /** PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9018) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9118) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9218) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9318) /** PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x901f) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x911f) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x921f) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x931f) /** PF 3 */
+
+ /** Gen 2 PF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9021) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9121) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9221) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9321) /** PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9024) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9124) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9224) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9324) /** PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9028) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9128) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9228) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9328) /** PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x902f) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x912f) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x922f) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x932f) /** PF 3 */
+
+ /** Gen 3 PF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9031) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9131) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9231) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9331) /** PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9034) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9134) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9234) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9334) /** PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9038) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9138) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9238) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9338) /** PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x903f) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x913f) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x923f) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x933f) /** PF 3 */
+
+ /** Gen 4 PF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9041) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9141) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9241) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9341) /** PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9044) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9144) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9244) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9344) /** PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9048) /** PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9148) /** PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9248) /** PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0x9348) /** PF 3 */
+
+ { .vendor_id = 0, /* sentinel */ },
+};
+
+static void qdma_device_attributes_get(struct qdma_pci_dev *qdma_dev)
+{
+ int mm_c2h_flag = 0;
+ int mm_h2c_flag = 0;
+ int st_c2h_flag = 0;
+ int st_h2c_flag = 0;
+ uint64_t bar_addr;
+ uint32_t v1;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ v1 = qdma_read_reg((uint64_t)(bar_addr + QDMA_REG_GLBL_QMAX));
+
+ /* DPDK limitation */
+ if (v1 > RTE_MAX_QUEUES_PER_PORT)
+ v1 = RTE_MAX_QUEUES_PER_PORT;
+ qdma_dev->qsets_max = v1;
+
+ v1 = qdma_read_reg((uint64_t)(bar_addr + QDMA_REG_GLBL_MDMA_CHANNEL));
+ mm_c2h_flag = (v1 & MDMA_CHANNEL_MM_C2H_ENABLED_MASK) ? 1 : 0;
+ mm_h2c_flag = (v1 & MDMA_CHANNEL_MM_H2C_ENABLED_MASK) ? 1 : 0;
+ st_c2h_flag = (v1 & MDMA_CHANNEL_ST_C2H_ENABLED_MASK) ? 1 : 0;
+ st_h2c_flag = (v1 & MDMA_CHANNEL_ST_H2C_ENABLED_MASK) ? 1 : 0;
+
+ qdma_dev->mm_mode_en = (mm_c2h_flag && mm_h2c_flag) ? 1 : 0;
+ qdma_dev->st_mode_en = (st_c2h_flag && st_h2c_flag) ? 1 : 0;
+
+ PMD_DRV_LOG(INFO, "qmax = %d, mm %d, st %d.\n", qdma_dev->qsets_max,
+ qdma_dev->mm_mode_en, qdma_dev->st_mode_en);
+}
+
+static inline uint8_t pcie_find_cap(const struct rte_pci_device *pci_dev,
+ uint8_t cap)
+{
+ uint8_t pcie_cap_pos = 0;
+ uint8_t pcie_cap_id = 0;
+
+ if (rte_pci_read_config(pci_dev, &pcie_cap_pos, sizeof(uint8_t),
+ PCI_CAPABILITY_LIST) < 0) {
+ PMD_DRV_LOG(ERR, "PCIe config space read failed..\n");
+ return 0;
+ }
+
+ while (pcie_cap_pos >= 0x40) {
+ pcie_cap_pos &= ~3;
+
+ if (rte_pci_read_config(pci_dev, &pcie_cap_id, sizeof(uint8_t),
+ pcie_cap_pos + PCI_CAP_LIST_ID) < 0) {
+ PMD_DRV_LOG(ERR, "PCIe config space read failed..\n");
+ goto ret;
+ }
+
+ if (pcie_cap_id == 0xff)
+ break;
+
+ if (pcie_cap_id == cap)
+ return pcie_cap_pos;
+
+ if (rte_pci_read_config(pci_dev, &pcie_cap_pos, sizeof(uint8_t),
+ pcie_cap_pos + PCI_CAP_LIST_NEXT) < 0) {
+ PMD_DRV_LOG(ERR, "PCIe config space read failed..\n");
+ goto ret;
+ }
+ }
+
+ret:
+ return 0;
+}
+
+static void pcie_perf_enable(const struct rte_pci_device *pci_dev)
+{
+ uint16_t value;
+ uint8_t pcie_cap_pos = pcie_find_cap(pci_dev, PCI_CAP_ID_EXP);
+
+ if (!pcie_cap_pos)
+ return;
+
+ if (pcie_cap_pos > 0) {
+ if (rte_pci_read_config(pci_dev, &value, sizeof(uint16_t),
+ pcie_cap_pos + PCI_EXP_DEVCTL) < 0) {
+ PMD_DRV_LOG(ERR, "PCIe config space read failed..\n");
+ return;
+ }
+
+ value |= (PCI_EXP_DEVCTL_EXT_TAG | PCI_EXP_DEVCTL_RELAX_EN);
+
+ if (rte_pci_write_config(pci_dev, &value, sizeof(uint16_t),
+ pcie_cap_pos + PCI_EXP_DEVCTL) < 0) {
+ PMD_DRV_LOG(ERR, "PCIe config space write failed..\n");
+ return;
+ }
+ }
+}
+
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function creates an Ethernet device for each port of a given
+ * PCI device.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int eth_qdma_dev_init(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *dma_priv;
+ uint8_t *baseaddr;
+ uint64_t bar_len;
+ int i, idx;
+ static bool once = true;
+ int vf_num;
+ uint64_t reg_addr, sz;
+ uint32_t reg_val;
+ uint32_t pfch_val;
+ struct rte_pci_device *pci_dev;
+
+ /* sanity checks */
+ if (dev == NULL)
+ return -EINVAL;
+ if (dev->data == NULL)
+ return -EINVAL;
+ if (dev->data->dev_private == NULL)
+ return -EINVAL;
+
+ pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ if (pci_dev == NULL)
+ return -EINVAL;
+
+ /* for secondary processes, we don't initialise any further as primary
+ * has already done this work.
+ */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ qdma_dev_ops_init(dev);
+ return 0;
+ }
+
+ if (once)
+ RTE_LOG(INFO, PMD, "QDMA PMD VERSION: %s\n", QDMA_PMD_VERSION);
+
+ /* allocate space for a single Ethernet MAC address */
+ dev->data->mac_addrs = rte_zmalloc("qdma", ETHER_ADDR_LEN * 1, 0);
+ if (dev->data->mac_addrs == NULL)
+ return -ENOMEM;
+
+ /* Copy some dummy Ethernet MAC address for QDMA device
+ * This will change in real NIC device...
+ */
+ for (i = 0; i < ETHER_ADDR_LEN; ++i)
+ dev->data->mac_addrs[0].addr_bytes[i] = 0x15 + i;
+
+ /* Init system & device */
+ dma_priv = (struct qdma_pci_dev *)dev->data->dev_private;
+ dma_priv->pf = PCI_DEVFN(pci_dev->addr.devid, pci_dev->addr.function);
+ dma_priv->queue_base = DEFAULT_QUEUE_BASE;
+ dma_priv->timer_count = DEFAULT_TIMER_CNT_TRIG_MODE_TIMER;
+
+ /* Setting default Mode to TRIG_MODE_USER_TIMER_COUNT */
+ dma_priv->trigger_mode = TRIG_MODE_USER_TIMER_COUNT;
+ if (dma_priv->trigger_mode == TRIG_MODE_USER_TIMER_COUNT)
+ dma_priv->timer_count = DEFAULT_TIMER_CNT_TRIG_MODE_COUNT_TIMER;
+
+ dma_priv->en_desc_prefetch = 0; //Keep prefetch default to 0
+ dma_priv->cmpt_desc_len = DEFAULT_QDMA_CMPT_DESC_LEN;
+ dma_priv->c2h_bypass_mode = C2H_BYPASS_MODE_NONE;
+ dma_priv->h2c_bypass_mode = 0;
+
+ /*Default write-back accumulation interval */
+ dma_priv->wb_acc_int = DEFAULT_WB_ACC_INT;
+
+ dma_priv->config_bar_idx = DEFAULT_PF_CONFIG_BAR;
+ dma_priv->bypass_bar_idx = BAR_ID_INVALID;
+ dma_priv->user_bar_idx = BAR_ID_INVALID;
+
+ /* Check and handle device devargs*/
+ if (qdma_check_kvargs(dev->device->devargs, dma_priv)) {
+ PMD_DRV_LOG(INFO, "devargs failed\n");
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ idx = qdma_identify_bars(dev);
+ if (idx < 0) {
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ PMD_DRV_LOG(INFO, "QDMA device driver probe:");
+
+ idx = qdma_get_hw_version(dev);
+ if (idx < 0) {
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ baseaddr = (uint8_t *)
+ pci_dev->mem_resource[dma_priv->config_bar_idx].addr;
+ bar_len = pci_dev->mem_resource[dma_priv->config_bar_idx].len;
+
+ dma_priv->bar_addr[dma_priv->config_bar_idx] = baseaddr;
+ dma_priv->bar_len[dma_priv->config_bar_idx] = bar_len;
+
+ if (dma_priv->user_bar_idx >= 0) {
+ baseaddr = (uint8_t *)
+ pci_dev->mem_resource[dma_priv->user_bar_idx].addr;
+ bar_len = pci_dev->mem_resource[dma_priv->user_bar_idx].len;
+
+ dma_priv->bar_addr[dma_priv->user_bar_idx] = baseaddr;
+ dma_priv->bar_len[dma_priv->user_bar_idx] = bar_len;
+ }
+
+ qdma_dev_ops_init(dev);
+ baseaddr = (uint8_t *)
+ pci_dev->mem_resource[dma_priv->config_bar_idx].addr;
+
+ /* Getting the device attributes from the Hardware */
+ qdma_device_attributes_get(dma_priv);
+
+ dma_priv->c2h_mm_control = (volatile uint32_t *)(baseaddr +
+ QDMA_TRQ_SEL_C2H_MM0 +
+ QDMA_C2H_MM0_CONTROL);
+ dma_priv->h2c_mm_control = (volatile uint32_t *)(baseaddr +
+ QDMA_TRQ_SEL_H2C_MM0 +
+ QDMA_H2C_MM0_CONTROL);
+ if (once) {
+ /** Write-back accumulation configuration **/
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_GLBL +
+ QDMA_GLBL_DSC_CFG);
+ reg_val = ((dma_priv->wb_acc_int <<
+ QDMA_GLBL_DSC_CFG_WB_ACC_SHFT) |
+ (DEFAULT_MAX_DESC_FETCH <<
+ QDMA_GLBL_DSC_CFG_MAX_DSC_FTCH_SHFT));
+ qdma_write_reg((uint64_t)reg_addr, reg_val);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " GLBL_DSC_CFG reg_val:0x%x, wb_acc: %d,"
+ " max_desc_fetch: %d\n",
+ reg_val, (reg_val & QDMA_GLBL_DSC_CFG_WB_ACC_MSK),
+ (reg_val & QDMA_GLBL_DSC_CFG_MAX_DSC_FTCH_MSK) >>
+ QDMA_GLBL_DSC_CFG_MAX_DSC_FTCH_SHFT);
+#endif
+ /* Ring-size configuration, need to set the same index
+ * in the H2C/C2H context structure
+ */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_GLBL +
+ QDMA_GLBL_RING_SZ);
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++, reg_addr += 4) {
+ qdma_write_reg((uint64_t)reg_addr, g_ring_sz[i]);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " Ring-size reg_val:0x%x written to "
+ "index %d", reg_val, i);
+#endif
+ }
+
+ /* C2H threshold count configuration, need to set the same index
+ * in the C2H completion context structure
+ */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_CNT_TH_BASE);
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++, reg_addr += 4) {
+ qdma_write_reg((uint64_t)reg_addr, g_c2h_cnt_th[i]);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " threshold-count reg_val:0x%x "
+ "written to index %d\n", reg_val, i);
+#endif
+ }
+
+ /* C2H Buf size, need to allocate the same buffer size
+ * in the C2H descriptor source addr
+ */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_BUF_SZ_BASE);
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++, reg_addr += 4) {
+ qdma_write_reg((uint64_t)reg_addr, g_c2h_buf_sz[i]);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " Buffer size for C2H, reg_val:0x%x "
+ "written to index %d\n", reg_val, i);
+#endif
+ }
+
+ /** C2H Prefetch configuration */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_PFCH_CACHE_DEPTH_OFFSET);
+ pfch_val = qdma_read_reg(reg_addr);
+
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_PFCH_CFG_OFFSET);
+ reg_val = ((DEFAULT_PFCH_STOP_THRESH <<
+ QDMA_C2H_PFCH_CFG_STOP_THRESH_SHIFT) |
+ (DEFAULT_PFCH_NUM_ENTRIES_PER_Q <<
+ QDMA_C2H_PFCH_CFG_NUM_ENTRIES_PER_Q_SHIFT) |
+ ((pfch_val / 2) <<
+ QDMA_C2H_PFCH_CFG_MAX_Q_CNT_SHIFT) |
+ (((pfch_val / 2) - 2) <<
+ QDMA_C2H_PFCH_CFG_EVICTION_Q_CNT_SHIFT));
+ qdma_write_reg((uint64_t)reg_addr, reg_val);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, "C2H prefetch config, reg_val:0x%x", reg_val);
+#endif
+
+ /** C2H timer tick configuration */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_INT_TIMER_TICK_OFFSET);
+ qdma_write_reg((uint64_t)reg_addr, DEFAULT_C2H_INTR_TIMER_TICK);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " timer tick for C2H, reg_val:0x%x", reg_val);
+#endif
+
+ /** C2H Writeback Coalesce configuration */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_CMPT_COAL_BUF_DEPTH_OFFSET);
+ pfch_val = qdma_read_reg(reg_addr);
+
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_CMPT_COAL_CFG_OFFSET);
+ reg_val = ((DEFAULT_CMPT_COAL_TIMER_CNT <<
+ QDMA_C2H_CMPT_COAL_TIMER_CNT_VAL_SHIFT) |
+ (DEFAULT_CMPT_COAL_TIMER_TICK <<
+ QDMA_C2H_CMPT_COAL_TIMER_TICK_VAL_SHIFT) |
+ (pfch_val <<
+ QDMA_C2H_CMPT_COAL_MAX_BUF_SZ_SHIFT));
+
+ qdma_write_reg((uint64_t)reg_addr, reg_val);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, "C2H writeback coalesce config, reg_val:0x%x",
+ reg_val);
+#endif
+ /** C2H timer configuration*/
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_C2H +
+ QDMA_C2H_TIMER_BASE);
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++, reg_addr += 4) {
+ qdma_write_reg((uint64_t)reg_addr, g_c2h_timer_cnt[i]);
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ reg_val = qdma_read_reg((uint64_t)reg_addr);
+ PMD_DRV_LOG(INFO, " timer count for C2H, reg_val:0x%x "
+ "written to index %d\n", reg_val, i);
+#endif
+ }
+
+ /* Data Threshold (Throttle) register setting when
+ * Prefetch is disabled
+ */
+ reg_addr = (uint64_t)(baseaddr + QDMA_TRQ_SEL_H2C +
+ QDMA_H2C_DATA_THRESHOLD_OFFSET);
+ qdma_write_reg((uint64_t)reg_addr, DEFAULT_H2C_THROTTLE);
+ once = false;
+ }
+
+ vf_num = pci_dev->max_vfs;
+ if (vf_num) {
+ sz = sizeof(struct qdma_vf_info) * vf_num;
+ dma_priv->vfinfo = rte_zmalloc("vfinfo", sz, 0);
+ if (dma_priv->vfinfo == NULL)
+ rte_panic("Cannot allocate memory for VF info\n");
+ /* Mark all VFs with invalid function id mapping*/
+ for (i = 0; i < vf_num; i++)
+ dma_priv->vfinfo[i].func_id = QDMA_FUNC_ID_INVALID;
+
+ rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_check_vf_msg,
+ (void *)dev);
+ }
+
+ pcie_perf_enable(pci_dev);
+
+ return 0;
+}
+
+/**
+ * DPDK callback to deregister PCI device.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int eth_qdma_dev_uninit(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint32_t numvf, i, count, v;
+ struct mbox_fsr *mbx_stat;
+ struct qdma_mbox_data send_msg;
+ uint64_t bar_addr, addr;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+
+ /* only uninitialize in the primary process */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return -EPERM;
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ mbx_stat = (struct mbox_fsr *)(bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_FSR);
+
+ numvf = pci_dev->max_vfs;
+
+ for (i = 0; i < numvf; i++) {
+ count = MAILBOX_PROG_POLL_COUNT;
+ memset(&send_msg, 0, sizeof(send_msg));
+ if (qdma_dev->vfinfo[i].qmax) {
+ qdma_write_reg((uint64_t)(bar_addr + QDMA_TRQ_EXT +
+ QDMA_MBOX_TRGT_FN),
+ qdma_dev->vfinfo[i].func_id);
+ while (mbx_stat->omsg_stat && count) {
+ rte_delay_ms(MAILBOX_PF_MSG_DELAY);
+ count--;
+ }
+ if (count) {
+ PMD_DRV_LOG(INFO, "PF-%d(DEVFN) send BYE "
+ "message to VF-%d(DEVFN)",
+ qdma_dev->pf,
+ qdma_dev->vfinfo[i].func_id);
+ send_msg.opcode = QDMA_MBOX_OPCD_BYE;
+ send_msg.debug_funid = qdma_dev->pf;
+ v = rte_cpu_to_le_32(send_msg.debug_funid << 8 |
+ send_msg.opcode);
+ addr = bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_OMSG;
+ qdma_write_reg(addr, v);
+
+ addr = bar_addr + QDMA_TRQ_EXT + QDMA_MBOX_FCMD;
+ qdma_write_reg(addr, QDMA_MBOX_CMD_SEND);
+
+ count = MAILBOX_PROG_POLL_COUNT;
+ while (mbx_stat->omsg_stat && count) {
+ rte_delay_ms(MAILBOX_PF_MSG_DELAY);
+ count--;
+ }
+ if (!count)
+ PMD_DRV_LOG(INFO, "Pf-%d(DEVFN) did "
+ "not receive ACK from "
+ "VF-%d(DEVFN)",
+ qdma_dev->pf,
+ qdma_dev->vfinfo[i].func_id);
+ } else {
+ PMD_DRV_LOG(INFO, "Pf-%d(DEVFN) could not send "
+ "BYE message to VF-%d(DEVFN)",
+ qdma_dev->pf,
+ qdma_dev->vfinfo[i].func_id);
+ break;
+ }
+ }
+ }
+
+ /* cancel pending polls*/
+ rte_eal_alarm_cancel(qdma_check_vf_msg, (void *)dev);
+ *qdma_dev->h2c_mm_control = 0; /* Stop Tx h2c engine */
+ *qdma_dev->c2h_mm_control = 0;
+
+ if (qdma_dev->vfinfo != NULL) {
+ rte_free(qdma_dev->vfinfo);
+ qdma_dev->vfinfo = NULL;
+ }
+
+ dev->dev_ops = NULL;
+ dev->rx_pkt_burst = NULL;
+ dev->tx_pkt_burst = NULL;
+ dev->data->nb_rx_queues = 0;
+ dev->data->nb_tx_queues = 0;
+
+ if (dev->data->mac_addrs != NULL) {
+ rte_free(dev->data->mac_addrs);
+ dev->data->mac_addrs = NULL;
+ }
+
+ if (qdma_dev->q_info != NULL) {
+ rte_free(qdma_dev->q_info);
+ qdma_dev->q_info = NULL;
+ }
+
+ return 0;
+}
+
+static int eth_qdma_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+ struct rte_pci_device *pci_dev)
+{
+ return rte_eth_dev_pci_generic_probe(pci_dev,
+ sizeof(struct qdma_pci_dev),
+ eth_qdma_dev_init);
+}
+
+/* Detach a ethdev interface */
+static int eth_qdma_pci_remove(struct rte_pci_device *pci_dev)
+{
+ return rte_eth_dev_pci_generic_remove(pci_dev, eth_qdma_dev_uninit);
+}
+
+static struct rte_pci_driver rte_qdma_pmd = {
+ .id_table = qdma_pci_id_tbl,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+ .probe = eth_qdma_pci_probe,
+ .remove = eth_qdma_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(net_qdma, rte_qdma_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(net_qdma, qdma_pci_id_tbl);
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_export.h b/QDMA/DPDK/drivers/net/qdma/qdma_export.h
new file mode 100644
index 0000000000000000000000000000000000000000..83143523df661021ec569f8ae0233ac1297666c9
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_export.h
@@ -0,0 +1,68 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __QDMA_EXPORT_H__
+#define __QDMA_EXPORT_H__
+
+#include <rte_dev.h>
+#include <rte_ethdev.h>
+#include <rte_spinlock.h>
+#include <rte_log.h>
+#include <rte_byteorder.h>
+#include <rte_memzone.h>
+#include <linux/pci.h>
+
+enum rte_xdebug_type {
+ ETH_XDEBUG_QDMA_GLOBAL_CSR,
+ ETH_XDEBUG_QUEUE_CONTEXT,
+ ETH_XDEBUG_QUEUE_STRUCT,
+ ETH_XDEBUG_QUEUE_DESC_DUMP,
+ ETH_XDEBUG_MAX,
+};
+
+enum rte_xdebug_desc_type {
+ ETH_XDEBUG_DESC_C2H,
+ ETH_XDEBUG_DESC_H2C,
+ ETH_XDEBUG_DESC_CMPT,
+ ETH_XDEBUG_DESC_MAX,
+};
+struct rte_xdebug_desc_param {
+ uint16_t queue;
+ int start;
+ int end;
+ enum rte_xdebug_desc_type type;
+};
+
+int qdma_xdebug(uint8_t port_id, enum rte_xdebug_type type,
+ void *params);
+
+#endif /* ifndef __QDMA_EXPORT_H__ */
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_regs.h b/QDMA/DPDK/drivers/net/qdma/qdma_regs.h
new file mode 100644
index 0000000000000000000000000000000000000000..5955e649939000aa69c6597b80ef5cf8b8311652
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_regs.h
@@ -0,0 +1,367 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __QDMA_REGS_H__
+#define __QDMA_REGS_H__
+
+#define QDMA_BAR_NUM (6)
+
+#define QDMA_GLOBAL_CSR_ARRAY_SZ (16)
+
+extern uint32_t g_ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+extern uint32_t g_c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ];
+extern uint32_t g_c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+extern uint32_t g_c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ];
+
+/** Target definations **/
+#define QDMA_TRQ_SEL_GLBL 0x00000200
+#define QDMA_TRQ_SEL_FMAP 0x00000400
+#define QDMA_TRQ_SEL_IND 0x00000800
+#define QDMA_TRQ_SEL_C2H 0x00000A00
+#define QDMA_TRQ_SEL_H2C 0x00000E00
+#define QDMA_TRQ_SEL_C2H_MM0 0x00001000
+#define QDMA_TRQ_SEL_H2C_MM0 0x00001200
+#define QDMA_TRQ_EXT 0x00002400
+//#define QDMA_TRQ_SEL_QUEUE_PF 0x00001800 /* for old Bitstreams **/
+#define QDMA_TRQ_SEL_QUEUE_PF 0x00018000
+#define QDMA_TRQ_EXT_VF 0x00001000
+#define QDMA_TRQ_SEL_QUEUE_VF 0x00003000
+
+#define QDMA_CONFIG_BLOCK_ID 0x1fd30000UL
+/** Global registers **/
+#define QDMA_GLBL_RING_SZ 0x04
+#define QDMA_GLBL_SCRATCH 0x44
+#define QDMA_GLBL_DSC_CFG 0x50
+#define QDMA_RING_SZ_MSK 0x0000ffff
+#define QDMA_GLBL_DSC_CFG_WB_ACC_MSK 0x00000007
+#define QDMA_GLBL_DSC_CFG_MAX_DSC_FTCH_MSK 0x00000038
+#define QDMA_GLBL_DSC_CFG_WB_ACC_SHFT 0
+#define QDMA_GLBL_DSC_CFG_MAX_DSC_FTCH_SHFT 3
+#define QDMA_GLBL2_PF_BARLITE_INT 0x104
+#define QDMA_GLBL2_PF_VF_BARLITE_INT 0x108
+#define QDMA_GLBL2_PF_BARLITE_EXT 0x10c
+#define QDMA_GLBL2_VF_BARLITE_EXT 0x1018
+#define QDMA_PF_RTL_VER 0x134
+#define QDMA_VF_RTL_VER 0x1014
+#define PF_RTL_VERSION_MASK 0xFF0000
+#define PF_RTL_VERSION_SHIFT 16
+#define PF_VIVADO_RELEASE_ID_MASK 0x0F000000
+#define PF_VIVADO_RELEASE_ID_SHIFT 24
+#define PF_EVEREST_IP_MASK 0x10000000
+#define PF_EVEREST_IP_SHIFT 28
+#define VF_RTL_VERSION_MASK 0xFF
+#define VF_RTL_VERSION_SHIFT 0
+#define VF_VIVADO_RELEASE_ID_MASK 0x0F00
+#define VF_VIVADO_RELEASE_ID_SHIFT 8
+#define VF_EVEREST_IP_MASK 0x1000
+#define VF_EVEREST_IP_SHIFT 12
+
+/** Fmap registers **/
+#define QID_BASE_MSK (0x000007ff)
+#define QID_MAX_MSK (0x003ff800)
+#define QID_MAX_SHIFT_B (11)
+
+/** Queue Indirect programming commands **/
+
+#define QDMA_IND_Q_PRG_OFF (0x4)
+
+#define QDMA_CTXT_CMD_CLR (0)
+#define QDMA_CTXT_CMD_WR (1)
+#define QDMA_CTXT_CMD_RD (2)
+#define QDMA_CTXT_CMD_INV (3)
+
+#define MASK_0BIT (0x0)
+#define MASK_32BIT (0xffffffffUL)
+
+#define QDMA_CTXT_SEL_DESC_SW_C2H (0)
+#define QDMA_CTXT_SEL_DESC_SW_H2C (1)
+#define QDMA_CTXT_SEL_DESC_HW_C2H (2)
+#define QDMA_CTXT_SEL_DESC_HW_H2C (3)
+#define QDMA_CTXT_SEL_CR_C2H (4)
+#define QDMA_CTXT_SEL_CR_H2C (5)
+#define QDMA_CTXT_SEL_DESC_CMPT (6)
+#define QDMA_CTXT_SEL_PFTCH (7)
+#define QDMA_CTXT_SEL_FMAP (12)
+
+#define QID_SHIFT_B (7)
+#define OP_CODE_SHIFT_B (5)
+#define CTXT_SEL_SHIFT_B (1)
+#define BUSY_BIT_MSK (1)
+
+#define QDMA_CIDX_STEP (0x10)
+#define QDMA_PIDX_STEP (0x10)
+
+#define MM_CHAN_SHIFT_B (19)
+#define DESC_RING_SZ_SHIFT_B (12)
+#define ST_H2C_DESC_SZ_SHIFT_B (16)
+#define MM_DESC_SZ_WB_SHIFT_B (29)
+//#define C2H_WB_CTXT_V_SHIFT_B (30)
+
+
+/** C2H target registers **/
+#define QDMA_C2H_CNT_TH_BASE 0x40
+#define QDMA_C2H_BUF_SZ_BASE 0xB0
+#define QDMA_C2H_TIMER_BASE 0x00
+
+#define QDMA_C2H_PFCH_CACHE_DEPTH_OFFSET 0x1E0
+#define QDMA_C2H_CMPT_COAL_BUF_DEPTH_OFFSET 0x1E4
+
+#define QDMA_C2H_PFCH_CFG_OFFSET 0x108
+#define QDMA_C2H_PFCH_CFG_STOP_THRESH_SHIFT (0)
+#define QDMA_C2H_PFCH_CFG_NUM_ENTRIES_PER_Q_SHIFT (8)
+#define QDMA_C2H_PFCH_CFG_MAX_Q_CNT_SHIFT (16)
+#define QDMA_C2H_PFCH_CFG_EVICTION_Q_CNT_SHIFT (25)
+
+#define QDMA_C2H_INT_TIMER_TICK_OFFSET 0x10C
+
+#define QDMA_C2H_CMPT_COAL_CFG_OFFSET 0x150
+#define QDMA_C2H_CMPT_COAL_TIMER_CNT_VAL_SHIFT (2)
+#define QDMA_C2H_CMPT_COAL_TIMER_TICK_VAL_SHIFT (14)
+#define QDMA_C2H_CMPT_COAL_MAX_BUF_SZ_SHIFT (26)
+
+/** H2C target registers **/
+#define QDMA_H2C_DATA_THRESHOLD_OFFSET 0x24 //Base address is QDMA_TRQ_SEL_H2C
+
+/** PF Queue index registers */
+#define QDMA_H2C_PIDX_Q_OFF (0x04)
+#define QDMA_C2H_PIDX_Q_OFF (0x08)
+#define QDMA_SEL_CMPT_CIDX_Q_OFF (0x0c)
+
+
+/** QDMA Target registers **/
+#define QDMA_C2H_MM0_CONTROL 0x00000004
+#define QDMA_H2C_MM0_CONTROL 0x00000004
+#define QDMA_MM_CTRL_START (1 << 0)
+
+/** QDMA Descriptor definations **/
+#define QDMA_DESC_SOP 0x1
+#define QDMA_DESC_EOP 0x1
+#define QDMA_DESC_VALID 0x1
+
+/** Mailbox Register **/
+#define QDMA_MBOX_FSR 0x00
+#define QDMA_MBOX_FCMD 0x04
+#define QDMA_MBOX_TRGT_FN 0x0c
+#define QDMA_MBOX_ACK_BASE 0x20
+#define QDMA_MBOX_IMSG 0x800
+#define QDMA_MBOX_OMSG 0xc00
+
+/** Mailbox commands **/
+#define QDMA_MBOX_CMD_SEND (1)
+#define QDMA_MBOX_CMD_RECV (2)
+
+/* Driver visible Attribute Space 0x100 */
+#define QDMA_REG_GLBL_PF_BARLITE_INT 0x104
+#define PF_BARLITE_INT_3_SHIFT 18
+#define PF_BARLITE_INT_3_MASK 0xFC0000
+#define PF_BARLITE_INT_2_SHIFT 12
+#define PF_BARLITE_INT_2_MASK 0x3F000
+#define PF_BARLITE_INT_1_SHIFT 6
+#define PF_BARLITE_INT_1_MASK 0xFC0
+#define PF_BARLITE_INT_0_SHIFT 0
+#define PF_BARLITE_INT_0_MASK 0x3F
+
+#define QDMA_REG_GLBL_QMAX 0x120
+#define QDMA_REG_GLBL_MISC_CAP 0x134
+#define QDMA_REG_GLBL_MDMA_CHANNEL 0x118
+#define MDMA_CHANNEL_ST_C2H_ENABLED_SHIFT 16
+#define MDMA_CHANNEL_ST_C2H_ENABLED_MASK 0x10000
+#define MDMA_CHANNEL_ST_H2C_ENABLED_SHIFT 17
+#define MDMA_CHANNEL_ST_H2C_ENABLED_MASK 0x20000
+#define MDMA_CHANNEL_MM_C2H_ENABLED_SHIFT 8
+#define MDMA_CHANNEL_MM_C2H_ENABLED_MASK 0x100
+#define MDMA_CHANNEL_MM_H2C_ENABLED_SHIFT 0
+#define MDMA_CHANNEL_MM_H2C_ENABLED_MASK 0x1
+
+/** Completion Descriptor config */
+#define CMPT_STATUS_DESC_EN (1)
+#define CMPT_STATUS_DESC_EN_SHIFT (27)
+#define CMPT_TRIGGER_MODE_SHIFT (24)
+#define CMPT_TIMER_CNT_SHIFT (20)
+#define CMPT_THREHOLD_CNT_SHIFT (16)
+
+/** Prefetch Context config */
+#define PREFETCH_EN (1)
+#define PREFETCH_EN_SHIFT (27)
+#define PREFETCH_BYPASS_SHIFT (0)
+#define BUFF_SIZE_INDEX_SHIFT (1)
+#define VALID_CNTXT (1)
+#define VALID_CNTXT_SHIFT (13)
+
+/** Completion Context config */
+#define CMPT_CNTXT_EN_STAT_DESC (1)
+#define CMPT_CNTXT_EN_STAT_DESC_SHIFT (0)
+#define CMPT_CNTXT_COLOR_BIT (1)
+#define CMPT_CNTXT_COLOR_BIT_SHIFT (27)
+#define CMPT_CNTXT_TRIGGER_MODE_SHIFT (2)
+#define CMPT_CNTXT_TIMER_INDEX_SHIFT (21)
+#define CMPT_CNTXT_THRESHOLD_MODE_SHIFT (17)
+#define CMPT_CNTXT_FUNC_ID_SHIFT (5)
+#define CMPT_CNTXT_RING_SIZE_INDEX_SHIFT (28)
+#define CMPT_CNTXT_DESC_SIZE_8B (0)
+#define CMPT_CNTXT_DESC_SIZE_16B (1)
+#define CMPT_CNTXT_DESC_SIZE_32B (2)
+#define CMPT_CNTXT_DESC_SIZE_64B (3)
+#define CMPT_CNTXT_DESC_SIZE_SHIFT (90 - (2 * 32))
+#define CMPT_CNTXT_CTXT_VALID (1)
+#define CMPT_CNTXT_OVF_CHK_DIS_SHIFT (130 - (4 * 32))
+#define CMPT_CNTXT_CTXT_VALID_SHIFT (124 - (3 * 32))
+#define CMPT_CNTX_FULL_UPDT (1)
+#define CMPT_CNTX_FULL_UPDT_SHIFT (129 - (4 * 32))
+
+/** SOFTWARE DESC CONTEXT */
+#define SW_DESC_CNTXT_WB_EN (1)
+#define SW_DESC_CNTXT_WB_EN_SHIFT_B (52 - (1 * 32))
+#define SW_DESC_CNTXT_8B_BYPASS_DMA (0)
+#define SW_DESC_CNTXT_16B_BYPASS_DMA (1)
+#define SW_DESC_CNTXT_32B_BYPASS_DMA (2)
+#define SW_DESC_CNTXT_64B_BYPASS_DMA (3)
+#define SW_DESC_CNTXT_MEMORY_MAP_DMA (2)
+#define SW_DESC_CNTXT_H2C_STREAM_DMA (1)
+#define SW_DESC_CNTXT_C2H_STREAM_DMA (0)
+#define SW_DESC_CNTXT_BYPASS_SHIFT (50 - (1 * 32))
+#define SW_DESC_CNTXT_DESC_SZ_SHIFT (48 - (1 * 32))
+#define SW_DESC_CNTXT_FUNC_ID_SHIFT (17)
+#define SW_DESC_CNTXT_RING_SIZE_ID_SHIFT (44 - (1 * 32))
+#define SW_DESC_CNTXT_QUEUE_ENABLE (1)
+#define SW_DESC_CNTXT_QUEUE_EN_SHIFT (32 - (1 * 32))
+#define SW_DESC_CNTXT_FETCH_CREDIT_EN (1)
+#define SW_DESC_CNTXT_FETCH_CREDIT_EN_SHIFT (33 - (1 * 32))
+#define SW_DESC_CNTXT_WBI_CHK (1)
+#define SW_DESC_CNTXT_WBI_CHK_SHIFT (34 - (1 * 32))
+#define SW_DESC_CNTXT_WBI_INTERVAL (1)
+#define SW_DESC_CNTXT_WBI_INTERVAL_SHIFT (35 - (1 * 32))
+#define SW_DESC_CNTXT_IS_MM (1)
+#define SW_DESC_CNTXT_IS_MM_SHIFT (63 - (1 * 32))
+
+/** QDMA Global registers **/
+struct __attribute__ ((packed)) qdma_global_regs
+{
+ volatile uint32_t ring_sz[16];
+ volatile uint32_t status[16];
+ volatile uint32_t config[16];
+ volatile uint32_t wb_acc;
+ volatile uint32_t pf_scratch_reg;
+};
+
+#define NUM_CONTEXT_REGS 8
+/** Queue Indirect programming registers **/
+struct __attribute__ ((packed)) queue_ind_prg
+{
+ volatile uint32_t ctxt_data[NUM_CONTEXT_REGS];
+ volatile uint32_t ctxt_mask[NUM_CONTEXT_REGS];
+ volatile uint32_t ctxt_cmd;
+};
+
+/** MM Write-back status structure **/
+struct __attribute__ ((packed)) wb_status
+{
+ volatile uint16_t pidx; /** in C2H WB **/
+ volatile uint16_t cidx; /** Consumer-index **/
+ uint32_t rsvd2; /** Reserved. **/
+};
+
+/** ST C2H Descriptor **/
+struct __attribute__ ((packed)) qdma_c2h_desc
+{
+ volatile uint64_t dst_addr;
+};
+
+#define S_H2C_DESC_F_SOP 1
+#define S_H2C_DESC_F_EOP 2
+
+/* pld_len and flags members are part of custom descriptor format needed
+ * by example design for ST loopback and desc bypass
+ */
+
+/** ST H2C Descriptor **/
+struct __attribute__ ((packed)) qdma_h2c_desc
+{
+ volatile uint16_t cdh_flags;
+ volatile uint16_t pld_len;
+ volatile uint16_t len;
+ volatile uint16_t flags;
+ volatile uint64_t src_addr;
+};
+
+/** MM Descriptor **/
+struct __attribute__ ((packed)) qdma_mm_desc
+{
+ volatile uint64_t src_addr;
+ volatile uint64_t len:28;
+ volatile uint64_t dv:1;
+ volatile uint64_t sop:1;
+ volatile uint64_t eop:1;
+ volatile uint64_t rsvd:33;
+ volatile uint64_t dst_addr;
+ volatile uint64_t rsvd2;
+
+};
+
+/** MBOX STATUS **/
+struct __attribute__ ((packed)) mbox_fsr
+{
+ volatile uint8_t imsg_stat:1;
+ volatile uint8_t omsg_stat:1;
+ volatile uint8_t ack_stat:2;
+ volatile uint16_t curr_src_fn:8;
+ volatile uint32_t rsvd:20;
+};
+
+struct __attribute__ ((packed)) mbox_ack_stat
+{
+ volatile uint32_t ack[8];
+};
+
+struct __attribute__ ((packed)) mbox_imsg
+{
+ volatile uint32_t imsg[32];
+};
+
+struct __attribute__ ((packed)) mbox_omsg
+{
+ volatile uint32_t omsg[32];
+};
+
+/** C2H/H2C Descriptor context structure **/
+
+#define PIDX_MSK (0)
+#define Q_STATUS_MSK (0)
+#define Q_STATUS_EN_MSK (3)
+#define Q_STATUS_RST_MSK (1)
+#define WBI_CHK_MSK (6)
+#define WBI_ACC_EN_MSK (7)
+#define FUNC_ID_MSK (8)
+#define RING_SZ_MSK (16)
+#define DESC_SZ_MSK (16)
+
+#endif /* ifndef __QDMA_REGS_H__ */
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_rxtx.c b/QDMA/DPDK/drivers/net/qdma/qdma_rxtx.c
new file mode 100644
index 0000000000000000000000000000000000000000..3ebd8d68e348097d976826a56737bb733b6f359a
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_rxtx.c
@@ -0,0 +1,814 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_mbuf.h>
+#include <rte_cycles.h>
+#include "qdma.h"
+#include "qdma_regs.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+
+int dma_wb_monitor(void *xq, uint8_t dir, uint16_t expected_count);
+
+/**
+ * Poll the QDMA engine for transfer completion.
+ *
+ * @param txq
+ * Generic pointer to either Rx/Tx queue structure based on the DMA direction.
+ * @param expected_count
+ * expected transfer count.
+ *
+ * @return
+ * Number of packets transferred successfully by the engine.
+ */
+int dma_wb_monitor(void *xq, uint8_t dir, uint16_t expected_count)
+{
+ struct wb_status *wb_status;
+ uint16_t mode, wb_tail;
+ uint32_t i = 0;
+
+ if (dir == DMA_TO_DEVICE) {
+ struct qdma_tx_queue *txq = (struct qdma_tx_queue *)xq;
+ wb_status = txq->wb_status;
+
+ while (i < WB_TIMEOUT) {
+ if (expected_count == wb_status->cidx) {
+ PMD_DRV_LOG(DEBUG, "Poll writeback count "
+ "matches to the expected count :%d",
+ expected_count);
+ return 0;
+ }
+ PMD_DRV_LOG(DEBUG, "poll wait on wb-count:%d and "
+ "expected-count:%d\n",
+ wb_status->cidx, expected_count);
+ i++;
+ }
+ PMD_DRV_LOG(ERR, "DMA Engine write-back monitor "
+ "timeout error occurred\n");
+ return -1;
+ }
+ /* dir == DMA_FROM_DEVICE */
+ struct qdma_rx_queue *rxq = (struct qdma_rx_queue *)xq;
+ wb_status = rxq->wb_status;
+ mode = rxq->st_mode;
+
+ /* Poll the writeback location until timeout Or, the expected
+ * count matches to the writeback count
+ */
+ while (i < WB_TIMEOUT) {
+ if (mode) {
+ wb_tail = (rxq->rx_cmpt_tail + expected_count) %
+ (rxq->nb_rx_cmpt_desc - 1);
+ if (wb_tail == wb_status->pidx) {
+ PMD_DRV_LOG(DEBUG, "ST: Poll cmpt count matches"
+ " to the expected count :%d",
+ expected_count);
+ return 0;
+ }
+ PMD_DRV_LOG(DEBUG, "ST: poll wait on wb-count:%d and"
+ " expected-count:%d\n",
+ wb_status->pidx, expected_count);
+ } else {
+ if (expected_count == wb_status->cidx) {
+ PMD_DRV_LOG(DEBUG, "MM: Poll writeback count "
+ "matches to the expected count"
+ " :%d", expected_count);
+ return 0;
+ }
+ PMD_DRV_LOG(DEBUG, "MM: poll wait on wb-count:%d "
+ "and expected-count:%d\n",
+ wb_status->cidx, expected_count);
+ }
+ i++;
+ }
+ return -1;
+}
+
+uint16_t qdma_recv_pkts_st(struct qdma_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct rte_mbuf *mb;
+ struct rte_mbuf *first_seg = NULL;
+ struct rte_mbuf *last_seg = NULL;
+ struct qdma_c2h_desc *rx_ring_st = NULL;
+ uint32_t count = 0, count_pkts = 0;
+ uint16_t id;
+ struct c2h_cmpt_ring *cmpt_ring;
+ struct wb_status *wb_status;
+ uint32_t pkt_length;
+ uint64_t phys_addr;
+ uint16_t nb_pkts_avail = 0;
+ uint16_t rx_cmpt_tail = 0;
+ uint16_t mbuf_index = 0;
+ uint16_t pkt_len[QDMA_MAX_BURST_SIZE];
+ uint16_t rx_buff_size;
+ uint16_t cmpt_pidx;
+ uint16_t pending_desc;
+#ifdef TEST_64B_DESC_BYPASS
+ int bypass_desc_sz_idx = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+#endif
+
+ id = rxq->rx_tail; /* Descriptor index */
+ PMD_DRV_LOG(DEBUG, "recv start on rx queue-id :%d, on "
+ "tail index:%d number of pkts %d",
+ rxq->queue_id, rxq->rx_tail, nb_pkts);
+
+ wb_status = rxq->wb_status;
+ rx_cmpt_tail = rxq->rx_cmpt_tail;
+ rx_buff_size = rxq->rx_buff_size;
+
+#ifdef TEST_64B_DESC_BYPASS
+ if (unlikely(rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)) {
+ PMD_DRV_LOG(DEBUG, "For RX ST-mode, example"
+ " design doesn't support 64byte descriptor\n");
+ return 0;
+ }
+#endif
+ rx_ring_st = (struct qdma_c2h_desc *)rxq->rx_ring;
+ cmpt_pidx = wb_status->pidx;
+
+ if (rx_cmpt_tail < cmpt_pidx)
+ nb_pkts_avail = cmpt_pidx - rx_cmpt_tail;
+ else if (rx_cmpt_tail > cmpt_pidx)
+ nb_pkts_avail = rxq->nb_rx_cmpt_desc - 1 - rx_cmpt_tail +
+ cmpt_pidx;
+
+ if (nb_pkts_avail == 0) {
+ PMD_DRV_LOG(DEBUG, "%s(): %d: nb_pkts_avail = 0\n",
+ __func__, __LINE__);
+ return 0;
+ }
+
+ if (nb_pkts > QDMA_MAX_BURST_SIZE)
+ nb_pkts = QDMA_MAX_BURST_SIZE;
+
+ if (nb_pkts > nb_pkts_avail)
+ nb_pkts = nb_pkts_avail;
+
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(DEBUG, "%s(): %d: queue id = %d, mbuf_avail_count = %d, "
+ "mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ /* Make sure reads to CMPT ring are synchronized before
+ * accessing the ring
+ */
+ rte_rmb();
+
+ while (count < nb_pkts) {
+ cmpt_ring = (struct c2h_cmpt_ring *)((uint64_t)rxq->cmpt_ring +
+ ((uint64_t)rx_cmpt_tail *
+ rxq->cmpt_desc_len));
+ pkt_length = cmpt_ring->length;
+
+ if (unlikely(!cmpt_ring->desc_used))
+ pkt_length = 0;
+ if (unlikely(rxq->dump_immediate_data)) {
+ int ofd, ret = 0;
+ char fln[50];
+#ifndef TEST_64B_DESC_BYPASS
+ uint16_t i = 0;
+#endif
+ sprintf(fln, "q_%d_%s", rxq->queue_id,
+ "immmediate_data.txt");
+ ofd = open(fln, O_RDWR | O_CREAT | O_APPEND |
+ O_SYNC, 0666);
+ if (ofd < 0) {
+ PMD_DRV_LOG(INFO, "recv on rxq[%d] CMPT, "
+ "unable to create outfile "
+ " to dump immediate data",
+ rxq->queue_id);
+ return 0;
+ }
+#ifdef TEST_64B_DESC_BYPASS
+ ret = write(ofd, cmpt_ring, rxq->cmpt_desc_len);
+#else
+ ret = dprintf(ofd, "%02x",
+ (*((uint8_t *)cmpt_ring) & 0xF0));
+ for (i = 1; i < (rxq->cmpt_desc_len) ; i++)
+ ret = dprintf(ofd, "%02x",
+ *((uint8_t *)cmpt_ring + i));
+#endif
+ if (ret < rxq->cmpt_desc_len)
+ PMD_DRV_LOG(DEBUG, "recv on rxq[%d] CMPT, "
+ "immediate data len: %d, "
+ "written to outfile :%d bytes",
+ rxq->queue_id, rxq->cmpt_desc_len,
+ ret);
+ close(ofd);
+ }
+ pkt_len[count] = pkt_length;
+ rx_cmpt_tail++;
+ if (unlikely(rx_cmpt_tail >= (rxq->nb_rx_cmpt_desc - 1)))
+ rx_cmpt_tail -= (rxq->nb_rx_cmpt_desc - 1);
+ count++;
+ }
+
+ rte_wmb();
+ rxq->rx_cmpt_tail = rx_cmpt_tail;
+ // Update the CPMT CIDX
+ *rxq->rx_cidx = rxq->rx_cmpt_tail;
+
+ if (rxq->status != RTE_ETH_QUEUE_STATE_STARTED) {
+ PMD_DRV_LOG(DEBUG, "%s(): %d: rxq->status = %d\n",
+ __func__, __LINE__, rxq->status);
+ return 0;
+ }
+
+ count = 0;
+ mbuf_index = 0;
+ while (count < nb_pkts) {
+ pkt_length = pkt_len[count];
+
+ if (unlikely(!pkt_length)) {
+ count++;
+ continue;
+ }
+
+ do {
+ mb = rxq->sw_ring[id];
+ rxq->sw_ring[id++] = NULL;
+
+ if (unlikely(id >= (rxq->nb_rx_desc - 1)))
+ id -= (rxq->nb_rx_desc - 1);
+ if (pkt_length > rx_buff_size) {
+ rte_pktmbuf_data_len(mb) = rx_buff_size;
+ pkt_length -= rx_buff_size;
+ } else {
+ rte_pktmbuf_data_len(mb) = pkt_length;
+ pkt_length = 0;
+ }
+ rte_mbuf_refcnt_set(mb, 1);
+
+ if (first_seg == NULL) {
+ first_seg = mb;
+ first_seg->nb_segs = 1;
+ first_seg->pkt_len = pkt_len[count];
+ first_seg->packet_type = 0;
+ first_seg->ol_flags = 0;
+ first_seg->port = rxq->port_id;
+ first_seg->vlan_tci = 0;
+ first_seg->hash.rss = 0;
+ } else {
+ first_seg->nb_segs++;
+ last_seg->next = mb;
+ }
+
+ last_seg = mb;
+ mb->next = NULL;
+ } while (pkt_length);
+ rxq->stats.pkts++;
+ rxq->stats.bytes += pkt_len[count++];
+ rx_pkts[count_pkts++] = first_seg;
+ first_seg = NULL;
+ }
+
+ rxq->rx_tail = id;
+
+ pending_desc = rxq->rx_tail - rxq->c2h_pidx;
+ if (rxq->rx_tail < rxq->c2h_pidx)
+ pending_desc = rxq->nb_rx_desc - 1 + rxq->rx_tail -
+ rxq->c2h_pidx;
+
+ /* Batch the PIDX updates, this minimizes overhead on
+ * descriptor engine
+ */
+ if (pending_desc >= MIN_RX_PIDX_UPDATE_THRESHOLD) {
+ struct rte_mbuf *tmp_sw_ring[pending_desc];
+ /* allocate new buffer */
+ if (rte_mempool_get_bulk(rxq->mb_pool, (void *)tmp_sw_ring,
+ pending_desc) != 0){
+ PMD_DRV_LOG(ERR, "%s(): %d: No MBUFS\n",
+ __func__, __LINE__);
+ return count_pkts;
+ }
+
+ id = rxq->c2h_pidx;
+ for (mbuf_index = 0; mbuf_index < pending_desc; mbuf_index++) {
+ mb = tmp_sw_ring[mbuf_index];
+ /* low 32-bits of phys addr must be 4KB aligned... */
+ phys_addr = (uint64_t)mb->buf_physaddr +
+ RTE_PKTMBUF_HEADROOM;
+
+ /* make it so the data pointer starts there too... */
+ mb->data_off = RTE_PKTMBUF_HEADROOM;
+
+ /* rearm descriptor */
+ rx_ring_st[id].dst_addr = phys_addr;
+ rxq->sw_ring[id++] = mb;
+ if (unlikely(id >= (rxq->nb_rx_desc - 1)))
+ id -= (rxq->nb_rx_desc - 1);
+ }
+
+ PMD_DRV_LOG(DEBUG, "%s(): %d: PIDX Update: queue id = %d, "
+ "pending_desc = %d",
+ __func__, __LINE__, rxq->queue_id,
+ pending_desc);
+
+ /* Make sure writes to the C2H descriptors are
+ * synchronized before updating PIDX
+ */
+ rte_wmb();
+
+ if (rxq->rx_tail == 0)
+ *rxq->rx_pidx = (rxq->nb_rx_desc - 2);
+ else
+ *rxq->rx_pidx = (rxq->rx_tail - 1);
+
+ rxq->c2h_pidx = rxq->rx_tail;
+ }
+
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(DEBUG, "%s(): %d: queue id = %d, mbuf_avail_count = %d,"
+ " mbuf_in_use_count = %d, count_pkts = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool), count_pkts);
+#endif //DUMP_MEMPOOL_USAGE_STATS
+
+ PMD_DRV_LOG(DEBUG, " Recv complete with hw cidx :%d",
+ rxq->wb_status->cidx);
+ PMD_DRV_LOG(DEBUG, " Recv complete with hw pidx :%d\n",
+ rxq->wb_status->pidx);
+
+ return count_pkts;
+}
+
+uint16_t qdma_recv_pkts_mm(struct qdma_rx_queue *rxq, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct rte_mbuf *mb;
+ uint32_t count, id;
+ struct qdma_mm_desc *desc;
+ uint64_t phys_addr;
+ uint32_t len;
+ int ret;
+#ifdef TEST_64B_DESC_BYPASS
+ int bypass_desc_sz_idx = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+#endif
+
+ if (rxq->status != RTE_ETH_QUEUE_STATE_STARTED)
+ return 0;
+
+ id = rxq->rx_tail; /* Descriptor index */
+
+ PMD_DRV_LOG(DEBUG, "recv start on rx queue-id :%d, on tail index:%d\n",
+ rxq->queue_id, rxq->rx_tail);
+
+#ifdef TEST_64B_DESC_BYPASS
+ if (unlikely(rxq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)) {
+ PMD_DRV_LOG(DEBUG, "For MM mode, example design doesn't "
+ "support 64byte descriptor\n");
+ return 0;
+ }
+#endif
+ /* Make 1 less available, otherwise if we allow all descriptors
+ * to be filled,when nb_pkts = nb_tx_desc - 1, pidx will be same
+ * as old pidx and HW will treat this as no new descriptors were added.
+ * Hence, DMA won't happen with new descriptors.
+ */
+ if (nb_pkts > rxq->nb_rx_desc - 2)
+ nb_pkts = rxq->nb_rx_desc - 2;
+
+ for (count = 0; count < nb_pkts; count++) {
+ /* allocate new buffer */
+ if (rte_mempool_get(rxq->mb_pool, (void *)&mb) != 0) {
+ rte_panic("No MBUFS\n");
+ break;
+ }
+
+ /* low 32-bits of phys addr must be 4KB aligned... */
+ phys_addr = (uint64_t)mb->buf_physaddr + RTE_PKTMBUF_HEADROOM;
+
+ /* make it so the data pointer starts there too... */
+ mb->data_off = RTE_PKTMBUF_HEADROOM;
+
+ desc = (struct qdma_mm_desc *)rxq->rx_ring;
+ desc += id;
+
+ desc->src_addr = rxq->ep_addr;
+ desc->dst_addr = phys_addr;
+ desc->dv = QDMA_DESC_VALID;
+ desc->sop = 0;
+ desc->eop = 0;
+ if ((count + 1) == nb_pkts)
+ desc->eop = QDMA_DESC_EOP;
+ if (count == 0)
+ desc->sop = QDMA_DESC_SOP;
+
+ desc->len = (int)rxq->rx_buff_size;
+ len = (int)rxq->rx_buff_size;
+ rte_pktmbuf_pkt_len(mb) = (int)rxq->rx_buff_size;
+
+ rte_mbuf_refcnt_set(mb, 1);
+ mb->packet_type = 0;
+ mb->ol_flags = 0;
+ mb->next = 0;
+ mb->nb_segs = 1;
+ mb->port = rxq->port_id;
+ mb->vlan_tci = 0;
+ mb->hash.rss = 0;
+
+ rx_pkts[count] = mb;
+
+ rxq->ep_addr = (rxq->ep_addr + len) % DMA_BRAM_SIZE;
+ rxq->rx_tail = (rxq->rx_tail + 1) % (rxq->nb_rx_desc - 1);
+ id = rxq->rx_tail;
+ }
+
+ /* Make sure writes to the C2H descriptors are synchronized
+ * before updating PIDX
+ */
+ rte_wmb();
+
+ /* update pidx pointer for MM-mode*/
+ if (count > 0)
+ *rxq->rx_pidx = id;
+
+ ret = dma_wb_monitor(rxq, DMA_FROM_DEVICE, id);
+ if (ret) {//Error
+ PMD_DRV_LOG(ERR, "DMA Engine write-back monitor "
+ "timeout error occurred\n");
+ return 0;
+ }
+ return count;
+}
+/**
+ * DPDK callback for receiving packets in burst.
+ *
+ * @param rx_queue
+ * Generic pointer to Rx queue structure.
+ * @param[out] rx_pkts
+ * Array to store received packets.
+ * @param nb_pkts
+ * Maximum number of packets in array.
+ *
+ * @return
+ * Number of packets successfully received (<= nb_pkts).
+ */
+uint16_t qdma_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct qdma_rx_queue *rxq = rx_queue;
+ uint32_t count;
+
+ if (rxq->st_mode)
+ count = qdma_recv_pkts_st(rxq, rx_pkts, nb_pkts);
+ else
+ count = qdma_recv_pkts_mm(rxq, rx_pkts, nb_pkts);
+
+ return count;
+}
+
+static void reclaim_tx_mbuf(struct qdma_tx_queue *txq, uint16_t cidx)
+{
+ int fl_desc = 0;
+ uint16_t count;
+ int id;
+
+ id = txq->tx_fl_tail;
+ fl_desc = (int)cidx - id;
+ if (fl_desc < 0)
+ fl_desc += (txq->nb_tx_desc - 1);
+
+ for (count = 0; count < fl_desc; count++) {
+ rte_pktmbuf_free(txq->sw_ring[id]);
+ txq->sw_ring[id] = NULL;
+ id++;
+ if (unlikely(id >= (txq->nb_tx_desc - 1)))
+ id -= (txq->nb_tx_desc - 1);
+ }
+ txq->tx_fl_tail = id;
+}
+
+#ifdef TEST_64B_DESC_BYPASS
+static uint16_t qdma_xmit_64B_desc_bypass(struct qdma_tx_queue *txq,
+ struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ uint16_t count, id;
+ uint8_t *tx_ring_st_bypass = NULL;
+ int ofd = -1, ret = 0;
+ char fln[50];
+
+ id = txq->tx_tail;
+
+ for (count = 0; count < nb_pkts; count++) {
+ tx_ring_st_bypass = (uint8_t *)txq->tx_ring;
+ memset(&tx_ring_st_bypass[id * (txq->bypass_desc_sz)],
+ ((id % 255) + 1), txq->bypass_desc_sz);
+
+ sprintf(fln, "q_%d_%s", txq->queue_id,
+ "h2c_desc_data.txt");
+ ofd = open(fln, O_RDWR | O_CREAT | O_APPEND | O_SYNC,
+ 0666);
+ if (ofd < 0) {
+ PMD_DRV_LOG(INFO, " txq[%d] unable to create "
+ "outfile to dump descriptor"
+ " data", txq->queue_id);
+ return 0;
+ }
+ ret = write(ofd, &(tx_ring_st_bypass[id *
+ (txq->bypass_desc_sz)]),
+ txq->bypass_desc_sz);
+ if (ret < txq->bypass_desc_sz)
+ PMD_DRV_LOG(DEBUG, "Txq[%d] descriptor data "
+ "len: %d, written to inputfile"
+ " :%d bytes", txq->queue_id,
+ txq->bypass_desc_sz, ret);
+ close(ofd);
+
+ rte_pktmbuf_free(tx_pkts[count]);
+
+ id++;
+ if (unlikely(id >= (txq->nb_tx_desc - 1)))
+ id -= (txq->nb_tx_desc - 1);
+ }
+
+ /* Make sure writes to the H2C descriptors are synchronized
+ * before updating PIDX
+ */
+ rte_wmb();
+
+ txq->tx_tail = id;
+ *txq->tx_pidx = txq->tx_tail;
+
+ PMD_DRV_LOG(DEBUG, " xmit completed with count:%d\n", count);
+
+ return count;
+}
+#endif
+
+uint16_t qdma_xmit_pkts_st(struct qdma_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ struct rte_mbuf *mb;
+ uint16_t count, id;
+ uint64_t phys_addr;
+ struct qdma_h2c_desc *desc;
+ int avail, in_use;
+ uint16_t pkt_len;
+ uint16_t cidx = 0;
+ struct qdma_h2c_desc *tx_ring_st = NULL;
+#ifdef TEST_64B_DESC_BYPASS
+ int bypass_desc_sz_idx = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+
+ if (unlikely(txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)) {
+ return qdma_xmit_64B_desc_bypass(txq, tx_pkts, nb_pkts);
+ }
+#endif
+
+ id = txq->tx_tail;
+ cidx = txq->wb_status->cidx;
+ PMD_DRV_LOG(DEBUG, "Xmit start on tx queue-id:%d, tail index:%d\n",
+ txq->queue_id, id);
+
+ /* Free transmitted mbufs back to pool */
+ reclaim_tx_mbuf(txq, cidx);
+
+ in_use = (int)txq->tx_tail - cidx;
+ if (in_use < 0)
+ in_use += (txq->nb_tx_desc - 1);
+
+ /* Make 1 less available, otherwise if we allow all descriptors
+ * to be filled, when nb_pkts = nb_tx_desc - 1, pidx will be same
+ * as old pidx and HW will treat this as no new descriptors were added.
+ * Hence, DMA won't happen with new descriptors.
+ */
+ avail = txq->nb_tx_desc - 2 - in_use;
+ if (!avail) {
+ PMD_DRV_LOG(DEBUG, "Tx queue full, in_use = %d", in_use);
+ return 0;
+ }
+
+ if (nb_pkts > avail)
+ nb_pkts = avail;
+
+ tx_ring_st = (struct qdma_h2c_desc *)txq->tx_ring;
+ for (count = 0; count < nb_pkts; count++) {
+ desc = (struct qdma_h2c_desc *)&tx_ring_st[id];
+
+ mb = tx_pkts[count];
+ txq->sw_ring[id] = mb;
+
+ phys_addr = mb->buf_physaddr + mb->data_off;
+
+ if (rte_pktmbuf_pkt_len(mb) !=
+ rte_pktmbuf_data_len(mb)) {
+ PMD_DRV_LOG(ERR, "Scatter buffers are not "
+ "supported");
+ break;
+ }
+ desc->len = rte_pktmbuf_data_len(mb);
+ pkt_len = rte_pktmbuf_data_len(mb);
+
+ desc->pld_len = pkt_len;
+ PMD_DRV_LOG(DEBUG, "xmit number of bytes:%d, count:%d ",
+ pkt_len, count);
+ desc->src_addr = phys_addr;
+ desc->flags = 0;
+ desc->cdh_flags = 0;
+ if (count == 0)
+ desc->flags |= S_H2C_DESC_F_SOP;
+ if ((count + 1) == nb_pkts)
+ desc->flags |= S_H2C_DESC_F_EOP;
+
+ txq->stats.pkts++;
+ txq->stats.bytes += pkt_len;
+
+ id++;
+ if (unlikely(id >= (txq->nb_tx_desc - 1)))
+ id -= (txq->nb_tx_desc - 1);
+ }
+
+ /* Make sure writes to the H2C descriptors are synchronized
+ * before updating PIDX
+ */
+ rte_wmb();
+
+ rte_spinlock_lock(&txq->pidx_update_lock);
+ txq->tx_tail = id;
+ txq->tx_desc_pend += count;
+
+ /* Send PIDX update only if pending desc is more than threshold
+ * Saves frequent Hardware transactions
+ */
+ if (txq->tx_desc_pend >= MIN_TX_PIDX_UPDATE_THRESHOLD) {
+ *txq->tx_pidx = txq->tx_tail;
+ txq->tx_desc_pend = 0;
+ }
+ rte_spinlock_unlock(&txq->pidx_update_lock);
+
+ PMD_DRV_LOG(DEBUG, " xmit completed with count:%d\n", count);
+
+ return count;
+}
+
+uint16_t qdma_xmit_pkts_mm(struct qdma_tx_queue *txq, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ struct rte_mbuf *mb;
+ uint32_t count, id;
+ uint64_t phys_addr;
+ struct qdma_mm_desc *desc;
+ int avail, in_use;
+ int ret;
+ struct qdma_mm_desc *tx_ring = (struct qdma_mm_desc *)txq->tx_ring;
+#ifdef TEST_64B_DESC_BYPASS
+ int bypass_desc_sz_idx = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+#endif
+
+ id = txq->tx_tail;
+ PMD_DRV_LOG(DEBUG, "Xmit start on tx queue-id:%d, tail index:%d\n",
+ txq->queue_id, id);
+
+#ifdef TEST_64B_DESC_BYPASS
+ if (unlikely(txq->en_bypass &&
+ bypass_desc_sz_idx == SW_DESC_CNTXT_64B_BYPASS_DMA)) {
+ PMD_DRV_LOG(DEBUG, "For MM mode, example design doesn't "
+ "support 64B bypass testing\n");
+ return 0;
+ }
+#endif
+ in_use = txq->tx_tail - txq->wb_status->cidx;
+ if (in_use < 0)
+ in_use += (txq->nb_tx_desc - 1);
+
+ /* Make 1 less available, otherwise if we allow all descriptors to be
+ * filled, when nb_pkts = nb_tx_desc - 1, pidx will be same as old pidx
+ * and HW will treat this as no new descriptors were added.
+ * Hence, DMA won't happen with new descriptors.
+ */
+ avail = txq->nb_tx_desc - 2 - in_use;
+ if (!avail) {
+ PMD_DRV_LOG(ERR, "Tx queue full, in_use = %d", in_use);
+ return 0;
+ }
+
+ if (nb_pkts > avail)
+ nb_pkts = avail;
+
+ // Set the xmit descriptors and control bits
+ for (count = 0; count < nb_pkts; count++) {
+ /* Free the n/w buffer associated with the descriptor 'id'
+ * to mpool
+ */
+ rte_pktmbuf_free(txq->sw_ring[id]);
+
+ /* see if there is something to receive */
+ desc = (struct qdma_mm_desc *)&tx_ring[id];
+
+ /* Set the descriptor control bits */
+ desc->dv = QDMA_DESC_VALID;
+ desc->sop = 0;
+ desc->eop = 0;
+ if ((count + 1) == nb_pkts)
+ desc->eop = QDMA_DESC_EOP;
+ if (count == 0)
+ desc->sop = QDMA_DESC_SOP;
+
+ mb = tx_pkts[count];
+ txq->sw_ring[id] = mb;
+
+ phys_addr = mb->buf_physaddr + mb->data_off;
+
+ desc->len = rte_pktmbuf_data_len(mb);
+
+ /*if (desc->len > (DMA_BRAM_SIZE / txq->num_queues))
+ * desc->len = DMA_BRAM_SIZE / txq->num_queues;
+ */
+
+ PMD_DRV_LOG(DEBUG, "xmit number of bytes:%d, count:%d ",
+ desc->len, count);
+ desc->src_addr = phys_addr;
+
+ desc->dst_addr = txq->ep_addr;
+
+ txq->ep_addr = (txq->ep_addr + desc->len) % DMA_BRAM_SIZE;
+ txq->tx_tail = (txq->tx_tail + 1) % (txq->nb_tx_desc - 1);
+ id = txq->tx_tail;
+ }
+
+ /* Make sure writes to the H2C descriptors are synchronized before
+ * updating PIDX
+ */
+ rte_wmb();
+
+ /* update pidx pointer */
+ if (count > 0)
+ *txq->tx_pidx = id;
+
+ ret = dma_wb_monitor(txq, DMA_TO_DEVICE, id);
+ if (ret) {
+ PMD_DRV_LOG(ERR, "DMA Engine write-back monitor "
+ "timeout error occurred\n");
+ return 0;
+ }
+
+ PMD_DRV_LOG(DEBUG, " xmit completed with count:%d", count);
+ return count;
+}
+/**
+ * DPDK callback for transmitting packets in burst.
+ *
+ * @param tx_queue
+ G* Generic pointer to TX queue structure.
+ * @param[in] tx_pkts
+ * Packets to transmit.
+ * @param nb_pkts
+ * Number of packets in array.
+ *
+ * @return
+ * Number of packets successfully transmitted (<= nb_pkts).
+ */
+uint16_t qdma_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ struct qdma_tx_queue *txq = tx_queue;
+ uint16_t count;
+
+ if (txq->status != RTE_ETH_QUEUE_STATE_STARTED)
+ return 0;
+
+ if (txq->st_mode)
+ count = qdma_xmit_pkts_st(txq, tx_pkts, nb_pkts);
+ else
+ count = qdma_xmit_pkts_mm(txq, tx_pkts, nb_pkts);
+
+ return count;
+}
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_user.h b/QDMA/DPDK/drivers/net/qdma/qdma_user.h
new file mode 100644
index 0000000000000000000000000000000000000000..520216c7deb355faad53db87ceeecd268b822c2a
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_user.h
@@ -0,0 +1,77 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __QDMA_USER_H__
+#define __QDMA_USER_H__
+/**
+ * @file
+ * @brief This file contains example design/user logic controlled
+ * data structures
+ * The driver is specific to an example design, if the example design
+ * changes user controlled parameters, this file needs to be modified
+ * appropriately.
+ * Structures for Completion entry, Descriptor bypass can be added here.
+ */
+
+ /**
+ * C2H Completion entry structure
+ * This structure is specific for the example design.
+ * Processing of this ring happens in qdma_rxtx.c.
+ */
+struct __attribute__ ((packed)) c2h_cmpt_ring
+{
+ volatile uint32_t data_frmt:1; /* For 2018.2 IP, this field
+ * determines the Standard or User
+ * format of completion entry
+ */
+ volatile uint32_t color:1; /* This field inverts every time
+ * PIDX wraps the completion ring
+ */
+ volatile uint32_t err:1; /* Indicates that C2H engine
+ * encountered a descriptor
+ * error
+ */
+ volatile uint32_t desc_used:1; /* Indicates that the completion
+ * packet consumes descriptor in
+ * C2H ring
+ */
+ volatile uint32_t length:16; /* Indicates length of the data
+ * packet
+ */
+ volatile uint32_t user_rsv:4; /* Reserved field */
+ volatile uint8_t user_def[]; /* User logic defined data of
+ * length based on CMPT entry
+ * length
+ */
+};
+
+#endif /* ifndef __QDMA_USER_H__ */
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_vf_ethdev.c b/QDMA/DPDK/drivers/net/qdma/qdma_vf_ethdev.c
new file mode 100644
index 0000000000000000000000000000000000000000..198ae20290746514e3e103bb4399016e3fd35ffb
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_vf_ethdev.c
@@ -0,0 +1,968 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
+#include <rte_ethdev_pci.h>
+#include <rte_malloc.h>
+#include <rte_dev.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_cycles.h>
+#include <rte_alarm.h>
+#include <unistd.h>
+#include <string.h>
+#include <linux/pci.h>
+
+#include "qdma.h"
+#include "qdma_regs.h"
+#include "version.h"
+
+/*
+ * The set of PCI devices this driver supports
+ */
+static struct rte_pci_id qdma_vf_pci_id_tbl[] = {
+#define RTE_PCI_DEV_ID_DECL_XNIC(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
+#ifndef PCI_VENDOR_ID_XILINX
+#define PCI_VENDOR_ID_XILINX 0x10ee
+#endif
+
+ /** Gen 1 VF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa011) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa111) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa211) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa311) /* VF on PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa014) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa114) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa214) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa314) /* VF on PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa018) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa118) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa218) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa318) /* VF on PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa01f) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa11f) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa21f) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa31f) /* VF on PF 3 */
+
+ /** Gen 2 VF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa021) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa121) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa221) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa321) /* VF on PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa024) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa124) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa224) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa324) /* VF on PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa028) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa128) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa228) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa328) /* VF on PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa02f) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa12f) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa22f) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa32f) /* VF on PF 3 */
+
+ /** Gen 3 VF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa031) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa131) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa231) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa331) /* VF on PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa034) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa134) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa234) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa334) /* VF on PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa038) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa138) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa238) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa338) /* VF on PF 3 */
+ /** PCIe lane width x16 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa03f) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa13f) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa23f) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa33f) /* VF on PF 3 */
+
+ /** Gen 4 VF */
+ /** PCIe lane width x1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa041) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa141) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa241) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa341) /* VF on PF 3 */
+ /** PCIe lane width x4 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa044) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa144) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa244) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa344) /* VF on PF 3 */
+ /** PCIe lane width x8 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa048) /* VF on PF 0 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa148) /* VF on PF 1 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa248) /* VF on PF 2 */
+ RTE_PCI_DEV_ID_DECL_XNIC(PCI_VENDOR_ID_XILINX, 0xa348) /* VF on PF 3 */
+
+ { .vendor_id = 0, /* sentinel */ },
+};
+
+static int qdma_vf_write_mbx(struct rte_eth_dev *dev,
+ struct qdma_mbox_data *send_msg)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct mbox_omsg *out_msg;
+ struct mbox_fsr *mbx_stat;
+ uint64_t bar_addr, reg_addr;
+ uint16_t i;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ reg_addr = bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_FSR;
+
+ mbx_stat = (struct mbox_fsr *)(reg_addr);
+
+ i = MAILBOX_PROG_POLL_COUNT;
+ while (mbx_stat->omsg_stat && i) {
+ rte_delay_ms(MAILBOX_VF_MSG_DELAY);
+ i--;
+ }
+ if (!i)
+ return -1;
+ out_msg = (struct mbox_omsg *)(bar_addr + QDMA_TRQ_EXT_VF +
+ QDMA_MBOX_OMSG);
+
+ qdma_write_reg((uint64_t)&out_msg->omsg[0],
+ rte_cpu_to_le_32(send_msg->err << 24 |
+ send_msg->filler << 20 |
+ send_msg->debug_funid << 8 |
+ send_msg->opcode));
+ for (i = 1; i < 32; i++)
+ qdma_write_reg((uint64_t)&out_msg->omsg[i],
+ rte_cpu_to_le_32(send_msg->data[i - 1]));
+
+ qdma_write_reg((uint64_t)(bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_FCMD),
+ QDMA_MBOX_CMD_SEND);
+
+ return 0;
+}
+
+static int check_outmsg_status(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct mbox_fsr *mbx_stat;
+ uint64_t bar_addr, reg_addr;
+ uint16_t i;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ reg_addr = bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_FSR;
+
+ mbx_stat = (struct mbox_fsr *)(reg_addr);
+
+ i = MAILBOX_PROG_POLL_COUNT;
+ while (mbx_stat->omsg_stat && i) {
+ rte_delay_ms(MAILBOX_VF_MSG_DELAY);
+ i--;
+ }
+ if (!i) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) did not receive ack from PF",
+ qdma_dev->pf);
+ return -1;
+ }
+ return 0;
+}
+
+static int qdma_vf_dev_start(struct rte_eth_dev *dev)
+{
+ struct qdma_tx_queue *txq;
+ struct qdma_rx_queue *rxq;
+ uint32_t qid;
+ int err;
+
+ PMD_DRV_LOG(INFO, "qdma_dev_start: Starting\n");
+
+ /* prepare descriptor rings for operation */
+ for (qid = 0; qid < dev->data->nb_tx_queues; qid++) {
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+
+ /*Deferred Queues should not start with dev_start*/
+ if (!txq->tx_deferred_start) {
+ err = qdma_vf_dev_tx_queue_start(dev, qid);
+ if (err != 0)
+ return err;
+ }
+ }
+
+ for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+
+ /*Deferred Queues should not start with dev_start*/
+ if (!rxq->rx_deferred_start) {
+ err = qdma_vf_dev_rx_queue_start(dev, qid);
+ if (err != 0)
+ return err;
+ }
+ }
+ return 0;
+}
+
+static int qdma_pf_bye(struct rte_eth_dev *dev)
+{
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) received BYE message from PF",
+ qdma_dev->pf);
+#endif
+ _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_VF_MBOX, NULL, NULL);
+ return 0;
+}
+
+static void qdma_read_pf_msg(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t bar_addr, reg_addr;
+ struct qdma_mbox_data recv_msg;
+ struct mbox_imsg *in_msg;
+ int32_t retval = -1;
+ uint32_t reg_val;
+ int i;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ reg_addr = bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_IMSG;
+
+ in_msg = (struct mbox_imsg *)(reg_addr);
+
+ reg_val = qdma_read_reg((uint64_t)&in_msg->imsg[0]);
+
+ recv_msg.opcode = reg_val & 0xff;
+ recv_msg.debug_funid = (reg_val >> 8) & 0xfff;
+ recv_msg.filler = (reg_val >> 20) & 0xf;
+ recv_msg.err = (reg_val >> 24) & 0xff;
+
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) received mbox message from PF %d",
+ qdma_dev->pf, recv_msg.debug_funid);
+ for (i = 1; i < 32; i++)
+ recv_msg.data[i - 1] = qdma_read_reg((uint64_t)&
+ in_msg->imsg[i]);
+
+ switch (recv_msg.opcode) {
+ case QDMA_MBOX_OPCD_BYE:
+ retval = qdma_pf_bye(dev);
+ break;
+ }
+ /* once the msg is handled ACK it */
+ if (!retval)
+ qdma_write_reg((bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_FCMD),
+ QDMA_MBOX_CMD_RECV);
+}
+
+static void qdma_check_pf_msg(void *arg)
+{
+ struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ uint64_t bar_addr, reg_addr;
+ struct mbox_fsr *mbx_stat;
+
+ if (!qdma_dev)
+ return;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ reg_addr = bar_addr + QDMA_TRQ_EXT_VF + QDMA_MBOX_FSR;
+ mbx_stat = (struct mbox_fsr *)(reg_addr);
+
+ if (mbx_stat->imsg_stat == 1)
+ qdma_read_pf_msg(dev);
+ rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_check_pf_msg, arg);
+}
+
+static int qdma_vf_dev_link_update(struct rte_eth_dev *dev,
+ __rte_unused int wait_to_complete)
+{
+ dev->data->dev_link.link_status = ETH_LINK_UP;
+ dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
+ dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
+
+ PMD_DRV_LOG(INFO, "Link update done\n");
+
+ return 0;
+}
+
+static void qdma_vf_dev_infos_get(__rte_unused struct rte_eth_dev *dev,
+ struct rte_eth_dev_info *dev_info)
+{
+ dev_info->max_rx_queues = QDMA_QUEUES_NUM_MAX;
+ dev_info->max_tx_queues = QDMA_QUEUES_NUM_MAX;
+ dev_info->min_rx_bufsize = QDMA_MIN_RXBUFF_SIZE;
+ dev_info->max_rx_pktlen = DMA_BRAM_SIZE;
+ dev_info->max_mac_addrs = 1;
+}
+
+static void qdma_vf_dev_stop(struct rte_eth_dev *dev)
+{
+#ifdef RTE_LIBRTE_QDMA_DEBUG_DRIVER
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+#endif
+ uint32_t qid;
+
+ /* reset driver's internal queue structures to default values */
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Stop H2C & C2H queues", qdma_dev->pf);
+ for (qid = 0; qid < dev->data->nb_tx_queues; qid++)
+ qdma_vf_dev_tx_queue_stop(dev, qid);
+ for (qid = 0; qid < dev->data->nb_rx_queues; qid++)
+ qdma_vf_dev_rx_queue_stop(dev, qid);
+}
+
+static void qdma_vf_dev_close(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_rx_queue *rxq;
+ struct qdma_mbox_data send_msg;
+ int ret = 0;
+ uint32_t qid;
+
+ PMD_DRV_LOG(INFO, "Closing all queues\n");
+
+ /* send MBOX Bye MSG for both TX and RX*/
+
+ memset(&send_msg, 0, sizeof(send_msg));
+ send_msg.opcode = QDMA_MBOX_OPCD_BYE;
+ send_msg.debug_funid = qdma_dev->pf;
+
+ ret = qdma_vf_write_mbx(dev, &send_msg);
+ if (ret != 0)
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) Failed sending QBYE MSG",
+ qdma_dev->pf);
+ ret = check_outmsg_status(dev);
+ if (ret != 0)
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) did not receive ack from PF",
+ qdma_dev->pf);
+
+ /* iterate over rx queues */
+ for (qid = 0; qid < dev->data->nb_rx_queues; ++qid) {
+ rxq = dev->data->rx_queues[qid];
+ if (rxq != NULL) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Remove C2H queue: %d",
+ qdma_dev->pf, qid);
+
+ if (rxq->sw_ring)
+ rte_free(rxq->sw_ring);
+//#ifdef QDMA_STREAM
+ if (rxq->st_mode) { /** if ST-mode **/
+ if (rxq->rx_cmpt_mz)
+ rte_memzone_free(rxq->rx_cmpt_mz);
+ }
+//#endif
+ if (rxq->rx_mz)
+ rte_memzone_free(rxq->rx_mz);
+ rte_free(rxq);
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) C2H queue %d removed",
+ qdma_dev->pf, qid);
+ }
+ }
+ /* iterate over tx queues */
+ for (qid = 0; qid < dev->data->nb_tx_queues; ++qid) {
+ txq = dev->data->tx_queues[qid];
+ if (txq != NULL) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Remove H2C queue: %d",
+ qdma_dev->pf, qid);
+
+ if (txq->sw_ring)
+ rte_free(txq->sw_ring);
+ if (txq->tx_mz)
+ rte_memzone_free(txq->tx_mz);
+ rte_free(txq);
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) H2C queue %d removed",
+ qdma_dev->pf, qid);
+ }
+ }
+}
+
+static int qdma_vf_dev_configure(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_data send_msg;
+ int32_t ret = 0;
+ uint32_t qid = 0;
+
+ /** FMAP configuration **/
+ qdma_dev->qsets_en = RTE_MAX(dev->data->nb_rx_queues,
+ dev->data->nb_tx_queues);
+
+ if (qdma_dev->queue_base + qdma_dev->qsets_en > qdma_dev->qsets_max) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Error: Number of Queues to be "
+ "configured are greater than the queues "
+ "supported by the hardware\n", qdma_dev->pf);
+ qdma_dev->qsets_en = 0;
+ return -1;
+ }
+
+ qdma_dev->q_info = rte_zmalloc("qinfo", sizeof(struct queue_info) *
+ qdma_dev->qsets_en, 0);
+ if (qdma_dev->q_info == NULL) {
+ PMD_DRV_LOG(INFO, "VF-%d fail to allocate queue info memory\n",
+ qdma_dev->pf);
+ return (-ENOMEM);
+ }
+
+ /* Initialize queue_modes to all 1's ( i.e. Streaming) */
+ for (qid = 0 ; qid < qdma_dev->qsets_en; qid++)
+ qdma_dev->q_info[qid].queue_mode = STREAMING_MODE;
+
+ for (qid = 0 ; qid < dev->data->nb_rx_queues; qid++) {
+ qdma_dev->q_info[qid].cmpt_desc_sz = qdma_dev->cmpt_desc_len;
+ qdma_dev->q_info[qid].rx_bypass_mode =
+ qdma_dev->c2h_bypass_mode;
+ }
+
+ for (qid = 0 ; qid < dev->data->nb_tx_queues; qid++)
+ qdma_dev->q_info[qid].tx_bypass_mode =
+ qdma_dev->h2c_bypass_mode;
+
+ memset(&send_msg, 0, sizeof(send_msg));
+
+ /* send hi msg to PF with queue-base and queue-max*/
+ send_msg.opcode = QDMA_MBOX_OPCD_HI;
+ send_msg.debug_funid = qdma_dev->pf;
+
+ send_msg.data[0] = qdma_dev->queue_base;
+ send_msg.data[1] = qdma_dev->qsets_en;
+
+ ret = qdma_vf_write_mbx(dev, &send_msg);
+ if (ret < 0) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Failed sending HI MSG",
+ qdma_dev->pf);
+ return ret;
+ }
+ ret = check_outmsg_status(dev);
+
+ return ret;
+}
+
+int qdma_vf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_mbox_data send_msg;
+ uint32_t queue_base = qdma_dev->queue_base;
+ uint64_t phys_addr;
+ int err;
+
+ memset(&send_msg, 0, sizeof(send_msg));
+ send_msg.opcode = QDMA_MBOX_OPCD_QADD;
+ send_msg.debug_funid = qdma_dev->pf;
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+ qdma_reset_tx_queue(txq);
+
+ phys_addr = (uint64_t)txq->tx_mz->phys_addr;
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Phys-addr for H2C queue-id:%d: "
+ "is Lo:0x%lx, Hi:0x%lx\n",
+ qdma_dev->pf, qid,
+ rte_cpu_to_le_32(phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(phys_addr >> 32));
+
+ /* send MBOX msg QADD for H2C*/
+ send_msg.data[0] = (qid + queue_base);
+ send_msg.data[1] = txq->st_mode;
+ send_msg.data[2] = 0;
+ send_msg.data[3] = txq->ringszidx;
+ send_msg.data[4] = 0;
+ send_msg.data[5] = 0;
+ send_msg.data[6] = 0;
+ send_msg.data[7] = 0;
+ send_msg.data[8] = rte_cpu_to_le_32(phys_addr & MASK_32BIT);
+ send_msg.data[9] = rte_cpu_to_le_32(phys_addr >> 32);
+ send_msg.data[15] = txq->en_bypass;
+ send_msg.data[16] = qmda_get_desc_sz_idx(txq->bypass_desc_sz);
+
+ err = qdma_vf_write_mbx(dev, &send_msg);
+ if (err != 0) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Failed sending H2C QADD MSG "
+ "for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+ err = check_outmsg_status(dev);
+ if (err != 0)
+ return err;
+
+ qdma_start_tx_queue(txq);
+
+ dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
+ txq->status = RTE_ETH_QUEUE_STATE_STARTED;
+
+ return 0;
+}
+
+int qdma_vf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ struct qdma_mbox_data send_msg;
+ uint32_t queue_base = qdma_dev->queue_base;
+ uint64_t phys_addr, cmpt_phys_addr;
+ uint8_t en_pfetch, cmpt_desc_fmt;
+ int err;
+
+ memset(&send_msg, 0, sizeof(send_msg));
+ send_msg.opcode = QDMA_MBOX_OPCD_QADD;
+ send_msg.debug_funid = qdma_dev->pf;
+
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+ qdma_reset_rx_queue(rxq);
+
+ en_pfetch = (rxq->en_prefetch) ? 1 : 0;
+ switch (rxq->cmpt_desc_len) {
+ case CMPT_DESC_LEN_8B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_8B;
+ break;
+ case CMPT_DESC_LEN_16B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_16B;
+ break;
+ case CMPT_DESC_LEN_32B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_32B;
+ break;
+ case CMPT_DESC_LEN_64B:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_64B;
+ break;
+ default:
+ cmpt_desc_fmt = CMPT_CNTXT_DESC_SIZE_8B;
+ break;
+ }
+
+ phys_addr = (uint64_t)rxq->rx_mz->phys_addr;
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Phys-addr for C2H queue-id:%d: "
+ "is Lo:0x%lx, Hi:0x%lx\n",
+ qdma_dev->pf, qid,
+ rte_cpu_to_le_32(phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(phys_addr >> 32));
+
+ if (rxq->st_mode)
+ cmpt_phys_addr = (uint64_t)rxq->rx_cmpt_mz->phys_addr;
+ else
+ cmpt_phys_addr = 0;
+
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) C2H Completion queue Phys-addr "
+ "for queue-id:%d: is Lo:0x%lx, Hi:0x%lx\n",
+ qdma_dev->pf, qid,
+ rte_cpu_to_le_32(cmpt_phys_addr & MASK_32BIT),
+ rte_cpu_to_le_32(cmpt_phys_addr >> 32));
+ err = qdma_start_rx_queue(rxq);
+ if (err != 0)
+ return err;
+
+ /* Send Mbox Msg QADD for C2H */
+ send_msg.data[0] = (qid + queue_base);
+ send_msg.data[1] = rxq->st_mode;
+ send_msg.data[2] = 1;
+ send_msg.data[3] = rxq->ringszidx;
+ send_msg.data[4] = rxq->buffszidx;
+ send_msg.data[5] = rxq->threshidx;
+ send_msg.data[6] = rxq->timeridx;
+ send_msg.data[7] = rxq->triggermode;
+ send_msg.data[8] = rte_cpu_to_le_32(phys_addr & MASK_32BIT);
+ send_msg.data[9] = rte_cpu_to_le_32(phys_addr >> 32);
+ send_msg.data[10] = rte_cpu_to_le_32(cmpt_phys_addr & MASK_32BIT);
+ send_msg.data[11] = rte_cpu_to_le_32(cmpt_phys_addr >> 32);
+ send_msg.data[12] = rxq->cmpt_ringszidx;
+ send_msg.data[13] = en_pfetch;
+ send_msg.data[14] = cmpt_desc_fmt;
+ send_msg.data[15] = rxq->en_bypass;
+ send_msg.data[16] = qmda_get_desc_sz_idx(rxq->bypass_desc_sz);
+ send_msg.data[17] = rxq->en_bypass_prefetch;
+ send_msg.data[19] = rxq->dis_overflow_check;
+
+ err = qdma_vf_write_mbx(dev, &send_msg);
+ if (err != 0) {
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) Failed sending C2H QADD MSG "
+ "for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+ err = check_outmsg_status(dev);
+ if (err != 0)
+ return err;
+
+ if (rxq->st_mode) {
+ *rxq->rx_cidx = (0x09000000) | 0x0;
+ *rxq->rx_pidx = (rxq->nb_rx_desc - 2);
+ }
+
+ dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
+ rxq->status = RTE_ETH_QUEUE_STATE_STARTED;
+ return 0;
+}
+
+int qdma_vf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_rx_queue *rxq;
+ struct qdma_mbox_data send_msg;
+ uint32_t queue_base = qdma_dev->queue_base;
+ int i = 0, err = 0, cnt = 0;
+
+ rxq = (struct qdma_rx_queue *)dev->data->rx_queues[qid];
+
+ rxq->status = RTE_ETH_QUEUE_STATE_STOPPED;
+
+ /* Wait for queue to recv all packets. */
+ if (rxq->st_mode) { /** ST-mode **/
+ while (rxq->wb_status->pidx != rxq->rx_cmpt_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+ } else { /* MM mode */
+ while (rxq->wb_status->cidx != rxq->rx_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+ }
+
+
+ memset(&send_msg, 0, sizeof(send_msg));
+ send_msg.opcode = QDMA_MBOX_OPCD_QDEL;
+ send_msg.debug_funid = qdma_dev->pf;
+ send_msg.data[0] = (qid + queue_base);
+ send_msg.data[1] = DMA_FROM_DEVICE;
+
+ err = qdma_vf_write_mbx(dev, &send_msg);
+ if (err != 0) {
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) Failed sending RX QDEL MSG "
+ "for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+
+ err = check_outmsg_status(dev);
+ if (err != 0) {
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) did not receive ack from PF "
+ "for RX QDEL MSG for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+
+ if (rxq->st_mode) { /** ST-mode **/
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: queue id = %d, mbuf_avail_count = "
+ "%d, mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+
+ for (i = 0; i < rxq->nb_rx_desc - 1; i++) {
+ rte_pktmbuf_free(rxq->sw_ring[i]);
+ rxq->sw_ring[i] = NULL;
+ }
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ PMD_DRV_LOG(INFO, "%s(): %d: queue id = %d, mbuf_avail_count = "
+ "%d, mbuf_in_use_count = %d",
+ __func__, __LINE__, rxq->queue_id,
+ rte_mempool_avail_count(rxq->mb_pool),
+ rte_mempool_in_use_count(rxq->mb_pool));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ }
+
+ qdma_reset_rx_queue(rxq);
+ dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
+ return 0;
+}
+
+
+int qdma_vf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_tx_queue *txq;
+ struct qdma_mbox_data send_msg;
+ uint32_t queue_base = qdma_dev->queue_base;
+ int err = 0;
+ int i = 0, cnt = 0;
+
+ txq = (struct qdma_tx_queue *)dev->data->tx_queues[qid];
+
+ txq->status = RTE_ETH_QUEUE_STATE_STOPPED;
+ /* Wait for TXQ to send out all packets. */
+ while (txq->wb_status->cidx != txq->tx_tail) {
+ usleep(10);
+ if (cnt++ > 10000)
+ break;
+ }
+
+ memset(&send_msg, 0, sizeof(send_msg));
+ send_msg.opcode = QDMA_MBOX_OPCD_QDEL;
+ send_msg.debug_funid = qdma_dev->pf;
+ send_msg.data[0] = (qid + queue_base);
+ send_msg.data[1] = DMA_TO_DEVICE;
+
+ err = qdma_vf_write_mbx(dev, &send_msg);
+ if (err != 0) {
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) Failed sending TX QDEL MSG "
+ "for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+
+ err = check_outmsg_status(dev);
+ if (err != 0) {
+ RTE_LOG(ERR, PMD, "VF-%d(DEVFN) did not receive ack from PF "
+ "for TX QDEL MSG for QID:%d", qdma_dev->pf, qid);
+ return err;
+ }
+
+ qdma_reset_tx_queue(txq);
+
+ /* Free mbufs if any pending in the ring */
+ for (i = 0; i < txq->nb_tx_desc - 1; i++) {
+ rte_pktmbuf_free(txq->sw_ring[i]);
+ txq->sw_ring[i] = NULL;
+ }
+
+ dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
+ return 0;
+}
+
+static struct eth_dev_ops qdma_vf_eth_dev_ops = {
+ .dev_configure = qdma_vf_dev_configure,
+ .dev_infos_get = qdma_vf_dev_infos_get,
+ .dev_start = qdma_vf_dev_start,
+ .dev_stop = qdma_vf_dev_stop,
+ .dev_close = qdma_vf_dev_close,
+ .link_update = qdma_vf_dev_link_update,
+ .rx_queue_setup = qdma_dev_rx_queue_setup,
+ .tx_queue_setup = qdma_dev_tx_queue_setup,
+ .rx_queue_release = qdma_dev_rx_queue_release,
+ .tx_queue_release = qdma_dev_tx_queue_release,
+ .rx_queue_start = qdma_vf_dev_rx_queue_start,
+ .rx_queue_stop = qdma_vf_dev_rx_queue_stop,
+ .tx_queue_start = qdma_vf_dev_tx_queue_start,
+ .tx_queue_stop = qdma_vf_dev_tx_queue_stop,
+};
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function creates an Ethernet device for each port of a given
+ * PCI device.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int eth_qdma_vf_dev_init(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *dma_priv;
+ uint8_t *baseaddr;
+ uint64_t bar_len;
+ int i, idx;
+ static bool once = true;
+ struct rte_pci_device *pci_dev;
+
+ /* sanity checks */
+ if (dev == NULL)
+ return -EINVAL;
+ if (dev->data == NULL)
+ return -EINVAL;
+ if (dev->data->dev_private == NULL)
+ return -EINVAL;
+
+ pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ if (pci_dev == NULL)
+ return -EINVAL;
+
+ /* for secondary processes, we don't initialise any further as primary
+ * has already done this work.
+ */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ dev->dev_ops = &qdma_vf_eth_dev_ops;
+ return 0;
+ }
+
+ if (once) {
+ RTE_LOG(INFO, PMD, "QDMA PMD VERSION: %s\n", QDMA_PMD_VERSION);
+ once = false;
+ }
+
+ /* allocate space for a single Ethernet MAC address */
+ dev->data->mac_addrs = rte_zmalloc("qdma_vf", ETHER_ADDR_LEN * 1, 0);
+ if (dev->data->mac_addrs == NULL)
+ return -ENOMEM;
+
+ /* Copy some dummy Ethernet MAC address for XDMA device
+ * This will change in real NIC device...
+ */
+ for (i = 0; i < ETHER_ADDR_LEN; ++i)
+ dev->data->mac_addrs[0].addr_bytes[i] = 0x15 + i;
+
+ /* Init system & device */
+ dma_priv = (struct qdma_pci_dev *)dev->data->dev_private;
+ dma_priv->pf = PCI_DEVFN(pci_dev->addr.devid, pci_dev->addr.function);
+ dma_priv->is_vf = 1;
+
+ dma_priv->qsets_max = QDMA_QUEUES_NUM_MAX;
+ dma_priv->queue_base = DEFAULT_QUEUE_BASE;
+ dma_priv->timer_count = DEFAULT_TIMER_CNT_TRIG_MODE_TIMER;
+
+ /* Setting default Mode to TRIG_MODE_USER_TIMER_COUNT */
+ dma_priv->trigger_mode = TRIG_MODE_USER_TIMER_COUNT;
+ if (dma_priv->trigger_mode == TRIG_MODE_USER_TIMER_COUNT)
+ dma_priv->timer_count = DEFAULT_TIMER_CNT_TRIG_MODE_COUNT_TIMER;
+
+ /* !! FIXME default to enabled for everything.
+ *
+ * Currently the registers are not available for VFs, so
+ * setting them as enabled.
+ */
+ dma_priv->st_mode_en = 1;
+ dma_priv->mm_mode_en = 1;
+
+ dma_priv->en_desc_prefetch = 0; //Keep prefetch default to 0
+ dma_priv->cmpt_desc_len = DEFAULT_QDMA_CMPT_DESC_LEN;
+ dma_priv->c2h_bypass_mode = C2H_BYPASS_MODE_NONE;
+ dma_priv->h2c_bypass_mode = 0;
+
+ dev->dev_ops = &qdma_vf_eth_dev_ops;
+ dev->rx_pkt_burst = &qdma_recv_pkts;
+ dev->tx_pkt_burst = &qdma_xmit_pkts;
+
+ dma_priv->config_bar_idx = DEFAULT_VF_CONFIG_BAR;
+ dma_priv->bypass_bar_idx = BAR_ID_INVALID;
+ dma_priv->user_bar_idx = BAR_ID_INVALID;
+
+ if (qdma_check_kvargs(dev->device->devargs, dma_priv)) {
+ PMD_DRV_LOG(INFO, "devargs failed\n");
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ idx = qdma_identify_bars(dev);
+ if (idx < 0) {
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ PMD_DRV_LOG(INFO, "VF-%d(DEVFN) QDMA device driver probe:",
+ dma_priv->pf);
+
+ idx = qdma_get_hw_version(dev);
+ if (idx < 0) {
+ rte_free(dev->data->mac_addrs);
+ return -EINVAL;
+ }
+
+ baseaddr = (uint8_t *)
+ pci_dev->mem_resource[dma_priv->config_bar_idx].addr;
+ bar_len = pci_dev->mem_resource[dma_priv->config_bar_idx].len;
+
+ dma_priv->bar_addr[dma_priv->config_bar_idx] = baseaddr;
+ dma_priv->bar_len[dma_priv->config_bar_idx] = bar_len;
+
+ if (dma_priv->user_bar_idx >= 0) {
+ baseaddr = (uint8_t *)
+ pci_dev->mem_resource[dma_priv->user_bar_idx].addr;
+ bar_len = pci_dev->mem_resource[dma_priv->user_bar_idx].len;
+ dma_priv->bar_addr[dma_priv->user_bar_idx] = baseaddr;
+ dma_priv->bar_len[dma_priv->user_bar_idx] = bar_len;
+ }
+ rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_check_pf_msg, (void *)dev);
+ return 0;
+}
+
+/**
+ * DPDK callback to deregister PCI device.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, negative errno value on failure.
+ */
+static int eth_qdma_vf_dev_uninit(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+ /* only uninitialize in the primary process */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return -EPERM;
+ /*Cancel pending polls*/
+ rte_eal_alarm_cancel(qdma_check_pf_msg, (void *)dev);
+ dev->dev_ops = NULL;
+ dev->rx_pkt_burst = NULL;
+ dev->tx_pkt_burst = NULL;
+ dev->data->nb_rx_queues = 0;
+ dev->data->nb_tx_queues = 0;
+
+ if (dev->data->mac_addrs != NULL) {
+ rte_free(dev->data->mac_addrs);
+ dev->data->mac_addrs = NULL;
+ }
+
+ if (qdma_dev->q_info != NULL) {
+ rte_free(qdma_dev->q_info);
+ qdma_dev->q_info = NULL;
+ }
+
+ return 0;
+}
+
+static int eth_qdma_vf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+ struct rte_pci_device *pci_dev)
+{
+ return rte_eth_dev_pci_generic_probe(pci_dev,
+ sizeof(struct qdma_pci_dev),
+ eth_qdma_vf_dev_init);
+}
+
+/* Detach a ethdev interface */
+static int eth_qdma_vf_pci_remove(struct rte_pci_device *pci_dev)
+{
+ return rte_eth_dev_pci_generic_remove(pci_dev, eth_qdma_vf_dev_uninit);
+}
+
+static struct rte_pci_driver rte_qdma_vf_pmd = {
+ .id_table = qdma_vf_pci_id_tbl,
+ .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
+ .probe = eth_qdma_vf_pci_probe,
+ .remove = eth_qdma_vf_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(net_qdma_vf, rte_qdma_vf_pmd);
+RTE_PMD_REGISTER_PCI_TABLE(net_qdma_vf, qdma_vf_pci_id_tbl);
diff --git a/QDMA/DPDK/drivers/net/qdma/qdma_xdebug.c b/QDMA/DPDK/drivers/net/qdma/qdma_xdebug.c
new file mode 100644
index 0000000000000000000000000000000000000000..4db917f658476788d725062cd9327239b3ee8050
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/qdma_xdebug.c
@@ -0,0 +1,1113 @@
+/*
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/fcntl.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
+#include <rte_ethdev_pci.h>
+#include <rte_malloc.h>
+#include <rte_dev.h>
+#include <rte_pci.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_cycles.h>
+#include <unistd.h>
+#include <string.h>
+#include <rte_hexdump.h>
+
+#include "qdma.h"
+#include "qdma_export.h"
+#include "qdma_regs.h"
+
+union __attribute__ ((packed)) h2c_c2h_ctxt
+{
+ struct __attribute__ ((packed)) ctxt_data
+ {
+ uint32_t data[5];
+ } c_data;
+ struct __attribute__ ((packed)) ctxt_fields
+ {
+ uint16_t pidx;
+
+ uint16_t irq_arm:1;
+ uint16_t fnc_id:8;
+ uint16_t rsv0:7;
+
+ uint16_t qen:1;
+ uint16_t fcrd_en:1;
+ uint16_t wbi_chk:1;
+ uint16_t wbi_intvl_en:1;
+ uint16_t at:1;
+ uint16_t fetch_max:3;
+ uint16_t rsv1:4;
+ uint16_t rng_sz:4;
+
+ uint16_t dsc_sz:2;
+ uint16_t byp:1;
+ uint16_t mm_chn:1;
+ uint16_t wbk_en:1;
+ uint16_t irq_en:1;
+ uint16_t port_id:3;
+ uint16_t irq_no_last:1;
+ uint16_t err:2;
+ uint16_t err_wb_sent:1;
+ uint16_t irq_req:1;
+ uint16_t mrkr_dis:1;
+ uint16_t is_mm:1;
+
+ uint64_t dsc_base;
+
+ uint16_t vec:10;
+ uint16_t int_aggr:1;
+ } c_fields;
+};
+
+union __attribute__ ((packed)) c2h_cmpt_ctxt
+{
+ struct __attribute__ ((packed)) c2h_cmpt_data
+ {
+ uint32_t data[5];
+ } c_data;
+ struct __attribute__ ((packed)) c2h_cmpt_fields
+ {
+ uint32_t en_stat_desc:1;
+ uint32_t en_int:1;
+ uint32_t trig_mode:3;
+ uint32_t fnc_id:8;
+ uint32_t rsv0:4;
+ uint32_t count_idx:4;
+ uint32_t timer_idx:4;
+ uint32_t int_st:2;
+ uint32_t color:1;
+ uint32_t qsize_idx:4;
+
+ uint64_t rsv1:6;
+ uint64_t baddr:52;
+ uint64_t desc_sz:2;
+ uint64_t pidx_l:4;
+
+ uint32_t pidx_h:12;
+ uint32_t cidx:16;
+ uint32_t valid:1;
+ uint32_t err:2;
+ uint32_t user_trig_pend:1;
+
+ uint32_t timer_running:1;
+ uint32_t full_upd:1;
+ uint32_t ovf_chk_dis:1;
+ uint32_t at:1;
+ uint32_t vec:11;
+ uint32_t int_aggr:1;
+ uint32_t rsv2:17;
+ } c_fields;
+};
+
+union __attribute__ ((packed)) h2c_c2h_hw_ctxt
+{
+ struct __attribute__ ((packed)) hw_ctxt_data
+ {
+ uint32_t data[2];
+ } c_data;
+ struct __attribute__ ((packed)) hw_ctxt_fields
+ {
+ uint16_t cidx;
+ uint16_t crd_use;
+
+ uint16_t rsv0:8;
+ uint16_t dsc_pnd:1;
+ uint16_t idl_stp_b:1;
+ uint16_t evt_pnd:1;
+ uint16_t fetch_pnd:4;
+ uint16_t rsv1:1;
+
+ } c_fields;
+};
+
+union __attribute__ ((packed)) prefetch_ctxt
+{
+ struct __attribute__ ((packed)) pref_ctxt_data
+ {
+ uint32_t data[2];
+ } c_data;
+
+ struct __attribute__ ((packed)) pref_ctxt_fields
+ {
+ uint64_t bypass:1;
+ uint64_t buf_sz_idx:4;
+ uint64_t port_id:3;
+ uint64_t rsvd:18;
+ uint64_t err:1;
+ uint64_t pfch_en:1;
+ uint64_t pfch:1;
+ uint64_t sw_crdt:16;
+ uint64_t valid:1;
+ } c_fields;
+};
+
+#define xdebug_info(x, args...) rte_log(RTE_LOG_INFO, RTE_LOGTYPE_USER1,\
+ ## args)
+
+static void print_header(const char *str)
+{
+ xdebug_info(adap, "\n\n%s\n\n", str);
+}
+
+static int qdma_struct_dump(uint8_t port_id, uint16_t queue)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+
+ if (queue >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(INFO, " RX queue_id=%d not configured\n", queue);
+ } else {
+ struct qdma_rx_queue *rx_q =
+ (struct qdma_rx_queue *)dev->data->rx_queues[queue];
+
+ if(rx_q) {
+ print_header(" ***********RX Queue struct********** ");
+ xdebug_info(dev, "\t\t wb_pidx :%x\n",
+ rx_q->wb_status->pidx);
+ xdebug_info(dev, "\t\t wb_cidx :%x\n",
+ rx_q->wb_status->cidx);
+ xdebug_info(dev, "\t\t rx_pidx :%x\n",
+ *rx_q->rx_pidx);
+ xdebug_info(dev, "\t\t rx_cidx :%x\n",
+ *rx_q->rx_cidx);
+ xdebug_info(dev, "\t\t rx_tail :%x\n",
+ rx_q->rx_tail);
+ xdebug_info(dev, "\t\t c2h_pidx :%x\n",
+ rx_q->c2h_pidx);
+ xdebug_info(dev, "\t\t rx_cmpt_tail :%x\n",
+ rx_q->rx_cmpt_tail);
+ xdebug_info(dev, "\t\t cmpt_desc_len :%x\n",
+ rx_q->cmpt_desc_len);
+ xdebug_info(dev, "\t\t rx_buff_size :%x\n",
+ rx_q->rx_buff_size);
+ xdebug_info(dev, "\t\t nb_rx_desc :%x\n",
+ rx_q->nb_rx_desc);
+ xdebug_info(dev, "\t\t nb_rx_cmpt_desc :%x\n",
+ rx_q->nb_rx_cmpt_desc);
+ xdebug_info(dev, "\t\t ep_addr :%x\n",
+ rx_q->ep_addr);
+ xdebug_info(dev, "\t\t st_mode :%x\n",
+ rx_q->st_mode);
+ xdebug_info(dev, "\t\t rx_deferred_start :%x\n",
+ rx_q->rx_deferred_start);
+ xdebug_info(dev, "\t\t en_prefetch :%x\n",
+ rx_q->en_prefetch);
+ xdebug_info(dev, "\t\t en_bypass :%x\n",
+ rx_q->en_bypass);
+ xdebug_info(dev, "\t\t dump_immediate_data :%x\n",
+ rx_q->dump_immediate_data);
+ xdebug_info(dev, "\t\t en_bypass_prefetch :%x\n",
+ rx_q->en_bypass_prefetch);
+ xdebug_info(dev, "\t\t dis_overflow_check :%x\n",
+ rx_q->dis_overflow_check);
+ xdebug_info(dev, "\t\t bypass_desc_sz :%x\n",
+ rx_q->bypass_desc_sz);
+ xdebug_info(dev, "\t\t ringszidx :%x\n",
+ rx_q->ringszidx);
+ xdebug_info(dev, "\t\t cmpt_ringszidx :%x\n",
+ rx_q->cmpt_ringszidx);
+ xdebug_info(dev, "\t\t buffszidx :%x\n",
+ rx_q->buffszidx);
+ xdebug_info(dev, "\t\t threshidx :%x\n",
+ rx_q->threshidx);
+ xdebug_info(dev, "\t\t timeridx :%x\n",
+ rx_q->timeridx);
+ xdebug_info(dev, "\t\t triggermode :%x\n",
+ rx_q->triggermode);
+ }
+ }
+
+ if (queue >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(INFO, " TX queue_id=%d not configured\n", queue);
+ } else {
+ struct qdma_tx_queue *tx_q =
+ (struct qdma_tx_queue *)dev->data->tx_queues[queue];
+
+ if(tx_q) {
+ print_header("***********TX Queue struct************");
+ xdebug_info(dev, "\t\t wb_pidx :%x\n",
+ tx_q->wb_status->pidx);
+ xdebug_info(dev, "\t\t wb_cidx :%x\n",
+ tx_q->wb_status->cidx);
+ xdebug_info(dev, "\t\t tx_pidx :%x\n",
+ *tx_q->tx_pidx);
+ xdebug_info(dev, "\t\t tx_tail :%x\n",
+ tx_q->tx_tail);
+ xdebug_info(dev, "\t\t tx_fl_tail :%x\n",
+ tx_q->tx_fl_tail);
+ xdebug_info(dev, "\t\t tx_desc_pend :%x\n",
+ tx_q->tx_desc_pend);
+ xdebug_info(dev, "\t\t nb_tx_desc :%x\n",
+ tx_q->nb_tx_desc);
+ xdebug_info(dev, "\t\t st_mode :%x\n",
+ tx_q->st_mode);
+ xdebug_info(dev, "\t\t tx_deferred_start :%x\n",
+ tx_q->tx_deferred_start);
+ xdebug_info(dev, "\t\t en_bypass :%x\n",
+ tx_q->en_bypass);
+ xdebug_info(dev, "\t\t bypass_desc_sz :%x\n",
+ tx_q->bypass_desc_sz);
+ xdebug_info(dev, "\t\t func_id :%x\n",
+ tx_q->func_id);
+ xdebug_info(dev, "\t\t port_id :%x\n",
+ tx_q->port_id);
+ xdebug_info(dev, "\t\t ringszidx :%x\n",
+ tx_q->ringszidx);
+ xdebug_info(dev, "\t\t ep_addr :%x\n",
+ tx_q->ep_addr);
+ }
+ }
+ return 0;
+}
+
+static int qdma_csr_dump(void)
+{
+ struct rte_eth_dev *dev;
+ struct qdma_pci_dev *qdma_dev;
+ uint64_t bar_addr;
+ uint32_t reg_val;
+
+ if (rte_eth_dev_count() < 1)
+ return -1;
+
+ dev = &rte_eth_devices[0];
+ qdma_dev = dev->data->dev_private;
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+
+ xdebug_info(dev, "FPGA Registers\n--------------\n");
+ xdebug_info(dev, "NAME offset "
+ "description register-Value\n");
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x0));
+ xdebug_info(dev, "FPGA_VER 0x00000000 "
+ "FPGA Version %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x204));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[0] 0x00000204 "
+ "Ring size index 0 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x208));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[1] 0x00000208 "
+ "Ring size index 1 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x20c));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[2] 0x0000020c "
+ "Ring size index 2 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x210));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[3] 0x00000210 "
+ "Ring size index 3 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x214));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[4] 0x00000214 "
+ "Ring size index 4 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x218));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[5] 0x00000218 "
+ "Ring size index 5 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x21c));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[6] 0x0000021c "
+ "Ring size index 6 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x220));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[7] 0x00000220 "
+ "Ring size index 7 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x224));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[8] 0x00000224 "
+ "Ring size index 8 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x228));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[9] 0x00000228 "
+ "Ring size index 9 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x22c));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[10] 0x0000022c "
+ "Ring size index 10 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x230));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[11] 0x00000230 "
+ "Ring size index 11 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x234));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[12] 0x00000234 "
+ "Ring size index 12 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x238));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[13] 0x00000238 "
+ "Ring size index 13 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x23c));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[14] 0x0000023c "
+ "Ring size index 14 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x240));
+ xdebug_info(dev, "QDMA_GLBL_RNG_SZ_A[15] 0x00000240 "
+ "Ring size index 15 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x248));
+ xdebug_info(dev, "QDMA_GLBL_ERR_STAT 0x00000248 "
+ "Global error status %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x24c));
+ xdebug_info(dev, "QDMA_GLBL_ERR_MASK 0x0000024c "
+ "Global error mask %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x250));
+ xdebug_info(dev, "QDMA_GLBL_DSC_CFG 0x00000250 "
+ "Global Desc configuration %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x254));
+ xdebug_info(dev, "QDMA_GLBL_DSC_ERR_STS 0x00000254 "
+ "Global Desc error status %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x258));
+ xdebug_info(dev, "QDMA_GLBL_DSC_ERR_MSK 0x00000258 "
+ "Global Desc error Mask %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x25c));
+ xdebug_info(dev, "QDMA_GLBL_DSC_ERR_LOG0 0x0000025c "
+ "Global Desc error log0 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x260));
+ xdebug_info(dev, "QDMA_GLBL_DSC_ERR_LOG1 0x00000260 "
+ "Global Desc error log1 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x264));
+ xdebug_info(dev, "QDMA_GLBL_TRQ_ERR_STS 0x00000264 "
+ "Glbl target error status %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x268));
+ xdebug_info(dev, "QDMA_GLBL_TRQ_ERR_MSK 0x00000268 "
+ "Glbl target error Mask %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x26c));
+ xdebug_info(dev, "QDMA_GLBL_TRQ_ERR_LOG 0x0000026c "
+ "Register access space %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x400));
+ xdebug_info(dev, "QDMA_TRQ_SEL_FMAP[0] 0x00000400 "
+ "FMAP target index-0 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0x404));
+ xdebug_info(dev, "QDMA_TRQ_SEL_FMAP[1] 0x00000404 "
+ "FMAP target index-1 %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa88));
+ xdebug_info(dev, "QDMA_C2H_STAT_AXIS_C2H_ACPTD 0x00000a88 "
+ "C2H pkts accepted %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa8c));
+ xdebug_info(dev, "QDMA_C2H_STAT_AXIS_CMPT_ACPTD 0x00000a8c "
+ "C2H CMPT pkts accepted %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa90));
+ xdebug_info(dev, "QDMA_C2H_STAT_DESC_RSP_PKT_ACPTD 0x00000a90 "
+ "desc rsp pkts accepted %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa94));
+ xdebug_info(dev, "QDMA_C2H_STAT_AXIS_PKG_CMP 0x00000a94 "
+ "C2H pkts completed %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa98));
+ xdebug_info(dev, "QDMA_C2H_STAT_DESC_RSP_ACPTD 0x00000a98 "
+ "dsc rsp pkts accepted %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xa9c));
+ xdebug_info(dev, "QDMA_C2H_STAT_DESC_RSP_CMP D 0x00000a9c "
+ "dsc rsp pkts completed %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xaa0));
+ xdebug_info(dev, "QDMA_C2H_STAT_WRQ_OUT 0x00000aa0 "
+ "Number of WRQ %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xaa4));
+ xdebug_info(dev, "QDMA_C2H_STAT_WPL_REN_ACPTD 0x00000aa4 "
+ "WPL REN accepted %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xaf0));
+ xdebug_info(dev, "QDMA_GLBL_ERR_STAT 0x00000af0 "
+ "C2H error status %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xaf4));
+ xdebug_info(dev, "QDMA_GLBL_ERR_MASK 0x00000af4 "
+ "C2H error Mask %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xaf8));
+ xdebug_info(dev, "QDMA_C2H_FATAL_ERR_STAT 0x00000af8 "
+ "C2H fatal error status %x\n", reg_val);
+ reg_val = qdma_read_reg((uint64_t)(bar_addr + 0xafc));
+ xdebug_info(dev, "QDMA_C2H_FATAL_ERR_MSK 0x00000afc "
+ "C2H fatal error Mask %x\n", reg_val);
+
+ return 0;
+}
+
+static int qdma_context_dump(uint8_t port_id, uint16_t queue)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct queue_ind_prg *q_regs;
+ uint32_t reg_val, ctxt_sel;
+ uint64_t bar_addr;
+ uint16_t flag, i;
+ uint16_t qid = qdma_dev->queue_base + queue;
+
+ union h2c_c2h_ctxt q_ctxt;
+ union c2h_cmpt_ctxt c2h_cmpt;
+ union h2c_c2h_hw_ctxt hw_ctxt;
+ union prefetch_ctxt pref_ctxt;
+
+ bar_addr = (uint64_t)qdma_dev->bar_addr[qdma_dev->config_bar_idx];
+ q_regs = (struct queue_ind_prg *)(bar_addr + QDMA_TRQ_SEL_IND +
+ QDMA_IND_Q_PRG_OFF);
+
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[0], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[1], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[2], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[3], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[4], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[5], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[6], 0xffffffff);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_mask[7], 0xffffffff);
+
+ if (queue >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(INFO, " RX queue_id=%d not configured\n", queue);
+ } else {
+ print_header(" *************C2H Queue Contexts************** ");
+ /* SW context */
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 5; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ q_ctxt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t SW "
+ ":0x%x 0x%x 0x%x 0x%x 0x%x\n",
+ qid, q_ctxt.c_data.data[4],
+ q_ctxt.c_data.data[3], q_ctxt.c_data.data[2],
+ q_ctxt.c_data.data[1], q_ctxt.c_data.data[0]);
+ xdebug_info(qdma_dev, "\t\t int_aggr :%x\n",
+ q_ctxt.c_fields.int_aggr);
+ xdebug_info(qdma_dev, "\t\t vec :%x\n",
+ q_ctxt.c_fields.vec);
+ xdebug_info(qdma_dev, "\t\t Base-addr :%lx\n",
+ q_ctxt.c_fields.dsc_base);
+ xdebug_info(qdma_dev, "\t\t is_mm :%x\n",
+ q_ctxt.c_fields.is_mm);
+ xdebug_info(qdma_dev, "\t\t mark_dis :%x\n",
+ q_ctxt.c_fields.mrkr_dis);
+ xdebug_info(qdma_dev, "\t\t irq_req :%x\n",
+ q_ctxt.c_fields.irq_req);
+ xdebug_info(qdma_dev, "\t\t err-cmpt-sent :%x\n",
+ q_ctxt.c_fields.err_wb_sent);
+ xdebug_info(qdma_dev, "\t\t Error status :%x\n",
+ q_ctxt.c_fields.err);
+ xdebug_info(qdma_dev, "\t\t irq_no_last :%x\n",
+ q_ctxt.c_fields.irq_no_last);
+ xdebug_info(qdma_dev, "\t\t port-id :%x\n",
+ q_ctxt.c_fields.port_id);
+ xdebug_info(qdma_dev, "\t\t irq-enable :%x\n",
+ q_ctxt.c_fields.irq_en);
+ xdebug_info(qdma_dev, "\t\t write-back enable :%x\n",
+ q_ctxt.c_fields.wbk_en);
+ xdebug_info(qdma_dev, "\t\t mm-channel-id :%x\n",
+ q_ctxt.c_fields.mm_chn);
+ xdebug_info(qdma_dev, "\t\t bypass :%x\n",
+ q_ctxt.c_fields.byp);
+ xdebug_info(qdma_dev, "\t\t desc-size index :%x\n",
+ q_ctxt.c_fields.dsc_sz);
+ xdebug_info(qdma_dev, "\t\t ring-size index :%x\n",
+ q_ctxt.c_fields.rng_sz);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ q_ctxt.c_fields.rsv1);
+ xdebug_info(qdma_dev, "\t\t fetch_max :%x\n",
+ q_ctxt.c_fields.fetch_max);
+ xdebug_info(qdma_dev, "\t\t at :%x\n",
+ q_ctxt.c_fields.at);
+ xdebug_info(qdma_dev, "\t\t wbi_intvl_en :%x\n",
+ q_ctxt.c_fields.wbi_intvl_en);
+ xdebug_info(qdma_dev, "\t\t wbi_chk :%x\n",
+ q_ctxt.c_fields.wbi_chk);
+ xdebug_info(qdma_dev, "\t\t fetch credits :%x\n",
+ q_ctxt.c_fields.fcrd_en);
+ xdebug_info(qdma_dev, "\t\t queue-enable :%x\n",
+ q_ctxt.c_fields.qen);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ q_ctxt.c_fields.rsv0);
+ xdebug_info(qdma_dev, "\t\t function-id :%x\n",
+ q_ctxt.c_fields.fnc_id);
+ xdebug_info(qdma_dev, "\t\t irq_arm :%x\n",
+ q_ctxt.c_fields.irq_arm);
+ xdebug_info(qdma_dev, "\t\t producer-index :%x\n\n",
+ q_ctxt.c_fields.pidx);
+
+ /* c2h cmpt context */
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 5; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ c2h_cmpt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t C2H-CMPT "
+ ":0x%x 0x%x 0x%x 0x%x 0x%x\n",
+ qid, c2h_cmpt.c_data.data[4],
+ c2h_cmpt.c_data.data[3],
+ c2h_cmpt.c_data.data[2],
+ c2h_cmpt.c_data.data[1],
+ c2h_cmpt.c_data.data[0]);
+
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ c2h_cmpt.c_fields.rsv2);
+ xdebug_info(qdma_dev, "\t\t intr aggr :%x\n",
+ c2h_cmpt.c_fields.int_aggr);
+ xdebug_info(qdma_dev, "\t\t atc :%x\n",
+ c2h_cmpt.c_fields.at);
+ xdebug_info(qdma_dev, "\t\t Overflow Chk Disable:%x\n",
+ c2h_cmpt.c_fields.ovf_chk_dis);
+ xdebug_info(qdma_dev, "\t\t Full Update :%x\n",
+ c2h_cmpt.c_fields.full_upd);
+ xdebug_info(qdma_dev, "\t\t timer_running :%x\n",
+ c2h_cmpt.c_fields.timer_running);
+ xdebug_info(qdma_dev, "\t\t user_trig_pend :%x\n",
+ c2h_cmpt.c_fields.user_trig_pend);
+ xdebug_info(qdma_dev, "\t\t error status d :%x\n",
+ c2h_cmpt.c_fields.err);
+ xdebug_info(qdma_dev, "\t\t valid :%x\n",
+ c2h_cmpt.c_fields.valid);
+ xdebug_info(qdma_dev, "\t\t consumer-index :%x\n",
+ c2h_cmpt.c_fields.cidx);
+ xdebug_info(qdma_dev, "\t\t producer-index :%x\n",
+ (c2h_cmpt.c_fields.pidx_h << 4 |
+ c2h_cmpt.c_fields.pidx_l));
+ xdebug_info(qdma_dev, "\t\t desc-size :%x\n",
+ c2h_cmpt.c_fields.desc_sz);
+ xdebug_info(qdma_dev, "\t\t baddr(4K/22 bits) :%lx\n",
+ (unsigned long int)c2h_cmpt.c_fields.baddr);
+ xdebug_info(qdma_dev, "\t\t size-index :%x\n",
+ c2h_cmpt.c_fields.qsize_idx);
+ xdebug_info(qdma_dev, "\t\t color :%x\n",
+ c2h_cmpt.c_fields.color);
+ xdebug_info(qdma_dev, "\t\t interrupt-state :%x\n",
+ c2h_cmpt.c_fields.int_st);
+ xdebug_info(qdma_dev, "\t\t timer-index :%x\n",
+ c2h_cmpt.c_fields.timer_idx);
+ xdebug_info(qdma_dev, "\t\t counter-index :%x\n",
+ c2h_cmpt.c_fields.count_idx);
+ xdebug_info(qdma_dev, "\t\t function-id :%x\n",
+ c2h_cmpt.c_fields.fnc_id);
+ xdebug_info(qdma_dev, "\t\t trigger-mode :%x\n",
+ c2h_cmpt.c_fields.trig_mode);
+ xdebug_info(qdma_dev, "\t\t cause interrupt :%x\n",
+ c2h_cmpt.c_fields.en_int);
+ xdebug_info(qdma_dev, "\t\t cause status descwr :%x\n\n",
+ c2h_cmpt.c_fields.en_stat_desc);
+
+ /* pfch context */
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 2; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ pref_ctxt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t PFCH "
+ ":0x%x 0x%x\n",
+ qid, c2h_cmpt.c_data.data[1],
+ c2h_cmpt.c_data.data[0]);
+ xdebug_info(qdma_dev, "\t\t valid :%x\n",
+ pref_ctxt.c_fields.valid);
+ xdebug_info(qdma_dev, "\t\t software credit :%x\n",
+ pref_ctxt.c_fields.sw_crdt);
+ xdebug_info(qdma_dev, "\t\t queue is in prefetch:%x\n",
+ pref_ctxt.c_fields.pfch);
+ xdebug_info(qdma_dev, "\t\t enable prefetch :%x\n",
+ pref_ctxt.c_fields.pfch_en);
+ xdebug_info(qdma_dev, "\t\t error :%x\n",
+ pref_ctxt.c_fields.err);
+ xdebug_info(qdma_dev, "\t\t port ID :%x\n",
+ pref_ctxt.c_fields.port_id);
+ xdebug_info(qdma_dev, "\t\t buffer size index :%x\n",
+ pref_ctxt.c_fields.buf_sz_idx);
+ xdebug_info(qdma_dev, "\t\t C2H in bypass mode :%x\n\n",
+ pref_ctxt.c_fields.bypass);
+
+ /* hwc c2h context */
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_C2H << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 2; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ hw_ctxt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t HW "
+ ":0x%x 0x%x\n",
+ qid, hw_ctxt.c_data.data[1],
+ hw_ctxt.c_data.data[0]);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ hw_ctxt.c_fields.rsv1);
+ xdebug_info(qdma_dev, "\t\t fetch pending :%x\n",
+ hw_ctxt.c_fields.fetch_pnd);
+ xdebug_info(qdma_dev, "\t\t event pending :%x\n",
+ hw_ctxt.c_fields.evt_pnd);
+ xdebug_info(qdma_dev, "\t\t Q-inval/no desc pend:%x\n",
+ hw_ctxt.c_fields.idl_stp_b);
+ xdebug_info(qdma_dev, "\t\t descriptor pending :%x\n",
+ hw_ctxt.c_fields.dsc_pnd);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ hw_ctxt.c_fields.rsv0);
+ xdebug_info(qdma_dev, "\t\t credit-use :%x\n",
+ hw_ctxt.c_fields.crd_use);
+ xdebug_info(qdma_dev, "\t\t consumer-index :%x\n\n",
+ hw_ctxt.c_fields.cidx);
+ }
+
+ if (queue >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(INFO, " TX queue_id=%d not configured\n", queue);
+ } else {
+ print_header(" *************H2C Queue Contexts************** ");
+ /* SW context */
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 5; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ q_ctxt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t SW "
+ ":0x%x 0x%x 0x%x 0x%x 0x%x\n",
+ qid, q_ctxt.c_data.data[4],
+ q_ctxt.c_data.data[3], q_ctxt.c_data.data[2],
+ q_ctxt.c_data.data[1], q_ctxt.c_data.data[0]);
+ xdebug_info(qdma_dev, "\t\t int_aggr :%x\n",
+ q_ctxt.c_fields.int_aggr);
+ xdebug_info(qdma_dev, "\t\t vec :%x\n",
+ q_ctxt.c_fields.vec);
+ xdebug_info(qdma_dev, "\t\t Base-addr :%lx\n",
+ q_ctxt.c_fields.dsc_base);
+ xdebug_info(qdma_dev, "\t\t is_mm :%x\n",
+ q_ctxt.c_fields.is_mm);
+ xdebug_info(qdma_dev, "\t\t mark_dis :%x\n",
+ q_ctxt.c_fields.mrkr_dis);
+ xdebug_info(qdma_dev, "\t\t irq_req :%x\n",
+ q_ctxt.c_fields.irq_req);
+ xdebug_info(qdma_dev, "\t\t err-cmpt-sent :%x\n",
+ q_ctxt.c_fields.err_wb_sent);
+ xdebug_info(qdma_dev, "\t\t Error status :%x\n",
+ q_ctxt.c_fields.err);
+ xdebug_info(qdma_dev, "\t\t irq_no_last :%x\n",
+ q_ctxt.c_fields.irq_no_last);
+ xdebug_info(qdma_dev, "\t\t port-id :%x\n",
+ q_ctxt.c_fields.port_id);
+ xdebug_info(qdma_dev, "\t\t irq-enable :%x\n",
+ q_ctxt.c_fields.irq_en);
+ xdebug_info(qdma_dev, "\t\t write-back enable :%x\n",
+ q_ctxt.c_fields.wbk_en);
+ xdebug_info(qdma_dev, "\t\t mm-channel-id :%x\n",
+ q_ctxt.c_fields.mm_chn);
+ xdebug_info(qdma_dev, "\t\t bypass :%x\n",
+ q_ctxt.c_fields.byp);
+ xdebug_info(qdma_dev, "\t\t desc-size index :%x\n",
+ q_ctxt.c_fields.dsc_sz);
+ xdebug_info(qdma_dev, "\t\t ring-size index :%x\n",
+ q_ctxt.c_fields.rng_sz);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ q_ctxt.c_fields.rsv1);
+ xdebug_info(qdma_dev, "\t\t fetch_max :%x\n",
+ q_ctxt.c_fields.fetch_max);
+ xdebug_info(qdma_dev, "\t\t at :%x\n",
+ q_ctxt.c_fields.at);
+ xdebug_info(qdma_dev, "\t\t wbi_intvl_en :%x\n",
+ q_ctxt.c_fields.wbi_intvl_en);
+ xdebug_info(qdma_dev, "\t\t wbi_chk :%x\n",
+ q_ctxt.c_fields.wbi_chk);
+ xdebug_info(qdma_dev, "\t\t fetch credits :%x\n",
+ q_ctxt.c_fields.fcrd_en);
+ xdebug_info(qdma_dev, "\t\t queue-enable :%x\n",
+ q_ctxt.c_fields.qen);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ q_ctxt.c_fields.rsv0);
+ xdebug_info(qdma_dev, "\t\t function-id :%x\n",
+ q_ctxt.c_fields.fnc_id);
+ xdebug_info(qdma_dev, "\t\t irq_arm :%x\n",
+ q_ctxt.c_fields.irq_arm);
+ xdebug_info(qdma_dev, "\t\t producer-index :%x\n\n",
+ q_ctxt.c_fields.pidx);
+
+ /* hwc h2c context */
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_H2C << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ qdma_write_reg((uint64_t)&q_regs->ctxt_cmd, ctxt_sel);
+
+ flag = 1;
+ i = CONTEXT_PROG_POLL_COUNT;
+ while (flag && i) {
+ reg_val = qdma_read_reg((uint64_t)&q_regs->ctxt_cmd);
+ PMD_DRV_LOG(DEBUG, " Read cmd for queue-id:%d: "
+ "reg_val:%x\n", qid, reg_val);
+ flag = reg_val & BUSY_BIT_MSK;
+ rte_delay_ms(CONTEXT_PROG_DELAY);
+ i--;
+ }
+ if (flag) {
+ PMD_DRV_LOG(ERR, "Error: Busy on queue-id:%d: "
+ "initailization with cmd reg_val:%x\n",
+ qid, reg_val);
+ return -1;
+ }
+
+ for (i = 0; i < 2; i++) {
+ reg_val = qdma_read_reg((uint64_t)&
+ q_regs->ctxt_data[i]);
+ hw_ctxt.c_data.data[i] = reg_val;
+ }
+
+ xdebug_info(qdma_dev, "Q-id: %d, \t HW "
+ ":0x%x 0x%x\n",
+ qid, hw_ctxt.c_data.data[1],
+ hw_ctxt.c_data.data[0]);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ hw_ctxt.c_fields.rsv1);
+ xdebug_info(qdma_dev, "\t\t fetch pending :%x\n",
+ hw_ctxt.c_fields.fetch_pnd);
+ xdebug_info(qdma_dev, "\t\t event pending :%x\n",
+ hw_ctxt.c_fields.evt_pnd);
+ xdebug_info(qdma_dev, "\t\t Q-inval/no desc pend:%x\n",
+ hw_ctxt.c_fields.idl_stp_b);
+ xdebug_info(qdma_dev, "\t\t descriptor pending :%x\n",
+ hw_ctxt.c_fields.dsc_pnd);
+ xdebug_info(qdma_dev, "\t\t reserved :%x\n",
+ hw_ctxt.c_fields.rsv0);
+ xdebug_info(qdma_dev, "\t\t credit-use :%x\n",
+ hw_ctxt.c_fields.crd_use);
+ xdebug_info(qdma_dev, "\t\t consumer-index :%x\n\n",
+ hw_ctxt.c_fields.cidx);
+ }
+
+ return 0;
+}
+static int qdma_queue_desc_dump(uint8_t port_id,
+ struct rte_xdebug_desc_param *param)
+{
+ struct rte_eth_dev *dev = &rte_eth_devices[port_id];
+ int x;
+ struct qdma_rx_queue *rxq;
+ struct qdma_tx_queue *txq;
+ char str[50];
+
+ switch (param->type) {
+ case ETH_XDEBUG_DESC_C2H:
+
+ if (param->queue >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(INFO, "queue_id=%d not configured",
+ param->queue);
+ return -1;
+ }
+
+ rxq = (struct qdma_rx_queue *)
+ dev->data->rx_queues[param->queue];
+
+ if(rxq) {
+ if (rxq->status != RTE_ETH_QUEUE_STATE_STARTED) {
+ printf("Queue_id %d is not yet started\n",
+ param->queue);
+ return -1;
+ }
+
+ if (param->start < 0 || param->start > rxq->nb_rx_desc)
+ param->start = 0;
+ if (param->end <= param->start ||
+ param->end > rxq->nb_rx_desc)
+ param->end = rxq->nb_rx_desc;
+
+ if (rxq->st_mode) {
+ struct qdma_c2h_desc *rx_ring_st =
+ (struct qdma_c2h_desc *)rxq->rx_ring;
+
+ printf("\n===== C2H ring descriptors=====\n");
+ for (x = param->start; x < param->end; x++) {
+ struct qdma_c2h_desc *rx_st =
+ &rx_ring_st[x];
+ sprintf(str, "\nDescriptor ID %d\t", x);
+ rte_hexdump(stdout, str,
+ (const void *)rx_st,
+ sizeof(struct qdma_c2h_desc));
+ }
+ } else {
+ struct qdma_mm_desc *rx_ring_mm =
+ (struct qdma_mm_desc *)rxq->rx_ring;
+ printf("\n====== C2H ring descriptors======\n");
+ for (x = param->start; x < param->end; x++) {
+ sprintf(str, "\nDescriptor ID %d\t", x);
+ rte_hexdump(stdout, str,
+ (const void *)&rx_ring_mm[x],
+ sizeof(struct qdma_mm_desc));
+ }
+ }
+ }
+ break;
+ case ETH_XDEBUG_DESC_CMPT:
+
+ if (param->queue >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(INFO, "queue_id=%d not configured",
+ param->queue);
+ return -1;
+ }
+
+ rxq = (struct qdma_rx_queue *)
+ dev->data->rx_queues[param->queue];
+
+ if(rxq) {
+ if (rxq->status != RTE_ETH_QUEUE_STATE_STARTED) {
+ printf("Queue_id %d is not yet started\n",
+ param->queue);
+ return -1;
+ }
+
+ if (param->start < 0 ||
+ param->start > rxq->nb_rx_cmpt_desc)
+ param->start = 0;
+ if (param->end <= param->start ||
+ param->end > rxq->nb_rx_cmpt_desc)
+ param->end = rxq->nb_rx_cmpt_desc;
+
+ if (!rxq->st_mode) {
+ printf("Queue_id %d is not initialized "
+ "in Stream mode\n", param->queue);
+ return -1;
+ }
+
+ printf("\n===== CMPT ring descriptors=====\n");
+ for (x = param->start; x < param->end; x++) {
+ uint32_t *cmpt_ring = (uint32_t *)
+ ((uint64_t)(rxq->cmpt_ring) +
+ ((uint64_t)x * rxq->cmpt_desc_len));
+ sprintf(str, "\nDescriptor ID %d\t", x);
+ rte_hexdump(stdout, str,
+ (const void *)cmpt_ring,
+ rxq->cmpt_desc_len);
+ }
+ }
+ break;
+ case ETH_XDEBUG_DESC_H2C:
+
+ if (param->queue >= dev->data->nb_tx_queues) {
+ PMD_DRV_LOG(INFO, "queue_id=%d not configured",
+ param->queue);
+ return -1;
+ }
+
+ txq = (struct qdma_tx_queue *)
+ dev->data->tx_queues[param->queue];
+
+ if(txq) {
+ if (txq->status != RTE_ETH_QUEUE_STATE_STARTED) {
+ printf("Queue_id %d is not yet started\n",
+ param->queue);
+ return -1;
+ }
+
+ if (param->start < 0 || param->start > txq->nb_tx_desc)
+ param->start = 0;
+ if (param->end <= param->start ||
+ param->end > txq->nb_tx_desc)
+ param->end = txq->nb_tx_desc;
+
+ if (txq->st_mode) {
+ struct qdma_h2c_desc *qdma_h2c_ring =
+ (struct qdma_h2c_desc *)txq->tx_ring;
+ printf("\n====== H2C ring descriptors=====\n");
+ for (x = param->start; x < param->end; x++) {
+ sprintf(str, "\nDescriptor ID %d\t", x);
+ rte_hexdump(stdout, str,
+ (const void *)&qdma_h2c_ring[x],
+ sizeof(struct qdma_h2c_desc));
+ }
+ } else {
+ struct qdma_mm_desc *tx_ring_mm =
+ (struct qdma_mm_desc *)txq->tx_ring;
+ printf("\n===== H2C ring descriptors=====\n");
+ for (x = param->start; x < param->end; x++) {
+ sprintf(str, "\nDescriptor ID %d\t", x);
+ rte_hexdump(stdout, str,
+ (const void *)&tx_ring_mm[x],
+ sizeof(struct qdma_mm_desc));
+ }
+ }
+ }
+ break;
+ default:
+ printf("Invalid ring selected\n");
+ break;
+ }
+ return 0;
+}
+
+int qdma_xdebug(uint8_t port_id, enum rte_xdebug_type type,
+ void *params)
+{
+ int err = -ENOTSUP;
+
+ switch (type) {
+ case ETH_XDEBUG_QDMA_GLOBAL_CSR:
+ err = qdma_csr_dump();
+ if (err) {
+ xdebug_info(adap, "Error dumping Global registers\n");
+ return err;
+ }
+ break;
+ case ETH_XDEBUG_QUEUE_CONTEXT:
+
+ if (!params) {
+ printf("QID required for Queue context dump\n");
+ return -1;
+ }
+
+ err = qdma_context_dump(port_id, *(uint16_t *)params);
+ if (err) {
+ xdebug_info(adap, "Error dumping %d: %d\n",
+ *(int *)params, err);
+ return err;
+ }
+ break;
+
+ case ETH_XDEBUG_QUEUE_STRUCT:
+
+ if (!params) {
+ printf("QID required for Queue Structure dump\n");
+ return -1;
+ }
+
+ err = qdma_struct_dump(port_id, *(uint16_t *)params);
+ if (err) {
+ xdebug_info(adap, "Error dumping %d: %d\n",
+ *(int *)params, err);
+ return err;
+ }
+ break;
+
+ case ETH_XDEBUG_QUEUE_DESC_DUMP:
+
+ if (!params) {
+ printf("Params required for Queue descriptor dump\n");
+ return -1;
+ }
+
+ err = qdma_queue_desc_dump(port_id,
+ (struct rte_xdebug_desc_param *)params);
+ if (err) {
+ xdebug_info(adap, "Error dumping %d: %d\n",
+ *(int *)params, err);
+ return err;
+ }
+ break;
+
+ default:
+ xdebug_info(adap, "Unsupported type\n");
+ return err;
+ }
+
+ return 0;
+}
diff --git a/QDMA/DPDK/drivers/net/qdma/rte_pmd_qdma_version.map b/QDMA/DPDK/drivers/net/qdma/rte_pmd_qdma_version.map
new file mode 100644
index 0000000000000000000000000000000000000000..a753031720cfddc79191d1a6e6bffc732b8efb94
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/rte_pmd_qdma_version.map
@@ -0,0 +1,3 @@
+DPDK_17.11 {
+ local: *;
+};
diff --git a/QDMA/DPDK/drivers/net/qdma/version.h b/QDMA/DPDK/drivers/net/qdma/version.h
new file mode 100644
index 0000000000000000000000000000000000000000..2b1ef0254ce3e7e46c4f441a24f039d3a930f4e0
--- /dev/null
+++ b/QDMA/DPDK/drivers/net/qdma/version.h
@@ -0,0 +1,51 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __QDMA_VERSION_H__
+#define __QDMA_VERSION_H__
+
+#define qdma_stringify1(x...) #x
+#define qdma_stringify(x...) qdma_stringify1(x)
+
+#define QDMA_PMD_MAJOR 2018
+#define QDMA_PMD_MINOR 3
+#define QDMA_PMD_PATCHLEVEL 19
+
+#define QDMA_PMD_VERSION \
+ qdma_stringify(QDMA_PMD_MAJOR) "." \
+ qdma_stringify(QDMA_PMD_MINOR) "." \
+ qdma_stringify(QDMA_PMD_PATCHLEVEL)
+
+#define QDMA_PMD_VERSION_NUMBER \
+ ((QDMA_PMD_MAJOR) * 1000 + (QDMA_PMD_MINOR) * 100 + QDMA_PMD_PATCHLEVEL)
+
+#endif /* ifndef __QDMA_VERSION_H__ */
diff --git a/QDMA/DPDK/examples/qdma_testapp/Makefile b/QDMA/DPDK/examples/qdma_testapp/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..6b619e01292eaf13fcfd08e378ef755edf848e2c
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/Makefile
@@ -0,0 +1,55 @@
+# BSD LICENSE
+#
+# Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overriden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+#Default BRAM size is set to 512K
+#if modified the BRAM_SIZE, the same need to be set to the driver Makefile
+BRAM_SIZE ?= 524288
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = qdma_testapp
+
+# all source are stored in SRCS-y
+SRCS-y := testapp.c pcierw.c commands.c
+
+CFLAGS += -O3 -DDPDK=1 -DBRAM_SIZE=$(BRAM_SIZE)
+CFLAGS += $(WERROR_FLAGS)
+
+# PERF benchmarking may be enabled by uncommenting following
+#CFLAGS += -DPERF_BENCHMARK
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/QDMA/DPDK/examples/qdma_testapp/commands.c b/QDMA/DPDK/examples/qdma_testapp/commands.c
new file mode 100644
index 0000000000000000000000000000000000000000..5ddd58114bea5c84e941f52699d37c45d49e8405
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/commands.c
@@ -0,0 +1,1532 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2009, Olivier MATZ <zer0@droids-corp.org>
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the University of California, Berkeley nor the
+ * names of its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <netinet/in.h>
+#include <termios.h>
+#ifndef __linux__
+#include <net/socket.h>
+#endif
+
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_parse_ipaddr.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+
+#include <rte_string_fns.h>
+#include <rte_ethdev.h>
+#include <fcntl.h>
+
+#include "parse_obj_list.h"
+#include "pcierw.h"
+#include "commands.h"
+#include "qdma_regs.h"
+#include "testapp.h"
+#include "../../drivers/net/qdma/qdma.h"
+
+/* Command help */
+struct cmd_help_result {
+ cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ cmdline_printf(cl,
+ "Demo example of command line interface in RTE\n\n"
+ "This is a readline-like interface that can be "
+ "used to\n debug your RTE application.\n\n"
+ "Supported Commands:\n"
+ "\tport_init <port-id> <queue-id base> "
+ "<num-queues> <st-queues> "
+ "<ring-depth> <pkt-buff-size> "
+ " :port-initailization\n"
+ "\tport_close <port-id> "
+ ":port-close\n"
+ "\treg_read <port-id> <bar-num> <address> "
+ ":Reads Specified Register\n"
+ "\treg_write <port-id> <bar-num> <address> "
+ "<value> "
+ ":Writes Specified Register\n"
+ "\tdma_to_device <port-id> <num-queues> "
+ "<input-filename> "
+ "<dst_addr> <size> <iterations> "
+ ":To Transmit\n"
+ "\tdma_from_device <port-id> <num-queues> "
+ "<output-filename> "
+ "<src_addr> <size> <iterations> "
+ ":To Receive\n"
+ "\treg_dump <port-id> "
+ ":To dump all the valid registers\n"
+ "\tqueue_dump <port-id> <queue-id> "
+ ":To dump the queue-context of a queue-number\n"
+ "\tdesc_dump <port-id> <queue-id> "
+ ":To dump the descriptor-fields of a "
+ "queue-number\n"
+ "\tload_cmds <file-name> "
+ ":To execute the list of commands from file\n"
+ "\thelp\n"
+ "\tCtrl-d "
+ ": To quit from this command-line type Ctrl+d\n"
+ "\n");
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+ TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+ .f = cmd_help_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "show help",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_help_help,
+ NULL,
+ },
+};
+
+struct cmd_obj_port_init_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t queue_base;
+ cmdline_fixed_string_t num_queues;
+ cmdline_fixed_string_t st_queues;
+ cmdline_fixed_string_t nb_descs;
+ cmdline_fixed_string_t buff_size;
+};
+
+static void cmd_obj_port_init_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_port_init_result *res = parsed_result;
+
+ cmdline_printf(cl, "Port-init Port:%s, num-queues:%s, st-queues: %s\n",
+ res->port_id, res->num_queues, res->st_queues);
+
+ int port_id = atoi(res->port_id);
+ if (pinfo[port_id].num_queues) {
+ cmdline_printf(cl, "Error: port-id:%d already initialized\n"
+ "To re-initialize please close the port\n",
+ port_id);
+ return;
+ }
+
+ int num_queues = atoi(res->num_queues);
+ int queue_base = atoi(res->queue_base);
+ int st_queues = atoi(res->st_queues);
+ int nb_descs = atoi(res->nb_descs);
+ int buff_size = atoi(res->buff_size);
+
+ if ((num_queues < 1) || (queue_base + num_queues > MAX_NUM_QUEUES)) {
+ cmdline_printf(cl, "Error: please enter valid number of queues,"
+ "entered: %d max allowed: %d\n",
+ num_queues, MAX_NUM_QUEUES);
+ return;
+ }
+
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: please enter valid port number: "
+ "%d\n", port_id);
+ return;
+ }
+ {
+ int result = port_init(port_id, queue_base, num_queues,
+ st_queues, nb_descs, buff_size);
+
+ if (result < 0)
+ cmdline_printf(cl, "Error: Port initialization on "
+ "port-id:%d failed\n", port_id);
+ else
+ cmdline_printf(cl, "Port initialization done "
+ "successfully on port-id:%d\n",
+ port_id);
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_port_init =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, action,
+ "port_init");
+cmdline_parse_token_string_t cmd_obj_port_init_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_port_init_queue_base =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, queue_base,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_port_init_num_queues =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, num_queues,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_port_init_st_queues =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, st_queues,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_port_init_nb_descs =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, nb_descs,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_port_init_buff_size =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_init_result, buff_size,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_port_init = {
+ .f = cmd_obj_port_init_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "port_init port-id queue-id-base num-queues st-queues "
+ "queue-ring-size buffer-size",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_port_init,
+ (void *)&cmd_obj_port_init_port_id,
+ (void *)&cmd_obj_port_init_queue_base,
+ (void *)&cmd_obj_port_init_num_queues,
+ (void *)&cmd_obj_port_init_st_queues,
+ (void *)&cmd_obj_port_init_nb_descs,
+ (void *)&cmd_obj_port_init_buff_size,
+ NULL,
+ },
+
+};
+
+/* Command port-close */
+struct cmd_obj_port_close_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+};
+
+static void cmd_obj_port_close_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_port_close_result *res = parsed_result;
+
+ cmdline_printf(cl, "Port-close on Port-id:%s\n", res->port_id);
+
+ int port_id = atoi(res->port_id);
+ if (pinfo[port_id].num_queues == 0) {
+ cmdline_printf(cl, "Error: port-id:%d already closed\n",
+ port_id);
+ return;
+ }
+
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: please enter valid port "
+ "number: %d\n", port_id);
+ return;
+ }
+ {
+ port_close(port_id);
+ pinfo[port_id].num_queues = 0;
+ cmdline_printf(cl, "Port-id:%d closed successfully\n", port_id);
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_port_close =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_close_result, action,
+ "port_close");
+cmdline_parse_token_string_t cmd_obj_port_close_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_port_close_result, port_id,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_port_close = {
+ .f = cmd_obj_port_close_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "port_close port-id ",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_port_close,
+ (void *)&cmd_obj_port_close_port_id,
+ NULL,
+ },
+
+};
+
+/* Command Read addr */
+struct cmd_obj_reg_read_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t bar_id;
+ cmdline_fixed_string_t addr;
+};
+
+static void cmd_obj_reg_read_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_reg_read_result *res = parsed_result;
+
+ cmdline_printf(cl, "Read Port:%s, BAR-index:%s, Address:%s\n\n",
+ res->port_id, res->bar_id, res->addr);
+
+ int addr = strtol(res->addr, NULL, 16);
+
+ if (addr % 4) {
+ cmdline_printf(cl, "ERROR: Read address must aligned to "
+ "a 4-byte boundary.\n\n");
+ } else {
+ int port_id = atoi(res->port_id);
+ int bar_id = atoi(res->bar_id);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n",
+ port_id);
+ return;
+ }
+ int result = PciRead(bar_id, addr, port_id);
+
+ cmdline_printf(cl, "Read (%d:0x%08x) = 0x%08x\n",
+ port_id, addr, result);
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_reg_read =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_read_result, action,
+ "reg_read");
+cmdline_parse_token_string_t cmd_obj_reg_read_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_read_result, port_id, NULL);
+cmdline_parse_token_string_t cmd_obj_reg_read_bar_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_read_result, bar_id, NULL);
+cmdline_parse_token_string_t cmd_obj_reg_read_addr =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_read_result, addr, NULL);
+
+cmdline_parse_inst_t cmd_obj_reg_read = {
+ .f = cmd_obj_reg_read_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "reg_read port-id bar-id address",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_reg_read,
+ (void *)&cmd_obj_reg_read_port_id,
+ (void *)&cmd_obj_reg_read_bar_id,
+ (void *)&cmd_obj_reg_read_addr,
+ NULL,
+ },
+
+};
+
+/* Command Write addr */
+struct cmd_obj_reg_write_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t bar_id;
+ cmdline_fixed_string_t address;
+ cmdline_fixed_string_t value;
+};
+
+static void cmd_obj_reg_write_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_reg_write_result *res = parsed_result;
+
+ cmdline_printf(cl, "Write Port:%s, Address:%s, Value:%s\n",
+ res->port_id, res->address, res->value);
+
+ int bar_id = atoi(res->bar_id);
+ int port_id = atoi(res->port_id);
+ int addr = strtol(res->address, NULL, 16);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n", port_id);
+ return;
+ }
+ if (addr % 4) {
+ cmdline_printf(cl, "ERROR: Write address must aligned to a "
+ "4-byte boundary.\n\n");
+ } else{
+ int value = strtol(res->value, NULL, 16);
+ PciWrite(bar_id, addr, value, port_id);
+ int result = PciRead(bar_id, addr, port_id);
+ cmdline_printf(cl, "Read (%d:0x%08x) = 0x%08x\n", port_id, addr,
+ result);
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_reg_write =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_write_result, action,
+ "reg_write");
+cmdline_parse_token_string_t cmd_obj_reg_write_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_write_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_reg_write_bar_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_write_result, bar_id, NULL);
+cmdline_parse_token_string_t cmd_obj_reg_write_address =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_write_result, address,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_reg_write_value =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_write_result, value, NULL);
+
+cmdline_parse_inst_t cmd_obj_reg_write = {
+ .f = cmd_obj_reg_write_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "reg_write port-id bar-id address value",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_reg_write,
+ (void *)&cmd_obj_reg_write_port_id,
+ (void *)&cmd_obj_reg_write_bar_id,
+ (void *)&cmd_obj_reg_write_address,
+ (void *)&cmd_obj_reg_write_value,
+ NULL,
+ },
+
+};
+
+/* Command do-xmit */
+struct cmd_obj_dma_to_device_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t queues;
+ cmdline_fixed_string_t filename;
+ cmdline_fixed_string_t dst_addr;
+ cmdline_fixed_string_t size;
+ cmdline_fixed_string_t loops;
+};
+
+static void cmd_obj_dma_to_device_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_dma_to_device_result *res = parsed_result;
+ int i, ifd, tot_num_desc, offset, size, r_size = 0, total_size = 0;
+ int ld_size = 0, loop = 0, ret, j, zbyte = 0, user_bar_idx;
+ off_t ret_val;
+ int port_id = 0, num_queues = 0, input_size = 0, num_loops = 0;
+ int dst_addr = 0;
+ uint32_t regval = 0;
+ unsigned int q_data_size = 0;
+
+ cmdline_printf(cl, "xmit on Port:%s, filename:%s, num-queues:%s\n\n",
+ res->port_id, res->filename, res->queues);
+
+ ifd = open(res->filename, O_RDWR);
+ if (ifd < 0) {
+ cmdline_printf(cl, "Error: Invalid filename: %s\n",
+ res->filename);
+ return;
+ }
+
+ {
+ port_id = atoi(res->port_id);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n",
+ port_id);
+ close(ifd);
+ return;
+ }
+ num_queues = atoi(res->queues);
+ if ((unsigned int)num_queues > pinfo[port_id].num_queues) {
+ cmdline_printf(cl, "Error: num-queues:%d are more than "
+ "the configured queues:%d,\n "
+ "Please enter valid number of queues\n",
+ num_queues, pinfo[port_id].num_queues);
+ close(ifd);
+ return;
+ }
+ if (num_queues == 0) {
+ cmdline_printf(cl, "Error: Please enter valid number "
+ "of queues\n");
+ close(ifd);
+ return;
+ }
+ user_bar_idx = pinfo[port_id].user_bar_idx;
+ regval = PciRead(user_bar_idx, C2H_CONTROL_REG, port_id);
+
+ input_size = atoi(res->size);
+ num_loops = atoi(res->loops);
+ dst_addr = atoi(res->dst_addr);
+
+#ifndef PERF_BENCHMARK
+ if (dst_addr + input_size > BRAM_SIZE) {
+ cmdline_printf(cl, "Error: (dst_addr %d + input size "
+ "%d) shall be less than "
+ "BRAM_SIZE %d.\n", dst_addr,
+ input_size, BRAM_SIZE);
+ close(ifd);
+ return;
+ }
+#endif
+ /* For zero-byte transfers, HW expects a
+ * buffer of length 4kb and with desc->len as 0.
+ */
+ if (input_size == 0) {
+ if ((unsigned int)num_queues <=
+ pinfo[port_id].st_queues) {
+ zbyte = 1;
+ } else {
+ cmdline_printf(cl, "Error: Zero-length support "
+ "is for queues with ST-mode "
+ "only\n");
+ close(ifd);
+ return;
+ }
+ }
+
+ if (input_size % num_queues) {
+ size = input_size / num_queues;
+ r_size = input_size % num_queues;
+ } else
+ size = input_size / num_queues;
+
+ do {
+ total_size = input_size;
+ /* transmit data on the number of Queues configured
+ * from the input file
+ */
+ for (i = 0, j = 0; i < num_queues; i++, j++) {
+
+ if (total_size == 0)
+ q_data_size = pinfo[port_id].buff_size;
+ else if (total_size == (r_size + size)) {
+ q_data_size = total_size;
+ total_size -= total_size;
+ } else {
+ q_data_size = size;
+ total_size -= size;
+ }
+
+ if (q_data_size >= pinfo[port_id].buff_size) {
+ if (q_data_size %
+ pinfo[port_id].buff_size) {
+ tot_num_desc = (q_data_size /
+ pinfo[port_id].buff_size) + 1;
+ ld_size = q_data_size %
+ pinfo[port_id].buff_size;
+ } else
+ tot_num_desc = (q_data_size /
+ pinfo[port_id].buff_size);
+ } else {
+ tot_num_desc = 1;
+ ld_size = q_data_size %
+ pinfo[port_id].buff_size;
+ }
+
+ if (port_id)
+ offset = (input_size/num_queues) * j;
+ else
+ offset = (input_size/num_queues) * i;
+
+ if ((unsigned int)i < (pinfo[port_id].st_queues)
+ && !(regval & ST_LOOPBACK_EN))
+ ret_val = lseek(ifd, 0, SEEK_SET);
+ else
+ ret_val = lseek(ifd, offset, SEEK_SET);
+
+ if (ret_val == (off_t)-1) {
+ cmdline_printf(cl, "DMA-to-Device: "
+ "lseek func failed\n");
+ close(ifd);
+ return;
+ }
+
+#ifdef PERF_BENCHMARK
+ dst_addr = (dst_addr + (i * q_data_size)) %
+ BRAM_SIZE;
+#else
+ dst_addr = dst_addr + (i * q_data_size);
+#endif
+ ret =
+ update_queue_param(&rte_eth_devices[port_id], i,
+ TX_DST_ADDRESS, dst_addr);
+ if (ret < 0) {
+ close(ifd);
+ return;
+ }
+
+ cmdline_printf(cl, "DMA-to-Device: with "
+ "input-size:%d, ld_size:%d,"
+ "tot_num_desc:%d\n",
+ input_size, ld_size,
+ tot_num_desc);
+ ret = do_xmit(port_id, ifd, i, ld_size,
+ tot_num_desc, zbyte);
+ if (ret < 0) {
+ close(ifd);
+ return;
+ }
+ }
+ ++loop;
+ } while (loop < num_loops);
+ close(ifd);
+ }
+ cmdline_printf(cl, "\n######## DMA transfer to device is completed "
+ "successfully #######\n");
+}
+
+cmdline_parse_token_string_t cmd_obj_action_dma_to_device =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, action,
+ "dma_to_device");
+cmdline_parse_token_string_t cmd_obj_dma_to_device_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_to_device_queues =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, queues,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_to_device_filename =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, filename,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_to_device_dst_addr =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, dst_addr,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_to_device_size =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, size,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_to_device_loops =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_to_device_result, loops,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_dma_to_device = {
+ .f = cmd_obj_dma_to_device_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "dma_to_device port-id num-queues filename dst_addr "
+ "size loops",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_dma_to_device,
+ (void *)&cmd_obj_dma_to_device_port_id,
+ (void *)&cmd_obj_dma_to_device_queues,
+ (void *)&cmd_obj_dma_to_device_filename,
+ (void *)&cmd_obj_dma_to_device_dst_addr,
+ (void *)&cmd_obj_dma_to_device_size,
+ (void *)&cmd_obj_dma_to_device_loops,
+ NULL,
+ },
+
+};
+
+/* Command do-recv */
+struct cmd_obj_dma_from_device_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t queues;
+ cmdline_fixed_string_t filename;
+ cmdline_fixed_string_t src_addr;
+ cmdline_fixed_string_t size;
+ cmdline_fixed_string_t loops;
+};
+
+static void cmd_obj_dma_from_device_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_dma_from_device_result *res = parsed_result;
+ int i, ofd, offset, size, total_size, r_size = 0;
+ int mm_tdesc, mm_ld_size = 0;
+ int loop = 0, ret, j;
+ off_t ret_val;
+ int port_id = 0, num_queues = 0, input_size = 0, num_loops = 0;
+ int src_addr = 0;
+ unsigned int q_data_size = 0;
+
+ cmdline_printf(cl, "recv on Port:%s, filename:%s\n",
+ res->port_id, res->filename);
+
+ ofd = open(res->filename, O_RDWR | O_CREAT | O_TRUNC | O_SYNC, 0666);
+ if (ofd < 0) {
+ cmdline_printf(cl, "Error: Invalid filename: %s\n",
+ res->filename);
+ return;
+ }
+
+ {
+ port_id = atoi(res->port_id);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n",
+ port_id);
+ close(ofd);
+ return;
+ }
+ num_queues = atoi(res->queues);
+ if ((unsigned int)num_queues > pinfo[port_id].num_queues) {
+ cmdline_printf(cl, "Error: num-queues:%d are more than "
+ "the configured queues:%d,\n"
+ "Please enter valid number of queues\n",
+ num_queues, pinfo[port_id].num_queues);
+ close(ofd);
+ return;
+ }
+ if (num_queues == 0) {
+ cmdline_printf(cl, "Error: Please enter valid number "
+ "of queues\n");
+ close(ofd);
+ return;
+ }
+ input_size = atoi(res->size);
+ num_loops = atoi(res->loops);
+ src_addr = atoi(res->src_addr);
+#ifndef PERF_BENCHMARK
+ if (src_addr + input_size > BRAM_SIZE) {
+ cmdline_printf(cl, "Error: (src_addr %d + input "
+ "size %d) shall be less than "
+ "BRAM_SIZE %d.\n", src_addr,
+ input_size, BRAM_SIZE);
+ close(ofd);
+ return;
+ }
+#endif
+ /* Restrict C2H zerobyte support to ST-mode queues*/
+ if (input_size == 0) {
+ if ((unsigned int)num_queues >
+ pinfo[port_id].st_queues) {
+ cmdline_printf(cl, "Error: Zero-length support "
+ "is for queues with ST-mode only\n");
+ close(ofd);
+ return;
+ }
+ }
+
+ if (input_size % num_queues) {
+ size = input_size / num_queues;
+ r_size = input_size % num_queues;
+ } else
+ size = input_size / num_queues;
+
+ do {
+ total_size = input_size;
+ /* Transmit data on the number of Queues configured
+ * from the input file
+ */
+ for (i = 0, j = 0; i < num_queues; i++, j++) {
+
+ if (total_size == (r_size + size)) {
+ q_data_size = total_size;
+ total_size -= total_size;
+ } else {
+ q_data_size = size;
+ total_size -= size;
+ }
+
+ if (q_data_size >= pinfo[port_id].buff_size) {
+ if (q_data_size %
+ pinfo[port_id].buff_size) {
+ mm_tdesc = (q_data_size /
+ pinfo[port_id].buff_size) + 1;
+
+ mm_ld_size = q_data_size %
+ pinfo[port_id].buff_size;
+ } else
+ mm_tdesc = (q_data_size /
+ pinfo[port_id].buff_size);
+ } else {
+ mm_tdesc = 1;
+ mm_ld_size = q_data_size %
+ pinfo[port_id].buff_size;
+ }
+
+ if (port_id)
+ offset = (input_size/num_queues) * j;
+ else
+ offset = (input_size/num_queues) * i;
+
+ ret_val = lseek(ofd, offset, SEEK_SET);
+ if (ret_val == (off_t)-1) {
+ cmdline_printf(cl, "DMA-to-Device: "
+ "lseek func failed\n");
+ close(ofd);
+ return;
+ }
+#ifdef PERF_BENCHMARK
+ src_addr = (src_addr + (i * q_data_size)) %
+ BRAM_SIZE;
+#else
+ src_addr = src_addr + (i * q_data_size);
+#endif
+ ret =
+ update_queue_param(&rte_eth_devices[port_id], i,
+ RX_SRC_ADDRESS, src_addr);
+ if (ret < 0) {
+ close(ofd);
+ return;
+ }
+
+ cmdline_printf(cl, "DMA-from-Device: with "
+ "input-size:%d, ld_size:%d, "
+ "tot_num_desc:%d\n",
+ input_size, mm_ld_size,
+ mm_tdesc);
+
+ if ((unsigned int)i <
+ (pinfo[port_id].st_queues))
+ ret = do_recv_st(port_id, ofd, i,
+ q_data_size);
+ else
+ ret = do_recv_mm(port_id, ofd, i,
+ mm_ld_size,
+ mm_tdesc);
+ if (ret < 0) {
+ close(ofd);
+ return;
+ }
+
+ ret_val = lseek(ofd, offset, SEEK_END);
+ if (ret_val == (off_t)-1) {
+ cmdline_printf(cl, "DMA-to-Device: "
+ "lseek func failed\n");
+ close(ofd);
+ return;
+ }
+ }
+ ++loop;
+ } while (loop < num_loops);
+ close(ofd);
+ }
+ cmdline_printf(cl, "\n####### DMA transfer from device is completed "
+ "successfully #######\n");
+}
+
+cmdline_parse_token_string_t cmd_obj_action_dma_from_device =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result, action,
+ "dma_from_device");
+cmdline_parse_token_string_t cmd_obj_dma_from_device_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_from_device_queues =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result, queues,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_from_device_filename =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result,
+ filename, NULL);
+cmdline_parse_token_string_t cmd_obj_dma_from_device_src_addr =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result,
+ src_addr, NULL);
+cmdline_parse_token_string_t cmd_obj_dma_from_device_size =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result, size,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_dma_from_device_loops =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_dma_from_device_result, loops,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_dma_from_device = {
+ .f = cmd_obj_dma_from_device_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "dma_from_device port_id num-queues filename "
+ "src_addr size loops",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_dma_from_device,
+ (void *)&cmd_obj_dma_from_device_port_id,
+ (void *)&cmd_obj_dma_from_device_queues,
+ (void *)&cmd_obj_dma_from_device_filename,
+ (void *)&cmd_obj_dma_from_device_src_addr,
+ (void *)&cmd_obj_dma_from_device_size,
+ (void *)&cmd_obj_dma_from_device_loops,
+ NULL,
+ },
+
+};
+
+/* Command reg_dump addr
+ * Register Name - Addr Structure & Values
+ */
+struct S_NAME_INT_PAIR {
+ cmdline_fixed_string_t name;
+ int offset;
+ cmdline_fixed_string_t desc;
+};
+
+struct S_NAME_INT_PAIR C_REG_LIST[] = {
+ // Build/Status Registers
+ { "FPGA_VER ", 0x00000000,
+ "FPGA Version" },
+ /* Global Ring-size registers */
+ { "QDMA_GLBL_RNG_SZ_A[0] ", 0x00000204,
+ "Ring size index 0" },
+ { "QDMA_GLBL_RNG_SZ_A[1] ", 0x00000208,
+ "Ring size index 1" },
+ { "QDMA_GLBL_RNG_SZ_A[2] ", 0x0000020c,
+ "Ring size index 2" },
+ { "QDMA_GLBL_RNG_SZ_A[3] ", 0x00000210,
+ "Ring size index 3" },
+ { "QDMA_GLBL_RNG_SZ_A[4] ", 0x00000214,
+ "Ring size index 4" },
+ { "QDMA_GLBL_RNG_SZ_A[5] ", 0x00000218,
+ "Ring size index 5" },
+ { "QDMA_GLBL_RNG_SZ_A[6] ", 0x0000021c,
+ "Ring size index 6" },
+ { "QDMA_GLBL_RNG_SZ_A[7] ", 0x00000220,
+ "Ring size index 7" },
+ { "QDMA_GLBL_RNG_SZ_A[8] ", 0x00000224,
+ "Ring size index 8" },
+ { "QDMA_GLBL_RNG_SZ_A[9] ", 0x00000228,
+ "Ring size index 9" },
+ { "QDMA_GLBL_RNG_SZ_A[10] ", 0x0000022c,
+ "Ring size index 10" },
+ { "QDMA_GLBL_RNG_SZ_A[11] ", 0x00000230,
+ "Ring size index 11" },
+ { "QDMA_GLBL_RNG_SZ_A[12] ", 0x00000234,
+ "Ring size index 12" },
+ { "QDMA_GLBL_RNG_SZ_A[13] ", 0x00000238,
+ "Ring size index 13" },
+ { "QDMA_GLBL_RNG_SZ_A[14] ", 0x0000023c,
+ "Ring size index 14" },
+ { "QDMA_GLBL_RNG_SZ_A[15] ", 0x00000240,
+ "Ring size index 15" },
+ /* Global Error Status register */
+ { "QDMA_GLBL_ERR_STAT ", 0x00000248,
+ "Global error status" },
+ /*Global error mask register*/
+ { "QDMA_GLBL_ERR_MASK ", 0x0000024c,
+ "Global error mask" },
+ /* Global Write-back accumulation */
+ { "QDMA_GLBL_WB_ACC ", 0x00000250,
+ "Global Write-back accumulation" },
+ /* */
+ { "QDMA_GLBL_DSC_ERR_STS ", 0x00000254,
+ "Global Descriptor error status" },
+ /**/
+ { "QDMA_GLBL_DSC_ERR_MSK ", 0x00000258,
+ "Global Descriptor error mask" },
+ /**/
+ { "QDMA_GLBL_DSC_ERR_LOG0 ", 0x0000025c,
+ "Global Descriptor error Log0" },
+ /**/
+ { "QDMA_GLBL_DSC_ERR_LOG1 ", 0x00000260,
+ "Global Descriptor error Log1" },
+ /**/
+ { "QDMA_GLBL_TRQ_ERR_STS ", 0x00000264,
+ "Global target error status" },
+ /**/
+ { "QDMA_GLBL_TRQ_ERR_MSK ", 0x00000268,
+ "Global target error mask"},
+ /**/
+ { "QDMA_GLBL_TRQ_ERR_LOG ", 0x0000026c,
+ "Register access space, functio, address" },
+ /*Function-map registers*/
+ { "QDMA_TRQ_SEL_FMAP[0] ", 0x00000400,
+ "FMAP target index-0" },
+ { "QDMA_TRQ_SEL_FMAP[1] ", 0x00000404,
+ "FMAP target index-1" },
+ { "QDMA_C2H_STAT_AXIS_C2H_ACPTD ", 0x00000a88,
+ "Number of C2H pkts accepted" },
+ { "QDMA_C2H_STAT_AXIS_CMPT_ACPTD ", 0x00000a8c,
+ "Number of C2H CMPT pkts accepted" },
+ { "QDMA_C2H_STAT_DESC_RSP_PKT_ACPTD", 0x00000a90,
+ "Number of desc rsp pkts accepted" },
+ { "QDMA_C2H_STAT_AXIS_PKG_CMP ", 0x00000a94,
+ "Number of C2H pkts accepted" },
+ { "QDMA_C2H_STAT_DESC_RSP_ACPTD ", 0x00000a98,
+ "Number of dsc rsp pkts accepted" },
+ { "QDMA_C2H_STAT_DESC_RSP_CMP ", 0x00000a9c,
+ "Number of dsc rsp pkts completed" },
+ { "QDMA_C2H_STAT_WRQ_OUT ", 0x00000aa0,
+ "Number of WRQ" },
+ { "QDMA_C2H_STAT_WPL_REN_ACPTD ", 0x00000aa4,
+ "Number of WPL REN accepted" },
+ { "QDMA_C2H_STAT_TOT_WRQ_LEN ", 0x00000aa8,
+ "Number of total WRQ len from C2H DMA write engine" },
+ { "QDMA_C2H_STAT_TOT_WPL_LEN ", 0x00000aac,
+ "Number of total WPL len from C2H DMA write engine" },
+ { "QDMA_C2H_ERR_STAT ", 0x00000af0,
+ "C2H error status" },
+ { "QDMA_C2H_ERR_MASK ", 0x00000af4,
+ "C2H error mask" },
+ { "QDMA_C2H_FATAL_ERR_STAT ", 0x00000af8,
+ "C2H fatal error status" },
+ { "QDMA_C2H_FATAL_ERR_MSK ", 0x00000afc,
+ "C2H fatal error mask" },
+ { "QDMA_C2H_FATAL_ERR_MSK ", 0x00000afc,
+ "C2H fatal error mask" },
+ // End Of Register List
+ { "EndOfList", -999, "EndOfList" }
+};
+
+
+struct cmd_obj_reg_dump_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+};
+
+static void cmd_obj_reg_dump_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_reg_dump_result *res = parsed_result;
+
+
+ int bar_id;
+ int port_id = atoi(res->port_id);
+ unsigned int i, reg_val;
+ i = 0;
+
+ bar_id = pinfo[port_id].config_bar_idx;
+ if (bar_id < 0) {
+ cmdline_printf(cl, "Error: fetching QDMA config BAR-id "
+ "on port-id:%d not supported\n Please enter "
+ "valid port-id\n", port_id);
+ return;
+ }
+ cmdline_printf(cl, "Register dump on cofig BAR-id:%d with Port-id:%s\n",
+ bar_id, res->port_id);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n",
+ port_id);
+ return;
+ }
+ cmdline_printf(cl, "FPGA Registers\n--------------\n");
+ cmdline_printf(cl, "\tName:\t Offset:\tDescription:\t "
+ "register-Value:\n--------------\n");
+ do {
+ cmdline_printf(cl, "%18s : 0x%08x : %s:",
+ C_REG_LIST[i].name,
+ C_REG_LIST[i].offset,
+ C_REG_LIST[i].desc);
+ reg_val = PciRead(bar_id, C_REG_LIST[i].offset, port_id);
+ cmdline_printf(cl, "\tValue:0x%08x\n", reg_val);
+ i++;
+ } while (C_REG_LIST[i].offset != -999);
+}
+
+cmdline_parse_token_string_t cmd_obj_action_reg_dump =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_dump_result, action,
+ "reg_dump");
+cmdline_parse_token_string_t cmd_obj_reg_dump_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_reg_dump_result, port_id, NULL);
+
+cmdline_parse_inst_t cmd_obj_reg_dump = {
+ .f = cmd_obj_reg_dump_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "reg_dump port-id",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_reg_dump,
+ (void *)&cmd_obj_reg_dump_port_id,
+ NULL,
+ },
+
+};
+
+/*Command queue-context dump*/
+
+#define QDMA_IND_CTXT_CMD_A 0x844
+
+void queue_context_dump(uint8_t bar_id, uint32_t qid, struct cmdline *cl)
+{
+ union h2c_c2h_ctxt q_ctxt;
+ union c2h_cmpt_ctxt c2h_cmpt;
+ union h2c_c2h_hw_ctxt hw_ctxt;
+ union prefetch_ctxt pref_ctxt;
+ uint32_t ctxt_sel, reg_val;
+ uint32_t base_offset, offset;
+ uint16_t i;
+ uint8_t port_id = 0;
+
+ base_offset = (QDMA_TRQ_SEL_IND + QDMA_IND_Q_PRG_OFF);
+
+ /** To read the H2C Queue **/
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_H2C<<CTXT_SEL_SHIFT_B) |
+ (qid<<QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD<<OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ cmdline_printf(cl, "\nH2C context-data structure on queue-id:%d:\n",
+ qid);
+ for (i = 0; i < 5; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ q_ctxt.c_data.data[i] = reg_val;
+ }
+ cmdline_printf(cl, "\t\t interrupt vector:%x\n",
+ q_ctxt.c_fields.int_vec);
+ cmdline_printf(cl, "\t\t interrupt aggregation:%x\n",
+ q_ctxt.c_fields.int_aggr);
+ cmdline_printf(cl, "\t\t Base-addr of Desc ring:%lx\n",
+ q_ctxt.c_fields.dsc_base);
+ cmdline_printf(cl, "\t\t is_mm:%x\n", q_ctxt.c_fields.is_mm);
+ cmdline_printf(cl, "\t\t mrkr_dis:%x\n", q_ctxt.c_fields.mrkr_dis);
+ cmdline_printf(cl, "\t\t irq_req:%x\n", q_ctxt.c_fields.irq_req);
+ cmdline_printf(cl, "\t\t err-wb-sent:%x\n",
+ q_ctxt.c_fields.err_wb_sent);
+ cmdline_printf(cl, "\t\t Error status:%x\n", q_ctxt.c_fields.err);
+ cmdline_printf(cl, "\t\t irq_no_last:%x\n",
+ q_ctxt.c_fields.irq_no_last);
+ cmdline_printf(cl, "\t\t port id:%x\n", q_ctxt.c_fields.port_id);
+ cmdline_printf(cl, "\t\t irq-enable:%x\n", q_ctxt.c_fields.irq_en);
+ cmdline_printf(cl, "\t\t write-back enable:%x\n",
+ q_ctxt.c_fields.wbk_en);
+ cmdline_printf(cl, "\t\t mm-channel-id:%x\n", q_ctxt.c_fields.mm_chn);
+ cmdline_printf(cl, "\t\t bypass:%x\n", q_ctxt.c_fields.byp);
+ cmdline_printf(cl, "\t\t desc-size index:%x\n", q_ctxt.c_fields.dsc_sz);
+ cmdline_printf(cl, "\t\t ring-size index:%x\n", q_ctxt.c_fields.rng_sz);
+ cmdline_printf(cl, "\t\t reserved:%x\n", q_ctxt.c_fields.rsv1);
+ cmdline_printf(cl, "\t\t fetch_max:%x\n", q_ctxt.c_fields.fetch_max);
+ cmdline_printf(cl, "\t\t address type of context :%x\n",
+ q_ctxt.c_fields.at);
+ cmdline_printf(cl, "\t\t wbi_acc_en:%x\n", q_ctxt.c_fields.wbi_acc_en);
+ cmdline_printf(cl, "\t\t wbi_chk:%x\n", q_ctxt.c_fields.wbi_chk);
+ cmdline_printf(cl, "\t\t fetch credits:%x\n", q_ctxt.c_fields.fcrd_en);
+ cmdline_printf(cl, "\t\t queue-enable:%x\n", q_ctxt.c_fields.qen);
+ cmdline_printf(cl, "\t\t reserved:%x\n", q_ctxt.c_fields.rsv0);
+ cmdline_printf(cl, "\t\t function-id:%x\n", q_ctxt.c_fields.fnc_id);
+ cmdline_printf(cl, "\t\t irq_ack:%x\n", q_ctxt.c_fields.irq_ack);
+ cmdline_printf(cl, "\t\t producer-index:0x%x\n", q_ctxt.c_fields.pidx);
+
+ cmdline_printf(cl, "\nH2C Hardware Descriptor context-data structure "
+ "on queue-id:%d:\n", qid);
+
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_H2C<<CTXT_SEL_SHIFT_B) |
+ (qid<<QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD<<OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ for (i = 0; i < 2; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ hw_ctxt.c_data.data[i] = reg_val;
+ }
+ cmdline_printf(cl, "\t\t reserved:0x%x\n", hw_ctxt.c_fields.rsvd1);
+ cmdline_printf(cl, "\t\t descriptor fetch pending:0x%x\n",
+ hw_ctxt.c_fields.fetch_pend);
+ cmdline_printf(cl, "\t\t event pending:0x%x\n",
+ hw_ctxt.c_fields.event_pend);
+ cmdline_printf(cl, "\t\t Queue invalid Or no descriptor pending:0x%x\n",
+ hw_ctxt.c_fields.idl_stp_b);
+ cmdline_printf(cl, "\t\t descriptor pending:0x%x\n",
+ hw_ctxt.c_fields.pnd);
+ cmdline_printf(cl, "\t\t reserved:0x%x\n", hw_ctxt.c_fields.rsvd0);
+ cmdline_printf(cl, "\t\t credit-use:0x%x\n", hw_ctxt.c_fields.crd_use);
+ cmdline_printf(cl, "\t\t consumer-index:0x%x\n", hw_ctxt.c_fields.cidx);
+
+ /** To read the C2H Queue **/
+ cmdline_printf(cl, "\nC2H context-data structure on queue-id:%d:\n",
+ qid);
+
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_SW_C2H<<CTXT_SEL_SHIFT_B) |
+ (qid<<QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD<<OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ for (i = 0; i < 5; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ q_ctxt.c_data.data[i] = reg_val;
+ }
+ cmdline_printf(cl, "\t\t interrupt vector:%x\n",
+ q_ctxt.c_fields.int_vec);
+ cmdline_printf(cl, "\t\t interrupt aggregation:%x\n",
+ q_ctxt.c_fields.int_aggr);
+ cmdline_printf(cl, "\t\t Base-addr of Desc ring:%lx\n",
+ q_ctxt.c_fields.dsc_base);
+ cmdline_printf(cl, "\t\t is_mm:%x\n", q_ctxt.c_fields.is_mm);
+ cmdline_printf(cl, "\t\t mrkr_dis:%x\n", q_ctxt.c_fields.mrkr_dis);
+ cmdline_printf(cl, "\t\t irq_req:%x\n", q_ctxt.c_fields.irq_req);
+ cmdline_printf(cl, "\t\t err-wb-sent:%x\n",
+ q_ctxt.c_fields.err_wb_sent);
+ cmdline_printf(cl, "\t\t Error status:%x\n", q_ctxt.c_fields.err);
+ cmdline_printf(cl, "\t\t irq_no_last:%x\n",
+ q_ctxt.c_fields.irq_no_last);
+ cmdline_printf(cl, "\t\t port id:%x\n", q_ctxt.c_fields.port_id);
+ cmdline_printf(cl, "\t\t irq-enable:%x\n", q_ctxt.c_fields.irq_en);
+ cmdline_printf(cl, "\t\t write-back enable:%x\n",
+ q_ctxt.c_fields.wbk_en);
+ cmdline_printf(cl, "\t\t mm-channel-id:%x\n", q_ctxt.c_fields.mm_chn);
+ cmdline_printf(cl, "\t\t bypass:%x\n", q_ctxt.c_fields.byp);
+ cmdline_printf(cl, "\t\t desc-size index:%x\n",
+ q_ctxt.c_fields.dsc_sz);
+ cmdline_printf(cl, "\t\t ring-size index:%x\n",
+ q_ctxt.c_fields.rng_sz);
+ cmdline_printf(cl, "\t\t reserved:%x\n", q_ctxt.c_fields.rsv1);
+ cmdline_printf(cl, "\t\t fetch_max:%x\n", q_ctxt.c_fields.fetch_max);
+ cmdline_printf(cl, "\t\t address type of context :%x\n",
+ q_ctxt.c_fields.at);
+ cmdline_printf(cl, "\t\t wbi_acc_en:%x\n", q_ctxt.c_fields.wbi_acc_en);
+ cmdline_printf(cl, "\t\t wbi_chk:%x\n", q_ctxt.c_fields.wbi_chk);
+ cmdline_printf(cl, "\t\t fetch credits:%x\n", q_ctxt.c_fields.fcrd_en);
+ cmdline_printf(cl, "\t\t queue-enable:%x\n", q_ctxt.c_fields.qen);
+ cmdline_printf(cl, "\t\t reserved:%x\n", q_ctxt.c_fields.rsv0);
+ cmdline_printf(cl, "\t\t function-id:%x\n", q_ctxt.c_fields.fnc_id);
+ cmdline_printf(cl, "\t\t irq_ack:%x\n", q_ctxt.c_fields.irq_ack);
+ cmdline_printf(cl, "\t\t producer-index:0x%x\n", q_ctxt.c_fields.pidx);
+
+
+ /** C2H Completion context **/
+ cmdline_printf(cl, "\nC2H Completion context-data structure "
+ "on queue-id:%d:\n", qid);
+
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_CMPT << CTXT_SEL_SHIFT_B) |
+ (qid<<QID_SHIFT_B) | (QDMA_CTXT_CMD_RD<<OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ for (i = 0; i < 5; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ c2h_cmpt.c_data.data[i] = reg_val;
+ }
+
+ cmdline_printf(cl, "\t\t at:%x\n", c2h_cmpt.c_fields.at);
+ cmdline_printf(cl, "\t\t ovf_chk_dis:%x\n",
+ c2h_cmpt.c_fields.ovf_chk_dis);
+ cmdline_printf(cl, "\t\t full_upd:%x\n", c2h_cmpt.c_fields.full_upd);
+ cmdline_printf(cl, "\t\t timer_run:%x\n", c2h_cmpt.c_fields.timer_run);
+ cmdline_printf(cl, "\t\t usr_trig_pend:%x\n",
+ c2h_cmpt.c_fields.usr_trig_pend);
+ cmdline_printf(cl, "\t\t err:%x\n", c2h_cmpt.c_fields.err);
+ cmdline_printf(cl, "\t\t valid:%x\n", c2h_cmpt.c_fields.valid);
+ cmdline_printf(cl, "\t\t consumer-index:0x%x\n",
+ c2h_cmpt.c_fields.cidx);
+ cmdline_printf(cl, "\t\t producer-index:0x%x\n",
+ c2h_cmpt.c_fields.pidx);
+ cmdline_printf(cl, "\t\t desc-size:%x\n", c2h_cmpt.c_fields.desc_sz);
+ cmdline_printf(cl, "\t\t cmpt-desc-base_h addr:0x%x\n",
+ (unsigned int)c2h_cmpt.c_fields.cmpt_dsc_base_h);
+ cmdline_printf(cl, "\t\t cmpt-desc-base_l addr:0x%x\n",
+ (unsigned int)c2h_cmpt.c_fields.cmpt_dsc_base_l);
+ cmdline_printf(cl, "\t\t size-index:%x\n", c2h_cmpt.c_fields.size);
+ cmdline_printf(cl, "\t\t color:%x\n", c2h_cmpt.c_fields.color);
+ cmdline_printf(cl, "\t\t interrupt-state:%x\n",
+ c2h_cmpt.c_fields.int_st);
+ cmdline_printf(cl, "\t\t timer-index:0x%x\n",
+ c2h_cmpt.c_fields.timer_idx);
+ cmdline_printf(cl, "\t\t counter-index:0x%x\n",
+ c2h_cmpt.c_fields.count_idx);
+ cmdline_printf(cl, "\t\t function-id:0x%x\n", c2h_cmpt.c_fields.fnc_id);
+ cmdline_printf(cl, "\t\t trigger-mode:0x%x\n",
+ c2h_cmpt.c_fields.trig_mode);
+ cmdline_printf(cl, "\t\t cause interrupt on completion:0x%x\n",
+ c2h_cmpt.c_fields.en_int);
+ cmdline_printf(cl, "\t\t cause status descriptor write on "
+ "completion:0x%x\n", c2h_cmpt.c_fields.en_stat_desc);
+
+ /* Prefetch Context */
+ cmdline_printf(cl, "\nPrefetch context-data structure on "
+ "queue-id:%d:\n", qid);
+
+ ctxt_sel = (QDMA_CTXT_SEL_PFTCH << CTXT_SEL_SHIFT_B) |
+ (qid << QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD << OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ for (i = 0; i < 2; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ pref_ctxt.c_data.data[i] = reg_val;
+ }
+ cmdline_printf(cl, "\t\t valid:0x%x\n", pref_ctxt.c_fields.valid);
+ cmdline_printf(cl, "\t\t software credit:0x%x\n",
+ pref_ctxt.c_fields.sw_crdt);
+ cmdline_printf(cl, "\t\t queue is in prefetch:0x%x\n",
+ pref_ctxt.c_fields.pfch);
+ cmdline_printf(cl, "\t\t enable prefetch:0x%x\n",
+ pref_ctxt.c_fields.pfch_en);
+ cmdline_printf(cl, "\t\t err:0x%x\n", pref_ctxt.c_fields.err);
+ cmdline_printf(cl, "\t\t rsvd:0x%x\n", pref_ctxt.c_fields.rsvd);
+ cmdline_printf(cl, "\t\t port ID:0x%x\n", pref_ctxt.c_fields.port_id);
+ cmdline_printf(cl, "\t\t buffer size index:0x%x\n",
+ pref_ctxt.c_fields.buf_sz_idx);
+ cmdline_printf(cl, "\t\t C2H is in bypass mode:0x%x\n",
+ pref_ctxt.c_fields.bypass);
+
+ /* C2H Hardware descriptor context */
+ cmdline_printf(cl, "\nC2H Hardware Descriptor context-data structure "
+ "on queue-id:%d:\n", qid);
+
+ ctxt_sel = (QDMA_CTXT_SEL_DESC_HW_C2H<<CTXT_SEL_SHIFT_B) |
+ (qid<<QID_SHIFT_B) |
+ (QDMA_CTXT_CMD_RD<<OP_CODE_SHIFT_B);
+ PciWrite(bar_id, QDMA_IND_CTXT_CMD_A, ctxt_sel, port_id);
+
+ for (i = 0; i < 2; i++) {
+ offset = base_offset + (i * 4);
+ reg_val = PciRead(bar_id, offset, port_id);
+ hw_ctxt.c_data.data[i] = reg_val;
+ }
+ cmdline_printf(cl, "\t\t reserved:0x%x\n", hw_ctxt.c_fields.rsvd1);
+ cmdline_printf(cl, "\t\t descriptor fetch pending:0x%x\n",
+ hw_ctxt.c_fields.fetch_pend);
+ cmdline_printf(cl, "\t\t event pending:0x%x\n",
+ hw_ctxt.c_fields.event_pend);
+ cmdline_printf(cl, "\t\t Queue invalid Or no descriptor pending:0x%x\n",
+ hw_ctxt.c_fields.idl_stp_b);
+ cmdline_printf(cl, "\t\t descriptor pending:0x%x\n",
+ hw_ctxt.c_fields.pnd);
+ cmdline_printf(cl, "\t\t reserved:0x%x\n", hw_ctxt.c_fields.rsvd0);
+ cmdline_printf(cl, "\t\t credit-use:0x%x\n", hw_ctxt.c_fields.crd_use);
+ cmdline_printf(cl, "\t\t consumer-index:0x%x\n", hw_ctxt.c_fields.cidx);
+}
+
+struct cmd_obj_queue_dump_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t queue_id;
+};
+
+static void cmd_obj_queue_dump_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_queue_dump_result *res = parsed_result;
+
+ cmdline_printf(cl, "queue-dump on Port:%s, queue-id:%s\n\n",
+ res->port_id, res->queue_id);
+
+ {
+ int port_id = atoi(res->port_id);
+ int qid = atoi(res->queue_id);
+ int bar_id = 0x0;
+
+ bar_id = pinfo[port_id].config_bar_idx;
+ if (bar_id < 0) {
+ cmdline_printf(cl, "Error: fetching QDMA config BAR-id "
+ "on port-id:%d not supported\n Please "
+ "enter valid port-id\n", port_id);
+ return;
+ }
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n "
+ "Please enter valid port-id\n",
+ port_id);
+ return;
+ }
+ if ((unsigned int)qid >= pinfo[port_id].num_queues) {
+ cmdline_printf(cl, "Error: queue-id:%d is greater than "
+ "the number of confgiured queues in "
+ "the port\n Please enter valid "
+ "queue-id\n", qid);
+ return;
+ }
+ qid = qid + pinfo[port_id].queue_base;
+ queue_context_dump(bar_id, qid, cl);
+
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_queue_dump =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_queue_dump_result, action,
+ "queue_dump");
+cmdline_parse_token_string_t cmd_obj_queue_dump_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_queue_dump_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_queue_dump_queue_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_queue_dump_result, queue_id,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_queue_dump = {
+ .f = cmd_obj_queue_dump_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "queue_dump port-id queue_id",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_queue_dump,
+ (void *)&cmd_obj_queue_dump_port_id,
+ (void *)&cmd_obj_queue_dump_queue_id,
+ NULL,
+ },
+
+};
+
+/* Command descriptor dump */
+
+struct cmd_obj_desc_dump_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t port_id;
+ cmdline_fixed_string_t queue_id;
+};
+
+static void cmd_obj_desc_dump_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_desc_dump_result *res = parsed_result;
+
+ cmdline_printf(cl, "Descriptor-dump on Port:%s, queue-id:%s\n\n",
+ res->port_id, res->queue_id);
+ {
+ int port_id = atoi(res->port_id);
+ int qid = atoi(res->queue_id);
+ if (port_id >= num_ports) {
+ cmdline_printf(cl, "Error: port-id:%d not supported\n"
+ "Please enter valid port-id\n",
+ port_id);
+ return;
+ }
+ if ((unsigned int)qid >= pinfo[port_id].num_queues) {
+ cmdline_printf(cl, "Error: queue-id:%d is greater than "
+ "the number of confgiured queues in "
+ "the port\n Please enter valid "
+ "queue-id\n", qid);
+ return;
+ }
+ qdma_desc_dump(&rte_eth_devices[port_id], qid);
+ }
+}
+
+cmdline_parse_token_string_t cmd_obj_action_desc_dump =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_desc_dump_result, action,
+ "desc_dump");
+cmdline_parse_token_string_t cmd_obj_desc_dump_port_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_desc_dump_result, port_id,
+ NULL);
+cmdline_parse_token_string_t cmd_obj_desc_dump_queue_id =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_desc_dump_result, queue_id,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_desc_dump = {
+ .f = cmd_obj_desc_dump_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "desc_dump port-id queue_id",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_desc_dump,
+ (void *)&cmd_obj_desc_dump_port_id,
+ (void *)&cmd_obj_desc_dump_queue_id,
+ NULL,
+ },
+
+};
+
+/*Command load commands from file */
+
+struct cmd_obj_load_cmds_result {
+ cmdline_fixed_string_t action;
+ cmdline_fixed_string_t filename;
+};
+
+static void cmd_obj_load_cmds_parsed(void *parsed_result,
+ struct cmdline *cl,
+ __attribute__((unused)) void *data)
+{
+ struct cmd_obj_load_cmds_result *res = parsed_result;
+ FILE *fp;
+ char buff[256];
+
+ cmdline_printf(cl, "load-cmds from file:%s\n\n", res->filename);
+ fp = fopen((const char *)res->filename, "r");
+ if (fp == NULL) {
+ cmdline_printf(cl, "Error: Invalid filename: %s\n",
+ res->filename);
+ return;
+ }
+
+ rdline_reset(&cl->rdl);
+ {
+ cmdline_in(cl, "\r", 1);
+ while (fgets(buff, sizeof(buff), fp))
+ cmdline_in(cl, buff, strlen(buff));
+
+ cmdline_in(cl, "\r", 1);
+ }
+ fclose(fp);
+}
+
+cmdline_parse_token_string_t cmd_obj_action_load_cmds =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_load_cmds_result, action,
+ "load_cmds");
+cmdline_parse_token_string_t cmd_obj_load_cmds_filename =
+ TOKEN_STRING_INITIALIZER(struct cmd_obj_load_cmds_result, filename,
+ NULL);
+
+cmdline_parse_inst_t cmd_obj_load_cmds = {
+ .f = cmd_obj_load_cmds_parsed, /* function to call */
+ .data = NULL, /* 2nd arg of func */
+ .help_str = "load_cmds file-name",
+ .tokens = { /* token list, NULL terminated */
+ (void *)&cmd_obj_action_load_cmds,
+ (void *)&cmd_obj_load_cmds_filename,
+ NULL,
+ },
+
+};
+
+/* CONTEXT (list of instruction) */
+
+cmdline_parse_ctx_t main_ctx[] = {
+ (cmdline_parse_inst_t *)&cmd_obj_port_init,
+ (cmdline_parse_inst_t *)&cmd_obj_port_close,
+ (cmdline_parse_inst_t *)&cmd_obj_reg_read,
+ (cmdline_parse_inst_t *)&cmd_obj_reg_write,
+ (cmdline_parse_inst_t *)&cmd_obj_dma_to_device,
+ (cmdline_parse_inst_t *)&cmd_obj_dma_from_device,
+ (cmdline_parse_inst_t *)&cmd_obj_reg_dump,
+ (cmdline_parse_inst_t *)&cmd_obj_queue_dump,
+ (cmdline_parse_inst_t *)&cmd_obj_desc_dump,
+ (cmdline_parse_inst_t *)&cmd_obj_load_cmds,
+ (cmdline_parse_inst_t *)&cmd_help,
+ NULL,
+};
diff --git a/QDMA/DPDK/examples/qdma_testapp/commands.h b/QDMA/DPDK/examples/qdma_testapp/commands.h
new file mode 100644
index 0000000000000000000000000000000000000000..4bae349b4997fd57fa1ecb816757c0599de11d8b
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/commands.h
@@ -0,0 +1,38 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _COMMANDS_H_
+#define _COMMANDS_H_
+
+extern cmdline_parse_ctx_t main_ctx[];
+#endif /* _COMMANDS_H_ */
diff --git a/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.c b/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.c
new file mode 100644
index 0000000000000000000000000000000000000000..66f4df0906c140406ccb61d44e2eec7e45ad99c6
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.c
@@ -0,0 +1,196 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2009, Olivier MATZ <zer0@droids-corp.org>
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the University of California, Berkeley nor the
+ * names of its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <ctype.h>
+#include <string.h>
+#include <netinet/in.h>
+
+#include <cmdline_parse.h>
+#include <cmdline_parse_ipaddr.h>
+
+#include <rte_string_fns.h>
+
+#include "parse_obj_list.h"
+
+/* This file is an example of extension of libcmdline. It provides an
+ * example of objects stored in a list
+ */
+
+struct cmdline_token_ops token_obj_list_ops = {
+ .parse = parse_obj_list,
+ .complete_get_nb = complete_get_nb_obj_list,
+ .complete_get_elt = complete_get_elt_obj_list,
+ .get_help = get_help_obj_list,
+};
+
+int
+parse_obj_list(cmdline_parse_token_hdr_t *tk, const char *buf, void *res,
+ unsigned int buf_len)
+{
+ struct token_obj_list *tk2 = (struct token_obj_list *)tk;
+ struct token_obj_list_data *tkd = &tk2->obj_list_data;
+ struct object *o;
+ unsigned int token_len = 0;
+
+ if (*buf == 0)
+ return -1;
+
+ while (!cmdline_isendoftoken(buf[token_len]))
+ token_len++;
+
+ SLIST_FOREACH(o, tkd->list, next) {
+ if (token_len != strnlen(o->name, OBJ_NAME_LEN_MAX))
+ continue;
+ if (strncmp(buf, o->name, token_len))
+ continue;
+ break;
+ }
+ if (!o) /* not found */
+ return -1;
+
+ /* store the address of object in structure */
+ if (res)
+ *(struct object **)res = o;
+
+ return token_len;
+}
+
+int complete_get_nb_obj_list(cmdline_parse_token_hdr_t *tk)
+{
+ struct token_obj_list *tk2 = (struct token_obj_list *)tk;
+ struct token_obj_list_data *tkd = &tk2->obj_list_data;
+ struct object *o;
+ int ret = 0;
+
+ SLIST_FOREACH(o, tkd->list, next) {
+ ret++;
+ }
+ return ret;
+}
+
+int complete_get_elt_obj_list(cmdline_parse_token_hdr_t *tk,
+ int idx, char *dstbuf, unsigned int size)
+{
+ struct token_obj_list *tk2 = (struct token_obj_list *)tk;
+ struct token_obj_list_data *tkd = &tk2->obj_list_data;
+ struct object *o;
+ int i = 0;
+ unsigned int len;
+
+ SLIST_FOREACH(o, tkd->list, next) {
+ if (i++ == idx)
+ break;
+ }
+ if (!o)
+ return -1;
+
+ len = strnlen(o->name, OBJ_NAME_LEN_MAX);
+ if ((len + 1) > size)
+ return -1;
+
+ if (dstbuf)
+ snprintf(dstbuf, size, "%s", o->name);
+
+ return 0;
+}
+
+
+int get_help_obj_list(__attribute__((unused)) cmdline_parse_token_hdr_t *tk,
+ char *dstbuf, unsigned int size)
+{
+ snprintf(dstbuf, size, "Obj-List");
+ return 0;
+}
diff --git a/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.h b/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.h
new file mode 100644
index 0000000000000000000000000000000000000000..aa754ba8581d9b1330ea6fa8f5b4edf4a0245601
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/parse_obj_list.h
@@ -0,0 +1,146 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 2009, Olivier MATZ <zer0@droids-corp.org>
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of the University of California, Berkeley nor the
+ * names of its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _PARSE_OBJ_LIST_H_
+#define _PARSE_OBJ_LIST_H_
+
+/* This file is an example of extension of libcmdline. It provides an
+ * example of objects stored in a list.
+ */
+
+#include <sys/queue.h>
+#include <cmdline_parse.h>
+
+#define OBJ_NAME_LEN_MAX 64
+
+struct object {
+ SLIST_ENTRY(object) next;
+ char name[OBJ_NAME_LEN_MAX];
+ cmdline_ipaddr_t ip;
+};
+
+/* define struct object_list */
+SLIST_HEAD(object_list, object);
+
+/* data is a pointer to a list */
+struct token_obj_list_data {
+ struct object_list *list;
+};
+
+struct token_obj_list {
+ struct cmdline_token_hdr hdr;
+ struct token_obj_list_data obj_list_data;
+};
+typedef struct token_obj_list parse_token_obj_list_t;
+
+extern struct cmdline_token_ops token_obj_list_ops;
+
+int parse_obj_list(cmdline_parse_token_hdr_t *tk, const char *srcbuf, void *res,
+ unsigned int buf_len);
+int complete_get_nb_obj_list(cmdline_parse_token_hdr_t *tk);
+int complete_get_elt_obj_list(cmdline_parse_token_hdr_t *tk, int idx,
+ char *dstbuf, unsigned int size);
+int get_help_obj_list(cmdline_parse_token_hdr_t *tk, char *dstbuf,
+ unsigned int size);
+
+#define TOKEN_OBJ_LIST_INITIALIZER(structure, field, obj_list_ptr) \
+{ \
+ .hdr = { \
+ .ops = &token_obj_list_ops, \
+ .offset = offsetof(structure, field), \
+ }, \
+ .obj_list_data = { \
+ .list = obj_list_ptr, \
+ }, \
+}
+
+#endif /* _PARSE_OBJ_LIST_H_ */
diff --git a/QDMA/DPDK/examples/qdma_testapp/pcierw.c b/QDMA/DPDK/examples/qdma_testapp/pcierw.c
new file mode 100644
index 0000000000000000000000000000000000000000..0e384dcb076cb79e0ddd50f1b1d9a300b9ed0af8
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/pcierw.c
@@ -0,0 +1,56 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <rte_ethdev.h>
+
+#include "pcierw.h"
+
+unsigned int PciRead(unsigned int Bar, unsigned int Offset, int portid)
+{
+
+ return qdma_pci_read_reg(&rte_eth_devices[portid], Bar, Offset);
+}
+
+
+void PciWrite(unsigned int Bar, unsigned int Offset, unsigned int RegVal,
+ int portid)
+{
+
+ qdma_pci_write_reg(&rte_eth_devices[portid], Bar, Offset, RegVal);
+
+}
diff --git a/QDMA/DPDK/examples/qdma_testapp/pcierw.h b/QDMA/DPDK/examples/qdma_testapp/pcierw.h
new file mode 100644
index 0000000000000000000000000000000000000000..f4a7b2a8fbab3b0b769115fb01c626c6a4771115
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/pcierw.h
@@ -0,0 +1,42 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __PCIERW_H__
+#define __PCIERW_H__
+
+unsigned int PciRead(unsigned int Bar, unsigned int Offset, int SockFD);
+void PciWrite(unsigned int Bar, unsigned int Offset, unsigned int RegVal,
+ int SockFD);
+void qdma_pci_write_reg(struct rte_eth_dev *dev, uint32_t bar, uint32_t reg,
+ uint32_t val);
+uint32_t qdma_pci_read_reg(struct rte_eth_dev *dev, uint32_t bar, uint32_t reg);
+#endif
diff --git a/QDMA/DPDK/examples/qdma_testapp/qdma_regs.h b/QDMA/DPDK/examples/qdma_testapp/qdma_regs.h
new file mode 100644
index 0000000000000000000000000000000000000000..c8e0d07b11bf1c8b325be069f002f360fa017c43
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/qdma_regs.h
@@ -0,0 +1,235 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/** Target definations **/
+#define QDMA_TRQ_SEL_GLBL 0x00000200
+#define QDMA_TRQ_SEL_FMAP 0x00000400
+#define QDMA_TRQ_SEL_IND 0x00000800
+#define QDMA_TRQ_SEL_C2H 0x00000A00
+#define QDMA_TRQ_SEL_H2H 0x00000E00
+#define QDMA_TRQ_SEL_C2H_MM0 0x00001000
+#define QDMA_TRQ_SEL_H2C_MM0 0x00001200
+#define QDMA_TRQ_SEL_QUEUE_PF 0x00006400
+
+#define QDMA_CONFIG_BLOCK_ID 0x1fd00000UL
+/** Global registers **/
+#define QDMA_GLBL_RING_SZ 0x04
+#define QDMA_GLBL_SCRATCH 0x44
+#define QDMA_GLBL_WB_ACC 0x50
+#define QDMA_RING_SZ_MSK 0x0000ffff
+#define QDMA_WB_ACC_MSK 0x00000007
+
+/** Fmap registers **/
+#define QID_BASE_MSK (0x000007ff)
+#define QID_MAX_MSK (0x003ff800)
+#define QID_MAX_SHIFT_B (11)
+
+/** Queue Indirect programming commands **/
+
+#define QDMA_IND_Q_PRG_OFF (0x4)
+
+#define QDMA_CTXT_CMD_CLR (0)
+#define QDMA_CTXT_CMD_WR (1)
+#define QDMA_CTXT_CMD_RD (2)
+#define QDMA_CTXT_CMD_INV (3)
+
+#define QDMA_CTXT_SEL_DESC_SW_C2H (0)
+#define QDMA_CTXT_SEL_DESC_SW_H2C (1)
+#define QDMA_CTXT_SEL_DESC_HW_C2H (2)
+#define QDMA_CTXT_SEL_DESC_HW_H2C (3)
+#define QDMA_CTXT_SEL_CR_C2H (4)
+#define QDMA_CTXT_SEL_CR_H2C (5)
+#define QDMA_CTXT_SEL_DESC_CMPT (6)
+#define QDMA_CTXT_SEL_PFTCH (7)
+
+#define QID_SHIFT_B (7)
+#define OP_CODE_SHIFT_B (5)
+#define CTXT_SEL_SHIFT_B (1)
+#define BUSY_BIT_MSK (1)
+
+#define WB_EN_SHIFT_B (20)
+#define MM_CHAN_SHIFT_B (19)
+#define MM_DESC_SZ_SHIFT_B (17)
+#define ST_H2C_DESC_SZ_SHIFT_B (16)
+#define DESC_RING_SZ_SHIFT_B (12)
+#define ST_H2C_DESC_SZ_SHIFT_B (16)
+#define MM_DESC_SZ_WB_SHIFT_B (29)
+#define C2H_WB_CTXT_V_SHIFT_B (24)
+
+/** C2H target registers **/
+#define QDMA_C2H_CNT_TH_BASE 0x40
+#define QDMA_C2H_BUF_SZ_BASE 0xB0
+
+/** PF Queue index registers */
+#define QDMA_H2C_PIDX_Q_OFF (0x04)
+#define QDMA_C2H_PIDX_Q_OFF (0x08)
+#define QDMA_SEL_CMPT_CIDX_Q_OFF (0x0c)
+
+
+/** QDMA Target registers **/
+#define QDMA_C2H_MM0_CONTROL 0x00000004
+#define QDMA_H2C_MM0_CONTROL 0x00000004
+#define QDMA_MM_CTRL_START (1 << 0)
+
+/** QDMA Descriptor definations **/
+#define QDMA_DESC_SOP 0x1
+#define QDMA_DESC_EOP 0x1
+#define QDMA_DESC_VALID 0x1
+
+
+/** Queue Indirect programming registers **/
+struct __attribute__ ((packed)) q_ind_prg
+{
+ uint32_t ctxt_data[8];
+ uint32_t ctxt_mask[8];
+ uint32_t ctxt_cmd;
+};
+
+union __attribute__ ((packed)) h2c_c2h_ctxt
+{
+ struct __attribute__ ((packed)) ctxt_data
+ {
+ uint32_t data[5];
+ } c_data;
+ struct __attribute__ ((packed)) ctxt_fields
+ {
+ uint16_t pidx;
+ uint16_t irq_ack:1;
+ uint16_t fnc_id:8;
+ uint16_t rsv0:7;
+ uint16_t qen:1;
+ uint16_t fcrd_en:1;
+ uint16_t wbi_chk:1;
+ uint16_t wbi_acc_en:1;
+ uint16_t at:1;
+ uint16_t fetch_max:3;
+ uint16_t rsv1:4;
+ uint16_t rng_sz:4;
+ uint16_t dsc_sz:2;
+ uint16_t byp:1;
+ uint16_t mm_chn:1;
+ uint16_t wbk_en:1;
+ uint16_t irq_en:1;
+ uint16_t port_id:3;
+ uint16_t irq_no_last:1;
+ uint16_t err:2;
+ uint16_t err_wb_sent:1;
+ uint16_t irq_req:1;
+ uint16_t mrkr_dis:1;
+ uint16_t is_mm:1;
+ uint64_t dsc_base;
+ uint16_t int_vec:11;
+ uint16_t int_aggr:1;
+ } c_fields;
+};
+
+union __attribute__ ((packed)) c2h_cmpt_ctxt
+{
+ struct __attribute__ ((packed)) c2h_cmpt_data
+ {
+ uint32_t data[5];
+ } c_data;
+ struct __attribute__ ((packed)) c2h_cmpt_fields
+ {
+ uint32_t en_stat_desc:1;
+ uint32_t en_int:1;
+ uint32_t trig_mode:3;
+ uint32_t fnc_id:12;
+ uint32_t count_idx:4;
+ uint32_t timer_idx:4;
+ uint32_t int_st:2;
+ uint32_t color:1;
+ uint32_t size:4;
+ uint32_t cmpt_dsc_base_l;
+ uint32_t cmpt_dsc_base_h:26;
+ uint32_t desc_sz:2;
+ uint32_t pidx:16;
+ uint32_t cidx:16;
+ uint32_t valid:1;
+ uint32_t err:2;
+ uint32_t usr_trig_pend:1;
+ uint32_t timer_run:1;
+ uint32_t full_upd:1;
+ uint32_t ovf_chk_dis:1;
+ uint32_t at:1;
+ } c_fields;
+};
+
+union __attribute__ ((packed)) h2c_c2h_hw_ctxt
+{
+ struct __attribute__ ((packed)) hw_ctxt_data
+ {
+ uint32_t data[2];
+ } c_data;
+ struct __attribute__ ((packed)) hw_ctxt_fields
+ {
+ uint32_t cidx:16;
+ uint32_t crd_use:16;
+ uint32_t rsvd0:8;
+ uint32_t pnd:1;
+ uint32_t idl_stp_b:1;
+ uint32_t event_pend:1;
+ uint32_t fetch_pend:4;
+ uint32_t rsvd1:1;
+ } c_fields;
+};
+
+union __attribute__ ((packed)) prefetch_ctxt
+{
+ struct __attribute__ ((packed)) pref_ctxt_data
+ {
+ uint32_t data[2];
+ } c_data;
+
+ struct __attribute__ ((packed)) pref_ctxt_fields
+ {
+ uint8_t bypass:1;
+ uint8_t buf_sz_idx:4;
+ uint8_t port_id:3;
+ uint32_t rsvd:18;
+ uint8_t err:1;
+ uint8_t pfch_en:1;
+ uint8_t pfch:1;
+ uint16_t sw_crdt:16;
+ uint8_t valid:1;
+ } c_fields;
+};
+
+#define PIDX_MSK (0)
+#define Q_STATUS_MSK (0)
+#define Q_STATUS_EN_MSK (3)
+#define Q_STATUS_RST_MSK (1)
+#define WBI_CHK_MSK (6)
+#define WBI_ACC_EN_MSK (7)
+#define FUNC_ID_MSK (8)
+#define RING_SZ_MSK (16)
+#define DESC_SZ_MSK (16)
diff --git a/QDMA/DPDK/examples/qdma_testapp/testapp.c b/QDMA/DPDK/examples/qdma_testapp/testapp.c
new file mode 100644
index 0000000000000000000000000000000000000000..5ec7ca047142fac7eb6317dd3c1e1b322b1aaafd
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/testapp.c
@@ -0,0 +1,795 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h> /**> memset */
+#include <signal.h>
+#include <termios.h>
+#include <rte_eal.h> /**> rte_eal_init */
+#include <rte_debug.h> /**> for rte_panic */
+#include <rte_ethdev.h> /**> rte_eth_rx_burst */
+#include <rte_errno.h> /**> rte_errno global var */
+#include <rte_memzone.h> /**> rte_memzone_dump */
+#include <rte_memcpy.h>
+#include <rte_malloc.h>
+#include <rte_cycles.h>
+#include <rte_log.h>
+#include <rte_string_fns.h>
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline.h>
+#include <time.h> /** For SLEEP **/
+#include <getopt.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <rte_mbuf.h>
+
+#include "pcierw.h"
+#include "commands.h"
+#include "qdma_regs.h"
+#include "testapp.h"
+#include "../../drivers/net/qdma/qdma.h"
+
+int num_ports;
+char *filename;
+
+struct port_info pinfo[QDMA_MAX_PORTS];
+
+int do_recv_mm(int portid, int fd, int queueid, int ld_size, int tot_num_desc)
+{
+ struct rte_mbuf *pkts[NUM_RX_PKTS] = { NULL };
+ int nb_rx = 0, i = 0, ret = 0, num_pkts;
+ int tdesc;
+#ifdef PERF_BENCHMARK
+ uint64_t prev_tsc, cur_tsc, diff_tsc;
+#endif
+
+ if (tot_num_desc == 0) {
+ printf("Error: tot_num_desc : invalid value\n");
+ return -1;
+ }
+
+ printf("recv start num-desc: %d, with data-len: %d, "
+ "last-desc-size:%d\n",
+ tot_num_desc, pinfo[portid].buff_size, ld_size);
+ tdesc = tot_num_desc;
+ if (ld_size)
+ tdesc--;
+
+ while (tdesc) {
+ if (tdesc > NUM_RX_PKTS)
+ num_pkts = NUM_RX_PKTS;
+ else
+ num_pkts = tdesc;
+#ifdef PERF_BENCHMARK
+ prev_tsc = rte_rdtsc_precise();
+#endif
+ /* try to receive RX_BURST_SZ packets */
+ nb_rx = rte_eth_rx_burst(portid, queueid, pkts, num_pkts);
+
+#ifdef PERF_BENCHMARK
+ cur_tsc = rte_rdtsc_precise();
+ diff_tsc = cur_tsc - prev_tsc;
+#endif
+
+ if (nb_rx == 0) {
+ printf("Error: dma_from_device failed to "
+ "receive packets\n");
+ return -1;
+ }
+#ifdef PERF_BENCHMARK
+ /* Calculate average operations processed per second */
+ double pkts_per_second = ((double)nb_rx * rte_get_tsc_hz() /
+ diff_tsc);
+
+ /* Calculate average throughput (Gbps) in bits per second */
+ double throughput_gbps = ((pkts_per_second *
+ pinfo[portid].buff_size) / 1000000000);
+ printf("Throughput GBps %lf\n", throughput_gbps);
+ printf("%16s%16s%16s%16s%16s%16s%16s\n\n",
+ "Buf Size", "Burst Size",
+ "pps", "Gbps", "freq", "Cycles",
+ "Cycles/Buf");
+
+ printf("%16u%16u%16.4lf%16.4lf%16 "
+ ""PRIu64"%16"PRIu64"%16"PRIu64"\n",
+ pinfo[portid].buff_size,
+ nb_rx,
+ pkts_per_second,
+ throughput_gbps,
+ rte_get_tsc_hz(),
+ diff_tsc,
+ diff_tsc/nb_rx);
+#endif
+
+ for (i = 0; i < nb_rx; i++) {
+ struct rte_mbuf *mb = pkts[i];
+ ret = write(fd, rte_pktmbuf_mtod(mb, void*),
+ pinfo[portid].buff_size);
+ rte_pktmbuf_free(mb);
+#ifndef PERF_BENCHMARK
+ printf("recv count: %d, with data-len: %d\n", i, ret);
+#endif
+ }
+ tdesc = tdesc - nb_rx;
+ }
+ if (ld_size) {
+ struct rte_mbuf *mb;
+ nb_rx = rte_eth_rx_burst(portid, queueid, pkts, 1);
+ if (nb_rx != 0) {
+ mb = pkts[0];
+ ret = write(fd, rte_pktmbuf_mtod(mb, void*), ld_size);
+ rte_pktmbuf_free(mb);
+ }
+ }
+ fsync(fd);
+ printf("\nDMA received number of packets: %d, on queue-id:%d\n",
+ nb_rx, queueid);
+ return 0;
+}
+
+int do_recv_st(int portid, int fd, int queueid, int input_size)
+{
+ struct rte_mbuf *pkts[NUM_RX_PKTS] = { NULL };
+ int nb_rx = 0, i, ret = 0, tmp = 0, num_pkts, nb_pkts;
+ int reg_val, num_pkts_recv = 0;
+ int regval;
+ int user_bar_idx;
+ struct rte_mbuf *nxtmb;
+ int qbase = pinfo[portid].queue_base, diag;
+ unsigned int max_completion_size;
+ unsigned int max_rx_retry;
+
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ struct rte_mempool *mp;
+ mp = rte_mempool_lookup(pinfo[portid].mem_pool);
+
+ /* get the mempool from which to acquire buffers */
+ if (mp == NULL)
+ rte_exit(EXIT_FAILURE, "Could not find mempool with name %s\n",
+ pinfo[portid].mem_pool);
+#endif //DUMP_MEMPOOL_USAGE_STATS
+
+ user_bar_idx = pinfo[portid].user_bar_idx;
+ PciWrite(user_bar_idx, C2H_ST_QID_REG, (queueid + qbase), portid);
+
+ /* As per hardware design a single completion will point to atmost
+ * 7 descriptors. So If the size of the buffer in descriptor is 4KB ,
+ * then a single completion which corresponds a packet can give you
+ * atmost 28KB data.
+ *
+ * As per this when testing sizes beyond 28KB, one needs to split it
+ * up in chunks of 28KB, example : to test 56KB data size, set 28KB
+ * as packet length in USER BAR 0x04 register and no of packets as 2
+ * in user BAR 0x20 register this would give you completions or
+ * packets, which needs to be combined as one in application.
+ */
+
+ max_completion_size = pinfo[portid].buff_size * 7;
+
+ /* Calculate number of packets to receive and programming user bar */
+ if (input_size == 0) /* zerobyte support uses one descriptor */
+ num_pkts = 1;
+ else if (input_size % max_completion_size != 0)
+ num_pkts = input_size / max_completion_size + 1;
+ else
+ num_pkts = input_size / max_completion_size;
+
+ reg_val = PciRead(user_bar_idx, C2H_CONTROL_REG, portid);
+ reg_val &= C2H_CONTROL_REG_MASK;
+ if (!(reg_val & ST_LOOPBACK_EN)) {
+ PciWrite(user_bar_idx, C2H_PACKET_COUNT_REG, num_pkts, portid);
+
+ if (num_pkts > 1)
+ PciWrite(user_bar_idx, C2H_ST_LEN_REG,
+ max_completion_size, portid);
+ else
+ PciWrite(user_bar_idx, C2H_ST_LEN_REG, input_size,
+ portid);
+
+ regval = PciRead(user_bar_idx, C2H_PACKET_COUNT_REG, portid);
+ printf("BAR-%d is the QDMA C2H number of packets:0x%x,\n",
+ user_bar_idx, regval);
+ }
+
+ while (num_pkts) {
+ if (num_pkts > NUM_RX_PKTS)
+ nb_pkts = NUM_RX_PKTS;
+ else
+ nb_pkts = num_pkts;
+
+ max_rx_retry = RX_TX_MAX_RETRY;
+ /* Start the C2H Engine */
+ reg_val = PciRead(user_bar_idx, C2H_CONTROL_REG, portid);
+ reg_val &= C2H_CONTROL_REG_MASK;
+ if (!(reg_val & ST_LOOPBACK_EN)) {
+ reg_val |= ST_C2H_START_VAL;
+ PciWrite(user_bar_idx, C2H_CONTROL_REG, reg_val,
+ portid);
+ }
+ /* Immediate data Enabled*/
+ if ((reg_val & ST_C2H_IMMEDIATE_DATA_EN)) {
+ /* payload received is zero for the immediate data case.
+ * Therefore, no need to call the rx_burst function
+ * again in this case and set the num_pkts to nb_rx
+ * which is always Zero.
+ */
+ diag = update_queue_param(&rte_eth_devices[portid],
+ queueid,
+ DUMP_IMMEDIATE_DATA, 1);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "updata_queue_param : "
+ "Passing of DUMP_IMMEDIATE_DATA "
+ "failed\n");
+ nb_rx = rte_eth_rx_burst(portid, queueid, pkts,
+ nb_pkts);
+ num_pkts = num_pkts_recv = nb_rx;
+
+ /* Reset the queue's DUMP_IMMEDIATE_DATA mode */
+ diag = update_queue_param(&rte_eth_devices[portid],
+ queueid,
+ DUMP_IMMEDIATE_DATA, 0);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "updata_queue_param : "
+ "Passing of DUMP_IMMEDIATE_DATA "
+ "failed\n");
+ } else {
+ /* try to receive RX_BURST_SZ packets */
+
+ nb_rx = rte_eth_rx_burst(portid, queueid, pkts,
+ nb_pkts);
+ tmp = nb_rx;
+ while ((nb_rx < nb_pkts) && max_rx_retry) {
+ printf("Couldn't receive all the packets: "
+ "Expected = %d Received = %d.\n"
+ "Calling rte_eth_rx_burst "
+ "again\n", nb_pkts, nb_rx);
+ nb_pkts -= nb_rx;
+ nb_rx = rte_eth_rx_burst(portid, queueid,
+ &pkts[tmp],
+ nb_pkts);
+ tmp += nb_rx;
+ max_rx_retry--;
+ }
+ num_pkts_recv = tmp;
+ if ((max_rx_retry == 0) && (num_pkts_recv == 0)) {
+ printf("ERROR: rte_eth_rx_burst failed for "
+ "port %d queue id %d\n",
+ portid, queueid);
+ break;
+ }
+ }
+
+ /* Stop the C2H Engine */
+ reg_val = PciRead(user_bar_idx, C2H_CONTROL_REG, portid);
+ reg_val &= C2H_CONTROL_REG_MASK;
+ if (!(reg_val & ST_LOOPBACK_EN)) {
+ reg_val &= ~(ST_C2H_START_VAL);
+ PciWrite(user_bar_idx, C2H_CONTROL_REG, reg_val,
+ portid);
+ }
+
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ printf("%s(): %d: queue id = %d, mbuf_avail_count = %d, "
+ "mbuf_in_use_count = %d\n",
+ __func__, __LINE__, queueid,
+ rte_mempool_avail_count(mp),
+ rte_mempool_in_use_count(mp));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ for (i = 0; i < num_pkts_recv; i++) {
+ struct rte_mbuf *mb = pkts[i];
+ while (mb != NULL) {
+ ret += write(fd, rte_pktmbuf_mtod(mb, void*),
+ rte_pktmbuf_data_len(mb));
+ nxtmb = mb->next;
+ mb = nxtmb;
+ }
+
+ mb = pkts[i];
+ rte_pktmbuf_free(mb);
+ printf("recv count: %d, with data-len: %d\n", i, ret);
+ ret = 0;
+ }
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ printf("%s(): %d: queue id = %d, mbuf_avail_count = %d, "
+ "mbuf_in_use_count = %d, num_pkts_recv = %d\n",
+ __func__, __LINE__, queueid,
+ rte_mempool_avail_count(mp),
+ rte_mempool_in_use_count(mp), num_pkts_recv);
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ num_pkts = num_pkts - num_pkts_recv;
+ printf("\nDMA received number of packets: %d, on queue-id:%d\n",
+ num_pkts_recv, queueid);
+ }
+
+ fsync(fd);
+ return 0;
+}
+
+int do_xmit(int portid, int fd, int queueid, int ld_size, int tot_num_desc,
+ int zbyte)
+{
+ struct rte_mempool *mp;
+ struct rte_mbuf *mb[NUM_RX_PKTS] = { NULL };
+ int ret = 0, nb_tx, i = 0, tdesc, num_pkts = 0, total_tx = 0, reg_val;
+ int tmp = 0, user_bar_idx;
+ int qbase = pinfo[portid].queue_base;
+ uint32_t max_tx_retry;
+
+#ifdef PERF_BENCHMARK
+ uint64_t prev_tsc, cur_tsc, diff_tsc;
+#endif
+ mp = rte_mempool_lookup(pinfo[portid].mem_pool);
+ /* get the mempool from which to acquire buffers */
+ if (mp == NULL)
+ rte_exit(EXIT_FAILURE, "Could not find mempool with name %s\n",
+ pinfo[portid].mem_pool);
+ tdesc = tot_num_desc;
+ user_bar_idx = pinfo[portid].user_bar_idx;
+
+ if (ld_size)
+ tdesc--;
+
+ while (tdesc) {
+ if (tdesc > NUM_RX_PKTS)
+ num_pkts = NUM_RX_PKTS;
+ else
+ num_pkts = tdesc;
+
+ max_tx_retry = RX_TX_MAX_RETRY;
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ printf("%s(): %d: queue id %d, mbuf_avail_count = %d, "
+ "mbuf_in_use_count = %d",
+ __func__, __LINE__, queueid,
+ rte_mempool_avail_count(mp),
+ rte_mempool_in_use_count(mp));
+#endif //DUMP_MEMPOOL_USAGE_STATS
+ for (i = 0; i < num_pkts; i++) {
+ mb[i] = rte_pktmbuf_alloc(mp);
+ if (mb[i] == NULL)
+ rte_exit(EXIT_FAILURE, " #####Cannot "
+ "allocate mbuf packet\n");
+
+ if (!zbyte)
+ ret = read(fd, rte_pktmbuf_mtod(mb[i], void *),
+ pinfo[portid].buff_size);
+ if (ret < 0) {
+ printf("Error: Could not the read "
+ "input-file\n");
+ return -1;
+ }
+ mb[i]->nb_segs = 1;
+ mb[i]->next = NULL;
+ rte_pktmbuf_data_len(mb[i]) = (uint16_t)ret;
+ rte_pktmbuf_pkt_len(mb[i]) = (uint16_t)ret;
+ }
+
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ printf("%s(): %d: queue id %d, mbuf_avail_count = %d, "
+ "mbuf_in_use_count = %d, num_pkts_tx = %d",
+ __func__, __LINE__, queueid,
+ rte_mempool_avail_count(mp),
+ rte_mempool_in_use_count(mp), num_pkts);
+#endif //DUMP_MEMPOOL_USAGE_STATS
+
+ total_tx = num_pkts;
+ PciWrite(user_bar_idx, C2H_ST_QID_REG, (queueid + qbase),
+ portid);
+ /* try to transmit TX_BURST_SZ packets */
+
+#ifdef PERF_BENCHMARK
+ prev_tsc = rte_rdtsc_precise();
+#endif
+ nb_tx = rte_eth_tx_burst(portid, queueid, mb, num_pkts);
+#ifdef PERF_BENCHMARK
+ cur_tsc = rte_rdtsc_precise();
+ diff_tsc = cur_tsc - prev_tsc;
+ /* Calculate average operations processed per second */
+ double pkts_per_second = ((double)nb_tx * rte_get_tsc_hz() /
+ diff_tsc);
+
+ /* Calculate average throughput (Gbps) in bits per second */
+ double throughput_gbps = ((pkts_per_second *
+ pinfo[portid].buff_size) / 1000000000);
+ printf("Throughput GBps %lf\n", throughput_gbps);
+ printf("%12s%12s%12s%12s%12s%12s%12s\n\n",
+ "Buf Size", "Burst Size",
+ "pps", "Gbps", "freq", "Cycles",
+ "Cycles/Buf");
+
+ printf("%12u%12u%12.4lf%12.4lf%12"
+ ""PRIu64"%12"PRIu64"%12"PRIu64"\n",
+ pinfo[portid].buff_size,
+ nb_tx,
+ pkts_per_second,
+ throughput_gbps,
+ rte_get_tsc_hz(),
+ diff_tsc,
+ diff_tsc/nb_tx);
+#endif
+ tmp = nb_tx;
+ while ((nb_tx < num_pkts) && max_tx_retry) {
+ printf("Couldn't transmit all the packets: Expected = %d "
+ "Transmitted = %d.\n"
+ "Calling rte_eth_tx_burst again\n",
+ num_pkts, nb_tx);
+ num_pkts -= nb_tx;
+ nb_tx = rte_eth_tx_burst(portid, queueid, &mb[tmp],
+ num_pkts);
+ tmp += nb_tx;
+ max_tx_retry--;
+ }
+
+ if ((max_tx_retry == 0)) {
+ for (i = tmp; i < total_tx; i++)
+ rte_pktmbuf_free(mb[i]);
+ if (tmp == 0) {
+ printf("ERROR: rte_eth_tx_burst failed "
+ "for port %d queue %d\n",
+ portid, queueid);
+ break;
+ }
+ }
+
+ tdesc = tdesc - tmp;
+ printf("\nDMA transmitted number of packets: %d, "
+ "on Queue-id:%d\n",
+ tmp, queueid);
+ }
+
+ if (ld_size) {
+ mb[0] = rte_pktmbuf_alloc(mp);
+ if (mb[0] == NULL) {
+ rte_exit(EXIT_FAILURE, " #####Cannot allocate mbuf "
+ "packet\n");
+ }
+ ret = read(fd, rte_pktmbuf_mtod(mb[0], void *), ld_size);
+ if (ret < 0) {
+ printf("Error: Could not read the input-file\n");
+ return -1;
+ }
+ mb[0]->nb_segs = 1;
+ mb[0]->next = NULL;
+ rte_pktmbuf_data_len(mb[0]) = (uint16_t)ret;
+ rte_pktmbuf_pkt_len(mb[0]) = (uint16_t)ret;
+
+ nb_tx = rte_eth_tx_burst(portid, queueid, mb, 1);
+ if (nb_tx == 0)
+ rte_pktmbuf_free(mb[0]);
+ }
+
+ reg_val = PciRead(user_bar_idx, C2H_CONTROL_REG, portid);
+ reg_val &= C2H_CONTROL_REG_MASK;
+ if (!(reg_val & ST_LOOPBACK_EN)) {
+ reg_val = PciRead(user_bar_idx, H2C_STATUS_REG, portid);
+ printf("BAR-%d is the QDMA H2C transfer match: 0x%x,\n",
+ user_bar_idx, reg_val);
+
+ /** TO clear H2C DMA write **/
+ PciWrite(user_bar_idx, H2C_CONTROL_REG, 0x1, portid);
+ }
+
+ return 0;
+}
+
+void port_close(int port_id)
+{
+ struct rte_mempool *mp;
+
+ rte_eth_dev_stop(port_id);
+ rte_eth_dev_close(port_id);
+ mp = rte_mempool_lookup(pinfo[port_id].mem_pool);
+
+ if (mp != NULL)
+ rte_mempool_free(mp);
+}
+
+static int mbox_event_callback(uint16_t portid,
+ enum rte_eth_event_type type __rte_unused,
+ void *param __rte_unused, void *ret_param)
+{
+ RTE_SET_USED(ret_param);
+ printf("%s is received\n", __func__);
+ pinfo[portid].num_queues = 0;
+ port_close(portid);
+ return 0;
+}
+
+static struct option const long_opts[] = {
+{"filename", 1, 0, 0},
+{NULL, 0, 0, 0}
+};
+
+int parse_cmdline(int argc, char **argv)
+{
+ int cmd_opt;
+ int option_index;
+ char **argvopt;
+
+ argvopt = argv;
+ while ((cmd_opt = getopt_long(argc, argvopt, "c:n:b:w", long_opts,
+ &option_index)) != EOF) {
+ switch (cmd_opt) {
+ case 'c':
+ /* eal option */
+ break;
+ case 'n':
+ /* eal option */
+ break;
+ case 'b':
+ /* eal option */
+ break;
+ case 'w':
+ /* eal option */
+ break;
+ case 0:
+ if (!strncmp(long_opts[option_index].name,
+ "filename",
+ sizeof("filename"))) {
+
+ filename = optarg;
+ }
+ break;
+ default:
+ printf("please pass valid parameters as follows:\n");
+ return -1;
+ }
+ }
+ return 0;
+}
+
+int port_init(int portid, int queue_base, int num_queues, int st_queues,
+ int nb_descs, int buff_size)
+{
+ struct rte_mempool *mbuf_pool;
+ struct rte_eth_conf port_conf;
+ struct rte_eth_txconf tx_conf;
+ struct rte_eth_rxconf rx_conf;
+ int diag, x;
+ uint32_t nb_buff;
+
+ printf("Setting up port :%d.\n", portid);
+ printf("Setting up queue-base :%d.\n", queue_base);
+
+ snprintf(pinfo[portid].mem_pool, RTE_MEMPOOL_NAMESIZE,
+ MBUF_POOL_NAME_PORT, portid);
+
+ /* Mbuf packet pool */
+ nb_buff = ((nb_descs) * num_queues * 2);
+
+ /* NUM_RX_PKTS should be added to every queue as that many descriptors
+ * can be pending with application after Rx processing but before
+ * consumed by application or sent to Tx
+ */
+ nb_buff += ((NUM_RX_PKTS) * num_queues);
+
+ mbuf_pool = rte_pktmbuf_pool_create(pinfo[portid].mem_pool, nb_buff,
+ MP_CACHE_SZ, 0, buff_size +
+ RTE_PKTMBUF_HEADROOM,
+ rte_socket_id());
+
+ if (mbuf_pool == NULL)
+ rte_exit(EXIT_FAILURE, " Cannot create mbuf pkt-pool\n");
+#ifdef DUMP_MEMPOOL_USAGE_STATS
+ printf("%s(): %d: mpool = %p, mbuf_avail_count = %d,"
+ " mbuf_in_use_count = %d,"
+ "nb_buff = %d\n", __func__, __LINE__, mbuf_pool,
+ rte_mempool_avail_count(mbuf_pool),
+ rte_mempool_in_use_count(mbuf_pool), nb_buff);
+#endif //DUMP_MEMPOOL_USAGE_STATS
+
+ /*
+ * Make sure the port is configured. Zero everything and
+ * hope for sane defaults
+ */
+ memset(&port_conf, 0x0, sizeof(struct rte_eth_conf));
+ memset(&tx_conf, 0x0, sizeof(struct rte_eth_txconf));
+ memset(&rx_conf, 0x0, sizeof(struct rte_eth_rxconf));
+ diag = get_param(&rte_eth_devices[portid], CONFIG_BAR,
+ &(pinfo[portid].config_bar_idx));
+
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "get_param : CONFIG_BAR failed\n");
+ diag = get_param(&rte_eth_devices[portid], USER_BAR,
+ &(pinfo[portid].user_bar_idx));
+ if ((diag < 0) || (pinfo[portid].user_bar_idx < 0))
+ rte_exit(EXIT_FAILURE, "get_param : USER_BAR failed\n");
+ diag = get_param(&rte_eth_devices[portid], BYPASS_BAR,
+ &(pinfo[portid].bypass_bar_idx));
+ if ((diag < 0) || (pinfo[portid].bypass_bar_idx < 0))
+ printf("get_param : BYPASS_BAR is failed\n");
+
+ printf("QDMA Config bar idx: %d\n", pinfo[portid].config_bar_idx);
+ printf("QDMA User bar idx: %d\n", pinfo[portid].user_bar_idx);
+ printf("QDMA Bypass bar idx: %d\n", pinfo[portid].bypass_bar_idx);
+
+ diag = update_param(&rte_eth_devices[portid], QUEUE_BASE, queue_base);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "updata_param : Passing of "
+ "QUEUE_BASE failed\n");
+
+ /* configure the device to use # queues */
+ diag = rte_eth_dev_configure(portid, num_queues, num_queues,
+ &port_conf);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "Cannot configure port %d (err=%d)\n",
+ portid, diag);
+
+ pinfo[portid].queue_base = queue_base;
+ pinfo[portid].num_queues = num_queues;
+ pinfo[portid].st_queues = st_queues;
+ pinfo[portid].buff_size = buff_size;
+
+ for (x = 0; x < num_queues; x++) {
+ if (x < st_queues) {
+ diag = update_queue_param(&rte_eth_devices[portid], x,
+ QUEUE_MODE, 1);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "updata_queue_param : "
+ "Passing of QUEUE_MODE "
+ "failed\n");
+ } else {
+ update_queue_param(&rte_eth_devices[portid], x,
+ QUEUE_MODE, 0);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "updata_queue_param : "
+ "Passing of QUEUE_MODE "
+ "failed\n");
+ }
+
+ diag = rte_eth_tx_queue_setup(portid, x, nb_descs, 0,
+ &tx_conf);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "Cannot setup port %d "
+ "TX Queue id:%d "
+ "(err=%d)\n", portid, x, diag);
+ diag = rte_eth_rx_queue_setup(portid, x, nb_descs, 0,
+ &rx_conf, mbuf_pool);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "Cannot setup port %d "
+ "RX Queue 0 (err=%d)\n", portid, diag);
+ }
+
+ diag = rte_eth_dev_start(portid);
+ if (diag < 0)
+ rte_exit(EXIT_FAILURE, "Cannot start port %d (err=%d)\n",
+ portid, diag);
+
+ rte_eth_dev_callback_register((uint16_t)portid, RTE_ETH_EVENT_VF_MBOX,
+ mbox_event_callback, NULL);
+
+ return 0;
+}
+
+static inline void do_sanity_checks(void)
+{
+#if (!defined(RTE_LIBRTE_QDMA_PMD))
+ rte_exit(EXIT_FAILURE, "CONFIG_RTE_LIBRTE_QDMA_PMD must be set "
+ "to 'Y' in the .config file\n");
+#endif /* RTE_LIBRTE_XDMA_PMD */
+
+}
+
+
+void load_file_cmds(struct cmdline *cl)
+{
+ FILE *fp;
+ char buff[256];
+
+ cmdline_printf(cl, "load commands from file:%s\n\n", filename);
+ fp = fopen((const char *)filename, "r");
+ if (fp == NULL) {
+ cmdline_printf(cl, "Error: Invalid filename: %s\n", filename);
+ return;
+ }
+
+ rdline_reset(&cl->rdl);
+ {
+ cmdline_in(cl, "\r", 1);
+ while (fgets(buff, sizeof(buff), fp))
+ cmdline_in(cl, buff, strlen(buff));
+
+ cmdline_in(cl, "\r", 1);
+ }
+ fclose(fp);
+
+}
+
+/** XDMA DPDK testapp */
+
+int main(int argc, char **argv)
+{
+ int port_id = 0;
+ int ret = 0;
+ const struct rte_memzone *mz = 0;
+ struct cmdline *cl;
+ char name[RTE_ETH_NAME_MAX_LEN];
+
+ /* Make sure the port is configured. Zero everything and
+ * hope for same defaults
+ */
+
+ printf("QDMA testapp rte eal init...\n");
+
+ /* Make sure things are initialized ... */
+ ret = rte_eal_init(argc, argv);
+ if (ret < 0)
+ rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
+ rte_log_set_global_level(RTE_LOG_DEBUG);
+
+ printf("Ethernet Device Count: %d\n", (int)rte_eth_dev_count());
+ printf("Logical Core Count: %d\n", rte_lcore_count());
+
+ num_ports = rte_eth_dev_count();
+ if (num_ports < 1)
+ rte_exit(EXIT_FAILURE, "No Ethernet devices found."
+ " Try updating the FPGA image.\n");
+
+#if 1
+ ret = parse_cmdline(argc, argv);
+ if (ret < 0)
+ rte_exit(EXIT_FAILURE, "Invalid argument\n");
+#endif
+
+ /* Make sure things are defined ... */
+ do_sanity_checks();
+
+ mz = rte_memzone_reserve_aligned("eth_devices", RTE_MAX_ETHPORTS *
+ sizeof(*rte_eth_devices), 0, 0, 4096);
+
+ memcpy(mz->addr, &rte_eth_devices[0], RTE_MAX_ETHPORTS *
+ sizeof(*rte_eth_devices));
+
+ cl = cmdline_stdin_new(main_ctx, "xilinx-app> ");
+ if (cl == NULL)
+ rte_panic("Cannot create cmdline instance\n");
+
+ /* if input commands file exists, then load commands from the file */
+ if (filename != NULL) {
+ load_file_cmds(cl);
+ rte_delay_ms(100);
+ } else
+ cmdline_interact(cl);
+ for (port_id = 0; port_id < num_ports; port_id++) {
+ if (pinfo[port_id].num_queues)
+ port_close(port_id);
+
+ /*Detach the port, it will invoke device remove/uninit */
+ if (rte_eth_dev_detach(port_id, name))
+ printf("Failed to detach port '%s'\n", name);
+ }
+ cmdline_stdin_exit(cl);
+
+ rte_delay_ms(5000);
+ return 0;
+}
diff --git a/QDMA/DPDK/examples/qdma_testapp/testapp.h b/QDMA/DPDK/examples/qdma_testapp/testapp.h
new file mode 100644
index 0000000000000000000000000000000000000000..167a169e8b35977be51888d84c85f8426894bd89
--- /dev/null
+++ b/QDMA/DPDK/examples/qdma_testapp/testapp.h
@@ -0,0 +1,86 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of the copyright holder nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define QDMA_MAX_PORTS 256
+
+#define PORT_0 0
+
+#define NUM_DESC_PER_RING 1024
+#ifdef PERF_BENCHMARK
+#define NUM_RX_PKTS (NUM_DESC_PER_RING - 2)
+#else
+#define NUM_RX_PKTS 32
+#endif
+#define MAX_NUM_QUEUES 2048
+#define DEFAULT_NUM_QUEUES 64
+#define RX_TX_MAX_RETRY 1000
+
+#define MP_CACHE_SZ 512
+
+#define MBUF_POOL_NAME_PORT "mbuf_pool_%d"
+
+/* User bar registers */
+#define C2H_ST_QID_REG 0x0
+#define C2H_ST_LEN_REG 0x4
+#define C2H_CONTROL_REG 0x8
+#define ST_LOOPBACK_EN 0x1
+#define ST_C2H_START_VAL 0x2
+#define ST_C2H_IMMEDIATE_DATA_EN 0x4
+#define C2H_CONTROL_REG_MASK 0xF
+#define H2C_CONTROL_REG 0xC
+#define H2C_STATUS_REG 0x10
+#define C2H_PACKET_COUNT_REG 0x20
+
+extern int num_ports;
+
+struct port_info {
+ int config_bar_idx;
+ int user_bar_idx;
+ int bypass_bar_idx;
+ unsigned int queue_base;
+ unsigned int num_queues;
+ unsigned int st_queues;
+ unsigned int buff_size;
+ char mem_pool[RTE_MEMPOOL_NAMESIZE];
+};
+
+extern struct port_info pinfo[QDMA_MAX_PORTS];
+int port_init(int portid, int queue_base, int num_queues, int st_queues,
+ int nb_descs, int buff_size);
+int do_recv_st(int portid, int fd, int queueid, int input_size);
+int do_recv_mm(int portid, int fd, int queueid, int size, int tot_num_desc);
+int do_xmit(int portid, int fd, int queueid, int size, int nb_desc, int zbyte);
+void load_file_cmds(struct cmdline *cl);
+void port_close(int port_id);
+void qdma_desc_dump(struct rte_eth_dev *dev, uint32_t qid);
+void queue_context_dump(uint8_t bar_id, uint32_t qid, struct cmdline *cl);
+int parse_cmdline(int argc, char **argv);
diff --git a/QDMA/DPDK/tools/0001-PKTGEN-3.4.7-Patch-to-add-Jumbo-packet-support.patch b/QDMA/DPDK/tools/0001-PKTGEN-3.4.7-Patch-to-add-Jumbo-packet-support.patch
new file mode 100644
index 0000000000000000000000000000000000000000..39f83bf9e77440cf26990a8fb95b520cb217cffc
--- /dev/null
+++ b/QDMA/DPDK/tools/0001-PKTGEN-3.4.7-Patch-to-add-Jumbo-packet-support.patch
@@ -0,0 +1,302 @@
+From ccc6a165e55b121ee39536c279af38e9866f3f68 Mon Sep 17 00:00:00 2001
+From: Nikhil Agarwal <nagarwal@xilinx.com>
+Date: Tue, 21 Aug 2018 00:24:51 -0600
+Subject: [PATCH] PKTGEN-3.4.5: Patch to add Jumbo packet support.
+
+This patch include:
+1. Jumbo frame support for Pktgen.
+2. Increase default number of RX_DESC to 2K.
+3. Disable RX classification.
+
+Signed-off-by: Kumar Sanghvi <kumars@xilinx.com>
+Signed-off-by: Nikhil Agarwal <nagarwal@xilinx.com>
+---
+ app/pktgen-cmds.c | 15 +++++++++++----
+ app/pktgen-constants.h | 3 ++-
+ app/pktgen-main.c | 7 ++++++-
+ app/pktgen-port-cfg.c | 14 ++++++++------
+ app/pktgen-range.c | 3 ++-
+ app/pktgen.c | 21 +++++++++++++++------
+ app/pktgen.h | 4 +++-
+ 7 files changed, 47 insertions(+), 20 deletions(-)
+
+diff --git a/app/pktgen-cmds.c b/app/pktgen-cmds.c
+index 66ea98a..be97d35 100644
+--- a/app/pktgen-cmds.c
++++ b/app/pktgen-cmds.c
+@@ -2443,6 +2443,10 @@ debug_set_tx_cycles(port_info_t *info, uint32_t cycles)
+ void
+ single_set_pkt_size(port_info_t *info, uint16_t size)
+ {
++ uint16_t pktsize = (pktgen.flags & MTU9K_SUPPORT_FLAG) ?
++ MAX_PKT_SIZE:
++ (ETHER_MAX_LEN - ETHER_CRC_LEN);
++
+ pkt_seq_t * pkt = &info->seq_pkt[SINGLE_PKT];
+
+ if (size < FCS_SIZE)
+@@ -2452,8 +2456,8 @@ single_set_pkt_size(port_info_t *info, uint16_t size)
+ if ( (size - FCS_SIZE) < MIN_PKT_SIZE)
+ size = (MIN_PKT_SIZE + FCS_SIZE);
+ }
+- if ( (size - FCS_SIZE) > MAX_PKT_SIZE)
+- size = MAX_PKT_SIZE + FCS_SIZE;
++ if ( (size - FCS_SIZE) > pktsize)
++ size = pktsize + FCS_SIZE;
+
+ if ((pkt->ethType == ETHER_TYPE_IPv6) && (size < (MIN_v6_PKT_SIZE + FCS_SIZE)))
+ size = MIN_v6_PKT_SIZE + FCS_SIZE;
+@@ -3007,6 +3011,9 @@ range_set_cos_id(port_info_t *info, char *what, uint8_t id)
+ void
+ range_set_pkt_size(port_info_t *info, char *what, uint16_t size)
+ {
++ uint32_t pktsize = (pktgen.flags & MTU9K_SUPPORT_FLAG) ?
++ MAX_9K_SIZE : ETHER_MAX_LEN;
++
+ if (!strcmp(what, "inc") || !strcmp(what, "increment")) {
+ if (size > ETHER_MIN_LEN)
+ size = ETHER_MIN_LEN;
+@@ -3015,8 +3022,8 @@ range_set_pkt_size(port_info_t *info, char *what, uint16_t size)
+ } else {
+ if (size < ETHER_MIN_LEN)
+ size = MIN_PKT_SIZE;
+- else if (size > ETHER_MAX_LEN)
+- size = MAX_PKT_SIZE;
++ else if (size > pktsize)
++ size = pktsize;
+ else
+ size -= FCS_SIZE;
+
+diff --git a/app/pktgen-constants.h b/app/pktgen-constants.h
+index 1dc7c1a..bd5ce07 100644
+--- a/app/pktgen-constants.h
++++ b/app/pktgen-constants.h
+@@ -17,7 +17,7 @@ extern "C" {
+ enum {
+ DEFAULT_PKT_BURST = 64, /* Increasing this number consumes memory very fast */
+ #ifdef RTE_LIBRTE_VMXNET3_PMD
+- DEFAULT_RX_DESC = (DEFAULT_PKT_BURST * 8 * 2),
++ DEFAULT_RX_DESC = (DEFAULT_PKT_BURST * 8 * 2 * 2),
+ DEFAULT_TX_DESC = DEFAULT_RX_DESC * 2,
+ #else
+ DEFAULT_RX_DESC = (DEFAULT_PKT_BURST * 8),
+@@ -30,6 +30,7 @@ enum {
+
+ DEFAULT_PRIV_SIZE = 0,
+ MBUF_SIZE = RTE_MBUF_DEFAULT_BUF_SIZE + DEFAULT_PRIV_SIZE, /* See: http://dpdk.org/dev/patchwork/patch/4479/ */
++ MBUF_9K_SIZE = 9018 + RTE_PKTMBUF_HEADROOM + DEFAULT_PRIV_SIZE,
+
+ NUM_Q = 8, /**< Number of cores per port. */
+ };
+diff --git a/app/pktgen-main.c b/app/pktgen-main.c
+index 8a10156..3ed1978 100644
+--- a/app/pktgen-main.c
++++ b/app/pktgen-main.c
+@@ -161,7 +161,7 @@ pktgen_parse_args(int argc, char **argv)
+ for (opt = 0; opt < argc; opt++)
+ pktgen.argv[opt] = strdup(argv[opt]);
+
+- while ((opt = getopt_long(argc, argvopt, "p:m:f:l:s:g:hPNGT",
++ while ((opt = getopt_long(argc, argvopt, "p:m:f:l:s:g:hPNGT9",
+ lgopts, &option_index)) != EOF)
+ switch (opt) {
+ case 'p':
+@@ -239,6 +239,11 @@ pktgen_parse_args(int argc, char **argv)
+ pktgen_usage(prgname);
+ return -1;
+
++ case '9': /* MTU 9K support */
++ pktgen_log_info("%s: case 9... \n", __func__);
++ pktgen.flags |= MTU9K_SUPPORT_FLAG;
++ break;
++
+ case 0: /* crc-strip for all ports */
+ pktgen_set_hw_strip_crc(1);
+ break;
+diff --git a/app/pktgen-port-cfg.c b/app/pktgen-port-cfg.c
+index f53d0ec..8b9d478 100644
+--- a/app/pktgen-port-cfg.c
++++ b/app/pktgen-port-cfg.c
+@@ -91,6 +91,8 @@ pktgen_mbuf_pool_create(const char *type, uint8_t pid, uint8_t queue_id,
+ uint32_t nb_mbufs, int socket_id, int cache_size){
+ struct rte_mempool *mp;
+ char name[RTE_MEMZONE_NAMESIZE];
++ uint16_t mbuf_sz = (pktgen.flags & MTU9K_SUPPORT_FLAG) ?
++ MBUF_9K_SIZE : MBUF_SIZE;
+
+ snprintf(name, sizeof(name), "%-12s%u:%u", type, pid, queue_id);
+ pktgen_log_info(
+@@ -98,23 +100,23 @@ pktgen_mbuf_pool_create(const char *type, uint8_t pid, uint8_t queue_id,
+ 16,
+ name,
+ nb_mbufs,
+- MBUF_SIZE,
++ mbuf_sz,
+ sizeof(struct rte_mbuf),
+ sizeof(struct rte_mempool),
+- (((nb_mbufs * (MBUF_SIZE + sizeof(struct rte_mbuf)) +
++ (((nb_mbufs * (mbuf_sz + sizeof(struct rte_mbuf)) +
+ sizeof(struct rte_mempool))) + 1023) / 1024,
+ RTE_PKTMBUF_HEADROOM,
+ RTE_MBUF_DEFAULT_BUF_SIZE);
+ pktgen.mem_used += ((nb_mbufs *
+- (MBUF_SIZE + sizeof(struct rte_mbuf)) +
++ (mbuf_sz + sizeof(struct rte_mbuf)) +
+ sizeof(struct rte_mempool)));
+ pktgen.total_mem_used += ((nb_mbufs *
+- (MBUF_SIZE + sizeof(struct rte_mbuf)) +
++ (mbuf_sz + sizeof(struct rte_mbuf)) +
+ sizeof(struct rte_mempool)));
+
+ /* create the mbuf pool */
+ mp = rte_pktmbuf_pool_create(name, nb_mbufs, cache_size,
+- DEFAULT_PRIV_SIZE, MBUF_SIZE, socket_id);
++ DEFAULT_PRIV_SIZE, mbuf_sz, socket_id);
+ if (mp == NULL)
+ pktgen_log_panic(
+ "Cannot create mbuf pool (%s) port %d, queue %d, nb_mbufs %d, socket_id %d: %s",
+@@ -250,7 +252,7 @@ pktgen_config_ports(void)
+ pktgen_log_info(
+ "Configuring %d ports, MBUF Size %d, MBUF Cache Size %d",
+ pktgen.nb_ports,
+- MBUF_SIZE,
++ (pktgen.flags & MTU9K_SUPPORT_FLAG) ? MBUF_9K_SIZE : MBUF_SIZE,
+ MBUF_CACHE_SIZE);
+
+ /* For each lcore setup each port that is handled by that lcore. */
+diff --git a/app/pktgen-range.c b/app/pktgen-range.c
+index e59ae01..36b5924 100644
+--- a/app/pktgen-range.c
++++ b/app/pktgen-range.c
+@@ -568,7 +568,8 @@ pktgen_range_setup(port_info_t *info)
+ range->pkt_size = MIN_PKT_SIZE;
+ range->pkt_size_inc = 0;
+ range->pkt_size_min = MIN_PKT_SIZE;
+- range->pkt_size_max = MAX_PKT_SIZE;
++ range->pkt_size_max = (pktgen.flags & MTU9K_SUPPORT_FLAG) ?
++ MAX_PKT_SIZE : (ETHER_MAX_LEN - ETHER_CRC_LEN);
+
+ info->seq_pkt[RANGE_PKT].pktSize = MIN_PKT_SIZE;
+
+diff --git a/app/pktgen.c b/app/pktgen.c
+index dd1a93a..68579b1 100644
+--- a/app/pktgen.c
++++ b/app/pktgen.c
+@@ -273,7 +273,7 @@ _send_burst_fast(port_info_t *info, uint16_t qid)
+
+ pkts = mtab->m_table;
+
+- retry = 100;
++ retry = 100000;
+ if (rte_atomic32_read(&info->port_flags) & PROCESS_TX_TAP_PKTS)
+ while (cnt && retry) {
+ ret = rte_eth_tx_burst(info->pid, qid, pkts, cnt);
+@@ -294,6 +294,8 @@ _send_burst_fast(port_info_t *info, uint16_t qid)
+ if (!ret)
+ retry--;
+ }
++ if(cnt)
++ rte_pktmbuf_free_bulk(pkts, cnt);
+ }
+
+ /**************************************************************************//**
+@@ -880,6 +882,9 @@ pktgen_setup_cb(struct rte_mempool *mp,
+ port_info_t *info;
+ pkt_seq_t *pkt;
+ uint16_t qid;
++ uint32_t pktsize = (pktgen.flags & MTU9K_SUPPORT_FLAG) ?
++ MAX_PKT_SIZE:
++ (ETHER_MAX_LEN - ETHER_CRC_LEN);
+
+ info = data->info;
+ qid = data->qid;
+@@ -898,7 +903,7 @@ pktgen_setup_cb(struct rte_mempool *mp,
+ pktgen_packet_ctor(info, SINGLE_PKT, -1);
+
+ rte_memcpy((uint8_t *)m->buf_addr + m->data_off,
+- (uint8_t *)&pkt->hdr, MAX_PKT_SIZE);
++ (uint8_t *)&pkt->hdr, pktsize);
+
+ m->pkt_len = pkt->pktSize;
+ m->data_len = pkt->pktSize;
+@@ -907,7 +912,7 @@ pktgen_setup_cb(struct rte_mempool *mp,
+ pktgen_packet_ctor(info, RANGE_PKT, -1);
+
+ rte_memcpy((uint8_t *)m->buf_addr + m->data_off,
+- (uint8_t *)&pkt->hdr, MAX_PKT_SIZE);
++ (uint8_t *)&pkt->hdr, pktsize);
+
+ m->pkt_len = pkt->pktSize;
+ m->data_len = pkt->pktSize;
+@@ -925,7 +930,7 @@ pktgen_setup_cb(struct rte_mempool *mp,
+ pktgen_packet_ctor(info, info->seqIdx, -1);
+
+ rte_memcpy((uint8_t *)m->buf_addr + m->data_off,
+- (uint8_t *)&pkt->hdr, MAX_PKT_SIZE);
++ (uint8_t *)&pkt->hdr, pktsize);
+ m->pkt_len = pkt->pktSize;
+ m->data_len = pkt->pktSize;
+ pkt = &info->seq_pkt[info->seqIdx];
+@@ -937,7 +942,7 @@ pktgen_setup_cb(struct rte_mempool *mp,
+ pktgen_packet_ctor(info, info->seqIdx, -1);
+
+ rte_memcpy((uint8_t *)m->buf_addr + m->data_off,
+- (uint8_t *)&pkt->hdr, MAX_PKT_SIZE);
++ (uint8_t *)&pkt->hdr, pktsize);
+
+ m->pkt_len = pkt->pktSize;
+ m->data_len = pkt->pktSize;
+@@ -1121,7 +1126,7 @@ pktgen_main_receive(port_info_t *info,
+ {
+ uint8_t pid;
+ uint16_t qid, nb_rx;
+- capture_t *capture;
++ __rte_unused capture_t *capture;
+
+ pid = info->pid;
+ qid = get_rxque(pktgen.l2p, lid, pid);
+@@ -1133,7 +1138,10 @@ pktgen_main_receive(port_info_t *info,
+ return;
+
+ info->q[qid].rx_cnt += nb_rx;
++ info->sizes._64 += nb_rx;
+
++ rte_pktmbuf_free_bulk(pkts_burst, nb_rx);
++#if 0
+ pktgen_recv_latency(info, pkts_burst, nb_rx);
+
+ /* packets are not freed in the next call. */
+@@ -1150,6 +1158,7 @@ pktgen_main_receive(port_info_t *info,
+ }
+
+ rte_pktmbuf_free_bulk(pkts_burst, nb_rx);
++#endif
+ }
+
+ static void
+diff --git a/app/pktgen.h b/app/pktgen.h
+index f830c81..892875e 100644
+--- a/app/pktgen.h
++++ b/app/pktgen.h
+@@ -219,8 +219,9 @@ enum {
+ INTER_FRAME_GAP = 12, /**< in bytes */
+ PKT_PREAMBLE_SIZE = 8, /**< in bytes */
+ FCS_SIZE = 4, /**< in bytes */
++ MAX_9K_SIZE = 9018,
+ MIN_PKT_SIZE = (ETHER_MIN_LEN - FCS_SIZE),
+- MAX_PKT_SIZE = (ETHER_MAX_LEN - FCS_SIZE),
++ MAX_PKT_SIZE = (MAX_9K_SIZE - FCS_SIZE),
+ MIN_v6_PKT_SIZE = (78 - FCS_SIZE),
+
+ MAX_RX_QUEUES = 16, /**< RX Queues per port */
+@@ -314,6 +315,7 @@ enum { /* Pktgen flags bits */
+ BLINK_PORTS_FLAG = (1 << 10), /**< Blink the port leds */
+ ENABLE_THEME_FLAG = (1 << 11), /**< Enable theme or color support */
+
++ MTU9K_SUPPORT_FLAG = (1 << 15), /**< MTU 9K support */
+ CONFIG_PAGE_FLAG = (1 << 16), /**< Display the configure page */
+ SEQUENCE_PAGE_FLAG = (1 << 17), /**< Display the Packet sequence page */
+ RANGE_PAGE_FLAG = (1 << 18), /**< Display the range page */
+--
+2.7.4
+
diff --git a/QDMA/DPDK/tools/README.txt b/QDMA/DPDK/tools/README.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d28800dab87e20653136be2067e24a1861dbf5dc
--- /dev/null
+++ b/QDMA/DPDK/tools/README.txt
@@ -0,0 +1,7 @@
+0001-PKTGEN-3.4.5-Patch-to-add-Jumbo-packet-support.patch is the patch file
+over dpdk-pktgen v3.4.7 that extends dpdk-pktgen application to handle packets
+with packet sizes more than 1518 bytes and it disables the packet size
+classification logic in dpdk-pktgen to remove application overhead in
+performance measurement.
+
+This patch is used for performance testing with dpdk-pktgen application.
diff --git a/LICENSE b/QDMA/linux-kernel/COPYING
similarity index 95%
rename from LICENSE
rename to QDMA/linux-kernel/COPYING
index d159169d1050894d3ea3b98e1c965c4058208fe1..3912109b5cd65a68039d473c11c9f7ac2303e06d 100644
--- a/LICENSE
+++ b/QDMA/linux-kernel/COPYING
@@ -1,12 +1,12 @@
- GNU GENERAL PUBLIC LICENSE
- Version 2, June 1991
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+ 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
- Preamble
+ Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
@@ -15,7 +15,7 @@ software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.) You can apply it to
+the GNU Library General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
@@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
-
- GNU GENERAL PUBLIC LICENSE
+
+ GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
@@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions:
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
-
+
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
@@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
-
+
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
@@ -225,7 +225,7 @@ impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
-
+
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
@@ -255,7 +255,7 @@ make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
- NO WARRANTY
+ NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
@@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
- END OF TERMS AND CONDITIONS
-
- How to Apply These Terms to Your New Programs
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
@@ -303,9 +303,10 @@ the "copyright" line and a pointer to where the full notice is found.
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
- You should have received a copy of the GNU General Public License along
- with this program; if not, write to the Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
Also add information on how to contact you by electronic and paper mail.
@@ -335,5 +336,5 @@ necessary. Here is a sample; alter the names:
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
-library. If this is what you want to do, use the GNU Lesser General
+library. If this is what you want to do, use the GNU Library General
Public License instead of this License.
diff --git a/QDMA/linux-kernel/LICENSE b/QDMA/linux-kernel/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..703e647155260d871353837a448fc5d765bbc363
--- /dev/null
+++ b/QDMA/linux-kernel/LICENSE
@@ -0,0 +1,30 @@
+BSD License
+
+For Xilinx DMA IP software
+
+Copyright (c) 2016-present, Xilinx, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ * Neither the name Xilinx nor the names of its contributors may be used to
+ endorse or promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/QDMA/linux-kernel/Makefile b/QDMA/linux-kernel/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..40a2cfe37b02aa21ddc713a00ad07e2a48a68de9
--- /dev/null
+++ b/QDMA/linux-kernel/Makefile
@@ -0,0 +1,307 @@
+SHELL = /bin/bash
+
+#
+# makefile parameters:
+# - KDIR/KSRC/KOBJ=, optional
+# - install_path=, override all install directories
+# - kernel_install_path=, override install directory for kernel module
+# - dev_install_path=, override install directory for development headers
+# - user_install_path=, override install directory for applications
+# - conf_install_path=, override install directory for config files
+# - udev_install_path=, override install directory for udev rules
+# - docs_install_path=, override install directory for man pages
+#
+# - enable_cmpt_immediate_data=<0|1> enable immediate data in writeback desc.
+# - disable_st_c2h_completion=<0|1> disable completion
+#
+
+# Define grep error output to NULL, since -s is not portable.
+grep = grep 2>/dev/null
+
+# ALL subdirectories
+ALLSUBDIRS := drv
+DRIVER_SRC_DIR := drv
+
+# subdirectories to be build
+SUBDIRS := $(ALLSUBDIRS)
+
+# Honor the -s (silent) make option.
+verbose := $(if $(filter s,$(MAKEFLAGS)),,-v)
+
+# Define paths.
+srcdir := $(shell pwd)
+topdir := $(shell cd $(srcdir)/.. && pwd)
+build_dir := $(srcdir)/build
+
+kernel_check = 1
+distro_check = 1
+
+ifeq ($(filter clean,$(MAKECMDGOALS)),clean)
+ kernel_check = 0
+ distro_check = 0
+endif
+
+ifeq ($(filter uninstall,$(MAKECMDGOALS)),uninstall)
+ distro_check = 0
+endif
+
+ifeq ($(kernel_check),1)
+ include make/kernel_check.mk
+
+ ifeq ($(distro_check),1)
+ include make/distro_check.mk
+ endif
+endif
+
+ifneq ($(wildcard $(KINC)/linux/kconfig.h),)
+ FLAGS += -DKERNEL_HAS_KCONFIG_H
+endif
+ifneq ($(wildcard $(KINC)/linux/export.h),)
+ FLAGS += -DKERNEL_HAS_EXPORT_H
+endif
+
+# Debug flags.
+ifeq ($(DEBUG),1)
+ FLAGS += -g -DDEBUG
+endif
+
+# Debugfs flags
+ifneq ($(DEBUGFS),0)
+ FLAGS += -DDEBUGFS
+endif
+
+# LOOPBACK flags.
+ifeq ($(LOOPBACK),1)
+ FLAGS += -DLOOPBACK_TEST
+endif
+
+ifeq ($(DEBUG_THREADS),1)
+ FLAGS += -DDEBUG -DDEBUG_THREADS
+endif
+
+ifeq ($(enable_cmpt_immediate_data),1)
+ FLAGS += -DXNL_IMM_DATA_EN
+endif
+
+ifeq ($(disable_st_c2h_completion),1)
+ FLAGS += -DXMP_DISABLE_ST_C2H_CMPL
+endif
+
+# LOOPBACK flags.
+ifeq ($(TEST_64B_DESC_BYPASS),1)
+ FLAGS += -DTEST_64B_DESC_BYPASS_FEATURE
+endif
+
+ifeq ($(ERR_DEBUG),1)
+ EXTRA_FLAGS += -DERR_DEBUG
+# FLAGS += -DERR_DEBUG
+ export EXTRA_FLAGS
+endif
+
+# Don't allow ARCH to overwrite the modified variable when passed to
+# the sub-makes.
+MAKEOVERRIDES := $(filter-out ARCH=%,$(MAKEOVERRIDES))
+# Don't allow CFLAGS/EXTRA_CFLAGS to clobber definitions in sub-make.
+MAKEOVERRIDES := $(filter-out CFLAGS=%,$(MAKEOVERRIDES))
+MAKEOVERRIDES := $(filter-out EXTRA_CFLAGS=%,$(MAKEOVERRIDES))
+
+# Exports.
+export grep
+export srcdir
+export topdir
+export build_dir
+export KERNELRELEASE
+export KSRC
+export KOBJ
+export KINC
+# arm64 specific fix to include <ksrc>/arch/<karch> folder properly.
+# This hack is motivated by the RHEL7.X/CentOS7.X release where the
+# uname Architecture is indicated as "aarch64" but the real Architecture
+# source directory is "arm64"
+ifeq ($(ARCH),aarch64)
+ ifeq ($(wildcard $(KOBJ)/arch/$(ARCH)/Makefile),)
+ override MAKECMDGOALS = $(MAKECMDGOALS) "ARCH=arm64"
+ else
+ export ARCH
+ endif
+else
+ export ARCH
+endif
+export FLAGS
+#export FLAGS += $(CFLAGS) $(EXTRA_CFLAGS) $(CPPFLAGS)
+export verbose
+export utsrelease
+export kversions
+export kseries
+export modulesymfile
+
+export enable_xvc
+
+# evaluate install paths
+ifeq ($(install_path),)
+ # defaults
+ kernel_install_path ?= $(PREFIX)/lib/modules/$(utsrelease)/updates/kernel/drivers/qdma
+ dev_install_path ?= /usr/local/include/qdma
+ user_install_path ?= /usr/local/sbin
+ conf_install_path ?= /etc/xilinx-dma
+ udev_install_path ?= /etc/udev/rules.d
+ docs_install_path ?= /usr/share/man/man8
+else # bundled install
+ kernel_install_path ?= $(install_path)/modules
+ dev_install_path ?= $(install_path)/include/qdma
+ user_install_path ?= $(install_path)/bin
+ conf_install_path ?= $(install_path)/etc
+ udev_install_path ?= $(install_path)/etc
+ docs_install_path ?= $(install_path)/doc
+endif
+
+$(shell rm -f $(srcdir)/drv/libqdma; ln -fs $(srcdir)/libqdma $(srcdir)/drv;)
+
+.PHONY: eval.mak
+
+.PHONY: default
+default: user tools mod_pf mod_vf post
+
+.PHONY: pf
+pf: user mod_pf
+
+.PHONY: vf
+vf: user mod_vf
+
+.PHONY: mods
+mod: mod_pf mod_vf
+
+.PHONY: install
+install: install-mods install-user install-etc install-dev
+
+.PHONY: uninstall
+uninstall: uninstall-mod uninstall-user uninstall-dev
+
+.PHONY: user
+user:
+ @echo "#######################";
+ @echo "#### user ####";
+ @echo "#######################";
+ $(MAKE) -C user
+
+.PHONY: tools
+tools:
+ @echo "#######################";
+ @echo "#### tools ####";
+ @echo "#######################";
+ $(MAKE) -C tools
+
+.PHONY: mod_pf
+mod_pf:
+ @if [ -n "$(verbose)" ]; then \
+ echo "#######################";\
+ printf "#### PF %-8s%5s####\n" $(DRIVER_SRC_DIR);\
+ echo "#######################";\
+ fi;
+ @drvdir=$(shell pwd)/$(DRIVER_SRC_DIR) $(MAKE) VF=0 -C $(DRIVER_SRC_DIR);
+
+.PHONY: mod_vf
+mod_vf:
+ @if [ -n "$(verbose)" ]; then \
+ echo "#######################";\
+ printf "#### VF %-8s%5s####\n" $(DRIVER_SRC_DIR);\
+ echo "#######################";\
+ fi;
+ @drvdir=$(shell pwd)/$(DRIVER_SRC_DIR) $(MAKE) VF=1 -C $(DRIVER_SRC_DIR);
+
+.PHONY: post
+post:
+ @if [ -n "$(post_msg)" ]; then \
+ echo -e "\nWARNING:\n $(post_msg)";\
+ fi;
+
+.PHONY: clean
+clean:
+ @echo "#######################";
+ @echo "#### user ####";
+ @echo "#######################";
+ $(MAKE) -C user clean;
+ @echo "#######################";
+ @echo "#### tools ####";
+ @echo "#######################";
+ $(MAKE) -C tools clean;
+ @for dir in $(ALLSUBDIRS); do \
+ echo "#######################";\
+ printf "#### %-8s%5s####\n" $$dir;\
+ echo "#######################";\
+ drvdir=$(shell pwd)/$$dir $(MAKE) -C $$dir clean;\
+ done;
+ @-/bin/rm -f *.symvers eval.mak 2>/dev/null;
+
+.PHONY: install-mods
+install-mods:
+ @echo "installing kernel modules to $(kernel_install_path) ..."
+ @mkdir -p -m 755 $(kernel_install_path)
+ @install -v -m 644 $(build_dir)/*.ko $(kernel_install_path)
+ @depmod -a || true
+
+
+.PHONY: install-user
+install-user:
+ @echo "installing user tools to $(user_install_path) ..."
+ @mkdir -p -m 755 $(user_install_path)
+ @install -v -m 755 build/dmactl* $(user_install_path)
+ @install -v -m 755 tools/dma_from_device $(user_install_path)
+ @install -v -m 755 tools/dma_to_device $(user_install_path)
+ @install -v -m 755 tools/dmautils $(user_install_path)
+ @echo "MAN PAGES:"
+ @mkdir -p -m 755 $(docs_install_path)
+ @install -v -m 644 docs/dmactl.8.gz $(docs_install_path)
+
+.PHONY: install-dev
+install-dev:
+ @echo "installing development headers to $(dev_install_path) ..."
+ @mkdir -p -m 755 $(dev_install_path)
+ @install -v -m 755 include/* $(dev_install_path)
+
+.PHONY: install-etc
+install-etc:
+ @echo "install Xilinx DMA udev rules:"
+ @mkdir -p -m 755 $(udev_install_path)
+ @if [ ! -d "$(udev_install_path)/30-qdma.rules" ]; then \
+ install -v -m 644 etc/30-qdma.rules $(udev_install_path); \
+ fi;
+
+.PHONY: uninstall-mod
+uninstall-mod:
+ @echo "Un-installing $(kernel_install_path) ..."
+ @/bin/rm -rf $(kernel_install_path)/*
+ @depmod -a
+
+.PHONY: uninstall-user
+uninstall-user:
+ @echo "Un-installing user tools under $(user_install_path) ..."
+ @/bin/rm -f $(user_install_path)/dmactl
+ @/bin/rm -f $(user_install_path)/dma_from_device
+ @/bin/rm -f $(user_install_path)/dma_to_device
+ @/bin/rm -f $(user_install_path)/dmautils
+
+.PHONY: uninstall-dev
+uninstall-dev:
+ @echo "Un-installing development headers under $(dev_install_path) ..."
+ @/bin/rm -r $(dev_install_path)
+
+.PHONY: help
+help:
+ @echo "Build Targets:";\
+ echo " install - Installs all compiled drivers.";\
+ echo " uninstall - Uninstalls drivers.";\
+ echo " clean - Removes all generated files.";\
+ echo;\
+ echo "Build Options:";\
+ echo " KOBJ=<path> - Kernel build (object) path.";\
+ echo " KSRC=<path> - Kernel source path.";\
+ echo " - Note: When using KSRC or KOBJ, both";\
+ echo " variables must be specified.";\
+ echo " KDIR=<path> - Kernel build and source path. Shortcut";\
+ echo " for KOBJ=KSRC=<path>.";\
+ echo " kernel_install_path=<path>";\
+ echo " - kernel module install path.";\
+ echo " user_install_path=<path>";\
+ echo " - user cli tool install path.";\
+ ech_;
diff --git a/QDMA/linux-kernel/README b/QDMA/linux-kernel/README
new file mode 120000
index 0000000000000000000000000000000000000000..a90f4af9594dfe9eebfb5c293050019e6d17035e
--- /dev/null
+++ b/QDMA/linux-kernel/README
@@ -0,0 +1 @@
+docs/README
\ No newline at end of file
diff --git a/QDMA/linux-kernel/RELEASE b/QDMA/linux-kernel/RELEASE
new file mode 100644
index 0000000000000000000000000000000000000000..f9cce4889a37f1400b971156db4e9450dab5609d
--- /dev/null
+++ b/QDMA/linux-kernel/RELEASE
@@ -0,0 +1,72 @@
+Release: 2018.3
+===========================
+
+NOTE:
+ This release is based on the 2018.3 QDMA design
+
+ Performance benchmark numbers are captured in perforamcne report
+ for this version of driver and released along with the driver package
+
+ QDMA Linux driver is supported in auto mode by default
+ (refer to 2018.3 Features below for more details on auto mode)
+
+ QDMA 2018.3 Linux driver is not backaward compatible with 2018.2 IP
+
+
+DRIVER LIMITATIONS:
+===================
+- Driver compilation on Fedora 28 with gcc8.1 results compilation warnings
+
+SUPPORTED FEATURES:
+===================
+- Support for both the AXI4 Memory Mapped(MM) and AXI4 Streaming(ST) Interfaces
+- 2048 Queue Sets
+ - 2048 H2C Descriptor Rings
+ - 2048 C2H Descriptor Rings
+ - 2048 C2H Completion Rings
+- Supports Polling Mode
+- Interrupts
+ - 2048 MSI-X Vectors
+ - Up to 8 MSI-X per Function
+ - Interrupt Aggregation
+ - User Interrupts
+ - Error Interrupts
+- C2H Stream Completion queue entry coalescing
+- Supports SR-IOV with upto 4 Physical Functions(PF) and 252 Virtual Functions(VF)
+- Allows only Privileged/Physical functions to program the contexts & registers
+- Function Level Reset(FLR) Support
+- Mailbox Support
+- Descriptor bypass(8,16,32 descriptor sizes)
+- Descriptor Prefetching
+- Asynchronous and Synchronous IO support
+- ST H2C to C2H and C2H to H2C loopback support
+- Driver configuration through sysfs
+- Zero byte transfer support
+- Dynamic queue configuration
+- Dynamic driver configuration
+
+2018.3 Features:
+- Auto mode support for PF & VF driver
+ - MM and ST H2C requests are services in poll mode
+ - ST C2H is services in Interrupt Aggregation mode
+- Support 64B descriptor format in byapss mode
+- Support for Completion queue descriptors of 64 bytes size
+- Support flexible BAR mapping for QDMA configuration register space
+- Indirect programming of FMAP registers
+- Support disabling overflow check in completion ring
+- Version for SW and HW
+- ECC Support
+- Flexible interrupt allocation between PF/VF
+- Greater than 32 vectors per function
+- Legacy Interrupt Support
+- VF driver support in direct and indirect interrupt mode
+
+
+KNOWN ISSUES:
+=============
+- Occasionally, C2H completions are not observed when
+ payload and immediate data transfers are combined
+- In the Host only mode, When VFs are loaded in mixed interrupt mode(i.e
+ some VFs are in direct interrupt mode and some VFs are in indirect
+ interrupt mode),C2H completions are not observed for the VF with direct
+ mode interrupt which follows a VF with indirect interrupt mode
diff --git a/QDMA/linux-kernel/docs/Doxyfile b/QDMA/linux-kernel/docs/Doxyfile
new file mode 100644
index 0000000000000000000000000000000000000000..fbf5b4453be91fd324481368396effa54f4844d6
--- /dev/null
+++ b/QDMA/linux-kernel/docs/Doxyfile
@@ -0,0 +1,2443 @@
+# Doxyfile 1.8.11
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME = "Linux QDMA Driver"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER = 2000-0142
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF = API
+
+# With the PROJECT_LOGO tag one can specify a logo or an icon that is included
+# in the documentation. The maximum height of the logo should not exceed 55
+# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
+# the logo to the output directory.
+
+PROJECT_LOGO =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY = qdma
+
+# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS = NO
+
+# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII
+# characters to appear in the names of generated files. If set to NO, non-ASCII
+# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode
+# U+3044.
+# The default value is: NO.
+
+ALLOW_UNICODE_NAMES = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC = NO
+
+# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF = "The $name class" \
+ "The $name widget" \
+ "The $name file" \
+ is \
+ provides \
+ specifies \
+ contains \
+ represents \
+ a \
+ an \
+ the
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB = NO
+
+# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES = NO
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH =
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH =
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit \brief command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new
+# page for each member. If set to NO, the documentation of a member will be part
+# of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE = 4
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C = YES
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
+# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
+# Fortran. In the later case the parser tries to guess whether the code is fixed
+# or free formatted code, this is the default for Fortran type files), VHDL. For
+# instance to make doxygen treat .inc files as Fortran files (default is PHP),
+# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+#
+# Note: For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC = NO
+
+# If one adds a struct or class to a group and this option is enabled, then also
+# any nested class or struct is added to the same group. By default this option
+# is disabled and one has to add nested compounds explicitly via \ingroup.
+# The default value is: NO.
+
+GROUP_NESTED_COMPOUNDS = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE = 0
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE = NO
+
+# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO,
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES = YES
+
+# This flag is only useful for Objective-C code. If set to YES, local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO, only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO, these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO, these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES, upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES = NO
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES, the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES = NO
+
+# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will
+# append additional text to a page's title, such as Class Reference. If set to
+# YES the compound reference will be hidden.
+# The default value is: NO.
+
+HIDE_COMPOUND_REFERENCE= NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES = NO
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS = NO
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO, the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo
+# list. This list is created by putting \todo commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test
+# list. This list is created by putting \test commands in the documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES, the
+# list will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES = NO
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. See also \cite for info how to create references.
+
+CITE_BIB_FILES =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS = YES
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED = YES
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO, doxygen will only warn about wrong or incomplete
+# parameter documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC = NO
+
+# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when
+# a warning is encountered.
+# The default value is: NO.
+
+WARN_AS_ERROR = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+INPUT = ../
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
+# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
+
+FILE_PATTERNS = *.h
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS = */apps/* \
+ */build/* \
+ */etc/* \
+ */example/* \
+ */scripts/* \
+ */user/* \
+ */tools/*
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH =
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS = *
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH =
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+INPUT_FILTER =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# properly processed by doxygen.
+
+FILTER_PATTERNS =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = NO
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION = NO
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS = YES
+
+# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the
+# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the
+# cost of reduced performance. This can be particularly helpful with template
+# rich C++ code for which doxygen's built-in parser lacks the necessary type
+# information.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse-libclang=ON option for CMake.
+# The default value is: NO.
+
+CLANG_ASSISTED_PARSING = NO
+
+# If clang assisted parsing is enabled you can provide the compiler with command
+# line options that you would normally use when invoking the compiler. Note that
+# the include paths will already be set by doxygen for the files and directories
+# specified with INPUT and INCLUDE_PATH.
+# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES.
+
+CLANG_OPTIONS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX = 5
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML = NO
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# cascading style sheets that are included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list). For an example see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the style sheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to YES can help to show when doxygen was last run and thus if the
+# documentation is up to date.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP = NO
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler (hhc.exe). If non-empty,
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated
+# (YES) or that it should be included in the master .chm file (NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated
+# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it
+# enables the Previous and Next buttons.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP = NO
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE =
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE = org.doxygen.Project
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME =
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS =
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION =
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW = YES
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE = 1
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH = 250
+
+# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE = YES
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavors of web server based searching depending on the EXTERNAL_SEARCH
+# setting. When disabled, doxygen will generate a PHP script for searching and
+# an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
+# and searching needs to be provided by external tools. See the section
+# "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH = NO
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH = YES
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer (doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL =
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID =
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX = YES
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME = makeindex
+
+# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. The package can be specified just
+# by its name or with the correct syntax as to be used with the LaTeX
+# \usepackage command. To get the times font for instance you can specify :
+# EXTRA_PACKAGES=times or EXTRA_PACKAGES={times}
+# To use the option intlimits with the amsmath package you can specify:
+# EXTRA_PACKAGES=[intlimits]{amsmath}
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber,
+# $projectbrief, $projectlogo. Doxygen will replace $title with the empty
+# string, for the replacement values of the other commands the user is referred
+# to HTML_HEADER.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer. See
+# LATEX_HEADER for more information on how to generate a default footer and what
+# special commands can be used inside the footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER =
+
+# The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined
+# LaTeX style sheets that are included after the standard style sheets created
+# by doxygen. Using this option one can overrule certain style aspects. Doxygen
+# will copy the style sheet files to the output directory.
+# Note: The order of the extra style sheet files is of importance (e.g. the last
+# style sheet in the list overrules the setting of the previous ones in the
+# list).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_STYLESHEET =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS = YES
+
+# If the USE_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES, to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE = plain
+
+# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_TIMESTAMP = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT = rtf
+
+# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF = YES
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE =
+
+# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code
+# with syntax highlighting in the RTF output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_SOURCE_CODE = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION = .3
+
+# The MAN_SUBDIR tag determines the name of the directory created within
+# MAN_OUTPUT in which the man pages are placed. If defaults to man followed by
+# MAN_EXTENSION with the initial . removed.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_SUBDIR =
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT = docbook
+
+# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the
+# program listings (including syntax highlighting and cross-referencing
+# information) to the DOCBOOK output. Note that enabling this will significantly
+# increase the size of the DOCBOOK output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_PROGRAMLISTING = NO
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
+# AutoGen Definitions (see http://autogen.sf.net) file that captures the
+# structure of the code including all documentation. Note that this feature is
+# still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD = NO
+
+# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX = NO
+
+# If the PERLMOD_PRETTY tag is set to YES, the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO, the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF = NO
+
+# If the SEARCH_INCLUDES tag is set to YES, the include files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH =
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have
+# an all uppercase name, and do not end with a semicolon. Such function macros
+# are typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have a unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE =
+
+# If the ALLEXTERNALS tag is set to YES, all external class will be listed in
+# the class index. If set to NO, only the inherited external classes will be
+# listed.
+# The default value is: NO.
+
+ALLEXTERNALS = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed
+# in the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS = YES
+
+# If the EXTERNAL_PAGES tag is set to YES, all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH =
+
+# If set to YES the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: YES.
+
+HAVE_DOT = NO
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS = 0
+
+# When you want a differently looking font in the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS = YES
+
+# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS = NO
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command. Disabling a call graph can be
+# accomplished by means of the command \hidecallgraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command. Disabling a caller graph can be
+# accomplished by means of the command \hidecallergraph.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot. For an explanation of the image formats see the section
+# output formats in the documentation of the dot tool (Graphviz (see:
+# http://www.graphviz.org/)).
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, png:cairo, png:cairo:cairo, png:cairo:gd, png:gd,
+# png:gd:gd, jpg, jpg:cairo, jpg:cairo:gd, jpg:gd, jpg:gd:gd, gif, gif:cairo,
+# gif:cairo:gd, gif:gd, gif:gd:gd, svg, png:gd, png:gd:gd, png:cairo,
+# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and
+# png:gdiplus:gdiplus.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT = png
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH =
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS =
+
+# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the
+# path where java can find the plantuml.jar file. If left blank, it is assumed
+# PlantUML is not used or called during a preprocessing step. Doxygen will
+# generate a warning when it encounters a \startuml command in this case and
+# will not generate output for the diagram.
+
+PLANTUML_JAR_PATH =
+
+# When using plantuml, the specified paths are searched for files specified by
+# the !include statement in a plantuml block.
+
+PLANTUML_INCLUDE_PATH =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS = NO
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND = YES
+
+# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP = YES
diff --git a/QDMA/linux-kernel/docs/QDMA_Driver_APIs.pdf b/QDMA/linux-kernel/docs/QDMA_Driver_APIs.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..2067210305a69349a653958acf90c91520b2ed25
Binary files /dev/null and b/QDMA/linux-kernel/docs/QDMA_Driver_APIs.pdf differ
diff --git a/QDMA/linux-kernel/docs/README b/QDMA/linux-kernel/docs/README
new file mode 100644
index 0000000000000000000000000000000000000000..a4b9e64b3fae008b47fc5f3aa547ce7e0c3b2b20
--- /dev/null
+++ b/QDMA/linux-kernel/docs/README
@@ -0,0 +1,267 @@
+###############################################################################
+
+ Xilinx QDMA Software README
+
+###############################################################################
+
+
+_____________________________________________________________________________
+Contents
+
+1. Installation
+ 1.1 Compiling the QDMA Software
+ 1.2 Installing the compiled binaries
+ 1.3 Loading the Kernel module
+2 Configuration
+ 2.1 Configuring Queues
+3. Xilinx "dmactl" Command-line Utility
+ 3.1 Using dmactl for query the QDMA devices/functions
+ 3.2 Using dmactl for Queue control
+4. QDMA test script
+ Sample scripts to run simple MM and ST transfers
+_____________________________________________________________________________
+
+1. Installation:
+
+ 1.1 Compiling the QDMA Software:
+ --------------------------------
+
+ In order to compile the Xilinx QDMA software, a configured and compiled
+ Linux kernel source tree is required. The source tree may be only compiled
+ header files, or a complete tree. The source tree needs to be configured
+ and the header files need to be compiled. And, the Linux kernel must be
+ configured to use modules.
+
+ Additionally, the /lib/modules must have been set up for this particular
+ kernel
+ (i.e. "make modules_install" has been run with the Linux kernel source tree).
+
+ a. QDMA Linux Kernel Driver is available in Xilinx github at
+ https://github.com/Xilinx/dma_ip_drivers/QDMA/linux-kernel
+
+ b. Compile the Xilinx QDMA Linux driver:
+
+ [xilinx@]# make
+
+ a sub-directory build/ will be created as a result of running "make".
+
+ By default, both PF driver (qdma.ko) and VF driver (qdma_vf.ko) will
+ be compiled.
+
+ If only PF driver is desired:
+ [xilinx@]# make pf
+
+ If only VF driver is desired:
+ [xilinx@]# make vf
+
+ 1.2 Installing the compiled binaries:
+ -------------------------------------
+
+ To install the QDMA software, the installer must be the root user, then
+ run "make install".
+
+ [xilinx@]# make install
+
+ The QDMA module will be installed in the
+ /lib/modules/<linux_kernel_version>/updates/kernel/drivers/qdma directory.
+
+ The dmactl tool will be installed in /user/local/sbin.
+
+ Example application software to issue DMA requests:
+ dma_to_device for H2C queues and
+ dma_from_device for C2H queues
+ will be installed in /user/local/sbin.
+
+
+ 1.3 Loading the Kernel module:
+ --------------------------------
+
+ Kernel module cane be loaded in following different modes
+ 0 - Auto Mode, driver decides to process the request in poll or interrupt mode
+ 1 - Poll Mode
+ 2 - Direct Interrupt Mode
+ 3 - Interrupt Aggregation Mode or Indirect Interrupt Mode
+ 4 - Legacy Interrupt Mode
+
+ To load the module in poll mode run modprobe as follows:
+
+ [root@]# modprobe qdma mode=1
+
+ Or on a VM:
+ [root@]# modprobe qdma_vf
+
+ To load the module in direct interrupt mode run modprobe as follows:
+
+ [root@]# modprobe qdma mode=2
+
+ To load the module in indirect interrupt mode run modprobe as follows:
+
+ [root@]# modprobe qdma mode=3
+
+ To set a master pf other than the first pf, load the module as follows:
+
+ [root@]# modprobe qdma master_pf=0x<bbddf>
+
+ Note:<bbddf> can figured out for a pf from 'lspci' output
+ bb - bus number
+ dd - device number
+ f - function number
+
+ Now the QDMA software is ready for use.
+
+2. Configuration
+
+ 2.1 Configuring Queues
+ -------------------------------------
+
+ To configure a QDMA queue, there are three minimum required parameters
+ (in the form of key value pairs) needed to be configured.
+
+ idx <N>: The 0-based queue index on the function.
+ mode <mm | st>: queue mode, default to "mm"
+ dir <h2c | c2h>: queue direction, default to "h2c"
+
+ - "idx" is a 0-based unique index on the function.
+ *With the 2018.2 release, the range is
+ 0 ~ (512 - 8 * total # of VFs) on a physical function and
+ 0 ~ 7 on a virtual function
+
+ - "mode" is the operation mode of the queue.
+ It is either memory mapped (mm) or streaming mode (st)
+
+ - "dir" is the direction of the queue.
+ It is either host to card (h2c) or card to host (c2h).
+
+ A h2c queue and a c2h queue can share the same index. In other word, a index
+ represents a pair of queues: one on h2c direction and the other on the c2h
+ direction.
+
+
+3. Xilinx "dmactl" Command-line Configuration Utility:
+
+ The Xilinx QDMA control tool, dmactl, is a Command Line utility
+ which is installed in /usr/local/sbin/ and allows administration of the
+ Xilinx QDMA queues. It can perform the following functions:
+
+ - query the qdma functions/devices the driver has bind into.
+
+ - list all of the queues on a device/function
+ - add/configure a new queues on a device/function
+ - start an already added/configured queue (i.e., bring the queue online)
+ - stop an started queue (i.e., bring the queue offline)
+ - delete an already added/configured queue
+
+ register access:
+ - read a register
+ - write a register
+ - dump the qdma config bar and user bar registers
+
+ debug helper
+ - display a queue's configuration parameters
+ - display a queue's descriptor ring entries
+ - display a ch2 queue's completion ring entries
+ - display the interrupt ring entries in indirect interrupt mode
+
+ Apart from the above basic commands, there are various other commands
+ available in dmactl to add/start/stop/del list of queues at the same time
+
+ For help run:
+ dmactl -h
+ (or)
+ man dmactl
+
+
+ 3.1 Using dmactl for query the QDMA devices/functions
+ -------------------------------------
+
+ Please refer User Guide/dmactl man page to find all options and parameters
+ available.
+
+ 1. Get the list of devices the driver has bind with
+
+ [root@]# dmactl dev list
+
+ qdma06000 0000:06:00.0 max QP: 32
+ qdma06001 0000:06:00.1 max QP: 32
+
+ The above example output shows 2 QDMA functions/devices.
+ one is at pci BDF: 06:00.0
+ the other is at pci BDF: 06:00.1
+
+
+ 3.2 Using dmactl for Queue control
+ -------------------------------------
+
+ Please refer User Guide to find all options and parameters available.
+
+ a. Add/Configure a queue
+
+ To add a MM H2C queue on qdma06000 in the above example:
+
+ [root@]# dmactl qdma06000 q add idx 0 mode mm dir h2c
+
+ *A character device /dev/qdma06000-MM-0 would be created.
+
+ To add a MM C2H queue on qdma06000:
+
+ [root@]# dmactl qdma06000 q add idx 0 mode mm dir c2h
+
+ *A character device /dev/qdma06000-MM-0 would be created.
+
+ b. Start an added queue
+
+ To start the MM H2C queue on qdma06000 added in the previous example:
+
+ [root@]# dmactl qdma06000 q start idx 0 dir h2c
+
+ *After the queue is started the normal read and write operation can be
+ performed on the character device /dev/qdma06000-MM-0.
+
+ To start the MM C2H queue on qdma06000 added in the previous example:
+
+ [root@]# dmactl qdma06000 q start idx 0 dir c2h
+
+ *After the queue is started the normal read and write operation can be
+ performed on the character device /dev/qdma06000-MM-0.
+
+ c. Stop a queue
+
+ [root@]# dmactl qdma06000 q start idx 0 dir h2c
+ [root@]# dmactl qdma06000 q start idx 0 dir c2h
+
+
+ d. Delete a queue
+
+ [root@]# dmactl qdma06000 q del idx 0 dir h2c
+ [root@]# dmactl qdma06000 q del idx 0 dir c2h
+
+
+4. QDMA test script
+
+ Test scripts are in scripts/ directory
+
+ To use the scripts in this directory, "make" and "make install" must be run
+ install the dma_from_device and dma_to_device applications.
+
+ qdma_run_test.sh
+ The script does the following dma operations:
+
+ PF AXI-MM : H2C/C2H AXI-MM transfer and check for data correctness.
+ PF AXI-ST-H2C : H2C AXI-ST transfer and read data register on user side
+ to check if the data if correct.
+ PF AXI-ST-C2H : C2H AXI-St transfer and check data for correctness.
+
+ All the above transfers are done for 4 [0 to 3] Queues.
+
+ qdma_run_test_mm_vf.sh
+ VF AXI-MM : Script runs H2C/C2H AXI-MM transfer and check for data
+ correctness.
+
+ All the above transfers are done for 4 [0 to 3] Queues.
+
+ qdma_run_test_st_vf.sh
+ VF AXI-ST-H2C : H2C AXI-ST transfer and read data register on user side
+ to check if the data if correct.
+ VF AXI-ST-C2H : C2H AXI-St transfer and check data for correctness.
+
+ All the above transfers are done for 4 [0 to 3] Queues.
diff --git a/QDMA/linux-kernel/docs/dmactl.8.gz b/QDMA/linux-kernel/docs/dmactl.8.gz
new file mode 100644
index 0000000000000000000000000000000000000000..b55007269369e7d42a69f4e8bdfd3e318f302085
Binary files /dev/null and b/QDMA/linux-kernel/docs/dmactl.8.gz differ
diff --git a/QDMA/linux-kernel/docs/linux_qdma_driver_user_guide.pdf b/QDMA/linux-kernel/docs/linux_qdma_driver_user_guide.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..813d82a5e87104f09803f9a51bce12033809de4f
Binary files /dev/null and b/QDMA/linux-kernel/docs/linux_qdma_driver_user_guide.pdf differ
diff --git a/QDMA/linux-kernel/drv/.cache.mk b/QDMA/linux-kernel/drv/.cache.mk
new file mode 100644
index 0000000000000000000000000000000000000000..938e7a5d69a6b9971e834b5d2c653ff15991a228
--- /dev/null
+++ b/QDMA/linux-kernel/drv/.cache.mk
@@ -0,0 +1,63 @@
+__cached_gcc_-v_2>&1_|_grep_-q_"clang_version"_&&_echo_clang_||_echo_gcc := gcc
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-mretpoline-external-thunk_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mretpoline-external-thunk";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" :=
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-mindirect-branch_thunk-extern_-mindirect-branch-register_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mindirect-branch_thunk-extern_-mindirect-branch-register";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mindirect-branch=thunk-extern -mindirect-branch-register
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-PIE";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-PIE
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-fno-PIE_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-PIE";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-PIE
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-Wmaybe-uninitialized_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-maybe-uninitialized";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-maybe-uninitialized
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE__-mpreferred-stack-boundary_4_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-mpreferred-stack-boundary_4";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mpreferred-stack-boundary=4
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE__-m16_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-m16";_else_echo_"_-m32_-Wa_./arch/x86/boot/code16gcc.h";_fi;_rm_-f_"_TMP"_"_TMPO" := -m16
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-m16_-g_-Os_-D__KERNEL___-DDISABLE_BRANCH_PROFILING_-Wall_-Wstrict-prototypes_-march_i386_-mregparm_3_-fno-strict-aliasing_-fomit-frame-pointer_-fno-pic_-mno-mmx_-mno-sse__-ffreestanding_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-ffreestanding";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -ffreestanding
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-m16_-g_-Os_-D__KERNEL___-DDISABLE_BRANCH_PROFILING_-Wall_-Wstrict-prototypes_-march_i386_-mregparm_3_-fno-strict-aliasing_-fomit-frame-pointer_-fno-pic_-mno-mmx_-mno-sse_-ffreestanding__-fno-stack-protector_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-fno-stack-protector";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-stack-protector
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-m16_-g_-Os_-D__KERNEL___-DDISABLE_BRANCH_PROFILING_-Wall_-Wstrict-prototypes_-march_i386_-mregparm_3_-fno-strict-aliasing_-fomit-frame-pointer_-fno-pic_-mno-mmx_-mno-sse_-ffreestanding_-fno-stack-protector__-mpreferred-stack-boundary_2_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-mpreferred-stack-boundary_2";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mpreferred-stack-boundary=2
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mno-avx";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mno-avx
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-falign-jumps_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -falign-jumps=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-falign-loops_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -falign-loops=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mno-80387";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mno-80387
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mno-fp-ret-in-387";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mno-fp-ret-in-387
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mpreferred-stack-boundary_3";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mpreferred-stack-boundary=3
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mskip-rax-setup";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mskip-rax-setup
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-mtune_generic";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mtune=generic
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-funit-at-a-time";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -funit-at-a-time
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI__-mfentry_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"y";_else_echo_"n";_fi;_rm_-f_"_TMP"_"_TMPO" := y
+__cached_/bin/bash_./scripts/gcc-version.sh_-p_gcc := 070300
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_".cfi_startproc_n.cfi_rel_offset_rsp_0_n.cfi_endproc"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_CFI_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_CFI=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_".cfi_startproc_n.cfi_signal_frame_n.cfi_endproc"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_CFI_SIGNAL_FRAME_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_CFI_SIGNAL_FRAME=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_".cfi_sections_.debug_frame"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_CFI_SECTIONS_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_CFI_SECTIONS=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"fxsaveq__%rax_"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_FXSAVEQ_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_FXSAVEQ=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"pshufb_%xmm0_%xmm0"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_SSSE3_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_SSSE3=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"crc32l_%eax_%eax"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_CRC32_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_CRC32=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"vxorps_%ymm0_%ymm1_%ymm2"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_AVX_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_AVX=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"vpbroadcastb_%xmm0_%ymm1"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_AVX2_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_AVX2=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"vpmovm2b_%k1_%zmm5"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_AVX512_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_AVX512=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"sha1msg1_%xmm0_%xmm1"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_SHA1_NI_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_SHA1_NI=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___printf_"%b_n"_"sha256msg1_%xmm0_%xmm1"_|_gcc_-D__ASSEMBLY___-fno-PIE_-m64_-DCONFIG_X86_X32_ABI_-c_-x_assembler_-o_"_TMP"_-__>/dev/null_2>&1;_then_echo_"-DCONFIG_AS_SHA256_NI_1";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -DCONFIG_AS_SHA256_NI=1
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-x_c_/dev/null_-c_-o_"_TMPO";_ld_-m_elf_x86_64__-z_max-page-size_0x200000_"_TMPO"_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-z_max-page-size_0x200000";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -z max-page-size=0x200000
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-delete-null-pointer-checks";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-delete-null-pointer-checks
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wframe-address_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-frame-address";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-frame-address
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wformat-truncation_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-format-truncation";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-format-truncation
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wformat-overflow_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-format-overflow";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-format-overflow
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wint-in-bool-context_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-int-in-bool-context";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-int-in-bool-context
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_-Wmaybe-uninitialized_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-maybe-uninitialized";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-maybe-uninitialized
+__cached_/bin/bash_./scripts/gcc-version.sh_gcc := 0703
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"--param_allow-store-data-races_0";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := --param=allow-store-data-races=0
+__cached_/bin/bash_./scripts/gcc-goto.sh_gcc_-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2__--param_allow-store-data-races_0 := y
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wframe-larger-than_1024";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wframe-larger-than=1024
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wunused-but-set-variable_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-unused-but-set-variable";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-unused-but-set-variable
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wunused-const-variable_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-unused-const-variable";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-unused-const-variable
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls__-fno-var-tracking-assignments_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-fno-var-tracking-assignments";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-var-tracking-assignments
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments__-mfentry_-DCC_USING_FENTRY_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-mfentry_-DCC_USING_FENTRY";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -mfentry -DCC_USING_FENTRY
+__cached_gcc_-print-file-name_include := /usr/lib/gcc/x86_64-linux-gnu/7/include
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wdeclaration-after-statement";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wdeclaration-after-statement
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc_-Werror_-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wpointer-sign_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Wno-pointer-sign";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wno-pointer-sign
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-strict-overflow";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-strict-overflow
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-merge-all-constants";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-merge-all-constants
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fmerge-constants";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fmerge-constants
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fno-stack-check";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fno-stack-check
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-fconserve-stack";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -fconserve-stack
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Werror_implicit-int";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Werror=implicit-int
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-Werror_strict-prototypes_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Werror_strict-prototypes";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Werror=strict-prototypes
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-Werror_strict-prototypes_-Werror_date-time_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Werror_date-time";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Werror=date-time
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-Werror_strict-prototypes_-Werror_date-time_-Werror_incompatible-pointer-types_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Werror_incompatible-pointer-types";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Werror=incompatible-pointer-types
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if____gcc_-Werror__-D__KERNEL___-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-Werror_strict-prototypes_-Werror_date-time_-Werror_incompatible-pointer-types_-Werror_designated-init_-c_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"-Werror_designated-init";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Werror=designated-init
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___ar_rcD_"_TMP"__>/dev/null_2>&1;_then_echo_"D";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := D
+__cached_set_-e;_TMP_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.tmp";_TMPO_"/home/karenx/MY_SYSPRO/GIT/dma_ip_drivers.git/QDMA/linux-kernel/drv/.__.o";_if___gcc__-Wl_--build-id_-D__KERNEL_____-Wall_-Wundef_-Wstrict-prototypes_-Wno-trigraphs_-fno-strict-aliasing_-fno-common_-fshort-wchar_-Werror-implicit-function-declaration_-Wno-format-security_-std_gnu89_-fno-PIE_-mno-sse_-mno-mmx_-mno-sse2_-mno-3dnow_-mno-avx_-m64_-falign-jumps_1_-falign-loops_1_-mno-80387_-mno-fp-ret-in-387_-mpreferred-stack-boundary_3_-mskip-rax-setup_-mtune_generic_-mno-red-zone_-mcmodel_kernel_-funit-at-a-time_-DCONFIG_X86_X32_ABI_-DCONFIG_AS_CFI_1_-DCONFIG_AS_CFI_SIGNAL_FRAME_1_-DCONFIG_AS_CFI_SECTIONS_1_-DCONFIG_AS_FXSAVEQ_1_-DCONFIG_AS_SSSE3_1_-DCONFIG_AS_CRC32_1_-DCONFIG_AS_AVX_1_-DCONFIG_AS_AVX2_1_-DCONFIG_AS_AVX512_1_-DCONFIG_AS_SHA1_NI_1_-DCONFIG_AS_SHA256_NI_1_-pipe_-Wno-sign-compare_-fno-asynchronous-unwind-tables_-mindirect-branch_thunk-extern_-mindirect-branch-register_-DRETPOLINE_-fno-delete-null-pointer-checks_-Wno-frame-address_-Wno-format-truncation_-Wno-format-overflow_-Wno-int-in-bool-context_-O2_--param_allow-store-data-races_0_-DCC_HAVE_ASM_GOTO_-Wframe-larger-than_1024_-fstack-protector-strong_-Wno-unused-but-set-variable_-Wno-unused-const-variable_-fno-omit-frame-pointer_-fno-optimize-sibling-calls_-fno-var-tracking-assignments_-pg_-mfentry_-DCC_USING_FENTRY_-Wdeclaration-after-statement_-Wno-pointer-sign_-fno-strict-overflow_-fno-merge-all-constants_-fmerge-constants_-fno-stack-check_-fconserve-stack_-Werror_implicit-int_-Werror_strict-prototypes_-Werror_date-time_-Werror_incompatible-pointer-types_-Werror_designated-init_-nostdlib_-x_c_/dev/null_-o_"_TMP"__>/dev/null_2>&1;_then_echo_"_-Wl_--build-id";_else_echo_"";_fi;_rm_-f_"_TMP"_"_TMPO" := -Wl,--build-id
diff --git a/QDMA/linux-kernel/drv/Makefile b/QDMA/linux-kernel/drv/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..b85c19b3eedb51b4a7d3fe108162674153735944
--- /dev/null
+++ b/QDMA/linux-kernel/drv/Makefile
@@ -0,0 +1,83 @@
+SHELL = /bin/bash
+
+# The top-level makefile defines required variables and flags.
+ifneq ($(shell [[ $(MAKELEVEL) -ge 1 ]] && echo 1),1)
+ $(error Please use the top-level Makefile to build this driver)
+endif
+
+ifneq ($(wildcard /etc/lsb-release),)
+ ifneq ($(shell $(grep) "Ubuntu" /etc/lsb-release),)
+ FLAGS += -DUBUNTU_VERSION_CODE
+ endif
+endif
+
+include $(srcdir)/make/common_flags.mk
+
+$(info srcdir = $(srcdir).)
+$(info KSRC = $(KSRC).)
+$(info VF = $(VF).)
+
+MOD_NAME := qdma$(PFVF_TYPE)
+
+EXTRA_CFLAGS += -DLINUX -D__KERNEL__ -DMODULE -O2 -pipe -Wall -Werror
+EXTRA_CFLAGS += $(FLAGS) $(CPPFLAGS) $(EXTRA_FLAGS)
+EXTRA_CFLAGS += -I$(srcdir)/include
+EXTRA_CFLAGS += -I$(KSRC)/include
+EXTRA_CFLAGS += -I.
+
+# linux >= 3.13 genl_ops become part of the genl_family. And although
+# genl_register_family_with_ops() is still retained until kernel 4.10,
+# its prototype changed from a extern function to a define.
+#
+ifneq ($(shell $(grep) -c 'int genl_register_family_with_ops' $(KINC)/net/genetlink.h),0)
+ ccflags-y += -D__GENL_REG_FAMILY_OPS_FUNC__
+endif
+
+# linux <= 3.13 pci_msix_vec_count is not defined.
+#
+ifeq ($(shell $(grep) -c 'int pci_msix_vec_count' $(KINC)/linux/pci.h),0)
+ ccflags-y += -D__PCI_MSI_VEC_COUNT__
+endif
+
+# linux < 3.13 list_next_entry is not defined.
+#
+ifeq ($(shell $(grep) -c 'list_next_entry' $(KINC)/linux/list.h),0)
+ ccflags-y += -D__LIST_NEXT_ENTRY__
+endif
+
+# linux < 3.18.13 READ_ONCE is not defined.
+#
+ifneq ($(shell $(grep) -c 'READ_ONCE' $(KINC)/linux/compiler.h),0)
+ ccflags-y += -D__READ_ONCE_DEFINED__
+endif
+
+$(info EXTRA_FLAGS = $(EXTRA_FLAGS).)
+$(info ccflags-y = $(ccflags-y).)
+#EXTRA_CFLAGS += -DDEBUG
+
+ifneq ($(modulesymfile),)
+ override symverfile = symverfile="$(topdir)/$(modulesymfile) \
+ -o $(drvdir)/$(modulesymfile)"
+else
+ override symverfile =
+endif
+
+ifneq ($(SUBDIRS),)
+ BASE_OBJS := $(patsubst $(SUBDIRS)/%.c,%.o,$(wildcard $(SUBDIRS)/*.c $(SUBDIRS)/*/*.c))
+endif
+obj-m := $(MOD_NAME).o
+$(MOD_NAME)-objs := $(BASE_OBJS)
+
+.PHONY: build
+build:
+ @mkdir -p -m 755 $(build_dir)
+ @rm -f $(MOD_NAME).ko $(build_dir)/$(MOD_NAME).ko
+ @$(MAKE) symverfile=$(symverfile) KBUILD_EXTRA_SYMBOLS=$(extra_symb) -C $(KOBJ) SUBDIRS=$(shell pwd) modules
+ @cp $(MOD_NAME).ko $(build_dir)
+
+.PHONY: clean
+clean:
+ @-/bin/rm -rf *.ko* ?odule* .tmp_versions *.mod.* *.o *.o.* .*.o.* .*.cmd
+ @-/bin/rm -rf */*.o */*.o.* */.*.o.* */.*.cmd
+ @-/bin/rm -rf $(build_dir)/*.ko
+ @-/bin/rm -f $(srcdir)/drv/libqdma
diff --git a/QDMA/linux-kernel/drv/cdev.c b/QDMA/linux-kernel/drv/cdev.c
new file mode 100644
index 0000000000000000000000000000000000000000..72eb57ba9c47ac17680bd9ff5a0d385c14cc1dcc
--- /dev/null
+++ b/QDMA/linux-kernel/drv/cdev.c
@@ -0,0 +1,800 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "cdev.h"
+
+#include <asm/cacheflush.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/fs.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <linux/aio.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+#include <linux/uio.h>
+#endif
+
+#include "qdma_mod.h"
+
+
+/*
+ * @struct - xlnx_phy_dev
+ * @brief xilinx board device data members
+ */
+struct xlnx_phy_dev {
+ struct list_head list_head; /**< board list */
+ unsigned int major; /**< major number per board */
+ unsigned int device_bus; /**< PCIe device bus number per board */
+};
+
+static LIST_HEAD(xlnx_phy_dev_list);
+static DEFINE_MUTEX(xlnx_phy_dev_mutex);
+
+struct cdev_async_io {
+ ssize_t res2;
+ unsigned long req_count;
+ unsigned long cmpl_count;
+ unsigned long err_cnt;
+ struct qdma_io_cb *qiocb;
+ struct qdma_request **reqv;
+ struct kiocb *iocb;
+ struct work_struct wrk_itm;
+};
+
+enum qdma_cdev_ioctl_cmd {
+ QDMA_CDEV_IOCTL_NO_MEMCPY,
+ QDMA_CDEV_IOCTL_CMDS
+};
+
+struct class *qdma_class;
+static struct kmem_cache *cdev_cache;
+
+static ssize_t cdev_gen_read_write(struct file *file, char __user *buf,
+ size_t count, loff_t *pos, bool write);
+static void unmap_user_buf(struct qdma_io_cb *iocb, bool write);
+static inline void iocb_release(struct qdma_io_cb *iocb);
+
+static inline void xlnx_phy_dev_list_remove(struct xlnx_phy_dev *phy_dev)
+{
+ if (!phy_dev)
+ return;
+
+ mutex_lock(&xlnx_phy_dev_mutex);
+ list_del(&phy_dev->list_head);
+ mutex_unlock(&xlnx_phy_dev_mutex);
+}
+
+static inline void xlnx_phy_dev_list_add(struct xlnx_phy_dev *phy_dev)
+{
+ if (!phy_dev)
+ return;
+
+ mutex_lock(&xlnx_phy_dev_mutex);
+ list_add_tail(&phy_dev->list_head, &xlnx_phy_dev_list);
+ mutex_unlock(&xlnx_phy_dev_mutex);
+}
+
+static int qdma_req_completed(struct qdma_request *req,
+ unsigned int bytes_done, int err)
+{
+ struct qdma_io_cb *qiocb = container_of(req,
+ struct qdma_io_cb,
+ req);
+ struct cdev_async_io *caio = NULL;
+ bool free_caio = false;
+ ssize_t res, res2;
+
+
+ if (qiocb) {
+ caio = (struct cdev_async_io *)qiocb->private;
+ } else {
+ pr_err("Invalid Data structure. Probable memory corruption");
+ return -EINVAL;
+ }
+
+ if (!caio) {
+ pr_err("Invalid Data structure. Probable memory corruption");
+ return -EINVAL;
+ }
+
+ unmap_user_buf(qiocb, req->write);
+ iocb_release(qiocb);
+ caio->res2 |= (err < 0) ? err : 0;
+ if (caio->res2)
+ caio->err_cnt++;
+ caio->cmpl_count++;
+ if (caio->cmpl_count == caio->req_count) {
+ res = caio->cmpl_count - caio->err_cnt;
+ res2 = caio->res2;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
+ caio->iocb->ki_complete(caio->iocb, res, res2);
+#else
+ aio_complete(caio->iocb, res, res2);
+#endif
+ kfree(caio->qiocb);
+ free_caio = true;
+ }
+ if (free_caio)
+ kmem_cache_free(cdev_cache, caio);
+
+ return 0;
+}
+
+/*
+ * character device file operations
+ */
+static int cdev_gen_open(struct inode *inode, struct file *file)
+{
+ struct qdma_cdev *xcdev = container_of(inode->i_cdev, struct qdma_cdev,
+ cdev);
+ file->private_data = xcdev;
+
+ if (xcdev->fp_open_extra)
+ return xcdev->fp_open_extra(xcdev);
+
+ return 0;
+}
+
+static int cdev_gen_close(struct inode *inode, struct file *file)
+{
+ struct qdma_cdev *xcdev = (struct qdma_cdev *)file->private_data;
+
+ if (xcdev && xcdev->fp_close_extra)
+ return xcdev->fp_close_extra(xcdev);
+
+ return 0;
+}
+
+static loff_t cdev_gen_llseek(struct file *file, loff_t off, int whence)
+{
+ struct qdma_cdev *xcdev = (struct qdma_cdev *)file->private_data;
+
+ loff_t newpos = 0;
+
+ switch (whence) {
+ case 0: /* SEEK_SET */
+ newpos = off;
+ break;
+ case 1: /* SEEK_CUR */
+ newpos = file->f_pos + off;
+ break;
+ case 2: /* SEEK_END, @TODO should work from end of address space */
+ newpos = UINT_MAX + off;
+ break;
+ default: /* can't happen */
+ return -EINVAL;
+ }
+ if (newpos < 0)
+ return -EINVAL;
+ file->f_pos = newpos;
+
+ pr_debug("%s: pos=%lld\n", xcdev->name, (signed long long)newpos);
+
+ return newpos;
+}
+
+static long cdev_gen_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ struct qdma_cdev *xcdev = (struct qdma_cdev *)file->private_data;
+
+ switch (cmd) {
+ case QDMA_CDEV_IOCTL_NO_MEMCPY:
+ get_user(xcdev->no_memcpy, (unsigned char *)arg);
+ return 0;
+ default:
+ break;
+ }
+ if (xcdev->fp_ioctl_extra)
+ return xcdev->fp_ioctl_extra(xcdev, cmd, arg);
+
+ pr_info("%s ioctl NOT supported.\n", xcdev->name);
+ return -EINVAL;
+}
+
+/*
+ * cdev r/w
+ */
+static inline void iocb_release(struct qdma_io_cb *iocb)
+{
+ if (iocb->pages)
+ iocb->pages = NULL;
+ kfree(iocb->sgl);
+ iocb->sgl = NULL;
+ iocb->buf = NULL;
+}
+
+static void unmap_user_buf(struct qdma_io_cb *iocb, bool write)
+{
+ int i;
+
+ if (!iocb->pages || !iocb->pages_nr)
+ return;
+
+ for (i = 0; i < iocb->pages_nr; i++) {
+ if (iocb->pages[i]) {
+ if (!write)
+ set_page_dirty_lock(iocb->pages[i]);
+ put_page(iocb->pages[i]);
+ } else
+ break;
+ }
+
+ if (i != iocb->pages_nr)
+ pr_err("sgl pages %d/%u.\n", i, iocb->pages_nr);
+
+ iocb->pages_nr = 0;
+}
+
+static int map_user_buf_to_sgl(struct qdma_io_cb *iocb, bool write)
+{
+ unsigned long len = iocb->len;
+ char *buf = iocb->buf;
+ struct qdma_sw_sg *sg;
+ unsigned int pg_off = offset_in_page(buf);
+ unsigned int pages_nr = (len + pg_off + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ int i;
+ int rv;
+
+ if (len == 0)
+ pages_nr = 1;
+ if (pages_nr == 0)
+ return -EINVAL;
+
+ iocb->pages_nr = 0;
+ sg = kmalloc(pages_nr * (sizeof(struct qdma_sw_sg) +
+ sizeof(struct page *)), GFP_KERNEL);
+ if (!sg) {
+ pr_err("sgl allocation failed for %u pages", pages_nr);
+ return -ENOMEM;
+ }
+ memset(sg, 0, pages_nr * (sizeof(struct qdma_sw_sg) +
+ sizeof(struct page *)));
+ iocb->sgl = sg;
+
+ iocb->pages = (struct page **)(sg + pages_nr);
+ rv = get_user_pages_fast((unsigned long)buf, pages_nr, 1/* write */,
+ iocb->pages);
+ /* No pages were pinned */
+ if (rv < 0) {
+ pr_err("unable to pin down %u user pages, %d.\n",
+ pages_nr, rv);
+ goto err_out;
+ }
+ /* Less pages pinned than wanted */
+ if (rv != pages_nr) {
+ pr_err("unable to pin down all %u user pages, %d.\n",
+ pages_nr, rv);
+ iocb->pages_nr = rv;
+ rv = -EFAULT;
+ goto err_out;
+ }
+
+ for (i = 1; i < pages_nr; i++) {
+ if (iocb->pages[i - 1] == iocb->pages[i]) {
+ pr_err("duplicate pages, %d, %d.\n",
+ i - 1, i);
+ iocb->pages_nr = pages_nr;
+ rv = -EFAULT;
+ goto err_out;
+ }
+ }
+
+ sg = iocb->sgl;
+ for (i = 0; i < pages_nr; i++, sg++) {
+ unsigned int offset = offset_in_page(buf);
+ unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset,
+ len);
+ struct page *pg = iocb->pages[i];
+
+ flush_dcache_page(pg);
+
+ sg->next = sg + 1;
+ sg->pg = pg;
+ sg->offset = offset;
+ sg->len = nbytes;
+ sg->dma_addr = 0UL;
+
+ buf += nbytes;
+ len -= nbytes;
+ }
+
+ iocb->sgl[pages_nr - 1].next = NULL;
+ iocb->pages_nr = pages_nr;
+ return 0;
+
+err_out:
+ unmap_user_buf(iocb, write);
+ iocb_release(iocb);
+
+ return rv;
+}
+
+static ssize_t cdev_gen_read_write(struct file *file, char __user *buf,
+ size_t count, loff_t *pos, bool write)
+{
+ struct qdma_cdev *xcdev = (struct qdma_cdev *)file->private_data;
+ struct qdma_io_cb iocb;
+ struct qdma_request *req = &iocb.req;
+ ssize_t res = 0;
+ int rv;
+ unsigned long qhndl;
+
+ if (!xcdev) {
+ pr_info("file 0x%p, xcdev NULL, 0x%p,%llu, pos %llu, W %d.\n",
+ file, buf, (u64)count, (u64)*pos, write);
+ return -EINVAL;
+ }
+
+ if (!xcdev->fp_rw) {
+ pr_info("file 0x%p, %s, NO rw, 0x%p,%llu, pos %llu, W %d.\n",
+ file, xcdev->name, buf, (u64)count, (u64)*pos, write);
+ return -EINVAL;
+ }
+
+ qhndl = write ? xcdev->h2c_qhndl : xcdev->c2h_qhndl;
+
+ pr_debug("%s, priv 0x%lx: buf 0x%p,%llu, pos %llu, W %d.\n",
+ xcdev->name, qhndl, buf, (u64)count, (u64)*pos,
+ write);
+
+ memset(&iocb, 0, sizeof(struct qdma_io_cb));
+ iocb.buf = buf;
+ iocb.len = count;
+ rv = map_user_buf_to_sgl(&iocb, write);
+ if (rv < 0)
+ return rv;
+
+ req->sgcnt = iocb.pages_nr;
+ req->sgl = iocb.sgl;
+ req->write = write ? 1 : 0;
+ req->dma_mapped = 0;
+ req->udd_len = 0;
+ req->ep_addr = (u64)*pos;
+ req->count = count;
+ req->timeout_ms = 10 * 1000; /* 10 seconds */
+ req->fp_done = NULL; /* blocking */
+ req->h2c_eot = 1; /* set to 1 for STM tests */
+
+ res = xcdev->fp_rw(xcdev->xcb->xpdev->dev_hndl, qhndl, req);
+
+ unmap_user_buf(&iocb, write);
+ iocb_release(&iocb);
+
+ return res;
+}
+
+static ssize_t cdev_gen_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *pos)
+{
+ return cdev_gen_read_write(file, (char *)buf, count, pos, 1);
+}
+
+static ssize_t cdev_gen_read(struct file *file, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ return cdev_gen_read_write(file, (char *)buf, count, pos, 0);
+}
+
+static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io,
+ unsigned long count, loff_t pos)
+{
+ struct qdma_cdev *xcdev =
+ (struct qdma_cdev *)iocb->ki_filp->private_data;
+ struct cdev_async_io *caio;
+ int rv = 0;
+ unsigned long i;
+ unsigned long qhndl;
+
+ if (!xcdev) {
+ pr_err("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
+ iocb->ki_filp, (u64)count, (u64)pos, 1);
+ return -EINVAL;
+ }
+
+ if (!xcdev->fp_rw) {
+ pr_err("No Read write handler assigned\n");
+ return -EINVAL;
+ }
+
+ caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
+ if (!caio) {
+ pr_info("OOM");
+ return -ENOMEM;
+ }
+ memset(caio, 0, sizeof(struct cdev_async_io));
+ caio->qiocb = kzalloc(count * (sizeof(struct qdma_io_cb) +
+ sizeof(struct qdma_request *)), GFP_KERNEL);
+ if (!caio->qiocb) {
+ pr_err("failed to allocate qiocb");
+ return -ENOMEM;
+ }
+ caio->reqv = (struct qdma_request **)(caio->qiocb + count);
+ for (i = 0; i < count; i++) {
+ caio->qiocb[i].private = caio;
+ caio->reqv[i] = &(caio->qiocb[i].req);
+ caio->qiocb[i].buf = io[i].iov_base;
+ caio->qiocb[i].len = io[i].iov_len;
+ rv = map_user_buf_to_sgl(&(caio->qiocb[i]), true);
+ if (rv < 0) {
+ break;
+ }
+
+ caio->reqv[i]->write = 1;
+ caio->reqv[i]->sgcnt = caio->qiocb[i].pages_nr;
+ caio->reqv[i]->sgl = caio->qiocb[i].sgl;
+ caio->reqv[i]->dma_mapped = false;
+ caio->reqv[i]->udd_len = 0;
+ caio->reqv[i]->ep_addr = (u64)pos;
+ caio->reqv[i]->no_memcpy = xcdev->no_memcpy ? 1 : 0;
+ caio->reqv[i]->count = io->iov_len;
+ caio->reqv[i]->timeout_ms = 10 * 1000; /* 10 seconds */
+ caio->reqv[i]->fp_done = qdma_req_completed;
+
+ }
+ if (i > 0) {
+ iocb->private = caio;
+ caio->iocb = iocb;
+ caio->req_count = i;
+ qhndl = xcdev->h2c_qhndl;
+ rv = xcdev->fp_aiorw(xcdev->xcb->xpdev->dev_hndl, qhndl,
+ caio->req_count, caio->reqv);
+ if (rv >= 0)
+ rv = -EIOCBQUEUED;
+ } else {
+ pr_err("failed with %d for %lu reqs", rv, caio->req_count);
+ kfree(caio->qiocb);
+ kmem_cache_free(cdev_cache, caio);
+ }
+
+ return rv;
+}
+
+static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io,
+ unsigned long count, loff_t pos)
+{
+ struct qdma_cdev *xcdev =
+ (struct qdma_cdev *)iocb->ki_filp->private_data;
+ struct cdev_async_io *caio;
+ int rv = 0;
+ unsigned long i;
+ unsigned long qhndl;
+
+ if (!xcdev) {
+ pr_err("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
+ iocb->ki_filp, (u64)count, (u64)pos, 1);
+ return -EINVAL;
+ }
+
+ if (!xcdev->fp_rw) {
+ pr_err("No Read write handler assigned\n");
+ return -EINVAL;
+ }
+
+ caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
+ if (!caio) {
+ pr_err("failed to allocate qiocb");
+ return -ENOMEM;
+ }
+ memset(caio, 0, sizeof(struct cdev_async_io));
+ caio->qiocb = kzalloc (count * (sizeof(struct qdma_io_cb) +
+ sizeof(struct qdma_request *)), GFP_KERNEL);
+ if (!caio->qiocb) {
+ pr_err("failed to allocate qiocb");
+ return -ENOMEM;
+ }
+ caio->reqv = (struct qdma_request **)(caio->qiocb + count);
+ for (i = 0; i < count; i++) {
+ caio->qiocb[i].private = caio;
+ caio->reqv[i] = &(caio->qiocb[i].req);
+ caio->qiocb[i].buf = io[i].iov_base;
+ caio->qiocb[i].len = io[i].iov_len;
+ rv = map_user_buf_to_sgl(&(caio->qiocb[i]), false);
+ if (rv < 0) {
+ break;
+ }
+
+ caio->reqv[i]->write = 0;
+ caio->reqv[i]->sgcnt = caio->qiocb[i].pages_nr;
+ caio->reqv[i]->sgl = caio->qiocb[i].sgl;
+ caio->reqv[i]->dma_mapped = false;
+ caio->reqv[i]->udd_len = 0;
+ caio->reqv[i]->ep_addr = (u64)pos;
+ caio->reqv[i]->no_memcpy = xcdev->no_memcpy ? 1 : 0;
+ caio->reqv[i]->count = io->iov_len;
+ caio->reqv[i]->timeout_ms = 10 * 1000; /* 10 seconds */
+ caio->reqv[i]->fp_done = qdma_req_completed;
+ }
+ if (i > 0) {
+ iocb->private = caio;
+ caio->iocb = iocb;
+ caio->req_count = i;
+ qhndl = xcdev->c2h_qhndl;
+ rv = xcdev->fp_aiorw(xcdev->xcb->xpdev->dev_hndl, qhndl,
+ caio->req_count, caio->reqv);
+ if (rv >= 0)
+ rv = -EIOCBQUEUED;
+ } else {
+ pr_err("failed with %d for %lu reqs", rv, caio->req_count);
+ kfree(caio->qiocb);
+ kmem_cache_free(cdev_cache, caio);
+ }
+
+ return rv;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+static ssize_t cdev_write_iter(struct kiocb *iocb, struct iov_iter *io)
+{
+ return cdev_aio_write(iocb, io->iov, io->nr_segs, iocb->ki_pos);
+}
+
+static ssize_t cdev_read_iter(struct kiocb *iocb, struct iov_iter *io)
+{
+ return cdev_aio_read(iocb, io->iov, io->nr_segs, iocb->ki_pos);
+}
+#endif
+
+static const struct file_operations cdev_gen_fops = {
+ .owner = THIS_MODULE,
+ .open = cdev_gen_open,
+ .release = cdev_gen_close,
+ .write = cdev_gen_write,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+ .write_iter = cdev_write_iter,
+#else
+ .aio_write = cdev_aio_write,
+#endif
+ .read = cdev_gen_read,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+ .read_iter = cdev_read_iter,
+#else
+ .aio_read = cdev_aio_read,
+#endif
+ .unlocked_ioctl = cdev_gen_ioctl,
+ .llseek = cdev_gen_llseek,
+};
+
+/*
+ * xcb: per pci device character device control info.
+ * xcdev: per queue character device
+ */
+int qdma_cdev_display(void *p, char *buf)
+{
+ struct qdma_cdev *xcdev = (struct qdma_cdev *)p;
+
+ return sprintf(buf, ", cdev %s", xcdev->name);
+}
+
+void qdma_cdev_destroy(struct qdma_cdev *xcdev)
+{
+ pr_debug("destroying cdev %p", xcdev);
+ if (!xcdev) {
+ pr_info("xcdev NULL.\n");
+ return;
+ }
+
+ if (xcdev->sys_device)
+ device_destroy(qdma_class, xcdev->cdevno);
+
+ cdev_del(&xcdev->cdev);
+
+ kfree(xcdev);
+}
+
+int qdma_cdev_create(struct qdma_cdev_cb *xcb, struct pci_dev *pdev,
+ struct qdma_queue_conf *qconf, unsigned int minor,
+ unsigned long qhndl, struct qdma_cdev **xcdev_pp,
+ char *ebuf, int ebuflen)
+{
+ struct qdma_cdev *xcdev;
+ int rv;
+ unsigned long *priv_data;
+
+ xcdev = kzalloc(sizeof(struct qdma_cdev) + strlen(qconf->name) + 1,
+ GFP_KERNEL);
+ if (!xcdev) {
+ pr_info("%s OOM %lu.\n", qconf->name, sizeof(struct qdma_cdev));
+ if (ebuf && ebuflen) {
+ rv = sprintf(ebuf, "%s cdev OOM %lu.\n",
+ qconf->name, sizeof(struct qdma_cdev));
+ ebuf[rv] = '\0';
+
+ }
+ return -ENOMEM;
+ }
+
+ xcdev->cdev.owner = THIS_MODULE;
+ xcdev->xcb = xcb;
+ priv_data = qconf->c2h ? &xcdev->c2h_qhndl : &xcdev->h2c_qhndl;
+ *priv_data = qhndl;
+ xcdev->dir_init = (1 << qconf->c2h);
+ strcpy(xcdev->name, qconf->name);
+
+ xcdev->minor = minor;
+ if (xcdev->minor >= xcb->cdev_minor_cnt) {
+ pr_info("%s: no char dev. left.\n", qconf->name);
+ if (ebuf && ebuflen) {
+ rv = sprintf(ebuf, "%s cdev no cdev left.\n",
+ qconf->name);
+ ebuf[rv] = '\0';
+ }
+ rv = -ENOSPC;
+ goto err_out;
+ }
+ xcdev->cdevno = MKDEV(xcb->cdev_major, xcdev->minor);
+
+ cdev_init(&xcdev->cdev, &cdev_gen_fops);
+
+ /* bring character device live */
+ rv = cdev_add(&xcdev->cdev, xcdev->cdevno, 1);
+ if (rv < 0) {
+ pr_info("cdev_add failed %d, %s.\n", rv, xcdev->name);
+ if (ebuf && ebuflen) {
+ int l = sprintf(ebuf, "%s cdev add failed %d.\n",
+ qconf->name, rv);
+ ebuf[l] = '\0';
+ }
+ goto err_out;
+ }
+
+ /* create device on our class */
+ if (qdma_class) {
+ xcdev->sys_device = device_create(qdma_class, &(pdev->dev),
+ xcdev->cdevno, NULL, "%s", xcdev->name);
+ if (IS_ERR(xcdev->sys_device)) {
+ rv = PTR_ERR(xcdev->sys_device);
+ pr_info("%s device_create failed %d.\n",
+ xcdev->name, rv);
+ if (ebuf && ebuflen) {
+ int l = sprintf(ebuf,
+ "%s device_create failed %d.\n",
+ qconf->name, rv);
+ ebuf[l] = '\0';
+ }
+ goto del_cdev;
+ }
+ }
+
+ xcdev->fp_rw = qdma_request_submit;
+ xcdev->fp_aiorw = qdma_batch_request_submit;
+
+ *xcdev_pp = xcdev;
+ return 0;
+
+del_cdev:
+ cdev_del(&xcdev->cdev);
+
+err_out:
+ kfree(xcdev);
+ return rv;
+}
+
+/*
+ * per device initialization & cleanup
+ */
+void qdma_cdev_device_cleanup(struct qdma_cdev_cb *xcb)
+{
+ if (!xcb->cdev_major)
+ return;
+
+ xcb->cdev_major = 0;
+}
+
+int qdma_cdev_device_init(struct qdma_cdev_cb *xcb)
+{
+ dev_t dev;
+ int rv;
+ struct xlnx_phy_dev *phy_dev, *tmp, *new_phy_dev;
+
+ spin_lock_init(&xcb->lock);
+
+ xcb->cdev_minor_cnt = QDMA_MINOR_MAX;
+
+ if (xcb->cdev_major) {
+ pr_warn("major %d already exist.\n", xcb->cdev_major);
+ return -EINVAL;
+ }
+
+ /* Check if same bus id device is added in global list
+ * If found then assign same major number */
+ mutex_lock(&xlnx_phy_dev_mutex);
+ list_for_each_entry_safe(phy_dev, tmp, &xlnx_phy_dev_list, list_head) {
+ if (phy_dev->device_bus == xcb->xpdev->pdev->bus->number) {
+ xcb->cdev_major = phy_dev->major;
+ mutex_unlock(&xlnx_phy_dev_mutex);
+ return 0;
+ }
+ }
+ mutex_unlock(&xlnx_phy_dev_mutex);
+
+ /* allocate a dynamically allocated char device node */
+ rv = alloc_chrdev_region(&dev, 0, xcb->cdev_minor_cnt,
+ QDMA_CDEV_CLASS_NAME);
+ if (rv) {
+ pr_err("unable to allocate cdev region %d.\n", rv);
+ return rv;
+ }
+ xcb->cdev_major = MAJOR(dev);
+
+ new_phy_dev = kzalloc(sizeof(struct xlnx_phy_dev), GFP_KERNEL);
+ if (!new_phy_dev) {
+ pr_err("unable to allocate xlnx_dev.\n");
+ return-ENOMEM;
+ }
+ new_phy_dev->major = xcb->cdev_major;
+ new_phy_dev->device_bus = xcb->xpdev->pdev->bus->number;
+ xlnx_phy_dev_list_add(new_phy_dev);
+
+ return 0;
+}
+
+/*
+ * driver-wide Initialization & cleanup
+ */
+
+int qdma_cdev_init(void)
+{
+ qdma_class = class_create(THIS_MODULE, QDMA_CDEV_CLASS_NAME);
+ if (IS_ERR(qdma_class)) {
+ pr_info("%s: failed to create class 0x%lx.",
+ QDMA_CDEV_CLASS_NAME, (unsigned long)qdma_class);
+ qdma_class = NULL;
+ return -1;
+ }
+ /* using kmem_cache_create to enable sequential cleanup */
+ cdev_cache = kmem_cache_create("cdev_cache",
+ sizeof(struct cdev_async_io),
+ 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!cdev_cache) {
+ pr_info("memory allocation for cdev_cache failed. OOM\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void qdma_cdev_cleanup(void)
+{
+ struct xlnx_phy_dev *phy_dev, *tmp;
+
+ list_for_each_entry_safe(phy_dev, tmp, &xlnx_phy_dev_list, list_head) {
+ unregister_chrdev_region(MKDEV(phy_dev->major, 0),
+ QDMA_MINOR_MAX);
+ xlnx_phy_dev_list_remove(phy_dev);
+ kfree(phy_dev);
+ }
+
+ kmem_cache_destroy(cdev_cache);
+ if (qdma_class)
+ class_destroy(qdma_class);
+
+}
diff --git a/QDMA/linux-kernel/drv/cdev.h b/QDMA/linux-kernel/drv/cdev.h
new file mode 100644
index 0000000000000000000000000000000000000000..822e14dda9c6812fee6026c83a088b12eba9388f
--- /dev/null
+++ b/QDMA/linux-kernel/drv/cdev.h
@@ -0,0 +1,182 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_CDEV_H__
+#define __QDMA_CDEV_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma pcie kernel module
+ *
+ */
+#include <linux/cdev.h>
+#include "version.h"
+#include <linux/spinlock_types.h>
+
+#include "libqdma/libqdma_export.h"
+#include <linux/workqueue.h>
+
+/** QDMA character device class name */
+#define QDMA_CDEV_CLASS_NAME DRV_MODULE_NAME
+/** QDMA character device max minor number*/
+#define QDMA_MINOR_MAX (2048)
+
+/* per pci device control */
+/**
+ * @struct - qdma_cdev_cb
+ * @brief QDMA character device call back data
+ */
+struct qdma_cdev_cb {
+ /** pointer to xilinx pcie device */
+ struct xlnx_pci_dev *xpdev;
+ /** character device lock */
+ spinlock_t lock;
+ /** character device major number */
+ int cdev_major;
+ /** character device minor number count */
+ int cdev_minor_cnt;
+};
+
+/**
+ * @struct - qdma_cdev
+ * @brief QDMA character device book keeping parameters
+ */
+struct qdma_cdev {
+ /** lsit of qdma character devices */
+ struct list_head list_head;
+ /** minor number */
+ int minor;
+ /** character device number */
+ dev_t cdevno;
+ /** pointer to qdma character device call back data */
+ struct qdma_cdev_cb *xcb;
+ /** pointer to kernel device(struct device) */
+ struct device *sys_device;
+ /** pointer to kernel cdev(struct cdev) */
+ struct cdev cdev;
+ /** c2h queue handle */
+ unsigned long c2h_qhndl;
+ /** hec queue handle */
+ unsigned long h2c_qhndl;
+ /** direction */
+ unsigned short dir_init;
+ /* flag to indicate if memcpy is required */
+ unsigned char no_memcpy;
+ /** call back function for open a device */
+ int (*fp_open_extra)(struct qdma_cdev *);
+ /** call back function for close a device */
+ int (*fp_close_extra)(struct qdma_cdev *);
+ /** call back function to handle ioctl message */
+ long (*fp_ioctl_extra)(struct qdma_cdev *, unsigned int, unsigned long);
+ /** call back function to handle read write request*/
+ ssize_t (*fp_rw)(unsigned long dev_hndl, unsigned long qhndl,
+ struct qdma_request *);
+ ssize_t (*fp_aiorw)(unsigned long dev_hndl, unsigned long qhndl,
+ unsigned long count, struct qdma_request **);
+ /** name of the character device*/
+ char name[0];
+};
+
+/**
+ * @struct - qdma_io_cb
+ * @brief QDMA character device io call back book keeping parameters
+ */
+struct qdma_io_cb {
+ void *private;
+ /** user buffer */
+ void __user *buf;
+ /** length of the user buffer */
+ size_t len;
+ /** page number */
+ unsigned int pages_nr;
+ /** scatter gather list */
+ struct qdma_sw_sg *sgl;
+ /** pages allocated to accommodate the scatter gather list */
+ struct page **pages;
+ /** qdma request */
+ struct qdma_request req;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_destroy() - handler to destroy the character device
+ *
+ * @param[in] xcdev: pointer to character device
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_cdev_destroy(struct qdma_cdev *xcdev);
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_create() - handler to create a character device
+ *
+ * @param[in] xcb: pointer to qdma character device call back data
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] qconf: queue configurations
+ * @param[in] minor: character device minor number
+ * @param[in] ebuflen: buffer length
+ * @param[in] qhndl: queue handle
+ * @param[out] xcdev_pp: pointer to struct qdma_cdev
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_cdev_create(struct qdma_cdev_cb *xcb, struct pci_dev *pdev,
+ struct qdma_queue_conf *qconf, unsigned int minor,
+ unsigned long qhndl, struct qdma_cdev **xcdev_pp,
+ char *ebuf, int ebuflen);
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_device_cleanup() - handler to clean up a character device
+ *
+ * @param[in] xcb: pointer to qdma character device call back data
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_cdev_device_cleanup(struct qdma_cdev_cb *xcb);
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_device_init() - handler to initialize a character device
+ *
+ * @param[in] xcb: pointer to qdma character device call back data
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_cdev_device_init(struct qdma_cdev_cb *xcb);
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_cleanup() - character device cleanup handler
+ *
+ *****************************************************************************/
+void qdma_cdev_cleanup(void);
+
+/*****************************************************************************/
+/**
+ * qdma_cdev_init() - character device initialization handler
+ *
+ *****************************************************************************/
+int qdma_cdev_init(void);
+
+#endif /* ifndef __QDMA_CDEV_H__ */
diff --git a/QDMA/linux-kernel/drv/nl.c b/QDMA/linux-kernel/drv/nl.c
new file mode 100644
index 0000000000000000000000000000000000000000..5e201d8d358565ef4194af78ff7e01561e0f7087
--- /dev/null
+++ b/QDMA/linux-kernel/drv/nl.c
@@ -0,0 +1,1862 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <net/genetlink.h>
+
+#include "libqdma/libqdma_export.h"
+#include "qdma_mod.h"
+#include "qdma_nl.h"
+#include "nl.h"
+#include "version.h"
+
+static int xnl_dev_list(struct sk_buff *skb_2, struct genl_info *info);
+
+static struct nla_policy xnl_policy[XNL_ATTR_MAX] = {
+ [XNL_ATTR_GENMSG] = { .type = NLA_NUL_STRING },
+
+ [XNL_ATTR_DRV_INFO] = { .type = NLA_NUL_STRING },
+
+ [XNL_ATTR_DEV_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_PCI_BUS] = { .type = NLA_U32 },
+ [XNL_ATTR_PCI_DEV] = { .type = NLA_U32 },
+ [XNL_ATTR_PCI_FUNC] = { .type = NLA_U32 },
+ [XNL_ATTR_DEV_CFG_BAR] = { .type = NLA_U32 },
+ [XNL_ATTR_DEV_USR_BAR] = { .type = NLA_U32 },
+ [XNL_ATTR_DEV_QSET_MAX] = { .type = NLA_U32 },
+ [XNL_ATTR_DEV_QSET_QBASE] = { .type = NLA_U32 },
+
+ [XNL_ATTR_REG_BAR_NUM] = { .type = NLA_U32 },
+ [XNL_ATTR_REG_ADDR] = { .type = NLA_U32 },
+ [XNL_ATTR_REG_VAL] = { .type = NLA_U32 },
+
+ [XNL_ATTR_QIDX] = { .type = NLA_U32 },
+ [XNL_ATTR_NUM_Q] = { .type = NLA_U32 },
+ [XNL_ATTR_QFLAG] = { .type = NLA_U32 },
+ [XNL_ATTR_CMPT_DESC_SIZE] = { .type = NLA_U32 },
+ [XNL_ATTR_SW_DESC_SIZE] = { .type = NLA_U32 },
+ [XNL_ATTR_QRNGSZ_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_C2H_BUFSZ_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_CMPT_TIMER_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_CMPT_CNTR_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_CMPT_TRIG_MODE] = { .type = NLA_U32 },
+ [XNL_ATTR_RANGE_START] = { .type = NLA_U32 },
+ [XNL_ATTR_RANGE_END] = { .type = NLA_U32 },
+
+ [XNL_ATTR_INTR_VECTOR_IDX] = { .type = NLA_U32 },
+ [XNL_ATTR_PIPE_GL_MAX] = { .type = NLA_U32 },
+ [XNL_ATTR_PIPE_FLOW_ID] = { .type = NLA_U32 },
+ [XNL_ATTR_PIPE_SLR_ID] = { .type = NLA_U32 },
+ [XNL_ATTR_PIPE_TDEST] = { .type = NLA_U32 },
+
+ [XNL_ATTR_DEV_STM_BAR] = { .type = NLA_U32 },
+#ifdef ERR_DEBUG
+ [XNL_ATTR_QPARAM_ERR_INFO] = { .type = NLA_U32 },
+#endif
+};
+
+static int xnl_dev_list(struct sk_buff *, struct genl_info *);
+static int xnl_dev_info(struct sk_buff *, struct genl_info *);
+static int xnl_dev_version(struct sk_buff *skb2, struct genl_info *info);
+static int xnl_dev_stat(struct sk_buff *, struct genl_info *);
+static int xnl_dev_stat_clear(struct sk_buff *, struct genl_info *);
+static int xnl_q_list(struct sk_buff *, struct genl_info *);
+static int xnl_q_add(struct sk_buff *, struct genl_info *);
+static int xnl_q_start(struct sk_buff *, struct genl_info *);
+static int xnl_q_stop(struct sk_buff *, struct genl_info *);
+static int xnl_q_del(struct sk_buff *, struct genl_info *);
+static int xnl_q_dump(struct sk_buff *, struct genl_info *);
+static int xnl_q_dump_desc(struct sk_buff *, struct genl_info *);
+static int xnl_q_dump_cmpt(struct sk_buff *, struct genl_info *);
+static int xnl_q_read_pkt(struct sk_buff *, struct genl_info *);
+static int xnl_q_read_udd(struct sk_buff *, struct genl_info *);
+static int xnl_intr_ring_dump(struct sk_buff *, struct genl_info *);
+static int xnl_register_read(struct sk_buff *, struct genl_info *);
+static int xnl_register_write(struct sk_buff *, struct genl_info *);
+static int xnl_get_global_csr(struct sk_buff *skb2, struct genl_info *info);
+
+#ifdef ERR_DEBUG
+static int xnl_err_induce(struct sk_buff *skb2, struct genl_info *info);
+#endif
+
+struct genl_ops xnl_ops[] = {
+ {
+ .cmd = XNL_CMD_DEV_LIST,
+ .policy = xnl_policy,
+ .doit = xnl_dev_list,
+ },
+ {
+ .cmd = XNL_CMD_VERSION,
+ .policy = xnl_policy,
+ .doit = xnl_dev_version,
+ },
+ {
+ .cmd = XNL_CMD_DEV_INFO,
+ .policy = xnl_policy,
+ .doit = xnl_dev_info,
+ },
+ {
+ .cmd = XNL_CMD_DEV_STAT,
+ .policy = xnl_policy,
+ .doit = xnl_dev_stat,
+ },
+ {
+ .cmd = XNL_CMD_DEV_STAT_CLEAR,
+ .policy = xnl_policy,
+ .doit = xnl_dev_stat_clear,
+ },
+ {
+ .cmd = XNL_CMD_Q_LIST,
+ .policy = xnl_policy,
+ .doit = xnl_q_list,
+ },
+ {
+ .cmd = XNL_CMD_Q_ADD,
+ .policy = xnl_policy,
+ .doit = xnl_q_add,
+ },
+ {
+ .cmd = XNL_CMD_Q_START,
+ .policy = xnl_policy,
+ .doit = xnl_q_start,
+ },
+ {
+ .cmd = XNL_CMD_Q_STOP,
+ .policy = xnl_policy,
+ .doit = xnl_q_stop,
+ },
+ {
+ .cmd = XNL_CMD_Q_DEL,
+ .policy = xnl_policy,
+ .doit = xnl_q_del,
+ },
+ {
+ .cmd = XNL_CMD_Q_DUMP,
+ .policy = xnl_policy,
+ .doit = xnl_q_dump,
+ },
+ {
+ .cmd = XNL_CMD_Q_DESC,
+ .policy = xnl_policy,
+ .doit = xnl_q_dump_desc,
+ },
+ {
+ .cmd = XNL_CMD_Q_CMPT,
+ .policy = xnl_policy,
+ .doit = xnl_q_dump_cmpt,
+ },
+ {
+ .cmd = XNL_CMD_Q_UDD,
+ .policy = xnl_policy,
+ .doit = xnl_q_read_udd,
+ },
+ {
+ .cmd = XNL_CMD_Q_RX_PKT,
+ .policy = xnl_policy,
+ .doit = xnl_q_read_pkt,
+ },
+ {
+ .cmd = XNL_CMD_INTR_RING_DUMP,
+ .policy = xnl_policy,
+ .doit = xnl_intr_ring_dump,
+ },
+ {
+ .cmd = XNL_CMD_REG_RD,
+ .policy = xnl_policy,
+ .doit = xnl_register_read,
+ },
+ {
+ .cmd = XNL_CMD_REG_WRT,
+ .policy = xnl_policy,
+ .doit = xnl_register_write,
+ },
+ {
+ .cmd = XNL_CMD_GLOBAL_CSR,
+ .policy = xnl_policy,
+ .doit = xnl_get_global_csr,
+ },
+#ifdef ERR_DEBUG
+ {
+ .cmd = XNL_CMD_Q_ERR_INDUCE,
+ .policy = xnl_policy,
+ .doit = xnl_err_induce,
+ }
+#endif
+};
+
+static struct genl_family xnl_family = {
+#ifdef GENL_ID_GENERATE
+ .id = GENL_ID_GENERATE,
+#endif
+ .hdrsize = 0,
+#ifdef __QDMA_VF__
+ .name = XNL_NAME_VF,
+#else
+ .name = XNL_NAME_PF,
+#endif
+#ifndef __GENL_REG_FAMILY_OPS_FUNC__
+ .ops = xnl_ops,
+ .n_ops = ARRAY_SIZE(xnl_ops),
+#endif
+ .maxattr = XNL_ATTR_MAX - 1,
+};
+
+static struct sk_buff *xnl_msg_alloc(enum xnl_op_t op, int min_sz,
+ void **hdr, struct genl_info *info)
+{
+ struct sk_buff *skb;
+ void *p;
+ unsigned long sz = min_sz < NLMSG_GOODSIZE ? NLMSG_GOODSIZE : min_sz;
+
+ skb = genlmsg_new(sz, GFP_KERNEL);
+ if (!skb) {
+ pr_info("OOM %lu.\n", sz);
+ return NULL;
+ }
+
+ p = genlmsg_put(skb, 0, info->snd_seq + 1, &xnl_family, 0, op);
+ if (!p) {
+ pr_info("skb too small.\n");
+ nlmsg_free(skb);
+ return NULL;
+ }
+
+ *hdr = p;
+ return skb;
+}
+
+static inline int xnl_msg_add_attr_str(struct sk_buff *skb,
+ enum xnl_attr_t type, char *s)
+{
+ int rv;
+
+ rv = nla_put_string(skb, type, s);
+ if (rv != 0) {
+ pr_info("nla_put_string return %d.\n", rv);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static inline int xnl_msg_add_attr_data(struct sk_buff *skb,
+ enum xnl_attr_t type, void *s, unsigned int len)
+{
+ int rv;
+
+ rv = nla_put(skb, type, len, s);
+ if (rv != 0) {
+ pr_info("nla_put return %d.\n", rv);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+
+static inline int xnl_msg_add_attr_uint(struct sk_buff *skb,
+ enum xnl_attr_t type, u32 v)
+{
+ int rv;
+
+ rv = nla_put_u32(skb, type, v);
+ if (rv != 0) {
+ pr_info("nla add dev_idx failed %d.\n", rv);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static inline int xnl_msg_send(struct sk_buff *skb_tx, void *hdr,
+ struct genl_info *info)
+{
+ int rv;
+
+ genlmsg_end(skb_tx, hdr);
+
+ rv = genlmsg_unicast(genl_info_net(info), skb_tx, info->snd_portid);
+ if (rv)
+ pr_info("send portid %d failed %d.\n", info->snd_portid, rv);
+
+ return 0;
+}
+
+#ifdef DEBUG
+static int xnl_dump_attrs(struct genl_info *info)
+{
+ int i;
+
+ pr_info("snd_seq 0x%x, portid 0x%x.\n",
+ info->snd_seq, info->snd_portid);
+#if 0
+ print_hex_dump_bytes("nlhdr", DUMP_PREFIX_OFFSET, info->nlhdr,
+ sizeof(struct nlmsghdr));
+ pr_info("\n");
+ print_hex_dump_bytes("genlhdr", DUMP_PREFIX_OFFSET, info->genlhdr,
+ sizeof(struct genlmsghdr));
+ pr_info("\n");
+#endif
+
+ pr_info("nlhdr: len %u, type %u, flags 0x%x, seq 0x%x, pid %u.\n",
+ info->nlhdr->nlmsg_len,
+ info->nlhdr->nlmsg_type,
+ info->nlhdr->nlmsg_flags,
+ info->nlhdr->nlmsg_seq,
+ info->nlhdr->nlmsg_pid);
+ pr_info("genlhdr: cmd 0x%x %s, version %u, reserved 0x%x.\n",
+ info->genlhdr->cmd, xnl_op_str[info->genlhdr->cmd],
+ info->genlhdr->version,
+ info->genlhdr->reserved);
+
+ for (i = 0; i < XNL_ATTR_MAX; i++) {
+ struct nlattr *na = info->attrs[i];
+
+ if (na) {
+ if (xnl_policy[i].type == NLA_NUL_STRING) {
+ char *s = (char *)nla_data(na);
+
+ if (s)
+ pr_info("attr %d, %s, str %s.\n",
+ i, xnl_attr_str[i], s);
+ else
+ pr_info("attr %d, %s, str NULL.\n",
+ i, xnl_attr_str[i]);
+
+ } else {
+ u32 v = nla_get_u32(na);
+
+ pr_info("attr %s, u32 0x%x.\n",
+ xnl_attr_str[i], v);
+ }
+ }
+ }
+
+ return 0;
+}
+#else
+#define xnl_dump_attrs(info)
+#endif
+
+int xnl_respond_buffer(struct genl_info *info, char *buf, int buflen)
+{
+ struct sk_buff *skb;
+ void *hdr;
+ int rv;
+
+ skb = xnl_msg_alloc(info->genlhdr->cmd, buflen, &hdr, info);
+ if (!skb)
+ return -ENOMEM;
+
+ rv = xnl_msg_add_attr_str(skb, XNL_ATTR_GENMSG, buf);
+ if (rv != 0) {
+ pr_err("xnl_msg_add_attr_str() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+
+ rv = xnl_msg_send(skb, hdr, info);
+
+ return rv;
+}
+
+int xnl_respond_data(struct genl_info *info, void *buf, int buflen)
+{
+ struct sk_buff *skb;
+ void *hdr;
+ int rv;
+
+ skb = xnl_msg_alloc(info->genlhdr->cmd, buflen, &hdr, info);
+ if (!skb)
+ return -ENOMEM;
+
+ rv = xnl_msg_add_attr_data(skb, XNL_ATTR_GLOBAL_CSR, buf, buflen);
+ if (rv != 0) {
+ pr_err("xnl_msg_add_attr_data() failed: %d", rv);
+ return rv;
+ }
+
+ rv = xnl_msg_send(skb, hdr, info);
+ return rv;
+}
+
+static char *xnl_mem_alloc(int l, struct genl_info *info)
+{
+ char ebuf[XNL_ERR_BUFLEN];
+ char *buf = kmalloc(l, GFP_KERNEL);
+ int rv;
+
+ if (buf) {
+ memset(buf, 0, l);
+ return buf;
+ }
+
+ pr_info("xnl OOM %d.\n", l);
+
+ rv = snprintf(ebuf, XNL_ERR_BUFLEN, "ERR! xnl OOM %d.\n", l);
+ ebuf[rv] = '\0';
+
+ xnl_respond_buffer(info, ebuf, XNL_ERR_BUFLEN);
+
+ return NULL;
+}
+
+static struct xlnx_pci_dev *xnl_rcv_check_xpdev(struct genl_info *info)
+{
+ u32 idx;
+ struct xlnx_pci_dev *xpdev;
+ char err[XNL_ERR_BUFLEN];
+ int rv;
+
+ if (info == NULL)
+ return NULL;
+
+ if (!info->attrs[XNL_ATTR_DEV_IDX]) {
+ rv = snprintf(err, sizeof(err),
+ "command %s missing attribute XNL_ATTR_DEV_IDX",
+ xnl_op_str[info->genlhdr->cmd]);
+ if (rv <= 0)
+ return NULL;
+ goto respond_error;
+ }
+ idx = nla_get_u32(info->attrs[XNL_ATTR_DEV_IDX]);
+
+ xpdev = xpdev_find_by_idx(idx, err, sizeof(err));
+ if (!xpdev) {
+ /* err buffer populated by xpdev_find_by_idx*/
+ goto respond_error;
+ }
+
+ return xpdev;
+
+respond_error:
+ xnl_respond_buffer(info, err, strlen(err));
+ return NULL;
+}
+
+static int qconf_get(struct qdma_queue_conf *qconf, struct genl_info *info,
+ char *err, int errlen, unsigned char *is_qp)
+{
+ u32 f = 0;
+
+ if (!qconf || !info)
+ return -EINVAL;
+
+ if (!info->attrs[XNL_ATTR_QFLAG]) {
+ snprintf(err, errlen, "Missing attribute 'XNL_ATTR_QFLAG'\n");
+ goto respond_error;
+ }
+ f = nla_get_u32(info->attrs[XNL_ATTR_QFLAG]);
+ if ((f & XNL_F_QMODE_ST) && (f & XNL_F_QMODE_MM)) {
+ snprintf(err, errlen, "ERR! Both ST and MM mode set.\n");
+ goto respond_error;
+ } else if (!(f & XNL_F_QMODE_ST) && !(f & XNL_F_QMODE_MM)) {
+ /* default to MM */
+ f |= XNL_F_QMODE_MM;
+ }
+
+ if (!(f & XNL_F_QDIR_H2C) && !(f & XNL_F_QDIR_C2H)) {
+ /* default to H2C */
+ f |= XNL_F_QDIR_H2C;
+ }
+
+ memset(qconf, 0, sizeof(*qconf));
+ qconf->st = (f & XNL_F_QMODE_ST) ? 1 : 0;
+ qconf->c2h = (f & XNL_F_QDIR_C2H) ? 1 : 0;
+ *is_qp = ((f & XNL_F_QDIR_BOTH) == XNL_F_QDIR_BOTH) ? 1 : 0;
+
+ if (!info->attrs[XNL_ATTR_QIDX]) {
+ snprintf(err, errlen, "Missing attribute 'XNL_ATTR_QIDX'");
+ goto respond_error;
+ }
+ qconf->qidx = nla_get_u32(info->attrs[XNL_ATTR_QIDX]);
+ if (qconf->qidx == XNL_QIDX_INVALID)
+ qconf->qidx = QDMA_QUEUE_IDX_INVALID;
+
+ return 0;
+
+respond_error:
+
+ xnl_respond_buffer(info, err, strlen(err));
+ return -EINVAL;
+}
+
+static struct xlnx_qdata *xnl_rcv_check_qidx(struct genl_info *info,
+ struct xlnx_pci_dev *xpdev,
+ struct qdma_queue_conf *qconf, char *buf,
+ int buflen)
+{
+ char ebuf[XNL_ERR_BUFLEN];
+ struct xlnx_qdata *qdata = xpdev_queue_get(xpdev, qconf->qidx,
+ qconf->c2h, 1, ebuf, XNL_ERR_BUFLEN);
+
+ if (!qdata) {
+ int l = snprintf(ebuf,
+ XNL_ERR_BUFLEN,
+ "ERR! qidx %u invalid.\n",
+ qconf->qidx);
+
+ ebuf[l] = '\0';
+ xnl_respond_buffer(info, ebuf, XNL_ERR_BUFLEN);
+ }
+
+ return qdata;
+}
+
+static int xnl_chk_attr(enum xnl_attr_t xnl_attr, struct genl_info *info,
+ unsigned short qidx, char *buf)
+{
+ int rv = 0;
+
+ if (!info->attrs[xnl_attr]) {
+ if (buf) {
+ rv += sprintf(buf,
+ "Missing attribute %s for qidx = %u\n",
+ xnl_attr_str[xnl_attr],
+ qidx);
+ buf[rv] = '\0';
+ }
+ rv = -1;
+ }
+
+ return rv;
+}
+
+static void xnl_extract_extra_config_attr(struct genl_info *info,
+ struct qdma_queue_conf *qconf)
+{
+ u32 f = nla_get_u32(info->attrs[XNL_ATTR_QFLAG]);
+
+ qconf->desc_bypass = (f & XNL_F_DESC_BYPASS_EN) ? 1 : 0;
+ qconf->pfetch_bypass = (f & XNL_F_PFETCH_BYPASS_EN) ? 1 : 0;
+ qconf->pfetch_en = (f & XNL_F_PFETCH_EN) ? 1 : 0;
+ qconf->cmpl_status_en = (f & XNL_F_CMPL_STATUS_EN) ? 1 : 0;
+ qconf->cmpl_status_acc_en = (f & XNL_F_CMPL_STATUS_ACC_EN) ? 1 : 0;
+ qconf->cmpl_status_pend_chk = (f & XNL_F_CMPL_STATUS_PEND_CHK) ? 1 : 0;
+ qconf->fetch_credit = (f & XNL_F_FETCH_CREDIT) ? 1 : 0;
+ qconf->cmpl_stat_en = (f & XNL_F_CMPL_STATUS_DESC_EN) ? 1 : 0;
+ qconf->cmpl_en_intr = (f & XNL_F_C2H_CMPL_INTR_EN) ? 1 : 0;
+ qconf->cmpl_udd_en = (f & XNL_F_CMPL_UDD_EN) ? 1 : 0;
+ qconf->cmpl_ovf_chk_dis = (f & XNL_F_CMPT_OVF_CHK_DIS) ? 1 : 0;
+
+ if (xnl_chk_attr(XNL_ATTR_QRNGSZ_IDX, info, qconf->qidx, NULL) == 0)
+ qconf->desc_rng_sz_idx = qconf->cmpl_rng_sz_idx =
+ nla_get_u32(info->attrs[XNL_ATTR_QRNGSZ_IDX]);
+ if (xnl_chk_attr(XNL_ATTR_C2H_BUFSZ_IDX, info, qconf->qidx, NULL) == 0)
+ qconf->c2h_buf_sz_idx =
+ nla_get_u32(info->attrs[XNL_ATTR_C2H_BUFSZ_IDX]);
+ if (xnl_chk_attr(XNL_ATTR_CMPT_TIMER_IDX, info, qconf->qidx, NULL) == 0)
+ qconf->cmpl_timer_idx =
+ nla_get_u32(info->attrs[XNL_ATTR_CMPT_TIMER_IDX]);
+ if (xnl_chk_attr(XNL_ATTR_CMPT_CNTR_IDX, info, qconf->qidx, NULL) == 0)
+ qconf->cmpl_cnt_th_idx =
+ nla_get_u32(info->attrs[XNL_ATTR_CMPT_CNTR_IDX]);
+ if (xnl_chk_attr(XNL_ATTR_CMPT_DESC_SIZE,
+ info, qconf->qidx, NULL) == 0)
+ qconf->cmpl_desc_sz =
+ nla_get_u32(info->attrs[XNL_ATTR_CMPT_DESC_SIZE]);
+ if (xnl_chk_attr(XNL_ATTR_SW_DESC_SIZE,
+ info, qconf->qidx, NULL) == 0)
+ qconf->sw_desc_sz =
+ nla_get_u32(info->attrs[XNL_ATTR_SW_DESC_SIZE]);
+ if (xnl_chk_attr(XNL_ATTR_CMPT_TRIG_MODE, info, qconf->qidx, NULL) == 0)
+ qconf->cmpl_trig_mode =
+ nla_get_u32(info->attrs[XNL_ATTR_CMPT_TRIG_MODE]);
+ else
+ qconf->cmpl_trig_mode = 1;
+ if (xnl_chk_attr(XNL_ATTR_PIPE_GL_MAX,
+ info, qconf->pipe_gl_max, NULL) == 0)
+ qconf->pipe_gl_max =
+ nla_get_u32(info->attrs[XNL_ATTR_PIPE_GL_MAX]);
+ if (xnl_chk_attr(XNL_ATTR_PIPE_FLOW_ID,
+ info, qconf->pipe_flow_id, NULL) == 0)
+ qconf->pipe_flow_id =
+ nla_get_u32(info->attrs[XNL_ATTR_PIPE_FLOW_ID]);
+ if (xnl_chk_attr(XNL_ATTR_PIPE_SLR_ID,
+ info, qconf->pipe_slr_id, NULL) == 0)
+ qconf->pipe_slr_id =
+ nla_get_u32(info->attrs[XNL_ATTR_PIPE_SLR_ID]);
+ if (xnl_chk_attr(XNL_ATTR_PIPE_TDEST,
+ info, qconf->pipe_tdest, NULL) == 0)
+ qconf->pipe_tdest =
+ nla_get_u32(info->attrs[XNL_ATTR_PIPE_TDEST]);
+}
+
+static int xnl_dev_list(struct sk_buff *skb2, struct genl_info *info)
+{
+ char *buf;
+ int rv;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MAX, info);
+ if (!buf)
+ return -ENOMEM;
+
+ rv = xpdev_list_dump(buf, XNL_RESP_BUFLEN_MAX);
+ if (rv < 0) {
+ pr_err("xpdev_list_dump() failed: %d", rv);
+ goto free_msg_buff;
+ }
+ rv = xnl_respond_buffer(info, buf, strlen(buf));
+
+free_msg_buff:
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_dev_info(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct sk_buff *skb;
+ void *hdr;
+ struct xlnx_pci_dev *xpdev;
+ struct pci_dev *pdev;
+ struct qdma_dev_conf conf;
+ int rv;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+ pdev = xpdev->pdev;
+
+ rv = qdma_device_get_config(xpdev->dev_hndl, &conf, NULL, 0);
+ if (rv < 0)
+ return rv;
+
+ skb = xnl_msg_alloc(XNL_CMD_DEV_INFO, 0, &hdr, info);
+ if (!skb)
+ return -ENOMEM;
+
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_PCI_BUS,
+ pdev->bus->number);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_PCI_DEV,
+ PCI_SLOT(pdev->devfn));
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_PCI_FUNC,
+ PCI_FUNC(pdev->devfn));
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_CFG_BAR,
+ conf.bar_num_config);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_USR_BAR,
+ conf.bar_num_user);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_QSET_MAX, conf.qsets_max);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_QSET_QBASE, conf.qsets_base);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STM_BAR,
+ conf.bar_num_stm);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ nlmsg_free(skb);
+ return rv;
+ }
+
+ rv = xnl_msg_send(skb, hdr, info);
+
+ return rv;
+}
+
+static int xnl_dev_version(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_version_info ver_info;
+ char *buf;
+ int rv = 0;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MAX, info);
+ if (!buf)
+ return -ENOMEM;
+
+ rv = qdma_device_version_info(xpdev->dev_hndl, &ver_info);
+ if (rv < 0) {
+ pr_err("qdma_device_version_info() failed: %d", rv);
+ goto free_buf;
+ }
+
+ rv = sprintf(buf + rv,"=============Hardware Version============\n\n");
+ rv += sprintf(buf + rv,"RTL Version : %s\n",ver_info.rtl_version_str);
+ rv += sprintf(buf + rv,"Vivado ReleaseID : %s\n",ver_info.vivado_release_str);
+ rv += sprintf(buf + rv,"Everest IP : %s\n\n",ver_info.everest_ip_str);
+ rv += sprintf(buf + rv,"============Software Version============\n\n");
+ rv += sprintf(buf + rv,"qdma driver version : %s\n",DRV_MODULE_VERSION);
+ buf[rv] = '\0';
+
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MAX);
+
+free_buf:
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_dev_stat(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct sk_buff *skb;
+ void *hdr;
+ struct xlnx_pci_dev *xpdev;
+ int rv;
+ unsigned long long mmh2c_pkts = 0;
+ unsigned long long mmc2h_pkts = 0;
+ unsigned long long sth2c_pkts = 0;
+ unsigned long long stc2h_pkts = 0;
+ unsigned int pkts;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+
+ skb = xnl_msg_alloc(XNL_CMD_DEV_STAT, 0, &hdr, info);
+ if (!skb)
+ return -ENOMEM;
+
+ qdma_device_get_mmh2c_pkts(xpdev->dev_hndl, &mmh2c_pkts);
+ qdma_device_get_mmc2h_pkts(xpdev->dev_hndl, &mmc2h_pkts);
+ qdma_device_get_sth2c_pkts(xpdev->dev_hndl, &sth2c_pkts);
+ qdma_device_get_stc2h_pkts(xpdev->dev_hndl, &stc2h_pkts);
+ pkts = mmh2c_pkts;
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_MMH2C_PKTS1, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = (mmh2c_pkts >> 32);
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_MMH2C_PKTS2, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = mmc2h_pkts;
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_MMC2H_PKTS1, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = (mmc2h_pkts >> 32);
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_MMC2H_PKTS2, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = sth2c_pkts;
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_STH2C_PKTS1, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = (sth2c_pkts >> 32);
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_STH2C_PKTS2, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = stc2h_pkts;
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_STC2H_PKTS1, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ pkts = (stc2h_pkts >> 32);
+ rv = xnl_msg_add_attr_uint(skb, XNL_ATTR_DEV_STAT_STC2H_PKTS2, pkts);
+ if (rv < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return rv;
+ }
+ rv = xnl_msg_send(skb, hdr, info);
+
+ return rv;
+}
+
+static int xnl_dev_stat_clear(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ int rv;
+ char *buf;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MIN, info);
+ if (!buf)
+ return -ENOMEM;
+
+ qdma_device_clear_stats(xpdev->dev_hndl);
+
+ buf[0] = '\0';
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MAX);
+
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_q_list(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ char *buf;
+ int rv = 0;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MAX, info);
+ if (!buf)
+ return -ENOMEM;
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl))) {
+ rv += snprintf(buf, 8, "Zero Qs");
+ goto send_rsp;
+ }
+
+ rv = qdma_queue_list(xpdev->dev_hndl, buf, XNL_RESP_BUFLEN_MAX);
+ if (rv < 0) {
+ pr_err("qdma_queue_list() failed: %d", rv);
+ goto free_buf;
+ }
+
+send_rsp:
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MAX);
+
+free_buf:
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_q_add(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev = NULL;
+ struct qdma_queue_conf qconf;
+ char *buf, *cur, *end;
+ int rv = 0;
+ int rv2 = 0;
+ unsigned char is_qp;
+ unsigned int num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return -EINVAL;
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf)
+ return -ENOMEM;
+ cur = buf;
+ end = buf + buf_len;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl))) {
+ pr_info("0 sized Qs\n");
+ rv += snprintf(cur, end - cur, "Zero Qs");
+ goto send_resp;
+ }
+ rv = qconf_get(&qconf, info, cur, end - cur, &is_qp);
+ if (rv < 0)
+ goto free_buf;
+
+
+ qidx = qconf.qidx;
+
+ rv = xnl_chk_attr(XNL_ATTR_NUM_Q, info, qidx, cur);
+ if (rv < 0)
+ goto send_resp;
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ is_c2h = qconf.c2h;
+ for (i = 0; i < num_q; i++) {
+ qconf.c2h = is_c2h;
+add_q:
+ if (qidx != QDMA_QUEUE_IDX_INVALID)
+ qconf.qidx = qidx + i;
+ rv = xpdev_queue_add(xpdev, &qconf, cur, end - cur);
+ if (rv < 0) {
+ pr_err("xpdev_queue_add() failed: %d\n", rv);
+ goto send_resp;
+ }
+ cur = buf + strlen(buf);
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto add_q;
+ }
+ }
+ cur += snprintf(cur, end - cur, "Added %d Queues.\n", i);
+
+send_resp:
+ rv2 = xnl_respond_buffer(info, buf, strlen(buf));
+free_buf:
+ kfree(buf);
+ return rv < 0 ? rv : rv2;
+}
+
+static int xnl_q_start(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ char buf[XNL_RESP_BUFLEN_MIN];
+ struct xlnx_qdata *qdata;
+ int rv = 0;
+ unsigned char is_qp;
+ unsigned short num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl))) {
+ rv += snprintf(buf, 8, "Zero Qs");
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+ rv = qconf_get(&qconf, info, buf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ goto send_resp;
+
+ qidx = qconf.qidx;
+
+ rv = xnl_chk_attr(XNL_ATTR_NUM_Q, info, qidx, buf);
+ if (rv < 0)
+ goto send_resp;
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ xnl_extract_extra_config_attr(info, &qconf);
+ is_c2h = qconf.c2h;
+
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+reconfig:
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf,
+ XNL_RESP_BUFLEN_MIN);
+ if (!qdata)
+ goto send_resp;
+
+ rv = qdma_queue_reconfig(xpdev->dev_hndl, qdata->qhndl, &qconf,
+ buf, XNL_RESP_BUFLEN_MIN);
+ if (rv < 0) {
+ pr_err("qdma_queue_reconfig() failed: %d", rv);
+ goto send_resp;
+ }
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto reconfig;
+ }
+ }
+
+ rv = xpdev_nl_queue_start(xpdev, info, is_qp, is_c2h, qidx, num_q);
+ if (rv < 0) {
+ snprintf(buf, XNL_RESP_BUFLEN_MIN, "qdma%05x OOM.\n",
+ xpdev->idx);
+ goto send_resp;
+ }
+
+ return 0;
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MIN);
+
+ return rv;
+}
+
+static int xnl_q_stop(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ char buf[XNL_RESP_BUFLEN_MIN];
+ struct xlnx_qdata *qdata;
+ int rv = 0;
+ unsigned char is_qp;
+ unsigned short num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl))) {
+ rv += snprintf(buf, 8, "Zero Qs");
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+ rv = qconf_get(&qconf, info, buf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ goto send_resp;
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ is_c2h = qconf.c2h;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+stop_q:
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf,
+ XNL_RESP_BUFLEN_MIN);
+ if (!qdata)
+ goto send_resp;
+ rv = qdma_queue_stop(xpdev->dev_hndl, qdata->qhndl,
+ buf, XNL_RESP_BUFLEN_MIN);
+ if (rv < 0) {
+ pr_err("qdma_queue_stop() failed: %d", rv);
+ goto send_resp;
+ }
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto stop_q;
+ }
+ }
+ rv += sprintf(buf + rv, "Stopped Queues %d -> %d.\n", qidx, i - 1);
+send_resp:
+ buf[rv] = '\0';
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MIN);
+ return rv;
+}
+
+static int xnl_q_del(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ char buf[XNL_RESP_BUFLEN_MIN];
+ int rv = 0;
+ unsigned char is_qp;
+ unsigned short num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl))) {
+ rv += snprintf(buf, 8, "Zero Qs");
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+ rv = qconf_get(&qconf, info, buf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ goto send_resp;
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ is_c2h = qconf.c2h;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+del_q:
+ qconf.qidx = i;
+ rv = xpdev_queue_delete(xpdev, qconf.qidx, qconf.c2h,
+ buf, XNL_RESP_BUFLEN_MIN);
+ if (rv < 0) {
+ pr_err("xpdev_queue_delete() failed: %d", rv);
+ goto send_resp;
+ }
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto del_q;
+ }
+ }
+ rv += sprintf(buf + rv, "Deleted Queues %d -> %d.\n", qidx, i - 1);
+send_resp:
+ buf[rv] = '\0';
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MIN);
+ return rv;
+}
+
+static int xnl_q_dump(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ struct xlnx_qdata *qdata;
+ char *buf;
+ char ebuf[XNL_RESP_BUFLEN_MIN];
+ int rv;
+ unsigned char is_qp;
+ unsigned int num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+ unsigned int buf_idx = 0;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ rv = qconf_get(&qconf, info, ebuf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ return rv;
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf)
+ return -ENOMEM;
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ kfree(buf);
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ is_c2h = qconf.c2h;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+dump_q:
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf + buf_idx,
+ buf_len - buf_idx);
+ if (!qdata)
+ goto send_resp;
+ rv = qdma_queue_dump(xpdev->dev_hndl, qdata->qhndl,
+ buf + buf_idx,
+ buf_len - buf_idx);
+ if (rv < 0) {
+ pr_err("qdma_queue_dump() failed: %d", rv);
+ goto send_resp;
+ }
+ buf_idx = strlen(buf);
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto dump_q;
+ }
+ }
+ rv = snprintf(buf + buf_idx, buf_len - buf_idx,
+ "Dumped Queues %d -> %d.\n", qidx, i - 1);
+send_resp:
+ buf[buf_idx + rv] = '\0';
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_q_dump_desc(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ struct xlnx_qdata *qdata;
+ u32 v1;
+ u32 v2;
+ char *buf;
+ char ebuf[XNL_RESP_BUFLEN_MIN];
+ int rv;
+ unsigned char is_qp;
+ unsigned int num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+ unsigned int buf_idx = 0;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ v1 = nla_get_u32(info->attrs[XNL_ATTR_RANGE_START]);
+ v2 = nla_get_u32(info->attrs[XNL_ATTR_RANGE_END]);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl)))
+ return 0;
+ rv = qconf_get(&qconf, info, ebuf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ return rv;
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf) {
+ rv = sprintf(ebuf, "%s OOM %d.\n",
+ __func__, buf_len);
+
+ ebuf[rv] = '\0';
+ xnl_respond_buffer(info, ebuf, XNL_RESP_BUFLEN_MIN);
+ return -ENOMEM;
+ }
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ is_c2h = qconf.c2h;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+dump_q:
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf + buf_idx,
+ buf_len - buf_idx);
+ if (!qdata)
+ goto send_resp;
+ rv = qdma_queue_dump_desc(xpdev->dev_hndl,
+ qdata->qhndl, v1, v2,
+ buf + buf_idx, buf_len - buf_idx);
+ if (rv < 0) {
+ pr_err("qdma_queue_dump_desc() failed: %d", rv);
+ goto send_resp;
+ }
+ buf_idx = strlen(buf);
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto dump_q;
+ }
+ }
+ rv = snprintf(buf + buf_idx, buf_len - buf_idx,
+ "Dumped descs of queues %d -> %d.\n",
+ qidx, i - 1);
+ buf[buf_idx + rv] = '\0';
+send_resp:
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_q_dump_cmpt(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ struct xlnx_qdata *qdata;
+ u32 v1;
+ u32 v2;
+ char *buf;
+ char ebuf[XNL_RESP_BUFLEN_MIN];
+ int rv;
+ unsigned char is_qp;
+ unsigned int num_q;
+ unsigned int i;
+ unsigned short qidx;
+ unsigned char is_c2h;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+ unsigned int buf_idx = 0;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ v1 = nla_get_u32(info->attrs[XNL_ATTR_RANGE_START]);
+ v2 = nla_get_u32(info->attrs[XNL_ATTR_RANGE_END]);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (unlikely(!qdma_get_qmax(xpdev->dev_hndl)))
+ return 0;
+
+ rv = qconf_get(&qconf, info, ebuf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ return rv;
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf) {
+ rv = sprintf(ebuf, "%s OOM %d.\n",
+ __func__, buf_len);
+ ebuf[rv] = '\0';
+ xnl_respond_buffer(info, ebuf, XNL_RESP_BUFLEN_MIN);
+ return -ENOMEM;
+ }
+ pr_info("response buf_len = %d", buf_len);
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ is_c2h = qconf.c2h;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = is_c2h;
+dump_q:
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf + buf_idx,
+ buf_len - buf_idx);
+ if (!qdata)
+ goto send_resp;
+ rv = qdma_queue_dump_cmpt(xpdev->dev_hndl,
+ qdata->qhndl, v1, v2,
+ buf + buf_idx, buf_len - buf_idx);
+ if (rv < 0) {
+ pr_err("qdma_queue_dump_cmpt() failed: %d", rv);
+ goto send_resp;
+ }
+ buf_idx = strlen(buf);
+ if (is_qp && (is_c2h == qconf.c2h)) {
+ qconf.c2h = ~qconf.c2h;
+ goto dump_q;
+ }
+ }
+ rv = snprintf(buf + buf_idx, buf_len - buf_idx,
+ "Dumped descs of queues %d -> %d.\n",
+ qidx, i - 1);
+send_resp:
+ buf[buf_idx + rv] = '\0';
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_q_read_udd(struct sk_buff *skb2, struct genl_info *info)
+{
+ int rv = 0;
+ struct qdma_queue_conf qconf;
+ char *buf;
+ unsigned char is_qp;
+ struct xlnx_pci_dev *xpdev;
+ struct xlnx_qdata *qdata;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MAX, info);
+ if (!buf)
+ return -ENOMEM;
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev) {
+ kfree(buf);
+ return -EINVAL;
+ }
+
+ rv = qconf_get(&qconf, info, buf, XNL_RESP_BUFLEN_MAX, &is_qp);
+ if (rv < 0)
+ goto send_resp;
+
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf,
+ XNL_RESP_BUFLEN_MAX);
+ if (!qdata)
+ goto send_resp;
+
+ rv = qdma_descq_get_cmpt_udd(xpdev->dev_hndl, qdata->qhndl, buf,
+ XNL_RESP_BUFLEN_MAX);
+ if (rv < 0)
+ goto send_resp;
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+}
+
+#ifdef ERR_DEBUG
+static int xnl_err_induce(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ struct xlnx_qdata *qdata;
+ char *buf;
+ char ebuf[XNL_RESP_BUFLEN_MIN];
+ unsigned char is_qp;
+ int rv;
+ u32 err;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+ rv = qconf_get(&qconf, info, ebuf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ return rv;
+
+ buf = xnl_mem_alloc(XNL_RESP_BUFLEN_MAX, info);
+ if (!buf) {
+ rv = sprintf(ebuf, "%s OOM %d.\n",
+ __func__, XNL_RESP_BUFLEN_MAX);
+ ebuf[rv] = '\0';
+ xnl_respond_buffer(info, ebuf, XNL_RESP_BUFLEN_MIN);
+ return -ENOMEM;
+ }
+
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf,
+ XNL_RESP_BUFLEN_MIN);
+ if (!qdata)
+ goto send_resp;
+ err = nla_get_u32(info->attrs[XNL_ATTR_QPARAM_ERR_INFO]);
+
+ if (qdma_queue_set_err_induction(xpdev->dev_hndl, qdata->qhndl, err,
+ buf, XNL_RESP_BUFLEN_MAX)) {
+ rv += sprintf(buf + rv, "Failed to set induce err\n");
+ goto send_resp;
+ }
+ rv += sprintf(buf + rv, "queue error induced\n");
+ buf[rv] = '\0';
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MAX);
+
+ kfree(buf);
+ return rv;
+}
+#endif
+
+static int xnl_q_read_pkt(struct sk_buff *skb2, struct genl_info *info)
+{
+#if 0
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_queue_conf qconf;
+ struct xlnx_qdata *qdata;
+ char *buf;
+ char ebuf[XNL_RESP_BUFLEN_MIN];
+ int rv;
+ unsigned char is_qp;
+ unsigned int num_q;
+ unsigned int i;
+ unsigned short qidx;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ rv = qconf_get(&qconf, info, ebuf, XNL_RESP_BUFLEN_MIN, &is_qp);
+ if (rv < 0)
+ return rv;
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf) {
+ rv = sprintf(ebuf, "%s OOM %d.\n",
+ __func__, buf_len);
+ ebuf[rv] = '\0';
+ xnl_respond_buffer(info, ebuf, XNL_RESP_BUFLEN_MIN);
+ return -ENOMEM;
+ }
+
+ if (!info->attrs[XNL_ATTR_NUM_Q]) {
+ pr_warn("Missing attribute 'XNL_ATTR_NUM_Q'");
+ return -1;
+ }
+ num_q = nla_get_u32(info->attrs[XNL_ATTR_NUM_Q]);
+
+ qidx = qconf.qidx;
+ for (i = qidx; i < (qidx + num_q); i++) {
+ qconf.c2h = 1;
+ qconf.qidx = i;
+ qdata = xnl_rcv_check_qidx(info, xpdev, &qconf, buf,
+ buf_len);
+ if (!qdata)
+ goto send_resp;
+ rv = qdma_queue_dump_rx_packet(xpdev->dev_hndl, qdata->qhndl,
+ buf, buf_len);
+ if (rv < 0) {
+ pr_err("qdma_queue_dump_rx_packet) failed: %d", rv);
+ goto send_resp;
+ }
+ }
+ buf[rv] = '\0';
+send_resp:
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+#endif
+ pr_info("NOT supported.\n");
+ return -EINVAL;
+}
+
+static int xnl_intr_ring_dump(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ char *buf;
+ unsigned int vector_idx = 0;
+ int start_idx = 0, end_idx = 0;
+ int rv = 0;
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ if (!info->attrs[XNL_ATTR_INTR_VECTOR_IDX]) {
+ pr_warn("Missing attribute 'XNL_ATTR_INTR_VECTOR_IDX'");
+ return -1;
+ }
+ vector_idx = nla_get_u32(info->attrs[XNL_ATTR_INTR_VECTOR_IDX]);
+ start_idx = nla_get_u32(info->attrs[XNL_ATTR_INTR_VECTOR_START_IDX]);
+ end_idx = nla_get_u32(info->attrs[XNL_ATTR_INTR_VECTOR_END_IDX]);
+
+ if (info->attrs[XNL_ATTR_RSP_BUF_LEN])
+ buf_len = nla_get_u32(info->attrs[XNL_ATTR_RSP_BUF_LEN]);
+
+ buf = xnl_mem_alloc(buf_len, info);
+ if (!buf)
+ return -ENOMEM;
+
+ if (xpdev->idx == 0) {
+ if (vector_idx == 0) {
+ rv += sprintf(buf + rv,
+ "vector_idx %d is for error interrupt\n",
+ vector_idx);
+ buf[rv] = '\0';
+ goto send_resp;
+ } else if (vector_idx == 1) {
+ rv += sprintf(buf + rv,
+ "vector_idx %d is for user interrupt\n",
+ vector_idx);
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+ } else {
+ if (vector_idx == 0) {
+ rv += sprintf(buf + rv,
+ "vector_idx %d is for user interrupt\n",
+ vector_idx);
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+ }
+
+ rv = qdma_intr_ring_dump(xpdev->dev_hndl,
+ vector_idx, start_idx,
+ end_idx, buf, buf_len);
+ if (rv < 0) {
+ pr_err("qdma_intr_ring_dump() failed: %d", rv);
+ goto send_resp;
+ }
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, buf_len);
+
+ kfree(buf);
+ return rv;
+}
+
+static int xnl_register_read(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct sk_buff *skb;
+ void *hdr;
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_dev_conf conf;
+ char buf[XNL_RESP_BUFLEN_MIN];
+ unsigned int bar_num = 0, reg_addr = 0;
+ uint32_t reg_val = 0;
+ int rv = 0, err = 0;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ rv = qdma_device_get_config(xpdev->dev_hndl, &conf, NULL, 0);
+ if (rv < 0)
+ return rv;
+
+ skb = xnl_msg_alloc(XNL_CMD_REG_RD, 0, &hdr, info);
+ if (!skb)
+ return -ENOMEM;
+
+ if (!info->attrs[XNL_ATTR_REG_BAR_NUM]) {
+ pr_warn("Missing attribute 'XNL_ATTR_REG_BAR_NUM'");
+ return -EINVAL;
+ }
+
+ if (!info->attrs[XNL_ATTR_REG_ADDR]) {
+ pr_warn("Missing attribute 'XNL_ATTR_REG_ADDR'");
+ return -EINVAL;
+ }
+
+ bar_num = nla_get_u32(info->attrs[XNL_ATTR_REG_BAR_NUM]);
+ reg_addr = nla_get_u32(info->attrs[XNL_ATTR_REG_ADDR]);
+
+ if (bar_num == conf.bar_num_config) {
+ reg_val = qdma_device_read_config_register(xpdev->dev_hndl,
+ reg_addr);
+ } else if (bar_num == conf.bar_num_user) {
+ reg_val = qdma_device_read_user_register(xpdev, reg_addr);
+ } else if (bar_num == conf.bar_num_bypass) {
+ reg_val = qdma_device_read_bypass_register(xpdev, reg_addr);
+ } else {
+ rv += sprintf(buf + rv, "Invalid bar number\n");
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+
+ err = xnl_msg_add_attr_uint(skb, XNL_ATTR_REG_VAL, reg_val);
+ if (err < 0) {
+ pr_err("xnl_msg_add_attr_uint() failed: %d", rv);
+ return err;
+ }
+
+ err = xnl_msg_send(skb, hdr, info);
+ return err;
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MIN);
+ nlmsg_free(skb);
+ return rv;
+}
+
+static int xnl_register_write(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_dev_conf conf;
+ char buf[XNL_RESP_BUFLEN_MIN];
+ unsigned int bar_num = 0, reg_addr = 0;
+ uint32_t reg_val = 0;
+ int rv = 0;
+
+ if (info == NULL)
+ return 0;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ rv = qdma_device_get_config(xpdev->dev_hndl, &conf, NULL, 0);
+ if (rv < 0)
+ return rv;
+
+ if (!info->attrs[XNL_ATTR_REG_BAR_NUM]) {
+ pr_warn("Missing attribute 'XNL_ATTR_REG_BAR_NUM'");
+ return -EINVAL;
+ }
+
+ if (!info->attrs[XNL_ATTR_REG_ADDR]) {
+ pr_warn("Missing attribute 'XNL_ATTR_REG_ADDR'");
+ return -EINVAL;
+ }
+
+ if (!info->attrs[XNL_ATTR_REG_VAL]) {
+ pr_warn("Missing attribute 'XNL_ATTR_REG_VAL'");
+ return -EINVAL;
+ }
+
+ bar_num = nla_get_u32(info->attrs[XNL_ATTR_REG_BAR_NUM]);
+ reg_addr = nla_get_u32(info->attrs[XNL_ATTR_REG_ADDR]);
+ reg_val = nla_get_u32(info->attrs[XNL_ATTR_REG_VAL]);
+
+ if (bar_num == conf.bar_num_config) {
+ qdma_device_write_config_register(xpdev->dev_hndl,
+ reg_addr, reg_val);
+ } else if (bar_num == conf.bar_num_user) {
+ qdma_device_write_user_register(xpdev, reg_addr, reg_val);
+ } else if (bar_num == conf.bar_num_bypass) {
+ qdma_device_write_bypass_register(xpdev, reg_addr, reg_val);
+ } else {
+ rv += sprintf(buf + rv, "Invalid bar number\n");
+ buf[rv] = '\0';
+ goto send_resp;
+ }
+
+ buf[rv] = '\0';
+
+send_resp:
+ rv = xnl_respond_buffer(info, buf, XNL_RESP_BUFLEN_MIN);
+ return rv;
+}
+static int xnl_get_global_csr(struct sk_buff *skb2, struct genl_info *info)
+{
+ struct global_csr_conf *csr;
+ struct xlnx_pci_dev *xpdev;
+ int rv;
+
+ if (info == NULL)
+ return -EINVAL;
+
+ xnl_dump_attrs(info);
+
+ xpdev = xnl_rcv_check_xpdev(info);
+ if (!xpdev)
+ return 0;
+
+ csr = kmalloc(sizeof(struct global_csr_conf), GFP_KERNEL);
+ if (!csr)
+ return -ENOMEM;
+
+ rv = qdma_global_csr_get(xpdev->dev_hndl, csr);
+ if (rv < 0) {
+ pr_err("qdma_global_csr_get() failed: %d", rv);
+ goto free_msg_buff;
+ }
+
+ rv = xnl_respond_data(info,
+ (void *)csr, sizeof(struct global_csr_conf));
+
+free_msg_buff:
+ kfree(csr);
+ return rv;
+}
+
+int xlnx_nl_init(void)
+{
+ int rv;
+#ifdef __GENL_REG_FAMILY_OPS_FUNC__
+ rv = genl_register_family_with_ops(&xnl_family,
+ xnl_ops,
+ ARRAY_SIZE(xnl_ops));
+#else
+ rv = genl_register_family(&xnl_family);
+#endif
+ if (rv)
+ pr_info("genl_register_family failed %d.\n", rv);
+
+ return rv;
+}
+
+void xlnx_nl_exit(void)
+{
+ int rv;
+
+ rv = genl_unregister_family(&xnl_family);
+ if (rv)
+ pr_info("genl_unregister_family failed %d.\n", rv);
+}
diff --git a/QDMA/linux-kernel/drv/nl.h b/QDMA/linux-kernel/drv/nl.h
new file mode 100644
index 0000000000000000000000000000000000000000..c9988cdcf9ec2b5b5cb138bf065f0fd897798450
--- /dev/null
+++ b/QDMA/linux-kernel/drv/nl.h
@@ -0,0 +1,45 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_DRV_NL_H__
+#define __QDMA_DRV_NL_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma netlink helper
+ * funnctions kernel module
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <net/genetlink.h>
+
+/*****************************************************************************/
+/**
+ * xnl_respond_buffer() - send a netlink string message
+ *
+ * @param[in] nl_info: pointer to netlink genl_info
+ * @param[in] buf: string buffer
+ * @param[in] buflen: length of the string buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xnl_respond_buffer(struct genl_info *info, char *buf, int buflen);
+
+#endif /* ifndef __QDMA_DRV_NL_H__ */
diff --git a/QDMA/linux-kernel/drv/pci_ids.h b/QDMA/linux-kernel/drv/pci_ids.h
new file mode 100644
index 0000000000000000000000000000000000000000..dbbf17e94f811cf8d9061c139415211f940b3b7a
--- /dev/null
+++ b/QDMA/linux-kernel/drv/pci_ids.h
@@ -0,0 +1,253 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __XDMA_PCI_ID_H__
+#define __XDMA_PCI_ID_H__
+/**
+ * @file
+ * @brief This file contains the list of pcie devices supported for qdma driver
+ *
+ */
+
+/**
+ * list of pcie devices supported for qdma driver
+ */
+static const struct pci_device_id pci_ids[] = {
+
+#ifdef __QDMA_VF__
+ /** Gen 1 VF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0xa011), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa111), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa211), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa311), }, /** VF on PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0xa012), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa112), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa212), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa312), }, /** VF on PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0xa014), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa114), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa214), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa314), }, /** VF on PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0xa018), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa118), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa218), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa318), }, /** VF on PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0xa01f), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa11f), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa21f), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa31f), }, /** VF on PF 3 */
+
+ /** Gen 2 VF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0xa021), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa121), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa221), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa321), }, /** VF on PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0xa022), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa122), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa222), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa322), }, /** VF on PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0xa024), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa124), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa224), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa324), }, /** VF on PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0xa028), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa128), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa228), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa328), }, /** VF on PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0xa02f), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa12f), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa22f), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa32f), }, /** VF on PF 3 */
+
+ /** Gen 3 VF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0xa031), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa131), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa231), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa331), }, /** VF on PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0xa032), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa132), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa232), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa332), }, /** VF on PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0xa034), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa134), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa234), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa334), }, /** VF on PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0xa038), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa138), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa238), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa338), }, /** VF on PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0xa03f), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa13f), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa23f), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa33f), }, /** VF on PF 3 */
+
+ /** Gen 4 VF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0xa041), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa141), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa241), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa341), }, /** VF on PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0xa042), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa142), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa242), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa342), }, /** VF on PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0xa044), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa144), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa244), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa344), }, /** VF on PF 3 */
+ { PCI_DEVICE(0x10ee, 0xa444), }, /** VF on PF 4 */
+ { PCI_DEVICE(0x10ee, 0xa544), }, /** VF on PF 5 */
+ { PCI_DEVICE(0x10ee, 0xa644), }, /** VF on PF 6 */
+ { PCI_DEVICE(0x10ee, 0xa744), }, /** VF on PF 7 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0xa048), }, /** VF on PF 0 */
+ { PCI_DEVICE(0x10ee, 0xa148), }, /** VF on PF 1 */
+ { PCI_DEVICE(0x10ee, 0xa248), }, /** VF on PF 2 */
+ { PCI_DEVICE(0x10ee, 0xa348), }, /** VF on PF 3 */
+#else
+ /** Gen 1 PF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0x9011), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9111), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9211), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9311), }, /** PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0x9012), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9112), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9212), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9312), }, /** PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0x9014), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9114), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9214), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9314), }, /** PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0x9018), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9118), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9218), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9318), }, /** PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0x901f), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x911f), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x921f), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x931f), }, /** PF 3 */
+
+ /** Gen 2 PF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0x9021), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9121), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9221), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9321), }, /** PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0x9022), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9122), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9222), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9322), }, /** PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0x9024), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9124), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9224), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9324), }, /** PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0x9028), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9128), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9228), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9328), }, /** PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0x902f), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x912f), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x922f), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x932f), }, /** PF 3 */
+
+ /** Gen 3 PF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0x9031), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9131), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9231), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9331), }, /** PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0x9032), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9132), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9232), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9332), }, /** PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0x9034), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9134), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9234), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9334), }, /** PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0x9038), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9138), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9238), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9338), }, /** PF 3 */
+ /** PCIe lane width x16 */
+ { PCI_DEVICE(0x10ee, 0x903f), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x913f), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x923f), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x933f), }, /** PF 3 */
+ /* { PCI_DEVICE(0x10ee, 0x6a9f), }, */ /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x6aa0), }, /** PF 1 */
+
+ /** Gen 4 PF */
+ /** PCIe lane width x1 */
+ { PCI_DEVICE(0x10ee, 0x9041), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9141), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9241), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9341), }, /** PF 3 */
+ /** PCIe lane width x2 */
+ { PCI_DEVICE(0x10ee, 0x9042), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9142), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9242), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9342), }, /** PF 3 */
+ /** PCIe lane width x4 */
+ { PCI_DEVICE(0x10ee, 0x9044), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9144), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9244), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9344), }, /** PF 3 */
+ /** PCIe lane width x8 */
+ { PCI_DEVICE(0x10ee, 0x9048), }, /** PF 0 */
+ { PCI_DEVICE(0x10ee, 0x9148), }, /** PF 1 */
+ { PCI_DEVICE(0x10ee, 0x9248), }, /** PF 2 */
+ { PCI_DEVICE(0x10ee, 0x9348), }, /** PF 3 */
+#endif
+
+ {0,}
+};
+
+/** module device table */
+MODULE_DEVICE_TABLE(pci, pci_ids);
+
+#endif /* ifndef __XDMA_PCI_ID_H__ */
diff --git a/QDMA/linux-kernel/drv/qdma_mod.c b/QDMA/linux-kernel/drv/qdma_mod.c
new file mode 100644
index 0000000000000000000000000000000000000000..f1ce60481379eec477a039b2d4b73babd82493d4
--- /dev/null
+++ b/QDMA/linux-kernel/drv/qdma_mod.c
@@ -0,0 +1,2257 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_mod.h"
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/version.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/aer.h>
+#include <linux/vmalloc.h>
+
+#include "nl.h"
+
+/* include early, to verify it depends only on the headers above */
+#include "version.h"
+
+static char version[] =
+ DRV_MODULE_DESC " v" DRV_MODULE_VERSION "\n";
+
+MODULE_AUTHOR("Xilinx, Inc.");
+MODULE_DESCRIPTION(DRV_MODULE_DESC);
+MODULE_VERSION(DRV_MODULE_VERSION);
+MODULE_LICENSE("Dual BSD/GPL");
+
+/* Module Parameters */
+/** 0 - AUTO_MODE,
+ * 1 - POLL_MODE,
+ * 2 - DIRECT_INTR_MODE,
+ * 3 - INDIRECT_INTR_MODE,
+ * 4 - LEGACY_INTR_MODE
+ */
+static unsigned int mode = 0;
+module_param(mode, uint, 0644);
+MODULE_PARM_DESC(mode,
+"use hw polling instead of interrupts for determining dma transfer completion");
+
+static unsigned int master_pf[MAX_DMA_DEV] = {0};
+static unsigned int master_cnt;
+module_param_array(master_pf, uint, &master_cnt, 0644);
+MODULE_PARM_DESC(master_pf, "Master PF for setting global CSRs, dflt PF 0");
+
+static unsigned int config_bar[MAX_DMA_DEV] = {0};
+static unsigned int config_bar_cnt;
+module_param_array(config_bar, uint, &config_bar_cnt, 0644);
+MODULE_PARM_DESC(config_bar, "Config bar number for all devices , dflt 0");
+
+static unsigned int num_threads = 0;
+module_param(num_threads, uint, 0644);
+MODULE_PARM_DESC(num_threads,
+"Number of threads to be created each for request and writeback processing");
+
+static unsigned int tm_mode_en = 0;
+module_param(tm_mode_en, uint, 0644);
+MODULE_PARM_DESC(tm_mode_en,
+ "Enable Traffic Manager mode for bypass ST H2C xmit. Default disabled");
+
+static unsigned int tm_one_cdh_en = 0;
+module_param(tm_one_cdh_en, uint, 0644);
+MODULE_PARM_DESC(tm_one_cdh_en,
+ "Enable 1 CDH for Traffic Manager mode. Default is Zero CDH");
+
+#include "pci_ids.h"
+
+/*
+ * xpdev helper functions
+ */
+static LIST_HEAD(xpdev_list);
+static DEFINE_MUTEX(xpdev_mutex);
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the intr_rngsz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: intr_rngsz configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the intr_rngsz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_intr_rngsz(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xlnx_pci_dev *xpdev;
+ int len;
+ unsigned int rngsz = 0;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ rngsz = qdma_get_intr_rngsz(xpdev->dev_hndl);
+ len = sprintf(buf, "%d\n", rngsz);
+ if (len <= 0)
+ pr_err("copying rngsz to buffer failed with err: %d\n", len);
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the intr_rngsz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: intr_rngsz configuration value
+ * @buf : buffer to hold the configured value
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the intr_rngsz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t set_intr_rngsz(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct xlnx_pci_dev *xpdev;
+ unsigned int rngsz = 0;
+ int err = 0;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ err = kstrtouint(buf, 0, &rngsz);
+ if (err < 0) {
+ pr_info("failed to set interrupt ring size\n");
+ return err;
+ }
+
+ err = qdma_set_intr_rngsz(xpdev->dev_hndl, (u32)rngsz);
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the qmax configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: qmax configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the qmax
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_qmax(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xlnx_pci_dev *xpdev;
+ int len;
+ unsigned int qmax = 0;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ qmax = qdma_get_qmax(xpdev->dev_hndl);
+ len = sprintf(buf, "%d\n", qmax);
+ if (len <= 0)
+ pr_err("copying qmax to buf failed with err: %d\n", len);
+
+ return len;
+}
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the qmax configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: qmax configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the qmax
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_qmax_vfs(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ int len;
+ int vf_qmax = qconf_get_qmax(PCI_TYPE_VF, 0);
+
+ len = sprintf(buf, "%d\n", vf_qmax);
+ if (len <= 0)
+ pr_err("copying qmax_vfs to buf failed with err: %d\n", len);
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_reconfigure_qmax() - handler to set the qmax configuration value
+ *
+ * @xpdev : PCIe device handle for
+ * @qmax_new : new value for qmax
+ * @conf_head: pointer to hold the device configuration
+ *
+ * Helper function to reconfigure qmax values using sysfs/qmax interface for
+ * the xlnx_pci_dev passed as parameter. Will change the starting of the
+ * subsequent PF Qs based on the modified qmax value of the xpdev. For
+ * preceeding devices, no changes will happen
+ *
+ * @note none
+ *
+ * Return: Returns number of devices succesfully configured on success,
+ * <0 on failure
+ *
+ *****************************************************************************/
+static int qdma_prep_reconfigure(struct xlnx_pci_dev *xpdev,
+ int qmax_new, struct qdma_dev_conf **conf_head)
+{
+ int reconfig_ok = 0;
+ unsigned int used_qs = 0;
+ unsigned int cfg_pending = 0;
+ unsigned int cfg_pending_qcnt = 0;
+ int err = 0;
+ int next_qbase = 0;
+ int total_pf = 0;
+ int count = 0;
+ struct xlnx_pci_dev *_xpdev, *tmp;
+ struct qdma_dev_conf *_conf_head = NULL;
+ struct qdma_dev_conf *conf_p = NULL;
+ int last_bus = 0;
+ int last_dev = 0;
+ int max_pf_qs = qconf_get_qmax(PCI_TYPE_PF, 0);
+
+ mutex_lock(&xpdev_mutex);
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+ struct qdma_dev_conf conf;
+ /** found a new card? */
+ if ((last_bus | last_dev) &&
+ ((last_bus != _xpdev->pdev->bus->number) ||
+ (last_dev != PCI_SLOT(_xpdev->pdev->devfn))))
+ break;
+
+ err = qdma_device_get_config(_xpdev->dev_hndl, &conf, NULL, 0);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%05x'\n",
+ _xpdev->idx);
+ reconfig_ok = err;
+ goto errout_reconfig;
+ }
+
+ if (conf.cur_cfg_state == QMAX_CFG_USER) {
+ if (xpdev != _xpdev)
+ used_qs += conf.qsets_max;
+ else {
+ if (qmax_new == conf.qsets_max) {
+ pr_info("WARN! New qmax(%d) same as current(%d)\n",
+ qmax_new,
+ conf.qsets_max);
+ reconfig_ok = 0;
+ goto errout_reconfig;
+ }
+ }
+ } else if (conf.cur_cfg_state == QMAX_CFG_INITIAL)
+ if (xpdev != _xpdev)
+ cfg_pending++;
+ total_pf++;
+ last_bus = _xpdev->pdev->bus->number;
+ last_dev = PCI_SLOT(_xpdev->pdev->devfn);
+ }
+ if ((used_qs + qmax_new) > max_pf_qs) {
+ pr_err("Not enough qs to allocate for qdma%05x\n",
+ _xpdev->idx);
+ pr_err("Requested size = %d, remaining size=%d\n",
+ qmax_new, max_pf_qs - used_qs);
+ reconfig_ok = -EINVAL;
+ goto errout_reconfig;
+ }
+
+ _conf_head = (struct qdma_dev_conf *)kzalloc(total_pf * sizeof(struct qdma_dev_conf),
+ GFP_KERNEL);
+ if (!_conf_head) {
+ reconfig_ok = -ENOMEM;
+ goto errout_reconfig;
+ }
+
+ if (cfg_pending)
+ cfg_pending_qcnt = (max_pf_qs - (used_qs + qmax_new))/cfg_pending;
+
+ count = total_pf;
+ conf_p = _conf_head;
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+ err = qdma_device_get_config(_xpdev->dev_hndl, conf_p, NULL, 0);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%05x'\n", _xpdev->idx);
+ reconfig_ok = -1;
+ goto errout_reconfig;
+ }
+
+ conf_p->qsets_base = next_qbase;
+ if (_xpdev == xpdev) {
+ conf_p->qsets_max = qmax_new;
+ conf_p->cur_cfg_state = QMAX_CFG_USER;
+ } else {
+ if (conf_p->cur_cfg_state == QMAX_CFG_INITIAL)
+ conf_p->qsets_max = cfg_pending_qcnt;
+ }
+ next_qbase = conf_p->qsets_max + conf_p->qsets_base;
+ /* if 0qs make qbase to 0 */
+ if (!conf_p->qsets_max)
+ conf_p->qsets_base = 0;
+ conf_p++;
+ /** we have to configure 'total_pf' PF devices */
+ if (!(count--))
+ break;
+ }
+
+ *conf_head = _conf_head;
+
+ reconfig_ok = total_pf;
+
+errout_reconfig:
+ mutex_unlock(&xpdev_mutex);
+
+ if ((reconfig_ok < 0) && (_conf_head != NULL))
+ kfree(_conf_head);
+
+ return reconfig_ok;
+}
+
+static int qdma_sysfs_reconfigure(void)
+{
+ int reconfig_ok = 0;
+ unsigned int qcnt = 0;
+ int err = 0;
+ int next_qbase = 0;
+ int total_pf = 0;
+ int cfgd_pf = 0;
+ int count = 0;
+ struct xlnx_pci_dev *_xpdev, *tmp;
+ int last_bus = 0;
+ int last_dev = 0;
+ int max_pf_qs = qconf_get_qmax(PCI_TYPE_PF, 0);
+ int total_qcnt_user = 0;
+ int qcnt_remaining = 0;
+
+ mutex_lock(&xpdev_mutex);
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+ struct qdma_dev_conf conf;
+ /** found a new card? */
+ if ((last_bus | last_dev) &&
+ ((last_bus != _xpdev->pdev->bus->number) ||
+ (last_dev != PCI_SLOT(_xpdev->pdev->devfn))))
+ break;
+
+ err = qdma_device_get_config(_xpdev->dev_hndl, &conf, NULL, 0);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%d'\n",
+ _xpdev->idx);
+ reconfig_ok = err;
+ goto errout_reconfig;
+ }
+
+ if (conf.cur_cfg_state == QMAX_CFG_USER) {
+ total_qcnt_user += conf.qsets_max;
+ } else
+ cfgd_pf++;
+
+ total_pf++;
+ last_bus = _xpdev->pdev->bus->number;
+ last_dev = PCI_SLOT(_xpdev->pdev->devfn);
+ }
+
+ qcnt_remaining = max_pf_qs - total_qcnt_user;
+ if (qcnt_remaining < 0) {
+ pr_err("Sysfs reconfiguration of vf-max failed!!!\n");
+ reconfig_ok = -EPERM;
+ goto errout_reconfig;
+ }
+
+ if (cfgd_pf)
+ qcnt = qcnt_remaining / cfgd_pf;
+
+ count = total_pf;
+
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+ struct qdma_dev_conf conf;
+
+ err = qdma_device_get_config(_xpdev->dev_hndl, &conf, NULL, 0);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%d'\n",
+ _xpdev->idx);
+ reconfig_ok = err;
+ goto errout_reconfig;
+ }
+
+ if (conf.cur_cfg_state == QMAX_CFG_INITIAL)
+ conf.qsets_max = qcnt;
+
+ if (conf.qsets_max)
+ conf.qsets_base = next_qbase;
+ else
+ conf.qsets_base = 0;
+ next_qbase += conf.qsets_max;
+
+ err = qdma_device_set_config(_xpdev->dev_hndl, &conf);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%d'\n", _xpdev->idx);
+ reconfig_ok = err;
+ goto errout_reconfig;
+ }
+
+ err = qdma_set_qmax(_xpdev->dev_hndl, conf.qsets_max, 1);
+ if (err < 0) {
+ pr_err("Failed to alter qmax for qdma%u %s max QP: %u, %u~%u\n",
+ _xpdev->idx, dev_name(&conf.pdev->dev),
+ conf.qsets_max, conf.qsets_base,
+ conf.qsets_base + conf.qsets_max - 1);
+ reconfig_ok = err;
+ goto errout_reconfig;
+ }
+
+ /** we have to configure 'total_pf' PF devices */
+ if (!(count--))
+ break;
+ }
+
+errout_reconfig:
+ mutex_unlock(&xpdev_mutex);
+
+ return reconfig_ok;
+}
+
+static int qdma_reconfigure_qmax(struct xlnx_pci_dev *xpdev,
+ struct qdma_dev_conf *conf_head)
+{
+ int err = 0, qmax = 0, count = 0;
+ struct xlnx_pci_dev *_xpdev, *tmp;
+ struct qdma_dev_conf *conf_p = conf_head;
+
+ if (!conf_head)
+ return -EINVAL;
+
+ mutex_lock(&xpdev_mutex);
+ /** start from the device we want to alter the qmax value
+ * TODO: failure will take the device to an inconsistant
+ * state and how to handle it.
+ */
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+
+ err = qdma_device_set_config(_xpdev->dev_hndl, conf_p);
+ if (err < 0) {
+ pr_err("Failed to get conf for qdma device '%05x'\n", _xpdev->idx);
+ break;
+ }
+ qmax = conf_p->qsets_max;
+ err = qdma_set_qmax(_xpdev->dev_hndl, qmax, 1);
+ if (err < 0) {
+ pr_err("Failed to alter qmax for qdma%05x %s max QP: %u, %u~%u\n",
+ _xpdev->idx, dev_name(&conf_p->pdev->dev),
+ conf_p->qsets_max, conf_p->qsets_base,
+ conf_p->qsets_base + conf_p->qsets_max - 1);
+ break;
+ }
+ if (_xpdev == xpdev)
+ qdma_device_set_cfg_state(_xpdev->dev_hndl, QMAX_CFG_USER);
+
+ conf_p++;
+ count++;
+ }
+ mutex_unlock(&xpdev_mutex);
+
+ if (!err)
+ pr_debug("Succesfully reconfigured '%d' devices\n", count);
+
+ kfree(conf_head);
+
+ return err;
+}
+
+static ssize_t set_qmax(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct xlnx_pci_dev *xpdev;
+ struct qdma_dev_conf *conf_head = NULL;
+ unsigned int qmax = 0;
+ int err = 0;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ err = kstrtouint(buf, 0, &qmax);
+ if (err < 0) {
+ pr_info("failed to set qmax to %d\n", qmax);
+ return err;
+ }
+
+ if (qmax > qconf_get_qmax(PCI_TYPE_PF, 0)) {
+ pr_err("Invalid qmax %u (max = %u)\n",
+ qmax,
+ qconf_get_qmax(PCI_TYPE_PF, 0));
+ return -EINVAL;
+ }
+
+ err = qdma_prep_reconfigure(xpdev, qmax, &conf_head);
+ if (!(err < 0))
+ err = qdma_reconfigure_qmax(xpdev, conf_head);
+
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the qmax_vf configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: qmax_vf configuration value
+ * @buf : buffer to hold the configured value
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the qmax_vf
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t set_qmax_vfs(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ unsigned int qmax = 0;
+ int err = 0;
+ int last_qmax = 0;
+
+ if (!is_vfqmax_configurable()) {
+ pr_err("Setting VF-qmax failed, VFs already registered.");
+ return -EPERM;
+ }
+
+ err = kstrtoint(buf, 0, &qmax);
+ if (err < 0) {
+ pr_err("Failed to set qmax VFs\n");
+ return err;
+ }
+
+ if (qmax > TOTAL_QDMA_QS) {
+ pr_err("Invalid vf qmax %d\n", qmax);
+ return -EINVAL;
+ }
+
+ last_qmax = set_vf_qmax(qmax);
+ err = qdma_sysfs_reconfigure();
+ if (err < 0)
+ set_vf_qmax(last_qmax);
+
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the cmpl_status_acc configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: cmpl_status_acc configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the cmpl_status_acc
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_cmpl_status_acc(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int len;
+ unsigned int cmpl_status_acc = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ cmpl_status_acc = qdma_get_cmpl_status_acc(xpdev->dev_hndl);
+ len = sprintf(buf, "%d\n", cmpl_status_acc);
+ if (len <= 0)
+ pr_err("copying cmpl status acc value to buf failed with err: %d\n", len);
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the cmpl_status_acc configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: cmpl_status_acc configuration value
+ * @buf : buffer to hold the configured value
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the cmpl_status_acc
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t set_cmpl_status_acc(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ unsigned int cmpl_status_acc = 0;
+ int err = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ err = kstrtoint(buf, 0, &cmpl_status_acc);
+ if (err < 0) {
+ pr_err("failed to set cmpl status accumulator to %d\n", cmpl_status_acc);
+ return err;
+ }
+
+ err = qdma_set_cmpl_status_acc(xpdev->dev_hndl, cmpl_status_acc);
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the buf_sz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: buf_sz configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the buf_sz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+
+static ssize_t show_c2h_buf_sz(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int len = 0;
+ int i;
+ unsigned int c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ qdma_get_buf_sz(xpdev->dev_hndl, c2h_buf_sz);
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++)
+ len += sprintf(buf + len, "%hu ", c2h_buf_sz[i]);
+ len += sprintf(buf + len, "%s", "\n");
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the buf_sz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: buf_sz configuration value
+ * @buf : buffer to hold the configured value
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the buf_sz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+
+static ssize_t set_c2h_buf_sz(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int err = 0;
+ char *s = (char *)buf, *p = NULL;
+ const char *tc = " "; /* token character here is a "space" */
+ unsigned int c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+ int i = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ /* First read the values in to the register
+ ** This helps to restore the values of entries if
+ ** user configures lesser than 16 values
+ */
+ qdma_get_buf_sz(xpdev->dev_hndl, c2h_buf_sz);
+
+ while ((p = strsep(&s, tc)) != NULL && i < QDMA_GLOBAL_CSR_ARRAY_SZ) {
+ if (*p == 0)
+ continue;
+
+ err = kstrtoint(p, 0, &c2h_buf_sz[i]);
+ if (err < 0)
+ goto input_err;
+
+ /* Check if buf_sz is configured as power of 2 */
+ if (!(c2h_buf_sz[i]) || ((c2h_buf_sz[i]) & (c2h_buf_sz[i]-1))) {
+ pr_warn("buf_sz at index %d is not power of 2\n", i);
+ err = -EINVAL;
+ goto input_err;
+ }
+ i++;
+ }
+
+ if (p) {
+ /*
+ check if the number of entries are more than 16 !
+ if yes, ignore the extra values
+ */
+ pr_warn("Found more than 16 entries. Ignoring extra entries\n");
+ }
+
+ err = qdma_set_buf_sz(xpdev->dev_hndl, c2h_buf_sz);
+
+input_err:
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show the ring_sz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: ring_sz configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the ring_sz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+
+static ssize_t show_glbl_rng_sz(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int len = 0;
+ int i;
+ unsigned int glbl_ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ qdma_get_glbl_rng_sz(xpdev->dev_hndl, glbl_ring_sz);
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++)
+ len += sprintf(buf + len, "%hu ", glbl_ring_sz[i]);
+ len += sprintf(buf + len, "%s", "\n");
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the glbl_ring_sz configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: buf_sz configuration value
+ * @buf : buffer to hold the configured value
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the glbl_ring_sz
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+
+static ssize_t set_glbl_rng_sz(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int err = 0;
+ char *s = (char *)buf, *p = NULL;
+ const char *tc = " "; /* token character here is a "space" */
+ unsigned int glbl_ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+ int i = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ /* First read the values in to the register
+ ** This helps to restore the values of entries if
+ ** user configures lesser than 16 values
+ */
+ qdma_get_glbl_rng_sz(xpdev->dev_hndl, glbl_ring_sz);
+
+ while ((p = strsep(&s, tc)) != NULL && i < QDMA_GLOBAL_CSR_ARRAY_SZ) {
+ if (*p == 0)
+ continue;
+
+ err = kstrtoint(p, 0, &glbl_ring_sz[i]);
+ if (err < 0)
+ goto input_err;
+
+ /* Check if rng_sz is configured as power of 2 */
+ if ((glbl_ring_sz[i]) & (glbl_ring_sz[i]-1)) {
+ pr_warn("glbl_ring_sz at index %d is not power of 2\n", i);
+ err = -EINVAL;
+ goto input_err;
+ }
+
+ /* Always add one for the completion status */
+ glbl_ring_sz[i] += 1;
+
+ i++;
+ }
+
+ if (p) {
+ /*
+ check if the number of entries are more than 16 !
+ if yes, ignore the extra values
+ */
+ pr_warn("Found more than 16 entries. Ignoring extra entries\n");
+ }
+
+ err = qdma_set_glbl_rng_sz(xpdev->dev_hndl, glbl_ring_sz);
+
+input_err:
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show global csr c2h_timer_cnt configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: c2h_timer_cnt configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the global csr c2h_timer_cnt
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_c2h_timer_cnt(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int len = 0;
+ int i;
+ unsigned int c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ qdma_get_timer_cnt(xpdev->dev_hndl, c2h_timer_cnt);
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++)
+ len += sprintf(buf + len, "%hhu ", c2h_timer_cnt[i]);
+ len += sprintf(buf + len, "%s", "\n");
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * set_c2h_timer_cnt() - handler to set global csr c2h_timer_cnt config
+ *
+ * @dev : PCIe device handle
+ * @attr: c2h_timer_cnt configuration value
+ * @buf : buffer containing new configuration
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the global csr c2h_timer_cnt
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t set_c2h_timer_cnt(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int err = 0;
+ char *s = (char *)buf, *p = NULL;
+ const char *tc = " "; /* token character here is a "space" */
+ unsigned int c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+ int i = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ /* First read the values in to the register
+ ** This helps to restore the values of entries if
+ ** user configures lesser than 16 values
+ */
+ qdma_get_timer_cnt(xpdev->dev_hndl, c2h_timer_cnt);
+
+ while ((p = strsep(&s, tc)) != NULL && i < QDMA_GLOBAL_CSR_ARRAY_SZ) {
+ if (*p == 0)
+ continue;
+
+ err = kstrtoint(p, 0, &c2h_timer_cnt[i]);
+ if (err < 0)
+ goto input_err;
+
+ if (c2h_timer_cnt[i] < 0 || c2h_timer_cnt[i] > 255) {
+ pr_warn("timer cnt at index %d is %d - out of range [0-255]\n",
+ i, c2h_timer_cnt[i]);
+ err = -EINVAL;
+ goto input_err;
+ }
+ i++;
+ }
+
+ if (p) {
+ /*
+ check if the number of entries are more than 16 !
+ if yes, ignore the extra values
+ */
+ pr_warn("Found more than 16 entries. Ignoring extra entries\n");
+ }
+ err = qdma_set_timer_cnt(xpdev->dev_hndl, c2h_timer_cnt);
+
+input_err:
+ return err ? err : count;
+}
+
+/*****************************************************************************/
+/**
+ * funcname() - handler to show global csr c2h_cnt_th_ configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: c2h_cnt_th configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to show the global csr c2h_cnt_th
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t show_c2h_cnt_th(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int len = 0;
+ int i;
+ unsigned int c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ qdma_get_cnt_thresh(xpdev->dev_hndl, c2h_cnt_th);
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++)
+ len += sprintf(buf + len, "%hhu ", c2h_cnt_th[i]);
+ len += sprintf(buf + len, "%s", "\n");
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * set_c2h_cnt_th() - handler to set global csr c2h_cnt_th configuration
+ *
+ * @dev : PCIe device handle
+ * @attr: c2h_cnt_th configuration value
+ * @buf : buffer containing new configuration
+ * @count : the number of bytes of data in the buffer
+ *
+ * Handler function to set the global csr c2h_cnt_th
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+static ssize_t set_c2h_cnt_th(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *xpdev;
+ int err = 0;
+ char *s = (char *)buf, *p = NULL;
+ const char *tc = " "; /* token character here is a "space" */
+ unsigned int c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ] = {0};
+ int i = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!xpdev)
+ return -EINVAL;
+
+ /* First read the values in to the register
+ ** This helps to restore the values of entries if
+ ** user configures lesser than 16 values
+ */
+ qdma_get_cnt_thresh(xpdev->dev_hndl, c2h_cnt_th);
+
+ while ((p = strsep(&s, tc)) != NULL && i < QDMA_GLOBAL_CSR_ARRAY_SZ) {
+ if (*p == 0)
+ continue;
+
+ err = kstrtoint(p, 0, &c2h_cnt_th[i]);
+ if (err < 0)
+ goto input_err;
+
+ if (c2h_cnt_th[i] < 0 || c2h_cnt_th[i] > 255) {
+ pr_warn("counter threshold at index %d is %d - out of range [0-255]\n",
+ i, c2h_cnt_th[i]);
+ err = -EINVAL;
+ goto input_err;
+ }
+ i++;
+ }
+
+ if (p) {
+ /*
+ check if the number of entries are more than 16 !
+ if yes, ignore the extra values
+ */
+ pr_warn("Found more than 16 entries. Ignoring extra entries\n");
+ }
+ err = qdma_set_cnt_thresh(xpdev->dev_hndl, c2h_cnt_th);
+
+input_err:
+ return err ? err : count;
+}
+
+/**
+ * Function to find the first PF device available in the card
+ */
+static bool is_first_pfdev(u32 bdf)
+{
+ struct xlnx_pci_dev *_xpdev, *tmp;
+ u32 bdf_tmp = 0;
+
+ mutex_lock(&xpdev_mutex);
+ if (list_empty(&xpdev_list)) {
+ mutex_unlock(&xpdev_mutex);
+ return 1;
+ }
+
+ list_for_each_entry_safe(_xpdev, tmp, &xpdev_list, list_head) {
+ struct pci_dev *pdev = _xpdev->pdev;
+ bdf_tmp = (pdev->bus->number << PCI_SHIFT_BUS)
+ | (PCI_SLOT(pdev->devfn) << PCI_SHIFT_DEV)
+ | PCI_FUNC(pdev->devfn);
+ /** find first bus and device are matching */
+ if ((bdf & ~0xf) == (bdf_tmp & ~0xf)) {
+ mutex_unlock(&xpdev_mutex);
+ /** if func matches, it returns 1, else 0*/
+ return (PCI_FUNC(pdev->devfn) == (bdf & 0xf));
+ }
+ }
+ mutex_unlock(&xpdev_mutex);
+
+ return 0;
+}
+
+/**
+ * function to find out master pf, if the parameter master_pf is passed.
+ * By default it will be PF0
+ */
+static u8 is_master_pf(struct pci_dev *pdev)
+{
+ int i = 0;
+ u32 bdf = (pdev->bus->number << PCI_SHIFT_BUS)
+ | (PCI_SLOT(pdev->devfn) << PCI_SHIFT_DEV)
+ | PCI_FUNC(pdev->devfn);
+ /**
+ * no arguments are passed for master_pf, default to PF0
+ * arguments passed but bus and device not in list,
+ * default to PF0
+ */
+ if (!master_cnt) {
+ if (is_first_pfdev(bdf))
+ return 1;
+ } else {
+ for (i = 0; i < master_cnt; i++) {
+ if ((bdf & ~0xf) == (master_pf[i] & ~0xf)) {
+ if ((master_pf[i] & 0xf) == (bdf & 0xf))
+ return 1;
+ else
+ return 0;
+ }
+ }
+
+ if (is_first_pfdev(bdf))
+ return 1;
+ }
+
+ return 0;
+}
+#else /** For VF #ifdef __QDMA_VF__ */
+/*****************************************************************************/
+/**
+ * funcname() - handler to set the qmax configuration value
+ *
+ * @dev : PCIe device handle
+ * @attr: qmax configuration value
+ * @buf : buffer to hold the configured value
+ *
+ * Handler function to set the qmax
+ *
+ * @note none
+ *
+ * Return: Returns length of the buffer on success, <0 on failure
+ *
+ *****************************************************************************/
+
+static ssize_t set_qmax(struct device *dev,
+ struct device_attribute *attr, const char *buf, size_t count)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ struct xlnx_pci_dev *_xpdev;
+ unsigned int qmax = 0;
+ int err = 0;
+
+ if (!pdev)
+ return -EINVAL;
+
+ _xpdev = (struct xlnx_pci_dev *)dev_get_drvdata(dev);
+ if (!_xpdev)
+ return -EINVAL;
+ err = kstrtoint(buf, 0, &qmax);
+ if (err < 0) {
+ pr_err("Failed to set qmax\n");
+ return err;
+ }
+
+
+ if (qmax > TOTAL_QDMA_QS) {
+ pr_err("Invalid vf qmax %d\n", qmax);
+ return -EINVAL;
+ }
+
+ err = qdma_vf_qconf(_xpdev->dev_hndl, qmax);
+ if (err < 0)
+ return err;
+
+ if (!err)
+ pr_debug("Succesfully reconfigured qdmavf%05x\n", _xpdev->idx);
+
+ return err ? err : count;
+}
+#endif
+
+static DEVICE_ATTR(qmax, S_IWUSR | S_IRUGO, show_qmax, set_qmax);
+static DEVICE_ATTR(intr_rngsz, S_IWUSR | S_IRUGO,
+ show_intr_rngsz, set_intr_rngsz);
+#ifndef __QDMA_VF__
+static DEVICE_ATTR(buf_sz, S_IWUSR | S_IRUGO,
+ show_c2h_buf_sz, set_c2h_buf_sz);
+static DEVICE_ATTR(glbl_rng_sz, S_IWUSR | S_IRUGO,
+ show_glbl_rng_sz, set_glbl_rng_sz);
+static DEVICE_ATTR(c2h_timer_cnt, S_IWUSR | S_IRUGO,
+ show_c2h_timer_cnt, set_c2h_timer_cnt);
+static DEVICE_ATTR(c2h_cnt_th, S_IWUSR | S_IRUGO,
+ show_c2h_cnt_th, set_c2h_cnt_th);
+static DEVICE_ATTR(cmpl_status_acc, S_IWUSR | S_IRUGO, show_cmpl_status_acc, set_cmpl_status_acc);
+static DEVICE_ATTR(qmax_vfs, S_IWUSR | S_IRUGO, show_qmax_vfs, set_qmax_vfs);
+#endif
+
+static struct attribute *pci_device_attrs[] = {
+ &dev_attr_qmax.attr,
+ &dev_attr_intr_rngsz.attr,
+ NULL,
+};
+
+static struct attribute *pci_master_device_attrs[] = {
+ &dev_attr_qmax.attr,
+ &dev_attr_intr_rngsz.attr,
+#ifndef __QDMA_VF__
+ &dev_attr_qmax_vfs.attr,
+ &dev_attr_buf_sz.attr,
+ &dev_attr_glbl_rng_sz.attr,
+ &dev_attr_c2h_timer_cnt.attr,
+ &dev_attr_c2h_cnt_th.attr,
+ &dev_attr_cmpl_status_acc.attr,
+#endif
+ NULL,
+};
+
+static struct attribute_group pci_device_attr_group = {
+ .name = "qdma",
+ .attrs = pci_device_attrs,
+
+};
+
+static struct attribute_group pci_master_device_attr_group = {
+ .name = "qdma",
+ .attrs = pci_master_device_attrs,
+
+};
+
+static inline void xpdev_list_remove(struct xlnx_pci_dev *xpdev)
+{
+ mutex_lock(&xpdev_mutex);
+ list_del(&xpdev->list_head);
+ mutex_unlock(&xpdev_mutex);
+}
+
+static inline void xpdev_list_add(struct xlnx_pci_dev *xpdev)
+{
+ mutex_lock(&xpdev_mutex);
+ list_add_tail(&xpdev->list_head, &xpdev_list);
+ mutex_unlock(&xpdev_mutex);
+}
+
+int xpdev_list_dump(char *buf, int buflen)
+{
+ struct xlnx_pci_dev *xpdev, *tmp;
+ char *cur = buf;
+ char *const end = buf + buflen;
+ int base_end = 0;
+ int qmax_val = 0;
+
+ if (!buf || !buflen)
+ return -EINVAL;
+
+ mutex_lock(&xpdev_mutex);
+ list_for_each_entry_safe(xpdev, tmp, &xpdev_list, list_head) {
+ struct pci_dev *pdev;
+ struct qdma_dev_conf conf;
+ int rv;
+
+ rv = qdma_device_get_config(xpdev->dev_hndl, &conf, NULL, 0);
+ if (rv < 0) {
+ cur += snprintf(cur, cur - end,
+ "ERR! unable to get device config for idx %05x\n",
+ xpdev->idx);
+ if (cur >= end)
+ goto handle_truncation;
+ break;
+ }
+
+ pdev = conf.pdev;
+
+ base_end = (int)(conf.qsets_base + conf.qsets_max - 1);
+ if (base_end < 0)
+ base_end = 0;
+#ifdef __QDMA_VF__
+ if ((conf.qsets_base == 0)
+ && (conf.qsets_max == 1)
+ && (conf.cur_cfg_state == QMAX_CFG_INITIAL))
+ qmax_val = 0;
+ else
+ qmax_val = conf.qsets_max;
+#else
+ qmax_val = conf.qsets_max;
+#endif
+
+ cur += snprintf(cur, end - cur,
+#ifdef __QDMA_VF__
+ "qdmavf%05x\t%s\tmax QP: %u, %u~%u\n",
+#else
+ "qdma%05x\t%s\tmax QP: %u, %u~%u\n",
+#endif
+ xpdev->idx, dev_name(&pdev->dev),
+ qmax_val, conf.qsets_base,
+ base_end);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ mutex_unlock(&xpdev_mutex);
+
+ return cur - buf;
+
+handle_truncation:
+ mutex_unlock(&xpdev_mutex);
+ pr_warn("ERR! str truncated. req=%lu, avail=%u", cur - buf, buflen);
+ *buf = '\0';
+ return cur - buf;
+}
+
+static void xpdev_unmap_user_bypass_bars(struct xlnx_pci_dev *xpdev)
+{
+ if (xpdev->user_bar_regs) {
+ /* unmap BAR */
+ pci_iounmap(xpdev->pdev, xpdev->user_bar_regs);
+ /* mark as unmapped */
+ xpdev->user_bar_regs = NULL;
+ }
+
+ if (xpdev->bypass_bar_regs) {
+ /* unmap BAR */
+ pci_iounmap(xpdev->pdev, xpdev->bypass_bar_regs);
+ /* mark as unmapped */
+ xpdev->bypass_bar_regs = NULL;
+ }
+
+}
+
+/*****************************************************************************/
+/**
+ * identify_config_bar() - Function to identify the config bar of the PCIE device
+ *
+ * @param[in] pdev: pcie device
+ *
+ * @return config bar
+ *****************************************************************************/
+static u8 identify_config_bar(struct pci_dev *pdev)
+{
+ int i = 0;
+ u8 bus_num = pdev->bus->number;
+#ifndef __QDMA_VF__
+ u8 dev_fn = PCI_FUNC(pdev->devfn);
+#else
+ u16 device_id = pdev->device;
+#endif
+#ifndef __QDMA_VF__
+ if (config_bar_cnt) {
+ for (i = 0; i < MAX_DMA_DEV; i++) {
+ if (((config_bar[i] & BUS_NUM_MASK) >> BUS_NUM_SHIFT) == bus_num) {
+ switch (dev_fn) {
+ case PF_DEVICE_0:
+ return ((config_bar[i] & PF_DEV_0_MASK)
+ >> PF_DEV_0_SHIFT);
+ case PF_DEVICE_1:
+ return ((config_bar[i] & PF_DEV_1_MASK)
+ >> PF_DEV_1_SHIFT);
+ case PF_DEVICE_2:
+ return ((config_bar[i] & PF_DEV_2_MASK)
+ >> PF_DEV_2_SHIFT);
+ case PF_DEVICE_3:
+ return ((config_bar[i] & PF_DEV_3_MASK)
+ >> PF_DEV_3_SHIFT);
+ }
+ }
+ }
+ }
+
+ return QDMA_CONFIG_BAR;
+#else
+ if (config_bar_cnt) {
+ for (i = 0; i < MAX_DMA_DEV; i++) {
+ if (((config_bar[i] & BUS_NUM_MASK) >> BUS_NUM_SHIFT) == bus_num) {
+ switch ((device_id >> VF_PF_IDENTIFIER_SHIFT) &
+ VF_PF_IDENTIFIER_MASK) {
+ case PF_DEVICE_0:
+ return ((config_bar[i] & PF_DEV_0_MASK)
+ >> PF_DEV_0_SHIFT);
+ case PF_DEVICE_1:
+ return ((config_bar[i] & PF_DEV_1_MASK)
+ >> PF_DEV_1_SHIFT);
+ case PF_DEVICE_2:
+ return ((config_bar[i] & PF_DEV_2_MASK)
+ >> PF_DEV_2_SHIFT);
+ case PF_DEVICE_3:
+ return ((config_bar[i] & PF_DEV_3_MASK)
+ >> PF_DEV_3_SHIFT);
+ }
+ }
+ }
+ }
+
+ return QDMA_CONFIG_BAR;
+#endif
+}
+
+static int xpdev_map_user_bypass_bars(struct xlnx_pci_dev *xpdev)
+{
+ struct xlnx_dma_dev *xdev = NULL;
+ int map_len;
+
+ if (!xpdev)
+ return -EINVAL;
+
+ xdev = (struct xlnx_dma_dev *)(xpdev->dev_hndl);
+
+ if (xdev->conf.bar_num_user < 0) {
+ pr_err("User and bypass bar is not present \n");
+ return -EINVAL;
+ }
+
+ map_len = pci_resource_len(xpdev->pdev, (int)xdev->conf.bar_num_user);
+ if (map_len > QDMA_MAX_BAR_LEN_MAPPED)
+ map_len = QDMA_MAX_BAR_LEN_MAPPED;
+
+ xpdev->user_bar_regs = pci_iomap(xpdev->pdev, xdev->conf.bar_num_user, map_len);
+ if (!xpdev->user_bar_regs) {
+ pr_err("unable to map user bar %d.\n", xdev->conf.bar_num_user);
+ return -EINVAL;
+ }
+
+ if (xdev->conf.bar_num_bypass >= 0) {
+ /* Mapping bypass bar */
+ map_len = pci_resource_len(xpdev->pdev, (int)xdev->conf.bar_num_bypass);
+ if (map_len > QDMA_MAX_BAR_LEN_MAPPED)
+ map_len = QDMA_MAX_BAR_LEN_MAPPED;
+
+ xpdev->bypass_bar_regs = pci_iomap(xpdev->pdev, xdev->conf.bar_num_bypass, map_len);
+ if (!xpdev->bypass_bar_regs) {
+ pr_err("unable to map bypass bar %d.\n", xdev->conf.bar_num_bypass);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+struct xlnx_pci_dev *xpdev_find_by_idx(unsigned int idx, char *buf, int buflen)
+{
+ struct xlnx_pci_dev *xpdev, *tmp;
+
+ mutex_lock(&xpdev_mutex);
+ list_for_each_entry_safe(xpdev, tmp, &xpdev_list, list_head) {
+ if (xpdev->idx == idx) {
+ mutex_unlock(&xpdev_mutex);
+ return xpdev;
+ }
+ }
+ mutex_unlock(&xpdev_mutex);
+
+ if (buf && buflen)
+ snprintf(buf, buflen, "NO device found at index %05x!\n", idx);
+
+ return NULL;
+}
+
+struct xlnx_qdata *xpdev_queue_get(struct xlnx_pci_dev *xpdev,
+ unsigned int qidx, bool c2h, bool check_qhndl,
+ char *ebuf, int ebuflen)
+{
+ struct xlnx_qdata *qdata;
+
+ if (qidx >= xpdev->qmax) {
+ pr_debug("qdma%05x QID %u too big, %05x.\n",
+ xpdev->idx, qidx, xpdev->qmax);
+ if (ebuf && ebuflen) {
+ snprintf(ebuf, ebuflen, "QID %u too big, %u.\n",
+ qidx, xpdev->qmax);
+ }
+ return NULL;
+ }
+
+ qdata = xpdev->qdata + qidx;
+ if (c2h)
+ qdata += xpdev->qmax;
+
+ if (check_qhndl && (!qdata->qhndl && !qdata->xcdev)) {
+ pr_debug("qdma%05x QID %u NOT configured.\n", xpdev->idx, qidx);
+ if (ebuf && ebuflen) {
+ snprintf(ebuf, ebuflen, "QID %u NOT configured.\n", qidx);
+ }
+
+ return NULL;
+ }
+
+ return qdata;
+}
+
+int xpdev_queue_delete(struct xlnx_pci_dev *xpdev, unsigned int qidx, bool c2h,
+ char *ebuf, int ebuflen)
+{
+ struct xlnx_qdata *qdata = xpdev_queue_get(xpdev, qidx, c2h, 1, ebuf,
+ ebuflen);
+ int rv = 0;
+
+ if (!qdata)
+ return -EINVAL;
+
+ if(!qdata->xcdev)
+ return -EINVAL;
+
+ spin_lock(&xpdev->cdev_lock);
+ qdata->xcdev->dir_init &= ~(1 << (c2h ? 1 : 0));
+ if (qdata->xcdev && !qdata->xcdev->dir_init)
+ qdma_cdev_destroy(qdata->xcdev);
+ spin_unlock(&xpdev->cdev_lock);
+
+ if (qdata->qhndl != QDMA_QUEUE_IDX_INVALID)
+ rv = qdma_queue_remove(xpdev->dev_hndl, qdata->qhndl,
+ ebuf, ebuflen);
+ else
+ pr_debug("qidx %u/%u, c2h %d, qhndl invalid.\n",
+ qidx, xpdev->qmax, c2h);
+ if (rv < 0)
+ goto exit;
+
+ memset(qdata, 0, sizeof(*qdata));
+exit:
+ return rv;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+static void xpdev_queue_delete_all(struct xlnx_pci_dev *xpdev)
+{
+ int i;
+
+ for (i = 0; i < xpdev->qmax; i++) {
+ xpdev_queue_delete(xpdev, i, 0, NULL, 0);
+ xpdev_queue_delete(xpdev, i, 1, NULL, 0);
+ }
+}
+#endif
+
+int xpdev_queue_add(struct xlnx_pci_dev *xpdev, struct qdma_queue_conf *qconf,
+ char *ebuf, int ebuflen)
+{
+ struct xlnx_qdata *qdata;
+ struct qdma_cdev *xcdev;
+ struct xlnx_qdata *qdata_tmp;
+ struct qdma_dev_conf dev_config;
+ u8 dir;
+ unsigned long qhndl;
+ int rv;
+
+ rv = qdma_queue_add(xpdev->dev_hndl, qconf, &qhndl, ebuf, ebuflen);
+ if (rv < 0)
+ return rv;
+
+ pr_debug("qdma%05x idx %u, st %d, c2h %d, added, qhndl 0x%lx.\n",
+ xpdev->idx, qconf->qidx, qconf->st, qconf->c2h, qhndl);
+
+ qdata = xpdev_queue_get(xpdev, qconf->qidx, qconf->c2h, 0, ebuf,
+ ebuflen);
+ if (!qdata) {
+ pr_info("q added 0x%lx, get failed, idx 0x%x.\n",
+ qhndl, qconf->qidx);
+ return rv;
+ }
+
+ dir = qconf->c2h ? 0 : 1;
+ spin_lock(&xpdev->cdev_lock);
+ qdata_tmp = xpdev_queue_get(xpdev, qconf->qidx, dir, 0, NULL, 0);
+ if (qdata_tmp) {
+ /* only 1 cdev per queue pair */
+ if (qdata_tmp->xcdev) {
+ unsigned long *priv_data;
+
+ qdata->qhndl = qhndl;
+ qdata->xcdev = qdata_tmp->xcdev;
+ priv_data = qconf->c2h ? &qdata->xcdev->c2h_qhndl :
+ &qdata->xcdev->h2c_qhndl;
+ *priv_data = qhndl;
+ qdata->xcdev->dir_init |= (1 << qconf->c2h);
+
+ spin_unlock(&xpdev->cdev_lock);
+ return 0;
+ }
+ }
+ spin_unlock(&xpdev->cdev_lock);
+
+ rv = qdma_device_get_config(xpdev->dev_hndl, &dev_config, NULL, 0);
+ if (rv < 0) {
+ pr_err("Failed to get conf for qdma device '%05x'\n",
+ xpdev->idx);
+ return rv;
+ }
+
+ /* always create the cdev
+ * Give HW QID as minor number with qsets_base calculation */
+ rv = qdma_cdev_create(&xpdev->cdev_cb, xpdev->pdev, qconf,
+ (dev_config.qsets_base + qconf->qidx),
+ qhndl, &xcdev, ebuf, ebuflen);
+
+ qdata->qhndl = qhndl;
+ qdata->xcdev = xcdev;
+
+ return rv;
+}
+
+static void nl_work_handler_q_start(struct work_struct *work)
+{
+ struct xlnx_nl_work *nl_work = container_of(work, struct xlnx_nl_work,
+ work);
+ struct xlnx_pci_dev *xpdev = nl_work->xpdev;
+ struct xlnx_nl_work_q_ctrl *qctrl = &nl_work->qctrl;
+ char ebuf[XNL_EBUFLEN];
+ unsigned int qidx = qctrl->qidx;
+ u8 is_qp = qctrl->is_qp;
+ u8 c2h = qctrl->is_c2h;
+ int i;
+
+ for (i = 0; i < qctrl->qcnt; i++, qidx++) {
+ int rv;
+ struct xlnx_qdata *qdata;
+
+q_start:
+ qdata = xpdev_queue_get(xpdev, qidx, c2h, 1, ebuf, XNL_EBUFLEN);
+ if (!qdata) {
+ pr_info("%s, idx %u, c2h %u, get failed.\n",
+ dev_name(&xpdev->pdev->dev), qidx, c2h);
+ snprintf(ebuf, XNL_EBUFLEN,
+ "Q idx %u, c2h %u, get failed.\n", qidx, c2h);
+ goto send_resp;
+ }
+
+ rv = qdma_queue_start(xpdev->dev_hndl, qdata->qhndl, ebuf,
+ XNL_EBUFLEN);
+ if (rv < 0) {
+ pr_info("%s, idx %u, c2h %u, start failed %d.\n",
+ dev_name(&xpdev->pdev->dev), qidx, c2h, rv);
+ snprintf(ebuf, XNL_EBUFLEN,
+ "Q idx %u, c2h %u, start failed %d.\n",
+ qidx, c2h, rv);
+ goto send_resp;
+ }
+
+#ifndef __QDMA_VF__
+ {
+ struct xlnx_dma_dev *xdev =
+ (struct xlnx_dma_dev *)(xpdev->dev_hndl);
+
+ if (xdev->stm_en) {
+ rv = qdma_queue_prog_stm(xpdev->dev_hndl,
+ qdata->qhndl,
+ ebuf, XNL_EBUFLEN);
+ if (rv < 0) {
+ pr_info("%s, idx %u, c2h %u, prog stm failed %d.\n",
+ dev_name(&xpdev->pdev->dev),
+ qidx, c2h, rv);
+ snprintf(ebuf, XNL_EBUFLEN,
+ "Q idx %u, c2h %u, prog stm failed %d.\n",
+ qidx, c2h, rv);
+ goto send_resp;
+ }
+ }
+ }
+#endif
+
+ if (is_qp && c2h == qctrl->is_c2h) {
+ c2h = !qctrl->is_c2h;
+ goto q_start;
+ }
+
+ c2h = qctrl->is_c2h;
+ }
+
+ snprintf(ebuf, XNL_EBUFLEN,
+ "%u Queues started, idx %u ~ %u.\n",
+ qctrl->qcnt, qctrl->qidx, qidx - 1);
+
+send_resp:
+ spin_lock(&nl_work->lock);
+ nl_work->q_start_handled = 1;
+ spin_unlock(&nl_work->lock);
+ xnl_respond_buffer(&nl_work->nl_info, ebuf, strlen(ebuf));
+ wake_up_interruptible(&nl_work->wq);
+}
+
+static struct xlnx_nl_work *xpdev_nl_work_alloc(struct xlnx_pci_dev *xpdev,
+ struct genl_info *nl_info)
+{
+ struct xlnx_nl_work *nl_work;
+
+ /* allocate work struct */
+ nl_work = kmalloc(sizeof(*nl_work), GFP_ATOMIC);
+ if (!nl_work) {
+ pr_info("qdma%05x %s: OOM.\n",
+ xpdev->idx, dev_name(&xpdev->pdev->dev));
+ return NULL;
+ }
+
+ memcpy(&nl_work->nl_info, nl_info, sizeof(*nl_info));
+ nl_work->xpdev = xpdev;
+
+ return nl_work;
+}
+
+int xpdev_nl_queue_start(struct xlnx_pci_dev *xpdev, void *nl_info, u8 is_qp,
+ u8 is_c2h, unsigned short qidx, unsigned short qcnt)
+{
+ struct xlnx_nl_work *nl_work = xpdev_nl_work_alloc(xpdev,
+ (struct genl_info *)nl_info);
+ struct xlnx_nl_work_q_ctrl *qctrl = &nl_work->qctrl;
+
+ if (!nl_work)
+ return -ENOMEM;
+
+ qctrl->is_qp = is_qp;
+ qctrl->is_c2h = is_c2h;
+ qctrl->qidx = qidx;
+ qctrl->qcnt = qcnt;
+
+ INIT_WORK(&nl_work->work, nl_work_handler_q_start);
+ init_waitqueue_head(&nl_work->wq);
+ spin_lock_init(&nl_work->lock);
+ nl_work->q_start_handled = 0;
+ spin_lock(&nl_work->lock);
+ queue_work(xpdev->nl_task_wq, &nl_work->work);
+ spin_unlock(&nl_work->lock);
+ wait_event_interruptible(nl_work->wq, nl_work->q_start_handled);
+ kfree(nl_work);
+
+ return 0;
+}
+
+static void xpdev_free(struct xlnx_pci_dev *p)
+{
+ xpdev_list_remove(p);
+
+ if (p->nl_task_wq)
+ destroy_workqueue(p->nl_task_wq);
+
+ if (((unsigned long)p) >= VMALLOC_START &&
+ ((unsigned long)p) < VMALLOC_END)
+ vfree(p);
+ else
+ kfree(p);
+}
+
+static struct xlnx_pci_dev *xpdev_alloc(struct pci_dev *pdev, unsigned int qmax)
+{
+ int sz = sizeof(struct xlnx_pci_dev) +
+ qmax * 2 * sizeof(struct xlnx_qdata);
+ struct xlnx_pci_dev *xpdev = kzalloc(sz, GFP_KERNEL);
+ char name[80];
+
+ if (!xpdev) {
+ xpdev = vmalloc(sz);
+ if (xpdev)
+ memset(xpdev, 0, sz);
+ }
+
+ if (!xpdev) {
+ pr_info("OMM, qmax %u, sz %u.\n", qmax, sz);
+ return NULL;
+ }
+ spin_lock_init(&xpdev->cdev_lock);
+ xpdev->pdev = pdev;
+ xpdev->qmax = qmax;
+ xpdev->idx = 0xFF;
+
+ snprintf(name, 80, "qdma_%s_nl_wq", dev_name(&pdev->dev));
+ xpdev->nl_task_wq = create_singlethread_workqueue(name);
+ if (!xpdev->nl_task_wq) {
+ pr_info("%s OOM, nl_task_wq.\n", dev_name(&pdev->dev));
+ goto free_xpdev;
+ }
+
+ xpdev_list_add(xpdev);
+ return xpdev;
+
+free_xpdev:
+ xpdev_free(xpdev);
+ return NULL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_user_register() - read user bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the user bar register
+ *****************************************************************************/
+unsigned int qdma_device_read_user_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr)
+{
+ if (!xpdev)
+ return -EINVAL;
+
+ return readl(xpdev->user_bar_regs + reg_addr);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_user_register() - write user bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ * @param[in] val: register value to be written
+ *
+ *****************************************************************************/
+void qdma_device_write_user_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr, unsigned int val)
+{
+ if (!xpdev)
+ return;
+
+ writel(val, xpdev->user_bar_regs + reg_addr);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_bypass_register() - read bypass bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the bypass bar register
+ *****************************************************************************/
+unsigned int qdma_device_read_bypass_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr)
+{
+ if (!xpdev)
+ return -EINVAL;
+
+ if (!xpdev->bypass_bar_regs) {
+ pr_err("bypass bar is not present\n");
+ return -EINVAL;
+ }
+
+ return readl(xpdev->bypass_bar_regs + reg_addr);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_bypass_register() - write bypass bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ * @param[in] val: register value to be written
+ *
+ *****************************************************************************/
+void qdma_device_write_bypass_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr, unsigned int val)
+{
+ if (!xpdev)
+ return;
+
+ if (!xpdev->bypass_bar_regs) {
+ pr_err("bypass bar is not present\n");
+ return;
+ }
+
+ writel(val, xpdev->bypass_bar_regs + reg_addr);
+}
+
+static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ struct qdma_dev_conf conf;
+ struct xlnx_pci_dev *xpdev = NULL;
+ unsigned long dev_hndl;
+ int rv;
+
+ pr_info("%s: func 0x%x/0x%x, p/v %d/%d,0x%p.\n",
+ dev_name(&pdev->dev), PCI_FUNC(pdev->devfn), QDMA_PF_MAX,
+ pdev->is_physfn, pdev->is_virtfn, pdev->physfn);
+
+ memset(&conf, 0, sizeof(conf));
+ conf.qdma_drv_mode = mode;
+ conf.vf_max = 0; /* enable via sysfs */
+
+#ifdef __QDMA_VF__
+ conf.qsets_max = TOTAL_QDMA_QS;
+#else
+ conf.master_pf = is_master_pf(pdev);
+ if (conf.master_pf)
+ pr_info("Configuring '%02x:%02x:%x' as master pf\n",
+ pdev->bus->number,
+ PCI_SLOT(pdev->devfn),
+ PCI_FUNC(pdev->devfn));
+#endif /* #ifdef __QDMA_VF__ */
+ pr_info("Driver is loaded in %s mode\n",
+ mode_name_list[conf.qdma_drv_mode].name);
+ conf.intr_rngsz = QDMA_INTR_COAL_RING_SIZE;
+ conf.tm_mode_en = tm_mode_en;
+ conf.tm_one_cdh_en = tm_one_cdh_en;
+ conf.pdev = pdev;
+
+ /* initialize all the bar numbers with -1 */
+ conf.bar_num_config = -1;
+ conf.bar_num_user = -1;
+ conf.bar_num_bypass = -1;
+
+ conf.bar_num_config = identify_config_bar(pdev);
+
+ rv = qdma_device_open(DRV_MODULE_NAME, &conf, &dev_hndl);
+ if (rv < 0)
+ return rv;
+
+#ifndef __QDMA_VF__
+ conf.qsets_max = TOTAL_PF_QS;
+#endif
+
+ xpdev = xpdev_alloc(pdev, conf.qsets_max);
+ if (!xpdev) {
+ rv = -EINVAL;
+ goto close_device;
+ }
+
+ xpdev->dev_hndl = dev_hndl;
+ xpdev->idx = conf.bdf;
+
+ rv = xpdev_map_user_bypass_bars(xpdev);
+ if (rv < 0)
+ goto close_device;
+
+ xpdev->cdev_cb.xpdev = xpdev;
+ rv = qdma_cdev_device_init(&xpdev->cdev_cb);
+ if (rv < 0)
+ goto close_device;
+
+ /* Create the files for attributes in sysfs */
+ if (conf.master_pf) {
+ rv = sysfs_create_group(&pdev->dev.kobj, &pci_master_device_attr_group);
+ if (rv < 0)
+ goto close_device;
+ } else {
+ rv = sysfs_create_group(&pdev->dev.kobj, &pci_device_attr_group);
+ if (rv < 0)
+ goto close_device;
+ }
+
+ dev_set_drvdata(&pdev->dev, xpdev);
+
+ return 0;
+
+close_device:
+ qdma_device_close(pdev, dev_hndl);
+
+ if (xpdev)
+ xpdev_free(xpdev);
+
+ return rv;
+}
+
+static void xpdev_device_cleanup(struct xlnx_pci_dev *xpdev)
+{
+ struct xlnx_qdata *qdata = xpdev->qdata;
+ struct xlnx_qdata *qmax = qdata + (xpdev->qmax * 2); /* h2c and c2h */
+
+ spin_lock(&xpdev->cdev_lock);
+ for (; qdata != qmax; qdata++) {
+ if (qdata->xcdev) {
+ /* if either h2c(1) or c2h(2) bit set, but not both */
+ if (qdata->xcdev->dir_init == 1 ||
+ qdata->xcdev->dir_init == 2) {
+ qdma_cdev_destroy(qdata->xcdev);
+ } else { /* both bits are set so remove one */
+ qdata->xcdev->dir_init >>= 1;
+ }
+ }
+ memset(qdata, 0, sizeof(*qdata));
+ }
+ spin_unlock(&xpdev->cdev_lock);
+}
+
+static void remove_one(struct pci_dev *pdev)
+{
+ struct xlnx_dma_dev *xdev = NULL;
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+ if (!xpdev) {
+ pr_info("%s NOT attached.\n", dev_name(&pdev->dev));
+ return;
+ }
+
+ xdev = (struct xlnx_dma_dev *)(xpdev->dev_hndl);
+
+ pr_info("%s pdev 0x%p, xdev 0x%p, hndl 0x%lx, qdma%05x.\n",
+ dev_name(&pdev->dev), pdev, xpdev, xpdev->dev_hndl, xpdev->idx);
+
+ if (xdev->conf.master_pf)
+ sysfs_remove_group(&pdev->dev.kobj, &pci_master_device_attr_group);
+ else
+ sysfs_remove_group(&pdev->dev.kobj, &pci_device_attr_group);
+
+ qdma_cdev_device_cleanup(&xpdev->cdev_cb);
+
+ xpdev_device_cleanup(xpdev);
+
+ xpdev_unmap_user_bypass_bars(xpdev);
+
+ qdma_device_close(pdev, xpdev->dev_hndl);
+
+ xpdev_free(xpdev);
+
+ dev_set_drvdata(&pdev->dev, NULL);
+}
+
+#if defined(CONFIG_PCI_IOV) && !defined(__QDMA_VF__)
+static int sriov_config(struct pci_dev *pdev, int num_vfs)
+{
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ if (!xpdev) {
+ pr_info("%s NOT attached.\n", dev_name(&pdev->dev));
+ return -EINVAL;
+ }
+
+ pr_debug("%s pdev 0x%p, xdev 0x%p, hndl 0x%lx, qdma%05x.\n",
+ dev_name(&pdev->dev), pdev, xpdev, xpdev->dev_hndl, xpdev->idx);
+
+ if (num_vfs > QDMA_VF_MAX) {
+ pr_info("%s, clamp down # of VFs %d -> %d.\n",
+ dev_name(&pdev->dev), num_vfs, QDMA_VF_MAX);
+ num_vfs = QDMA_VF_MAX;
+ }
+
+ return qdma_device_sriov_config(pdev, xpdev->dev_hndl, num_vfs);
+}
+#endif
+
+static pci_ers_result_t qdma_error_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ switch (state) {
+ case pci_channel_io_normal:
+ return PCI_ERS_RESULT_CAN_RECOVER;
+ case pci_channel_io_frozen:
+ pr_warn("dev 0x%p,0x%p, frozen state error, reset controller\n",
+ pdev, xpdev);
+ pci_disable_device(pdev);
+ return PCI_ERS_RESULT_NEED_RESET;
+ case pci_channel_io_perm_failure:
+ pr_warn("dev 0x%p,0x%p, failure state error, req. disconnect\n",
+ pdev, xpdev);
+ return PCI_ERS_RESULT_DISCONNECT;
+ }
+ return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t qdma_slot_reset(struct pci_dev *pdev)
+{
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ if (!xpdev) {
+ pr_info("%s NOT attached.\n", dev_name(&pdev->dev));
+ return PCI_ERS_RESULT_DISCONNECT;
+ }
+
+ pr_info("0x%p restart after slot reset\n", xpdev);
+ if (pci_enable_device_mem(pdev)) {
+ pr_info("0x%p failed to renable after slot reset\n", xpdev);
+ return PCI_ERS_RESULT_DISCONNECT;
+ }
+
+ pci_set_master(pdev);
+ pci_restore_state(pdev);
+ pci_save_state(pdev);
+
+ return PCI_ERS_RESULT_RECOVERED;
+}
+
+static void qdma_error_resume(struct pci_dev *pdev)
+{
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ if (!xpdev) {
+ pr_info("%s NOT attached.\n", dev_name(&pdev->dev));
+ return;
+ }
+
+ pr_info("dev 0x%p,0x%p.\n", pdev, xpdev);
+ pci_cleanup_aer_uncorrect_error_status(pdev);
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+static void qdma_reset_prepare(struct pci_dev *pdev)
+{
+ struct qdma_version_info version_info;
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ pr_info("%s pdev 0x%p, xdev 0x%p, hndl 0x%lx, qdma%05x.\n",
+ dev_name(&pdev->dev), pdev, xpdev, xpdev->dev_hndl, xpdev->idx);
+
+ qdma_device_version_info(xpdev->dev_hndl, &version_info);
+
+ qdma_device_offline(pdev, xpdev->dev_hndl);
+
+ /* FLR setting is not required for 2018.3 IP */
+ if (version_info.vivado_release_id != VIVADO_RELEASE_2018_3)
+ qdma_device_flr_quirk_set(pdev, xpdev->dev_hndl);
+ xpdev_queue_delete_all(xpdev);
+ if (version_info.vivado_release_id != VIVADO_RELEASE_2018_3)
+ qdma_device_flr_quirk_check(pdev, xpdev->dev_hndl);
+}
+
+static void qdma_reset_done(struct pci_dev *pdev)
+{
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ pr_info("%s pdev 0x%p, xdev 0x%p, hndl 0x%lx, qdma%05x.\n",
+ dev_name(&pdev->dev), pdev, xpdev, xpdev->dev_hndl, xpdev->idx);
+ qdma_device_online(pdev, xpdev->dev_hndl);
+}
+
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+static void qdma_reset_notify(struct pci_dev *pdev, bool prepare)
+{
+ struct qdma_version_info version_info;
+ struct xlnx_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
+
+ pr_info("%s prepare %d, pdev 0x%p, xdev 0x%p, hndl 0x%lx, qdma%05x.\n",
+ dev_name(&pdev->dev), prepare, pdev, xpdev, xpdev->dev_hndl,
+ xpdev->idx);
+
+ qdma_device_version_info(xpdev->dev_hndl, &version_info);
+
+ if (prepare) {
+ qdma_device_offline(pdev, xpdev->dev_hndl);
+ /* FLR setting is not required for 2018.3 IP */
+ if (version_info.vivado_release_id != VIVADO_RELEASE_2018_3)
+ qdma_device_flr_quirk_set(pdev, xpdev->dev_hndl);
+ xpdev_queue_delete_all(xpdev);
+ if (version_info.vivado_release_id != VIVADO_RELEASE_2018_3)
+ qdma_device_flr_quirk_check(pdev, xpdev->dev_hndl);
+ } else
+ qdma_device_online(pdev, xpdev->dev_hndl);
+}
+#endif
+static const struct pci_error_handlers qdma_err_handler = {
+ .error_detected = qdma_error_detected,
+ .slot_reset = qdma_slot_reset,
+ .resume = qdma_error_resume,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0)
+ .reset_prepare = qdma_reset_prepare,
+ .reset_done = qdma_reset_done,
+#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+ .reset_notify = qdma_reset_notify,
+#endif
+};
+
+static struct pci_driver pci_driver = {
+ .name = DRV_MODULE_NAME,
+ .id_table = pci_ids,
+ .probe = probe_one,
+ .remove = remove_one,
+/* .shutdown = shutdown_one, */
+#if defined(CONFIG_PCI_IOV) && !defined(__QDMA_VF__)
+ .sriov_configure = sriov_config,
+#endif
+ .err_handler = &qdma_err_handler,
+};
+
+int xlnx_nl_init(void);
+void xlnx_nl_exit(void);
+
+static int __init qdma_mod_init(void)
+{
+ int rv;
+
+ pr_info("%s", version);
+#ifdef __QDMA_VF__
+ if (mode == LEGACY_INTR_MODE) {
+ pr_err("Legacy interrupt not supported\n");
+ return -EINVAL;
+ }
+#endif
+ rv = libqdma_init(mode, num_threads);
+ if (rv < 0)
+ return rv;
+
+ rv = xdev_qconf_init();
+ if (rv < 0)
+ return rv;
+
+ rv = xlnx_nl_init();
+ if (rv < 0)
+ return rv;
+
+ rv = qdma_cdev_init();
+ if (rv < 0)
+ return rv;
+
+ return pci_register_driver(&pci_driver);
+}
+
+static void __exit qdma_mod_exit(void)
+{
+ /* unregister this driver from the PCI bus driver */
+ pci_unregister_driver(&pci_driver);
+
+ xlnx_nl_exit();
+
+ qdma_cdev_cleanup();
+
+ xdev_qconf_cleanup();
+
+ libqdma_exit();
+}
+
+module_init(qdma_mod_init);
+module_exit(qdma_mod_exit);
diff --git a/QDMA/linux-kernel/drv/qdma_mod.h b/QDMA/linux-kernel/drv/qdma_mod.h
new file mode 100644
index 0000000000000000000000000000000000000000..11a46236e46ff7c41a7c7f7351ea339612b79f9a
--- /dev/null
+++ b/QDMA/linux-kernel/drv/qdma_mod.h
@@ -0,0 +1,215 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_MODULE_H__
+#define __QDMA_MODULE_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma pcie kernel module
+ *
+ */
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/workqueue.h>
+#include <net/genetlink.h>
+
+#include "libqdma/libqdma_export.h"
+#include "libqdma/xdev.h"
+#include "cdev.h"
+
+/**
+ * @struct - xlnx_qdata
+ * @brief queue data variables send while read/write request
+ */
+struct xlnx_qdata {
+ unsigned long qhndl; /**< Queue handle */
+ struct qdma_cdev *xcdev; /**< qdma character device details */
+};
+
+#define XNL_EBUFLEN 256
+struct xlnx_nl_work_q_ctrl {
+ unsigned short qidx;
+ unsigned short qcnt;
+ u8 is_qp:1;
+ u8 is_c2h:1;
+};
+
+struct xlnx_nl_work {
+ struct work_struct work;
+ struct genl_info nl_info;
+ struct xlnx_pci_dev *xpdev;
+ wait_queue_head_t wq;
+ spinlock_t lock;
+ unsigned int q_start_handled;
+ union {
+ struct xlnx_nl_work_q_ctrl qctrl;
+ };
+};
+
+/**
+ * @struct - xlnx_pci_dev
+ * @brief xilinx pcie device data members
+ */
+struct xlnx_pci_dev {
+ struct list_head list_head; /**< device list */
+ struct pci_dev *pdev; /**< pointer to struct pci_dev */
+ unsigned long dev_hndl; /**< device handle*/
+ struct workqueue_struct *nl_task_wq; /**< netlink request work queue */
+ struct qdma_cdev_cb cdev_cb; /**< character device call back data*/
+ spinlock_t cdev_lock; /**< character device lock*/
+ unsigned int qmax; /**< max number of queues for device*/
+ unsigned int idx; /**< device index*/
+ void __iomem *user_bar_regs; /**< PCIe user bar */
+ void __iomem *bypass_bar_regs; /**< PCIe bypass bar */
+ struct xlnx_qdata qdata[0]; /**< queue data*/
+};
+
+/*****************************************************************************/
+/**
+ * xpdev_list_dump() - list the qdma devices
+ *
+ * @param[in] buflen: buffer length
+ * @param[out] buf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xpdev_list_dump(char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * xpdev_find_by_idx() - qdma pcie kernel module api to
+ * find the qdma device by index
+ *
+ * @param[in] idx: qdma device index
+ * @param[in] ebuflen: buffer length
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: pointer to xlnx_pci_dev
+ * @return NULL: failure
+ *****************************************************************************/
+struct xlnx_pci_dev *xpdev_find_by_idx(unsigned int idx, char *ebuf,
+ int ebuflen);
+
+/*****************************************************************************/
+/**
+ * xpdev_queue_get() - qdma pcie kernel module api to get a queue information
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] qidx: queue index
+ * @param[in] c2h: flag to indicate the queue direction (c2h/h2c)
+ * @param[in] check_qhndl: flag for validating the data
+ * @param[in] ebuflen: buffer length
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: queue information
+ * @return NULL: failure
+ *****************************************************************************/
+struct xlnx_qdata *xpdev_queue_get(struct xlnx_pci_dev *xpdev,
+ unsigned int qidx, bool c2h, bool check_qhndl,
+ char *ebuf, int ebuflen);
+
+/*****************************************************************************/
+/**
+ * xpdev_queue_add() - qdma pcie kernel module api to add a queue
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] qconf: queue configuration
+ * @param[in] ebuflen: buffer length
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xpdev_queue_add(struct xlnx_pci_dev *xpdev, struct qdma_queue_conf *qconf,
+ char *ebuf, int ebuflen);
+
+/*****************************************************************************/
+/**
+ * xpdev_queue_delete() - qdma pcie kernel module api to delete a queue
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] qidx: queue index
+ * @param[in] c2h: flag to indicate the queue direction (c2h/h2c)
+ * @param[in] ebuflen: buffer length
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xpdev_queue_delete(struct xlnx_pci_dev *xpdev, unsigned int qidx, bool c2h,
+ char *ebuf, int ebuflen);
+
+int xpdev_nl_queue_start(struct xlnx_pci_dev *xpdev, void *nl_info, u8 is_qp,
+ u8 is_c2h, unsigned short qidx, unsigned short qcnt);
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_user_register() - read user bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the user bar register
+ *****************************************************************************/
+unsigned int qdma_device_read_user_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr);
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_user_register() - write user bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ * @param[in] value: register value to be written
+ *
+ *****************************************************************************/
+void qdma_device_write_user_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr, u32 value);
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_bypass_register() - read bypass bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the bypass bar register
+ *****************************************************************************/
+unsigned int qdma_device_read_bypass_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr);
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_bypass_register() - write bypass bar register
+ *
+ * @param[in] xpdev: pointer to xlnx_pci_dev
+ * @param[in] reg_addr: register address
+ * @param[in] value: register value to be written
+ *
+ *****************************************************************************/
+void qdma_device_write_bypass_register(struct xlnx_pci_dev *xpdev,
+ unsigned int reg_addr, u32 value);
+
+#endif /* ifndef __QDMA_MODULE_H__ */
diff --git a/QDMA/linux-kernel/drv/version.h b/QDMA/linux-kernel/drv/version.h
new file mode 100644
index 0000000000000000000000000000000000000000..15dad71903d7a8651a45783cad6fab4be0b235af
--- /dev/null
+++ b/QDMA/linux-kernel/drv/version.h
@@ -0,0 +1,40 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef __QDMA_VERSION_H__
+#define __QDMA_VERSION_H__
+
+#include "libqdma/version.h"
+
+#ifdef __QDMA_VF__
+#define DRV_MODULE_NAME "qdma_vf"
+#define DRV_MODULE_DESC "Xilinx QDMA VF Reference Driver"
+#else
+#define DRV_MODULE_NAME "qdma_pf"
+#define DRV_MODULE_DESC "Xilinx QDMA PF Reference Driver"
+#endif /* #ifdef __QDMA_VF__ */
+#define DRV_MODULE_RELDATE "Sep. 2018"
+
+#define DRV_MOD_MAJOR 2018
+#define DRV_MOD_MINOR 3
+#define DRV_MOD_PATCHLEVEL 97
+
+#define DRV_MODULE_VERSION \
+ __stringify(DRV_MOD_MAJOR) "." \
+ __stringify(DRV_MOD_MINOR) "." \
+ __stringify(DRV_MOD_PATCHLEVEL) "." \
+ __stringify(LIBQDMA_VERSION_PATCH) "." \
+
+#define DRV_MOD_VERSION_NUMBER \
+ ((DRV_MOD_MAJOR)*1000 + (DRV_MOD_MINOR)*100 + DRV_MOD_PATCHLEVEL)
+
+#endif /* ifndef __QDMA_VERSION_H__ */
diff --git a/QDMA/linux-kernel/etc/30-qdma.rules b/QDMA/linux-kernel/etc/30-qdma.rules
new file mode 100644
index 0000000000000000000000000000000000000000..f1cb73674bbbfc1973c661d97f6b7775c3446773
--- /dev/null
+++ b/QDMA/linux-kernel/etc/30-qdma.rules
@@ -0,0 +1,3 @@
+KERNEL=="qdma[0-4]-MM-[0-9]*", MODE="0666"
+KERNEL=="qdma[0-4]-ST-[0-9]*", MODE="0666"
+
diff --git a/QDMA/linux-kernel/include/qdma_nl.h b/QDMA/linux-kernel/include/qdma_nl.h
new file mode 100644
index 0000000000000000000000000000000000000000..748d7be418caa3490a647e533c930034adb39e98
--- /dev/null
+++ b/QDMA/linux-kernel/include/qdma_nl.h
@@ -0,0 +1,406 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef QDMA_NL_H__
+#define QDMA_NL_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma netlink interfaces
+ *
+ */
+/** physical function name (no more than 15 characters) */
+#define XNL_NAME_PF "xnl_pf"
+/** virtual function name */
+#define XNL_NAME_VF "xnl_vf"
+/** qdma netlink interface version number */
+#define XNL_VERSION 0x1
+
+/** qdma nl interface minimum response buffer length*/
+#define XNL_RESP_BUFLEN_MIN 256
+/** qdma nl interface maximum response buffer length*/
+#define XNL_RESP_BUFLEN_MAX (2048 * 6)
+/** qdma nl interface error buffer length*/
+#define XNL_ERR_BUFLEN 64
+/** qdma nl command parameter length*/
+#define XNL_STR_LEN_MAX 20
+
+/** Q parameter: value to indicate invalid qid*/
+#define XNL_QIDX_INVALID 0xFFFF
+/** Q parameter: streaming mode*/
+#define XNL_F_QMODE_ST 0x00000001
+/** Q parameter: memory management mode*/
+#define XNL_F_QMODE_MM 0x00000002
+/** Q parameter: queue in h2c direction*/
+#define XNL_F_QDIR_H2C 0x00000004
+/** Q parameter: queue in c2h direction*/
+#define XNL_F_QDIR_C2H 0x00000008
+/** Q parameter: queue in both directions*/
+#define XNL_F_QDIR_BOTH (XNL_F_QDIR_H2C | XNL_F_QDIR_C2H)
+/** Q parameter: queue in prefetch mode*/
+#define XNL_F_PFETCH_EN 0x00000010
+/** Q parameter: enable the bypass for the queue*/
+#define XNL_F_DESC_BYPASS_EN 0x00000020
+/** Q parameter: fetch credits*/
+#define XNL_F_FETCH_CREDIT 0x00000040
+/** Q parameter: enable writeback accumulation*/
+#define XNL_F_CMPL_STATUS_ACC_EN 0x00000080
+/** Q parameter: enable writeback*/
+#define XNL_F_CMPL_STATUS_EN 0x00000100
+/** Q parameter: enable writeback pending check*/
+#define XNL_F_CMPL_STATUS_PEND_CHK 0x00000200
+/** Q parameter: enable writeback status descriptor*/
+#define XNL_F_CMPL_STATUS_DESC_EN 0x00000400
+/** Q parameter: enable queue completion interrupt*/
+#define XNL_F_C2H_CMPL_INTR_EN 0x00000800
+/** Q parameter: enable user defined data*/
+#define XNL_F_CMPL_UDD_EN 0x00001000
+/** Q parameter: enable the pfetch bypass for the queue */
+#define XNL_F_PFETCH_BYPASS_EN 0x00002000
+/** Q parameter: disable CMPT overflow check */
+#define XNL_F_CMPT_OVF_CHK_DIS 0x00004000
+
+/** maximum number of queue flags to control queue configuration*/
+#define MAX_QFLAGS 15
+
+/** maximum number of interrupt ring entries*/
+#define QDMA_MAX_INT_RING_ENTRIES 512
+
+/**
+ * xnl_attr_t netlink attributes for qdma(variables):
+ * the index in this enum is used as a reference for the type,
+ * userspace application has to indicate the corresponding type
+ * the policy is used for security considerations
+ */
+enum xnl_attr_t {
+ XNL_ATTR_GENMSG, /**< generatl message */
+ XNL_ATTR_DRV_INFO, /**< device info */
+
+ XNL_ATTR_DEV_IDX, /**< device index */
+ XNL_ATTR_PCI_BUS, /**< pci bus number */
+ XNL_ATTR_PCI_DEV, /**< pci device number */
+ XNL_ATTR_PCI_FUNC, /**< pci function id */
+
+ XNL_ATTR_DEV_STAT_MMH2C_PKTS1, /**< number of MM H2C packkts */
+ XNL_ATTR_DEV_STAT_MMH2C_PKTS2, /**< number of MM H2C packkts */
+ XNL_ATTR_DEV_STAT_MMC2H_PKTS1, /**< number of MM C2H packkts */
+ XNL_ATTR_DEV_STAT_MMC2H_PKTS2, /**< number of MM C2H packkts */
+ XNL_ATTR_DEV_STAT_STH2C_PKTS1, /**< number of ST H2C packkts */
+ XNL_ATTR_DEV_STAT_STH2C_PKTS2, /**< number of ST H2C packkts */
+ XNL_ATTR_DEV_STAT_STC2H_PKTS1, /**< number of ST C2H packkts */
+ XNL_ATTR_DEV_STAT_STC2H_PKTS2, /**< number of ST C2H packkts */
+
+ XNL_ATTR_DEV_CFG_BAR, /**< device config bar number */
+ XNL_ATTR_DEV_USR_BAR, /**< device user bar number */
+ XNL_ATTR_DEV_QSET_MAX, /**< max queue sets */
+ XNL_ATTR_DEV_QSET_QBASE, /**< queue base start */
+
+ XNL_ATTR_REG_BAR_NUM, /**< register bar number */
+ XNL_ATTR_REG_ADDR, /**< register address */
+ XNL_ATTR_REG_VAL, /**< register value */
+
+ XNL_ATTR_QIDX, /**< queue index */
+ XNL_ATTR_NUM_Q, /**< number of queues */
+ XNL_ATTR_QFLAG, /**< queue config flags */
+
+ XNL_ATTR_CMPT_DESC_SIZE, /**< completion descriptor size */
+ XNL_ATTR_SW_DESC_SIZE, /**< software descriptor size */
+ XNL_ATTR_QRNGSZ_IDX, /**< queue ring index */
+ XNL_ATTR_C2H_BUFSZ_IDX, /**< c2h buffer idex */
+ XNL_ATTR_CMPT_TIMER_IDX, /**< completion timer index */
+ XNL_ATTR_CMPT_CNTR_IDX, /**< completion counter index */
+ XNL_ATTR_CMPT_TRIG_MODE, /**< completion trigger mode */
+
+ XNL_ATTR_RANGE_START, /**< range start */
+ XNL_ATTR_RANGE_END, /**< range end */
+
+ XNL_ATTR_INTR_VECTOR_IDX, /**< interrupt vector index */
+ XNL_ATTR_INTR_VECTOR_START_IDX, /**< interrupt vector start index */
+ XNL_ATTR_INTR_VECTOR_END_IDX, /**< interrupt vector end index */
+ XNL_ATTR_RSP_BUF_LEN, /**< response buffer length */
+ XNL_ATTR_GLOBAL_CSR, /**< global csr data */
+ XNL_ATTR_PIPE_GL_MAX, /**< max no. of gl for pipe */
+ XNL_ATTR_PIPE_FLOW_ID, /**< pipe flow id */
+ XNL_ATTR_PIPE_SLR_ID, /**< pipe slr id */
+ XNL_ATTR_PIPE_TDEST, /**< pipe tdest */
+ XNL_ATTR_DEV_STM_BAR, /**< device STM bar number */
+#ifdef ERR_DEBUG
+ XNL_ATTR_QPARAM_ERR_INFO, /**< queue param info */
+#endif
+ XNL_ATTR_MAX,
+};
+
+/**
+ * xnl_st_c2h_cmpt_desc_size
+ * c2h writeback descriptor sizes
+ */
+enum xnl_st_c2h_cmpt_desc_size {
+ XNL_ST_C2H_CMPT_DESC_SIZE_8B, /**< 8B descriptor */
+ XNL_ST_C2H_CMPT_DESC_SIZE_16B, /**< 16B descriptor */
+ XNL_ST_C2H_CMPT_DESC_SIZE_32B, /**< 32B descriptor */
+ XNL_ST_C2H_CMPT_DESC_SIZE_64B, /**< 64B descriptor */
+ XNL_ST_C2H_NUM_CMPT_DESC_SIZES /**< Num of desc sizes */
+};
+
+enum xnl_qdma_rngsz_idx {
+ XNL_QDMA_RNGSZ_64_IDX,
+ XNL_QDMA_RNGSZ_128_IDX,
+ XNL_QDMA_RNGSZ_256_IDX,
+ XNL_QDMA_RNGSZ_512_IDX,
+ XNL_QDMA_RNGSZ_1024_IDX,
+ XNL_QDMA_RNGSZ_2048_IDX,
+ XNL_QDMA_RNGSZ_4096_IDX,
+ XNL_QDMA_RNGSZ_8192_IDX,
+ XNL_QDMA_RNGSZ_16384_IDX,
+ XNL_QDMA_RNGSZ_32768_IDX,
+ XNL_QDMA_RNGSZ_65536_IDX,
+ XNL_QDMA_RNGSZ_131072_IDX,
+ XNL_QDMA_RNGSZ_262177_IDX,
+ XNL_QDMA_RNGSZ_524288_IDX,
+ XNL_QDMA_RNGSZ_1048576_IDX,
+ XNL_QDMA_RNGSZ_2097152_IDX,
+ XNL_QDMA_RNGSZ_IDXS
+};
+
+/** XNL attributes list */
+static const char *xnl_attr_str[XNL_ATTR_MAX] = {
+ "GENMSG", /**< XNL_ATTR_GENMSG */
+ "DRV_INFO", /**< XNL_ATTR_DRV_INFO */
+
+ "DEV_IDX", /**< XNL_ATTR_DEV_IDX */
+
+ "DEV_PCIBUS", /**< XNL_ATTR_PCI_BUS */
+ "DEV_PCIDEV", /**< XNL_ATTR_PCI_DEV */
+ "DEV_PCIFUNC", /**< XNL_ATTR_PCI_FUNC */
+
+ "DEV_CFG_BAR", /**< XNL_ATTR_DEV_CFG_BAR */
+ "DEV_USR_BAR", /**< XNL_ATTR_DEV_USER_BAR */
+ "DEV_QSETMAX", /**< XNL_ATTR_DEV_QSET_MAX */
+ "DEV_QBASE", /**< XNL_ATTR_DEV_QSET_QBASE */
+
+ "REG_BAR", /**< XNL_ATTR_REG_BAR_NUM */
+ "REG_ADDR", /**< XNL_ATTR_REG_ADDR */
+ "REG_VAL", /**< XNL_ATTR_REG_VAL */
+
+ "QIDX", /**< XNL_ATTR_QIDX */
+ "NUM_Q", /**< XNL_ATTR_NUM_Q */
+ "QFLAG", /**< XNL_ATTR_QFLAG */
+
+ "CMPT_DESC_SZ", /**< XNL_ATTR_CMPT_DESC_SIZE */
+ "SW_DESC_SIZE", /**< XNL_ATTR_SW_DESC_SIZE */
+ "QRINGSZ_IDX", /**< XNL_ATTR_QRNGSZ */
+ "C2H_BUFSZ_IDX", /**< XNL_ATTR_QBUFSZ */
+ "CMPT_TIMER_IDX", /**< XNL_ATTR_CMPT_TIMER_IDX */
+ "CMPT_CNTR_IDX", /**< XNL_ATTR_CMPT_CNTR_IDX */
+ "CMPT_TRIG_MODE", /**< XNL_ATTR_CMPT_TRIG_MODE */
+
+ "RANGE_START", /**< XNL_ATTR_RANGE_START */
+ "RANGE_END", /**< XNL_ATTR_RANGE_END */
+ "INTR_VECTOR_IDX", /**< XNL_ATTR_INTR_VECTOR_IDX */
+ "INTR_VECTOR_START_IDX", /**< XNL_ATTR_INTR_VECTOR_START_IDX */
+ "INTR_VECTOR_END_IDX", /**< XNL_ATTR_INTR_VECTOR_END_IDX */
+ "RSP_BUF_LEN", /**< XNL_ATTR_RSP_BUF_LEN */
+ "GLOBAL_CSR", /**< global csr data */
+#ifdef ERR_DEBUG
+ "QPARAM_ERR_INFO",
+#endif
+
+};
+
+/* commands, 0 ~ 0x7F */
+/**
+ * xnl_op_t - XNL command types
+ */
+enum xnl_op_t {
+ XNL_CMD_DEV_LIST, /**< list all the qdma devices */
+ XNL_CMD_DEV_INFO, /**< dump the device information */
+ XNL_CMD_DEV_STAT, /**< dump the device statistics */
+ XNL_CMD_DEV_STAT_CLEAR, /**< reset the device statistics */
+
+ XNL_CMD_REG_DUMP, /**< dump the register information */
+ XNL_CMD_REG_RD, /**< read a register value */
+ XNL_CMD_REG_WRT, /**< write value to a register */
+
+ XNL_CMD_Q_LIST, /**< list all the queue present in the system */
+ XNL_CMD_Q_ADD, /**< add a queue */
+ XNL_CMD_Q_START, /**< start a queue */
+ XNL_CMD_Q_STOP, /**< stop a queue */
+ XNL_CMD_Q_DEL, /**< delete a queue */
+ XNL_CMD_Q_DUMP, /**< dump queue information*/
+ XNL_CMD_Q_DESC, /**< dump descriptor information*/
+ XNL_CMD_Q_CMPT, /**< dump writeback descriptor information*/
+ XNL_CMD_Q_RX_PKT, /**< dump packet information*/
+#ifdef ERR_DEBUG
+ XNL_CMD_Q_ERR_INDUCE, /**< induce an error*/
+#endif
+
+ XNL_CMD_INTR_RING_DUMP, /**< dump interrupt ring information*/
+ XNL_CMD_Q_UDD, /**< dump the user defined data */
+ XNL_CMD_GLOBAL_CSR, /**< get all global csr register values */
+ XNL_CMD_VERSION, /**< list RTL version and Vivado release ID */
+ XNL_CMD_MAX, /**< max number of XNL commands*/
+};
+
+/**
+ * XNL command operation type
+ */
+static const char *xnl_op_str[XNL_CMD_MAX] = {
+ "DEV_LIST", /** XNL_CMD_DEV_LIST */
+ "DEV_INFO", /** XNL_CMD_DEV_INFO */
+ "DEV_STAT", /** XNL_CMD_DEV_STAT */
+ "DEV_STAT_CLEAR", /** XNL_CMD_DEV_STAT_CLEAR */
+
+ "REG_DUMP", /** XNL_CMD_REG_DUMP */
+ "REG_READ", /** XNL_CMD_REG_RD */
+ "REG_WRITE", /** XNL_CMD_REG_WRT */
+
+ "Q_LIST", /** XNL_CMD_Q_LIST */
+ "Q_ADD", /** XNL_CMD_Q_ADD */
+ "Q_START", /** XNL_CMD_Q_START */
+ "Q_STOP", /** XNL_CMD_Q_STOP */
+ "Q_DEL", /** XNL_CMD_Q_DEL */
+ "Q_DUMP", /** XNL_CMD_Q_DUMP */
+ "Q_DESC", /** XNL_CMD_Q_DESC */
+ "Q_CMPT", /** XNL_CMD_Q_CMPT */
+ "Q_RX_PKT", /** XNL_CMD_Q_RX_PKT */
+
+ "INTR_RING_DUMP", /** XNL_CMD_INTR_RING_DUMP */
+#ifdef ERR_DEBUG
+ "Q_ERR_INDUCE" /** XNL_CMD_Q_ERR_INDUCE */
+#endif
+};
+
+/**
+ * qdma_err_idx - Induce error
+ */
+enum qdma_err_idx {
+ err_ram_sbe,
+ err_ram_dbe,
+ err_dsc,
+ err_trq,
+ err_h2c_mm_0,
+ err_h2c_mm_1,
+ err_c2h_mm_0,
+ err_c2h_mm_1,
+ err_c2h_st,
+ ind_ctxt_cmd_err,
+ err_bdg,
+ err_h2c_st,
+ poison,
+ ur_ca,
+ param,
+ addr,
+ tag,
+ flr,
+ timeout,
+ dat_poison,
+ flr_cancel,
+ dma,
+ dsc,
+ rq_cancel,
+ dbe,
+ sbe,
+ unmapped,
+ qid_range,
+ vf_access_err,
+ tcp_timeout,
+ mty_mismatch,
+ len_mismatch,
+ qid_mismatch,
+ desc_rsp_err,
+ eng_wpl_data_par_err,
+ msi_int_fail,
+ err_desc_cnt,
+ portid_ctxt_mismatch,
+ portid_byp_in_mismatch,
+ cmpt_inv_q_err,
+ cmpt_qfull_err,
+ cmpt_cidx_err,
+ cmpt_prty_err,
+ fatal_mty_mismatch,
+ fatal_len_mismatch,
+ fatal_qid_mismatch,
+ timer_fifo_ram_rdbe,
+ fatal_eng_wpl_data_par_err,
+ pfch_II_ram_rdbe,
+ cmpt_ctxt_ram_rdbe,
+ pfch_ctxt_ram_rdbe,
+ desc_req_fifo_ram_rdbe,
+ int_ctxt_ram_rdbe,
+ cmpt_coal_data_ram_rdbe,
+ tuser_fifo_ram_rdbe,
+ qid_fifo_ram_rdbe,
+ payload_fifo_ram_rdbe,
+ wpl_data_par_err,
+ zero_len_desc_err,
+ csi_mop_err,
+ no_dma_dsc_err,
+ sb_mi_h2c0_dat,
+ sb_mi_c2h0_dat,
+ sb_h2c_rd_brg_dat,
+ sb_h2c_wr_brg_dat,
+ sb_c2h_rd_brg_dat,
+ sb_c2h_wr_brg_dat,
+ sb_func_map,
+ sb_dsc_hw_ctxt,
+ sb_dsc_crd_rcv,
+ sb_dsc_sw_ctxt,
+ sb_dsc_cpli,
+ sb_dsc_cpld,
+ sb_pasid_ctxt_ram,
+ sb_timer_fifo_ram,
+ sb_payload_fifo_ram,
+ sb_qid_fifo_ram,
+ sb_tuser_fifo_ram,
+ sb_wrb_coal_data_ram,
+ sb_int_qid2vec_ram,
+ sb_int_ctxt_ram,
+ sb_desc_req_fifo_ram,
+ sb_pfch_ctxt_ram,
+ sb_wrb_ctxt_ram,
+ sb_pfch_ll_ram,
+ sb_h2c_pend_fifo,
+ db_mi_h2c0_dat,
+ db_mi_c2h0_dat,
+ db_h2c_rd_brg_dat,
+ db_h2c_wr_brg_dat,
+ db_c2h_rd_brg_dat,
+ db_c2h_wr_brg_dat,
+ db_func_map,
+ db_dsc_hw_ctxt,
+ db_dsc_crd_rcv,
+ db_dsc_sw_ctxt,
+ db_dsc_cpli,
+ db_dsc_cpld,
+ db_pasid_ctxt_ram,
+ db_timer_fifo_ram,
+ db_payload_fifo_ram,
+ db_qid_fifo_ram,
+ db_tuser_fifo_ram,
+ db_wrb_coal_data_ram,
+ db_int_qid2vec_ram,
+ db_int_ctxt_ram,
+ db_desc_req_fifo_ram,
+ db_pfch_ctxt_ram,
+ db_wrb_ctxt_ram,
+ db_pfch_ll_ram,
+ db_h2c_pend_fifo,
+ qdma_errs
+};
+
+#endif /* ifndef QDMA_NL_H__ */
diff --git a/QDMA/linux-kernel/include/xdev_regs.h b/QDMA/linux-kernel/include/xdev_regs.h
new file mode 100644
index 0000000000000000000000000000000000000000..26776dc7634a3c048a8d69a52b0733419541c399
--- /dev/null
+++ b/QDMA/linux-kernel/include/xdev_regs.h
@@ -0,0 +1,240 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ *
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef __XDEV_REGS_H__
+#define __XDEV_REGS_H__
+
+struct xreg_info {
+ const char name[32];
+ uint32_t addr;
+ unsigned int repeat;
+ unsigned int step;
+ unsigned char shift;
+ unsigned char len;
+ unsigned char filler[2];
+};
+
+static struct xreg_info qdma_config_regs[] = {
+
+ /* QDMA_TRQ_SEL_GLBL1 (0x00000) */
+ {"CFG_BLOCK_ID", 0x00, 0, 0, 0, 0,},
+ {"CFG_BUSDEV", 0x04, 0, 0, 0, 0,},
+ {"CFG_PCIE_MAX_PL_SZ", 0x08, 0, 0, 0, 0,},
+ {"CFG_PCIE_MAX_RDRQ_SZ", 0x0C, 0, 0, 0, 0,},
+ {"CFG_SYS_ID", 0x10, 0, 0, 0, 0,},
+ {"CFG_MSI_EN", 0x14, 0, 0, 0, 0,},
+ {"CFG_PCIE_DATA_WIDTH", 0x18, 0, 0, 0, 0,},
+ {"CFG_PCIE_CTRL", 0x1C, 0, 0, 0, 0,},
+ {"CFG_AXI_USR_MAX_PL_SZ", 0x40, 0, 0, 0, 0,},
+ {"CFG_AXI_USR_MAX_RDRQ_SZ", 0x44, 0, 0, 0, 0,},
+ {"CFG_MISC_CTRL", 0x4C, 0, 0, 0, 0,},
+ {"CFG_SCRATCH_REG", 0x80, 8, 0, 0, 0,},
+ {"QDMA_RAM_SBE_MSK_A", 0xF0, 0, 0, 0, 0,},
+ {"QDMA_RAM_SBE_STS_A", 0xF4, 0, 0, 0, 0,},
+ {"QDMA_RAM_DBE_MSK_A", 0xF8, 0, 0, 0, 0,},
+ {"QDMA_RAM_DBE_STS_A", 0xFC, 0, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_GLBL2 (0x00100) */
+ {"GLBL2_ID", 0x100, 0, 0, 0, 0,},
+ {"GLBL2_PF_BL_INT", 0x104, 0, 0, 0, 0,},
+ {"GLBL2_PF_VF_BL_INT", 0x108, 0, 0, 0, 0,},
+ {"GLBL2_PF_BL_EXT", 0x10C, 0, 0, 0, 0,},
+ {"GLBL2_PF_VF_BL_EXT", 0x110, 0, 0, 0, 0,},
+ {"GLBL2_CHNL_INST", 0x114, 0, 0, 0, 0,},
+ {"GLBL2_CHNL_QDMA", 0x118, 0, 0, 0, 0,},
+ {"GLBL2_CHNL_STRM", 0x11C, 0, 0, 0, 0,},
+ {"GLBL2_QDMA_CAP", 0x120, 0, 0, 0, 0,},
+ {"GLBL2_PASID_CAP", 0x128, 0, 0, 0, 0,},
+ {"GLBL2_FUNC_RET", 0x12C, 0, 0, 0, 0,},
+ {"GLBL2_SYS_ID", 0x130, 0, 0, 0, 0,},
+ {"GLBL2_MISC_CAP", 0x134, 0, 0, 0, 0,},
+ {"GLBL2_DBG_PCIE_RQ", 0x1B8, 2, 0, 0, 0,},
+ {"GLBL2_DBG_AXIMM_WR", 0x1C0, 2, 0, 0, 0,},
+ {"GLBL2_DBG_AXIMM_RD", 0x1C8, 2, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_GLBL (0x00200) */
+ {"GLBL_RNGSZ", 0x204, 16, 0, 0, 0,},
+ {"GLBL_ERR_STAT", 0x248, 0, 0, 0, 0,},
+ {"GLBL_ERR_MASK", 0x24C, 0, 0, 0, 0,},
+ {"GLBL_DSC_CFG", 0x250, 0, 0, 0, 0,},
+ {"GLBL_DSC_ERR_STS", 0x254, 0, 0, 0, 0,},
+ {"GLBL_DSC_ERR_MSK", 0x258, 0, 0, 0, 0,},
+ {"GLBL_DSC_ERR_LOG", 0x25C, 2, 0, 0, 0,},
+ {"GLBL_TRQ_ERR_STS", 0x264, 0, 0, 0, 0,},
+ {"GLBL_TRQ_ERR_MSK", 0x268, 0, 0, 0, 0,},
+ {"GLBL_TRQ_ERR_LOG", 0x26C, 0, 0, 0, 0,},
+ {"GLBL_DSC_DBG_DAT", 0x270, 2, 0, 0, 0,},
+ {"GLBL_DSC_ERR_LOG2", 0x27C, 0, 0, 0, 0,},
+ {"GLBL_INTERRUPT_CFG", 0x288, 0, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_FMAP (0x00400 - 0x7FC) */
+ /* TODO: max 256, display 4 for now */
+ {"TRQ_SEL_FMAP", 0x400, 4, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_IND (0x00800) */
+ {"IND_CTXT_DATA", 0x804, 8, 0, 0, 0,},
+ {"IND_CTXT_MASK", 0x824, 8, 0, 0, 0,},
+ {"IND_CTXT_CMD", 0x844, 0, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_C2H (0x00A00) */
+ {"C2H_TIMER_CNT", 0xA00, 16, 0, 0, 0,},
+ {"C2H_CNT_THRESH", 0xA40, 16, 0, 0, 0,},
+ {"C2H_STAT_S_AXIS_C2H_ACCEPTED", 0xA88, 0, 0, 0, 0,},
+ {"C2H_STAT_S_AXIS_CMPT_ACCEPTED", 0xA8C, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_RSP_PKT_ACCEPTED", 0xA90, 0, 0, 0, 0,},
+ {"C2H_STAT_AXIS_PKG_CMP", 0xA94, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_RSP_ACCEPTED", 0xA98, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_RSP_CMP", 0xA9C, 0, 0, 0, 0,},
+ {"C2H_STAT_WRQ_OUT", 0xAA0, 0, 0, 0, 0,},
+ {"C2H_STAT_WPL_REN_ACCEPTED", 0xAA4, 0, 0, 0, 0,},
+ {"C2H_STAT_TOTAL_WRQ_LEN", 0xAA8, 0, 0, 0, 0,},
+ {"C2H_STAT_TOTAL_WPL_LEN", 0xAAC, 0, 0, 0, 0,},
+ {"C2H_BUF_SZ", 0xAB0, 16, 0, 0, 0,},
+ {"C2H_ERR_STAT", 0xAF0, 0, 0, 0, 0,},
+ {"C2H_ERR_MASK", 0xAF4, 0, 0, 0, 0,},
+ {"C2H_FATAL_ERR_STAT", 0xAF8, 0, 0, 0, 0,},
+ {"C2H_FATAL_ERR_MASK", 0xAFC, 0, 0, 0, 0,},
+ {"C2H_FATAL_ERR_ENABLE", 0xB00, 0, 0, 0, 0,},
+ {"GLBL_ERR_INT", 0xB04, 0, 0, 0, 0,},
+ {"C2H_PFCH_CFG", 0xB08, 0, 0, 0, 0,},
+ {"C2H_INT_TIMER_TICK", 0xB0C, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_RSP_DROP_ACCEPTED", 0xB10, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_RSP_ERR_ACCEPTED", 0xB14, 0, 0, 0, 0,},
+ {"C2H_STAT_DESC_REQ", 0xB18, 0, 0, 0, 0,},
+ {"C2H_STAT_DEBUG_DMA_ENG", 0xB1C, 4, 0, 0, 0,},
+ {"C2H_DBG_PFCH_ERR_CTXT", 0xB2C, 0, 0, 0, 0,},
+ {"C2H_FIRST_ERR_QID", 0xB30, 0, 0, 0, 0,},
+ {"STAT_NUM_CMPT_IN", 0xB34, 0, 0, 0, 0,},
+ {"STAT_NUM_CMPT_OUT", 0xB38, 0, 0, 0, 0,},
+ {"STAT_NUM_CMPT_DRP", 0xB3C, 0, 0, 0, 0,},
+ {"STAT_NUM_STAT_DESC_OUT", 0xB40, 0, 0, 0, 0,},
+ {"STAT_NUM_DSC_CRDT_SENT", 0xB44, 0, 0, 0, 0,},
+ {"STAT_NUM_FCH_DSC_RCVD", 0xB48, 0, 0, 0, 0,},
+ {"STAT_NUM_BYP_DSC_RCVD", 0xB4C, 0, 0, 0, 0,},
+ {"C2H_CMPT_COAL_CFG", 0xB50, 0, 0, 0, 0,},
+ {"C2H_INTR_H2C_REQ", 0xB54, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_MM_REQ", 0xB58, 0, 0, 0, 0,},
+ {"C2H_INTR_ERR_INT_REQ", 0xB5C, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_ST_REQ", 0xB60, 0, 0, 0, 0,},
+ {"C2H_INTR_H2C_ERR_MM_MSIX_ACK", 0xB64, 0, 0, 0, 0,},
+ {"C2H_INTR_H2C_ERR_MM_MSIX_FAIL", 0xB68, 0, 0, 0, 0,},
+ {"C2H_INTR_H2C_ERR_MM_NO_MSIX", 0xB6C, 0, 0, 0, 0,},
+ {"C2H_INTR_H2C_ERR_MM_CTXT_INVAL", 0xB70, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_ST_MSIX_ACK", 0xB74, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_ST_MSIX_FAIL", 0xB78, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_ST_NO_MSIX", 0xB7C, 0, 0, 0, 0,},
+ {"C2H_INTR_C2H_ST_CTXT_INVAL", 0xB80, 0, 0, 0, 0,},
+ {"C2H_STAT_WR_CMP", 0xB84, 0, 0, 0, 0,},
+ {"C2H_STAT_DEBUG_DMA_ENG_4", 0xB88, 0, 0, 0, 0,},
+ {"C2H_STAT_DEBUG_DMA_ENG_5", 0xB8C, 0, 0, 0, 0,},
+ {"C2H_DBG_PFCH_QID", 0xB90, 0, 0, 0, 0,},
+ {"C2H_DBG_PFCH", 0xB94, 0, 0, 0, 0,},
+ {"C2H_INT_DEBUG", 0xB98, 0, 0, 0, 0,},
+ {"C2H_STAT_IMM_ACCEPTED", 0xB9C, 0, 0, 0, 0,},
+ {"C2H_STAT_MARKER_ACCEPTED", 0xBA0, 0, 0, 0, 0,},
+ {"C2H_STAT_DISABLE_CMP_ACCEPTED", 0xBA4, 0, 0, 0, 0,},
+ {"C2H_C2H_PAYLOAD_FIFO_CRDT_CNT", 0xBA8, 0, 0, 0, 0,},
+
+ {"C2H_INTR_DYN_REQ", 0xBAC, 0, 0, 0, 0,},
+ {"C2H_INTR_DYN_MSIX", 0xBB0, 0, 0, 0, 0,},
+ {"C2H_DROP_LEN_MISMATCH", 0xBB4, 0, 0, 0, 0,},
+ {"C2H_DROP_DESC_RSP_LEN", 0xBB8, 0, 0, 0, 0,},
+ {"C2H_DROP_QID_FIFO_LEN", 0xBBC, 0, 0, 0, 0,},
+ {"C2H_DROP_PAYLOAD_CNT", 0xBC0, 0, 0, 0, 0,},
+ {"QDMA_C2H_CMPT_FORMAT", 0xBC4, 7, 0, 0, 0,},
+ {"C2H_PFCH_CACHE_DEPTH", 0xBE0, 0, 0, 0, 0,},
+ {"C2H_CMPT_COAL_BUF_DEPTH", 0xBE4, 0, 0, 0, 0,},
+ {"C2H_PFCH_CRDT", 0xBE8, 0, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_H2C(0x00E00) Register Space*/
+ {"H2C_ERR_STAT", 0xE00, 0, 0, 0, 0,},
+ {"H2C_ERR_MASK", 0xE04, 0, 0, 0, 0,},
+ {"H2C_FIRST_ERR_QID", 0xE08, 0, 0, 0, 0,},
+ {"H2C_DBG_REG", 0xE0C, 5, 0, 0, 0,},
+ {"H2C_FATAL_ERR_EN", 0xE20, 0, 0, 0, 0,},
+ {"H2C_REQ_THROT", 0xE24, 0, 0, 0, 0,},
+ {"H2C_ALN_DBG_REG0", 0xE28, 0, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_C2H_MM (0x1000) */
+ {"C2H_MM_CONTROL", 0x1004, 3, 0, 0, 0,},
+ {"C2H_MM_STATUS", 0x1040, 2, 0, 0, 0,},
+ {"C2H_MM_CMPL_DSC_CNT", 0x1048, 0, 0, 0, 0,},
+ {"C2H_MM_ERR_CODE_EN_MASK", 0x1054, 0, 0, 0, 0,},
+ {"C2H_MM_ERR_CODE", 0x1058, 0, 0, 0, 0,},
+ {"C2H_MM_ERR_INFO", 0x105C, 0, 0, 0, 0,},
+ {"C2H_MM_PERF_MON_CTRL", 0x10C0, 0, 0, 0, 0,},
+ {"C2H_MM_PERF_MON_CY_CNT", 0x10C4, 2, 0, 0, 0,},
+ {"C2H_MM_PERF_MON_DATA_CNT", 0x10CC, 2, 0, 0, 0,},
+ {"C2H_MM_DBG_INFO", 0x10E8, 2, 0, 0, 0,},
+
+ /* QDMA_TRQ_SEL_H2C_MM (0x1200)*/
+ {"H2C_MM_CONTROL", 0x1204, 3, 0, 0, 0,},
+ {"H2C_MM_STATUS", 0x1240, 0, 0, 0, 0,},
+ {"H2C_MM_CMPL_DSC_CNT", 0x1248, 0, 0, 0, 0,},
+ {"H2C_MM_ERR_CODE_EN_MASK", 0x1254, 0, 0, 0, 0,},
+ {"H2C_MM_ERR_CODE", 0x1258, 0, 0, 0, 0,},
+ {"H2C_MM_ERR_INFO", 0x125C, 0, 0, 0, 0,},
+ {"H2C_MM_PERF_MON_CTRL", 0x12C0, 0, 0, 0, 0,},
+ {"H2C_MM_PERF_MON_CY_CNT", 0x12C4, 2, 0, 0, 0,},
+ {"H2C_MM_PERF_MON_DATA_CNT", 0x12CC, 2, 0, 0, 0,},
+ {"H2C_MM_DBG_INFO", 0x12E8, 0, 0, 0, 0,},
+ {"H2C_MM_REQ_THROT", 0x12EC, 0, 0, 0, 0,},
+
+ /* QDMA_PF_MAILBOX (0x2400) */
+ {"FUNC_STATUS", 0x2400, 0, 0, 0, 0,},
+ {"FUNC_CMD", 0x2404, 0, 0, 0, 0,},
+ {"FUNC_INTR_VEC", 0x2408, 0, 0, 0, 0,},
+ {"TARGET_FUNC", 0x240C, 0, 0, 0, 0,},
+ {"INTR_CTRL", 0x2410, 0, 0, 0, 0,},
+ {"PF_ACK", 0x2420, 8, 0, 0, 0,},
+ {"FLR_CTRL_STATUS", 0x2500, 0, 0, 0, 0,},
+ {"MSG_IN", 0x2800, 32, 0, 0, 0,},
+ {"MSG_OUT", 0x2C00, 32, 0, 0, 0,},
+
+ {"", 0, 0, 0 }
+};
+
+static struct xreg_info qdma_user_regs[] = {
+ {"ST_C2H_QID", 0x0, 0, 0, 0, 0,},
+ {"ST_C2H_PKTLEN", 0x4, 0, 0, 0, 0,},
+ {"ST_C2H_CONTROL", 0x8, 0, 0, 0, 0,},
+ /* ST_C2H_CONTROL:
+ * [1] : start C2H
+ * [2] : immediate data
+ * [3] : every packet statrs with 00 instead of continuous data
+ * stream until # of packets is complete
+ * [31]: gen_user_reset_n
+ */
+ {"ST_H2C_CONTROL", 0xC, 0, 0, 0, 0,},
+ /* ST_H2C_CONTROL:
+ * [0] : clear match for H2C transfer
+ */
+ {"ST_H2C_STATUS", 0x10, 0, 0, 0, 0,},
+ {"ST_H2C_XFER_CNT", 0x14, 0, 0, 0, 0,},
+ {"ST_C2H_PKT_CNT", 0x20, 0, 0, 0, 0,},
+ {"ST_C2H_CMPT_DATA", 0x30, 8, 0, 0, 0,},
+ {"ST_C2H_CMPT_SIZE", 0x50, 0, 0, 0, 0,},
+ {"ST_SCRATCH_REG", 0x60, 2, 0, 0, 0,},
+ {"ST_C2H_PKT_DROP", 0x88, 0, 0, 0, 0,},
+ {"ST_C2H_PKT_ACCEPT", 0x8C, 0, 0, 0, 0,},
+ {"DSC_BYPASS_LOOP", 0x90, 0, 0, 0, 0,},
+ {"USER_INTERRUPT", 0x94, 0, 0, 0, 0,},
+ {"USER_INTERRUPT_MASK", 0x98, 0, 0, 0, 0,},
+ {"USER_INTERRUPT_VEC", 0x9C, 0, 0, 0, 0,},
+ {"DMA_CONTROL", 0xA0, 0, 0, 0, 0,},
+ {"VDM_MSG_READ", 0xA4, 0, 0, 0, 0,},
+
+ {"", 0, 0, 0 }
+};
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/Makefile b/QDMA/linux-kernel/libqdma/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..208a20ad68ac81a605299813de7b89462871e308
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/Makefile
@@ -0,0 +1,47 @@
+SHELL = /bin/bash
+
+# The top-level makefile defines required variables and flags.
+ifneq ($(shell [[ $(MAKELEVEL) -ge 1 ]] && echo 1),1)
+ $(error Please use the top-level Makefile to build this driver)
+endif
+
+ifneq ($(wildcard /etc/lsb-release),)
+ ifneq ($(shell $(grep) "Ubuntu" /etc/lsb-release),)
+ FLAGS += -DUBUNTU_VERSION_CODE
+ endif
+endif
+
+include $(srcdir)/make/common_flags.mk
+
+$(info srcdir = $(srcdir).)
+$(info KSRC = $(KSRC).)
+
+EXTRA_CFLAGS += -DLINUX -D__KERNEL__ -DMODULE -O2 -pipe -Wall -Werror
+EXTRA_CFLAGS += $(FLAGS) $(CPPFLAGS)
+EXTRA_CFLAGS += -I$(srcdir)/include
+EXTRA_CFLAGS += -I$(KSRC)/include
+EXTRA_CFLAGS += -I.
+
+#EXTRA_CFLAGS += -DDEBUG
+
+ifneq ($(modulesymfile),)
+ override symverfile = symverfile="$(topdir)/$(modulesymfile) \
+ -o $(drvdir)/$(modulesymfile)"
+else
+ override symverfile =
+endif
+
+MOD_NAME := libqdma
+ifneq ($(SUBDIRS),)
+ BASE_OBJS := $(patsubst $(SUBDIRS)/%.c,%.o,$(wildcard $(SUBDIRS)/*.c))
+endif
+obj-m := $(MOD_NAME).o
+$(MOD_NAME)-objs := $(BASE_OBJS)
+
+.PHONY: build
+build:
+ @$(MAKE) symverfile=$(symverfile) KBUILD_EXTRA_SYMBOLS=$(extra_symb) -C $(KOBJ) SUBDIRS=$(shell pwd) modules
+
+.PHONY: clean
+clean:
+ @-/bin/rm -rf *.ko* ?odule* .tmp_versions *.mod.* *.o .*.o.* .*.cmd
diff --git a/QDMA/linux-kernel/libqdma/libqdma_config.c b/QDMA/linux-kernel/libqdma/libqdma_config.c
new file mode 100644
index 0000000000000000000000000000000000000000..cb9cadfdc058a5908f6f0345ea3fa0cc07f813aa
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/libqdma_config.c
@@ -0,0 +1,572 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+/**
+ * @file
+ * @brief This file contains the definitions for qdma configuration apis
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "libqdma_export.h"
+
+#include "qdma_descq.h"
+#include "qdma_device.h"
+#include "qdma_thread.h"
+#include "qdma_regs.h"
+#include "qdma_context.h"
+#include "qdma_intr.h"
+#include "thread.h"
+#include "version.h"
+
+/*****************************************************************************/
+/**
+ * qdma_set_qmax() - Handler function to set the qmax configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] qsets_max: qmax configuration value
+ * @param[in] forced: whether to force set the value
+ *
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return < 0 on failure
+ *****************************************************************************/
+int qdma_set_qmax(unsigned long dev_hndl, u32 qsets_max, bool forced)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev;
+ int rv = -1;
+
+ /**
+ * If xdev is NULL or qdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev)
+ return -EINVAL;
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That means at least one queue is added in the system.
+ * qmax is not allowed to change after FMAP programming is done.
+ * Hence, If qdev->init_qrange is set, return error as qmax
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, can not modify qmax [%d]\n",
+ xdev,
+ qdev->qmax);
+ return rv;
+ }
+
+ /** If the input qsets_max is same as the current xdev->conf.qsets_max,
+ * return, as there is nothing to be changed
+ */
+ if ((qsets_max == xdev->conf.qsets_max) && !forced) {
+ pr_err("xdev 0x%p, Current qsets_max is same as [%d].Nothing to be done\n",
+ xdev, xdev->conf.qsets_max);
+ return rv;
+ }
+
+
+ /** FMAP programming is not done yet
+ * remove the already created qdev and recreate it using the
+ * newly configured size
+ */
+ qdma_device_cleanup(xdev);
+ xdev->conf.qsets_max = qsets_max;
+ rv = qdma_device_init(xdev);
+ if (rv < 0) {
+ pr_warn("qdma_init failed %d.\n", rv);
+ qdma_device_cleanup(xdev);
+ }
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+/*****************************************************************************/
+/**
+ * qdma_get_qmax() - Handler function to get the qmax configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ * @return qmax value on success
+ * @return < 0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_qmax(unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ /**
+ * Return the current qsets_max value of the device
+ */
+ return xdev->conf.qsets_max;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_set_intr_rngsz() - Handler function to set the intr_ring_size value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] intr_rngsz: interrupt aggregation ring size
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_intr_rngsz(unsigned long dev_hndl, u32 intr_rngsz)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev;
+ int rv = -1;
+
+ /**
+ * If xdev is NULL or qdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev)
+ return -EINVAL;
+
+ /** If the input intr_rngsz is same as the
+ * current xdev->conf.intr_rngsz,
+ * return, as there is nothing to be changed
+ */
+ if (intr_rngsz == xdev->conf.intr_rngsz) {
+ pr_err("xdev 0x%p, Current intr_rngsz is same as [%d].Nothing to be done\n",
+ xdev, intr_rngsz);
+ return rv;
+ }
+
+ /** If interrupt aggregation is not enabled, then no need to change the
+ * interrupt ring size. Retrun error in this case.
+ */
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE)) {
+ pr_err("xdev 0x%p, interrupt aggregation is disabled\n", xdev);
+ return rv;
+ }
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That means at least one queue is added in the system.
+ * intr_rngsz is not allowed to change after FMAP programming is done.
+ * Hence, If qdev->init_qrange is set, return error as intr_rngsz
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, cannot modify intr ring size [%d]\n",
+ xdev,
+ xdev->conf.intr_rngsz);
+ return rv;
+ }
+
+ /**
+ * FMAP programming is not done yet, update the intr_rngsz
+ */
+ xdev->conf.intr_rngsz = intr_rngsz;
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_intr_rngsz() - Handler function to get the intr_ring_size value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return interrupt ring size on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_intr_rngsz(unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ /** If interrupt aggregation is not enabled, then return 0
+ * As the intr_rngsz value is irrelevant in this case
+ */
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE)) {
+ pr_info("xdev 0x%p, interrupt aggregation is disabled\n", xdev);
+ return 0;
+ }
+
+ pr_info("xdev 0x%p, intr ring_size = %d\n",
+ xdev,
+ xdev->conf.intr_rngsz);
+ /**
+ * Return the current intr_rngsz value of the device
+ */
+ return xdev->conf.intr_rngsz;
+}
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_set_buf_sz() - Handler function to set the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] buf_sz: interrupt aggregation ring size
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_buf_sz(unsigned long dev_hndl, u32 *buf_sz)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = -1;
+
+ /**
+ * If xdev is NULL or qdev is NULL,
+ * return error as Invalid parameter
+ */
+ if (!xdev || !qdev)
+ return -EINVAL;
+
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That means at least one queue is added in the system.
+ * intr_rngsz is not allowed to change after FMAP programming is done.
+ * Hence, If qdev->init_qrange is set, return error as buf_sz
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, cannot modify buf size\n",
+ xdev);
+ return rv;
+ }
+
+ /**
+ * Write the given buf sizes to the registers
+ */
+ rv = qdma_csr_write_bufsz(xdev, buf_sz);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_buf_sz() - Handler function to get the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return buffer size on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_buf_sz(unsigned long dev_hndl, u32 *buf_sz)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdma_csr_read_bufsz(xdev, buf_sz);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_set_glbl_rng_sz() - Handler function to set the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] buf_sz: interrupt aggregation ring size
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_glbl_rng_sz(unsigned long dev_hndl, u32 *glbl_rng_sz)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = -1;
+
+ /**
+ * If xdev is NULL or qdev is NULL,
+ * return error as Invalid parameter
+ */
+ if (!xdev || !qdev)
+ return -EINVAL;
+
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That means at least one queue is added in the system.
+ * intr_rngsz is not allowed to change after FMAP programming is done.
+ * Hence, If qdev->init_qrange is set, return error as glbl_rng_sz
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, cannot modify glbl_rng_sz\n",
+ xdev);
+ return rv;
+ }
+
+ /**
+ * Write the given buf sizes to the registers
+ */
+ rv = qdma_csr_write_rngsz(xdev, glbl_rng_sz);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_glbl_rng_sz() - Handler function to get the glbl_rng_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return glbl_rng_sz size on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_glbl_rng_sz(unsigned long dev_hndl, u32 *glbl_rng_sz)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdma_csr_read_rngsz(xdev, glbl_rng_sz);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_set_timer_cnt() - Handler function to set the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] tmr_cnt: Array of 16 timer count values
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_timer_cnt(unsigned long dev_hndl, u32 *tmr_cnt)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = -1;
+
+ if (!xdev || !qdev)
+ return -EINVAL;
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That meansat least one queue is added in the system.
+ * qmax is not allowed to change after FMAP programming is done.
+ * Hence, If qdev->init_qrange is set, return error as qmax
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, can not modify timer count\n",
+ xdev);
+ return rv;
+ }
+
+ rv = qdma_csr_write_timer_cnt(xdev, tmr_cnt);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_timer_cnt() - Handler function to get the timer_cnt value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return timer_cnt on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_timer_cnt(unsigned long dev_hndl, u32 *tmr_cnt)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdma_csr_read_timer_cnt(xdev, tmr_cnt);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_set_cnt_thresh() - Handler function to set the counter threshold value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] cnt_th: Array of 16 timer count values
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_cnt_thresh(unsigned long dev_hndl, unsigned int *cnt_th)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = -1;
+
+ if (!xdev || !qdev)
+ return -EINVAL;
+
+ /** If qdev->init_qrange is set,
+ * it indicates that FMAP programming is done
+ * That means at least one queue is added in the system.
+ * csr count threshold is not allowed to change after FMAP
+ * programming is done.
+ * Hence, If qdev->init_qrange is set, return error as csr count threshold
+ * cannot be changed now.
+ */
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, can not modify threshold count\n",
+ xdev);
+ return rv;
+ }
+
+ rv = qdma_csr_write_cnt_thresh(xdev, cnt_th);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_cnt_thresh() - Handler function to get the counter thresh value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return counter threshold values on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_cnt_thresh(unsigned long dev_hndl, u32 *cnt_th)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ qdma_csr_read_cnt_thresh(xdev, cnt_th);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_set_cmpl_status_acc() - Handler function to set the cmpl_status_acc
+ * configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] cmpl_status_acc: Writeback Accumulation value
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_cmpl_status_acc(unsigned long dev_hndl, u32 cmpl_status_acc)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ unsigned int reg;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ if (qdev->init_qrange) {
+ pr_err("xdev 0x%p, FMAP prog done, cannot modify cmpt acc\n",
+ xdev);
+ return -EINVAL;
+ }
+ /**
+ * Write the given cmpl_status_acc value to the register
+ */
+ reg = __read_reg(xdev, QDMA_REG_GLBL_DSC_CFG);
+ reg &= ~QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK;
+ reg |= cmpl_status_acc;
+ __write_reg(xdev, QDMA_REG_GLBL_DSC_CFG, reg);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_get_cmpl_status_acc() - Handler function to get the cmpl_status_acc
+ * configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ * Handler function to get the writeback accumulation value
+ *
+ * @return cmpl_status_acc on success
+ * @return <0 on failure
+ *
+ *****************************************************************************/
+unsigned int qdma_get_cmpl_status_acc(unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ unsigned int cs_acc;
+
+ /**
+ * If xdev is NULL, return error as Invalid parameter
+ */
+ if (!xdev)
+ return -EINVAL;
+
+ /**
+ * Read the current cmpl_status_acc value from the register and return
+ */
+ cs_acc = __read_reg(xdev, QDMA_REG_GLBL_DSC_CFG) &
+ QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK;
+
+ return cs_acc;
+}
+#endif
diff --git a/QDMA/linux-kernel/libqdma/libqdma_config.h b/QDMA/linux-kernel/libqdma/libqdma_config.h
new file mode 100644
index 0000000000000000000000000000000000000000..31f2374db4ce5840281a35f37338f27d779a8280
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/libqdma_config.h
@@ -0,0 +1,301 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __LIBQDMA_CONFIG_H__
+#define __LIBQDMA_CONFIG_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma configuration apis
+ *
+ */
+/*****************************************************************************
+ * GLOBAL CONSTANTS
+ *****************************************************************************/
+#include <linux/types.h>
+
+/**
+ * QDMA config bar number
+ */
+#define QDMA_CONFIG_BAR 0
+
+/**
+ * STM bar
+ */
+#define STM_BAR 2
+
+/**
+ * Maximum number of physical functions
+ */
+#define QDMA_PF_MAX 4 /* 4 PFs */
+/**
+ * Maximum number of virtual functions
+ */
+#define QDMA_VF_MAX 252
+
+/**
+ * Maximum number of queues per physical function
+ */
+#define QDMA_Q_PER_PF_MAX 512
+
+/**
+ * Maximum number of QDMA devices in the system
+ */
+#define MAX_DMA_DEV 32
+
+/**
+ * Total number of qdma qs
+ */
+#define TOTAL_QDMA_QS (QDMA_PF_MAX * QDMA_Q_PER_PF_MAX)
+
+/**
+ * Maximum number of queues per virtual function
+ */
+#define QDMA_Q_PER_VF_MAX 1
+
+/**
+ * Total number of qs for all VF
+ */
+#define TOTAL_VF_QS 0
+
+/**
+ * Total number of qs for all PFs
+ */
+#define TOTAL_PF_QS (TOTAL_QDMA_QS - TOTAL_VF_QS)
+
+/**
+ * Maximum number of qs for PF
+ */
+#define MAX_QS_PER_PF (TOTAL_PF_QS/QDMA_PF_MAX)
+
+/**
+ * Shift for bus 'B' in B:D:F
+ */
+#define PCI_SHIFT_BUS 12
+
+/**
+ * Shift for device 'D' in B:D:F
+ */
+#define PCI_SHIFT_DEV 4
+
+/**
+ * To shift the Bus number for getting BDF
+ */
+#define SHIFT_DEC_PCI_BUS 1000
+
+/**
+ * To shift the device number for getting BDF
+ */
+#define SHIFT_DEC_PCI_DEV 10
+
+/**
+ * Maximum number of MSI-X vector per function
+ */
+#define QDMA_DEV_MSIX_VEC_MAX 8
+
+/**
+ * ring size is 4KB, i.e 512 entries
+ */
+#define QDMA_INTR_COAL_RING_SIZE INTR_RING_SZ_4KB
+
+/** Maximum data vectors to be used for each function
+ * TODO: Please note that for 2018.2 only one vector would be used
+ * per pf and only one ring would be created for this vector
+ * It is also assumed that all functions have the same number of data vectors
+ * and currently different number of vectors per PF is not supported
+ */
+#define QDMA_NUM_DATA_VEC_FOR_INTR_CXT 1
+
+/*****************************************************************************
+ * Function Declaration
+ *****************************************************************************/
+
+/*****************************************************************************/
+/**
+ * qdma_set_qmax() - Handler function to set the qmax configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] qsets_max: qmax configuration value
+ * @param[in] forced: flag to force qmax change
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_qmax(unsigned long dev_hndl, u32 qsets_max, bool forced);
+
+/*****************************************************************************/
+/**
+ * qdma_get_qmax() - Handler function to get the qmax configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ * @return qmax value on success
+ * @return < 0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_qmax(unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_set_intr_rngsz() - Handler function to set the intr_ring_size value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] rngsz: interrupt aggregation ring size
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_intr_rngsz(unsigned long dev_hndl, u32 rngsz);
+
+/*****************************************************************************/
+/**
+ * qdma_get_intr_rngsz() - Handler function to get the intr_ring_size value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return interrupt ring size on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_intr_rngsz(unsigned long dev_hndl);
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_set_cmpl_status_acc() - Handler function to set the cmpl_status_acc
+ * configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] cmpl_status_acc: Writeback Accumulation value
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_cmpl_status_acc(unsigned long dev_hndl, u32 cmpl_status_acc);
+
+/*****************************************************************************/
+/**
+ * qdma_get_cmpl_status_acc() - Handler function to get the cmpl_status_acc
+ * configuration value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ * Handler function to get the writeback accumulation value
+ *
+ * @return cmpl_status_acc on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_cmpl_status_acc(unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_set_buf_sz() - Handler function to set the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] buf_sz: buf sizes
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_buf_sz(unsigned long dev_hndl, u32 *buf_sz);
+
+/*****************************************************************************/
+/**
+ * qdma_get_buf_sz() - Handler function to get the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] buf_sz: buf sizes
+ *
+ * @return buf sizes on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_buf_sz(unsigned long dev_hndl, u32 *buf_sz);
+
+/*****************************************************************************/
+/**
+ * qdma_set_glbl_rng_sz() - Handler function to set the glbl_rng_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] glbl_rng_sz: glbl_rng_sizes
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_glbl_rng_sz(unsigned long dev_hndl, u32 *glbl_rng_sz);
+
+/*****************************************************************************/
+/**
+ * qdma_get_glbl_rng_sz() - Handler function to get the glbl_rng_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] glbl_rng_sz: glbl_rng sizes
+ *
+ * @return glbl_rng_sz on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_glbl_rng_sz(unsigned long dev_hndl, u32 *glbl_rng_sz);
+
+/*****************************************************************************/
+/**
+ * qdma_set_timer_cnt() - Handler function to set the buf_sz value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] tmr_cnt: Array of 16 timer count values
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_timer_cnt(unsigned long dev_hndl, u32 *tmr_cnt);
+
+/*****************************************************************************/
+/**
+ * qdma_get_timer_cnt() - Handler function to get the timer_cnt value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return timer_cnt on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_timer_cnt(unsigned long dev_hndl, u32 *tmr_cnt);
+
+/*****************************************************************************/
+/**
+ * qdma_set_cnt_thresh() - Handler function to set the counter threshold value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] cnt_th: Array of 16 timer count values
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL on success
+ * @return <0 on failure
+ *****************************************************************************/
+int qdma_set_cnt_thresh(unsigned long dev_hndl, unsigned int *cnt_th);
+
+/*****************************************************************************/
+/**
+ * qdma_get_cnt_thresh() - Handler function to get the counter thresh value
+ *
+ * @param[in] dev_hndl: qdma device handle
+ *
+ *
+ * @return counter threshold values on success
+ * @return <0 on failure
+ *****************************************************************************/
+unsigned int qdma_get_cnt_thresh(unsigned long dev_hndl, u32 *cnt_th);
+#endif
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/libqdma_export.c b/QDMA/linux-kernel/libqdma/libqdma_export.c
new file mode 100644
index 0000000000000000000000000000000000000000..50a88bcb14177f825e30db01ad6002bff96753fc
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/libqdma_export.c
@@ -0,0 +1,2037 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+/**
+ * @file
+ * @brief This file contains the definitions for libqdma interfaces
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "libqdma_export.h"
+
+#include "qdma_descq.h"
+#include "qdma_device.h"
+#include "qdma_thread.h"
+#include "qdma_regs.h"
+#include "qdma_context.h"
+#include "qdma_intr.h"
+#include "qdma_st_c2h.h"
+#include "thread.h"
+#include "version.h"
+
+#ifdef DEBUGFS
+#include "qdma_debugfs_queue.h"
+
+/** debugfs root */
+struct dentry *qdma_debugfs_root;
+#endif
+
+#define QDMA_Q_PEND_LIST_COMPLETION_TIMEOUT 1000 /* msec */
+
+struct drv_mode_name mode_name_list[] = {
+ { AUTO_MODE, "auto"},
+ { POLL_MODE, "poll"},
+ { DIRECT_INTR_MODE, "direct interrupt"},
+ { INDIRECT_INTR_MODE, "indirect interrupt"},
+ { LEGACY_INTR_MODE, "legacy interrupt"}
+};
+/* ********************* static function definitions ************************ */
+/*****************************************************************************/
+/**
+ * qdma_request_wait_for_cmpl() - static function to monitor the
+ * wait completion
+ *
+ * @param[in] xdev: pointer to xlnx_dma_dev structure
+ * @param[in] descq: pointer to qdma_descq structure
+ * @param[in] req: qdma request
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int qdma_request_wait_for_cmpl(struct xlnx_dma_dev *xdev,
+ struct qdma_descq *descq, struct qdma_request *req)
+{
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+
+ /** if timeout is mentioned in the request,
+ * wait until the timeout occurs or wait until the
+ * call back is completed
+ */
+
+ if (req->timeout_ms)
+ qdma_waitq_wait_event_timeout(cb->wq, cb->done,
+ msecs_to_jiffies(req->timeout_ms));
+ else
+ qdma_waitq_wait_event(cb->wq, cb->done);
+
+ lock_descq(descq);
+ /** if the call back is not done, request timed out
+ * delete the request list
+ */
+ if (!cb->done)
+ list_del(&cb->list);
+
+ /** if the call back is not done but the status is updated
+ * return i/o error
+ */
+ if (!cb->done || cb->status) {
+ pr_info("%s: req 0x%p, %c,%u,%u/%u,0x%llx, done %d, err %d, tm %u.\n",
+ descq->conf.name,
+ req, req->write ? 'W' : 'R',
+ cb->offset,
+ cb->left,
+ req->count,
+ req->ep_addr,
+ cb->done,
+ cb->status,
+ req->timeout_ms);
+ qdma_descq_dump(descq, NULL, 0, 1);
+ unlock_descq(descq);
+
+ return -EIO;
+ }
+
+ unlock_descq(descq);
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_request_submit_st_c2h() - static function to handle the
+ * st c2h request
+ *
+ * @param[in] xdev: pointer to xlnx_dma_dev structure
+ * @param[in] descq: pointer to qdma_descq structure
+ * @param[in] req: qdma request
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t qdma_request_submit_st_c2h(struct xlnx_dma_dev *xdev,
+ struct qdma_descq *descq, struct qdma_request *req)
+{
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+ int wait = req->fp_done ? 0 : 1;
+ int rv = 0;
+
+ /** make sure that qdev is not NULL, else return error */
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ pr_debug("%s: data len %u, sgl 0x%p, sgl cnt %u, tm %u ms.\n",
+ descq->conf.name,
+ req->count, req->sgl, req->sgcnt, req->timeout_ms);
+
+ /** get the request count */
+ cb->left = req->count;
+
+ lock_descq(descq);
+ if (descq->q_stop_wait) {
+ unlock_descq(descq);
+ return 0;
+ }
+ if ((descq->q_state == Q_STATE_ONLINE) &&
+ !descq->q_stop_wait) {
+ /* add to pend list even before cidx/pidx update as it could
+ * cause an interrupt and may miss processing of writeback
+ */
+ list_add_tail(&cb->list, &descq->pend_list);
+ /* any rcv'ed packet not yet read ? */
+ /** read the data from the device */
+ descq_st_c2h_read(descq, req, 1, 1);
+ if (!cb->left) {
+ list_del(&cb->list);
+ unlock_descq(descq);
+ return req->count;
+ }
+ descq->pend_list_empty = 0;
+ unlock_descq(descq);
+ } else {
+ unlock_descq(descq);
+ pr_info("%s descq %s NOT online.\n",
+ xdev->conf.name, descq->conf.name);
+ return -EINVAL;
+ }
+
+ /** if there is a completion thread associated,
+ * wake up the completion thread to process the
+ * completion status
+ */
+ if (descq->cmplthp)
+ qdma_kthread_wakeup(descq->cmplthp);
+
+ if (!wait) {
+ pr_debug("%s: cb 0x%p, data len 0x%x NO wait.\n",
+ descq->conf.name, cb, req->count);
+ return 0;
+ }
+
+ /** wait for the request completion */
+ rv = qdma_request_wait_for_cmpl(xdev, descq, req);
+ if (rv < 0) {
+ if (!req->dma_mapped)
+ sgl_unmap(xdev->conf.pdev, req->sgl, req->sgcnt,
+ DMA_FROM_DEVICE);
+ return rv;
+ }
+
+ /** Once the request completion received,
+ * return with the number of processed requests
+ */
+ return req->count - cb->left;
+}
+
+/* ********************* public function definitions ************************ */
+
+/*****************************************************************************/
+/**
+ * qdma_queue_get_config() - retrieve the configuration of a queue
+ *
+ * @param[in] dev_hndl: dev_hndl retured from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+struct qdma_queue_conf *qdma_queue_get_config(unsigned long dev_hndl,
+ unsigned long id, char *buf, int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 0);
+
+ /** make sure that descq is not NULL
+ * return error is it is null else return the config data
+ */
+ if (descq)
+ return &descq->conf;
+
+ return NULL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump() - display a queue's state in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl, id,
+ buf, buflen, 0);
+ struct hw_descq_context ctxt;
+ int len = 0;
+ int rv;
+#ifndef __QDMA_VF__
+ int ring_index = 0;
+ u32 intr_ctxt[4];
+ int i = 0;
+#endif
+
+ if (!descq)
+ return -EINVAL;
+
+ /** Make sure that buf and buflen is not invalid */
+ if (!buf || !buflen)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+
+ /** read the descq data to dump */
+ qdma_descq_dump(descq, buf, buflen, 1);
+ len = strlen(buf);
+
+ /** read the descq context for the given qid */
+ rv = qdma_descq_context_read(descq->xdev, descq->qidx_hw,
+ descq->conf.st, descq->conf.c2h, &ctxt);
+ if (rv < 0) {
+ len += sprintf(buf + len, "%s read context failed %d.\n",
+ descq->conf.name, rv);
+ return rv;
+ }
+
+ /** format the output for all contexts */
+ len += sprintf(buf + len,
+ "\tSW CTXT: [4]:0x%08x [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ ctxt.sw[4], ctxt.sw[3], ctxt.sw[2], ctxt.sw[1], ctxt.sw[0]);
+
+ len += sprintf(buf + len,
+ "\tHW CTXT: [1]:0x%08x [0]:0x%08x\n",
+ ctxt.hw[1], ctxt.hw[0]);
+
+ len += sprintf(buf + len,
+ "\tCR CTXT: 0x%08x\n", ctxt.cr[0]);
+
+ /** incase of ST C2H,
+ * add completion and prefetch context to the output
+ */
+ if (descq->conf.c2h && descq->conf.st) {
+ len += sprintf(buf + len,
+ "\tCMPT CTXT: [4]:0x%08x [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ ctxt.cmpt[4], ctxt.cmpt[3], ctxt.cmpt[2], ctxt.cmpt[1], ctxt.cmpt[0]);
+
+ len += sprintf(buf + len,
+ "\tPFTCH CTXT: [1]:0x%08x [0]:0x%08x\n",
+ ctxt.prefetch[1], ctxt.prefetch[0]);
+ }
+
+#ifndef __QDMA_VF__
+ /** if interrupt aggregation is enabled
+ * add the interrupt context
+ */
+ if ((descq->xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (descq->xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ for (i = 0; i < QDMA_NUM_DATA_VEC_FOR_INTR_CXT; i++) {
+ ring_index = get_intr_ring_index(
+ descq->xdev,
+ (i + descq->xdev->dvec_start_idx));
+ rv = qdma_intr_context_read(descq->xdev,
+ ring_index,
+ 4,
+ intr_ctxt);
+ if (rv < 0) {
+ len += sprintf(buf + len,
+ "%s read intr context failed %d.\n",
+ descq->conf.name, rv);
+ return rv;
+ }
+
+ len += sprintf(buf + len,
+ "\tINTR_CTXT[%d]: [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ ring_index,
+ intr_ctxt[3],
+ intr_ctxt[2],
+ intr_ctxt[1],
+ intr_ctxt[0]);
+ }
+ }
+
+ if (descq->xdev->stm_en) {
+ if (descq->conf.st &&
+ (descq->conf.c2h || descq->conf.desc_bypass)) {
+ struct stm_descq_context stm_hw;
+
+ memset(&stm_hw, 0, sizeof(stm_hw));
+ /** read stm context */
+ rv = qdma_descq_stm_read(descq->xdev, descq->qidx_hw,
+ descq->conf.pipe_flow_id,
+ descq->conf.c2h, false, true,
+ &stm_hw);
+ if (rv < 0) {
+ len += sprintf(buf + len,
+ "%s read stm-context failed %d.\n",
+ descq->conf.name, rv);
+ return rv;
+ }
+
+ len += sprintf(buf + len,
+ "\tSTM CTXT: [5]:0x%08x [4]:0x%08x [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ stm_hw.stm[5], stm_hw.stm[4],
+ stm_hw.stm[3], stm_hw.stm[2],
+ stm_hw.stm[1], stm_hw.stm[0]);
+
+ /** read stm can */
+ rv = qdma_descq_stm_read(descq->xdev, descq->qidx_hw,
+ descq->conf.pipe_flow_id,
+ descq->conf.c2h, false, false,
+ &stm_hw);
+ if (rv < 0) {
+ len += sprintf(buf + len,
+ "%s read stm-can failed %d.\n",
+ descq->conf.name, rv);
+ return rv;
+ }
+
+ len += sprintf(buf + len,
+ "\tSTM CAN: [4]:0x%08x [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ stm_hw.stm[4], stm_hw.stm[3],
+ stm_hw.stm[2], stm_hw.stm[1],
+ stm_hw.stm[0]);
+
+ /** read stm c2h map or h2c map */
+ rv = qdma_descq_stm_read(descq->xdev, descq->qidx_hw,
+ descq->conf.pipe_flow_id,
+ descq->conf.c2h, true, false,
+ &stm_hw);
+ if (rv < 0) {
+ len += sprintf(buf + len,
+ "%s read stm-map failed %d.\n",
+ descq->conf.name, rv);
+ return rv;
+ }
+
+ if (descq->conf.c2h)
+ len += sprintf(buf + len,
+ "\tSTM C2H MAP: [0]:0x%08x\n",
+ stm_hw.stm[0]);
+ else
+ len += sprintf(buf + len,
+ "\tSTM H2C MAP: [4]:0x%08x [3]:0x%08x [2]:0x%08x [1]:0x%08x [0]:0x%08x\n",
+ stm_hw.stm[4], stm_hw.stm[3],
+ stm_hw.stm[2], stm_hw.stm[1],
+ stm_hw.stm[0]);
+ }
+ }
+#endif
+ len += sprintf(buf + len,
+ "\ttotal descriptor processed: %llu\n",
+ descq->total_cmpl_descs);
+ /** set the buffer end with \0 and return the buffer length */
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump_desc() - display a queue's descriptor ring from index start
+ * ~ end in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] start: start index
+ * @param[in] end: end index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump_desc(unsigned long dev_hndl, unsigned long id,
+ unsigned int start, unsigned int end, char *buf,
+ int buflen)
+{
+ struct qdma_descq *descq = NULL;
+ int len = 0;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /** Assume that sufficient buflen is provided
+ * make sure that device handle provided is valid
+ */
+ if (!xdev || !buf || !buflen)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+
+ /** get the descq details based on the qid provided */
+ descq = qdma_device_get_descq_by_id(xdev, id, buf, buflen, 1);
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ /** dump the queue state */
+ len = qdma_descq_dump_state(descq, buf, buflen);
+ if (descq->q_state != Q_STATE_ONLINE)
+ return len;
+
+ /** dump the queue descriptor state */
+ len += qdma_descq_dump_desc(descq, start, end, buf + len, buflen - len);
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump_cmpt() - display a queue's descriptor ring from index start
+ * ~ end in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] start: start index
+ * @param[in] end: end index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump_cmpt(unsigned long dev_hndl, unsigned long id,
+ unsigned int start, unsigned int end, char *buf,
+ int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ int len = 0;
+
+ /** make sure that descq is not NULL, else return error */
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ /** dump the descriptor state */
+ len = qdma_descq_dump_state(descq, buf, buflen);
+ /** if the descriptor is in online state,
+ * then, dump the completion state
+ */
+ if (descq->q_state == Q_STATE_ONLINE)
+ len += qdma_descq_dump_cmpt(descq, start, end, buf + len,
+ buflen - len);
+
+ return len;
+}
+
+/* TODO: Currently this interface is not being used by any application.
+ * Enable the code when needed !
+ */
+#if 0
+/*****************************************************************************/
+/**
+ * qdma_queue_get_buf_sz() - to get queue's ch_buf_sz
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_get_buf_sz(unsigned long dev_hndl, unsigned long id,
+ char *buf,
+ int buflen)
+{
+ struct qdma_descq *descq = NULL;
+ int len = 0;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /** Assume that sufficient buflen is provided
+ * make sure that device handle provided is valid
+ */
+ if (!xdev || !buf || !buflen)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+
+ /** get the descq details based on the qid provided */
+ descq = qdma_device_get_descq_by_id(xdev, id, buf, buflen, 1);
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ /** dump the queue state */
+ len = qdma_descq_dump_state(descq, buf, buflen);
+ if (descq->q_state != Q_STATE_ONLINE)
+ return len;
+
+ /** dump the queue buf_sz */
+
+ /* reg = QDMA_REG_C2H_BUF_SZ_BASE + (descq->conf.c2h_buf_sz_idx)*4; */
+ /* buf_sz = __read_reg(xdev, reg); */
+ buflen = snprintf(buf, buflen, "%d", descq->conf.c2h_bufsz);
+ return descq->conf.c2h_bufsz;
+}
+#endif
+/*****************************************************************************/
+/**
+ * qdma_queue_remove() - remove a queue (i.e., offline, NOT ready for dma)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_remove(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_descq *descq =
+ qdma_device_get_descq_by_id(xdev, id, buf, buflen, 1);
+#ifdef DEBUGFS
+ struct qdma_descq *pair_descq =
+ qdma_device_get_pair_descq_by_id(xdev, id, buf, buflen, 1);
+#endif
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+
+ /** make sure that qdev is not NULL, else return error */
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ /** make sure that descq is not NULL, else return error */
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (descq->q_state != Q_STATE_ENABLED) {
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen,
+ "queue %s, id %u cannot be deleted. Invalid q state\n",
+ descq->conf.name, descq->conf.qidx);
+
+ buf[len] = '\0';
+ }
+ return QDMA_ERR_INVALID_DESCQ_STATE;
+
+ }
+
+#ifdef DEBUGFS
+ if (pair_descq)
+ /** if pair_descq is not NULL, it means the queue
+ * is in ENABLED state
+ */
+ dbgfs_queue_exit(&descq->conf, pair_descq);
+ else
+ dbgfs_queue_exit(&descq->conf, NULL);
+#endif
+ qdma_descq_cleanup(descq);
+
+ lock_descq(descq);
+ descq->q_state = Q_STATE_DISABLED;
+ unlock_descq(descq);
+
+ spin_lock(&qdev->lock);
+ if (descq->conf.c2h)
+ qdev->c2h_qcnt--;
+ else
+ qdev->h2c_qcnt--;
+ spin_unlock(&qdev->lock);
+#ifndef __QDMA_VF__
+ if (xdev->conf.qdma_drv_mode == LEGACY_INTR_MODE)
+ intr_legacy_clear(descq);
+#endif
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen, "queue %s, id %u deleted.\n",
+ descq->conf.name, descq->conf.qidx);
+ buf[len] = '\0';
+ }
+ pr_debug("queue %s, id %u deleted.\n",
+ descq->conf.name, descq->conf.qidx);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_config() - configure the queue with qcong parameters
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qid: queue id
+ * @param[in] qconf: queue configuration parameters
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_config(unsigned long dev_hndl, unsigned long qid,
+ struct qdma_queue_conf *qconf, char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct qdma_descq *descq = NULL;
+
+ /** make sure that qdev is not NULL, else return error */
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ /** get the descq for the given qid */
+ descq = qdma_device_get_descq_by_id(xdev, qid, NULL, 0, 0);
+ if (!descq) {
+ pr_err("Invalid queue ID! qid=%lu, max=%u\n", qid, qdev->qmax);
+ return QDMA_ERR_INVALID_QIDX;
+ }
+
+ lock_descq(descq);
+ /** if descq is not in disabled state, return error as
+ * queue is enabled/in use, else enable the descq
+ */
+ if (descq->q_state != Q_STATE_DISABLED) {
+ pr_err("queue_%lu already configured!\n", qid);
+ unlock_descq(descq);
+ return -EINVAL;
+ }
+ descq->q_state = Q_STATE_ENABLED;
+ unlock_descq(descq);
+
+ spin_lock(&qdev->lock);
+ /** increment the queue count */
+ if (qconf->c2h)
+ qdev->c2h_qcnt += 1;
+ else
+ qdev->h2c_qcnt += 1;
+ spin_unlock(&qdev->lock);
+
+ /** configure descriptor queue */
+ qdma_descq_config(descq, qconf, 0);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_list() - display all configured queues in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_list(unsigned long dev_hndl, char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct qdma_descq *descq = NULL;
+ char *cur = buf;
+ char * const end = buf + buflen;
+ int i;
+
+ /** make sure that qdev is not NULL, else return error */
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n", dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ /** make sure that input buffer is not empty, else return error */
+ if (!buf || !buflen) {
+ pr_warn("invalid argument: buf=%p, buflen=%d", buf, buflen);
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+ }
+
+ cur += snprintf(cur, end - cur, "H2C Q: %u, C2H Q: %u.\n",
+ qdev->h2c_qcnt, qdev->c2h_qcnt);
+ if (cur >= end)
+ goto handle_truncation;
+
+ /** traverse through the h2c and c2h queue list
+ * and dump the descriptors
+ */
+ if (qdev->h2c_qcnt) {
+ descq = qdev->h2c_descq;
+ for (i = 0; i < qdev->qmax; i++, descq++) {
+ lock_descq(descq);
+ if (descq->q_state != Q_STATE_DISABLED)
+ cur +=
+ qdma_descq_dump(descq, cur, end - cur, 0);
+ unlock_descq(descq);
+
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ }
+
+ if (qdev->c2h_qcnt) {
+ descq = qdev->c2h_descq;
+ for (i = 0; i < qdev->qmax; i++, descq++) {
+ lock_descq(descq);
+ if (descq->q_state != Q_STATE_DISABLED)
+ cur +=
+ qdma_descq_dump(descq, cur, end - cur, 0);
+ unlock_descq(descq);
+
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ }
+
+ return cur - buf;
+
+handle_truncation:
+
+ return cur - buf;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_reconfig() - reconfigure the existing queue with
+ * modified configuration
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: existing queue id
+ * @param[in] qconf: queue configuration parameters
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_reconfig(unsigned long dev_hndl, unsigned long id,
+ struct qdma_queue_conf *qconf,
+ char *buf, int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+
+ /** make sure that descq is not NULL, else return error */
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ lock_descq(descq);
+ /** if descq is not in disabled state then,
+ * return error as queue configuration can not be changed
+ */
+ if (descq->q_state != Q_STATE_ENABLED) {
+ pr_info("%s invalid state, q_state %d.\n",
+ descq->conf.name, descq->q_state);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += sprintf(buf + l,
+ "%s invalid state, q_state %d.\n",
+ descq->conf.name, descq->q_state);
+ buf[l] = '\0';
+ }
+ unlock_descq(descq);
+ return QDMA_ERR_INVALID_DESCQ_STATE;
+ }
+ /** Update the qconfig with new configuration provided */
+ qdma_descq_config(descq, qconf, 1);
+ unlock_descq(descq);
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_add() - add a queue
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qconf: queue configuration parameters
+ * @param[in] qhndl: list of unsigned long values that are the opaque qhndl
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_add(unsigned long dev_hndl, struct qdma_queue_conf *qconf,
+ unsigned long *qhndl, char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct qdma_descq *descq;
+ struct qdma_descq *pairq;
+ unsigned int qcnt;
+ char *cur = buf;
+ char * const end = buf + buflen;
+ int rv = QDMA_OPERATION_SUCCESSFUL;
+
+ /** If qconf is NULL, return error*/
+ if (!qconf)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+
+#ifdef __QDMA_VF__
+ /** allow only Q0 for if the VF Qmax is not set */
+ if ((xdev->conf.cur_cfg_state == QMAX_CFG_INITIAL) &&
+ (qconf->qidx > 0)) {
+ rv = QDMA_ERR_INVALID_QIDX;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "qdma%05x invalid idx %u >= 1.\n",
+ xdev->conf.bdf, qconf->qidx);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ return rv;
+ }
+#endif
+
+ /** If qhndl is NULL, return error*/
+ if (!qhndl) {
+ pr_warn("qhndl NULL.\n");
+ rv = QDMA_ERR_INVALID_QIDX;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "%s, add, qhndl NULL.\n",
+ xdev->conf.name);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ return rv;
+ }
+
+ /** reset qhandle to an invalid value
+ * can't use 0 or NULL here because queue idx 0 has same value
+ */
+ *qhndl = QDMA_QUEUE_IDX_INVALID;
+
+ /** check if the requested mode is enabled?
+ * the request modes are read from the HW
+ * before serving any request, first check if the
+ * HW has the capability or not, else return error
+ */
+ if ((qconf->st && !xdev->st_mode_en) ||
+ (!qconf->st && !xdev->mm_mode_en)) {
+ pr_warn("%s, %s mode not enabled.\n",
+ xdev->conf.name, qconf->st ? "ST" : "MM");
+ rv = QDMA_ERR_INTERFACE_NOT_ENABLED_IN_DEVICE;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "qdma%05x %s mode not enabled.\n",
+ xdev->conf.bdf, qconf->st ? "ST" : "MM");
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ return rv;
+ }
+
+ spin_lock(&qdev->lock);
+ /** if incase the qidx is not QDMA_QUEUE_IDX_INVALID
+ * then, make sure that the qidx range falls between
+ * 0 - qdev->qmax, else return error
+ */
+ if ((qconf->qidx != QDMA_QUEUE_IDX_INVALID) &&
+ (qconf->qidx >= qdev->qmax)) {
+ spin_unlock(&qdev->lock);
+ rv = QDMA_ERR_INVALID_QIDX;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "qdma%05x invalid idx %u >= %u.\n",
+ xdev->conf.bdf, qconf->qidx, qdev->qmax);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ return rv;
+ }
+
+ /** check if any free qidx available
+ * if qcnt is >= qdev->qmax, return error as
+ * no free queues found and descq is full
+ */
+ qcnt = qconf->c2h ? qdev->c2h_qcnt : qdev->h2c_qcnt;
+ if (qcnt >= qdev->qmax) {
+ spin_unlock(&qdev->lock);
+ pr_warn("No free queues %u/%u.\n", qcnt, qdev->qmax);
+ rv = QDMA_ERR_DESCQ_FULL;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "qdma%05x No free queues %u/%u.\n",
+ xdev->conf.bdf, qcnt, qdev->qmax);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ return rv;
+ }
+
+ /** add to the count first, need to rewind if failed later*/
+ if (qconf->c2h)
+ qdev->c2h_qcnt++;
+ else
+ qdev->h2c_qcnt++;
+ spin_unlock(&qdev->lock);
+
+ if (qconf->c2h) {
+ descq = qdev->c2h_descq;
+ pairq = qdev->h2c_descq;
+ } else {
+ descq = qdev->h2c_descq;
+ pairq = qdev->c2h_descq;
+ }
+ /** need to allocate a free qidx if it has an invalid id
+ * ie. qidx is not specified in the add request
+ */
+ if (qconf->qidx == QDMA_QUEUE_IDX_INVALID) {
+ int i;
+
+ /** loop through the qlist till qmax and find an empty descq*/
+ for (i = 0; i < qdev->qmax; i++, descq++, pairq++) {
+ /** make sure the queue pair are the same mode*/
+ lock_descq(pairq);
+ if ((pairq->q_state != Q_STATE_DISABLED)
+ && qconf->st != pairq->conf.st) {
+ unlock_descq(pairq);
+ continue;
+ }
+ unlock_descq(pairq);
+
+ lock_descq(descq);
+ if (descq->q_state != Q_STATE_DISABLED) {
+ unlock_descq(descq);
+ continue;
+ }
+ /** set the descq as enabled*/
+ descq->q_state = Q_STATE_ENABLED;
+ /** assign the qidx */
+ qconf->qidx = i;
+ unlock_descq(descq);
+
+ break;
+ }
+
+ /** we are reached here means no free descq found
+ * decrement the queue count and return error
+ */
+ if (i == qdev->qmax) {
+ pr_warn("no free %s qp found, %u.\n",
+ qconf->st ? "ST" : "MM", qdev->qmax);
+ rv = QDMA_ERR_DESCQ_FULL;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "qdma%05x no %s QP, %u.\n",
+ xdev->conf.bdf,
+ qconf->st ? "ST" : "MM", qdev->qmax);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ goto rewind_qcnt;
+ }
+ } else { /** qidx specified in the given add request*/
+ /** find the queue pair for the given qidx*/
+ pairq += qconf->qidx;
+ descq += qconf->qidx;
+
+ /** make sure the queue pair are the same mode*/
+ lock_descq(pairq);
+ if ((pairq->q_state != Q_STATE_DISABLED)
+ && (qconf->st != pairq->conf.st)) {
+ unlock_descq(pairq);
+ rv = -EINVAL;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "Need to have same mode for Q pair.\n");
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ goto rewind_qcnt;
+ }
+ unlock_descq(pairq);
+
+ lock_descq(descq);
+ /** if the descq for the given qidx is already in enabled state,
+ * then the queue is in use, return error
+ */
+ if (descq->q_state != Q_STATE_DISABLED) {
+ unlock_descq(descq);
+ pr_info("descq idx %u already added.\n", qconf->qidx);
+ rv = QDMA_ERR_DESCQ_IDX_ALREADY_ADDED;
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur,
+ "q idx %u already added.\n",
+ qconf->qidx);
+ }
+ goto rewind_qcnt;
+ }
+ descq->q_state = Q_STATE_ENABLED;
+ unlock_descq(descq);
+ }
+
+ /** prepare the queue resources*/
+ rv = qdma_device_prep_q_resource(xdev);
+ if (rv < 0) {
+#ifdef __QDMA_VF__
+ lock_descq(descq);
+ descq->q_state = Q_STATE_DISABLED;
+ unlock_descq(descq);
+#endif
+ goto rewind_qcnt;
+ }
+#ifndef __QDMA_VF__
+ if (xdev->conf.qdma_drv_mode == LEGACY_INTR_MODE) {
+ rv = intr_legacy_setup(descq);
+ if ((rv > 0) && buf && buflen) {
+ /** support only 1 queue in legacy interrupt mode */
+ intr_legacy_clear(descq);
+ descq->q_state = Q_STATE_DISABLED;
+ pr_debug("qdma%05x - Q%u - No free queues %u/%u.\n",
+ xdev->conf.bdf, descq->conf.qidx,
+ rv, 1);
+ rv = -EINVAL;
+ cur += snprintf(cur, end - cur,
+ "qdma%05x No free queues %u/%u.\n",
+ xdev->conf.bdf, qcnt, 1);
+ goto rewind_qcnt;
+ } else if ((rv < 0) && buf && buflen) {
+ rv = -EINVAL;
+ descq->q_state = Q_STATE_DISABLED;
+ pr_debug("qdma%05x Legacy interrupt setup failed.\n",
+ xdev->conf.bdf);
+ cur += snprintf(cur, end - cur,
+ "qdma%05x Legacy interrupt setup failed.\n",
+ xdev->conf.bdf);
+ goto rewind_qcnt;
+ }
+ }
+#endif
+
+ /** fill in config. info */
+ qdma_descq_config(descq, qconf, 0);
+
+ /** copy back the name in config*/
+ memcpy(qconf->name, descq->conf.name, QDMA_QUEUE_NAME_MAXLEN);
+ *qhndl = (unsigned long)descq->conf.qidx;
+ if (qconf->c2h)
+ *qhndl += qdev->qmax;
+
+#ifdef DEBUGFS
+ rv = dbgfs_queue_init(&descq->conf, pairq, xdev->dbgfs_queues_root);
+ if (rv < 0) {
+ pr_err("failed to create queue debug files for the queueu %d\n",
+ descq->conf.qidx);
+ }
+#endif
+
+ pr_debug("added %s, %s, qidx %u.\n",
+ descq->conf.name, qconf->c2h ? "C2H" : "H2C", qconf->qidx);
+ if (buf && buflen) {
+ cur += snprintf(cur, end - cur, "%s %s added.\n",
+ descq->conf.name, qconf->c2h ? "C2H" : "H2C");
+ if (cur >= end)
+ goto handle_truncation;
+ }
+
+ return QDMA_OPERATION_SUCCESSFUL;
+
+rewind_qcnt:
+ spin_lock(&qdev->lock);
+ if (qconf->c2h)
+ qdev->c2h_qcnt--;
+ else
+ qdev->h2c_qcnt--;
+ spin_unlock(&qdev->lock);
+
+ return rv;
+
+handle_truncation:
+ *buf = '\0';
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_queue_start() - start a queue (i.e, online, ready for dma)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_start(unsigned long dev_hndl, unsigned long id,
+ char *buf, int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ int rv;
+
+ /** make sure that descq is not NULL, else return error*/
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ /** complete the queue configuration*/
+ rv = qdma_descq_config_complete(descq);
+ if (rv < 0) {
+ pr_err("%s 0x%x setup failed.\n",
+ descq->conf.name, descq->qidx_hw);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s config failed.\n", descq->conf.name);
+ buf[l] = '\0';
+ }
+ goto free_resource;
+ }
+
+ lock_descq(descq);
+ /** if the descq is not enabled,
+ * it is in invalid state, return error
+ */
+ if (descq->q_state != Q_STATE_ENABLED) {
+ pr_info("%s invalid state, q_status%d.\n",
+ descq->conf.name, descq->q_state);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s invalid state, q_state %d.\n",
+ descq->conf.name, descq->q_state);
+ }
+ unlock_descq(descq);
+ return QDMA_ERR_INVALID_DESCQ_STATE;
+ }
+ unlock_descq(descq);
+ /** allocate the queue resources*/
+ rv = qdma_descq_alloc_resource(descq);
+ if (rv < 0) {
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s alloc resource failed.\n",
+ descq->conf.name);
+ buf[l] = '\0';
+ }
+ goto free_resource;
+ }
+
+ /** program the hw contexts*/
+ rv = qdma_descq_prog_hw(descq);
+ if (rv < 0) {
+ pr_err("%s 0x%x setup failed.\n",
+ descq->conf.name, descq->qidx_hw);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s prog. context failed.\n",
+ descq->conf.name);
+ buf[l] = '\0';
+ }
+ goto clear_context;
+ }
+
+ /** Interrupt mode */
+ if (descq->xdev->num_vecs) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&descq->xdev->lock, flags);
+ list_add_tail(&descq->intr_list,
+ &descq->xdev->dev_intr_info_list[descq->intr_id].intr_list);
+ spin_unlock_irqrestore(&descq->xdev->lock, flags);
+ }
+
+ qdma_thread_add_work(descq);
+
+ if (buf && buflen) {
+ rv = snprintf(buf, buflen, "%s started\n", descq->conf.name);
+ if (rv <= 0 || rv >= buflen)
+ goto clear_context;
+ }
+ /** set the descq to online state*/
+ lock_descq(descq);
+ descq->q_state = Q_STATE_ONLINE;
+ unlock_descq(descq);
+
+ return QDMA_OPERATION_SUCCESSFUL;
+
+clear_context:
+ qdma_descq_context_clear(descq->xdev, descq->qidx_hw, descq->conf.st,
+ descq->conf.c2h, 1);
+free_resource:
+ qdma_descq_free_resource(descq);
+
+ return rv;
+}
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_queue_prog_stm() - Program STM for queue (context, map, etc)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_prog_stm(unsigned long dev_hndl, unsigned long id,
+ char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_descq *descq =
+ qdma_device_get_descq_by_id((struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ int rv;
+
+ /** make sure that descq is not NULL, else return error*/
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (!descq->conf.st) {
+ pr_info("%s Skipping STM prog for MM queue.\n",
+ descq->conf.name);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s Skipping STM prog for MM queue.\n",
+ descq->conf.name);
+ }
+ return QDMA_OPERATION_SUCCESSFUL;
+ }
+
+ if (!xdev->stm_en) {
+ pr_info("%s No STM present; stm_rev %d.\n",
+ descq->conf.name, xdev->stm_rev);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s No STM present; stm_rev %d.\n",
+ descq->conf.name, xdev->stm_rev);
+ }
+ return QDMA_ERR_INVALID_PCI_DEV;
+ }
+
+ lock_descq(descq);
+ /** if the descq is not online,
+ * it is in invalid state, return error
+ */
+ if (descq->q_state != Q_STATE_ONLINE) {
+ pr_info("%s invalid state, q_status%d.\n",
+ descq->conf.name, descq->q_state);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s invalid state, q_state %d.\n",
+ descq->conf.name, descq->q_state);
+ }
+ unlock_descq(descq);
+ return QDMA_ERR_INVALID_DESCQ_STATE;
+ }
+
+ unlock_descq(descq);
+
+ /** program the stm */
+ rv = qdma_descq_prog_stm(descq, false);
+ if (rv < 0) {
+ pr_err("%s 0x%x STM setup failed.\n",
+ descq->conf.name, descq->qidx_hw);
+ if (buf && buflen) {
+ int l = strlen(buf);
+
+ l += snprintf(buf + l, buflen,
+ "%s prog. STM failed.\n",
+ descq->conf.name);
+ buf[l] = '\0';
+ }
+ return rv;
+ }
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+#endif
+
+/*****************************************************************************/
+/**
+ * qdma_queue_stop() - stop a queue (i.e., offline, NOT ready for dma)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_stop(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ struct qdma_sgt_req_cb *cb, *tmp;
+ struct qdma_request *req;
+ unsigned int pend_list_empty = 0;
+
+ /** make sure that descq is not NULL, else return error */
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ lock_descq(descq);
+ /** if the descq not online donot proceed */
+ if (descq->q_state != Q_STATE_ONLINE) {
+ pr_info("%s invalid state, q_state %d.\n",
+ descq->conf.name, descq->q_state);
+ if (buf && buflen) {
+ int l = snprintf(buf, buflen,
+ "queue %s, idx %u stop failed.\n",
+ descq->conf.name, descq->conf.qidx);
+ if (l <= 0 || l >= buflen) {
+ unlock_descq(descq);
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+ }
+ }
+ unlock_descq(descq);
+ return QDMA_ERR_INVALID_DESCQ_STATE;
+ }
+ pend_list_empty = descq->pend_list_empty;
+
+ descq->q_stop_wait = 1;
+ unlock_descq(descq);
+ if (!pend_list_empty) {
+ qdma_waitq_wait_event_timeout(descq->pend_list_wq,
+ descq->pend_list_empty,
+ msecs_to_jiffies(QDMA_Q_PEND_LIST_COMPLETION_TIMEOUT));
+ }
+ lock_descq(descq);
+ /** free the descq by updating the state */
+ descq->q_state = Q_STATE_ENABLED;
+ descq->q_stop_wait = 0;
+ list_for_each_entry_safe(cb, tmp, &descq->pend_list, list) {
+ req = (struct qdma_request *)cb;
+ cb->req_state = QDMA_REQ_COMPLETE;
+ cb->done = 1;
+ cb->status = -ENXIO;
+ if (req->fp_done) {
+ list_del(&cb->list);
+ req->fp_done(req, 0, -ENXIO);
+ } else
+ qdma_waitq_wakeup(&cb->wq);
+ }
+ list_for_each_entry_safe(cb, tmp, &descq->work_list, list) {
+ req = (struct qdma_request *)cb;
+ cb->req_state = QDMA_REQ_COMPLETE;
+ cb->done = 1;
+ cb->status = -ENXIO;
+ if (req->fp_done) {
+ list_del(&cb->list);
+ req->fp_done(req, 0, -ENXIO);
+ } else
+ qdma_waitq_wakeup(&cb->wq);
+ }
+ unlock_descq(descq);
+
+ /** remove the work thread associated with the current queue */
+ qdma_thread_remove_work(descq);
+
+ /** clear the queue context */
+ qdma_descq_context_clear(descq->xdev, descq->qidx_hw, descq->conf.st,
+ descq->conf.c2h, 0);
+
+ lock_descq(descq);
+ /** if the device is in direct/indirect interrupt mode,
+ * delete the interrupt list for the queue
+ */
+ if ((descq->xdev->conf.qdma_drv_mode != POLL_MODE) &&
+ (descq->xdev->conf.qdma_drv_mode != LEGACY_INTR_MODE)) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&descq->xdev->lock, flags);
+ list_del(&descq->intr_list);
+ spin_unlock_irqrestore(&descq->xdev->lock, flags);
+ }
+#ifndef __QDMA_VF__
+ /** clear stm context/maps */
+ if (descq->xdev->stm_en)
+ qdma_descq_prog_stm(descq, true);
+#endif
+
+ /** free the queue resources */
+ qdma_descq_free_resource(descq);
+ /** free the descq by updating the state */
+ descq->total_cmpl_descs = 0;
+ unlock_descq(descq);
+
+ /** fill the return buffer indicating that queue is stopped */
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen, "queue %s, idx %u stopped.\n",
+ descq->conf.name, descq->conf.qidx);
+ if (len <= 0 || len >= buflen)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+ }
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_intr_ring_dump() - display the interrupt ring info of a vector
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] vector_idx: vector number
+ * @param[in] start_idx: interrupt ring start idx
+ * @param[in] end_idx: interrupt ring end idx
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_intr_ring_dump(unsigned long dev_hndl, unsigned int vector_idx,
+ int start_idx, int end_idx, char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_intr_ring *ring_entry;
+ struct intr_coal_conf *coal_entry;
+ int counter = 0;
+ int len = 0;
+ u32 data[2];
+
+ /** if, interrupt aggregation is not enabled,
+ * interrupt ring is not created, return error
+ */
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE)) {
+ pr_info("Interrupt aggregation not enabled\n");
+ if (buf) {
+ len += sprintf(buf + len,
+ "Interrupt aggregation not enabled\n");
+ buf[len] = '\0';
+ }
+ return -1;
+ }
+
+ /** make sure that vector index is with in the
+ * start and end vector limit, else return error
+ */
+ if ((vector_idx < xdev->dvec_start_idx) ||
+ (vector_idx >=
+ (xdev->dvec_start_idx + QDMA_NUM_DATA_VEC_FOR_INTR_CXT))) {
+ pr_info("Vector idx %d is invalid. Shall be in range: %d - %d.\n",
+ vector_idx,
+ xdev->dvec_start_idx,
+ (xdev->dvec_start_idx +
+ QDMA_NUM_DATA_VEC_FOR_INTR_CXT - 1));
+ if (buf) {
+ len += sprintf(buf + len,
+ "Vector idx %d is invalid. Shall be in range: %d - %d.\n",
+ vector_idx,
+ xdev->dvec_start_idx,
+ (xdev->dvec_start_idx +
+ QDMA_NUM_DATA_VEC_FOR_INTR_CXT - 1));
+ buf[len] = '\0';
+ }
+ return -1;
+ }
+
+ /** get the intr entry based on vector index */
+ coal_entry = xdev->intr_coal_list + (vector_idx - xdev->dvec_start_idx);
+
+ /** make sure that intr ring entry indexes
+ * given are with in the range
+ */
+ if (start_idx > coal_entry->intr_rng_num_entries) {
+ pr_info("start_idx %d is invalid. Shall be less than: %d\n",
+ start_idx,
+ coal_entry->intr_rng_num_entries);
+ if (buf) {
+ len += sprintf(buf + len,
+ "start_idx %d is invalid. Shall be less than: %d\n",
+ start_idx,
+ coal_entry->intr_rng_num_entries);
+ buf[len] = '\0';
+ }
+ return -1;
+ }
+
+ if (end_idx == -1 || end_idx >= coal_entry->intr_rng_num_entries)
+ end_idx = coal_entry->intr_rng_num_entries - 1;
+
+ if (start_idx == -1)
+ start_idx = 0;
+
+ if (start_idx > end_idx) {
+ pr_info("start_idx can't be greater than end_idx\n");
+ if (buf) {
+ len += sprintf(buf + len,
+ "start_idx can't be greater than end_idx\n");
+ buf[len] = '\0';
+ }
+ return -1;
+ }
+
+ /** read the ring entries based on the range given and
+ * update the input buffer with details
+ */
+ for (counter = start_idx; counter <= end_idx; counter++) {
+ ring_entry = coal_entry->intr_ring_base + counter;
+ memcpy(data, ring_entry, sizeof(u32) * 2);
+ if (buf) {
+ len += sprintf(buf + len,
+ "intr_ring_entry = %d: 0x%08x 0x%08x\n",
+ counter, data[1], data[0]);
+ buf[len] = '\0';
+ }
+ }
+
+ return 0;
+}
+
+ /*****************************************************************************/
+ /**
+ * qdma_software_version_info Provides the qdma software version
+ *
+ * @param[out] software_version: libqdma software version
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_software_version_info(char *software_version)
+{
+ if (!software_version)
+ return -EINVAL;
+
+ sprintf(software_version, "%s", LIBQDMA_VERSION_STR);
+
+ return 0;
+}
+
+
+#ifdef __QDMA_VF__
+ /*****************************************************************************/
+ /**
+ * qdma_vf_qconf call for VF to request qmax number of Qs
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qmax: number of qs requested by vf
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_vf_qconf(unsigned long dev_hndl, int qmax)
+{
+ int err = 0;
+ u32 qbase = 0;
+ struct qdma_dev *qdev;
+ u32 last_qbase = 0;
+ u32 last_qmax = 0;
+
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!xdev) {
+ pr_err("Invalid dev handle\n");
+ return -EINVAL;
+ }
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev) {
+ pr_err("Invalid qdev\n");
+ return -EINVAL;
+ }
+
+ /** save the last qbase values for restoring
+ * if the qconf command failed
+ */
+ last_qbase = xdev->conf.qsets_base;
+ last_qmax = xdev->conf.qsets_max;
+
+ qdma_device_cleanup(xdev);
+ err = device_set_qconf(xdev, qmax, &qbase);
+ if (err < 0) {
+ pr_err("Setting qconf failed\n");
+ /** upon failure, go back to the last set qmax value */
+ xdev->conf.qsets_base = last_qbase;
+ xdev->conf.qsets_max = last_qmax;
+ qdma_device_init(xdev);
+ return err;
+ }
+ xdev->conf.qsets_base = qbase;
+ xdev->conf.qsets_max = qmax;
+ err = qdma_device_init(xdev);
+ if (err < 0) {
+ pr_warn("qdma_init failed %d.\n", err);
+ qdma_device_cleanup(xdev);
+ }
+
+ if (err < 0) {
+ pr_err("Failed to set qmax %d for %s\n",
+ qmax, dev_name(&xdev->conf.pdev->dev));
+ return err;
+ }
+
+ return err;
+}
+#endif
+
+/*****************************************************************************/
+/**
+ * sgl_unmap() - unmap the sg list from host pages
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] sg: qdma sg request list
+ * @param[in] sgcnt: number of sg lists
+ * @param[in] dir: direction of the dma transfer
+ * DMA_BIDIRECTIONAL = 0, DMA_TO_DEVICE = 1,
+ * DMA_FROM_DEVICE = 2, DMA_NONE = 3,
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+void sgl_unmap(struct pci_dev *pdev, struct qdma_sw_sg *sg, unsigned int sgcnt,
+ enum dma_data_direction dir)
+{
+ int i;
+
+ /** unmap the sg list and set the dma_addr to 0 all sg entries */
+ for (i = 0; i < sgcnt; i++, sg++) {
+ if (!sg->pg)
+ break;
+ if (sg->dma_addr) {
+ pci_unmap_page(pdev, sg->dma_addr - sg->offset,
+ PAGE_SIZE, dir);
+ sg->dma_addr = 0UL;
+ }
+ }
+}
+
+/*****************************************************************************/
+/**
+ * sgl_map() - map sg list to host pages
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] sg: qdma sg request list
+ * @param[in] sgcnt: number of sg lists
+ * @param[in] dir: direction of the dma transfer
+ * DMA_BIDIRECTIONAL = 0, DMA_TO_DEVICE = 1,
+ * DMA_FROM_DEVICE = 2, DMA_NONE = 3,
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int sgl_map(struct pci_dev *pdev, struct qdma_sw_sg *sgl, unsigned int sgcnt,
+ enum dma_data_direction dir)
+{
+ int i;
+ struct qdma_sw_sg *sg = sgl;
+
+ /** Map the sg list onto a dma pages where
+ * each page has max of PAGE_SIZE i.e 4K
+ */
+ for (i = 0; i < sgcnt; i++, sg++) {
+ /* !! TODO page size !! */
+ sg->dma_addr = pci_map_page(pdev, sg->pg, 0, PAGE_SIZE, dir);
+ if (unlikely(pci_dma_mapping_error(pdev, sg->dma_addr))) {
+ pr_info("map sgl failed, sg %d, %u.\n", i, sg->len);
+ if (i)
+ sgl_unmap(pdev, sgl, i, dir);
+ return -EIO;
+ }
+ sg->dma_addr += sg->offset;
+ }
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_request_submit() - submit a scatter-gather list of data for dma
+ * operation (for both read and write)
+ * This is a callback function called from upper layer(character device)
+ * to handle the read/write request on the queues
+ *
+ * @param[in] dev_hndl: hndl retured from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] req: qdma request
+ *
+ * @return # of bytes transferred
+ * @return <0: error
+ *****************************************************************************/
+ssize_t qdma_request_submit(unsigned long dev_hndl, unsigned long id,
+ struct qdma_request *req)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_descq *descq =
+ qdma_device_get_descq_by_id(xdev, id, NULL, 0, 1);
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+ enum dma_data_direction dir;
+ int wait = req->fp_done ? 0 : 1;
+ int rv = 0;
+
+ if (!descq)
+ return -EINVAL;
+
+ pr_debug("%s %s-%s, data len %u, sg cnt %u.\n",
+ descq->conf.name, descq->conf.st ? "ST" : "MM",
+ descq->conf.c2h ? "C2H" : "H2C", req->count, req->sgcnt);
+
+ /** Identify the direction of the transfer */
+ dir = descq->conf.c2h ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+ /** If write request is given on the C2H direction
+ * OR, a read request given on non C2H direction
+ * then, its an invalid request, return error in this case
+ */
+ if ((req->write && descq->conf.c2h) ||
+ (!req->write && !descq->conf.c2h)) {
+ pr_info("%s: bad direction, %c.\n",
+ descq->conf.name, req->write ? 'W' : 'R');
+ return -EINVAL;
+ }
+
+ /** Reset the local cb request with 0's */
+ memset(cb, 0, QDMA_REQ_OPAQUE_SIZE);
+ /** Initialize the wait queue */
+ qdma_waitq_init(&cb->wq);
+
+ pr_debug("%s: data len %u, ep 0x%llx, sgl 0x%p, sgl cnt %u, tm %u ms.\n",
+ descq->conf.name, req->count, req->ep_addr, req->sgl,
+ req->sgcnt, req->timeout_ms);
+
+ /** If the request is streaming mode C2H, invoke the
+ * handler to perform the read operation
+ */
+ if (descq->conf.st && descq->conf.c2h)
+ return qdma_request_submit_st_c2h(xdev, descq, req);
+
+ if (!req->dma_mapped) {
+ rv = sgl_map(xdev->conf.pdev, req->sgl, req->sgcnt, dir);
+ if (rv < 0) {
+ pr_info("%s map sgl %u failed, %u.\n",
+ descq->conf.name, req->sgcnt, req->count);
+ goto unmap_sgl;
+ }
+ cb->unmap_needed = 1;
+ }
+
+ lock_descq(descq);
+ /** if the descq is already in online state*/
+ if (descq->q_state != Q_STATE_ONLINE) {
+ unlock_descq(descq);
+ pr_info("%s descq %s NOT online.\n",
+ xdev->conf.name, descq->conf.name);
+ rv = -EINVAL;
+ goto unmap_sgl;
+ }
+ list_add_tail(&cb->list, &descq->work_list);
+ descq->pend_req_desc += ((req->count + PAGE_SIZE - 1) >> PAGE_SHIFT);
+ unlock_descq(descq);
+
+ pr_debug("%s: cb 0x%p submitted.\n", descq->conf.name, cb);
+
+ qdma_descq_proc_sgt_request(descq);
+
+ if (!wait)
+ return 0;
+
+ rv = qdma_request_wait_for_cmpl(xdev, descq, req);
+ if (rv < 0)
+ goto unmap_sgl;
+
+ return cb->offset;
+
+unmap_sgl:
+ if (!req->dma_mapped)
+ sgl_unmap(xdev->conf.pdev, req->sgl, req->sgcnt, dir);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_batch_request_submit() - submit a batch of scatter-gather list of data
+ * for dma operation (for both read and write)
+ * This is a callback function called from upper layer(character device)
+ * to handle the read/write request on the queues
+ *
+ * @param[in] dev_hndl: hndl retured from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] count: Number of qdma requests
+ * @param[in] reqv: qdma request vector
+ *
+ * @return # of bytes transferred
+ * @return <0: error
+ *****************************************************************************/
+ssize_t qdma_batch_request_submit(unsigned long dev_hndl, unsigned long id,
+ unsigned long count, struct qdma_request **reqv)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_descq *descq =
+ qdma_device_get_descq_by_id(xdev, id, NULL, 0, 0);
+ struct qdma_sgt_req_cb *cb;
+ enum dma_data_direction dir;
+ int rv = 0;
+ unsigned long i;
+ struct qdma_request *req;
+ int st_c2h = 0;
+
+ if (!descq)
+ return -EINVAL;
+
+ st_c2h = (descq->conf.st && descq->conf.c2h) ? 1 : 0;
+
+ /** Identify the direction of the transfer */
+ dir = descq->conf.c2h ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+ req = reqv[0];
+ /** If write request is given on the C2H direction
+ * OR, a read request given on non C2H direction
+ * then, its an invalid request, return error in this case
+ */
+ if ((req->write && descq->conf.c2h)
+ || (!req->write && !descq->conf.c2h)) {
+ pr_info("%s: bad direction, %c.\n", descq->conf.name,
+ req->write ? 'W' : 'R');
+ return -EINVAL;
+ }
+ if (!req->fp_done) {
+ pr_info("%s: missing fp_done.\n", descq->conf.name);
+ return -EINVAL;
+ }
+
+ if (st_c2h) {
+ for (i = 0; i < count; i++) {
+ req = reqv[i];
+ cb = qdma_req_cb_get(req);
+ /** Reset the local cb request with 0's */
+ memset(cb, 0, QDMA_REQ_OPAQUE_SIZE);
+
+ rv = qdma_request_submit_st_c2h(xdev, descq, req);
+ if ((rv < 0) || (rv == req->count))
+ req->fp_done(req, rv, rv);
+ }
+
+ return 0;
+
+ } else {
+ struct pci_dev *pdev = xdev->conf.pdev;
+
+ for (i = 0; i < count; i++) {
+ req = reqv[i];
+ cb = qdma_req_cb_get(req);
+ /** Reset the local cb request with 0's */
+ memset(cb, 0, QDMA_REQ_OPAQUE_SIZE);
+
+ if (!req->dma_mapped) {
+ rv = sgl_map(pdev, req->sgl, req->sgcnt, dir);
+ if (unlikely(rv < 0)) {
+ pr_info("%s map sgl %u failed, %u.\n",
+ descq->conf.name,
+ req->sgcnt,
+ req->count);
+ req->fp_done(req, 0, rv);
+ }
+ cb->unmap_needed = 1;
+ }
+ }
+ }
+
+ lock_descq(descq);
+ /** if the descq is already in online state*/
+ if (unlikely(descq->q_state != Q_STATE_ONLINE)) {
+ unlock_descq(descq);
+ pr_info("%s descq %s NOT online.\n", xdev->conf.name,
+ descq->conf.name);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < count; i++) {
+ req = reqv[i];
+ cb = qdma_req_cb_get(req);
+
+ list_add_tail(&cb->list, &descq->work_list);
+ }
+ unlock_descq(descq);
+
+ qdma_descq_proc_sgt_request(descq);
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * libqdma_init() initialize the QDMA core library
+ *
+ * @param[in] num_threads - number of threads to be created each for request
+ * processing and writeback processing
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int libqdma_init(enum qdma_drv_mode qdma_drv_mode, unsigned int num_threads)
+{
+
+ /** Make sure that the size of qdma scatter gather request size
+ * is less than the QDMA_REQ_OPAQUE_SIZE
+ * If not, return error
+ */
+ if (sizeof(struct qdma_sgt_req_cb) > QDMA_REQ_OPAQUE_SIZE) {
+ pr_err("dma req. opaque data size too big %lu > %d.\n",
+ sizeof(struct qdma_sgt_req_cb), QDMA_REQ_OPAQUE_SIZE);
+ return -1;
+ }
+ if (sizeof(struct qdma_flq) > QDMA_FLQ_SIZE) {
+ pr_err("qdma flq data size too big %lu > %d",
+ sizeof(struct qdma_flq), QDMA_FLQ_SIZE);
+ return -1;
+ }
+
+ if (qdma_drv_mode == LEGACY_INTR_MODE)
+ intr_legacy_init();
+
+ /** Create the qdma threads */
+ qdma_threads_create(num_threads);
+#ifdef DEBUGFS
+ return qdma_debugfs_init(&qdma_debugfs_root);
+#else
+ return 0;
+#endif
+}
+
+/*****************************************************************************/
+/**
+ * libqdma_exit() cleanup the QDMA core library before exiting
+ *
+ * @return none
+ *****************************************************************************/
+void libqdma_exit(void)
+{
+#ifdef DEBUGFS
+ qdma_debugfs_exit(qdma_debugfs_root);
+#endif
+ /** Destroy the qdma threads */
+ qdma_threads_destroy();
+}
+
+#ifdef __LIBQDMA_MOD__
+/** for module support only */
+#include "version.h"
+
+static char version[] =
+ DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
+
+MODULE_AUTHOR("Xilinx, Inc.");
+MODULE_DESCRIPTION(DRV_MODULE_DESC);
+MODULE_VERSION(DRV_MODULE_VERSION);
+MODULE_LICENSE("Dual BSD/GPL");
+
+/*****************************************************************************/
+/**
+ * libqdma_mod_init() libqdma module initialization routine
+ *
+ * Returns: none
+ *****************************************************************************/
+static int __init libqdma_mod_init(void)
+{
+ pr_info("%s", version);
+
+ /** invoke libqdma_init to initialize the libqdma library */
+ return libqdma_init();
+}
+
+/*****************************************************************************/
+/**
+ * libqdma_mod_exit() cleanup the QDMA core library before exiting
+ *
+ * Returns: none
+ *****************************************************************************/
+static void __exit libqdma_mod_exit(void)
+{
+ /** invoke libqdma_exit to cleanup the libqdma library resources */
+ libqdma_exit();
+}
+
+module_init(libqdma_mod_init);
+module_exit(libqdma_mod_exit);
+#endif /* ifdef __LIBQDMA_MOD__ */
diff --git a/QDMA/linux-kernel/libqdma/libqdma_export.h b/QDMA/linux-kernel/libqdma/libqdma_export.h
new file mode 100644
index 0000000000000000000000000000000000000000..dd1128cdf0c218488daedc07d9c7c90efe7f3b31
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/libqdma_export.h
@@ -0,0 +1,1228 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __LIBQDMA_EXPORT_API_H__
+#define __LIBQDMA_EXPORT_API_H__
+/**
+ * @file
+ * @brief This file contains the declarations for libqdma interfaces
+ *
+ */
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include "libqdma_config.h"
+
+/** Invalid QDMA function number */
+#define QDMA_FUNC_ID_INVALID (QDMA_PF_MAX + QDMA_VF_MAX)
+
+/**
+ * QDMA Error codes
+ */
+enum qdma_error_codes {
+ QDMA_OPERATION_SUCCESSFUL = 0,
+ /*!< QDMA driver API operation successful */
+ QDMA_ERR_PCI_DEVICE_NOT_FOUND = -1,
+ /*!< QDMA PCI device not found on the PCIe bus */
+ QDMA_ERR_PCI_DEVICE_ALREADY_ATTACHED = -2,
+ /*!< QDMA PCI device already attached */
+ QDMA_ERR_PCI_DEVICE_ENABLE_FAILED = -3,
+ /*!< Failed to enable the QDMA PCIe device */
+ QDMA_ERR_PCI_DEVICE_INIT_FAILED = -4,
+ /*!< Failed to initialize the QDMA PCIe device */
+ QDMA_ERR_INVALID_INPUT_PARAM = -5,
+ /*!< Invalid input parameter given to QDMA API */
+ QDMA_ERR_INVALID_PCI_DEV = -6,
+ /*!< Invalid PCIe device */
+ QDMA_ERR_INVALID_QIDX = -7,
+ /*!< Invalid Queue ID provided as input */
+ QDMA_ERR_INVALID_DESCQ_STATE = -8,
+ /*!< Invalid descriptor queue state */
+ QDMA_ERR_INVALID_DIRECTION = -9,
+ /*!< Invalid descriptor direction provided */
+ QDMA_ERR_DESCQ_SETUP_FAILED = -10,
+ /*!< Failed to setup the descriptor queue */
+ QDMA_ERR_DESCQ_FULL = -11,
+ /*!< Descriptor queue is full */
+ QDMA_ERR_DESCQ_IDX_ALREADY_ADDED = -12,
+ /*!< Descriptor queue index is already added */
+ QDMA_ERR_QUEUE_ALREADY_CONFIGURED = -13,
+ /*!< Queue is already configured */
+ QDMA_ERR_OUT_OF_MEMORY = -14,
+ /*!< Out of memory */
+ QDMA_ERR_INVALID_QDMA_DEVICE = -15,
+ /*!< Invalid QDMA device, QDMA device is not yet created */
+ QDMA_ERR_INTERFACE_NOT_ENABLED_IN_DEVICE = -16,
+ /*!< The ST or MM or Both interface not enabled in the device */
+};
+
+
+/**
+ * qdma_drv_state_t - qdma driver state
+ */
+enum qdma_drv_mode {
+ /*!< auto mode decided automatically, mix of poll and interrupt mode */
+ AUTO_MODE,
+ /*!< driver is inserted in poll mode */
+ POLL_MODE,
+ /*!< driver is inserted in direct interrupt mode */
+ DIRECT_INTR_MODE,
+ /*!< driver is inserted in indirect interrupt mode */
+ INDIRECT_INTR_MODE,
+ /*!< driver is inserted in legacy interrupt mode */
+ LEGACY_INTR_MODE
+};
+
+struct drv_mode_name {
+ enum qdma_drv_mode drv_mode;
+ char name[20];
+};
+extern struct drv_mode_name mode_name_list[];
+
+struct pci_dev;
+
+/*****************************************************************************/
+/**
+ * libqdma_init() initialize the QDMA core library
+ *
+ * @param[in] qdma_drv_mode - mode in which the driver needs to operate
+ * @param[in] num_threads - number of threads to be created each for request
+ * processing and writeback processing
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int libqdma_init(enum qdma_drv_mode qdma_drv_mode, unsigned int num_threads);
+
+/*****************************************************************************/
+/**
+ * libqdma_exit() cleanup the QDMA core library before exiting
+ *
+ * @return none
+ *****************************************************************************/
+void libqdma_exit(void);
+
+/**
+ * intr_ring_size_sel - qdma interrupt ring size selection
+ */
+enum intr_ring_size_sel {
+ /**
+ * 0 - INTR_RING_SZ_4KB, Accommodates 512 entries
+ */
+ INTR_RING_SZ_4KB = 0,
+ /**
+ * 1 - INTR_RING_SZ_8KB, Accommodates 1024 entries
+ */
+ INTR_RING_SZ_8KB,
+ /**
+ * 2 - INTR_RING_SZ_12KB, Accommodates 1536 entries
+ */
+ INTR_RING_SZ_12KB,
+ /**
+ * 3 - INTR_RING_SZ_16KB, Accommodates 2048 entries
+ */
+ INTR_RING_SZ_16KB,
+ /**
+ * 4 - INTR_RING_SZ_20KB, Accommodates 2560 entries
+ */
+ INTR_RING_SZ_20KB,
+ /**
+ * 5 - INTR_RING_SZ_24KB, Accommodates 3072 entries
+ */
+ INTR_RING_SZ_24KB,
+ /**
+ * 6 - INTR_RING_SZ_24KB, Accommodates 3584 entries
+ */
+ INTR_RING_SZ_28KB,
+ /**
+ * 7 - INTR_RING_SZ_24KB, Accommodates 4096 entries
+ */
+ INTR_RING_SZ_32KB,
+};
+
+enum qdma_dev_qmax_state {
+ /** device qmax not configured */
+ QMAX_CFG_UNCONFIGURED,
+ /** device qmax configured with initial values */
+ QMAX_CFG_INITIAL,
+ /** device qmax configured from sysfs */
+ QMAX_CFG_USER,
+};
+
+/**
+ * Maxinum length of the QDMA device name
+ */
+#define QDMA_DEV_NAME_MAXLEN 32
+
+/**
+ * qdma_dev_conf defines the per-device qdma property.
+ *
+ * NOTE: if any of the max requested is less than supported, the value will
+ * be updated
+ */
+struct qdma_dev_conf {
+ /** pointer to pci_dev */
+ struct pci_dev *pdev;
+ /** Maximum number of queue pairs per device */
+ unsigned short qsets_max;
+ /** Reserved */
+ unsigned short rsvd2;
+ /** Indicates whether zero length DMA is allowed or not */
+ u8 zerolen_dma:1;
+ /** Indicates whether the current pf is master_pf or not */
+ u8 master_pf:1;
+ /** extra handling of per descq handling in
+ * top half (i.e., qdma_descq.fp_descq_isr_top will be set)
+ */
+ u8 isr_top_q_en:1;
+ /** Reserved1 */
+ u8 rsvd1:5;
+ /** Maximum number of virtual functions for current physical function */
+ u8 vf_max;
+ /** Interrupt ring size */
+ u8 intr_rngsz;
+ /**
+ * interrupt:
+ * - MSI-X only
+ * max of QDMA_DEV_MSIX_VEC_MAX per function, 32 in Everest
+ * - 1 vector is reserved for user interrupt
+ * - 1 vector is reserved mailbox
+ * - 1 vector on pf0 is reserved for error interrupt
+ * - the remaining vectors will be used for queues
+ */
+
+ /** max. of vectors used for queues. libqdma update w/ actual # */
+ u8 msix_qvec_max;
+ /** upper layer data, i.e. callback data */
+ unsigned long uld;
+ /** qdma driver mode */
+ enum qdma_drv_mode qdma_drv_mode;
+ /**
+ * an unique string to identify the dev.
+ * current format: qdma[pf|vf][idx] filled in by libqdma
+ */
+ char name[QDMA_DEV_NAME_MAXLEN];
+
+#ifdef RTL2_FEATURE_CONFIGURABLE_BAR
+ /** dma config bar #, < 0 not present */
+ char bar_num_config;
+ /** user bar */
+ char bar_num_user;
+ /** bypass bar */
+ char bar_num_bypass;
+#endif
+ /** dma config bar #, < 0 not present */
+ char bar_num_config;
+ /** user bar */
+ char bar_num_user;
+ /** bypass bar */
+ char bar_num_bypass;
+ /** STM bar, PF only */
+ char bar_num_stm;
+ /** user bar, PF only */
+ unsigned int qsets_base;
+ /** device index */
+ u32 bdf;
+ /** index of device in device list */
+ u32 idx;
+ /**current configuration state of device*/
+ enum qdma_dev_qmax_state cur_cfg_state;
+ /** xmit in traffic manager mode */
+ u8 tm_mode_en;
+ /** enable 1 CDH for Traffic Manager */
+ u8 tm_one_cdh_en;
+
+ /** user interrupt, if null, default libqdma handler is used */
+ void (*fp_user_isr_handler)(unsigned long dev_hndl, unsigned long uld);
+
+ /**
+ * example flow of ST C2H:
+ * a. interrupt fires
+ * b. Hard IRQ: libqdma isr top -> dev->fp_q_isr_top_dev ->
+ * isr_top_qproc && Q->fp_descq_isr_top
+ * c. Soft IRQ:
+ * irq handler
+ * qdma_queue_service_bh() ->
+ * if rx: Q->fp_descq_rx_packet() called for each packet
+ * qdma_queue_cmpl_ctrl(set=true) to update h/w and re-enable
+ * interrupt
+ */
+
+ /** Q interrupt top, per-device addtional handling code */
+ void (*fp_q_isr_top_dev)(unsigned long dev_hndl, unsigned long uld);
+};
+
+/*****************************************************************************/
+/**
+ * qdma_device_open() - read the pci bars and configure the fpga
+ * This API should be called from probe()
+ *
+ * User interrupt will not be enabled until qdma_user_isr_enable() is called
+ *
+ * @param[in] mod_name: the module name, used for request_irq
+ * @param[in] conf: device configuration
+ * @param[out] dev_hndl: an opaque handle
+ * for libqdma to identify the device
+ *
+ * @return QDMA_OPERATION_SUCCESSFUL success
+ * @return <0 in case of error
+ *****************************************************************************/
+int qdma_device_open(const char *mod_name, struct qdma_dev_conf *conf,
+ unsigned long *dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_close() - prepare fpga for removal: disable all interrupts (users
+ * and qdma) and release all resources.This API should be called from remove()
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl retured from qdma_device_open()
+ *
+ *****************************************************************************/
+void qdma_device_close(struct pci_dev *pdev, unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_offline() - Set the device in offline mode
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl retured from qdma_device_open()
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+void qdma_device_offline(struct pci_dev *pdev, unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_online() - Set the device in online mode and re-initialze it
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl retured from qdma_device_open()
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_online(struct pci_dev *pdev, unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_flr_quirk_set() - start pre-flr processing
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_flr_quirk_set(struct pci_dev *pdev, unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_flr_quirk_check() - check if pre-flr processing completed
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_flr_quirk_check(struct pci_dev *pdev, unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_config() - retrieve the current device configuration
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ * @param[in] conf: device configuration
+ * @param[in] ebuflen: input buffer length
+ * @param[out] ebuf:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_get_config(unsigned long dev_hndl, struct qdma_dev_conf *conf,
+ char *ebuf, int ebuflen);
+
+/*****************************************************************************/
+/**
+ * qdma_device_clear_stats() - clear device statistics
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_clear_stats(unsigned long dev_hndl);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_mmh2c_pkts() - get mm h2c packets processed
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ * @param[out] mmh2c_pkts: number of mm h2c packets processed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_get_mmh2c_pkts(unsigned long dev_hndl,
+ unsigned long long *mmh2c_pkts);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_mmc2h_pkts() - get mm c2h packets processed
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ * @param[out] mmc2h_pkts: number of mm c2h packets processed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_get_mmc2h_pkts(unsigned long dev_hndl,
+ unsigned long long *mmc2h_pkts);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_sth2c_pkts() - get st h2c packets processed
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ * @param[out] sth2c_pkts: number of st h2c packets processed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_get_sth2c_pkts(unsigned long dev_hndl,
+ unsigned long long *sth2c_pkts);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_stc2h_pkts() - get st c2h packets processed
+ *
+ * @param[in] dev_hndl: dev_hndl retunred from qdma_device_open()
+ * @param[out] stc2h_pkts: number of st c2h packets processed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_get_stc2h_pkts(unsigned long dev_hndl,
+ unsigned long long *stc2h_pkts);
+/*****************************************************************************/
+/**
+ * qdma_device_set_config() - set the current device configuration
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] conf: device configuration to set
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_set_config(unsigned long dev_hndl, struct qdma_dev_conf *conf);
+
+/*****************************************************************************/
+/**
+ * qdma_device_set_cfg_state - set the device configuration state
+ *
+ * @param[in] dev_hndl: device handle
+ * @param[in] new_cfg_state: dma device conf state to set
+ *
+ *
+ * @return 0 on success ,<0 on failure
+ *****************************************************************************/
+int qdma_device_set_cfg_state(unsigned long dev_hndl,
+ enum qdma_dev_qmax_state new_cfg_state);
+
+/*****************************************************************************/
+/**
+ * qdma_device_sriov_config() - configure sriov
+ *
+ * @param[in] pdev: ptr to struct pci_dev
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] num_vfs: # of VFs to be instantiated
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_sriov_config(struct pci_dev *pdev, unsigned long dev_hndl,
+ int num_vfs);
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_config_register() - read dma config. register
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the config register
+ *****************************************************************************/
+unsigned int qdma_device_read_config_register(unsigned long dev_hndl,
+ unsigned int reg_addr);
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_config_register() - write dma config. register
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] reg_addr: register address
+ * @param[in] value: register value to be writen
+ *
+ *****************************************************************************/
+void qdma_device_write_config_register(unsigned long dev_hndl,
+ unsigned int reg_addr, u32 value);
+
+
+/**
+ * qdma_version_info defines the per-device version information
+ *
+ */
+#define DEVICE_VERSION_INFO_STR_LENGTH 10
+struct qdma_version_info {
+ u8 rtl_version;
+ char rtl_version_str[DEVICE_VERSION_INFO_STR_LENGTH];
+ u8 vivado_release_id;
+ char vivado_release_str[DEVICE_VERSION_INFO_STR_LENGTH];
+ u8 everest_ip;
+ char everest_ip_str[DEVICE_VERSION_INFO_STR_LENGTH];
+};
+
+/*****************************************************************************/
+/**
+ * qdma_device_version_info() - retrieve the RTL version , Vivado Release ID and
+ * Everest IP info
+ *
+ * @param[in] dev_hndl: handle returned from qdma_device_open()
+ * @param[out] version_info: pointer to hold all the version details
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_device_version_info(unsigned long dev_hndl,
+ struct qdma_version_info *version_info);
+
+/*****************************************************************************/
+/**
+ * qdma_software_version_info() - retrieve the software version
+ *
+ * @param[out] software_version: A pointer to a null-terminated string
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_software_version_info(char *software_version);
+
+/** QDMA Global CSR array size */
+#define QDMA_GLOBAL_CSR_ARRAY_SZ 16
+
+/**
+ * struct global_csr_conf - global CSR configuration
+ */
+struct global_csr_conf {
+ /** Descriptor ring size ie. queue depth */
+ unsigned int ring_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+ /** C2H timer count list */
+ unsigned int c2h_timer_cnt[QDMA_GLOBAL_CSR_ARRAY_SZ];
+ /** C2H counter threshold list*/
+ unsigned int c2h_cnt_th[QDMA_GLOBAL_CSR_ARRAY_SZ];
+ /** C2H buffer size list */
+ unsigned int c2h_buf_sz[QDMA_GLOBAL_CSR_ARRAY_SZ];
+ /** wireback acculation enable/disable */
+ unsigned int cmpl_status_acc;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_glbal_csr_get() - retrieve the global csr settings
+ *
+ * @param[in] dev_hndl: handle returned from qdma_device_open()
+ * @param[out] csr: data structures to hold the csr values
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_global_csr_get(unsigned long dev_hndl, struct global_csr_conf *csr);
+
+/*****************************************************************************/
+/**
+ * qdma_glbal_csr_set() - set the global csr values
+ * NOTE: for set, libqdma will enforce the access control
+ *
+ * @param[in] dev_hndl: handle returned from qdma_device_open()
+ * @param[in] csr: data structures to hold the csr values
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_global_csr_set(unsigned long dev_hndl, struct global_csr_conf *csr);
+
+/**
+ * cmpt_desc_sz_t - descriptor sizes
+ */
+enum cmpt_desc_sz_t {
+ /** 0 - completion size 8B */
+ CMPT_DESC_SZ_8B = 0,
+ /** 0 - completion size 16B */
+ CMPT_DESC_SZ_16B,
+ /** 0 - completion size 32B */
+ CMPT_DESC_SZ_32B,
+ /** 0 - completion size 64B */
+ CMPT_DESC_SZ_64B
+};
+
+/**
+ * desc_sz_t - descriptor sizes
+ */
+enum desc_sz_t {
+ /** 0 - size 8B */
+ DESC_SZ_8B = 0,
+ /** 0 - size 16B */
+ DESC_SZ_16B,
+ /** 0 - size 32B */
+ DESC_SZ_32B,
+ /** 0 - size 64B */
+ DESC_SZ_64B
+};
+
+/**
+ * tigger_mode_t - trigger modes
+ */
+enum tigger_mode_t {
+ /** 0 - disable trigger mode */
+ TRIG_MODE_DISABLE,
+ /** 1 - any trigger mode */
+ TRIG_MODE_ANY,
+ /** 2 - counter trigger mode */
+ TRIG_MODE_COUNTER,
+ /** 3 - trigger mode of user choice */
+ TRIG_MODE_USER,
+ /** 4 - timer trigger mode */
+ TRIG_MODE_TIMER,
+ /** 5 - timer and counter combo trigger mode */
+ TRIG_MODE_COMBO,
+};
+
+/**
+ * struct qdma_sw_sg - qdma scatter gather request
+ */
+struct qdma_sw_sg {
+ /** pointer to next page */
+ struct qdma_sw_sg *next;
+ /** pointer to current page */
+ struct page *pg;
+ /** offset in current page */
+ unsigned int offset;
+ /** length of the page */
+ unsigned int len;
+ /** dma address of the allocated page */
+ dma_addr_t dma_addr;
+};
+
+/**
+ * maximum queue name length
+ */
+#define QDMA_QUEUE_NAME_MAXLEN 32
+/**
+ * invalid queue index
+ */
+#define QDMA_QUEUE_IDX_INVALID 0xFFFF
+
+/**
+ * invalid MSI-x vector index
+ */
+#define QDMA_QUEUE_VEC_INVALID 0xFF
+
+/**
+ * struct qdma_queue_conf - qdma configuration parameters
+ * qdma_queue_conf defines the per-dma Q property.
+ * if any of the max requested is less than supported, the value will
+ * be updated
+ */
+struct qdma_queue_conf {
+ /** 0xFFFF: libqdma choose the queue idx
+ * 0 ~ (qdma_dev_conf.qsets_max - 1)
+ * the calling function choose the queue idx
+ */
+ u32 qidx:24;
+
+ /** config flags: byte #1 */
+ /** st mode */
+ u32 st:1;
+ /** c2h direction */
+ u32 c2h:1;
+ /** SDx only: inter-kernel communication pipe */
+ u32 pipe:1;
+ /** poll or interrupt */
+ u32 irq_en:1;
+
+ /** descriptor ring */
+ /** global_csr_conf.ringsz[N] */
+ u32 desc_rng_sz_idx:4;
+
+ /** config flags: byte #2 */
+ /** writeback enable, disabled for ST C2H */
+ u8 cmpl_status_en:1;
+ /** sw context.cmpl_status_acc_en */
+ u8 cmpl_status_acc_en:1;
+ /** sw context.cmpl_stauts_pend_chk */
+ u8 cmpl_status_pend_chk:1;
+ /** send descriptor to bypass out */
+ u8 desc_bypass:1;
+ /** descriptor prefetch enable control */
+ u8 pfetch_en:1;
+ /** sw context.frcd_en[32] */
+ u8 fetch_credit:1;
+ /** SDx only: ST packet mode (i.e., with TLAST
+ * to identify the packet boundary)
+ */
+ u8 st_pkt_mode:1;
+ /** c2h use pre-alloc free list */
+ u8 c2h_use_fl:1;
+
+ /** config flags: byte #3 */
+ /** global_csr_conf.c2h_buf_sz[N] */
+ u8 c2h_buf_sz_idx:4;
+
+ /** ST C2H Completion/Writeback ring */
+ /** global_csr_conf.ringsz[N] */
+ u8 cmpl_rng_sz_idx:4;
+
+ /** config flags: byte #4 */
+ /** C2H ST cmpt + immediate data, desc_sz_t */
+ u8 cmpl_desc_sz:2;
+ /** enable status desc. for CMPT */
+ u8 cmpl_stat_en:1;
+ /** C2H Completion entry user-defined data */
+ u8 cmpl_udd_en:1;
+ /** global_csr_conf.c2h_timer_cnt[N] */
+ u8 cmpl_timer_idx:4;
+
+ /** config flags: byte #5 */
+ /** global_csr_conf.c2h_cnt_th[N] */
+ u8 cmpl_cnt_th_idx:4;
+ /** tigger_mode_t */
+ u8 cmpl_trig_mode:3;
+ /** enable interrupt for CMPT */
+ u8 cmpl_en_intr:1;
+
+ /** config flags: byte #6 */
+ /** SW Context desc size, desc_sz_t */
+ u8 sw_desc_sz:2;
+ /** prefetch bypass en */
+ u8 pfetch_bypass:1;
+ /** OVF_DIS C2H ST over flow disable */
+ u8 cmpl_ovf_chk_dis:1;
+ /** Port ID */
+ u8 port_id:3;
+ /** Address Translation */
+ u8 at:1;
+
+ /*
+ * TODO: for Platform streaming DSA
+ */
+ /** only if pipe = 1 */
+ /** max 16. CDH length per packet */
+ u8 cdh_max;
+ /** <= 7, max # gather buf. per packet */
+ u8 pipe_gl_max;
+ /** pipe flow id */
+ u8 pipe_flow_id;
+ /** pipe SLR id */
+ u8 pipe_slr_id;
+ /** pipe route id */
+ u16 pipe_tdest;
+
+ /** user provided per-Q irq handler */
+ unsigned long quld; /* set by user for per Q data */
+
+ /** TBA: Q interrupt top, per-queue additional handling code
+ * for example, network rx: napi_schedule(&Q->napi)
+ */
+ void (*fp_descq_isr_top)(unsigned long qhndl, unsigned long quld);
+ /**
+ * optional rx packet handler:
+ * called from irq BH (i.e.qdma_queue_service_bh())
+ * - udd: user defined data in the completion entry
+ * - sgcnt / sgl: packet data in scatter-gather list
+ * NOTE: a. do NOT modify any field of sgl
+ * b. if zero copy, do a get_page() to prevent page freeing
+ * c. do loop through the sgl with sg->next and stop
+ * at sgcnt. the last sg may not have sg->next = NULL
+ * Returns:
+ * - 0 to allow libqdma free/re-task the sgl
+ * - < 0, libqdma will keep the packet for processing again
+ *
+ * A single packet could contain:
+ * in the case of c2h_udd_en = 1:
+ * - udd only (udd valid, sgcnt = 0, sgl = NULL), or
+ * - udd + packdet data
+ * in the case of c2h_udd_en = 0:
+ * - packet data (udd = NULL, sgcnt > 0 and sgl valid)
+ */
+ int (*fp_descq_c2h_packet)(unsigned long qhndl, unsigned long quld,
+ unsigned int len, unsigned int sgcnt,
+ struct qdma_sw_sg *sgl, void *udd);
+
+ /** fill in by libqdma */
+ /** name of the qdma device */
+ char name[QDMA_QUEUE_NAME_MAXLEN];
+ /** ring size of the queue */
+ unsigned int rngsz;
+ /** completion ring size of the queue */
+ unsigned int rngsz_cmpt;
+ /** C2H buffer size */
+ unsigned int c2h_bufsz;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_queue_add() - add a queue
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qconf: queue configuration parameters
+ * @param[in] qhndl: list of unsigned long values that are the opaque qhndl
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_add(unsigned long dev_hndl, struct qdma_queue_conf *qconf,
+ unsigned long *qhndl, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_reconfig() - reconfigure the queue
+ *
+ * @param[in] dev_hndl: qdma device handle
+ * @param[in] id: queue index
+ * @param[in] qconf: queue configuration
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_queue_reconfig(unsigned long dev_hndl, unsigned long id,
+ struct qdma_queue_conf *qconf, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_start() - start a queue (i.e, online, ready for dma)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: the opaque qhndl
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_start(unsigned long dev_hndl, unsigned long id,
+ char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_stop() - stop a queue (i.e., offline, NOT ready for dma)
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: the opaque qhndl
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_stop(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen);
+
+/*****************************************************************************/
+/**
+ * * qdma_queue_prog_stm() - Program STM for queue (context, map, etc)
+ * *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_prog_stm(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_remove() - remove a queue
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: the opaque qhndl
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_remove(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen);
+
+/**
+ * queue helper/debug functions
+ */
+
+/*****************************************************************************/
+/**
+ * qdma_queue_get_config() - retrieve the configuration of a queue
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: an opaque queue handle of type unsigned long
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+struct qdma_queue_conf *qdma_queue_get_config(unsigned long dev_hndl,
+ unsigned long id, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_list() - display all configured queues in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_list(unsigned long dev_hndl, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump() - display a queue's state in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: an opaque queue handle of type unsigned long
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump(unsigned long dev_hndl, unsigned long id, char *buf,
+ int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump_desc() - display a queue's descriptor ring from index start
+ * ~ end in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: an opaque queue handle of type unsigned long
+ * @param[in] start: start index
+ * @param[in] end: end index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump_desc(unsigned long dev_hndl, unsigned long id,
+ unsigned int start, unsigned int end, char *buf,
+ int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_dump_cmpt() - display a queue's descriptor ring from index start
+ * ~ end in a string buffer
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: an opaque queue handle of type unsigned long
+ * @param[in] start: start index
+ * @param[in] end: end index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_dump_cmpt(unsigned long dev_hndl, unsigned long id,
+ unsigned int start, unsigned int end, char *buf,
+ int buflen);
+
+#ifdef ERR_DEBUG
+/*****************************************************************************/
+/**
+ * qdma_queue_set_err_induction() - Induce the error
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] id: error id
+ * @param[in] err: error info
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return success: if optional message buffer used then strlen of buf,
+ * otherwise QDMA_OPERATION_SUCCESSFUL
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_set_err_induction(unsigned long dev_hndl, unsigned long id,
+ u32 err, char *buf, int buflen);
+#endif
+
+
+/**
+ * maximum request length
+ */
+#define QDMA_REQ_OPAQUE_SIZE 72
+/**
+ * Max length of the user defined data
+ */
+#define QDMA_UDD_MAXLEN 32
+
+/**
+ * struct qdma_request - qdma request for read or write
+ */
+struct qdma_request {
+ /** private to the dma driver, do NOT touch */
+ unsigned char opaque[QDMA_REQ_OPAQUE_SIZE];
+ /** filled in by the calling function */
+ /** for the calling function */
+ unsigned long uld_data;
+ /** set fp_done for non-blocking mode */
+ int (*fp_done)(struct qdma_request *, unsigned int bytes_done, int err);
+ /** timeout in mili-seconds, 0 - no timeout */
+ unsigned int timeout_ms;
+ /** total data size */
+ unsigned int count;
+ /** MM only, DDR/BRAM memory addr */
+ u64 ep_addr;
+ /* flag to indicate if memcpy is required */
+ u8 no_memcpy:1;
+ /** write: if write to the device */
+ u8 write:1;
+ /** if sgt is already dma mapped */
+ u8 dma_mapped:1;
+ /** user defined data present */
+ u8 h2c_eot:1;
+ /** indicates end of transfer towards user kernel */
+ u8 udd_len;
+ /** # of scatter-gather entries < 64K */
+ unsigned int sgcnt;
+ /** scatter-gather list of data bufs */
+ struct qdma_sw_sg *sgl;
+ /** udd data */
+ u8 udd[QDMA_UDD_MAXLEN];
+};
+
+/*****************************************************************************/
+/**
+ * qdma_request_submit() - submit a scatter-gather list of data for dma
+ * operation (for both read and write)
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] req: qdma request
+ *
+ * @return # of bytes transferred
+ * @return <0: error
+ *****************************************************************************/
+ssize_t qdma_request_submit(unsigned long dev_hndl, unsigned long id,
+ struct qdma_request *req);
+
+/*****************************************************************************/
+/**
+ * qdma_batch_request_submit() - submit a scatter-gather list of data for dma
+ * operation (for both read and write)
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] id: queue index
+ * @param[in] count: number of requests
+ * @param[in] reqv: qdma request
+ *
+ * @return # of bytes transferred
+ * @return <0: error
+ *****************************************************************************/
+ssize_t qdma_batch_request_submit(unsigned long dev_hndl, unsigned long id,
+ unsigned long count, struct qdma_request **reqv);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_c2h_peek() - peek a receive (c2h) queue
+ *
+ * @param dev_hndl: hndl returned from qdma_device_open()
+ * @param qhndl: hndl returned from qdma_queue_add()
+ *
+ * filled in by libqdma:
+ * @param[in] udd_cnt: # of udd received
+ * @param[in] pkt_cnt: # of packets received
+ * @param[in] data_len: # of bytes of packet data received
+ *
+ * @return 0: success and # of packets received in the Q
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_c2h_peek(unsigned long dev_hndl, unsigned long qhndl,
+ unsigned int *udd_cnt, unsigned int *pkt_cnt,
+ unsigned int *data_len);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_avail_desc() - query of # of free descriptor
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] qhndl: hndl returned from qdma_queue_add()
+ *
+ * @return # of available desc in the queue
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_avail_desc(unsigned long dev_hndl, unsigned long qhndl);
+
+/** packet/streaming interfaces */
+
+/**
+ * struct qdma_cmpl_ctrl - completion control
+ */
+struct qdma_cmpl_ctrl {
+ /** global_csr_conf.c2h_cnt_th[N] */
+ u8 cnt_th_idx:4;
+ /** global_csr_conf.c2h_timer_cnt[N] */
+ u8 timer_idx:4;
+ /** tigger_mode_t */
+ u8 trigger_mode:3;
+ /** enable status desc. for CMPT */
+ u8 en_stat_desc:1;
+ /** enable interrupt for CMPT */
+ u8 cmpl_en_intr:1;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_queue_cmpl_ctrl() - read/set the c2h Q's completion control
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] qhndl: hndl returned from qdma_queue_add()
+ * @param[in] cctrl: completion control
+ * @param[in] set: read or set
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_cmpl_ctrl(unsigned long dev_hndl, unsigned long qhndl,
+ struct qdma_cmpl_ctrl *cctrl, bool set);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_packet_read() - read rcv'ed data (ST C2H dma operation)
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] qhndl: hndl returned from qdma_queue_add()
+ * @param[in] req: pointer to the request data
+ * @param[out] cctrl: completion control, if no change is desired,
+ * set it to NULL
+ *
+ * @return number of packets read
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_packet_read(unsigned long dev_hndl, unsigned long qhndl,
+ struct qdma_request *req, struct qdma_cmpl_ctrl *cctrl);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_packet_write() - submit data for h2c dma operation
+ *
+ * @param[in] dev_hndl: hndl returned from qdma_device_open()
+ * @param[in] qhndl: hndl returned from qdma_queue_add()
+ * @param[in] req: pointer to the list of packet data
+ *
+ * @return number of desc posted
+ * @return <0: error
+ *****************************************************************************/
+int qdma_queue_packet_write(unsigned long dev_hndl, unsigned long qhndl,
+ struct qdma_request *req);
+
+/*****************************************************************************/
+/**
+ * qdma_queue_service() - service the queue
+ * in the case of irq handler is registered by the user, the user should
+ * call qdma_queue_service() in its interrupt handler to service the queue
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qhndl: hndl returned from qdma_queue_add()
+ * @param[in] budget: ST C2H only, max # of completions
+ * to be processed. 0 - no limit
+ * @param[in] c2h_upd_cmpl: To keep intrrupt disabled, set to false,
+ * Set to true to re-enable the interrupt:
+ * ST C2H only, max # of completions
+ * to be processed. 0 - no limit
+ *
+ *****************************************************************************/
+void qdma_queue_service(unsigned long dev_hndl, unsigned long qhndl,
+ int budget, bool c2h_upd_cmpl);
+
+/*****************************************************************************/
+/**
+ * qdma_intr_ring_dump() - display the interrupt ring info of a vector
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] vector_idx: vector number
+ * @param[in] start_idx: interrupt ring start idx
+ * @param[in] end_idx: interrupt ring end idx
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message bufferuffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_intr_ring_dump(unsigned long dev_hndl, unsigned int vector_idx,
+ int start_idx, int end_idx, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_get_cmpt_udd() - function to receive the user defined data
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qhndl: queue handle
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message bufferuffer
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+
+int qdma_descq_get_cmpt_udd(unsigned long dev_hndl, unsigned long qhndl,
+ char *buf, int buflen);
+
+#ifdef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_vf_qconf call for VF to request qmax number of Qs
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] qmax: number of qs requested by vf
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_vf_qconf(unsigned long dev_hndl, int qmax);
+#endif
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_compat.h b/QDMA/linux-kernel/libqdma/qdma_compat.h
new file mode 100644
index 0000000000000000000000000000000000000000..b3514f0b2e10deea9057a6e5c42fb4e813042109
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_compat.h
@@ -0,0 +1,97 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+/**
+ * @file
+ * @brief This file is used to allow the driver to be compiled under multiple
+ * versions of Linux with as few obtrusive in-line ifdef's as possible.
+ */
+
+#ifndef __QDMA_COMPAT_H
+#define __QDMA_COMPAT_H
+
+#include <linux/version.h>
+#include <asm/barrier.h>
+
+/**
+ * if linux kernel version is < 3.19.0
+ * then define the dma_rmb and dma_wmb
+ */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0)
+
+#ifndef dma_rmb
+#define dma_rmb rmb
+#endif /* #ifndef dma_rmb */
+
+#ifndef dma_wmb
+#define dma_wmb wmb
+#endif /* #ifndef dma_wmb */
+
+#endif
+
+/* use simple wait queue (swaitq) with newer kernels */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 6, 0)
+#include <linux/swait.h>
+
+#define qdma_wait_queue struct swait_queue_head
+#define qdma_waitq_init init_swait_queue_head
+#define qdma_waitq_wakeup swake_up
+#define qdma_waitq_wait_event swait_event_interruptible
+#define qdma_waitq_wait_event_timeout swait_event_interruptible_timeout
+
+#else
+#include <linux/wait.h>
+
+#define qdma_wait_queue wait_queue_head_t
+#define qdma_waitq_init init_waitqueue_head
+#define qdma_waitq_wakeup wake_up_interruptible
+#define qdma_waitq_wait_event wait_event_interruptible
+#define qdma_waitq_wait_event_timeout wait_event_interruptible_timeout
+
+#endif /* swaitq */
+
+/* timer */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)
+#define qdma_timer_setup(timer, fp_handler, data) \
+ timer_setup(timer, fp_handler, 0)
+
+#define qdma_timer_start(timer, expires) \
+ mod_timer(timer, round_jiffies(jiffies + (expires)))
+
+#else
+#define qdma_timer_setup(timer, fp_handler, priv) \
+ do { \
+ init_timer(timer); \
+ timer->data = (unsigned long)priv; \
+ timer->function = fp_handler; \
+ del_timer(timer); \
+ } while (0)
+
+#define qdma_timer_start(timer, timeout) \
+ do { \
+ del_timer(timer); \
+ timer->expires = jiffies + (timeout); \
+ add_timer(timer); \
+ } while (0)
+
+
+#endif /* timer */
+
+
+#endif /* #ifndef __QDMA_COMPAT_H */
diff --git a/QDMA/linux-kernel/libqdma/qdma_context.c b/QDMA/linux-kernel/libqdma/qdma_context.c
new file mode 100644
index 0000000000000000000000000000000000000000..2e3081f24bf0b7aa6b42e3984152818ac51467c9
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_context.c
@@ -0,0 +1,909 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include "qdma_device.h"
+#include "qdma_descq.h"
+#include "qdma_intr.h"
+#include "qdma_regs.h"
+#include "qdma_context.h"
+
+/**
+ * Make the interrupt context
+ */
+static int make_intr_context(struct xlnx_dma_dev *xdev, u32 *data, int cnt)
+{
+ int i, j;
+
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE)) {
+ memset(data, 0, cnt * sizeof(u32));
+ return 0;
+ }
+
+ /** Each context size is QDMA_REG_IND_CTXT_WCNT_3
+ * data[QDMA_REG_IND_CTXT_WCNT_3 * QDMA_NUM_DATA_VEC_FOR_INTR_CXT]
+ */
+ if (cnt <
+ (QDMA_NUM_DATA_VEC_FOR_INTR_CXT * QDMA_REG_IND_CTXT_WCNT_3)) {
+ pr_warn("%s, intr context %d < (%d * %d).\n",
+ xdev->conf.name, cnt,
+ QDMA_NUM_DATA_VEC_FOR_INTR_CXT,
+ QDMA_REG_IND_CTXT_WCNT_3);
+ return -EINVAL;
+ }
+
+ memset(data, 0, cnt * sizeof(u32));
+ /** program the coalescing context
+ * i -> Number of vectors
+ * j -> number of words for each vector context
+ */
+ for (i = 0, j = 0; i < QDMA_NUM_DATA_VEC_FOR_INTR_CXT; i++) {
+ u64 bus_64;
+ u32 v;
+ struct intr_coal_conf *entry = (xdev->intr_coal_list + i);
+
+ /* TBD:
+ * Assume that Qid is irrelevant for interrupt context
+ * programming because, interrupt context is done per vector
+ * which for the function and not for each queue
+ */
+
+ bus_64 = (PCI_DMA_H(entry->intr_ring_bus) << 20) |
+ ((PCI_DMA_L(entry->intr_ring_bus)) >> 12);
+
+ v = bus_64 & M_INT_AGGR_W0_BADDR_64;
+
+ /** Data[0] */
+ data[j] = (1 << S_INT_AGGR_W0_F_VALID |
+ V_INT_AGGR_W0_VEC_ID(entry->vec_id)|
+ V_INT_AGGR_W0_BADDR_64(v) |
+ 1 << S_INT_AGGR_W0_F_COLOR);
+
+ /** Data[1] */
+ v = (bus_64 >> L_INT_AGGR_W0_BADDR_64);
+ data[++j] = v;
+
+ v = (bus_64 >>
+ (L_INT_AGGR_W0_BADDR_64 + 32)) & M_INT_AGGR_W2_BADDR_64;
+ /** Data[3] */
+ data[++j] = ((V_INT_AGGR_W2_BADDR_64(v)) |
+ V_INT_AGGR_W2_VEC_SIZE(xdev->conf.intr_rngsz));
+
+ j++;
+ }
+
+ return 0;
+}
+
+#ifndef __QDMA_VF__
+static int make_stm_c2h_context(struct qdma_descq *descq, u32 *data)
+{
+ int pipe_slr_id = descq->conf.pipe_slr_id;
+ int pipe_flow_id = descq->conf.pipe_flow_id;
+ int pipe_tdest = descq->conf.pipe_tdest;
+
+ /* 191..160 */
+ data[5] = F_STM_C2H_CTXT_ENTRY_VALID;
+
+ /* 128..159 */
+ data[1] = (pipe_slr_id << S_STM_CTXT_C2H_SLR) |
+ ((pipe_tdest & 0xFF00) << S_STM_CTXT_C2H_TDEST_H);
+
+ /* 96..127 */
+ data[0] = ((pipe_tdest & 0xFF) << S_STM_CTXT_C2H_TDEST_L) |
+ (pipe_flow_id << S_STM_CTXT_C2H_FID);
+
+ pr_debug("%s, STM 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x.\n",
+ descq->conf.name, data[0], data[1], data[2], data[3], data[4],
+ data[5]);
+ return 0;
+}
+
+static int make_stm_h2c_context(struct qdma_descq *descq, u32 *data)
+{
+ int pipe_slr_id = descq->conf.pipe_slr_id;
+ int pipe_flow_id = descq->conf.pipe_flow_id;
+ int pipe_tdest = descq->conf.pipe_tdest;
+ int dppkt = 1;
+ int log2_dppkt = ilog2(dppkt);
+ int pkt_lim = 0;
+ int max_ask = 8;
+
+ /* 191..160 */
+ data[5] = F_STM_H2C_CTXT_ENTRY_VALID;
+
+ /* 128..159 */
+ data[4] = (descq->qidx_hw << S_STM_CTXT_QID);
+
+ /* 96..127 */
+ data[3] = (pipe_slr_id << S_STM_CTXT_H2C_SLR) |
+ ((pipe_tdest & 0xFF00) << S_STM_CTXT_H2C_TDEST_H);
+
+ /* 64..95 */
+ data[2] = ((pipe_tdest & 0xFF) << S_STM_CTXT_H2C_TDEST_L) |
+ (pipe_flow_id << S_STM_CTXT_H2C_FID) |
+ (pkt_lim << S_STM_CTXT_PKT_LIM) |
+ (max_ask << S_STM_CTXT_MAX_ASK);
+
+ /* 32..63 */
+ data[1] = (dppkt << S_STM_CTXT_DPPKT) |
+ (log2_dppkt << S_STM_CTXT_LOG2_DPPKT);
+
+ /* 0..31 */
+ /** explicitly init to 8 to workaround hw issue due to which the value
+ * is getting initialized to zero instead of its reset value of 8
+ */
+ data[0] = 8;
+
+ pr_debug("%s, STM 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x.\n",
+ descq->conf.name, data[0], data[1], data[2], data[3], data[4],
+ data[5]);
+ return 0;
+}
+#endif
+
+static int make_sw_context(struct qdma_descq *descq, u32 *data, int cnt)
+{
+ int ring_index;
+
+ if (cnt < QDMA_REG_IND_CTXT_WCNT_5) {
+ pr_warn("%s, sw context count %d < %d.\n",
+ descq->xdev->conf.name, cnt, QDMA_REG_IND_CTXT_WCNT_5);
+ return -EINVAL;
+ }
+ memset(data, 0, cnt * sizeof(u32));
+
+ /* sw context */
+ if ((descq->xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (descq->xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ ring_index = get_intr_ring_index(descq->xdev, descq->intr_id);
+ data[4] = V_DESC_CTXT_W4_VEC(ring_index) |
+ (0x01 << S_DESC_CTXT_W1_F_INTR_AGGR);
+ } else {
+ data[4] = V_DESC_CTXT_W4_VEC(descq->intr_id);
+ }
+
+ data[3] = PCI_DMA_H(descq->desc_bus);
+ data[2] = PCI_DMA_L(descq->desc_bus);
+ data[1] = (1 << S_DESC_CTXT_W1_F_QEN) |
+ (descq->conf.cmpl_status_pend_chk <<
+ S_DESC_CTXT_W1_F_CMPL_STATUS_PEND_CHK) |
+ (descq->conf.cmpl_status_acc_en <<
+ S_DESC_CTXT_W1_F_CMPL_STATUS_ACC_EN) |
+ V_DESC_CTXT_W1_RNG_SZ(descq->conf.desc_rng_sz_idx) |
+ (descq->conf.desc_bypass << S_DESC_CTXT_W1_F_BYP) |
+ (descq->conf.cmpl_status_en <<
+ S_DESC_CTXT_W1_F_CMPL_STATUS_EN) |
+ (descq->conf.irq_en << S_DESC_CTXT_W1_F_IRQ_EN) |
+ (~descq->conf.st << S_DESC_CTXT_W1_F_IS_MM);
+
+ if (descq->conf.desc_bypass &&
+ (descq->conf.sw_desc_sz == DESC_SZ_64B)) {
+ data[1] |= V_DESC_CTXT_W1_DSC_SZ(descq->conf.sw_desc_sz);
+ } else {
+ if (!descq->conf.st) { /* mm h2c/c2h */
+ data[1] |= (V_DESC_CTXT_W1_DSC_SZ(DESC_SZ_32B)) |
+ (descq->channel << S_DESC_CTXT_W1_F_MM_CHN);
+ } else if (descq->conf.c2h) { /* st c2h */
+ data[1] |= (descq->conf.fetch_credit <<
+ S_DESC_CTXT_W1_F_FCRD_EN) |
+ (V_DESC_CTXT_W1_DSC_SZ(DESC_SZ_8B));
+ } else { /* st h2c */
+ data[1] |= V_DESC_CTXT_W1_DSC_SZ(DESC_SZ_16B);
+
+ /* For STM & TM mode, set fcrd_en for ST H2C */
+ if (descq->xdev->stm_en ||
+ descq->xdev->conf.tm_mode_en)
+ data[1] |= (descq->conf.fetch_credit <<
+ S_DESC_CTXT_W1_F_FCRD_EN);
+ }
+ }
+
+ /* pidx = 0; irq_ack = 0 */
+ data[0] = ((V_DESC_CTXT_W0_FUNC_ID(descq->xdev->func_id)) |
+ (((descq->xdev->conf.qdma_drv_mode !=
+ POLL_MODE) ? 1 : 0) <<
+ S_DESC_CTXT_W0_F_INTR_ARM));
+
+#ifdef ERR_DEBUG
+ if (descq->induce_err & (1 << param)) {
+ data[0] |= (0xFFF << S_DESC_CTXT_W0_FUNC_ID);
+ pr_info("induced error %d", ind_ctxt_cmd_err);
+ }
+#endif
+
+ pr_debug("%s, SW 0x%08x 0x%08x, 0x%08x, 0x%08x, 0x%08x.\n",
+ descq->conf.name, data[4], data[3], data[2], data[1], data[0]);
+
+ return 0;
+}
+
+/* ST: prefetch context setup */
+static int make_prefetch_context(struct qdma_descq *descq, u32 *data, int cnt)
+{
+ BUG_ON(!descq);
+ BUG_ON(!data);
+
+ if (cnt < QDMA_REG_IND_CTXT_WCNT_2) {
+ pr_warn("%s, prefetch context count %d < %d.\n",
+ descq->conf.name, cnt, QDMA_REG_IND_CTXT_WCNT_2);
+ return -EINVAL;
+ }
+ memset(data, 0, cnt * sizeof(u32));
+
+ /* prefetch context */
+ data[1] = 1 << S_PFTCH_W1_F_VALID;
+ data[0] = (descq->conf.pfetch_bypass << S_PFTCH_W0_F_BYPASS) |
+ (descq->conf.c2h_buf_sz_idx << S_PFTCH_W0_BUF_SIZE_IDX) |
+ /** TBD: this code needs to be deleted once the PG is updated.
+ * func_id bits are going to be reserved
+ * need to get clarity on port id
+ */
+ //(descq->conf.port_id << S_PFTCH_W0_PORT_ID) |
+ // (descq->xdev->func_id << S_PFTCH_W0_FUNC_ID) |
+ (descq->conf.pfetch_en << S_PFTCH_W0_F_EN_PFTCH);
+
+ pr_debug("%s, PFTCH 0x%08x 0x%08x\n",
+ descq->conf.name, data[1], data[0]);
+
+ return 0;
+}
+
+/* ST C2H : writeback context setup */
+static int make_cmpt_context(struct qdma_descq *descq, u32 *data, int cnt)
+{
+ u64 bus_64;
+ u32 v;
+ int ring_index;
+
+ if (cnt < QDMA_REG_IND_CTXT_WCNT_5) {
+ pr_warn("%s, cmpt context count %d < %d.\n",
+ descq->xdev->conf.name, cnt, QDMA_REG_IND_CTXT_WCNT_5);
+ return -EINVAL;
+ }
+ memset(data, 0, cnt * sizeof(u32));
+
+ /* writeback context */
+ bus_64 = (PCI_DMA_H(descq->desc_cmpt_bus) << 26) |
+ ((PCI_DMA_L(descq->desc_cmpt_bus)) >> 6);
+
+ data[0] = (descq->conf.cmpl_stat_en << S_CMPT_CTXT_W0_F_EN_STAT_DESC) |
+ (descq->conf.irq_en << S_CMPT_CTXT_W0_F_EN_INT) |
+ (V_CMPT_CTXT_W0_TRIG_MODE(descq->conf.cmpl_trig_mode)) |
+ (V_CMPT_CTXT_W0_FNC_ID(descq->xdev->func_id)) |
+ (descq->conf.cmpl_timer_idx << S_CMPT_CTXT_W0_TIMER_IDX) |
+ (descq->conf.cmpl_cnt_th_idx << S_CMPT_CTXT_W0_COUNTER_IDX) |
+ (1 << S_CMPT_CTXT_W0_F_COLOR) |
+ (descq->conf.cmpl_rng_sz_idx << S_CMPT_CTXT_W0_RNG_SZ);
+
+ data[1] = bus_64 & 0xFFFFFFFF;
+
+ v = PCI_DMA_H(bus_64) & M_CMPT_CTXT_W2_BADDR_64;
+ data[2] = (V_CMPT_CTXT_W2_BADDR_64(v)) |
+ (V_CMPT_CTXT_W2_DESC_SIZE(descq->conf.cmpl_desc_sz));
+
+ data[3] = (1 << S_CMPT_CTXT_W3_F_VALID);
+
+ if ((descq->xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (descq->xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ ring_index = get_intr_ring_index(descq->xdev, descq->intr_id);
+ data[4] = (descq->conf.cmpl_ovf_chk_dis <<
+ S_CMPT_CTXT_W4_F_OVF_CHK_DIS) |
+ V_CMPT_CTXT_W4_VEC(ring_index) |
+ (0x01 << S_CMPT_CTXT_W4_F_INTR_AGGR);
+ } else {
+ data[4] = (descq->conf.cmpl_ovf_chk_dis <<
+ S_CMPT_CTXT_W4_F_OVF_CHK_DIS) |
+ V_CMPT_CTXT_W4_VEC(descq->intr_id);
+ }
+
+ pr_debug("%s, CMPT 0x%08x, 0x%08x, 0x%08x, 0x%08x, 0x%08x.\n",
+ descq->conf.name, data[4], data[3], data[2], data[1], data[0]);
+
+ return 0;
+}
+
+#ifdef __QDMA_VF__
+int qdma_intr_context_setup(struct xlnx_dma_dev *xdev)
+{
+ int i = 0;
+ int rv;
+ struct mbox_msg *m = NULL;
+ struct mbox_msg_hdr *hdr = NULL;
+ struct mbox_msg_intr_ctxt *ictxt = NULL;
+
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE))
+ return 0;
+
+ m = qdma_mbox_msg_alloc(xdev,
+ MBOX_OP_INTR_CTXT);
+ if (!m)
+ return -ENOMEM;
+
+ hdr = &m->hdr;
+ ictxt = &m->intr_ctxt;
+
+ ictxt->clear = 1;
+ ictxt->vec_base = xdev->dvec_start_idx;
+ ictxt->num_rings = QDMA_NUM_DATA_VEC_FOR_INTR_CXT;
+
+ for (i = 0; i < QDMA_NUM_DATA_VEC_FOR_INTR_CXT; i++) {
+ ictxt->ring_index_list[i] =
+ get_intr_ring_index(xdev, xdev->dvec_start_idx + i);
+ }
+
+ rv = make_intr_context(xdev, ictxt->w,
+ (QDMA_NUM_DATA_VEC_FOR_INTR_CXT *
+ QDMA_REG_IND_CTXT_WCNT_3));
+ if (rv < 0)
+ return rv;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_INTR_CTXT_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ if (rv != -ENODEV)
+ pr_err("%s, mbox failed for interrupt context %d.\n",
+ xdev->conf.name, rv);
+ goto free_msg;
+ }
+
+ rv = hdr->status;
+
+free_msg:
+ qdma_mbox_msg_free(m);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+
+int qdma_descq_context_clear(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h, bool clr)
+{
+ struct mbox_msg *m = qdma_mbox_msg_alloc(xdev, MBOX_OP_QCTXT_CLR);
+ struct mbox_msg_hdr *hdr = m ? &m->hdr : NULL;
+ struct mbox_msg_qctxt *qctxt = m ? &m->qctxt : NULL;
+ int rv;
+
+ if (!m)
+ return -ENOMEM;
+
+ qctxt->qid = qid_hw;
+ qctxt->st = st;
+ qctxt->c2h = c2h;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_QCTXT_CLR_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ if (rv != -ENODEV)
+ pr_info("%s, qid_hw 0x%x mbox failed %d.\n",
+ xdev->conf.name, qid_hw, rv);
+ goto err_out;
+ }
+
+ rv = hdr->status;
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int qdma_descq_context_read(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h,
+ struct hw_descq_context *context)
+{
+ struct mbox_msg *m = qdma_mbox_msg_alloc(xdev, MBOX_OP_QCTXT_RD);
+ struct mbox_msg_hdr *hdr = m ? &m->hdr : NULL;
+ struct mbox_msg_qctxt *qctxt = m ? &m->qctxt : NULL;
+ int rv;
+
+ if (!m)
+ return -ENOMEM;
+
+ qctxt->qid = qid_hw;
+ qctxt->st = st;
+ qctxt->c2h = c2h;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_QCTXT_RD_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ if (rv != -ENODEV)
+ pr_info("%s, qid_hw 0x%x mbox failed %d.\n",
+ xdev->conf.name, qid_hw, rv);
+ goto err_out;
+ }
+
+ if (hdr->status) {
+ rv = hdr->status;
+ goto err_out;
+ }
+
+ memcpy(context, &qctxt->context, sizeof(struct hw_descq_context));
+
+ return 0;
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int qdma_descq_context_setup(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ struct mbox_msg *m = qdma_mbox_msg_alloc(xdev, MBOX_OP_QCTXT_WRT);
+ struct mbox_msg_hdr *hdr = m ? &m->hdr : NULL;
+ struct mbox_msg_qctxt *qctxt = m ? &m->qctxt : NULL;
+ struct hw_descq_context *context = m ? &qctxt->context : NULL;
+ int rv;
+
+ if (!m)
+ return -ENOMEM;
+
+ rv = qdma_descq_context_read(xdev, descq->qidx_hw,
+ descq->conf.st, descq->conf.c2h,
+ context);
+ if (rv < 0) {
+ pr_info("%s, qid_hw 0x%x, %s mbox failed %d.\n",
+ xdev->conf.name, descq->qidx_hw, descq->conf.name, rv);
+ goto err_out;
+ }
+
+ make_sw_context(descq, context->sw, QDMA_REG_IND_CTXT_WCNT_5);
+ if (descq->conf.st && descq->conf.c2h) {
+ make_prefetch_context(descq, context->prefetch, 2);
+ make_cmpt_context(descq, context->cmpt,
+ QDMA_REG_IND_CTXT_WCNT_5);
+ }
+
+ qctxt->clear = 1;
+ qctxt->verify = 1;
+ qctxt->st = descq->conf.st;
+ qctxt->c2h = descq->conf.c2h;
+ qctxt->qid = descq->qidx_hw;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_QCTXT_WRT_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ if (rv != -ENODEV)
+ pr_info("%s, qid_hw 0x%x, %s mbox failed %d.\n",
+ xdev->conf.name, descq->qidx_hw,
+ descq->conf.name, rv);
+ goto err_out;
+ }
+
+ if (hdr->status) {
+ rv = hdr->status;
+ pr_err("Failed to set intr ctxt message\n");
+ goto err_out;
+ }
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+#else /* PF only */
+
+int qdma_prog_intr_context(struct xlnx_dma_dev *xdev,
+ struct mbox_msg_intr_ctxt *ictxt)
+{
+ int i = 0;
+ int rv;
+ int ring_index;
+
+ for (i = 0; i < ictxt->num_rings; i++) {
+ ring_index = ictxt->ring_index_list[i];
+
+ /* clear the interrupt context for each vector */
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_COAL,
+ NULL,
+ QDMA_REG_IND_CTXT_WCNT_8,
+ 0);
+ if (rv < 0)
+ return rv;
+
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_SEL_COAL,
+ (ictxt->w +
+ (QDMA_REG_IND_CTXT_WCNT_3 * i)),
+ QDMA_REG_IND_CTXT_WCNT_3,
+ 1);
+ if (rv < 0)
+ return rv;
+#if 0
+ /** print interrupt context */
+ pr_debug("intr_ctxt WR: ring_index(Qid) = %d, data[2] = %x data[1] = %x data[0] = %x\n",
+ ring_index,
+ *(ictxt->w +
+ ((QDMA_REG_IND_CTXT_WCNT_3*i) + 2)),
+ *(ictxt->w +
+ ((QDMA_REG_IND_CTXT_WCNT_3*i) + 1)),
+ *(ictxt->w + (QDMA_REG_IND_CTXT_WCNT_3*i)));
+
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_SEL_COAL,
+ intr_ctxt,
+ QDMA_REG_IND_CTXT_WCNT_3,
+ 1);
+ if (rv < 0)
+ return rv;
+
+ pr_debug("intr_ctxt RD: ring_index(Qid) = %d, data[2] = %x data[1] = %x data[0] = %x\n",
+ ring_index,
+ intr_ctxt[2],
+ intr_ctxt[1],
+ intr_ctxt[0]);
+#endif
+ }
+
+ return 0;
+}
+
+int qdma_intr_context_setup(struct xlnx_dma_dev *xdev)
+{
+ u32 data[(QDMA_NUM_DATA_VEC_FOR_INTR_CXT *
+ QDMA_REG_IND_CTXT_WCNT_3)];
+ int i = 0;
+ int rv;
+ int ring_index;
+
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE))
+ return 0;
+
+ /** Preparing the interrupt context for all the vectors
+ * each vector's context width is QDMA_REG_IND_CTXT_WCNT_3(3)
+ */
+ rv = make_intr_context(xdev, data,
+ (QDMA_NUM_DATA_VEC_FOR_INTR_CXT *
+ QDMA_REG_IND_CTXT_WCNT_3));
+ if (rv < 0)
+ return rv;
+
+ for (i = 0; i < QDMA_NUM_DATA_VEC_FOR_INTR_CXT; i++) {
+ ring_index = get_intr_ring_index(xdev,
+ (i + xdev->dvec_start_idx));
+ /* clear the interrupt context for each vector */
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_COAL,
+ NULL,
+ QDMA_REG_IND_CTXT_WCNT_8,
+ 0);
+ if (rv < 0)
+ return rv;
+
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_SEL_COAL,
+ (data + (QDMA_REG_IND_CTXT_WCNT_3 * i)),
+ QDMA_REG_IND_CTXT_WCNT_3,
+ 1);
+ if (rv < 0)
+ return rv;
+#if 0
+ /** print interrupt context */
+ pr_debug("intr_ctxt WR: ring_index(Qid) = %d, data[2] = %x data[1] = %x data[0] = %x\n",
+ ring_index,
+ *(data + ((QDMA_REG_IND_CTXT_WCNT_3*i) + 2)),
+ *(data + ((QDMA_REG_IND_CTXT_WCNT_3*i) + 1)),
+ *(data + (QDMA_REG_IND_CTXT_WCNT_3*i)));
+
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index,
+ QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_SEL_COAL,
+ intr_ctxt,
+ QDMA_REG_IND_CTXT_WCNT_3,
+ 1);
+ if (rv < 0)
+ return rv;
+
+ pr_debug("intr_ctxt RD: ring_index(Qid) = %d, data[2] = %x data[1] = %x data[0] = %x\n",
+ ring_index,
+ intr_ctxt[2],
+ intr_ctxt[1],
+ intr_ctxt[0]);
+#endif
+ }
+
+ return 0;
+}
+
+int qdma_descq_context_clear(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h, bool clr)
+{
+ u8 sel;
+ int rv = 0;
+
+ sel = c2h ? QDMA_CTXT_SEL_SW_C2H : QDMA_CTXT_SEL_SW_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw,
+ clr ? QDMA_CTXT_CMD_CLR : QDMA_CTXT_CMD_INV,
+ sel, NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+
+ sel = c2h ? QDMA_CTXT_SEL_HW_C2H : QDMA_CTXT_SEL_HW_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_CLR, sel,
+ NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+
+ sel = c2h ? QDMA_CTXT_SEL_CR_C2H : QDMA_CTXT_SEL_CR_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_CLR, sel,
+ NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+
+ /* Only clear prefetch and writeback contexts if this queue is ST C2H */
+ if (st && c2h) {
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_PFTCH, NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_CMPT, NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+ }
+
+ return 0;
+}
+
+int qdma_descq_context_setup(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ struct hw_descq_context context;
+ int rv;
+
+ rv = qdma_descq_context_clear(xdev, descq->qidx_hw, descq->conf.st,
+ descq->conf.c2h, 1);
+ if (rv < 0)
+ return rv;
+
+ memset(&context, 0, sizeof(context));
+
+ make_sw_context(descq, context.sw, QDMA_REG_IND_CTXT_WCNT_5);
+
+ if (descq->conf.st && descq->conf.c2h) {
+ make_prefetch_context(descq,
+ context.prefetch, QDMA_REG_IND_CTXT_WCNT_2);
+ make_cmpt_context(descq, context.cmpt,
+ QDMA_REG_IND_CTXT_WCNT_5);
+ }
+
+ return qdma_descq_context_program(descq->xdev, descq->qidx_hw,
+ descq->conf.st, descq->conf.c2h, &context);
+}
+
+int qdma_descq_stm_setup(struct qdma_descq *descq)
+{
+ struct stm_descq_context context;
+
+ memset(&context, 0, sizeof(context));
+ if (descq->conf.c2h)
+ make_stm_c2h_context(descq, context.stm);
+ else
+ make_stm_h2c_context(descq, context.stm);
+
+ return qdma_descq_stm_program(descq->xdev, descq->qidx_hw,
+ descq->conf.pipe_flow_id,
+ descq->conf.c2h, false,
+ &context);
+}
+
+int qdma_descq_stm_clear(struct qdma_descq *descq)
+{
+ struct stm_descq_context context;
+
+ memset(&context, 0, sizeof(context));
+ return qdma_descq_stm_program(descq->xdev, descq->qidx_hw,
+ descq->conf.pipe_flow_id,
+ descq->conf.c2h, true,
+ &context);
+}
+
+int qdma_descq_context_read(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h,
+ struct hw_descq_context *context)
+{
+ u8 sel;
+ int rv = 0;
+
+ memset(context, 0, sizeof(struct hw_descq_context));
+
+ sel = c2h ? QDMA_CTXT_SEL_SW_C2H : QDMA_CTXT_SEL_SW_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_RD, sel,
+ context->sw,
+ QDMA_REG_IND_CTXT_WCNT_5, 0);
+ if (rv < 0)
+ return rv;
+
+ sel = c2h ? QDMA_CTXT_SEL_HW_C2H : QDMA_CTXT_SEL_HW_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_RD, sel,
+ context->hw,
+ QDMA_REG_IND_CTXT_WCNT_2, 0);
+ if (rv < 0)
+ return rv;
+
+ sel = c2h ? QDMA_CTXT_SEL_CR_C2H : QDMA_CTXT_SEL_CR_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_RD, sel,
+ context->cr,
+ QDMA_REG_IND_CTXT_WCNT_1, 0);
+ if (rv < 0)
+ return rv;
+
+ if (st && c2h) {
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_SEL_CMPT,
+ context->cmpt,
+ QDMA_REG_IND_CTXT_WCNT_5, 0);
+ if (rv < 0)
+ return rv;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_SEL_PFTCH,
+ context->prefetch,
+ QDMA_REG_IND_CTXT_WCNT_2, 0);
+ if (rv < 0)
+ return rv;
+ }
+
+ return 0;
+}
+
+int qdma_intr_context_read(struct xlnx_dma_dev *xdev,
+ int ring_index, unsigned int ctxt_sz, u32 *context)
+{
+ int rv = 0;
+
+ memset(context, 0, (sizeof(u32) * ctxt_sz));
+
+ rv = hw_indirect_ctext_prog(xdev, ring_index,
+ QDMA_CTXT_CMD_RD, QDMA_CTXT_SEL_COAL,
+ context, ctxt_sz, 0);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+
+int qdma_descq_context_program(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h,
+ struct hw_descq_context *context)
+
+{
+ u8 sel;
+ int rv;
+
+ /* always clear first */
+ rv = qdma_descq_context_clear(xdev, qid_hw, st, c2h, 1);
+ if (rv < 0)
+ return rv;
+
+ sel = c2h ? QDMA_CTXT_SEL_SW_C2H : QDMA_CTXT_SEL_SW_H2C;
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_WR,
+ sel,
+ context->sw,
+ QDMA_REG_IND_CTXT_WCNT_5, 1);
+ if (rv < 0)
+ return rv;
+
+ /* Only c2h st specific setup done below*/
+ if (!st || !c2h)
+ return 0;
+
+ /* prefetch context */
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_SEL_PFTCH,
+ context->prefetch,
+ QDMA_REG_IND_CTXT_WCNT_2, 1);
+ if (rv < 0)
+ return rv;
+
+ /* writeback context */
+ rv = hw_indirect_ctext_prog(xdev, qid_hw, QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_SEL_CMPT,
+ context->cmpt,
+ QDMA_REG_IND_CTXT_WCNT_5, 1);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+
+
+int qdma_descq_stm_read(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ u8 pipe_flow_id, bool c2h, bool map, bool ctxt,
+ struct stm_descq_context *context)
+{
+ int rv = 0;
+
+ if (!map) {
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_RD,
+ ctxt ? STM_IND_ADDR_Q_CTX_H2C :
+ STM_IND_ADDR_FORCED_CAN,
+ context->stm, 5, false);
+ } else {
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_RD,
+ c2h ? STM_IND_ADDR_C2H_MAP :
+ STM_IND_ADDR_H2C_MAP,
+ context->stm, c2h ? 1 : 5,
+ false);
+ }
+ return rv;
+}
+
+int qdma_descq_stm_program(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ u8 pipe_flow_id, bool c2h, bool clear,
+ struct stm_descq_context *context)
+{
+ int rv;
+
+ if (!c2h) {
+ /* need to program stm context */
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_WR,
+ STM_IND_ADDR_Q_CTX_H2C,
+ context->stm, 5, clear);
+ if (rv < 0)
+ return rv;
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_WR,
+ STM_IND_ADDR_H2C_MAP,
+ context->stm, 1, clear);
+ if (rv < 0)
+ return rv;
+ }
+
+ /* Only c2h st specific setup done below*/
+ if (!c2h)
+ return 0;
+
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_WR,
+ STM_IND_ADDR_Q_CTX_C2H,
+ context->stm, 2, clear);
+ if (rv < 0)
+ return rv;
+
+ rv = hw_indirect_stm_prog(xdev, qid_hw, pipe_flow_id,
+ STM_CSR_CMD_WR, STM_IND_ADDR_C2H_MAP,
+ context->stm, 1, clear);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_context.h b/QDMA/linux-kernel/libqdma/qdma_context.h
new file mode 100644
index 0000000000000000000000000000000000000000..7d413217b6b94d5c4f2d5f6f7b08ab38f2bccedb
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_context.h
@@ -0,0 +1,198 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __LIBQDMA_CONTEXT_H__
+#define __LIBQDMA_CONTEXT_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma context handlers
+ *
+ */
+#include "xdev.h"
+#include "qdma_mbox.h"
+
+/*****************************************************************************/
+/**
+ * qdma_intr_context_setup() - handler to set the qdma interrupt context
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_intr_context_setup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * qdma_prog_intr_context() -
+ * handler to program the qdma interrupt context for
+ * VF from PF
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] ictxt: interrupt context
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+
+int qdma_prog_intr_context(struct xlnx_dma_dev *xdev,
+ struct mbox_msg_intr_ctxt *ictxt);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_context_setup() - handler to set the qdma sw descriptor context
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_context_setup(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_stm_setup() - handler to set the qdma stm
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_stm_setup(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_stm_clear() - handler to clear the qdma stm
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_stm_clear(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_context_clear() - handler to clear the qdma sw descriptor context
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qid_hw: hw qidx
+ * @param[in] st: indicated whether the mm mode or st mode
+ * @param[in] c2h: indicates whether the h2c or c2h direction
+ * @param[in] clr: flag to indicate whether to clear the context or not
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_context_clear(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h, bool clr);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_context_read() - handler to read the queue context
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qid_hw: hw qidx
+ * @param[in] st: indicated whether the mm mode or st mode
+ * @param[in] c2h: indicates whether the h2c or c2h direction
+ * @param[out] ctxt: pointer to context data
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_context_read(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h,
+ struct hw_descq_context *ctxt);
+
+/*****************************************************************************/
+/**
+ * qdma_intr_context_read() - handler to read the interrupt context
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] ring_index: interrupt ring index
+ * @param[in] ctxt_sz: context size
+ * @param[out] context: pointer to interrupt context*
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_intr_context_read(struct xlnx_dma_dev *xdev,
+ int ring_index, unsigned int ctxt_sz,
+ u32 *context);
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_descq_context_read() - handler to program the context for vf
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qid_hw: hw qidx
+ * @param[in] st: indicated whether the mm mode or st mode
+ * @param[in] c2h: indicates whether the h2c or c2h direction
+ * @param[out] ctxt: pointer to context data
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_context_program(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ bool st, bool c2h,
+ struct hw_descq_context *ctxt);
+
+
+/*****************************************************************************/
+/**
+ * qdma_descq_stm_read() - handler to read stm context, can, maps
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qid_hw: hw qidx
+ * @param[in] pipe_flow_id: pipe_flow_id for queue
+ * @param[in] c2h: indicates whether the h2c or c2h direction
+ * @param[in] map: indicates whether to read map or ctxt/can
+ * @param[in] ctxt: indicates whether to read ctxt or can
+ * @param[out] context: pointer to context data
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_stm_read(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ u8 pipe_flow_id, bool c2h, bool map, bool ctxt,
+ struct stm_descq_context *context);
+
+
+/*****************************************************************************/
+/**
+ * qdma_descq_stm_program() - handler to program the stm
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qid_hw: hw qidx
+ * @param[in] pipe_flow_id: flow id for pipe
+ * @param[in] c2h: indicates whether the h2c or c2h direction
+ * @param[in] clear: flag to prog/clear stm context/maps
+ * @param[out] stm: pointer to stm data
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_stm_program(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ uint8_t pipe_flow_id, bool c2h, bool clear,
+ struct stm_descq_context *context);
+
+#endif
+
+#endif /* ifndef __LIBQDMA_CONTEXT_H__ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs.c b/QDMA/linux-kernel/libqdma/qdma_debugfs.c
new file mode 100644
index 0000000000000000000000000000000000000000..07e5e430a9b863a0d1e8761b3dcc1da22e34a083
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs.c
@@ -0,0 +1,74 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_debugfs.h"
+
+/*****************************************************************************/
+/**
+ * qdma_debugfs_init() - function to initialize debugfs
+ *
+ * param[in]: qdma_debugfs_root - debugfs root
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_debugfs_init(struct dentry **qdma_debugfs_root)
+{
+ struct dentry *debugfs_root = NULL;
+ /* create a directory by the name qdma in
+ * /sys/kernel/debugfs
+ */
+#ifndef __QDMA_VF__
+ debugfs_root = debugfs_create_dir("qdma_pf", NULL);
+ if (!debugfs_root)
+ return -ENOENT;
+ pr_debug("created qdma_pf dir in Linux debug file system\n");
+
+#else
+ debugfs_root = debugfs_create_dir("qdma_vf", NULL);
+ if (!debugfs_root)
+ return -ENOENT;
+ pr_debug("created qdma_vf dir in Linux debug file system\n");
+
+#endif
+
+ *qdma_debugfs_root = debugfs_root;
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_debugfs_exit() - function to cleanup debugfs
+ *
+ * param[in]: qdma_debugfs_root - debugfs root
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+void qdma_debugfs_exit(struct dentry *qdma_debugfs_root)
+{
+ debugfs_remove_recursive(qdma_debugfs_root);
+#ifndef __QDMA_VF__
+ pr_debug("removed qdma_pf directory from Linux debug file system\n");
+#else
+ pr_debug("removed qdma_vf directory from Linux debug file system\n");
+#endif
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs.h b/QDMA/linux-kernel/libqdma/qdma_debugfs.h
new file mode 100644
index 0000000000000000000000000000000000000000..f81dce43db586121e73da26513538d9da43f8f79
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs.h
@@ -0,0 +1,52 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_DEBUGFS_H__
+#define __QDMA_DEBUGFS_H__
+
+#include <linux/pci.h>
+#include <linux/debugfs.h>
+#include <linux/fs.h>
+#include <asm/uaccess.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+
+#define DBGFS_DBG_FNAME_SZ (64)
+#define DBGFS_CTXT_ENTRY_NAME_SZ (64)
+#define QDMA_DEV_NAME_SZ (64)
+
+/*****************************************************************************/
+/**
+ * qdma_debugfs_init() - function to initialize debugfs
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int qdma_debugfs_init(struct dentry **qdma_debugfs_root);
+
+/*****************************************************************************/
+/**
+ * qdma_debugfs_exit() - function to cleanup debugfs
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+void qdma_debugfs_exit(struct dentry *qdma_debugfs_root);
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.c b/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.c
new file mode 100644
index 0000000000000000000000000000000000000000..77a73e906852873c0a42c3a283cfd1e1afd2f062
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.c
@@ -0,0 +1,655 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifdef DEBUGFS
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_debugfs_dev.h"
+#include "xdev_regs.h"
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include <linux/mutex.h>
+
+#define DEBUGFS_DEV_INFO_SZ (256)
+
+#define DBGFS_ERR_BUFLEN (64)
+#define DEBGFS_LINE_SZ (81)
+#define DEBGFS_GEN_NAME_SZ (32)
+#define REG_DUMP_SIZE_PER_LINE (256)
+
+
+enum dbgfs_dev_dbg_file_type {
+ DBGFS_DEV_DBGF_INFO = 0,
+ DBGFS_DEV_DBGF_REGS = 1,
+ DBGFS_DEV_DBGF_END,
+};
+
+struct dbgfs_dev_dbgf {
+ char name[DBGFS_DBG_FNAME_SZ];
+ struct file_operations fops;
+};
+
+struct dbgfs_dev_priv {
+ unsigned long dev_hndl;
+ char dev_name[QDMA_DEV_NAME_SZ];
+ char *data;
+ int datalen;
+};
+
+enum bar_type {
+ DEBUGFS_BAR_CONFIG = 0,
+ DEBUGFS_BAR_USER = 1,
+ DEBUGFS_BAR_BYPASS = 2,
+};
+
+extern struct dentry *qdma_debugfs_root;
+
+/** structure to hold file ops */
+static struct dbgfs_dev_dbgf dbgf[DBGFS_DEV_DBGF_END];
+
+/*****************************************************************************/
+/**
+ * dump_banner() - static helper function to dump a device banner
+ *
+ * @param[in] dev_name: qdma device name
+ * @param[out] buf: buffer to which banner is dumped
+ * @param[in] buf_sz: size of the buffer passed to func
+ *
+ * @return len: length of the buffer printed
+ *****************************************************************************/
+static int dump_banner(char *dev_name, char *buf, int buf_sz)
+{
+ int len = 0;
+ char seperator[81] = {0};
+
+ memset(seperator, '#', 80);
+
+ /** Banner consumes three lines, so size should be min 240 (80 * 3)
+ * If changed, check the minimum buffer size required
+ */
+ if (buf_sz < (3 * DEBGFS_LINE_SZ))
+ return -1;
+
+ len += snprintf(buf + len, buf_sz - len, "%s\n", seperator);
+ len += snprintf(buf + len, buf_sz - len,
+ "###\t\t\t\tqdma%s, reg dump\n",
+ dev_name);
+ len += snprintf(buf + len, buf_sz - len, "%s\n", seperator);
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * dump_reg() - static helper function to dump a specific reg
+ *
+ * @param[out] buf: buffer to dump the registers
+ * @param[in] buf_sz: size of the buffer passed to func
+ * @param[in] rname: register name
+ * @param[in] rval: register value
+ *
+ * @return len: length of the buffer printed
+ *****************************************************************************/
+static int dump_reg(char *buf, int buf_sz, unsigned int raddr, char *rname,
+ unsigned int rval)
+{
+ int len = 0;
+
+ /* length of the line should not exceed 80 chars, so, checking
+ * for min 80 chars. If below print pattern is changed, check for
+ * new the buffer size requirement
+ */
+ if (buf_sz < DEBGFS_LINE_SZ)
+ return -1;
+
+ len += sprintf(buf + len, "[%#7x] %-47s %#-10x %u\n",
+ raddr, rname, rval, rval);
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * dump_bar_regs() - static helper function to dump a specific bar regs
+ *
+ * @param[in] dev_hndl: pointer to xdev device structure
+ * @param[out] buf: buffer to dump the registers
+ * @param[in] buf_len: size of the buffer passed
+ * @param[in] type: bar type
+ *
+ * @return len: returns the length buffer printed
+ *****************************************************************************/
+static int dump_bar_regs(unsigned long dev_hndl, char *buf, int buf_len,
+ enum bar_type type)
+{
+ int num_regs = 0;
+ int i = 0, j = 0;
+ unsigned int val = 0;
+ unsigned int addr = 0;
+ int len = 0;
+ int rv;
+ char name[DEBGFS_GEN_NAME_SZ] = {0};
+ char bar_name[DEBGFS_GEN_NAME_SZ] = {0};
+ struct xreg_info *reg_info;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (type == DEBUGFS_BAR_CONFIG) {
+ num_regs = sizeof(qdma_config_regs)/sizeof(qdma_config_regs[0]);
+ reg_info = qdma_config_regs;
+ snprintf(bar_name, DEBGFS_GEN_NAME_SZ, "%s #%d", "CONFIG BAR",
+ xdev->conf.bar_num_config);
+ } else if (type == DEBUGFS_BAR_USER) {
+ num_regs = sizeof(qdma_user_regs)/sizeof(qdma_user_regs[0]);
+ reg_info = qdma_user_regs;
+ snprintf(bar_name, DEBGFS_GEN_NAME_SZ, "%s #%d", "USER BAR",
+ xdev->conf.bar_num_user);
+ }
+
+ num_regs -= 1; /* last entry is empty */
+
+ /* print the bar name */
+ len += snprintf(buf + len, buf_len - len, "\n%s\n", bar_name);
+
+ /* now print all the registers of the corresponding bar */
+ for (i = 0; i < num_regs; i++) {
+ /* if there are more than one register of same type
+ * loop and print 'repeat' number of times
+ */
+ if (reg_info[i].repeat) {
+ for (j = 0; j < reg_info[i].repeat; j++) {
+ memset(name, 0, DEBGFS_GEN_NAME_SZ);
+ snprintf(name, DEBGFS_GEN_NAME_SZ, "%s_%d",
+ reg_info[i].name, j);
+
+ addr = reg_info[i].addr + (j * 4);
+
+ /* read the register value */
+ val = qdma_device_read_config_register(dev_hndl,
+ addr);
+
+ /* num print address, name & value of reg */
+ rv = dump_reg(buf + len, buf_len - len, addr,
+ name, val);
+ if (rv < 0) {
+ pr_warn("insufficient space to dump reg vals\n");
+ return len;
+ }
+ len += rv;
+ }
+ } else {
+ addr = reg_info[i].addr;
+ val = qdma_device_read_config_register(dev_hndl,
+ addr);
+ rv = dump_reg(buf + len, buf_len - len, addr,
+ (char *)reg_info[i].name, val);
+ if (rv < 0) {
+ pr_warn("inusfficient space to dump register values\n");
+ return len;
+ }
+ len += rv;
+ }
+ }
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_dump_qdma_regs() - static function to dump qdma device registers
+ *
+ * @param[in] xpdev: pointer to qdma pci device structure
+ * @param[in] buf: buffer to dump the registers
+ * @param[in] buf_len:size of the buffer passed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dbgfs_dump_qdma_regs(unsigned long dev_hndl, char *dev_name,
+ char **data, int *data_len)
+{
+ int len = 0;
+ int rv;
+ char *buf = NULL;
+ int num_elem =
+ ((sizeof(qdma_config_regs)/sizeof(struct xreg_info)) + 1) +
+ ((sizeof(qdma_user_regs)/sizeof(struct xreg_info)) + 1);
+ int buflen = num_elem * REG_DUMP_SIZE_PER_LINE;
+
+ /** allocate memory */
+ buf = (char *) kzalloc(buflen, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ /* print the banner with device info */
+ rv = dump_banner(dev_name, buf + len, buflen - len);
+ if (rv < 0) {
+ pr_warn("insufficient space to dump register banner\n");
+ return len;
+ }
+ len += rv;
+
+ rv = dump_bar_regs(dev_hndl, buf + len, buflen - len,
+ DEBUGFS_BAR_USER);
+ if (rv < 0) {
+ pr_warn("insufficient space to dump User bar register values\n");
+ return len;
+ }
+ len += rv;
+
+ rv = dump_bar_regs(dev_hndl, buf + len, buflen - len,
+ DEBUGFS_BAR_CONFIG);
+ if (rv < 0) {
+ pr_warn("insufficient space to dump Config Bar register values\n");
+ return len;
+ }
+ len += rv;
+
+ *data = buf;
+ *data_len = buflen;
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_dump_qdma_info() - static function to dump qdma device registers
+ *
+ * @xpdev: pointer to qdma pci device structure
+ * @buf: buffer to dump the registers
+ * @buf_len:size of the buffer passed
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dbgfs_dump_qdma_info(unsigned long dev_hndl, char *dev_name,
+ char **data, int *data_len)
+{
+ int len = 0;
+ char *buf = NULL;
+ int buflen = DEBUGFS_DEV_INFO_SZ;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_dev_conf *conf = NULL;
+
+ if (!xdev)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ conf = &xdev->conf;
+
+ /** allocate memory */
+ buf = (char *) kzalloc(buflen, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ len += snprintf(buf + len, buflen - len, "%-36s: %s\n", "Master PF",
+ (conf->master_pf) ? "True" : "False");
+ len += snprintf(buf + len, buflen - len, "%-36s: %d\n", "QBase",
+ conf->qsets_base);
+ len += snprintf(buf + len, buflen - len, "%-36s: %d\n", "Max Qsets",
+ conf->qsets_max);
+ len += snprintf(buf + len, buflen - len, "%-36s: %d\n",
+ "Number of VFs", xdev->vf_count);
+ len += snprintf(buf + len, buflen - len, "%-36s: %d\n",
+ "Max number of VFs", conf->vf_max);
+ len += snprintf(buf + len, buflen - len, "%-36s: %s mode\n",
+ "Driver Mode",
+ mode_name_list[conf->qdma_drv_mode].name);
+
+ *data = buf;
+ *data_len = buflen;
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * dev_dbg_file_open() - static function that provides generic open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dev_dbg_file_open(struct inode *inode, struct file *fp)
+{
+ int dev_id = -1;
+ int rv = 0;
+ unsigned char dev_name[QDMA_DEV_NAME_SZ] = {0};
+ unsigned char *lptr = NULL, *rptr = NULL;
+ struct dentry *dev_dir = NULL;
+ struct dbgfs_dev_priv *priv = NULL;
+ struct xlnx_dma_dev *xdev = NULL;
+
+ if (!inode || !fp)
+ return -EINVAL;
+ dev_dir = fp->f_path.dentry->d_parent;
+ xdev = inode->i_private;
+ if (!xdev)
+ return -EINVAL;
+
+ /* convert colon sepearted b:d:f to hex */
+ rptr = dev_dir->d_iname;
+ lptr = dev_name;
+ while (*rptr) {
+ if (*rptr == ':') {
+ rptr++;
+ continue;
+ }
+ *lptr++ = *rptr++;
+ }
+
+ /* convert this string as hex integer */
+ rv = kstrtoint((const char *)dev_name, 16, &dev_id);
+ if (rv < 0) {
+ rv = -ENODEV;
+ return rv;
+ }
+
+
+ priv = (struct dbgfs_dev_priv *) kzalloc(sizeof(struct dbgfs_dev_priv),
+ GFP_KERNEL);
+ if (!priv) {
+ rv = -ENOMEM;
+ return rv;
+ }
+
+ priv->dev_hndl = (unsigned long)xdev;
+ snprintf(priv->dev_name, QDMA_DEV_NAME_SZ, "%s", dev_name);
+ fp->private_data = priv;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * dev_dbg_file_release() - static function that provides generic release
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dev_dbg_file_release(struct inode *inode, struct file *fp)
+{
+ kfree(fp->private_data);
+
+ fp->private_data = NULL;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * dev_dbg_file_read() - static function that provides common read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ * @param[in] type: information type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t dev_dbg_file_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos, enum dbgfs_dev_dbg_file_type type)
+{
+ char *buf = NULL;
+ int buf_len = 0;
+ int len = 0;
+ int rv = 0;
+ struct dbgfs_dev_priv *dev_priv =
+ (struct dbgfs_dev_priv *)fp->private_data;
+
+ if (dev_priv->data == NULL && dev_priv->datalen == 0) {
+ if (type == DBGFS_DEV_DBGF_INFO) {
+ rv = dbgfs_dump_qdma_info(dev_priv->dev_hndl,
+ dev_priv->dev_name, &buf, &buf_len);
+ } else if (type == DBGFS_DEV_DBGF_REGS) {
+ rv = dbgfs_dump_qdma_regs(dev_priv->dev_hndl,
+ dev_priv->dev_name, &buf, &buf_len);
+ }
+
+ if (rv < 0)
+ goto dev_dbg_file_read_exit;
+
+ dev_priv->datalen = rv;
+ dev_priv->data = buf;
+ }
+
+ buf = dev_priv->data;
+ len = dev_priv->datalen;
+
+ if (!buf)
+ goto dev_dbg_file_read_exit;
+
+ /** write to user buffer */
+ if (*ppos >= len) {
+ rv = 0;
+ goto dev_dbg_file_read_exit;
+ }
+
+ if (*ppos + count > len)
+ count = len - *ppos;
+
+ if (copy_to_user(user_buffer, buf + *ppos, count)) {
+ rv = -EFAULT;
+ goto dev_dbg_file_read_exit;
+ }
+
+ *ppos += count;
+ rv = count;
+
+ pr_debug("nuber of bytes written to user space is %zu\n", count);
+
+dev_dbg_file_read_exit:
+ kfree(buf);
+ dev_priv->data = NULL;
+ dev_priv->datalen = 0;
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * dev_info_open() - static function that executes info open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dev_info_open(struct inode *inode, struct file *fp)
+{
+ return dev_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * dev_info_read() - static function that executes info read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t dev_info_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return dev_dbg_file_read(fp, user_buffer, count, ppos,
+ DBGFS_DEV_DBGF_INFO);
+}
+
+/*****************************************************************************/
+/**
+ * dev_regs_open() - static function that opens regs debug file
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int dev_regs_open(struct inode *inode, struct file *fp)
+{
+ return dev_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * dev_regs_read() - static function that executes info read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t dev_regs_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return dev_dbg_file_read(fp, user_buffer, count, ppos,
+ DBGFS_DEV_DBGF_REGS);
+}
+
+/*****************************************************************************/
+/**
+ * create_dev_dbg_files() - static function to create queue debug files
+ *
+ * @param[in] dev_root: debugfs root for the device
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int create_dev_dbg_files(struct xlnx_dma_dev *xdev, struct dentry *dev_root)
+{
+ struct dentry *fp[DBGFS_DEV_DBGF_END] = { NULL };
+ struct file_operations *fops = NULL;
+ int i = 0;
+
+ memset(dbgf, 0, sizeof(struct dbgfs_dev_dbgf) * DBGFS_DEV_DBGF_END);
+
+ for (i = 0; i < DBGFS_DEV_DBGF_END; i++) {
+ fops = &dbgf[i].fops;
+ fops->owner = THIS_MODULE;
+ switch (i) {
+ case DBGFS_DEV_DBGF_INFO:
+ snprintf(dbgf[i].name, 64, "%s", "info");
+ fops->open = dev_info_open;
+ fops->read = dev_info_read;
+ fops->release = dev_dbg_file_release;
+ break;
+ case DBGFS_DEV_DBGF_REGS:
+ snprintf(dbgf[i].name, 64, "%s", "regs");
+ fops->open = dev_regs_open;
+ fops->read = dev_regs_read;
+ fops->release = dev_dbg_file_release;
+ break;
+ }
+ }
+
+ for (i = 0; i < DBGFS_DEV_DBGF_END; i++) {
+ fp[i] = debugfs_create_file(dbgf[i].name, 0644, dev_root,
+ xdev, &dbgf[i].fops);
+ if (!fp[i])
+ return -1;
+ }
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_dev_init() - function to initialize device debugfs files
+ *
+ * @param[in] xdev: Xilinx dma device
+ * @param[in] qdma_debugfs_root: root file in debugfs
+ *
+ * @return =0: success
+ * @return <0: error
+ *****************************************************************************/
+int dbgfs_dev_init(struct xlnx_dma_dev *xdev)
+{
+ char dname[QDMA_DEV_NAME_SZ] = {0};
+ struct dentry *dbgfs_dev_root = NULL;
+ struct dentry *dbgfs_queues_root = NULL;
+ struct pci_dev *pdev = xdev->conf.pdev;
+ int rv = 0;
+
+ snprintf(dname, QDMA_DEV_NAME_SZ, "%02x:%02x:%x",
+ pdev->bus->number,
+ PCI_SLOT(pdev->devfn),
+ PCI_FUNC(pdev->devfn));
+ /* create a directory for the device in debugfs */
+ dbgfs_dev_root = debugfs_create_dir(dname, qdma_debugfs_root);
+ if (!dbgfs_dev_root) {
+ pr_err("Failed to create device direcotry\n");
+ return -1;
+ }
+ xdev->dbgfs_dev_root = dbgfs_dev_root;
+
+ /* create debug files for qdma device */
+ rv = create_dev_dbg_files(xdev, dbgfs_dev_root);
+ if (rv < 0) {
+ pr_err("Failed to create device debug files\n");
+ goto dbgfs_dev_init_fail;
+ }
+
+ /* create a directory for the queues in debugfs */
+ dbgfs_queues_root = debugfs_create_dir("queues", xdev->dbgfs_dev_root);
+ if (!dbgfs_queues_root) {
+ pr_err("Failed to create queueus directory under device directory\n");
+ goto dbgfs_dev_init_fail;
+ }
+
+ xdev->dbgfs_queues_root = dbgfs_queues_root;
+ spin_lock_init(&xdev->qidx_lock);
+
+ return 0;
+
+dbgfs_dev_init_fail:
+
+ debugfs_remove_recursive(dbgfs_dev_root);
+ return -1;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_dev_exit() - function to cleanup device debugfs files
+ *
+ * @param[in] xdev: Xilinx dma device
+ *
+ *****************************************************************************/
+void dbgfs_dev_exit(struct xlnx_dma_dev *xdev)
+{
+ if (xdev->dbgfs_dev_root)
+ debugfs_remove_recursive(xdev->dbgfs_dev_root);
+ xdev->dbgfs_dev_root = NULL;
+ xdev->dbgfs_queues_root = NULL;
+}
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.h b/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.h
new file mode 100644
index 0000000000000000000000000000000000000000..47af008049779b85f3d641c288d2f0beadc13f51
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs_dev.h
@@ -0,0 +1,29 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_DEBUGFS_DEV_H__
+#define __QDMA_DEBUGFS_DEV_H__
+
+#include "qdma_debugfs.h"
+#include "xdev.h"
+
+int dbgfs_dev_init(struct xlnx_dma_dev *xdev);
+void dbgfs_dev_exit(struct xlnx_dma_dev *xdev);
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.c b/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.c
new file mode 100644
index 0000000000000000000000000000000000000000..62c59ed2ac7715d4fb6118625752dd405258e583
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.c
@@ -0,0 +1,1190 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_debugfs_queue.h"
+#include "libqdma_export.h"
+#include "qdma_regs.h"
+#include "qdma_context.h"
+#include "qdma_descq.h"
+#include "qdma_regs.h"
+#include <linux/uaccess.h>
+
+#ifdef DEBUGFS
+#define DEBUGFS_QUEUE_DESC_SZ (100)
+#define DEBUGFS_QUEUE_INFO_SZ (256)
+#define DEBUGFS_QUEUE_CTXT_SZ (2 * 4096)
+
+#define DEBUGFS_CTXT_ELEM(reg, pos, size) \
+ ((reg >> pos) & ~(~0 << size))
+
+#define DBGFS_QUEUE_INFO_SZ 256
+#define DBGFS_ERR_BUFLEN (64)
+
+enum dbgfs_queue_info_type {
+ DBGFS_QINFO_INFO = 0,
+ DBGFS_QINFO_CNTXT = 1,
+ DBGFS_QINFO_DESC = 2,
+ DBGFS_QINFO_END,
+};
+
+enum dbgfs_cntxt_word {
+ DBGFS_CNTXT_W0 = 0,
+ DBGFS_CNTXT_W1 = 1,
+ DBGFS_CNTXT_W2 = 2,
+ DBGFS_CNTXT_W3 = 3,
+ DBGFS_CNTXT_W4 = 4,
+ DBGFS_CNTXT_W5 = 5,
+ DBGFS_CNTXT_W6 = 6,
+ DBGFS_CNTXT_W7 = 7,
+};
+
+enum dbgfs_cmpt_queue_info_type {
+ DBGFS_CMPT_QINFO_INFO = 0,
+ DBGFS_CMPT_QINFO_CNTXT = 1,
+ DBGFS_CMPT_QINFO_DESC = 2,
+ DBGFS_CMPT_QINFO_END,
+};
+
+struct dbgfs_q_dbgf {
+ char name[DBGFS_DBG_FNAME_SZ];
+ struct file_operations fops;
+};
+
+struct dbgfs_q_priv {
+ unsigned long dev_hndl;
+ unsigned long qhndl;
+ char *data;
+ int datalen;
+};
+
+struct dbgfs_qctxt_entry {
+ char name[DBGFS_CTXT_ENTRY_NAME_SZ];
+ unsigned short word;
+ u32 pos;
+ u32 len;
+};
+
+enum dbgfs_desc_type {
+ DBGFS_DESC_TYPE_C2H = 0,
+ DBGFS_DESC_TYPE_H2C = DBGFS_DESC_TYPE_C2H,
+ DBGFS_DESC_TYPE_CMPT = 1,
+ DBGFS_DESC_TYPE_END = 2,
+};
+
+
+static struct dbgfs_qctxt_entry app_ctxt[] = {
+ /** format:
+ *{<name>, <word>, <start_bit_index>, <len>}
+ */
+
+ {"PIDX", DBGFS_CNTXT_W0, S_DESC_CTXT_W0_PIDX,
+ L_DESC_CTXT_W0_PIDX},
+ {"IRQ Arm", DBGFS_CNTXT_W0, S_DESC_CTXT_W0_F_INTR_ARM, 1},
+ {"Function Id", DBGFS_CNTXT_W0, S_DESC_CTXT_W0_FUNC_ID,
+ L_DESC_CTXT_W0_FUNC_ID},
+
+ {"Queue Enable", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_F_QEN, 1},
+ {"Fetch Credit Enable", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_FCRD_EN, 1},
+ {"Write back/Intr Check", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_CMPL_STATUS_PEND_CHK, 1},
+ {"Write back Acc Enable", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_CMPL_STATUS_ACC_EN, 1},
+ {"Address Translation", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_AT, 1},
+ {"Fetch Max", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_FETCH_MAX, L_DESC_CTXT_W1_FETCH_MAX},
+ {"Ring Size", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_RNG_SZ,
+ L_DESC_CTXT_W1_RNG_SZ},
+ {"Descriptor Size", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_DSC_SZ,
+ L_DESC_CTXT_W1_DSC_SZ},
+ {"Bypass Enable", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_F_BYP, 1},
+ {"MM Channel", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_F_MM_CHN, 1},
+ {"Writeback Enable", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_CMPL_STATUS_EN, 1},
+ {"Interrupt Enable", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_IRQ_EN, 1},
+ {"Port Id", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_PORT_ID,
+ L_DESC_CTXT_W1_PORT_ID},
+ {"Interrupt No Last", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_IRQ_NO_LAST, 1},
+ {"Error", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_ERR,
+ L_DESC_CTXT_W1_ERR},
+ {"Writeback Error Sent", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_CMPL_STATUS_ERR_SNT, 1},
+ {"IRQ Request", DBGFS_CNTXT_W1, S_DESC_CTXT_W1_F_IRQ_REQ, 1},
+ {"Marker Disable", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_MRKR_DIS, 1},
+ {"Is Memory Mapped", DBGFS_CNTXT_W1,
+ S_DESC_CTXT_W1_F_IS_MM, 1},
+ {"Interrupt Aggregation", DBGFS_CNTXT_W4,
+ S_DESC_CTXT_W1_F_INTR_AGGR, 1},
+
+ {"Descriptor Ring Base Addr (Low)", DBGFS_CNTXT_W2, 0, 32},
+
+ {"Descriptor Ring Base Addr (High)", DBGFS_CNTXT_W3, 0, 32},
+};
+
+static struct dbgfs_qctxt_entry hw_ctxt[] = {
+ {"CIDX", DBGFS_CNTXT_W0, 0, 16},
+ {"Credits Consumed", DBGFS_CNTXT_W0, 16, 16},
+
+ {"Descriptors Pending", DBGFS_CNTXT_W1, 8, 1},
+ {"Queue Invalid No Desc Pending", DBGFS_CNTXT_W1, 9, 1},
+ {"Eviction Pending", DBGFS_CNTXT_W1, 10, 1},
+ {"Fetch Peding", DBGFS_CNTXT_W1, 10, 1},
+};
+
+static struct dbgfs_qctxt_entry credit_ctxt[] = {
+ {"Credit", DBGFS_CNTXT_W0, 0, 16},
+};
+
+static struct dbgfs_qctxt_entry cmpt_ctxt[] = {
+ {"Enable Status Desc Update", DBGFS_CNTXT_W0,
+ S_CMPT_CTXT_W0_F_EN_STAT_DESC, 1},
+ {"Enable Interrupt", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_F_EN_INT, 1},
+ {"Trigger Mode", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_TRIG_MODE,
+ L_CMPT_CTXT_W0_TRIG_MODE},
+ {"Function Id", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_FNC_ID,
+ L_CMPT_CTXT_W0_FNC_ID},
+ {"Counter Index", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_COUNTER_IDX,
+ L_CMPT_CTXT_W0_COUNTER_IDX},
+ {"Timer Index", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_TIMER_IDX,
+ L_CMPT_CTXT_W0_TIMER_IDX},
+ {"Interrupt State", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_INT_ST,
+ L_CMPT_CTXT_W0_INT_ST},
+ {"Color", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_F_COLOR, 1},
+ {"Ring Size", DBGFS_CNTXT_W0, S_CMPT_CTXT_W0_RNG_SZ,
+ L_CMPT_CTXT_W0_RNG_SZ},
+
+ {"Base Address (Low)", DBGFS_CNTXT_W1, 0, 32},
+
+ {"Base Address (High)", DBGFS_CNTXT_W2, S_CMPT_CTXT_W2_BADDR_64,
+ L_CMPT_CTXT_W2_BADDR_64},
+ {"Descriptor Size", DBGFS_CNTXT_W2, S_CMPT_CTXT_W2_DESC_SIZE,
+ L_CMPT_CTXT_W2_DESC_SIZE},
+ {"PIDX (Low)", DBGFS_CNTXT_W2, S_CMPT_CTXT_W2_PIDX_L,
+ L_CMPT_CTXT_W3_PIDX_H},
+
+ {"PIDX (High)", DBGFS_CNTXT_W3, S_CMPT_CTXT_W3_PIDX_H,
+ L_CMPT_CTXT_W3_PIDX_H},
+ {"CIDX", DBGFS_CNTXT_W3, S_CMPT_CTXT_W3_CIDX, L_CMPT_CTXT_W3_CIDX},
+ {"Valid", DBGFS_CNTXT_W3, S_CMPT_CTXT_W3_F_VALID, 1},
+ {"ERROR", DBGFS_CNTXT_W3, S_CMPT_CTXT_W3_ERR, L_CMPT_CTXT_W3_ERR},
+ {"Trigger Pending", DBGFS_CNTXT_W3, S_CMPT_CTXT_W3_F_TRIG_PEND, 1},
+
+ {"Timer Running", DBGFS_CNTXT_W4, S_CMPT_CTXT_W4_F_TMR_RUNNING, 1},
+ {"Full Update", DBGFS_CNTXT_W4, S_CMPT_CTXT_W4_F_FULL_UPDATE, 1},
+ {"Over Flow Check Disable", DBGFS_CNTXT_W4,
+ S_CMPT_CTXT_W4_F_OVF_CHK_DIS, 1},
+ {"Address Translation", DBGFS_CNTXT_W4, S_CMPT_CTXT_W4_F_AT, 1},
+};
+
+/**TBD: enable them when RTL2 is stable */
+static struct dbgfs_qctxt_entry c2h_pftch_ctxt[] = {
+ {"Bypass", DBGFS_CNTXT_W0, S_PFTCH_W0_F_BYPASS, 1},
+ {"Buffer Size Index", DBGFS_CNTXT_W0, S_PFTCH_W0_BUF_SIZE_IDX,
+ L_PFTCH_W0_BUF_SIZE_IDX},
+ {"Port Id", DBGFS_CNTXT_W0, S_PFTCH_W0_PORT_ID, L_PFTCH_W0_PORT_ID},
+ {"Error", DBGFS_CNTXT_W0, S_PFTCH_W0_F_ERR, 1},
+ {"Prefetch Enable", DBGFS_CNTXT_W0, S_PFTCH_W0_F_EN_PFTCH, 1},
+ {"In Prefetch", DBGFS_CNTXT_W0, S_PFTCH_W0_F_Q_IN_PFTCH, 1},
+ {"Software Credit (Low)", DBGFS_CNTXT_W0, S_PFTCH_W0_SW_CRDT_L,
+ L_PFTCH_W0_SW_CRDT_L},
+
+ {"Software Credit (High)", DBGFS_CNTXT_W1, S_PFTCH_W1_SW_CRDT_H,
+ L_PFTCH_W1_SW_CRDT_H},
+ {"Valid", DBGFS_CNTXT_W1, S_PFTCH_W1_F_VALID, 1},
+};
+
+/** structure to hold file ops */
+static struct dbgfs_q_dbgf qf[DBGFS_QINFO_END];
+
+/** structure to hold file ops */
+static struct dbgfs_q_dbgf cmpt_qf[DBGFS_CMPT_QINFO_END];
+int q_dbg_file_open(struct inode *inode, struct file *fp);
+int q_dbg_file_release(struct inode *inode, struct file *fp);
+int qdbg_info_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type);
+int qdbg_desc_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type);
+int qdbg_cntxt_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type);
+
+/*****************************************************************************/
+/**
+ * cmpt_q_dbg_file_open() - static function that provides generic open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int cmpt_q_dbg_file_open(struct inode *inode, struct file *fp)
+{
+ return q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_dbg_file_release() - static function that provides generic release
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int cmpt_q_dbg_file_release(struct inode *inode, struct file *fp)
+{
+ return q_dbg_file_release(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_dbg_file_read() - static function that provides common read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ * @param[in] type: information type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t cmpt_q_dbg_file_read(struct file *fp,
+ char __user *user_buffer,
+ size_t count, loff_t *ppos,
+ enum dbgfs_queue_info_type type)
+{
+ char *buf = NULL;
+ int buf_len = 0;
+ int len = 0;
+ int rv = 0;
+ struct dbgfs_q_priv *qpriv =
+ (struct dbgfs_q_priv *)fp->private_data;
+
+ if (qpriv->data == NULL && qpriv->datalen == 0) {
+ if (type == DBGFS_QINFO_INFO) {
+ rv = qdbg_info_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_CMPT);
+ } else if (type == DBGFS_QINFO_CNTXT) {
+ rv = qdbg_cntxt_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_CMPT);
+ } else if (type == DBGFS_QINFO_DESC) {
+ rv = qdbg_desc_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_CMPT);
+ }
+
+ if (rv < 0)
+ goto cmpt_q_dbg_file_read_exit;
+
+ qpriv->datalen = rv;
+ qpriv->data = buf;
+ }
+
+ buf = qpriv->data;
+ len = qpriv->datalen;
+
+ if (!buf)
+ goto cmpt_q_dbg_file_read_exit;
+
+ /** write to user buffer */
+ if (*ppos >= len) {
+ rv = 0;
+ goto cmpt_q_dbg_file_read_exit;
+ }
+
+ if (*ppos + count > len)
+ count = len - *ppos;
+
+ if (copy_to_user(user_buffer, buf + *ppos, count)) {
+ rv = -EFAULT;
+ goto cmpt_q_dbg_file_read_exit;
+ }
+
+ *ppos += count;
+
+ pr_debug("cmpt q read size %zu\n", count);
+
+ return count;
+
+cmpt_q_dbg_file_read_exit:
+ kfree(buf);
+ qpriv->data = NULL;
+ qpriv->datalen = 0;
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_info_open() - static function that executes info open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int cmpt_q_info_open(struct inode *inode, struct file *fp)
+{
+ return cmpt_q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_info_read() - static function that executes info read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t cmpt_q_info_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return cmpt_q_dbg_file_read(fp, user_buffer, count, ppos,
+ DBGFS_QINFO_INFO);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_cntxt_open() - static function that executes info open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int cmpt_q_cntxt_open(struct inode *inode, struct file *fp)
+{
+ return cmpt_q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_cntxt_read() - static function that executes info read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t cmpt_q_cntxt_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return cmpt_q_dbg_file_read(fp, user_buffer, count, ppos,
+ DBGFS_QINFO_CNTXT);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_desc_open() - static function that executes descriptor open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int cmpt_q_desc_open(struct inode *inode, struct file *fp)
+{
+ return cmpt_q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * cmpt_q_desc_read() - static function that executes descriptor read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t cmpt_q_desc_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return cmpt_q_dbg_file_read(fp, user_buffer, count, ppos,
+ DBGFS_QINFO_DESC);
+}
+
+/*****************************************************************************/
+/**
+ * create_cmpt_q_dbg_files() - static function to create cmpt queue dbg files
+ *
+ * @param[in] queue_root: debugfs root for a queue
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int create_cmpt_q_dbg_files(struct qdma_descq *descq, struct dentry *queue_root)
+{
+ struct dentry *fp[DBGFS_QINFO_END] = { NULL };
+ struct file_operations *fops = NULL;
+ int i = 0;
+
+ memset(cmpt_qf, 0, sizeof(struct dbgfs_q_dbgf) * DBGFS_CMPT_QINFO_END);
+
+ for (i = 0; i < DBGFS_CMPT_QINFO_END; i++) {
+ fops = &cmpt_qf[i].fops;
+ fops->owner = THIS_MODULE;
+ switch (i) {
+ case DBGFS_CMPT_QINFO_INFO:
+ snprintf(cmpt_qf[i].name, DBGFS_DBG_FNAME_SZ,
+ "%s", "info");
+ fops->open = cmpt_q_info_open;
+ fops->read = cmpt_q_info_read;
+ fops->release = cmpt_q_dbg_file_release;
+ break;
+ case DBGFS_CMPT_QINFO_CNTXT:
+ snprintf(cmpt_qf[i].name, DBGFS_DBG_FNAME_SZ,
+ "%s", "cntxt");
+ fops->open = cmpt_q_cntxt_open;
+ fops->read = cmpt_q_cntxt_read;
+ fops->release = cmpt_q_dbg_file_release;
+ break;
+ case DBGFS_CMPT_QINFO_DESC:
+ snprintf(cmpt_qf[i].name, DBGFS_DBG_FNAME_SZ,
+ "%s", "desc");
+ fops->open = cmpt_q_desc_open;
+ fops->read = cmpt_q_desc_read;
+ fops->release = cmpt_q_dbg_file_release;
+ break;
+ }
+ }
+
+ for (i = 0; i < DBGFS_CMPT_QINFO_END; i++) {
+ fp[i] = debugfs_create_file(cmpt_qf[i].name, 0644, queue_root,
+ descq, &cmpt_qf[i].fops);
+ if (!fp[i])
+ return -1;
+ }
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * q_dbg_file_open() - generic queue debug file open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int q_dbg_file_open(struct inode *inode, struct file *fp)
+{
+ int dev_id = -1;
+ int qidx = -1;
+ struct dbgfs_q_priv *priv = NULL;
+ int rv = 0;
+ int c2h = 0;
+ unsigned char dev_name[QDMA_DEV_NAME_SZ] = {0};
+ unsigned char *lptr = NULL, *rptr = NULL;
+ struct dentry *direction_dir = NULL;
+ struct dentry *qid_dir = NULL;
+ struct dentry *qroot_dir = NULL;
+ struct dentry *dev_dir = NULL;
+ struct qdma_descq *descq = NULL;
+
+ if (!inode || !fp)
+ return -EINVAL;
+ descq = inode->i_private;
+ if (!descq)
+ return -EINVAL;
+
+ direction_dir = fp->f_path.dentry->d_parent;
+ qid_dir = direction_dir->d_parent;
+ qroot_dir = qid_dir->d_parent;
+ dev_dir = qroot_dir->d_parent;
+
+ /* check the direction */
+ if (!strncmp((const char *)direction_dir->d_iname,
+ "c2h", strlen("c2h")) ||
+ !strncmp((const char *)direction_dir->d_iname,
+ "cmpt", strlen("cmpt")))
+ c2h = 1;
+
+ /* convert this string as integer */
+ rv = kstrtoint((const char *)qid_dir->d_iname, 0, &qidx);
+ if (rv < 0) {
+ rv = -ENODEV;
+ return rv;
+ }
+
+ /* convert colon sepearted b:d:f to hex */
+ rptr = dev_dir->d_iname;
+ lptr = dev_name;
+ while (*rptr) {
+ if (*rptr == ':') {
+ rptr++;
+ continue;
+ }
+ *lptr++ = *rptr++;
+ }
+
+ /* convert this string as hex integer */
+ rv = kstrtoint((const char *)dev_name, 16, &dev_id);
+ if (rv < 0) {
+ rv = -ENODEV;
+ return rv;
+ }
+
+ priv = (struct dbgfs_q_priv *) kzalloc(sizeof(struct dbgfs_q_priv),
+ GFP_KERNEL);
+ if (!priv) {
+ rv = -ENOMEM;
+ return rv;
+ }
+
+ priv->dev_hndl = (unsigned long)descq->xdev;
+ priv->qhndl = qdma_device_get_id_from_descq(descq->xdev, descq);
+ if (priv->qhndl < 0) {
+ kfree(priv);
+ return -EINVAL;
+ }
+
+ fp->private_data = priv;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * q_dbg_file_release() - function that provides generic release
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int q_dbg_file_release(struct inode *inode, struct file *fp)
+{
+ kfree(fp->private_data);
+
+ fp->private_data = NULL;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdbg_parse_ctxt_to_buf() - parses queue context to human readable format
+ *
+ * @param[in] ctxt: raw context info
+ * @param[in] entries: context entries structure pointer
+ * @param[in] num_entries: number of context entries
+ * @param[out] buf: buffer to write parsed context
+ * @param[in] buflen: buffer len
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int qdbg_parse_ctxt_to_buf(u32 *ctxt,
+ struct dbgfs_qctxt_entry *entries,
+ int num_entries,
+ char *buf, int buflen)
+{
+ int i = 0, w;
+ u32 v;
+ u64 mask = 0;
+ int len = 0;
+
+ for (i = num_entries - 1; i >= 0; i--) {
+ w = entries[i].word;
+ v = ((ctxt[w] >> entries[i].pos) &
+ (~((~mask) << entries[i].len)));
+ len += snprintf(buf + len, buflen - len,
+ "\t%-47s %#-10x %u\n",
+ entries[i].name, v, v);
+ }
+ len += snprintf(buf + len, buflen - len, "\n");
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdbg_cntxt_read() - reads queue context for a queue
+ *
+ * @param[in] dev_hndl: xdev device handle
+ * @param[in] id: queue handle
+ * @param[out] buf: buffer to collect the context info
+ * @param[in] buflen: buffer len
+ * @param[in] type: context type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int qdbg_cntxt_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type)
+{
+ int rv = 0;
+ int len = 0;
+ int num_entries = 0;
+ char *buf = NULL;
+ int buflen = DEBUGFS_QUEUE_CTXT_SZ;
+ struct hw_descq_context ctxt;
+ struct qdma_descq *descq = NULL;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!xdev)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ /* allocate memory */
+ buf = (char *) kzalloc(buflen, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ /** get descq by id */
+ descq = qdma_device_get_descq_by_id(xdev, id, buf, buflen, 0);
+ if (!descq) {
+ kfree(buf);
+ return QDMA_ERR_INVALID_QIDX;
+ }
+
+ /** initialize the context */
+ memset(&ctxt, 0, sizeof(struct hw_descq_context));
+ /** read the descq context for the given qid */
+ rv = qdma_descq_context_read(descq->xdev, descq->qidx_hw,
+ descq->conf.st, descq->conf.c2h, &ctxt);
+ if (rv < 0) {
+ len += sprintf(buf + len, "%s read context failed %d.\n",
+ descq->conf.name, rv);
+ buf[len] = '\0';
+
+ *data = buf;
+ *data_len = buflen;
+ return rv;
+ }
+
+ if (type == DBGFS_DESC_TYPE_CMPT) {
+ /** convert CMPT context to human readable text */
+ len += snprintf(buf + len, buflen - len, "CMPT CTXT:\n");
+ num_entries = sizeof(cmpt_ctxt)/sizeof(cmpt_ctxt[0]);
+ len += qdbg_parse_ctxt_to_buf(ctxt.cmpt, cmpt_ctxt,
+ num_entries, buf + len, buflen - len);
+ } else {
+ /** convert SW context to human readable text */
+ len += snprintf(buf + len, buflen - len, "SOFTWARE CTXT:\n");
+ num_entries = sizeof(app_ctxt)/sizeof(app_ctxt[0]);
+ len += qdbg_parse_ctxt_to_buf(ctxt.sw, app_ctxt,
+ num_entries, buf + len, buflen - len);
+
+ /** convert hardware context to human readable text */
+ len += snprintf(buf + len, buflen - len, "HARDWARE CTXT:\n");
+ num_entries = sizeof(hw_ctxt)/sizeof(hw_ctxt[0]);
+ len += qdbg_parse_ctxt_to_buf(ctxt.hw, hw_ctxt,
+ num_entries, buf + len, buflen - len);
+ if (!(descq->conf.st && descq->conf.c2h))
+ goto cntxt_exit;
+
+ /** convert credit context to human readable text */
+ len += snprintf(buf + len, buflen - len, "CREDIT CTXT:\n");
+ num_entries = sizeof(credit_ctxt)/sizeof(credit_ctxt[0]);
+ len += qdbg_parse_ctxt_to_buf(ctxt.cr, credit_ctxt,
+ num_entries, buf + len, buflen - len);
+
+ if (type == DBGFS_DESC_TYPE_C2H) {
+ /** convert prefetch context to human readable text */
+ len += snprintf(buf + len, buflen - len,
+ "PREFETCH CTXT:\n");
+ num_entries =
+ sizeof(c2h_pftch_ctxt)/sizeof(c2h_pftch_ctxt[0]);
+ len += qdbg_parse_ctxt_to_buf(ctxt.prefetch,
+ c2h_pftch_ctxt,
+ num_entries, buf + len, buflen - len);
+ }
+ }
+
+cntxt_exit:
+ buf[len] = '\0';
+
+ *data = buf;
+ *data_len = buflen;
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdbg_info_read() - reads queue info for a queue
+ *
+ * @param[in] dev_hndl: xdev device handle
+ * @param[in] id: queue handle
+ * @param[out] data: buffer pointer to collect the queue info
+ * @param[out] data_len: buffer len pointer
+ * @param[in] type: ring type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int qdbg_info_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type)
+{
+ int len = 0;
+ char *buf = NULL;
+ int buflen = DEBUGFS_QUEUE_INFO_SZ;
+ struct qdma_descq *descq = NULL;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ /** allocate memory */
+ buf = (char *) kzalloc(buflen, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ descq = qdma_device_get_descq_by_id(xdev, id, buf, buflen, 0);
+ if (!descq) {
+ kfree(buf);
+ return QDMA_ERR_INVALID_QIDX;
+ }
+
+ len = qdma_descq_dump_state(descq, buf + len, buflen - len);
+
+ *data = buf;
+ *data_len = buflen;
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * qdbg_desc_read() - reads descriptors of a queue
+ *
+ * @param[in] dev_hndl: xdev device handle
+ * @param[in] id: queue handle
+ * @param[out] data: buffer pointer to collect the queue descriptors
+ * @param[out] data_len: buffer len pointer
+ * @param[in] type: ring type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int qdbg_desc_read(unsigned long dev_hndl, unsigned long id, char **data,
+ int *data_len, enum dbgfs_desc_type type)
+{
+ int len = 0;
+ int rngsz = 0;
+ struct qdma_descq *descq = NULL;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ char *buf = NULL;
+ int buflen = 0;
+
+ descq = qdma_device_get_descq_by_id(xdev, id, buf, buflen, 0);
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ /** get the ring size */
+ if (type != DBGFS_DESC_TYPE_CMPT)
+ rngsz = descq->conf.rngsz;
+ else
+ rngsz = descq->conf.rngsz_cmpt;
+
+ /** 128 bytes is to accomodate header printed in the begining */
+ buflen = (rngsz * DEBUGFS_QUEUE_DESC_SZ) + 128;
+
+ /* allocate memory */
+ buf = (char *) kzalloc(buflen, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ if (type != DBGFS_DESC_TYPE_CMPT) {
+ len += qdma_queue_dump_desc(dev_hndl, id,
+ 0, rngsz-1, buf + len, buflen - len);
+ } else {
+ len += qdma_queue_dump_cmpt(dev_hndl, id,
+ 0, rngsz-1, buf + len, buflen - len);
+ }
+
+ *data = buf;
+ *data_len = buflen;
+
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * q_dbg_file_read() - static function that provides common read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ * @param[in] type: information type
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t q_dbg_file_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos, enum dbgfs_queue_info_type type)
+{
+ char *buf = NULL;
+ int buf_len = 0, len = 0, rv = 0;
+ struct dbgfs_q_priv *qpriv = (struct dbgfs_q_priv *)fp->private_data;
+
+ if (qpriv->data == NULL && qpriv->datalen == 0) {
+ if (type == DBGFS_QINFO_INFO) {
+ rv = qdbg_info_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_C2H);
+ } else if (type == DBGFS_QINFO_CNTXT) {
+ rv = qdbg_cntxt_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_C2H);
+ } else if (type == DBGFS_QINFO_DESC) {
+ rv = qdbg_desc_read(qpriv->dev_hndl, qpriv->qhndl,
+ &buf, &buf_len, DBGFS_DESC_TYPE_C2H);
+ }
+
+ if (rv < 0)
+ goto q_dbg_file_read_exit;
+
+ qpriv->datalen = rv;
+ qpriv->data = buf;
+ }
+
+ buf = qpriv->data;
+ len = qpriv->datalen;
+
+ if (!buf)
+ goto q_dbg_file_read_exit;
+
+ /** write to user buffer */
+ if (*ppos >= len) {
+ rv = 0;
+ goto q_dbg_file_read_exit;
+ }
+
+ if (*ppos + count > len)
+ count = len - *ppos;
+
+ if (copy_to_user(user_buffer, buf + *ppos, count)) {
+ rv = -EFAULT;
+ goto q_dbg_file_read_exit;
+ }
+
+ *ppos += count;
+ rv = count;
+
+ pr_debug("number of bytes written to user space is %zu\n", count);
+
+q_dbg_file_read_exit:
+ kfree(buf);
+ qpriv->data = NULL;
+ qpriv->datalen = 0;
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * q_info_open() - static function that executes info file open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int q_info_open(struct inode *inode, struct file *fp)
+{
+ return q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * q_info_read() - static function that executes info file read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t q_info_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return q_dbg_file_read(fp, user_buffer, count, ppos, DBGFS_QINFO_INFO);
+}
+
+/*****************************************************************************/
+/**
+ * q_cntxt_open() - static function that executes cntxt file open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int q_cntxt_open(struct inode *inode, struct file *fp)
+{
+ return q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * q_cntxt_read() - static function that performs cntxt file read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t q_cntxt_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return q_dbg_file_read(fp, user_buffer, count, ppos, DBGFS_QINFO_CNTXT);
+}
+
+/*****************************************************************************/
+/**
+ * q_desc_open() - static function that executes desc file open
+ *
+ * @param[in] inode: pointer to file inode
+ * @param[in] fp: pointer to file structure
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+static int q_desc_open(struct inode *inode, struct file *fp)
+{
+ return q_dbg_file_open(inode, fp);
+}
+
+/*****************************************************************************/
+/**
+ * q_desc_read() - static function that executes desc read
+ *
+ * @param[in] fp: pointer to file structure
+ * @param[out] user_buffer: pointer to user buffer
+ * @param[in] count: size of data to read
+ * @param[in/out] ppos: pointer to offset read
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+static ssize_t q_desc_read(struct file *fp, char __user *user_buffer,
+ size_t count, loff_t *ppos)
+{
+ return q_dbg_file_read(fp, user_buffer, count, ppos, DBGFS_QINFO_DESC);
+}
+
+/*****************************************************************************/
+/**
+ * create_q_dbg_files() - static function to create queue debug files
+ *
+ * @param[in] queue_root: debugfs root for a queue
+ *
+ * @return >0: size read
+ * @return <0: error
+ *****************************************************************************/
+int create_q_dbg_files(struct qdma_descq *descq, struct dentry *queue_root)
+{
+ struct dentry *fp[DBGFS_QINFO_END] = { NULL };
+ struct file_operations *fops = NULL;
+ int i = 0;
+
+ memset(qf, 0, sizeof(struct dbgfs_q_dbgf) * DBGFS_QINFO_END);
+
+ for (i = 0; i < DBGFS_QINFO_END; i++) {
+ fops = &qf[i].fops;
+ fops->owner = THIS_MODULE;
+ switch (i) {
+ case DBGFS_QINFO_INFO:
+ snprintf(qf[i].name, DBGFS_DBG_FNAME_SZ, "%s", "info");
+ fops->open = q_info_open;
+ fops->read = q_info_read;
+ fops->release = q_dbg_file_release;
+ break;
+ case DBGFS_QINFO_CNTXT:
+ snprintf(qf[i].name, DBGFS_DBG_FNAME_SZ, "%s", "cntxt");
+ fops->open = q_cntxt_open;
+ fops->read = q_cntxt_read;
+ fops->release = q_dbg_file_release;
+ break;
+ case DBGFS_QINFO_DESC:
+ snprintf(qf[i].name, DBGFS_DBG_FNAME_SZ, "%s", "desc");
+ fops->open = q_desc_open;
+ fops->read = q_desc_read;
+ fops->release = q_dbg_file_release;
+ break;
+ }
+ }
+
+ for (i = 0; i < DBGFS_QINFO_END; i++) {
+ fp[i] = debugfs_create_file(qf[i].name, 0644, queue_root,
+ descq, &qf[i].fops);
+ if (!fp[i])
+ return -1;
+ }
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_queue_init() - queue initialization function
+ *
+ * @param[in] conf: queue configuration
+ * @param[in] pair_conf: pair queue configuration
+ * @param[in] dbgfs_queues_root: root directory for all queues
+ *
+ * @return 0: success
+ * @return <0: error
+ *****************************************************************************/
+int dbgfs_queue_init(struct qdma_queue_conf *conf,
+ struct qdma_descq *pairq,
+ struct dentry *dbgfs_queues_root)
+{
+ char qname[16] = {0};
+ char qdir[8] = {0};
+ struct dentry *dbgfs_qidx_root = NULL;
+ struct dentry *dbgfs_queue_root = NULL;
+ struct dentry *dbgfs_cmpt_queue_root = NULL;
+ struct qdma_descq *descq = container_of(conf, struct qdma_descq, conf);
+ int rv = 0;
+
+ if (!descq)
+ return -EINVAL;
+ snprintf(qname, 16, "%u", conf->qidx);
+
+ spin_lock(&descq->xdev->qidx_lock);
+ /** create queue root only if it is not created */
+ if (pairq->dbgfs_qidx_root) {
+ dbgfs_qidx_root = pairq->dbgfs_qidx_root;
+ } else {
+ /* create a directory for the queue in debugfs */
+ dbgfs_qidx_root = debugfs_create_dir(qname,
+ dbgfs_queues_root);
+ if (!dbgfs_qidx_root) {
+ pr_err("Failed to create queue [%s] directory\n",
+ qname);
+ spin_unlock(&descq->xdev->qidx_lock);
+ return -1;
+ }
+ }
+
+ /* create a directory for direction */
+ if (conf->c2h)
+ snprintf(qdir, 8, "%s", "c2h");
+ else
+ snprintf(qdir, 8, "%s", "h2c");
+
+ dbgfs_queue_root = debugfs_create_dir(qdir,
+ dbgfs_qidx_root);
+ if (!dbgfs_queue_root) {
+ pr_err("Failed to create %s directory under %s",
+ qdir, qname);
+ goto dbgfs_queue_init_fail;
+ }
+
+ if (conf->c2h && conf->st) {
+ /* create a directory for the cmpt in debugfs */
+ dbgfs_cmpt_queue_root = debugfs_create_dir("cmpt",
+ dbgfs_qidx_root);
+ if (!dbgfs_cmpt_queue_root) {
+ pr_err("Failed to create cmpt directory under %s",
+ qname);
+ goto dbgfs_queue_init_fail;
+ }
+ }
+
+ /* intialize fops and create all the files */
+ rv = create_q_dbg_files(descq, dbgfs_queue_root);
+ if (rv < 0) {
+ pr_err("Failed to create qdbg files, removing %s dir\n",
+ qdir);
+ debugfs_remove_recursive(dbgfs_queue_root);
+ goto dbgfs_queue_init_fail;
+ }
+
+ if (dbgfs_cmpt_queue_root) {
+ rv = create_cmpt_q_dbg_files(descq, dbgfs_cmpt_queue_root);
+ if (rv < 0) {
+ pr_err("Failed to create cmptq dbg files,removing cmpt dir\n");
+ debugfs_remove_recursive(dbgfs_cmpt_queue_root);
+ goto dbgfs_queue_init_fail;
+ }
+ }
+
+ descq->dbgfs_qidx_root = dbgfs_qidx_root;
+ descq->dbgfs_queue_root = dbgfs_queue_root;
+ descq->dbgfs_cmpt_queue_root = dbgfs_cmpt_queue_root;
+ spin_unlock(&descq->xdev->qidx_lock);
+
+ return 0;
+
+dbgfs_queue_init_fail:
+ if (pairq->dbgfs_qidx_root) {
+ spin_unlock(&descq->xdev->qidx_lock);
+ return -1;
+ }
+ pr_err("Failed to init q debug files, removing [%s] dir\n", qname);
+ debugfs_remove_recursive(dbgfs_qidx_root);
+ spin_unlock(&descq->xdev->qidx_lock);
+ return -1;
+}
+
+/*****************************************************************************/
+/**
+ * dbgfs_queue_exit() - debugfs queue teardown function
+ *
+ * @param[in] conf: queue configuration
+ * @param[in] conf: pair queue configuration
+ *
+ *****************************************************************************/
+void dbgfs_queue_exit(struct qdma_queue_conf *conf,
+ struct qdma_descq *pairq)
+{
+ struct qdma_descq *descq = container_of(conf, struct qdma_descq, conf);
+
+ if (!descq)
+ return;
+ debugfs_remove_recursive(descq->dbgfs_queue_root);
+ debugfs_remove_recursive(descq->dbgfs_cmpt_queue_root);
+ descq->dbgfs_queue_root = NULL;
+ descq->dbgfs_cmpt_queue_root = NULL;
+ if (!pairq)
+ debugfs_remove_recursive(descq->dbgfs_qidx_root);
+
+ descq->dbgfs_qidx_root = NULL;
+}
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.h b/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.h
new file mode 100644
index 0000000000000000000000000000000000000000..1dac21d388fc9bdaee905d1a7ee7b624d8e3c1c8
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_debugfs_queue.h
@@ -0,0 +1,33 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_DEBUGFS_QUEUE_H__
+#define __QDMA_DEBUGFS_QUEUE_H__
+
+#include "qdma_debugfs.h"
+#include "qdma_descq.h"
+
+
+int dbgfs_queue_init(struct qdma_queue_conf *conf,
+ struct qdma_descq *pairq,
+ struct dentry *dbgfs_queues_root);
+void dbgfs_queue_exit(struct qdma_queue_conf *conf,
+ struct qdma_descq *pairq);
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_descq.c b/QDMA/linux-kernel/libqdma/qdma_descq.c
new file mode 100644
index 0000000000000000000000000000000000000000..bbe1704a5930dfaf8de55a8765865dc85be5f0ac
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_descq.c
@@ -0,0 +1,1602 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_descq.h"
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+
+#include "qdma_device.h"
+#include "qdma_intr.h"
+#include "qdma_regs.h"
+#include "qdma_thread.h"
+#include "qdma_context.h"
+#include "qdma_st_c2h.h"
+#include "thread.h"
+#include "version.h"
+#ifdef ERR_DEBUG
+#include "qdma_nl.h"
+#endif
+
+void intr_cidx_update(struct qdma_descq *descq, unsigned int sw_cidx,
+ int ring_index)
+{
+ unsigned int cidx = 0;
+
+ cidx |= V_INTR_CIDX_UPD_SW_CIDX(sw_cidx) |
+ V_INTR_CIDX_UPD_RING_INDEX(ring_index);
+
+ __write_reg(descq->xdev,
+ QDMA_REG_INT_CIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP,
+ cidx);
+}
+
+/*
+ * dma transfer requests
+ */
+#ifdef DEBUG
+static void sgl_dump(struct qdma_sw_sg *sgl, unsigned int sgcnt)
+{
+ struct qdma_sw_sg *sg = sgl;
+ int i;
+
+ pr_info("sgl 0x%p, sgcntt %u.\n", sgl, sgcnt);
+
+ for (i = 0; i < sgcnt; i++, sg++)
+ pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx.\n",
+ i, sg, sg->pg, sg->offset, sg->len, sg->dma_addr);
+}
+#endif
+
+static int sgl_find_offset(struct qdma_sw_sg *sgl, unsigned int sgcnt,
+ unsigned int offset, struct qdma_sw_sg **sg_p,
+ unsigned int *sg_offset)
+{
+ struct qdma_sw_sg *sg = sgl;
+ unsigned int len = 0;
+ int i;
+
+ for (i = 0; i < sgcnt; i++, sg++) {
+ len += sg->len;
+
+ if (len == offset) {
+ *sg_p = sg + 1;
+ *sg_offset = 0;
+ return ++i;
+ } else if (len > offset) {
+ *sg_p = sg;
+ *sg_offset = sg->len - (len - offset);
+ return i;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static inline void req_submitted(struct qdma_descq *descq,
+ struct qdma_sgt_req_cb *cb)
+{
+ cb->req_state = QDMA_REQ_SUBMITTED;
+ list_del(&cb->list);
+ list_add_tail(&cb->list, &descq->pend_list);
+}
+
+static int descq_mm_n_h2c_cmpl_status(struct qdma_descq *descq);
+
+static int descq_poll_mm_n_h2c_cmpl_status(struct qdma_descq *descq)
+{
+ enum qdma_drv_mode drv_mode = descq->xdev->conf.qdma_drv_mode;
+
+ if ((drv_mode == POLL_MODE) || (drv_mode == AUTO_MODE)) {
+ descq->proc_req_running = 1;
+ return descq_mm_n_h2c_cmpl_status(descq);
+ } else
+ return 0;
+}
+
+static ssize_t descq_mm_proc_request(struct qdma_descq *descq)
+{
+ int c2h = descq->conf.c2h;
+ unsigned int desc_written = 0;
+ unsigned int rngsz = descq->conf.rngsz;
+ unsigned int pidx;
+ struct qdma_mm_desc *desc;
+
+ lock_descq(descq);
+ /* process completion of submitted requests */
+ if (descq->q_stop_wait) {
+ descq_mm_n_h2c_cmpl_status(descq);
+ unlock_descq(descq);
+ return 0;
+ }
+ if (unlikely(descq->q_state != Q_STATE_ONLINE)) {
+ unlock_descq(descq);
+ return 0;
+ }
+
+ if (descq->proc_req_running) {
+ unlock_descq(descq);
+ return 0;
+ }
+
+ pidx = descq->pidx;
+ desc = (struct qdma_mm_desc *)descq->desc + pidx;
+
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+
+ while (!list_empty(&descq->work_list)) {
+ struct qdma_sgt_req_cb *cb = list_first_entry(&descq->work_list,
+ struct qdma_sgt_req_cb, list);
+ struct qdma_request *req = (struct qdma_request *)cb;
+ struct qdma_sw_sg *sg = req->sgl;
+ unsigned int sg_offset = 0;
+ unsigned int sg_max = req->sgcnt;
+ u64 ep_addr = req->ep_addr + cb->offset;
+ struct qdma_mm_desc *desc_start = NULL;
+ struct qdma_mm_desc *desc_end = NULL;
+ unsigned int desc_max = descq->avail;
+ unsigned int data_cnt = 0;
+ unsigned int desc_cnt = 0;
+ unsigned int len = 0;
+ int i = 0;
+
+ if (!desc_max) {
+ descq->pidx = pidx;
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+ desc_max = descq->avail;
+ }
+
+ if (!desc_max)
+ break;
+
+ if (cb->offset) {
+ int rv = sgl_find_offset(req->sgl, req->sgcnt,
+ cb->offset, &sg, &sg_offset);
+ if (rv < 0 || rv >= sg_max) {
+ pr_info("descq %s, req 0x%p, OOR %u/%u, %d/%u.\n",
+ descq->conf.name, req, cb->offset,
+ req->count, rv, sg_max);
+ req_submitted(descq, cb);
+ continue;
+ }
+ i = rv;
+ pr_debug("%s, req 0x%p, offset %u/%u -> sg %d, 0x%p,%u.\n",
+ descq->conf.name, req, cb->offset, req->count,
+ i, sg, sg_offset);
+ } else
+ i = 0;
+
+ desc_start = desc;
+ for (; i < sg_max && desc_cnt < desc_max; i++, sg++) {
+ unsigned int tlen = sg->len;
+ dma_addr_t addr = sg->dma_addr;
+ unsigned int pg_off = sg->offset;
+
+ pr_debug("desc %u/%u, sgl %d, len %u,%u, offset %u.\n",
+ desc_cnt, desc_max, i, len, tlen, sg_offset);
+
+ if (sg_offset) {
+ tlen -= sg_offset;
+ addr += sg_offset;
+ pg_off += sg_offset;
+ sg_offset = 0;
+ }
+
+ while (tlen) {
+ unsigned int len = min_t(unsigned int, tlen,
+ QDMA_DESC_BLEN_MAX);
+ desc_end = desc;
+
+ desc->rsvd1 = 0UL;
+ desc->rsvd0 = 0U;
+
+ if (descq->conf.c2h) {
+ desc->src_addr = ep_addr;
+ desc->dst_addr = addr;
+ } else {
+ desc->dst_addr = ep_addr;
+ desc->src_addr = addr;
+ }
+
+ desc->flag_len = len;
+ desc->flag_len |= (1 << S_DESC_F_DV);
+
+ ep_addr += len;
+ data_cnt += len;
+ addr += len;
+ tlen -= len;
+ pg_off += len;
+
+ if (++pidx == rngsz) {
+ pidx = 0;
+ desc =
+ (struct qdma_mm_desc *)descq->desc;
+ } else {
+ desc++;
+ }
+
+ desc_cnt++;
+ if (desc_cnt == desc_max)
+ break;
+ }
+ }
+
+ if (!desc_end || !desc_start) {
+ pr_info("descq %s, %u, pidx 0x%x, desc 0x%p ~ 0x%p.\n",
+ descq->conf.name, descq->qidx_hw, pidx,
+ desc_start, desc_end);
+ break;
+ }
+
+ /* set eop */
+ desc_end->flag_len |= (1 << S_DESC_F_EOP);
+ /* set sop */
+ desc_start->flag_len |= (1 << S_DESC_F_SOP);
+
+ cb->desc_nr += desc_cnt;
+ cb->offset += data_cnt;
+
+ desc_written += desc_cnt;
+
+ pr_debug("descq %s, +%u,%u, avail %u, ep_addr 0x%llx + 0x%x(%u).\n",
+ descq->conf.name, desc_cnt, pidx, descq->avail,
+ req->ep_addr, data_cnt, data_cnt);
+
+ descq->avail -= desc_cnt;
+ descq->pend_req_desc -= desc_cnt;
+ if (cb->offset == req->count)
+ req_submitted(descq, cb);
+ else
+ cb->req_state = QDMA_REQ_SUBMIT_PARTIAL;
+ }
+
+ if (desc_written) {
+ descq->pend_list_empty = 0;
+ if (c2h)
+ descq_c2h_pidx_update(descq, pidx);
+ else
+ descq_h2c_pidx_update(descq, pidx);
+
+ descq->pidx = pidx;
+
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+ }
+
+ descq->proc_req_running = 0;
+ unlock_descq(descq);
+
+ if (desc_written && descq->cmplthp)
+ qdma_kthread_wakeup(descq->cmplthp);
+
+ return 0;
+}
+
+static ssize_t descq_proc_st_h2c_request(struct qdma_descq *descq)
+{
+ struct qdma_h2c_desc *desc;
+ unsigned int rngsz = descq->conf.rngsz;
+ unsigned int pidx;
+ unsigned int desc_written = 0;
+
+ lock_descq(descq);
+ /* process completion of submitted requests */
+ if (descq->q_stop_wait) {
+ descq_mm_n_h2c_cmpl_status(descq);
+ unlock_descq(descq);
+ return 0;
+ }
+ if (unlikely(descq->q_state != Q_STATE_ONLINE)) {
+ unlock_descq(descq);
+ return 0;
+ }
+
+ if (descq->proc_req_running) {
+ unlock_descq(descq);
+ return 0;
+ }
+
+ /* service completion first */
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+
+ pidx = descq->pidx;
+ desc = (struct qdma_h2c_desc *)descq->desc + pidx;
+ while (!list_empty(&descq->work_list)) {
+ struct qdma_sgt_req_cb *cb = list_first_entry(&descq->work_list,
+ struct qdma_sgt_req_cb, list);
+ struct qdma_request *req = (struct qdma_request *)cb;
+ struct qdma_sw_sg *sg = req->sgl;
+ unsigned int sg_offset = 0;
+ unsigned int sg_max = req->sgcnt;
+ unsigned int desc_max = descq->avail;
+ unsigned int data_cnt = 0;
+ unsigned int desc_cnt = 0;
+ unsigned int pktsz = req->ep_addr ?
+ min_t(unsigned int, req->ep_addr, PAGE_SIZE) :
+ PAGE_SIZE;
+ int i = 0;
+
+ if (!desc_max) {
+ descq->pidx = pidx;
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+ desc_max = descq->avail;
+ }
+
+ if (!desc_max)
+ break;
+
+#ifdef DEBUG
+ pr_info("%s, req %u.\n", descq->conf.name, req->count);
+ sgl_dump(req->sgl, sg_max);
+#endif
+
+ if (cb->offset) {
+ int rv = sgl_find_offset(req->sgl, sg_max, cb->offset,
+ &sg, &sg_offset);
+
+ if (rv < 0 || rv >= sg_max) {
+ pr_info("descq %s, req 0x%p, OOR %u/%u, %d/%u.\n",
+ descq->conf.name, req, cb->offset,
+ req->count, rv, sg_max);
+ req_submitted(descq, cb);
+ continue;
+ }
+ i = rv;
+ pr_debug("%s, req 0x%p, offset %u/%u -> sg %d, 0x%p,%u.\n",
+ descq->conf.name, req, cb->offset, req->count,
+ i, sg, sg_offset);
+ } else {
+ i = 0;
+ desc->flags |= S_H2C_DESC_F_SOP;
+ }
+
+ for (; i < sg_max && desc_cnt < desc_max; i++, sg++) {
+ unsigned int tlen = sg->len;
+ dma_addr_t addr = sg->dma_addr;
+
+ if (sg_offset) {
+ tlen -= sg_offset;
+ addr += sg_offset;
+ sg_offset = 0;
+ }
+
+ do { /* to support zero byte transfer */
+ unsigned int len = min_t(unsigned int, tlen,
+ pktsz);
+
+ desc->src_addr = addr;
+ desc->len = len;
+ desc->pld_len = len;
+ desc->cdh_flags |= S_H2C_DESC_F_ZERO_CDH;
+
+ if (descq->xdev->stm_en) {
+ desc->pld_len = len;
+ desc->cdh_flags =
+ (1 << S_H2C_DESC_F_ZERO_CDH);
+ desc->cdh_flags |= V_H2C_DESC_NUM_GL(1);
+
+ /* if we are about to exhaust ring,
+ * request h2c-wb
+ */
+ if (desc_cnt == (desc_max - 1))
+ desc->cdh_flags |=
+ (1 <<
+ S_H2C_DESC_F_REQ_CMPL_STATUS);
+ } else if (unlikely
+ (descq->xdev->conf.tm_mode_en)) {
+ /** Check if we are running in TM mode to test
+ * sending TMH and CDH with Traffic Manager
+ * example design for Streaming H2C queue in
+ * bypass mode
+ */
+
+ /** In TM mode, check if we are to run
+ * with 1 CDH or Zero CDH
+ */
+ if (descq->xdev->conf.tm_one_cdh_en) {
+ /** In 1 CDH case, this desc
+ * carries CDH - comprising of
+ * 4B TMH (pld_len and
+ * cdh_flags fields) and 12B CDH.
+ *
+ * Next desc carries the GL. So,
+ * we advance to next desc.
+ *
+ * Max supported payload with GL
+ * is 4K.
+ */
+ desc->pld_len = len;
+ desc->cdh_flags =
+ (V_H2C_DESC_NUM_CDH(1) |
+ V_H2C_DESC_NUM_GL(1) |
+ (1 << S_H2C_DESC_F_REQ_CMPL_STATUS) |
+ (1 << S_H2C_DESC_F_EOT));
+ desc->flags = 0;
+ desc->len = 0;
+ desc->src_addr = 0;
+
+ if (++pidx == descq->conf.rngsz) {
+ pidx = 0;
+ desc =
+ (struct qdma_h2c_desc *)descq->desc;
+ } else {
+ desc++;
+ }
+ desc_cnt++;
+ /* below due to one CDH desc */
+ descq->pend_req_desc += 1;
+ desc->pld_len = 0;
+ desc->cdh_flags = 0;
+ desc->src_addr = addr;
+ desc->len = len;
+ } else {
+ /** In zero CDH case, this desc
+ * holds the 4B TMH (pld_len and
+ * cdh_flags fields).
+ * And the desc carries the GL too.
+ *
+ * Max supported payload with GL
+ * is 4K.
+ */
+ desc->pld_len = len;
+ desc->cdh_flags =
+ (1 << S_H2C_DESC_F_ZERO_CDH);
+ desc->cdh_flags |=
+ V_H2C_DESC_NUM_GL(1);
+ desc->cdh_flags |=
+ ((1 <<
+ S_H2C_DESC_F_REQ_CMPL_STATUS) |
+ (1 << S_H2C_DESC_F_EOT));
+ }
+ }
+
+#ifdef ER_DEBUG
+ if (descq->induce_err & (1 << len_mismatch)) {
+ desc->len = 0xFFFFFFFF;
+ pr_info("inducing %d err",
+ len_mismatch);
+ }
+#endif
+ data_cnt += len;
+ addr += len;
+ tlen -= len;
+
+ if ((i == sg_max - 1)) {
+ desc->flags |= S_H2C_DESC_F_EOP;
+
+ if (descq->xdev->stm_en)
+ desc->cdh_flags |=
+ (req->h2c_eot <<
+ S_H2C_DESC_F_EOT) |
+ (1 <<
+ S_H2C_DESC_F_REQ_CMPL_STATUS);
+ }
+
+#if 0
+ pr_info("desc %d, pidx 0x%x, data_cnt %u, cb off %u:\n",
+ i, pidx, data_cnt, cb->offset);
+ print_hex_dump(KERN_INFO, "desc",
+ DUMP_PREFIX_OFFSET, 16, 1,
+ (void *)desc, 16, false);
+#endif
+
+ if (++pidx == rngsz) {
+ pidx = 0;
+ desc =
+ (struct qdma_h2c_desc *)descq->desc;
+ } else {
+ desc++;
+ }
+
+ desc_cnt++;
+ if (desc_cnt == desc_max)
+ break;
+ } while (tlen);
+
+ }
+
+ cb->desc_nr += desc_cnt;
+ cb->offset += data_cnt;
+
+ desc_written += desc_cnt;
+
+ pr_debug("descq %s, +%u,%u, avail %u, 0x%x(%u), cb off %u.\n",
+ descq->conf.name, desc_cnt, pidx, descq->avail,
+ data_cnt, data_cnt, cb->offset);
+
+ descq->avail -= desc_cnt;
+ descq->pend_req_desc -= desc_cnt;
+
+ if (cb->offset == req->count)
+ req_submitted(descq, cb);
+ else
+ cb->req_state = QDMA_REQ_SUBMIT_PARTIAL;
+ }
+
+ if (desc_written) {
+ descq->pend_list_empty = 0;
+ descq_h2c_pidx_update(descq, pidx);
+ descq->pidx = pidx;
+
+ descq_poll_mm_n_h2c_cmpl_status(descq);
+
+ }
+
+ descq->proc_req_running = 0;
+
+ if (descq->cmplthp)
+ qdma_kthread_wakeup(descq->cmplthp);
+
+ unlock_descq(descq);
+
+ return 0;
+}
+
+/*
+ * descriptor Queue
+ */
+static inline int get_desc_size(struct qdma_descq *descq)
+{
+ if (descq->conf.desc_bypass && (descq->conf.sw_desc_sz == DESC_SZ_64B))
+ return DESC_SZ_64B_BYTES;
+ if (!descq->conf.st)
+ return (int)sizeof(struct qdma_mm_desc);
+
+ if (descq->conf.c2h)
+ return (int)sizeof(struct qdma_c2h_desc);
+
+ return (int)sizeof(struct qdma_h2c_desc);
+}
+
+static inline int get_desc_cmpl_status_size(struct qdma_descq *descq)
+{
+ return (int)sizeof(struct qdma_desc_cmpl_status);
+}
+
+static inline void desc_ring_free(struct xlnx_dma_dev *xdev, int ring_sz,
+ int desc_sz, int cs_sz, u8 *desc, dma_addr_t desc_bus)
+{
+ unsigned int len = ring_sz * desc_sz + cs_sz;
+
+ pr_debug("free %u(0x%x)=%d*%u+%d, 0x%p, bus 0x%llx.\n",
+ len, len, desc_sz, ring_sz, cs_sz, desc, desc_bus);
+
+ dma_free_coherent(&xdev->conf.pdev->dev, ring_sz * desc_sz + cs_sz,
+ desc, desc_bus);
+}
+
+static void *desc_ring_alloc(struct xlnx_dma_dev *xdev, int ring_sz,
+ int desc_sz, int cs_sz, dma_addr_t *bus, u8 **cs_pp)
+{
+ unsigned int len = ring_sz * desc_sz + cs_sz;
+ u8 *p = dma_alloc_coherent(&xdev->conf.pdev->dev, len, bus, GFP_KERNEL);
+
+ if (!p) {
+ pr_info("%s, OOM, sz ring %d, desc %d, cmpl status sz %d.\n",
+ xdev->conf.name, ring_sz, desc_sz, cs_sz);
+ return NULL;
+ }
+
+ *cs_pp = p + ring_sz * desc_sz;
+ memset(p, 0, len);
+
+ pr_debug("alloc %u(0x%x)=%d*%u+%d, 0x%p, bus 0x%llx, cmpl status 0x%p.\n",
+ len, len, desc_sz, ring_sz, cs_sz, p, *bus, *cs_pp);
+
+ return p;
+}
+
+static void desc_free_irq(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ unsigned long flags;
+
+ if (!xdev->num_vecs)
+ return;
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ if (xdev->dev_intr_info_list[descq->intr_id].intr_list_cnt)
+ xdev->dev_intr_info_list[descq->intr_id].intr_list_cnt--;
+ spin_unlock_irqrestore(&xdev->lock, flags);
+}
+
+static void desc_alloc_irq(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ unsigned long flags;
+ int i, idx = 0, min = 0;
+
+ if (!xdev->num_vecs)
+ return;
+
+ /** Pick the MSI-X vector that currently has the fewest queues
+ * on PF0, vector#0 is dedicated for Error interrupts and
+ * vector #1 is dedicated for User interrupts
+ * For all other PFs, vector#0 is dedicated for User interrupts
+ */
+ min = xdev->dev_intr_info_list[xdev->dvec_start_idx].intr_list_cnt;
+ idx = xdev->dvec_start_idx;
+ if (xdev->conf.qdma_drv_mode == DIRECT_INTR_MODE) {
+ spin_lock_irqsave(&xdev->lock, flags);
+ for (i = xdev->dvec_start_idx; i < xdev->num_vecs; i++) {
+ if (xdev->dev_intr_info_list[i].intr_list_cnt < min) {
+ min = xdev->dev_intr_info_list[i].intr_list_cnt;
+ idx = i;
+ }
+
+ if (!min)
+ break;
+ }
+ xdev->dev_intr_info_list[idx].intr_list_cnt++;
+ spin_unlock_irqrestore(&xdev->lock, flags);
+ }
+ descq->intr_id = idx;
+ pr_debug("descq->intr_id = %d allocated to qidx = %d\n",
+ descq->intr_id, descq->conf.qidx);
+}
+
+/*
+ * writeback handling
+ */
+static int descq_mm_n_h2c_cmpl_status(struct qdma_descq *descq)
+{
+ unsigned int cidx, cidx_hw;
+ unsigned int cr;
+
+ pr_debug("descq 0x%p, %s, pidx %u, cidx %u.\n",
+ descq, descq->conf.name, descq->pidx, descq->cidx);
+
+ if (descq->pidx == descq->cidx) { /* queue empty? */
+ pr_debug("descq %s empty, return.\n", descq->conf.name);
+ return 0;
+ }
+
+ cidx = descq->cidx;
+#ifdef __READ_ONCE_DEFINED__
+ cidx_hw = READ_ONCE(((struct qdma_desc_cmpl_status *)
+ descq->desc_cmpl_status)->cidx);
+#else
+ cidx_hw = ((struct qdma_desc_cmpl_status *)
+ descq->desc_cmpl_status)->cidx;
+#endif
+
+ if (cidx_hw == cidx) /* no new writeback? */
+ return 0;
+
+ /* completion credits */
+ cr = (cidx_hw < cidx) ? (descq->conf.rngsz - cidx) + cidx_hw :
+ cidx_hw - cidx;
+
+ pr_debug("descq %s, cidx 0x%x -> 0x%x, avail 0x%x + 0x%x.\n",
+ descq->conf.name, cidx, cidx_hw, descq->avail, cr);
+
+ descq->cidx = cidx_hw;
+ descq->avail += cr;
+ descq->credit += cr;
+
+ incr_cmpl_desc_cnt(descq, cr);
+
+ /* completes requests */
+ pr_debug("%s, 0x%p, credit %u + %u.\n",
+ descq->conf.name, descq, cr, descq->credit);
+
+ cr = descq->credit;
+
+ while (!list_empty(&descq->pend_list)) {
+ struct qdma_sgt_req_cb *cb = list_first_entry(&descq->pend_list,
+ struct qdma_sgt_req_cb, list);
+
+ pr_debug("%s, 0x%p, cb 0x%p, desc_nr %u, credit %u.\n",
+ descq->conf.name, descq, cb, cb->desc_nr, cr);
+
+ if (cr >= cb->desc_nr) {
+ pr_debug("%s, cb 0x%p done, credit %u > %u.\n",
+ descq->conf.name, cb, cr, cb->desc_nr);
+ cr -= cb->desc_nr;
+ qdma_sgt_req_done(descq, cb, 0);
+ } else {
+ pr_debug("%s, cb 0x%p not done, credit %u < %u.\n",
+ descq->conf.name, cb, cr, cb->desc_nr);
+ cb->desc_nr -= cr;
+ cr = 0;
+ }
+
+ if (!cr)
+ break;
+ }
+
+ descq->credit = cr;
+ pr_debug("%s, 0x%p, credit %u.\n",
+ descq->conf.name, descq, descq->credit);
+
+ if (descq->conf.c2h)
+ descq_c2h_pidx_update(descq, descq->pidx);
+ else
+ descq_h2c_pidx_update(descq, descq->pidx);
+
+ /* Request thread may have only setup a fraction of the transfer (e.g.
+ * there wasn't enough space in desc ring). We now have more space
+ * available again so we can continue programming the
+ * dma transfer by resuming the thread here.
+ */
+
+
+ return 1;
+}
+
+/* ************** public function definitions ******************************* */
+
+void qdma_descq_init(struct qdma_descq *descq, struct xlnx_dma_dev *xdev,
+ int idx_hw, int idx_sw)
+{
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+
+ memset(descq, 0, sizeof(struct qdma_descq));
+
+ spin_lock_init(&descq->lock);
+ INIT_LIST_HEAD(&descq->work_list);
+ INIT_LIST_HEAD(&descq->pend_list);
+ qdma_waitq_init(&descq->pend_list_wq);
+ INIT_LIST_HEAD(&descq->intr_list);
+ INIT_LIST_HEAD(&descq->legacy_intr_q_list);
+ INIT_WORK(&descq->work, intr_work);
+ descq->xdev = xdev;
+ descq->channel = 0;
+ descq->qidx_hw = qdev->qbase + idx_hw;
+ descq->conf.qidx = idx_sw;
+}
+
+void qdma_descq_cleanup(struct qdma_descq *descq)
+{
+ lock_descq(descq);
+
+ if (descq->q_state == Q_STATE_ONLINE) {
+ descq->q_state = Q_STATE_ENABLED;
+ qdma_descq_context_clear(descq->xdev, descq->qidx_hw,
+ descq->conf.st, descq->conf.c2h, 0);
+ } else
+ goto end;
+ desc_free_irq(descq);
+
+ qdma_descq_free_resource(descq);
+
+end:
+ unlock_descq(descq);
+}
+
+int qdma_descq_alloc_resource(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ int rv;
+#ifdef TEST_64B_DESC_BYPASS_FEATURE
+ int i = 0;
+ u8 *desc_bypass;
+ u8 bypass_data[DESC_SZ_64B_BYTES];
+#endif
+ /* descriptor ring */
+ descq->desc = desc_ring_alloc(xdev, descq->conf.rngsz,
+ get_desc_size(descq),
+ get_desc_cmpl_status_size(descq),
+ &descq->desc_bus, &descq->desc_cmpl_status);
+ if (!descq->desc) {
+ pr_info("dev %s, descq %s, sz %u, desc ring OOM.\n",
+ xdev->conf.name, descq->conf.name, descq->conf.rngsz);
+ goto err_out;
+ }
+
+
+ if (descq->conf.st && descq->conf.c2h) {
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+
+ descq->color = 1;
+ flq->desc = (struct qdma_c2h_desc *)descq->desc;
+ flq->size = descq->conf.rngsz;
+ flq->pg_shift = fls(descq->conf.c2h_bufsz) - 1;
+
+ /* These code changes are to accomodate buf_sz
+ * of less than 4096
+ */
+ if (flq->pg_shift < PAGE_SHIFT) {
+ flq->pg_shift = PAGE_SHIFT;
+ flq->pg_order = 0;
+ } else
+ flq->pg_order = flq->pg_shift - PAGE_SHIFT;
+
+ /* writeback ring */
+ descq->desc_cmpt = desc_ring_alloc(xdev,
+ descq->conf.rngsz_cmpt,
+ descq->cmpt_entry_len,
+ sizeof(struct
+ qdma_c2h_cmpt_cmpl_status),
+ &descq->desc_cmpt_bus,
+ &descq->desc_cmpt_cmpl_status);
+ if (!descq->desc_cmpt) {
+ pr_warn("dev %s, descq %s, sz %u, cmpt ring OOM.\n",
+ xdev->conf.name, descq->conf.name,
+ descq->conf.rngsz_cmpt);
+ goto err_out;
+ }
+ descq->desc_cmpt_cur = descq->desc_cmpt;
+
+ /* freelist / rx buffers */
+ rv = descq_flq_alloc_resource(descq);
+ if (rv < 0)
+ goto err_out;
+ }
+
+ pr_debug("%s: %u/%u, rng %u,%u, desc 0x%p, cmpl status 0x%p.\n",
+ descq->conf.name, descq->conf.qidx, descq->qidx_hw,
+ descq->conf.rngsz, descq->conf.rngsz_cmpt, descq->desc,
+ descq->desc_cmpt);
+
+ /* interrupt vectors */
+ desc_alloc_irq(descq);
+
+ /* Fill in the descriptors with some hard coded value for testing */
+#ifdef TEST_64B_DESC_BYPASS_FEATURE
+ desc_bypass = descq->desc;
+ if (descq->conf.st && !(descq->conf.c2h)) {
+ if (descq->conf.desc_bypass &&
+ (descq->conf.sw_desc_sz == DESC_SZ_64B)) {
+ for (i = 0; i < descq->conf.rngsz; i++) {
+ memset(bypass_data, i+1, DESC_SZ_64B_BYTES);
+ memcpy(&desc_bypass[i*DESC_SZ_64B_BYTES],
+ bypass_data, DESC_SZ_64B_BYTES);
+ }
+ }
+ }
+#endif
+ return 0;
+
+err_out:
+ qdma_descq_free_resource(descq);
+ return QDMA_ERR_OUT_OF_MEMORY;
+}
+
+void qdma_descq_free_resource(struct qdma_descq *descq)
+{
+ if (!descq)
+ return;
+
+ if (descq->desc) {
+
+ int desc_sz = get_desc_size(descq);
+ int cs_sz = get_desc_cmpl_status_size(descq);
+
+ pr_debug("%s: desc 0x%p, cmpt 0x%p.\n",
+ descq->conf.name, descq->desc, descq->desc_cmpt);
+
+ desc_ring_free(descq->xdev, descq->conf.rngsz, desc_sz, cs_sz,
+ descq->desc, descq->desc_bus);
+
+ descq->desc_cmpl_status = NULL;
+ descq->desc = NULL;
+ descq->desc_bus = 0UL;
+ }
+
+ if (descq->desc_cmpt) {
+ descq_flq_free_resource(descq);
+ desc_ring_free(descq->xdev, descq->conf.rngsz_cmpt,
+ descq->cmpt_entry_len,
+ sizeof(struct qdma_c2h_cmpt_cmpl_status),
+ descq->desc_cmpt, descq->desc_cmpt_bus);
+
+ descq->desc_cmpt_cmpl_status = NULL;
+ descq->desc_cmpt = NULL;
+ descq->desc_cmpt_bus = 0UL;
+ }
+}
+
+void qdma_descq_config(struct qdma_descq *descq, struct qdma_queue_conf *qconf,
+ int reconfig)
+{
+ if (!reconfig) {
+ int len;
+
+ memcpy(&descq->conf, qconf, sizeof(struct qdma_queue_conf));
+ /* descq->conf.st = qconf->st;
+ * descq->conf.c2h = qconf->c2h;
+ */
+
+ /* qdma[vf]<255>-MM/ST-H2C/C2H-Q[2048] */
+#ifdef __QDMA_VF__
+ len = sprintf(descq->conf.name, "qdmavf");
+#else
+ len = sprintf(descq->conf.name, "qdma");
+#endif
+ len += sprintf(descq->conf.name + len, "%05x-%s-%u",
+ descq->xdev->conf.bdf, descq->conf.st ? "ST" : "MM",
+ descq->conf.qidx);
+ descq->conf.name[len] = '\0';
+
+ descq->conf.st = qconf->st;
+ descq->conf.c2h = qconf->c2h;
+
+ } else {
+ descq->conf.desc_rng_sz_idx = qconf->desc_rng_sz_idx;
+ descq->conf.cmpl_rng_sz_idx = qconf->cmpl_rng_sz_idx;
+ descq->conf.c2h_buf_sz_idx = qconf->c2h_buf_sz_idx;
+
+ descq->conf.irq_en = (descq->xdev->conf.qdma_drv_mode !=
+ POLL_MODE) ? 1 : 0;
+ descq->conf.cmpl_status_en = qconf->cmpl_status_en;
+ descq->conf.cmpl_status_acc_en = qconf->cmpl_status_acc_en;
+ descq->conf.cmpl_status_pend_chk = qconf->cmpl_status_pend_chk;
+ descq->conf.cmpl_stat_en = qconf->cmpl_stat_en;
+ descq->conf.cmpl_trig_mode = qconf->cmpl_trig_mode;
+ descq->conf.cmpl_timer_idx = qconf->cmpl_timer_idx;
+ descq->conf.fetch_credit = qconf->fetch_credit;
+ descq->conf.cmpl_cnt_th_idx = qconf->cmpl_cnt_th_idx;
+
+ descq->conf.desc_bypass = qconf->desc_bypass;
+ descq->conf.pfetch_bypass = qconf->pfetch_bypass;
+ descq->conf.pfetch_en = qconf->pfetch_en;
+ descq->conf.cmpl_udd_en = qconf->cmpl_udd_en;
+ descq->conf.cmpl_desc_sz = qconf->cmpl_desc_sz;
+ descq->conf.sw_desc_sz = qconf->sw_desc_sz;
+ descq->conf.pipe_gl_max = qconf->pipe_gl_max;
+ descq->conf.pipe_flow_id = qconf->pipe_flow_id;
+ descq->conf.pipe_slr_id = qconf->pipe_slr_id;
+ descq->conf.pipe_tdest = qconf->pipe_tdest;
+ descq->conf.cmpl_ovf_chk_dis = qconf->cmpl_ovf_chk_dis;
+ }
+}
+
+int qdma_descq_config_complete(struct qdma_descq *descq)
+{
+ struct qdma_csr_info csr_info;
+ struct qdma_queue_conf *qconf = &descq->conf;
+ int rv;
+
+ memset(&csr_info, 0, sizeof(csr_info));
+ csr_info.type = QDMA_CSR_TYPE_RNGSZ;
+ csr_info.idx_rngsz = qconf->desc_rng_sz_idx;
+ csr_info.idx_bufsz = qconf->c2h_buf_sz_idx;
+ csr_info.idx_timer_cnt = qconf->cmpl_timer_idx;
+ csr_info.idx_cnt_th = qconf->cmpl_cnt_th_idx;
+
+ rv = qdma_csr_read(descq->xdev, &csr_info, QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0)
+ return rv;
+
+ qconf->rngsz = csr_info.array[qconf->desc_rng_sz_idx] - 1;
+
+ /* <= 2018.2 IP
+ * make the cmpl ring size bigger if possible to avoid run out of
+ * cmpl entry while desc. ring still have free entries
+ */
+ if (qconf->st && qconf->c2h) {
+ int i;
+ unsigned int v = csr_info.array[qconf->cmpl_rng_sz_idx];
+ int best_fit_idx = -1;
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++) {
+ if (csr_info.array[i] > v) {
+ if (best_fit_idx < 0)
+ best_fit_idx = i;
+ else if ((best_fit_idx >= 0) &&
+ (csr_info.array[i] <
+ csr_info.array[best_fit_idx]))
+ best_fit_idx = i;
+ }
+ }
+
+ if (best_fit_idx >= 0)
+ qconf->cmpl_rng_sz_idx = best_fit_idx;
+
+ qconf->rngsz_cmpt = csr_info.array[qconf->cmpl_rng_sz_idx] - 1;
+ qconf->c2h_bufsz = csr_info.bufsz;
+ }
+
+ /* we can never use the full ring because then cidx would equal pidx
+ * and thus the ring would be interpreted as empty. Thus max number of
+ * usable entries is ring_size - 1
+ */
+ descq->avail = descq->conf.rngsz - 1;
+ descq->pend_list_empty = 1;
+
+ descq->pidx = 0;
+ descq->cidx = 0;
+ descq->cidx_cmpt = 0;
+ descq->pidx_cmpt = 0;
+ descq->credit = 0;
+ descq->io_batch_cnt = (descq->conf.st && descq->conf.c2h) ?
+ (1 << descq->conf.cmpl_cnt_th_idx) :
+ (1 << (csr_info.cmpl_status_acc + 3)); /* tuned value */
+ descq->pend_req_desc = 0;
+
+ /* ST C2H only */
+ if (qconf->c2h && qconf->st) {
+ descq->cmpt_entry_len = 8 << qconf->cmpl_desc_sz;
+
+ pr_debug("%s: cmpl sz %u(%d), udd_en %d.\n",
+ descq->conf.name, descq->cmpt_entry_len,
+ descq->conf.cmpl_desc_sz, descq->conf.cmpl_udd_en);
+ }
+
+ if (qconf->fp_descq_isr_top)
+ descq->xdev->conf.isr_top_q_en = 1;
+
+ return 0;
+}
+
+int qdma_descq_prog_hw(struct qdma_descq *descq)
+{
+ int rv = qdma_descq_context_setup(descq);
+
+ if (rv < 0) {
+ pr_warn("%s failed to program contexts", descq->conf.name);
+ return rv;
+ }
+
+ /* update pidx/cidx */
+ if (descq->conf.st && descq->conf.c2h) {
+ if (!descq->xdev->stm_en) {
+ /* For STM enabled device, these updates need to be
+ * delayed until STM context is setup for the queue
+ */
+ descq_cmpt_cidx_update(descq, 0);
+ descq_c2h_pidx_update(descq, descq->conf.rngsz - 1);
+ }
+ }
+
+ return rv;
+}
+
+#ifndef __QDMA_VF__
+int qdma_descq_prog_stm(struct qdma_descq *descq, bool clear)
+{
+ int rv;
+
+ if (!descq->conf.st) {
+ pr_err("%s: STM programming called for MM-mode\n",
+ descq->conf.name);
+ return -EINVAL;
+ }
+
+ if (descq->qidx_hw > STM_MAX_SUPPORTED_QID) {
+ pr_err("%s: QID for STM cannot be > %d\n",
+ descq->conf.name, STM_MAX_SUPPORTED_QID);
+ return -EINVAL;
+ }
+
+ if (descq->xdev->stm_rev != STM_SUPPORTED_REV) {
+ pr_err("%s: No supported STM rev found in hw\n",
+ descq->conf.name);
+ return -ENODEV;
+ }
+
+ if (!descq->conf.c2h && !descq->conf.desc_bypass) {
+ pr_err("%s: H2C queue needs to be in bypass with STM\n",
+ descq->conf.name);
+ return -EINVAL;
+ }
+
+ if (clear)
+ rv = qdma_descq_stm_clear(descq);
+ else
+ rv = qdma_descq_stm_setup(descq);
+ if (rv < 0) {
+ pr_warn("%s: failed to program stm", descq->conf.name);
+ return rv;
+ }
+
+ /* update pidx/cidx for C2H */
+ if (descq->conf.c2h) {
+ descq_cmpt_cidx_update(descq, 0);
+ descq_c2h_pidx_update(descq, descq->conf.rngsz - 1);
+ }
+
+ return rv;
+}
+#endif
+
+void qdma_descq_service_cmpl_update(struct qdma_descq *descq, int budget,
+ bool c2h_upd_cmpl)
+{
+ if (descq->conf.st && descq->conf.c2h) {
+ lock_descq(descq);
+ if (descq->q_state == Q_STATE_ONLINE)
+ descq_process_completion_st_c2h(descq, budget,
+ c2h_upd_cmpl);
+ unlock_descq(descq);
+ } else if ((descq->xdev->conf.qdma_drv_mode == POLL_MODE) ||
+ (descq->xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ if (!descq->proc_req_running)
+ qdma_descq_proc_sgt_request(descq);
+ } else {
+ lock_descq(descq);
+ descq_mm_n_h2c_cmpl_status(descq);
+ if (descq->pend_req_desc) {
+ unlock_descq(descq);
+ qdma_descq_proc_sgt_request(descq);
+ return;
+ }
+ unlock_descq(descq);
+ }
+}
+
+ssize_t qdma_descq_proc_sgt_request(struct qdma_descq *descq)
+{
+ if (!descq->conf.st) /* MM H2C/C2H */
+ return descq_mm_proc_request(descq);
+ else if (descq->conf.st && !descq->conf.c2h) /* ST H2C */
+ return descq_proc_st_h2c_request(descq);
+ else /* ST C2H - should not happen - handled separately */
+ return -1;
+}
+
+void incr_cmpl_desc_cnt(struct qdma_descq *descq, unsigned int cnt)
+{
+ descq->total_cmpl_descs += cnt;
+ switch ((descq->conf.st << 1) | descq->conf.c2h) {
+ case 0:
+ descq->xdev->total_mm_h2c_pkts += cnt;
+ break;
+ case 1:
+ descq->xdev->total_mm_c2h_pkts += cnt;
+ break;
+ case 2:
+ descq->xdev->total_st_h2c_pkts += cnt;
+ break;
+ case 3:
+ descq->xdev->total_st_c2h_pkts += cnt;
+ break;
+ default:
+ break;
+ }
+}
+
+void qdma_sgt_req_done(struct qdma_descq *descq, struct qdma_sgt_req_cb *cb,
+ int error)
+{
+ struct qdma_request *req = (struct qdma_request *)cb;
+
+ if (error)
+ pr_info("req 0x%p, cb 0x%p, fp_done 0x%p done, err %d.\n",
+ req, cb, req->fp_done, error);
+
+ list_del(&cb->list);
+ if (cb->unmap_needed) {
+ sgl_unmap(descq->xdev->conf.pdev, req->sgl, req->sgcnt,
+ descq->conf.c2h ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ cb->unmap_needed = 0;
+ }
+ cb->req_state = QDMA_REQ_COMPLETE;
+ if (req->fp_done) {
+ if ((cb->offset != req->count) &&
+ !(descq->conf.st && descq->conf.c2h)) {
+ pr_info("req 0x%p not completed %u != %u.\n",
+ req, cb->offset, req->count);
+ error = -EINVAL;
+ }
+ cb->status = error;
+ cb->done = 1;
+ req->fp_done(req, cb->offset, error);
+ } else {
+ pr_debug("req 0x%p, cb 0x%p, wake up.\n", req, cb);
+ cb->status = error;
+ cb->done = 1;
+ qdma_waitq_wakeup(&cb->wq);
+ }
+
+ if (descq->conf.st && descq->conf.c2h)
+ descq->pend_list_empty = (descq->avail == 0);
+ else
+ descq->pend_list_empty = (descq->avail ==
+ (descq->conf.rngsz - 1));
+
+ if (descq->q_stop_wait && descq->pend_list_empty)
+ qdma_waitq_wakeup(&descq->pend_list_wq);
+}
+
+int qdma_descq_dump_desc(struct qdma_descq *descq, int start,
+ int end, char *buf, int buflen)
+{
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ int desc_sz = get_desc_size(descq);
+ u8 *p = descq->desc + start * desc_sz;
+ struct qdma_sw_sg *fl = (descq->conf.st && descq->conf.c2h) ?
+ flq->sdesc + start : NULL;
+ int i = start;
+ int len = strlen(buf);
+
+ if (!descq->desc)
+ return 0;
+
+ for (; i < end && i < descq->conf.rngsz; i++, p += desc_sz) {
+ len += sprintf(buf + len, "%d: 0x%p ", i, p);
+ hex_dump_to_buffer(p, desc_sz,
+ (desc_sz < DESC_SZ_16B_BYTES) ? 16 : 32,
+ 4, buf + len, buflen - len, 0);
+ len = strlen(buf);
+ if (desc_sz > DESC_SZ_32B_BYTES) {
+ len += sprintf(buf + len, " ");
+ hex_dump_to_buffer(p + DESC_SZ_32B_BYTES, desc_sz,
+ 32, 4, buf + len, buflen - len, 0);
+ len = strlen(buf);
+ }
+ if (fl) {
+ len += sprintf(buf + len, " fl pg 0x%p, 0x%llx.\n",
+ fl->pg, fl->dma_addr);
+ fl++;
+ } else
+ buf[len++] = '\n';
+ }
+
+ p = descq->desc_cmpl_status;
+
+ dma_rmb();
+
+ len += sprintf(buf + len, "CMPL STATUS: 0x%p ", p);
+ hex_dump_to_buffer(p, get_desc_cmpl_status_size(descq), 16, 4,
+ buf + len, buflen - len, 0);
+ len = strlen(buf);
+ buf[len++] = '\n';
+
+ if (descq->conf.st && descq->conf.c2h) {
+ p = page_address(fl->pg);
+ len += sprintf(buf + len, "data 0: 0x%p ", p);
+ hex_dump_to_buffer(p, descq->cmpt_entry_len,
+ (descq->cmpt_entry_len < DESC_SZ_16B_BYTES) ? 16 : 32,
+ 4, buf + len,
+ buflen - len, 0);
+ len = strlen(buf);
+ if (descq->cmpt_entry_len > DESC_SZ_32B_BYTES) {
+ len += sprintf(buf + len, " ");
+ hex_dump_to_buffer(p + DESC_SZ_32B_BYTES,
+ descq->cmpt_entry_len, 32, 4,
+ buf + len, buflen - len, 0);
+ len = strlen(buf);
+ }
+ buf[len++] = '\n';
+ }
+
+ return len;
+}
+
+int qdma_descq_dump_cmpt(struct qdma_descq *descq, int start,
+ int end, char *buf, int buflen)
+{
+ uint8_t *cmpt = descq->desc_cmpt;
+ u8 *p;
+ int i = start;
+ int len = strlen(buf);
+ int stride = descq->cmpt_entry_len;
+
+ if (!descq->desc_cmpt)
+ return 0;
+
+ for (cmpt += (start * stride);
+ i < end && i < descq->conf.rngsz_cmpt; i++,
+ cmpt += stride) {
+ mdelay(100);
+ len += sprintf(buf + len, "%d: 0x%p ", i, cmpt);
+ hex_dump_to_buffer(cmpt, descq->cmpt_entry_len,
+ 32, 4, buf + len, buflen - len, 0);
+ mdelay(100);
+ len = strlen(buf);
+ if (descq->cmpt_entry_len > DESC_SZ_32B_BYTES) {
+ mdelay(100);
+ len += sprintf(buf + len, " ");
+ hex_dump_to_buffer(cmpt + DESC_SZ_32B_BYTES,
+ descq->cmpt_entry_len,
+ 32, 4, buf + len, buflen - len, 0);
+ mdelay(100);
+ len = strlen(buf);
+ }
+ mdelay(100);
+ buf[len++] = '\n';
+ }
+
+ len += sprintf(buf + len,
+ "CMPL STATUS: 0x%p ",
+ descq->desc_cmpt_cmpl_status);
+
+ p = descq->desc_cmpt_cmpl_status;
+ dma_rmb();
+ hex_dump_to_buffer(p, sizeof(struct qdma_c2h_cmpt_cmpl_status),
+ 16, 4, buf + len, buflen - len, 0);
+ len = strlen(buf);
+ buf[len++] = '\n';
+
+ return len;
+}
+
+int qdma_descq_dump_state(struct qdma_descq *descq, char *buf, int buflen)
+{
+ char *cur = buf;
+ char *const end = buf + buflen;
+
+ if (!buf || !buflen) {
+ pr_warn("incorrect arguments buf=%p buflen=%d", buf, buflen);
+ return 0;
+ }
+
+ cur += snprintf(cur, end - cur, "%s %s ",
+ descq->conf.name, descq->conf.c2h ? "C2H" : "H2C");
+ if (cur >= end)
+ goto handle_truncation;
+
+ if (descq->err)
+ cur += snprintf(cur, end - cur, "ERR\n");
+ else if (descq->q_state == Q_STATE_ONLINE)
+ cur += snprintf(cur, end - cur, "online\n");
+ else if (descq->q_state == Q_STATE_ENABLED)
+ cur += snprintf(cur, end - cur, "cfg'ed\n");
+ else
+ cur += snprintf(cur, end - cur, "un-initialized\n");
+ if (cur >= end)
+ goto handle_truncation;
+
+ return cur - buf;
+
+handle_truncation:
+ *buf = '\0';
+ return cur - buf;
+}
+
+int qdma_descq_dump(struct qdma_descq *descq, char *buf, int buflen, int detail)
+{
+ char *cur = buf;
+ char *const end = buf + buflen;
+
+ if (!buf || !buflen) {
+ pr_info("%s:%s 0x%x/0x%x, desc sz %u/%u, pidx %u, cidx %u\n",
+ descq->conf.name, descq->err ? "ERR" : "",
+ descq->conf.qidx, descq->qidx_hw, descq->conf.rngsz,
+ descq->avail, descq->pidx, descq->cidx);
+ return 0;
+ }
+
+ cur += qdma_descq_dump_state(descq, cur, end - cur);
+ if (cur >= end)
+ goto handle_truncation;
+
+ if (descq->q_state == Q_STATE_DISABLED)
+ return cur - buf;
+
+ cur += snprintf(cur, end - cur,
+ "\thw_ID %d, thp %s, desc 0x%p/0x%llx, %u\n",
+ descq->qidx_hw,
+ descq->cmplthp ? descq->cmplthp->name : "?",
+ descq->desc, descq->desc_bus, descq->conf.rngsz);
+ if (cur >= end)
+ goto handle_truncation;
+
+ if (descq->conf.st && descq->conf.c2h) {
+ cur += snprintf(cur, end - cur,
+ "\tcmpt desc 0x%p/0x%llx, %u\n",
+ descq->desc_cmpt, descq->desc_cmpt_bus,
+ descq->conf.rngsz_cmpt);
+ if (cur >= end)
+ goto handle_truncation;
+ }
+
+ if (!detail)
+ return cur - buf;
+
+ if (descq->desc_cmpl_status) {
+ u8 *cs = descq->desc_cmpl_status;
+
+ cur += snprintf(cur, end - cur, "\n\tcmpl status: 0x%p, ", cs);
+ if (cur >= end)
+ goto handle_truncation;
+
+ dma_rmb();
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
+ cur += hex_dump_to_buffer(cs,
+ sizeof(struct qdma_desc_cmpl_status),
+ 16, 4, cur, end - cur, 0);
+#else
+ hex_dump_to_buffer(cs, sizeof(struct qdma_desc_cmpl_status),
+ 16, 4, cur, end - cur, 0);
+ cur += strlen(cur);
+#endif
+ if (cur >= end)
+ goto handle_truncation;
+
+ cur += snprintf(cur, end - cur, "\n");
+ if (cur >= end)
+ goto handle_truncation;
+ }
+ if (descq->desc_cmpt_cmpl_status) {
+ u8 *cs = descq->desc_cmpt_cmpl_status;
+
+ cur += snprintf(cur, end - cur, "\tCMPT CMPL STATUS: 0x%p, ",
+ cs);
+ if (cur >= end)
+ goto handle_truncation;
+
+ dma_rmb();
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 0, 0)
+ cur += hex_dump_to_buffer(cs,
+ sizeof(struct qdma_c2h_cmpt_cmpl_status),
+ 16, 4, cur, end - cur, 0);
+#else
+ hex_dump_to_buffer(cs, sizeof(struct qdma_c2h_cmpt_cmpl_status),
+ 16, 4, cur, end - cur, 0);
+ cur += strlen(cur);
+#endif
+ if (cur >= end)
+ goto handle_truncation;
+
+ cur += snprintf(cur, end - cur, "\n");
+ if (cur >= end)
+ goto handle_truncation;
+ }
+
+ return cur - buf;
+
+handle_truncation:
+ *buf = '\0';
+ return cur - buf;
+}
+
+int qdma_queue_avail_desc(unsigned long dev_hndl, unsigned long id)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, NULL, 0, 1);
+ int avail;
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ lock_descq(descq);
+ avail = descq->avail;
+ unlock_descq(descq);
+
+ return avail;
+}
+
+#ifdef ERR_DEBUG
+int qdma_queue_set_err_induction(unsigned long dev_hndl, unsigned long id,
+ u32 err, char *buf, int buflen)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ const char *dummy; /* to avoid compiler warnings */
+ unsigned int err_en = (err >> 31);
+ unsigned int err_no = (err & 0x7FFFFFFF);
+
+ dummy = xnl_attr_str[0];
+ dummy = xnl_op_str[0];
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+ if (err_no < sb_mi_h2c0_dat) {
+ descq->induce_err &= ~(1 << err_no);
+ descq->induce_err |= (err_en << err_no);
+ } else {
+ descq->ecc_err &= ~(1 << (err_no - sb_mi_h2c0_dat));
+ descq->ecc_err |= (err_en << (err_no - sb_mi_h2c0_dat));
+ }
+ pr_info("Errs enabled = [QDMA]: 0x%08x 0x%08x\n [ECC]: 0x%08x 0x%08x",
+ (u32)(descq->induce_err >> 32),
+ (u32)(descq->induce_err & 0xFFFFFFFF),
+ (u32)(descq->ecc_err >> 32),
+ (u32)(descq->ecc_err & 0xFFFFFFFF));
+
+ return 0;
+}
+#endif
+
+int qdma_queue_packet_write(unsigned long dev_hndl, unsigned long id,
+ struct qdma_request *req)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, NULL, 0, 1);
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+ int rv;
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (!descq->conf.st || descq->conf.c2h) {
+ pr_info("%s: st %d, c2h %d.\n",
+ descq->conf.name, descq->conf.st, descq->conf.c2h);
+ return -EINVAL;
+ }
+
+ memset(cb, 0, QDMA_REQ_OPAQUE_SIZE);
+ qdma_waitq_init(&cb->wq);
+
+ if (!req->dma_mapped) {
+ rv = sgl_map(descq->xdev->conf.pdev, req->sgl, req->sgcnt,
+ DMA_TO_DEVICE);
+ if (rv < 0) {
+ pr_info("%s map sgl %u failed, %u.\n",
+ descq->conf.name, req->sgcnt, req->count);
+ goto unmap_sgl;
+ }
+ cb->unmap_needed = 1;
+ }
+
+ lock_descq(descq);
+ if (descq->q_state != Q_STATE_ONLINE) {
+ unlock_descq(descq);
+ pr_info("%s descq %s NOT online.\n",
+ descq->xdev->conf.name, descq->conf.name);
+ rv = -EINVAL;
+ goto unmap_sgl;
+ }
+
+ list_add_tail(&cb->list, &descq->work_list);
+ unlock_descq(descq);
+
+ pr_debug("%s: cb 0x%p submitted.\n", descq->conf.name, cb);
+
+ return req->count;
+
+unmap_sgl:
+ if (cb->unmap_needed)
+ sgl_unmap(descq->xdev->conf.pdev, req->sgl, req->sgcnt,
+ DMA_TO_DEVICE);
+ return rv;
+}
+
+int qdma_descq_get_cmpt_udd(unsigned long dev_hndl, unsigned long id,
+ char *buf, int buflen)
+{
+ uint8_t *cmpt;
+ uint8_t i = 0;
+ int len = 0;
+ int print_len = 0;
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, buf, buflen, 1);
+ struct qdma_c2h_cmpt_cmpl_status *cs;
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (!buf || !buflen)
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+
+ cs = (struct qdma_c2h_cmpt_cmpl_status *)
+ descq->desc_cmpt_cmpl_status;
+
+ cmpt = descq->desc_cmpt + ((cs->pidx - 1) * descq->cmpt_entry_len);
+
+ /*
+ * Ignoring the first 4 bits of the completion entry as they represent
+ * the error and color bits.
+ * TODO: May need to change the masking logic and move that in thegtest,
+ * as error and color bit positions may change in the future releases.
+ */
+ for (i = 0; i < descq->cmpt_entry_len; i++) {
+ if (buf && buflen) {
+ if (i == 0)
+ print_len = sprintf(buf + len, "%02x",
+ (cmpt[i] & 0xF0));
+ else
+ print_len = sprintf(buf + len, "%02x",
+ cmpt[i]);
+
+ }
+ buflen -= print_len;
+ len += print_len;
+ }
+ buf[len] = '\0';
+
+ return 0;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_descq.h b/QDMA/linux-kernel/libqdma/qdma_descq.h
new file mode 100644
index 0000000000000000000000000000000000000000..47919abd46e737c5e8328a4974112b68e69279fc
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_descq.h
@@ -0,0 +1,554 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_DESCQ_H__
+#define __QDMA_DESCQ_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma descq processing
+ *
+ */
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include "qdma_compat.h"
+#include "libqdma_export.h"
+#include "qdma_regs.h"
+#ifdef ERR_DEBUG
+#include "qdma_nl.h"
+#endif
+
+enum q_state_t {
+ Q_STATE_DISABLED = 0, /** Queue is not taken */
+ Q_STATE_ENABLED, /** Assigned/taken. Partial config is done */
+ Q_STATE_ONLINE, /** Resource/context is initialized for the
+ * queue and is available for data consumption
+ */
+};
+
+enum qdma_req_state {
+ QDMA_REQ_NOT_SUBMITTED,
+ QDMA_REQ_SUBMIT_PARTIAL,
+ QDMA_REQ_SUBMITTED,
+ QDMA_REQ_COMPLETE
+};
+
+#define QDMA_FLQ_SIZE 80
+
+/**
+ * @struct - qdma_descq
+ * @brief qdma software descriptor book keeping fields
+ */
+struct qdma_descq {
+ /** qdma queue configuration */
+ struct qdma_queue_conf conf;
+ /** lock to protect access to software descriptor */
+ spinlock_t lock;
+ /** pointer to dma device */
+ struct xlnx_dma_dev *xdev;
+ /** number of channels */
+ u8 channel;
+ /** flag to indicate error on the Q, in halted state */
+ u8 err:1;
+ /** color bit for the queue */
+ u8 color:1;
+ /** cpu attached */
+ u8 cpu_assigned:1;
+ /** state of the proc req */
+ u8 proc_req_running;
+ /** Indicate q state */
+ enum q_state_t q_state;
+ /** hw qidx associated for this queue */
+ unsigned int qidx_hw;
+ /** cpu attached to intr_work */
+ unsigned int intr_work_cpu;
+ /** queue handler */
+ struct work_struct work;
+ /** interrupt list */
+ struct list_head intr_list;
+ /** leagcy interrupt list */
+ struct list_head legacy_intr_q_list;
+ /** interrupt id associated for this queue */
+ int intr_id;
+ /** work list for the queue */
+ struct list_head work_list;
+ /** write back therad list */
+ struct qdma_kthread *cmplthp;
+ /** completion status thread list for the queue */
+ struct list_head cmplthp_list;
+ /** pending qork thread list */
+ struct list_head pend_list;
+ /** wait queue for pending list clear */
+ qdma_wait_queue pend_list_wq;
+ /** pending list empty count */
+ unsigned int pend_list_empty;
+ /* flag to indicate wwaiting for transfers to complete before q stop*/
+ unsigned int q_stop_wait;
+ /** availed count */
+ unsigned int avail;
+ /** IO batching cunt */
+ unsigned int io_batch_cnt;
+ /** current req count */
+ unsigned int pend_req_desc;
+ /** current producer index */
+ unsigned int pidx;
+ /** current consumer index */
+ unsigned int cidx;
+ /** number of descrtors yet to be processed*/
+ unsigned int credit;
+ /** desctor to be processed*/
+ u8 *desc;
+ /** desctor dma address*/
+ dma_addr_t desc_bus;
+ /** desctor writeback*/
+ u8 *desc_cmpl_status;
+
+ /* ST C2H */
+ /** programming order of the data in ST c2h mode*/
+ unsigned char fl_pg_order;
+ /** cmpt entry length*/
+ unsigned char cmpt_entry_len;
+ /** 2 bits reserved*/
+ unsigned char rsvd[2];
+ /** qdma free list q*/
+ unsigned char flq[QDMA_FLQ_SIZE];
+ /**total # of udd outstanding */
+ unsigned int udd_cnt;
+ /** packet count/number of packets to be processed*/
+ unsigned int pkt_cnt;
+ /** packet data length */
+ unsigned int pkt_dlen;
+ /** pidx of the completion entry */
+ unsigned int pidx_cmpt;
+ /** completion cidx */
+ unsigned int cidx_cmpt;
+ /** pending writeback cidx */
+ unsigned int cidx_cmpt_pend;
+ /** number of packets processed in q */
+ unsigned long long total_cmpl_descs;
+ /** descriptor writeback, data type depends on the cmpt_entry_len */
+ void *desc_cmpt_cur;
+ /** pointer to completion entry */
+ u8 *desc_cmpt;
+ /** descriptor dma bus address*/
+ dma_addr_t desc_cmpt_bus;
+ /** descriptor writeback dma bus address*/
+ u8 *desc_cmpt_cmpl_status;
+
+#ifdef ERR_DEBUG
+ /** flag to indicate error inducing */
+ u64 induce_err;
+ u64 ecc_err;
+#endif
+#ifdef DEBUGFS
+ /** debugfs queue index root */
+ struct dentry *dbgfs_qidx_root;
+ /** debugfs queue root */
+ struct dentry *dbgfs_queue_root;
+ /** debugfs cmpt queue root */
+ struct dentry *dbgfs_cmpt_queue_root;
+#endif
+};
+#ifdef DEBUG_THREADS
+#define lock_descq(descq) \
+ do { \
+ pr_debug("locking descq %s ...\n", (descq)->conf.name); \
+ spin_lock_bh(&(descq)->lock); \
+ } while (0)
+
+#define unlock_descq(descq) \
+ do { \
+ pr_debug("unlock descq %s ...\n", (descq)->conf.name); \
+ spin_unlock_bh(&(descq)->lock); \
+ } while (0)
+#else
+/** macro to lock descq */
+#define lock_descq(descq) spin_lock_bh(&(descq)->lock)
+/** macro to un lock descq */
+#define unlock_descq(descq) spin_unlock_bh(&(descq)->lock)
+#endif
+
+static inline unsigned int ring_idx_delta(unsigned int new, unsigned int old,
+ unsigned int rngsz)
+{
+ return new >= old ? (new - old) : new + (rngsz - old);
+}
+
+static inline unsigned int ring_idx_incr(unsigned int idx, unsigned int cnt,
+ unsigned int rngsz)
+{
+ idx += cnt;
+ return idx >= rngsz ? idx - rngsz : idx;
+}
+
+static inline unsigned int ring_idx_decr(unsigned int idx, unsigned int cnt,
+ unsigned int rngsz)
+{
+ return idx >= cnt ? idx - cnt : rngsz - (cnt - idx);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_descq_init() - initialize the sw descq entry
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] xdev: pointer to xdev
+ * @param[in] idx_hw: hw queue index
+ * @param[in] idx_sw: sw queue index
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_descq_init(struct qdma_descq *descq, struct xlnx_dma_dev *xdev,
+ int idx_hw, int idx_sw);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_config() - configure the sw descq entry
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] qconf: queue configuration
+ * @param[in] reconfig: flag to indicate whether to refig the sw descq
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_descq_config(struct qdma_descq *descq, struct qdma_queue_conf *qconf,
+ int reconfig);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_config_complete() - initialize the descq with default values
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_config_complete(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_cleanup() - clean up the resources assigned to a request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_descq_cleanup(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_alloc_resource() - allocate the resources for a request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_alloc_resource(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_free_resource() - free up the resources assigned to a request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_descq_free_resource(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_prog_hw() - program the hw descriptors
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_prog_hw(struct qdma_descq *descq);
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_descq_prog_stm() - program the STM
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] clear: flag to program/clear stm context
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_prog_stm(struct qdma_descq *descq, bool clear);
+#endif
+
+/*****************************************************************************/
+/**
+ * qdma_descq_context_cleanup() - clean up the queue context
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_context_cleanup(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_service_cmpl_update() - process completion data for the request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] budget: number of descriptors to process
+ * @param[in] c2h_upd_cmpl: C2H only: if update completion needed
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_descq_service_cmpl_update(struct qdma_descq *descq, int budget,
+ bool c2h_upd_cmpl);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_dump() - dump the queue sw desciptor data
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] detail: indicate whether full details or abstact details
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_dump(struct qdma_descq *descq,
+ char *buf, int buflen, int detail);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_dump_desc() - dump the queue hw descriptors
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] start: start descriptor index
+ * @param[in] end: end descriptor index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_dump_desc(struct qdma_descq *descq, int start, int end,
+ char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * qdma_descq_dump_state() - dump the queue desciptor state
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_dump_state(struct qdma_descq *descq, char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * intr_cidx_update() - update the interrupt cidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] sw_cidx: sw cidx
+ * @param[in] ring_index: ring index
+ *
+ *****************************************************************************/
+void intr_cidx_update(struct qdma_descq *descq, unsigned int sw_cidx,
+ int ring_index);
+
+/*****************************************************************************/
+/**
+ * incr_cmpl_desc_cnt() - update the interrupt cidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ *****************************************************************************/
+void incr_cmpl_desc_cnt(struct qdma_descq *descq, unsigned int cnt);
+
+/**
+ * @struct - qdma_sgt_req_cb
+ * @brief qdma_sgt_req_cb fits in qdma_request.opaque
+ */
+struct qdma_sgt_req_cb {
+ /** qdma read/write request list*/
+ struct list_head list;
+ /** request wait queue */
+ qdma_wait_queue wq;
+ /** number of descriptors to proccess*/
+ unsigned int desc_nr;
+ /** offset in the page*/
+ unsigned int offset;
+ /** number of data byte not yet proccessed*/
+ unsigned int left;
+ /** offset in the scatter gather list*/
+ unsigned int sg_offset;
+ /** scatter gather ebtry index*/
+ unsigned int sg_idx;
+ /** status of the request*/
+ int status;
+ /** indicates whether request processing is done or not*/
+ u8 done;
+ /** indicates whether to unmap the kernel pages*/
+ u8 unmap_needed:1;
+ /* flag to indicate partial req submit */
+ enum qdma_req_state req_state;
+};
+
+/** macro to get the request call back data */
+#define qdma_req_cb_get(req) (struct qdma_sgt_req_cb *)((req)->opaque)
+
+/*****************************************************************************/
+/**
+ * qdma_descq_proc_sgt_request() - handler to process the qdma
+ * read/write request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] cb: scatter gather list call back data
+ *
+ * @return size of the request
+ *****************************************************************************/
+ssize_t qdma_descq_proc_sgt_request(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_sgt_req_done() - handler to track the progress of the request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] cb: scatter gather list call back data
+ * @param[out] error: indicates the error status
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_sgt_req_done(struct qdma_descq *descq, struct qdma_sgt_req_cb *cb,
+ int error);
+
+/*****************************************************************************/
+/**
+ * sgl_map() - handler to map the scatter gather list to kernel pages
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] sg: scatter gather list
+ * @param[in] sgcnt: number of entries in scatter gather list
+ * @param[in] dir: direction of the request
+ *
+ * @return none
+ *****************************************************************************/
+int sgl_map(struct pci_dev *pdev, struct qdma_sw_sg *sg, unsigned int sgcnt,
+ enum dma_data_direction dir);
+
+/*****************************************************************************/
+/**
+ * sgl_unmap() - handler to unmap the scatter gather list and free
+ * the kernel pages
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] sg: scatter gather list
+ * @param[in] sgcnt: number of entries in scatter gather list
+ * @param[in] dir: direction of the request
+ *
+ * @return none
+ *****************************************************************************/
+void sgl_unmap(struct pci_dev *pdev, struct qdma_sw_sg *sg, unsigned int sgcnt,
+ enum dma_data_direction dir);
+
+/*****************************************************************************/
+/**
+ * descq_flq_free_resource() - handler to free the pages for the request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void descq_flq_free_resource(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * descq_h2c_pidx_update() - inline function to update the h2c pidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] pidx: c2h producer index
+ *
+ * @return none
+ *****************************************************************************/
+static inline void descq_h2c_pidx_update(struct qdma_descq *descq,
+ unsigned int pidx)
+{
+#ifdef ERR_DEBUG
+ const char *dummy; /* to avoid compiler warnings */
+
+ dummy = xnl_attr_str[0];
+ dummy = xnl_op_str[0];
+ if (descq->induce_err & (1 << qid_range)) {
+ __write_reg(descq->xdev,
+ QDMA_REG_H2C_PIDX_BASE +
+ descq->xdev->conf.qsets_max * QDMA_REG_PIDX_STEP,
+ pidx | (descq->conf.irq_en <<
+ S_CMPL_STATUS_PIDX_UPD_EN_INT));
+ pr_info("Inducing err %d", qid_range);
+ } else
+#endif
+ {
+ pr_debug("%s: pidx %u -> 0x%x, reg 0x%x.\n", descq->conf.name, pidx,
+ pidx | (descq->conf.irq_en << S_CMPL_STATUS_PIDX_UPD_EN_INT),
+ QDMA_REG_H2C_PIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP);
+
+ __write_reg(descq->xdev,
+ QDMA_REG_H2C_PIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP,
+ pidx | (descq->conf.irq_en << S_CMPL_STATUS_PIDX_UPD_EN_INT));
+ }
+}
+
+/*****************************************************************************/
+/**
+ * descq_c2h_pidx_update() - inline function to update the c2h pidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] pidx: c2h producer index
+ *
+ * @return none
+ *****************************************************************************/
+static inline void descq_c2h_pidx_update(struct qdma_descq *descq,
+ unsigned int pidx)
+{
+#ifdef ERR_DEBUG
+ if (descq->induce_err & (1 << qid_range)) {
+ __write_reg(descq->xdev,
+ QDMA_REG_C2H_PIDX_BASE +
+ descq->xdev->conf.qsets_max * QDMA_REG_PIDX_STEP,
+ pidx | (descq->conf.irq_en <<
+ S_CMPL_STATUS_PIDX_UPD_EN_INT));
+ pr_info("Inducing err %d", qid_range);
+ } else
+#endif
+ {
+ pr_debug("%s: pidx 0x%x -> 0x%x, reg 0x%x.\n", descq->conf.name, pidx,
+ pidx | (descq->conf.irq_en << S_CMPL_STATUS_PIDX_UPD_EN_INT),
+ QDMA_REG_C2H_PIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP);
+
+ __write_reg(descq->xdev,
+ QDMA_REG_C2H_PIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP,
+ pidx | (descq->conf.irq_en << S_CMPL_STATUS_PIDX_UPD_EN_INT));
+ }
+}
+
+#endif /* ifndef __QDMA_DESCQ_H__ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_device.c b/QDMA/linux-kernel/libqdma/qdma_device.c
new file mode 100644
index 0000000000000000000000000000000000000000..64ea0070224065130f7a0afa19d640887b3da5cf
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_device.c
@@ -0,0 +1,584 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/module.h>
+#include <linux/gfp.h>
+#include "qdma_device.h"
+#include "qdma_context.h"
+#include "qdma_descq.h"
+#include "qdma_intr.h"
+#include "qdma_regs.h"
+#include "qdma_mbox.h"
+
+#ifdef __QDMA_VF__
+static int device_set_qrange(struct xlnx_dma_dev *xdev)
+{
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct mbox_msg *m;
+ struct mbox_msg_hdr *hdr;
+ struct mbox_msg_fmap *fmap;
+ int rv = 0;
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ if (xdev->conf.cur_cfg_state == QMAX_CFG_INITIAL) {
+ qdev->qmax = 1;
+ qdev->qbase = 0;
+ }
+
+ m = qdma_mbox_msg_alloc(xdev, MBOX_OP_FMAP);
+ if (!m)
+ return -ENOMEM;
+
+ hdr = &m->hdr;
+ fmap = &m->fmap;
+
+ fmap->qbase = qdev->qbase;
+ fmap->qmax = qdev->qmax;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_FMAP_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ if (rv != -ENODEV)
+ pr_info("%s set q range (fmap) failed %d.\n",
+ xdev->conf.name, rv);
+ goto err_out;
+ }
+
+ if (hdr->status) {
+ rv = hdr->status;
+ if (xdev->conf.cur_cfg_state == QMAX_CFG_INITIAL) {
+ qdev->qbase = xdev->conf.qsets_base = fmap->qbase;
+ qdev->qmax = xdev->conf.qsets_max = 0;
+ }
+ pr_err("Failed to set qconf %d max/base = %u/%u\n", rv,
+ qdev->qmax, qdev->qbase);
+ goto err_out;
+ }
+
+ qdev->qbase = xdev->conf.qsets_base = fmap->qbase;
+
+ if (xdev->conf.cur_cfg_state == QMAX_CFG_INITIAL) {
+ qdev->h2c_descq->qidx_hw = fmap->qbase;
+ qdev->c2h_descq->qidx_hw = fmap->qbase;
+ xdev->conf.cur_cfg_state = QMAX_CFG_USER;
+ }
+
+ pr_debug("%s, func id %u/%u, Q 0x%x + 0x%x.\n",
+ xdev->conf.name, xdev->func_id, xdev->func_id_parent,
+ qdev->qbase, qdev->qmax);
+
+ qdev->init_qrange = 1;
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int device_set_qconf(struct xlnx_dma_dev *xdev, int qmax, u32 *qbase)
+{
+ struct mbox_msg *m;
+ struct mbox_msg_hdr *hdr;
+ struct mbox_msg_fmap *fmap;
+ int rv = 0;
+
+ m = qdma_mbox_msg_alloc(xdev, MBOX_OP_QCONF);
+ if (!m)
+ return -ENOMEM;
+
+ hdr = &m->hdr;
+ fmap = &m->fmap;
+
+ fmap->qbase = 0;
+ fmap->qmax = qmax;
+
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_QCONF_RESP,
+ QDMA_MBOX_MSG_TIMEOUT_MS);
+ if (rv < 0) {
+ pr_info("%s set q range (qconf) failed %d.\n",
+ xdev->conf.name, rv);
+ goto err_out;
+ }
+
+ if (hdr->status) {
+ rv = hdr->status;
+ pr_err("Failed to set qconf\n");
+ goto err_out;
+ }
+
+ *qbase = fmap->qbase;
+
+ qdma_device_set_cfg_state((unsigned long)xdev, QMAX_CFG_USER);
+ pr_debug("%s, func id %u/%u, Q 0x%x + 0x%x.\n",
+ xdev->conf.name, xdev->func_id, xdev->func_id_parent,
+ *qbase, qmax);
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+#else
+static int device_set_qrange(struct xlnx_dma_dev *xdev)
+{
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = 0;
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return QDMA_ERR_INVALID_QDMA_DEVICE;
+ }
+
+ hw_set_fmap(xdev, xdev->func_id, qdev->qbase, qdev->qmax);
+
+ qdev->init_qrange = 1;
+
+ pr_debug("%s, func id %u, Q 0x%x + 0x%x.\n",
+ xdev->conf.name, xdev->func_id, qdev->qbase, qdev->qmax);
+
+ return rv;
+}
+#endif /* ifndef __QDMA_VF__ */
+
+#ifndef __QDMA_VF__
+#ifdef ERR_DEBUG
+static void qdma_err_mon(struct work_struct *work)
+{
+ struct delayed_work *dwork = container_of(work,
+ struct delayed_work, work);
+ struct xlnx_dma_dev *xdev = container_of(dwork,
+ struct xlnx_dma_dev, err_mon);
+
+ if (!xdev) {
+ pr_err("Invalid xdev");
+ return;
+ }
+ spin_lock(&xdev->err_lock);
+
+ if (xdev->err_mon_cancel == 0) {
+ err_stat_handler(xdev);
+ schedule_delayed_work(dwork, msecs_to_jiffies(50)); /* 50msec */
+ }
+ spin_unlock(&xdev->err_lock);
+}
+#endif
+#endif
+
+int qdma_device_prep_q_resource(struct xlnx_dma_dev *xdev)
+{
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ int rv = 0;
+
+ spin_lock(&qdev->lock);
+
+ if (qdev->init_qrange)
+ goto done;
+
+ rv = device_set_qrange(xdev);
+ if (rv < 0)
+ goto done;
+
+ rv = intr_ring_setup(xdev);
+ if (rv)
+ goto done;
+
+ if ((xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ if (xdev->intr_coal_list != NULL) {
+ rv = qdma_intr_context_setup(xdev);
+ if (rv)
+ goto done;
+ } else {
+ pr_info("dev %s intr vec[%d] >= queues[%d], No aggregation\n",
+ dev_name(&xdev->conf.pdev->dev),
+ (xdev->num_vecs - xdev->dvec_start_idx),
+ xdev->conf.qsets_max);
+ pr_warn("Changing the system mode to direct interrupt mode");
+ xdev->conf.qdma_drv_mode = DIRECT_INTR_MODE;
+ }
+ }
+
+
+#ifndef __QDMA_VF__
+ if (((xdev->conf.qdma_drv_mode != POLL_MODE) &&
+ (xdev->conf.qdma_drv_mode != LEGACY_INTR_MODE)) &&
+ xdev->conf.master_pf)
+ qdma_err_intr_setup(xdev, 0);
+
+#ifdef ERR_DEBUG
+ else {
+ if ((xdev->conf.master_pf) &&
+ (xdev->conf.qdma_drv_mode == POLL_MODE)) {
+ spin_lock_init(&xdev->err_lock);
+ xdev->err_mon_cancel = 0;
+ INIT_DELAYED_WORK(&xdev->err_mon, qdma_err_mon);
+ schedule_delayed_work(&xdev->err_mon,
+ msecs_to_jiffies(50));
+ }
+ }
+#endif
+#endif
+
+done:
+ spin_unlock(&qdev->lock);
+
+ return rv;
+}
+
+int qdma_device_init(struct xlnx_dma_dev *xdev)
+{
+ int i;
+ int rv = 0;
+ int qmax = xdev->conf.qsets_max;
+ struct qdma_descq *descq;
+ struct qdma_dev *qdev;
+
+#ifdef __QDMA_VF__
+ xdev->func_id = xdev->func_id_parent = 0; /* filled later */
+#else
+ if (xdev->conf.master_pf) {
+ pr_info("%s master PF clearing memory.\n", xdev->conf.name);
+ rv = hw_init_global_context_memory(xdev);
+ if (rv)
+ return rv;
+ }
+#endif
+
+ if (xdev->conf.qdma_drv_mode != POLL_MODE &&
+ xdev->conf.qdma_drv_mode != LEGACY_INTR_MODE) {
+ rv = intr_setup(xdev);
+ if (rv)
+ return -EINVAL;
+ }
+ qdev = kzalloc(sizeof(struct qdma_dev) +
+ sizeof(struct qdma_descq) * qmax * 2, GFP_KERNEL);
+ if (!qdev) {
+ pr_err("dev %s qmax %d OOM.\n",
+ dev_name(&xdev->conf.pdev->dev), qmax);
+ intr_teardown(xdev);
+ return -ENOMEM;
+ }
+
+ spin_lock_init(&qdev->lock);
+
+ descq = (struct qdma_descq *)(qdev + 1);
+ qdev->h2c_descq = descq;
+ qdev->c2h_descq = descq + qmax;
+
+ xdev->dev_priv = (void *)qdev;
+ qdev->qmax = qmax;
+ qdev->init_qrange = 0;
+
+#ifdef __QDMA_VF__
+ if (xdev->conf.cur_cfg_state == QMAX_CFG_UNCONFIGURED) {
+ qdev->qbase = TOTAL_QDMA_QS - TOTAL_VF_QS
+ + (xdev->conf.idx - 1) * QDMA_Q_PER_VF_MAX;
+ } else
+ qdev->qbase = xdev->conf.qsets_base;
+#else
+ /*
+ * for the first device make qbase as 0 indicated by <0 value
+ * for others initial configuration, go for QDMA_Q_PER_PF_MAX
+ * for qbase
+ * if modified using sysfs/qmax, take it from calculated qbase
+ */
+ if (xdev->conf.cur_cfg_state == QMAX_CFG_UNCONFIGURED) {
+ qdev->qbase = (xdev->conf.idx - 1) * MAX_QS_PER_PF;
+ xdev->conf.cur_cfg_state = QMAX_CFG_INITIAL;
+ } else
+ qdev->qbase = xdev->conf.qsets_base;
+#endif
+ xdev->conf.qsets_base = qdev->qbase;
+
+ for (i = 0, descq = qdev->h2c_descq; i < qdev->qmax; i++, descq++)
+ qdma_descq_init(descq, xdev, i, i);
+ for (i = 0, descq = qdev->c2h_descq; i < qdev->qmax; i++, descq++)
+ qdma_descq_init(descq, xdev, i, i);
+#ifndef __QDMA_VF__
+ if (xdev->conf.master_pf) {
+ pr_info("%s master PF.\n", xdev->conf.name);
+ hw_set_global_csr(xdev);
+ qdma_trq_c2h_config(xdev);
+ for (i = 0; i < xdev->mm_channel_max; i++) {
+ hw_mm_channel_enable(xdev, i, 1);
+ hw_mm_channel_enable(xdev, i, 0);
+ }
+ }
+#endif
+ /** STM specific init */
+ xdev->pipe_stm_max_pkt_size = STM_MAX_PKT_SIZE;
+
+ return 0;
+}
+
+void qdma_device_cleanup(struct xlnx_dma_dev *xdev)
+{
+ int i;
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+ struct qdma_descq *descq;
+
+ if (!qdev) {
+ pr_info("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return;
+ }
+
+#ifndef __QDMA_VF__
+#ifdef ERR_DEBUG
+ if ((xdev->conf.master_pf) &&
+ (xdev->conf.qdma_drv_mode == POLL_MODE)) {
+ pr_info("Cancelling delayed work");
+ spin_lock(&xdev->err_lock);
+ xdev->err_mon_cancel = 1;
+ cancel_delayed_work_sync(&xdev->err_mon);
+ spin_unlock(&xdev->err_lock);
+ }
+#endif
+#endif
+
+ for (i = 0, descq = qdev->h2c_descq; i < qdev->qmax; i++, descq++) {
+ if (descq->q_state == Q_STATE_ONLINE)
+ qdma_queue_stop((unsigned long int)xdev, i, NULL, 0);
+ }
+
+ for (i = 0, descq = qdev->c2h_descq; i < qdev->qmax; i++, descq++) {
+ if (descq->q_state == Q_STATE_ONLINE)
+ qdma_queue_stop((unsigned long int)xdev,
+ i + qdev->qmax, NULL, 0);
+ }
+
+ intr_teardown(xdev);
+
+ if ((xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (xdev->conf.qdma_drv_mode == AUTO_MODE)) {
+ pr_info("dev %s teardown interrupt coalescing ring\n",
+ dev_name(&xdev->conf.pdev->dev));
+ intr_ring_teardown(xdev);
+ }
+
+#ifndef __QDMA_VF__
+ if (xdev->func_id == 0) {
+ for (i = 0; i < xdev->mm_channel_max; i++) {
+ hw_mm_channel_disable(xdev, i, DMA_TO_DEVICE);
+ hw_mm_channel_disable(xdev, i, DMA_FROM_DEVICE);
+ }
+ }
+#endif
+
+ for (i = 0, descq = qdev->h2c_descq; i < qdev->qmax; i++, descq++) {
+ if (descq->q_state == Q_STATE_ENABLED)
+ qdma_queue_remove((unsigned long int)xdev, i, NULL, 0);
+ }
+
+ for (i = 0, descq = qdev->c2h_descq; i < qdev->qmax; i++, descq++) {
+ if (descq->q_state == Q_STATE_ENABLED)
+ qdma_queue_remove((unsigned long int)xdev,
+ i + qdev->qmax, NULL, 0);
+ }
+
+ xdev->dev_priv = NULL;
+ kfree(qdev);
+}
+
+long qdma_device_get_id_from_descq(struct xlnx_dma_dev *xdev,
+ struct qdma_descq *descq)
+{
+ struct qdma_dev *qdev;
+ unsigned long idx;
+ unsigned long idx_max;
+ struct qdma_descq *_descq;
+
+ if (!xdev) {
+ pr_info("xdev NULL.\n");
+ return -EINVAL;
+ }
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return -EINVAL;
+ }
+ _descq = descq->conf.c2h ? qdev->c2h_descq : qdev->h2c_descq;
+ idx = descq->conf.c2h ? qdev->qmax : 0;
+
+ idx_max = (idx + qdev->qmax);
+ for ( ; idx < idx_max; idx++, _descq++)
+ if (_descq == descq)
+ return idx;
+
+ return -EINVAL;
+
+}
+
+struct qdma_descq *qdma_device_get_descq_by_id(struct xlnx_dma_dev *xdev,
+ unsigned long idx, char *buf, int buflen, int init)
+{
+ struct qdma_dev *qdev;
+ struct qdma_descq *descq;
+
+ if (!xdev) {
+ pr_info("xdev NULL.\n");
+ return NULL;
+ }
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return NULL;
+ }
+
+ if (idx >= qdev->qmax) {
+ idx -= qdev->qmax;
+ if (idx >= qdev->qmax) {
+ pr_info("%s, q idx too big 0x%lx > 0x%x.\n",
+ xdev->conf.name, idx, qdev->qmax);
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen,
+ "%s, q idx too big 0x%lx > 0x%x.\n",
+ xdev->conf.name, idx, qdev->qmax);
+ buf[len] = '\0';
+ buflen -= (len + 1);
+ }
+ return NULL;
+ }
+ descq = qdev->c2h_descq + idx;
+ } else {
+ descq = qdev->h2c_descq + idx;
+ }
+
+ if (init) {
+ lock_descq(descq);
+ if (descq->q_state == Q_STATE_DISABLED) {
+ pr_info("%s, idx 0x%lx, q 0x%p state invalid.\n",
+ xdev->conf.name, idx, descq);
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen,
+ "%s, idx 0x%lx, q 0x%p state invalid.\n",
+ xdev->conf.name, idx, descq);
+ buf[len] = '\0';
+ buflen -= (len + 1);
+ }
+ unlock_descq(descq);
+ return NULL;
+ }
+ unlock_descq(descq);
+ }
+
+ return descq;
+}
+
+#ifdef DEBUGFS
+struct qdma_descq *qdma_device_get_pair_descq_by_id(struct xlnx_dma_dev *xdev,
+ unsigned long idx, char *buf, int buflen, int init)
+{
+ struct qdma_dev *qdev;
+ struct qdma_descq *pair_descq;
+
+ if (!xdev) {
+ pr_info("xdev NULL.\n");
+ return NULL;
+ }
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return NULL;
+ }
+
+ if (idx >= qdev->qmax) {
+ idx -= qdev->qmax;
+ if (idx >= qdev->qmax) {
+ pr_debug("%s, q idx too big 0x%lx > 0x%x.\n",
+ xdev->conf.name, idx, qdev->qmax);
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen,
+ "%s, q idx too big 0x%lx > 0x%x.\n",
+ xdev->conf.name, idx, qdev->qmax);
+ buf[len] = '\0';
+ buflen -= len + 1;
+ }
+ return NULL;
+ }
+ pair_descq = qdev->h2c_descq + idx;
+ } else {
+ pair_descq = qdev->c2h_descq + idx;
+ }
+
+ if (init) {
+ lock_descq(pair_descq);
+ if (pair_descq->q_state == Q_STATE_DISABLED) {
+ pr_debug("%s, idx 0x%lx, q 0x%p state invalid.\n",
+ xdev->conf.name, idx, pair_descq);
+ if (buf && buflen) {
+ int len = snprintf(buf, buflen,
+ "%s, idx 0x%lx, q 0x%p state invalid.\n",
+ xdev->conf.name, idx, pair_descq);
+ buf[len] = '\0';
+ buflen -= len + 1;
+ }
+ unlock_descq(pair_descq);
+ return NULL;
+ }
+ unlock_descq(pair_descq);
+ }
+
+ return pair_descq;
+}
+#endif
+
+struct qdma_descq *qdma_device_get_descq_by_hw_qid(struct xlnx_dma_dev *xdev,
+ unsigned long qidx_hw, u8 c2h)
+{
+ struct qdma_dev *qdev;
+ struct qdma_descq *descq;
+ unsigned long qidx_sw = 0;
+
+ if (!xdev) {
+ pr_info("xdev NULL.\n");
+ return NULL;
+ }
+
+ qdev = xdev_2_qdev(xdev);
+
+ if (!qdev) {
+ pr_err("dev %s, qdev null.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ return NULL;
+ }
+
+ qidx_sw = qidx_hw - qdev->qbase;
+ if (c2h)
+ descq = &qdev->c2h_descq[qidx_sw];
+ else
+ descq = &qdev->h2c_descq[qidx_sw];
+
+ return descq;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_device.h b/QDMA/linux-kernel/libqdma/qdma_device.h
new file mode 100644
index 0000000000000000000000000000000000000000..fdc32c5eef4f5f63927df1efb9a8b0c08c778746
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_device.h
@@ -0,0 +1,276 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef LIBQDMA_QDMA_DEVICE_H_
+#define LIBQDMA_QDMA_DEVICE_H_
+/**
+ * @file
+ * @brief This file contains the declarations for QDMA device
+ *
+ */
+#include <linux/spinlock_types.h>
+#include "libqdma_export.h"
+
+/**
+ * forward declaration for qdma descriptor
+ */
+struct qdma_descq;
+/**
+ * forward declaration for xlnx_dma_dev
+ */
+struct xlnx_dma_dev;
+
+/**
+ * @struct - qdma_dev
+ * @brief qdma device per function
+ */
+struct qdma_dev {
+ /** flag indicates whether the fmap programming is completed or not */
+ u8 init_qrange:1;
+ /** filler */
+ u8 filler[3];
+ /** max number of queues per function */
+ unsigned short qmax;
+ /** queue start number for this function */
+ unsigned short qbase;
+ /** qdma_dev lock */
+ spinlock_t lock;
+ /** number of h2c queues for this function */
+ unsigned short h2c_qcnt;
+ /** number of c2h queues for this function */
+ unsigned short c2h_qcnt;
+ /** h2c descq list */
+ struct qdma_descq *h2c_descq;
+ /** c2h descq list */
+ struct qdma_descq *c2h_descq;
+};
+
+/**
+ * macro to convert the given xdev to qdev
+ */
+#define xdev_2_qdev(xdev) (struct qdma_dev *)((xdev)->dev_priv)
+
+/*****************************************************************************/
+/**
+ * qdma_device_init() - initializes the qdma device
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return -1: failure
+ *****************************************************************************/
+int qdma_device_init(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * qdma_device_cleanup() - clean up the qdma device
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_device_cleanup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_descq_by_id() - get the qhndl for descq
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] descq: pointer to the descq
+ *
+ * @return qhndl for descq on success
+ * @return <0 on failure
+ *****************************************************************************/
+long qdma_device_get_id_from_descq(struct xlnx_dma_dev *xdev,
+ struct qdma_descq *descq);
+/*****************************************************************************/
+/**
+ * qdma_device_get_descq_by_id() - get the descq using the qid
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] idx: sw qidx
+ * @param[in] init: indicates whether to initialize the device or not
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return pointer to descq on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct qdma_descq *qdma_device_get_descq_by_id(struct xlnx_dma_dev *xdev,
+ unsigned long idx, char *buf, int buflen, int init);
+
+#ifdef DEBUGFS
+/*****************************************************************************/
+/**
+ * qdma_device_get_pair_descq_by_id() - get the descq using the qid
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] idx: sw qidx
+ * @param[in] init: indicates whether to initialize the device or not
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return pointer to descq on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct qdma_descq *qdma_device_get_pair_descq_by_id(struct xlnx_dma_dev *xdev,
+ unsigned long idx, char *buf, int buflen, int init);
+#endif
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_descq_by_hw_qid() - get the descq using the hw qid
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qidx_hw: hw qidx
+ * @param[in] c2h: indicates whether hw qidx belongs to c2h or h2c
+ *
+ * @return pointer to descq on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct qdma_descq *qdma_device_get_descq_by_hw_qid(struct xlnx_dma_dev *xdev,
+ unsigned long qidx_hw, u8 c2h);
+
+/*****************************************************************************/
+/**
+ * qdma_device_prep_q_resource() - Prepare queue resources
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_device_prep_q_resource(struct xlnx_dma_dev *xdev);
+
+#ifndef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * qdma_csr_read_cmpl_status_acc() - Read the completion status
+ * accumulation value
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[out] cs_acc: cs_acc value
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_csr_read_cmpl_status_acc(struct xlnx_dma_dev *xdev,
+ unsigned int *cs_acc);
+
+/*****************************************************************************/
+/**
+ * qdma_csr_read_rngsz() - Read the queue ring size
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[out] rngsz: queue ring size
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_csr_read_rngsz(struct xlnx_dma_dev *xdev, unsigned int *rngsz);
+
+/*****************************************************************************/
+/**
+ * qdma_csr_read_bufsz() - Read the buffer size
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[out] bufsz: buffer size
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_csr_read_bufsz(struct xlnx_dma_dev *xdev, unsigned int *bufsz);
+
+/*****************************************************************************/
+/**
+ * qdma_csr_read_timer_cnt() - Read the timer count
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[out] cnt: timer count
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_csr_read_timer_cnt(struct xlnx_dma_dev *xdev, unsigned int *cnt);
+
+/*****************************************************************************/
+/**
+ * qdma_csr_read_timer_cnt() - Read the timer threshold
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[out] th: timer threshold
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_csr_read_cnt_thresh(struct xlnx_dma_dev *xdev, unsigned int *th);
+#else
+/*****************************************************************************/
+/**
+ * device_set_qconf() - set device conf
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] qmax: maximum request qsize for VF instance
+ *
+ * @return 0: success
+ * @return < 0: failure
+ *****************************************************************************/
+int device_set_qconf(struct xlnx_dma_dev *xdev, int qmax, u32 *qbase);
+
+#endif
+
+
+enum csr_type {
+ QDMA_CSR_TYPE_NONE,
+ QDMA_CSR_TYPE_RNGSZ, /** all global csr ring size settings */
+ QDMA_CSR_TYPE_BUFSZ, /** all global csr buffer size settings */
+ QDMA_CSR_TYPE_TIMER_CNT, /** all global csr timer count settings */
+ QDMA_CSR_TYPE_CNT_TH, /** all global csr counter thresh settings */
+
+ QDMA_CSR_TYPE_MAX
+};
+
+/**
+ * @struct - descq_csr_info
+ * @brief qdma Q csr register settings
+ */
+struct qdma_csr_info {
+ enum csr_type type; /** one csr register array */
+ u32 array[QDMA_GLOBAL_CSR_ARRAY_SZ];
+ u8 idx_rngsz; /** 1x index-value pair for each type */
+ u8 idx_bufsz;
+ u8 idx_timer_cnt;
+ u8 idx_cnt_th;
+ u32 rngsz;
+ u32 bufsz;
+ u32 timer_cnt;
+ u32 cnt_th;
+ u32 cmpl_status_acc;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_csr_read() - Read specific global csr registers
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] csr: csr type & index
+ * @param[out] csr: csr value
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_csr_read(struct xlnx_dma_dev *xdev, struct qdma_csr_info *csr_info,
+ unsigned int timeout_ms);
+
+#endif /* LIBQDMA_QDMA_DEVICE_H_ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_intr.c b/QDMA/linux-kernel/libqdma/qdma_intr.c
new file mode 100644
index 0000000000000000000000000000000000000000..55db5eb5b8e68402e8cd505e2572e5982a3bbd2a
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_intr.c
@@ -0,0 +1,1007 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_intr.h"
+
+#include <linux/kernel.h>
+#include "qdma_descq.h"
+#include "qdma_device.h"
+#include "qdma_regs.h"
+#include "thread.h"
+#include "version.h"
+#include "qdma_regs.h"
+
+struct qdma_err_info {
+ u32 intr_mask;
+ char **stat;
+};
+
+struct qdma_err_stat_info {
+ char *err_name;
+ u32 stat_reg_addr;
+ u32 mask_reg_addr;
+ struct qdma_err_info err_info;
+};
+
+/** List shall be from Bit 0 - Bit31 */
+char *glbl_err_info[] = {
+ "err_ram_sbe",
+ "err_ram_dbe",
+ "err_dsc",
+ "err_trq",
+ "err_h2c_mm_0",
+ "err_h2c_mm_1",
+ "err_c2h_mm_0",
+ "err_c2h_mm_1",
+ "err_c2h_st",
+ "ind_ctxt_cmd_err",
+ "err_bdg",
+ "err_h2c_st"
+};
+
+char *dsc_err_info[] = {
+ "poison",
+ "ur_ca",
+ "param",
+ "addr",
+ "tag",
+ "flr",
+ "timeout",
+ "dat_poison",
+ "flr_cancel",
+ "dma",
+ "dsc",
+ "rq_cancel",
+ "dbe",
+ "sbe"
+};
+
+char *trq_err_info[] = {
+ "unmapped",
+ "qid_range",
+ "vf_access_err",
+ "tcp_timeout"
+};
+
+char *c2h_err_info[] = {
+ "mty_mismatch",
+ "len_mismatch",
+ "qid_mismatch",
+ "desc_rsp_err",
+ "eng_wpl_data_par_err",
+ "msi_int_fail",
+ "err_desc_cnt",
+ "portid_ctxt_mismatch",
+ "portid_byp_in_mismatch",
+ "cmpt_inv_q_err",
+ "cmpt_qfull_err",
+ "cmpt_cidx_err",
+ "cmpt_prty_err"
+};
+
+char *c2h_fatal_err_info[] = {
+ "mty_mismatch",
+ "len_mismatch",
+ "qid_mismatch",
+ "timer_fifo_ram_rdbe",
+ "eng_wpl_data_par_err",
+ "pfch_II_ram_rdbe",
+ "cmpt_ctxt_ram_rdbe",
+ "pfch_ctxt_ram_rdbe",
+ "desc_req_fifo_ram_rdbe",
+ "int_ctxt_ram_rdbe",
+ "cmpt_coal_data_ram_rdbe",
+ "tuser_fifo_ram_rdbe",
+ "qid_fifo_ram_rdbe",
+ "payload_fifo_ram_rdbe",
+ "wpl_data_par_err"
+};
+
+char *h2c_err_info[] = {
+ "zero_len_desc_err",
+ "sdi_mrkr_req_mop_err",
+ "no_dma_dsc_err",
+};
+
+/** ECC Single bit errors from Bit 0 -Bit 31 */
+char *ecc_sb_err_info[] = {
+ "mi_h2c0_dat",
+ "mi_c2h0_dat",
+ "h2c_rd_brg_dat",
+ "h2c_wr_brg_dat",
+ "c2h_rd_brg_dat",
+ "c2h_wr_brg_dat",
+ "func_map",
+ "dsc_hw_ctxt",
+ "dsc_crd_rcv",
+ "dsc_sw_ctxt",
+ "dsc_cpli",
+ "dsc_cpld",
+ "pasid_ctxt_ram",
+ "timer_fifo_ram",
+ "payload_fifo_ram",
+ "qid_fifo_ram",
+ "tuser_fifo_ram",
+ "wrb_coal_data_ram",
+ "int_qid2vec_ram",
+ "int_ctxt_ram",
+ "desc_req_fifo_ram",
+ "pfch_ctxt_ram",
+ "wrb_ctxt_ram",
+ "pfch_ll_ram",
+ "h2c_pend_fifo"
+};
+
+/** ECC Double bit errors from Bit 0 -Bit 31 */
+char *ecc_db_err_info[] = {
+ "mi_h2c0_dat",
+ "mi_c2h0_dat",
+ "h2c_rd_brg_dat",
+ "h2c_wr_brg_dat",
+ "c2h_rd_brg_dat",
+ "c2h_wr_brg_dat",
+ "func_map",
+ "dsc_hw_ctxt",
+ "dsc_crd_rcv",
+ "dsc_sw_ctxt",
+ "dsc_cpli",
+ "dsc_cpld",
+ "pasid_ctxt_ram",
+ "timer_fifo_ram",
+ "payload_fifo_ram",
+ "qid_fifo_ram",
+ "tuser_fifo_ram",
+ "wrb_coal_data_ram",
+ "int_qid2vec_ram",
+ "int_ctxt_ram",
+ "desc_req_fifo_ram",
+ "pfch_ctxt_ram",
+ "wrb_ctxt_ram",
+ "pfch_ll_ram",
+ "h2c_pend_fifo",
+};
+
+struct qdma_err_stat_info err_stat_info[HW_ERRS] = {
+ { "glbl_err", QDMA_REG_GLBL_ERR_STAT, QDMA_REG_GLBL_ERR_MASK,
+ { QDMA_REG_GLBL_ERR_MASK_VALUE, glbl_err_info } },
+ { "dsc_err", QDMA_GLBL_DSC_ERR_STS, QDMA_GLBL_DSC_ERR_MSK,
+ { QDMA_GLBL_DSC_ERR_MSK_VALUE, dsc_err_info } },
+ { "trq_err", QDMA_GLBL_TRQ_ERR_STS, QDMA_GLBL_TRQ_ERR_MSK,
+ { QDMA_GLBL_TRQ_ERR_MSK_VALUE, trq_err_info } },
+ { "c2h_err", QDMA_REG_C2H_ERR_STAT, QDMA_REG_C2H_ERR_MASK,
+ { QDMA_REG_C2H_ERR_MASK_VALUE, c2h_err_info } },
+ { "c2h_fatal_err", QDMA_C2H_FATAL_ERR_STAT, QDMA_C2H_FATAL_ERR_MASK,
+ { QDMA_C2H_FATAL_ERR_MASK_VALUE, c2h_fatal_err_info } },
+ { "h2c_err", QDMA_H2C_ERR_STAT, QDMA_H2C_ERR_MASK,
+ { QDMA_H2C_ERR_MASK_VALUE, h2c_err_info } },
+ { "ecc_sb_err", QDMA_RAM_SBE_STAT_A, QDMA_RAM_SBE_MASK_A,
+ { QDMA_RAM_SBE_MASK_VALUE, ecc_sb_err_info } },
+ { "ecc_sb_err", QDMA_RAM_DBE_STAT_A, QDMA_RAM_DBE_MASK_A,
+ { QDMA_RAM_DBE_MASK_VALUE, ecc_db_err_info } },
+};
+
+#ifndef __QDMA_VF__
+static LIST_HEAD(legacy_intr_q_list);
+static spinlock_t legacy_intr_lock;
+static spinlock_t legacy_q_add_lock;
+static unsigned long legacy_intr_flags = IRQF_SHARED;
+#endif
+
+void err_stat_handler(struct xlnx_dma_dev *xdev)
+{
+ u32 i;
+ u32 err_stat;
+ u32 glb_err_stat = 0;
+
+ for (i = 0; i < HW_ERRS; i++) {
+ err_stat = __read_reg(xdev, err_stat_info[i].stat_reg_addr);
+ if ((i == 0) && err_stat_info[i].err_info.intr_mask)
+ glb_err_stat = err_stat;
+ if (err_stat & err_stat_info[i].err_info.intr_mask) {
+ uint8_t bit = 0;
+ uint32_t intr_mask =
+ err_stat_info[i].err_info.intr_mask;
+ uint32_t chk_mask = 0x01;
+
+ pr_info("%s[0x%x] : 0x%x", err_stat_info[i].err_name,
+ err_stat_info[i].stat_reg_addr,
+ err_stat);
+ while (intr_mask) {
+ if (((intr_mask & 0x01)) &&
+ (err_stat & chk_mask))
+ pr_err("\t%s detected",
+ err_stat_info[i].err_info.stat[bit]);
+
+ if (intr_mask & 0x01)
+ bit++;
+ intr_mask >>= 1;
+ chk_mask <<= 1;
+ }
+ __write_reg(xdev, err_stat_info[i].stat_reg_addr,
+ err_stat);
+ }
+ }
+ if (glb_err_stat) {
+ __write_reg(xdev, err_stat_info[0].stat_reg_addr,
+ glb_err_stat);
+ qdma_err_intr_setup(xdev, 1);
+ }
+}
+
+static irqreturn_t user_intr_handler(int irq_index, int irq, void *dev_id)
+{
+ struct xlnx_dma_dev *xdev = dev_id;
+
+ pr_info("User IRQ fired on PF#%d: index=%d, vector=%d\n",
+ xdev->func_id, irq_index, irq);
+
+ if (xdev->conf.fp_user_isr_handler)
+ xdev->conf.fp_user_isr_handler((unsigned long)xdev,
+ xdev->conf.uld);
+
+ return IRQ_HANDLED;
+}
+
+#ifndef __QDMA_VF__
+static irqreturn_t error_intr_handler(int irq_index, int irq, void *dev_id)
+{
+ struct xlnx_dma_dev *xdev = dev_id;
+ unsigned long flags;
+
+ pr_info("Error IRQ fired on PF#%d: index=%d, vector=%d\n",
+ xdev->func_id, irq_index, irq);
+
+ spin_lock_irqsave(&xdev->lock, flags);
+
+ err_stat_handler(xdev);
+
+ spin_unlock_irqrestore(&xdev->lock, flags);
+
+ return IRQ_HANDLED;
+}
+#endif
+
+static void data_intr_aggregate(struct xlnx_dma_dev *xdev, int vidx, int irq)
+{
+ struct qdma_descq *descq = NULL;
+ u32 counter = 0;
+ int ring_index = 0;
+ struct intr_coal_conf *coal_entry =
+ (xdev->intr_coal_list + vidx - xdev->dvec_start_idx);
+ struct qdma_intr_ring *ring_entry;
+
+ if (!coal_entry) {
+ pr_err("Failed to locate the coalescing entry for vector = %d\n",
+ vidx);
+ return;
+ }
+ pr_debug("INTR_COAL: msix[%d].vector=%d, msix[%d].entry=%d, rngsize=%d, cidx = %d\n",
+ vidx, xdev->msix[vidx].vector,
+ vidx,
+ xdev->msix[vidx].entry,
+ coal_entry->intr_rng_num_entries,
+ coal_entry->cidx);
+
+ pr_debug("vidx = %d, dvec_start_idx = %d\n", vidx,
+ xdev->dvec_start_idx);
+
+ if ((xdev->msix[vidx].entry) != coal_entry->vec_id) {
+ pr_err("msix[%d].entry[%d] != vec_id[%d]\n",
+ vidx, xdev->msix[vidx].entry,
+ coal_entry->vec_id);
+
+ return;
+ }
+
+ counter = coal_entry->cidx;
+ ring_entry = (coal_entry->intr_ring_base + counter);
+ if (!ring_entry) {
+ pr_err("Failed to locate the ring entry for vector = %d\n",
+ vidx);
+ return;
+ }
+ while (ring_entry->coal_color == coal_entry->color) {
+ pr_debug("IRQ[%d]: IVE[%d], Qid = %d, e_color = %d, c_color = %d, intr_type = %d\n",
+ irq, vidx, ring_entry->qid, coal_entry->color,
+ ring_entry->coal_color, ring_entry->intr_type);
+
+ descq = qdma_device_get_descq_by_hw_qid(xdev, ring_entry->qid,
+ ring_entry->intr_type);
+ if (!descq) {
+ pr_err("IRQ[%d]: IVE[%d], Qid = %d: desc not found\n",
+ irq, vidx, ring_entry->qid);
+ return;
+ }
+
+ if (descq->conf.fp_descq_isr_top) {
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+
+ descq->conf.fp_descq_isr_top(descq->conf.qidx +
+ (descq->conf.c2h ? qdev->qmax : 0),
+ descq->conf.quld);
+ } else {
+ if (descq->cpu_assigned)
+ schedule_work_on(descq->intr_work_cpu,
+ &descq->work);
+ else
+ schedule_work(&descq->work);
+ }
+
+ if (++coal_entry->cidx == coal_entry->intr_rng_num_entries) {
+ counter = 0;
+ xdev->intr_coal_list->color =
+ (xdev->intr_coal_list->color) ? 0 : 1;
+ coal_entry->cidx = 0;
+ } else
+ counter++;
+
+ ring_entry = (coal_entry->intr_ring_base + counter);
+ }
+
+ if (descq) {
+ ring_index = get_intr_ring_index(descq->xdev,
+ coal_entry->vec_id);
+ intr_cidx_update(descq, coal_entry->cidx, ring_index);
+ }
+}
+
+static void data_intr_direct(struct xlnx_dma_dev *xdev, int vidx, int irq)
+{
+ struct qdma_descq *descq;
+
+ list_for_each_entry(descq, &xdev->dev_intr_info_list[vidx].intr_list,
+ intr_list)
+ if (descq->conf.fp_descq_isr_top) {
+ struct qdma_dev *qdev = xdev_2_qdev(xdev);
+
+ descq->conf.fp_descq_isr_top(descq->conf.qidx +
+ (descq->conf.c2h ? qdev->qmax : 0),
+ descq->conf.quld);
+ } else {
+ if (descq->cpu_assigned)
+ schedule_work_on(descq->intr_work_cpu,
+ &descq->work);
+ else
+ schedule_work(&descq->work);
+ }
+}
+
+static irqreturn_t data_intr_handler(int vector_index, int irq, void *dev_id)
+{
+ struct xlnx_dma_dev *xdev = dev_id;
+ unsigned long flags;
+
+ pr_debug("Data IRQ fired on PF#%d: index=%d, vector=%d\n",
+ xdev->func_id, vector_index, irq);
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ if ((xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) ||
+ (xdev->conf.qdma_drv_mode == AUTO_MODE))
+ data_intr_aggregate(xdev, vector_index, irq);
+ else
+ data_intr_direct(xdev, vector_index, irq);
+ spin_unlock_irqrestore(&xdev->lock, flags);
+
+ return IRQ_HANDLED;
+}
+
+static inline void intr_ring_free(struct xlnx_dma_dev *xdev, int ring_sz,
+ int intr_desc_sz, u8 *intr_desc, dma_addr_t desc_bus)
+{
+ unsigned int len = ring_sz * intr_desc_sz;
+
+ pr_debug("free %u(0x%x)=%d*%u, 0x%p, bus 0x%llx.\n",
+ len, len, intr_desc_sz, ring_sz, intr_desc, desc_bus);
+
+ dma_free_coherent(&xdev->conf.pdev->dev, (size_t)ring_sz * intr_desc_sz,
+ intr_desc, desc_bus);
+}
+
+static void *intr_ring_alloc(struct xlnx_dma_dev *xdev, int ring_sz,
+ int intr_desc_sz, dma_addr_t *bus)
+{
+ unsigned int len = ring_sz * intr_desc_sz;
+ u8 *p = dma_alloc_coherent(&xdev->conf.pdev->dev, len, bus, GFP_KERNEL);
+
+ if (!p) {
+ pr_err("%s, OOM, sz ring %d, intr_desc %d.\n",
+ xdev->conf.name, ring_sz, intr_desc_sz);
+ return NULL;
+ }
+
+ memset(p, 0, len);
+
+ pr_debug("alloc %u(0x%x)=%d*%u, bus 0x%llx .\n",
+ len, len, intr_desc_sz, ring_sz, *bus);
+
+ return p;
+}
+
+void intr_ring_teardown(struct xlnx_dma_dev *xdev)
+{
+ int i = 0;
+ struct intr_coal_conf *ring_entry;
+
+
+#ifndef __QDMA_VF__
+ int rv = 0;
+ unsigned int ring_index = 0;
+#endif
+
+ while (i < QDMA_NUM_DATA_VEC_FOR_INTR_CXT) {
+#ifndef __QDMA_VF__
+ ring_index = get_intr_ring_index(xdev,
+ (i + xdev->dvec_start_idx));
+#endif
+ ring_entry = (xdev->intr_coal_list + i);
+ if (ring_entry) {
+ intr_ring_free(xdev,
+ ring_entry->intr_rng_num_entries,
+ sizeof(struct qdma_intr_ring),
+ (u8 *)ring_entry->intr_ring_base,
+ ring_entry->intr_ring_bus);
+#ifndef __QDMA_VF__
+ pr_debug("Clearing intr_ctxt for ring_index =%d\n",
+ ring_index);
+ /* clear interrupt context (0x8) */
+ rv = hw_indirect_ctext_prog(xdev,
+ ring_index, QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_COAL, NULL, 0, 0);
+ if (rv < 0) {
+ pr_err("Failed to clear interrupt context, rv = %d\n",
+ rv);
+ }
+#endif
+ }
+ i++;
+ }
+
+ kfree(xdev->intr_coal_list);
+ pr_debug("dev %s interrupt coalescing ring teardown successful\n",
+ dev_name(&xdev->conf.pdev->dev));
+}
+
+static void data_vector_handler(int irq, struct xlnx_dma_dev *xdev)
+{
+ int i;
+
+ for (i = 0; i < xdev->num_vecs; i++) {
+ if (xdev->msix[i].vector == irq) {
+ xdev->dev_intr_info_list[i].intr_vec_map.intr_handler(i,
+ irq, (void *)xdev);
+ break;
+ }
+ }
+}
+
+static irqreturn_t irq_bottom(int irq, void *dev_id)
+{
+ struct xlnx_dma_dev *xdev = dev_id;
+
+ data_vector_handler(irq, xdev);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t irq_top(int irq, void *dev_id)
+{
+ struct xlnx_dma_dev *xdev = dev_id;
+
+ if (xdev->conf.fp_q_isr_top_dev) {
+ xdev->conf.fp_q_isr_top_dev((unsigned long)xdev,
+ xdev->conf.uld);
+ }
+
+ return IRQ_WAKE_THREAD;
+}
+
+
+void intr_teardown(struct xlnx_dma_dev *xdev)
+{
+ int i = xdev->num_vecs;
+
+ while (--i >= 0)
+ free_irq(xdev->msix[i].vector, xdev);
+
+ if (xdev->num_vecs)
+ pci_disable_msix(xdev->conf.pdev);
+
+ kfree(xdev->msix);
+ kfree(xdev->dev_intr_info_list);
+}
+
+
+static int intr_vector_setup(struct xlnx_dma_dev *xdev, int idx,
+ enum intr_type_list type, f_intr_handler handler)
+{
+ int rv;
+
+ if (type == INTR_TYPE_ERROR)
+ snprintf(xdev->dev_intr_info_list[idx].msix_name,
+ QDMA_DEV_NAME_MAXLEN + 16, "%s-error",
+ xdev->conf.name);
+ else if (type == INTR_TYPE_USER)
+ snprintf(xdev->dev_intr_info_list[idx].msix_name,
+ QDMA_DEV_NAME_MAXLEN + 16, "%s-user", xdev->conf.name);
+ else if (type == INTR_TYPE_DATA)
+ snprintf(xdev->dev_intr_info_list[idx].msix_name,
+ QDMA_DEV_NAME_MAXLEN + 16, "%s-data", xdev->conf.name);
+ else
+ snprintf(xdev->dev_intr_info_list[idx].msix_name,
+ QDMA_DEV_NAME_MAXLEN + 16, "%s", xdev->conf.name);
+
+ xdev->dev_intr_info_list[idx].intr_vec_map.intr_type = type;
+ xdev->dev_intr_info_list[idx].intr_vec_map.intr_vec_index = idx;
+ xdev->dev_intr_info_list[idx].intr_vec_map.intr_handler = handler;
+
+ if (type == INTR_TYPE_DATA)
+ rv = request_irq(xdev->msix[idx].vector, irq_bottom, 0,
+ xdev->dev_intr_info_list[idx].msix_name, xdev);
+ else
+ rv = request_threaded_irq(xdev->msix[idx].vector, irq_top,
+ irq_bottom, 0,
+ xdev->dev_intr_info_list[idx].msix_name,
+ xdev);
+
+ pr_info("%s requesting IRQ vector #%d: vec %d, type %d, %s.\n",
+ xdev->conf.name, idx, xdev->msix[idx].vector,
+ type, xdev->dev_intr_info_list[idx].msix_name);
+
+ if (rv) {
+ pr_err("%s requesting IRQ vector #%d: vec %d failed %d.\n",
+ xdev->conf.name, idx, xdev->msix[idx].vector, rv);
+ return rv;
+ }
+
+ return 0;
+}
+#ifdef __PCI_MSI_VEC_COUNT__
+
+#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
+
+static int pci_msix_vec_count(struct pci_dev *dev)
+{
+ u16 control;
+
+ if (!dev->msix_cap)
+ return 0;
+
+ pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
+ return msix_table_size(control);
+}
+#endif
+
+int intr_setup(struct xlnx_dma_dev *xdev)
+{
+ int rv = 0;
+ int i;
+
+ if ((xdev->conf.qdma_drv_mode == POLL_MODE) ||
+ (xdev->conf.qdma_drv_mode == LEGACY_INTR_MODE)) {
+ goto exit;
+ }
+ xdev->num_vecs = pci_msix_vec_count(xdev->conf.pdev);
+ pr_debug("dev %s, xdev->num_vecs = %d\n",
+ dev_name(&xdev->conf.pdev->dev), xdev->num_vecs);
+
+ if (!xdev->num_vecs) {
+ pr_warn("MSI-X not supported, running in polled mode\n");
+ return 0;
+ }
+
+ xdev->msix = kzalloc((sizeof(struct msix_entry) * xdev->num_vecs),
+ GFP_KERNEL);
+ if (!xdev->msix) {
+ pr_err("dev %s xdev->msix OOM.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ rv = -ENOMEM;
+ goto exit;
+ }
+
+ xdev->dev_intr_info_list =
+ kzalloc((sizeof(struct intr_info_t) * xdev->num_vecs),
+ GFP_KERNEL);
+ if (!xdev->dev_intr_info_list) {
+ pr_err("dev %s xdev->dev_intr_info_list OOM.\n",
+ dev_name(&xdev->conf.pdev->dev));
+ rv = -ENOMEM;
+ goto free_msix;
+ }
+
+ for (i = 0; i < xdev->num_vecs; i++) {
+ xdev->msix[i].entry = i;
+ INIT_LIST_HEAD(&xdev->dev_intr_info_list[i].intr_list);
+ }
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
+ rv = pci_enable_msix_exact(xdev->conf.pdev, xdev->msix, xdev->num_vecs);
+#else
+ rv = pci_enable_msix(xdev->conf.pdev, xdev->msix, xdev->num_vecs);
+#endif
+ if (rv < 0) {
+ pr_err("Error enabling MSI-X (%d)\n", rv);
+ goto free_intr_info;
+ }
+
+ /** On master PF0, vector#0 is dedicated for Error interrupts and
+ * vector #1 is dedicated for User interrupts
+ * For all other PFs and VFs, vector#0 is dedicated for User interrupts
+ * The remaining vectors are for Data interrupts
+ */
+ i = 0;
+#ifndef __QDMA_VF__
+ /* global error interrupt */
+ if (xdev->conf.master_pf) {
+ rv = intr_vector_setup(xdev, 0, INTR_TYPE_ERROR,
+ error_intr_handler);
+ if (rv)
+ goto cleanup_irq;
+ i = 1;
+ }
+#endif
+
+ /* user interrupt */
+ rv = intr_vector_setup(xdev, i, INTR_TYPE_USER, user_intr_handler);
+ if (rv)
+ goto cleanup_irq;
+
+ /* data interrupt */
+ xdev->dvec_start_idx = ++i;
+ for (; i < xdev->num_vecs; i++) {
+ rv = intr_vector_setup(xdev, i, INTR_TYPE_DATA,
+ data_intr_handler);
+ if (rv)
+ goto cleanup_irq;
+ }
+
+ xdev->flags |= XDEV_FLAG_IRQ;
+ return rv;
+
+cleanup_irq:
+ while (--i >= 0)
+ free_irq(xdev->msix[i].vector, xdev);
+
+ pci_disable_msix(xdev->conf.pdev);
+ xdev->num_vecs = 0;
+free_intr_info:
+ kfree(xdev->dev_intr_info_list);
+free_msix:
+ kfree(xdev->msix);
+exit:
+ return rv;
+}
+
+#ifndef __QDMA_VF__
+static irqreturn_t irq_legacy(int irq, void *param)
+{
+ struct list_head *entry, *tmp;
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)param;
+ irqreturn_t ret = IRQ_NONE;
+
+ if (!xdev) {
+ pr_err("Invalid Xdev");
+ goto irq_return;
+ }
+
+ spin_lock_irqsave(&legacy_intr_lock, legacy_intr_flags);
+ if (__read_reg(xdev, QDMA_GLBL_INTERRUPT_CFG) &
+ QDMA_GLBL_INTERRUPT_LGCY_INTR_PEND) {
+
+ list_for_each_safe(entry, tmp, &legacy_intr_q_list) {
+ struct qdma_descq *descq =
+ container_of(entry,
+ struct qdma_descq,
+ legacy_intr_q_list);
+
+ qdma_descq_service_cmpl_update(descq, 0, 1);
+ }
+ __write_reg(xdev, QDMA_GLBL_INTERRUPT_CFG,
+ QDMA_GLBL_INTERRUPT_CFG_EN_LGCY_INTR |
+ QDMA_GLBL_INTERRUPT_LGCY_INTR_PEND);
+ ret = IRQ_HANDLED;
+ }
+ spin_unlock_irqrestore(&legacy_intr_lock, legacy_intr_flags);
+
+irq_return:
+ return ret;
+}
+
+void intr_legacy_clear(struct qdma_descq *descq)
+{
+
+ if (!descq) {
+ pr_err("Invalid descq received");
+ return;
+ }
+ list_del(&descq->legacy_intr_q_list);
+
+ if (list_empty(&legacy_intr_q_list)) {
+
+ pr_info("un-registering legacy interrupt from qdma%05x\n",
+ descq->xdev->conf.bdf);
+ __write_reg(descq->xdev, QDMA_GLBL_INTERRUPT_CFG,
+ QDMA_GLBL_INTERRUPT_LGCY_INTR_PEND);
+ free_irq(descq->xdev->conf.pdev->irq, descq->xdev);
+ }
+}
+
+int intr_legacy_setup(struct qdma_descq *descq)
+{
+ int req_irq = 0;
+ int rv = 0;
+
+ if (!descq) {
+ pr_err("Invalid descq received");
+ return -EINVAL;
+ }
+
+ spin_lock(&legacy_q_add_lock);
+ req_irq = list_empty(&legacy_intr_q_list);
+ rv = req_irq ? 0 : 1;
+
+ if (req_irq != 0) {
+ spin_lock_init(&legacy_intr_lock);
+ pr_debug("registering legacy interrupt for irq-%d from qdma%05x\n",
+ descq->xdev->conf.pdev->irq, descq->xdev->conf.bdf);
+ __write_reg(descq->xdev, QDMA_GLBL_INTERRUPT_CFG,
+ QDMA_GLBL_INTERRUPT_LGCY_INTR_PEND);
+ rv = request_threaded_irq(descq->xdev->conf.pdev->irq, irq_top,
+ irq_legacy, legacy_intr_flags,
+ "qdma legacy intr",
+ descq->xdev);
+
+ if (rv < 0)
+ goto exit_intr_setup;
+ else {
+ list_add_tail(&descq->legacy_intr_q_list,
+ &legacy_intr_q_list);
+ rv = 0;
+ }
+ __write_reg(descq->xdev, QDMA_GLBL_INTERRUPT_CFG,
+ QDMA_GLBL_INTERRUPT_CFG_EN_LGCY_INTR);
+ } else
+ list_add_tail(&descq->legacy_intr_q_list,
+ &legacy_intr_q_list);
+
+exit_intr_setup:
+ spin_unlock(&legacy_q_add_lock);
+ return rv;
+}
+#endif
+
+int intr_ring_setup(struct xlnx_dma_dev *xdev)
+{
+ int num_entries = 0;
+ int counter = 0;
+ struct intr_coal_conf *intr_coal_list;
+ struct intr_coal_conf *intr_coal_list_entry;
+
+ if ((xdev->conf.qdma_drv_mode != INDIRECT_INTR_MODE) &&
+ (xdev->conf.qdma_drv_mode != AUTO_MODE)) {
+ pr_debug("skipping interrupt aggregation: driver is loaded in %s mode\n",
+ mode_name_list[xdev->conf.qdma_drv_mode].name);
+ xdev->intr_coal_list = NULL;
+ return 0;
+ }
+
+ /** For master_pf, vec0 and vec1 is used for
+ * error and user interrupts
+ * for other pfs, vec0 is used for user interrupts
+ */
+ if ((xdev->num_vecs != 0) &&
+ ((xdev->num_vecs - xdev->dvec_start_idx) < xdev->conf.qsets_max)) {
+ pr_debug("dev %s num_vectors[%d] < num_queues [%d]\n",
+ dev_name(&xdev->conf.pdev->dev),
+ xdev->num_vecs,
+ xdev->conf.qsets_max);
+ pr_debug("Enabling Interrupt aggregation\n");
+
+ /** obtain the number of queue entries
+ * in each inr_ring based on ring size
+ */
+ num_entries = ((xdev->conf.intr_rngsz + 1) * 512);
+
+ pr_debug("%s interrupt coalescing ring with %d entries\n",
+ dev_name(&xdev->conf.pdev->dev), num_entries);
+ /**
+ * Initially assuming that each vector has the same size of the
+ * ring, In practical it is possible to have different ring
+ * size of different vectors (?)
+ */
+ intr_coal_list = kzalloc(
+ sizeof(struct intr_coal_conf) *
+ QDMA_NUM_DATA_VEC_FOR_INTR_CXT,
+ GFP_KERNEL);
+ if (!intr_coal_list) {
+ pr_err("dev %s num_vecs %d OOM.\n",
+ dev_name(&xdev->conf.pdev->dev),
+ QDMA_NUM_DATA_VEC_FOR_INTR_CXT);
+ return -ENOMEM;
+ }
+
+ for (counter = 0;
+ counter < QDMA_NUM_DATA_VEC_FOR_INTR_CXT;
+ counter++) {
+ intr_coal_list_entry = (intr_coal_list + counter);
+ intr_coal_list_entry->intr_rng_num_entries =
+ num_entries;
+ intr_coal_list_entry->intr_ring_base = intr_ring_alloc(
+ xdev, num_entries,
+ sizeof(struct qdma_intr_ring),
+ &intr_coal_list_entry->intr_ring_bus);
+ if (!intr_coal_list_entry->intr_ring_base) {
+ pr_err("dev %s, sz %u, intr_desc ring OOM.\n",
+ xdev->conf.name,
+ intr_coal_list_entry->intr_rng_num_entries);
+ goto err_out;
+ }
+
+ intr_coal_list_entry->vec_id =
+ xdev->msix[counter + xdev->dvec_start_idx].entry;
+ intr_coal_list_entry->cidx = 0;
+ intr_coal_list_entry->color = 1;
+ pr_debug("ring_number = %d, vector_index = %d, ring_size = %d, ring_base = 0x%08x",
+ counter, intr_coal_list_entry->vec_id,
+ intr_coal_list_entry->intr_rng_num_entries,
+ (unsigned int)intr_coal_list_entry->intr_ring_bus);
+ }
+
+ pr_debug("dev %s interrupt coalescing ring setup successful\n",
+ dev_name(&xdev->conf.pdev->dev));
+
+ xdev->intr_coal_list = intr_coal_list;
+ } else {
+ pr_info("dev %s intr vec[%d] >= queues[%d], No aggregation\n",
+ dev_name(&xdev->conf.pdev->dev),
+ (xdev->num_vecs - xdev->dvec_start_idx),
+ xdev->conf.qsets_max);
+
+ xdev->intr_coal_list = NULL;
+ /* Fallback from indirect interrupt mode */
+ if (xdev->num_vecs != 0)
+ xdev->conf.qdma_drv_mode = DIRECT_INTR_MODE;
+ else
+ xdev->conf.qdma_drv_mode = POLL_MODE;
+ }
+ return 0;
+
+err_out:
+ while (--counter >= 0) {
+ intr_coal_list_entry = (intr_coal_list + counter);
+ intr_ring_free(xdev, intr_coal_list_entry->intr_rng_num_entries,
+ sizeof(struct qdma_intr_ring),
+ (u8 *)intr_coal_list_entry->intr_ring_base,
+ intr_coal_list_entry->intr_ring_bus);
+ }
+ kfree(intr_coal_list);
+ return -ENOMEM;
+}
+
+void intr_work(struct work_struct *work)
+{
+ struct qdma_descq *descq;
+
+ descq = container_of(work, struct qdma_descq, work);
+ qdma_descq_service_cmpl_update(descq, 0, 1);
+}
+
+/**
+ * qdma_queue_service - service the queue
+ * in the case of irq handler is registered by the user, the user should
+ * call qdma_queue_service() in its interrupt handler to service the queue
+ * @dev_hndl: hndl retured from qdma_device_open()
+ * @qhndl: hndl retured from qdma_queue_add()
+ */
+void qdma_queue_service(unsigned long dev_hndl, unsigned long id, int budget,
+ bool c2h_upd_cmpl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(xdev, id,
+ NULL, 0, 1);
+
+ if (descq)
+ qdma_descq_service_cmpl_update(descq, budget, c2h_upd_cmpl);
+}
+
+static u8 get_intr_vec_index(struct xlnx_dma_dev *xdev, u8 intr_type)
+{
+ u8 i = 0;
+
+ for (i = 0; i < xdev->num_vecs; i++) {
+ if (xdev->dev_intr_info_list[i].intr_vec_map.intr_type ==
+ intr_type)
+ return xdev->dev_intr_info_list[i].intr_vec_map.intr_vec_index;
+ }
+ return 0;
+}
+
+void qdma_err_intr_setup(struct xlnx_dma_dev *xdev, u8 rearm)
+{
+ u32 val = 0;
+ u8 err_intr_index = 0;
+ u8 i;
+
+ val = xdev->func_id;
+ err_intr_index = get_intr_vec_index(xdev, INTR_TYPE_ERROR);
+ val |= V_QDMA_C2H_ERR_INT_VEC(err_intr_index);
+
+ val |= (1 << S_QDMA_C2H_ERR_INT_F_ERR_INT_ARM);
+
+ __write_reg(xdev, QDMA_C2H_ERR_INT, val);
+
+ if (rearm)
+ return;
+
+ pr_debug("Error interrupt setup: val = 0x%08x, readback = 0x%08x err_intr_index = %d func_id = %d\n",
+ val, __read_reg(xdev, QDMA_C2H_ERR_INT),
+ err_intr_index, xdev->func_id);
+
+ for (i = 0; i < HW_ERRS; i++)
+ qdma_enable_hw_err(xdev, i);
+}
+
+
+void qdma_enable_hw_err(struct xlnx_dma_dev *xdev, u8 hw_err_type)
+{
+
+ switch (hw_err_type) {
+ case GLBL_ERR:
+ case GLBL_DSC_ERR:
+ case GLBL_TRQ_ERR:
+ case C2H_ERR:
+ case C2H_FATAL_ERR:
+ case H2C_ERR:
+ case ECC_SB_ERR:
+ case ECC_DB_ERR:
+ break;
+ default:
+ hw_err_type = 0;
+ break;
+ }
+
+ __write_reg(xdev,
+ err_stat_info[hw_err_type].mask_reg_addr,
+ err_stat_info[hw_err_type].err_info.intr_mask);
+ pr_info("%s interrupts enabled: reg -> 0x%08x, value = 0x%08x\n",
+ err_stat_info[hw_err_type].err_name,
+ err_stat_info[hw_err_type].mask_reg_addr,
+ __read_reg(xdev, err_stat_info[hw_err_type].mask_reg_addr));
+}
+
+
+int get_intr_ring_index(struct xlnx_dma_dev *xdev, u32 vector_index)
+{
+ int ring_index = 0;
+
+ ring_index = (vector_index - xdev->dvec_start_idx) +
+ (xdev->func_id * QDMA_NUM_DATA_VEC_FOR_INTR_CXT);
+ pr_debug("func_id = %d, vector_index = %d, ring_index = %d\n",
+ xdev->func_id, vector_index, ring_index);
+
+ return ring_index;
+}
+
+void intr_legacy_init(void)
+{
+#ifndef __QDMA_VF__
+ spin_lock_init(&legacy_q_add_lock);
+#endif
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_intr.h b/QDMA/linux-kernel/libqdma/qdma_intr.h
new file mode 100644
index 0000000000000000000000000000000000000000..da407653f25397efcaae1d555ea49b9f78ccc406
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_intr.h
@@ -0,0 +1,217 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef LIBQDMA_QDMA_INTR_H_
+#define LIBQDMA_QDMA_INTR_H_
+/**
+ * @file
+ * @brief This file contains the declarations for qdma dev interrupt handlers
+ *
+ */
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include "qdma_descq.h"
+/**
+ * forward declaration for xlnx_dma_dev
+ */
+struct xlnx_dma_dev;
+
+/**
+ * @struct - qdma_intr_ring
+ * @brief Interrupt ring entry definition
+ */
+struct qdma_intr_ring {
+ /** producer index. This is from Interrupt source.
+ * Cumulative pointer of total interrupt Aggregation
+ * Ring entry written
+ */
+ __be64 pidx:16;
+ /** consumer index. This is from Interrupt source.
+ * Cumulative consumed pointer
+ */
+ __be64 cidx:16;
+ /** source color. This is from Interrupt source.
+ * This bit inverts every time pidx wraps around
+ * and this field gets copied to color field of descriptor.
+ */
+ __be64 s_color:1;
+ /** This is from Interrupt source.
+ * Interrupt state, 0: CMPT_INT_ISR; 1: CMPT_INT_TRIG; 2: CMPT_INT_ARMED
+ */
+ __be64 intr_satus:2;
+ /** error. This is from interrupt source
+ * {C2h_err[1:0], h2c_err[1:0]}
+ */
+ __be64 error:2;
+ /** 1 reserved bits*/
+ __be64 rsvd:1;
+ /** interrupt type, 0: H2C; 1: C2H*/
+ __be64 intr_type:1;
+ /** This is from Interrupt source. Queue ID*/
+ __be64 qid:24;
+ /** The color bit of the Interrupt Aggregation Ring.
+ * This bit inverts every time pidx wraps around on the
+ * Interrupt Aggregation Ring.
+ */
+ __be64 coal_color:1;
+};
+
+/*****************************************************************************/
+/**
+ * intr_teardown() - un register the interrupts for the device
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return none
+ *****************************************************************************/
+void intr_teardown(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * intr_setup() - register the interrupts for the device
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int intr_setup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * intr_ring_teardown() - delete the interrupt ring
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return none
+ *****************************************************************************/
+void intr_ring_teardown(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * intr_context_setup() - set up the interrupt context
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int intr_context_setup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * intr_ring_setup() - create the interrupt ring
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int intr_ring_setup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * intr_legacy_init() - legacy interrupt init
+ *
+ *****************************************************************************/
+void intr_legacy_init(void);
+
+/*****************************************************************************/
+/**
+ * intr_legacy_setup() - setup the legacy interrupt handler
+ *
+ * @param[in] descq: descq on which the interrupt needs to be setup
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int intr_legacy_setup(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * intr_legacy_clear() - clear the legacy interrupt handler
+ *
+ * @param[in] descq: descq on which the interrupt needs to be cleared
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+void intr_legacy_clear(struct qdma_descq *descq);
+
+
+/*****************************************************************************/
+/**
+ * intr_work() - attach the top half for the interrupt
+ *
+ * @param[in] work: pointer to struct work_struct
+ *
+ * @return none
+ *****************************************************************************/
+void intr_work(struct work_struct *work);
+
+/*****************************************************************************/
+/**
+ * qdma_err_intr_setup() - set up the error interrupt
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] rearm: flag to control the error interrupt arming
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_err_intr_setup(struct xlnx_dma_dev *xdev, u8 rearm);
+
+/*****************************************************************************/
+/**
+ * qdma_enable_hw_err() - enable the hw errors
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] hw_err_type: hw error type
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_enable_hw_err(struct xlnx_dma_dev *xdev, u8 hw_err_type);
+
+/*****************************************************************************/
+/**
+ * get_intr_ring_index() - get the interrupt ring index based on vector index
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] vector_index: vector index
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int get_intr_ring_index(struct xlnx_dma_dev *xdev, u32 vector_index);
+
+#ifndef __QDMA_VF__
+#ifdef ERR_DEBUG
+/*****************************************************************************/
+/**
+ * err_stat_handler() - error interrupt handler
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return none
+ *****************************************************************************/
+void err_stat_handler(struct xlnx_dma_dev *xdev);
+#endif
+#endif
+
+#endif /* LIBQDMA_QDMA_DEVICE_H_ */
+
diff --git a/QDMA/linux-kernel/libqdma/qdma_mbox.c b/QDMA/linux-kernel/libqdma/qdma_mbox.c
new file mode 100644
index 0000000000000000000000000000000000000000..ccfd3a9d8e9603c2efc5a6ddb2a6a6503c7f1e53
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_mbox.c
@@ -0,0 +1,762 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/version.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/jiffies.h>
+#include <linux/timer.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+
+#include "qdma_compat.h"
+#include "xdev.h"
+#include "qdma_device.h"
+#include "qdma_regs.h"
+#include "qdma_context.h"
+#include "qdma_mbox.h"
+#include "qdma_qconf_mgr.h"
+
+#define MBOX_TIMER_INTERVAL (1)
+
+/*
+ * mailbox opcode string
+ */
+static const char *mbox_op_str[MBOX_OP_MAX] = {
+ "noop",
+ "bye",
+ "hello",
+
+ "fmap",
+ "csr",
+ "intr_ctrx",
+ "qctrx_wrt",
+ "qctrx_rd",
+ "qctrx_clr",
+ "qctrx_inv",
+
+ "qconf",
+ "rsvd_0x0b",
+ "rsvd_0x0c",
+ "rsvd_0x0d",
+ "rsvd_0x0e",
+ "rsvd_0x0f",
+ "rsvd_0x10",
+ "rsvd_0x11",
+
+ "hello_resp",
+ "fmap_resp",
+ "csr_resp",
+ "intr_ctrx_resp",
+ "qctrx_wrt_resp",
+ "qctrx_rd_resp",
+ "qctrx_clr_resp",
+ "qctrx_inv_resp",
+ "qconf_resp",
+};
+
+/*
+ * mailbox h/w registers access
+ */
+
+#ifndef __QDMA_VF__
+static inline void mbox_pf_hw_clear_func_ack(struct xlnx_dma_dev *xdev,
+ u8 func_id)
+{
+ int idx = func_id / 32; /* bitmask, u32 reg */
+ int bit = func_id % 32;
+
+ /* clear the function's ack status */
+ __write_reg(xdev,
+ MBOX_BASE + MBOX_PF_ACK_BASE + idx * MBOX_PF_ACK_STEP,
+ (1 << bit));
+}
+
+static void mbox_pf_hw_clear_ack(struct xlnx_dma_dev *xdev)
+{
+ u32 v = __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS);
+ u32 reg = MBOX_BASE + MBOX_PF_ACK_BASE;
+ int i;
+
+ if ((v & F_MBOX_FN_STATUS_ACK) == 0)
+ return;
+
+ for (i = 0; i < MBOX_PF_ACK_COUNT; i++, reg += MBOX_PF_ACK_STEP) {
+ u32 v = __read_reg(xdev, reg);
+
+ if (!v)
+ continue;
+
+ /* clear the ack status */
+ pr_debug("%s, PF_ACK %d, 0x%x.\n", xdev->conf.name, i, v);
+ __write_reg(xdev, reg, v);
+ }
+}
+#endif
+
+static int mbox_hw_send(struct qdma_mbox *mbox, struct mbox_msg *m)
+{
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ struct mbox_msg_hdr *hdr = &m->hdr;
+ u32 fn_id = hdr->dst;
+ int i;
+ u32 reg = MBOX_OUT_MSG_BASE;
+ u32 v;
+ int rv = -EAGAIN;
+
+ pr_debug("%s, dst 0x%x, op 0x%x, %s, status reg 0x%x.\n",
+ xdev->conf.name, fn_id, m->hdr.op, mbox_op_str[m->hdr.op],
+ __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS));
+
+ spin_lock_bh(&mbox->hw_tx_lock);
+
+#ifndef __QDMA_VF__
+ __write_reg(xdev, MBOX_BASE + MBOX_FN_TARGET,
+ V_MBOX_FN_TARGET_ID(fn_id));
+#endif
+
+ v = __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS);
+ if (v & F_MBOX_FN_STATUS_OUT_MSG) {
+ pr_debug("%s, func 0x%x, outgoing message busy, 0x%x.\n",
+ xdev->conf.name, fn_id, v);
+ goto unlock;
+ }
+
+ for (i = 0; i < MBOX_MSG_REG_MAX; i++, reg += MBOX_MSG_STEP)
+ __write_reg(xdev, MBOX_BASE + reg, m->raw[i]);
+
+ pr_debug("%s, send op 0x%x,%s, src 0x%x, dst 0x%x, s 0x%x:\n",
+ xdev->conf.name, m->hdr.op, mbox_op_str[m->hdr.op], m->hdr.src,
+ m->hdr.dst, m->hdr.status);
+#if 0
+ print_hex_dump(KERN_INFO, "mbox snd: ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)m, 64, false);
+#endif
+
+#ifndef __QDMA_VF__
+ /* clear the outgoing ack */
+ mbox_pf_hw_clear_func_ack(xdev, fn_id);
+#endif
+
+ __write_reg(xdev, MBOX_BASE + MBOX_FN_CMD, F_MBOX_FN_CMD_SND);
+ rv = 0;
+
+unlock:
+ spin_unlock_bh(&mbox->hw_tx_lock);
+
+ return rv;
+}
+
+static int mbox_hw_rcv(struct qdma_mbox *mbox, struct mbox_msg *m)
+{
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ u32 reg = MBOX_IN_MSG_BASE;
+ u32 v = 0;
+ int all_zero_msg = 1;
+ int i;
+ int rv = -EAGAIN;
+#ifndef __QDMA_VF__
+ unsigned int from_id = 0;
+#endif
+
+ spin_lock_bh(&mbox->hw_rx_lock);
+
+ v = __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS);
+
+#if 0
+ if ((v & MBOX_FN_STATUS_MASK))
+ pr_debug("%s, base 0x%x, status 0x%x.\n",
+ xdev->conf.name, MBOX_BASE, v);
+#endif
+
+ if (!(v & M_MBOX_FN_STATUS_IN_MSG))
+ goto unlock;
+
+#ifndef __QDMA_VF__
+ from_id = G_MBOX_FN_STATUS_SRC(v);
+ __write_reg(xdev, MBOX_BASE + MBOX_FN_TARGET, from_id);
+#endif
+
+ for (i = 0; i < MBOX_MSG_REG_MAX; i++, reg += MBOX_MSG_STEP) {
+ m->raw[i] = __read_reg(xdev, MBOX_BASE + reg);
+ /* if rcv'ed message is all zero, stop and disable the mbox,
+ * the h/w mbox is not working properly
+ */
+ if (m->raw[i])
+ all_zero_msg = 0;
+ }
+ if (all_zero_msg) {
+ rv = -EPIPE;
+ goto unlock;
+ }
+
+ pr_debug("%s, rcv op 0x%x, %s, src 0x%x, dst 0x%x, s 0x%x:\n",
+ xdev->conf.name, m->hdr.op, mbox_op_str[m->hdr.op], m->hdr.src,
+ m->hdr.dst, m->hdr.status);
+#if 0
+ print_hex_dump(KERN_INFO, "mbox rcv: ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)m, 64, false);
+#endif
+
+#ifndef __QDMA_VF__
+ if (from_id != m->hdr.src) {
+ pr_debug("%s, src 0x%x -> func_id 0x%x.\n",
+ xdev->conf.name, m->hdr.src, from_id);
+ m->hdr.src = from_id;
+ }
+#endif
+
+ /* ack'ed the sender */
+ __write_reg(xdev, MBOX_BASE + MBOX_FN_CMD, F_MBOX_FN_CMD_RCV);
+ rv = 0;
+
+unlock:
+ spin_unlock_bh(&mbox->hw_rx_lock);
+ return rv;
+}
+
+/*
+ * mbox tx message processing
+ */
+static void mbox_msg_destroy(struct kref *kref)
+{
+ struct mbox_msg *m = container_of(kref, struct mbox_msg, refcnt);
+
+ kfree(m);
+}
+
+void __qdma_mbox_msg_free(const char *f, struct mbox_msg *m)
+{
+ kref_put(&m->refcnt, mbox_msg_destroy);
+}
+
+struct mbox_msg *qdma_mbox_msg_alloc(struct xlnx_dma_dev *xdev,
+ enum mbox_msg_op op)
+{
+ struct mbox_msg *m = kmalloc(sizeof(struct mbox_msg), GFP_KERNEL);
+ struct mbox_msg_hdr *hdr;
+
+ if (!m) {
+ pr_info("%s OOM, %lu, op 0x%x, %s.\n",
+ xdev->conf.name, sizeof(struct mbox_msg), op,
+ mbox_op_str[op]);
+ return NULL;
+ }
+
+ memset(m, 0, sizeof(struct mbox_msg));
+
+ kref_init(&m->refcnt);
+ qdma_waitq_init(&m->waitq);
+
+ hdr = &m->hdr;
+ hdr->op = op;
+ hdr->src = xdev->func_id;
+#ifdef __QDMA_VF__
+ hdr->dst = xdev->func_id_parent;
+#endif
+
+ return m;
+}
+
+void qdma_mbox_msg_cancel(struct xlnx_dma_dev *xdev, struct mbox_msg *m)
+{
+ struct qdma_mbox *mbox = &xdev->mbox;
+
+ /* delete from mbox list */
+ spin_lock_bh(&mbox->list_lock);
+ list_del(&m->list);
+ spin_unlock_bh(&mbox->list_lock);
+}
+
+int qdma_mbox_msg_send(struct xlnx_dma_dev *xdev, struct mbox_msg *m,
+ bool wait_resp, u8 resp_op, unsigned int timeout_ms)
+{
+ struct qdma_mbox *mbox = &xdev->mbox;
+ struct mbox_msg_hdr *hdr = &m->hdr;
+
+
+ m->wait_resp = wait_resp ? 1 : 0;
+ m->wait_op = resp_op;
+
+
+ if (unlikely(xlnx_dma_device_flag_check(mbox->xdev, XDEV_FLAG_OFFLINE))
+ && (hdr->op != MBOX_OP_BYE))
+ /* allow BYE message even if the device is going down */
+ return -ENODEV;
+
+ /* queue up to ensure order */
+ spin_lock_bh(&mbox->list_lock);
+ list_add_tail(&m->list, &mbox->tx_todo_list);
+ spin_unlock_bh(&mbox->list_lock);
+
+ /* kick start the tx */
+ queue_work(mbox->workq, &mbox->tx_work);
+
+ if (!wait_resp)
+ return 0;
+
+ qdma_waitq_wait_event_timeout(m->waitq, hdr->op == resp_op,
+ msecs_to_jiffies(timeout_ms));
+
+ if (hdr->op != resp_op) {
+ /* delete from mbox list */
+ spin_lock_bh(&mbox->list_lock);
+ list_del(&m->list);
+ spin_unlock_bh(&mbox->list_lock);
+
+ pr_err("%s mbox timed out. op 0x%x, %s, timeout %u ms.\n",
+ xdev->conf.name, hdr->op, mbox_op_str[hdr->op],
+ timeout_ms);
+
+ return -EPIPE;
+ }
+
+ if (hdr->status) {
+ pr_err("%s mbox msg op failed %d. op 0x%x, %s.\n",
+ xdev->conf.name, hdr->status, hdr->op,
+ mbox_op_str[hdr->op]);
+ return hdr->status;
+ }
+
+ return 0;
+}
+
+/*
+ * mbox rx message processing
+ */
+#ifdef __QDMA_VF__
+
+void dump_rx_pend_list(struct qdma_mbox *mbox)
+{
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ struct mbox_msg *_msg = NULL, *_tmp = NULL;
+
+ spin_lock_bh(&mbox->list_lock);
+ list_for_each_entry_safe(_msg, _tmp, &mbox->rx_pend_list, list) {
+ pr_debug("m_snd(%u) : wait_op/src/dst = %x/%u/%u\n",
+ xdev->func_id, _msg->wait_op,
+ _msg->hdr.src, _msg->hdr.dst);
+ }
+ spin_unlock_bh(&mbox->list_lock);
+}
+
+static int mbox_rcv_one_msg(struct qdma_mbox *mbox)
+{
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ struct mbox_msg *m_rcv = &mbox->rx;
+ struct mbox_msg_hdr *hdr_rcv = &m_rcv->hdr;
+ struct mbox_msg *m_snd = NULL;
+
+ u8 op = hdr_rcv->op;
+
+ if (xdev->func_id == xdev->func_id_parent) {
+ /* fill in VF's func_id */
+ xdev->func_id = hdr_rcv->dst;
+ xdev->func_id_parent = hdr_rcv->src;
+ }
+
+ spin_lock_bh(&mbox->list_lock);
+ if (!list_empty(&mbox->rx_pend_list))
+ m_snd = list_first_entry(&mbox->rx_pend_list, struct mbox_msg,
+ list);
+ spin_unlock_bh(&mbox->list_lock);
+
+
+ if (!m_snd || op != m_snd->wait_op) {
+ if (op != MBOX_OP_HELLO_RESP) {
+ pr_err("%s: unexpected op 0x%x, %s.\n",
+ xdev->conf.name, hdr_rcv->op,
+ mbox_op_str[hdr_rcv->op]);
+ if (m_snd)
+ pr_err("m_snd : wait_op/src/dst = %x/%u/%u\n",
+ m_snd->wait_op,
+ m_snd->hdr.src,
+ m_snd->hdr.dst);
+ dump_rx_pend_list(mbox);
+ }
+ return 0;
+ }
+
+ /* a matching request is found */
+ spin_lock_bh(&mbox->list_lock);
+ list_del(&m_snd->list);
+ spin_unlock_bh(&mbox->list_lock);
+
+ memcpy(m_snd->raw, m_rcv->raw, sizeof(u32) * MBOX_MSG_REG_MAX);
+ /* wake up anyone waiting on the response */
+ qdma_waitq_wakeup(&m_snd->waitq);
+
+ return 0;
+}
+
+#else
+static int mbox_rcv_one_msg(struct qdma_mbox *mbox)
+{
+ struct mbox_msg *m = &mbox->rx;
+ struct mbox_msg_hdr *hdr = &m->hdr;
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ u8 op = hdr->op;
+ struct mbox_msg *m_resp;
+ struct mbox_msg_hdr *hdr_resp;
+ int rv = 0;
+ struct qconf_entry *entry = NULL;
+
+ pr_debug("%s, src 0x%x op 0x%x,%s.\n",
+ xdev->conf.name, hdr->src, hdr->op, mbox_op_str[hdr->op]);
+
+ switch (op) {
+ case MBOX_OP_BYE:
+ {
+ unsigned int qbase = 0;
+ unsigned int qmax = 0;
+
+ hw_read_fmap(xdev, hdr->src, &qbase, &qmax);
+ hw_init_qctxt_memory(xdev, qbase, qmax);
+
+ hw_set_fmap(xdev, hdr->src, 0, 0);
+ xdev_sriov_vf_offline(xdev, hdr->src);
+
+ pr_debug("%s: clear 0x%x FMAP, Q 0x%x+0x%x.\n",
+ xdev->conf.name, hdr->src, qbase, qmax);
+
+ xdev_destroy_qconf(PCI_TYPE_VF, hdr->src);
+ /* no response needed */
+ return 0;
+ }
+ break;
+ case MBOX_OP_HELLO:
+ {
+ xdev_sriov_vf_online(xdev, hdr->src);
+ entry = xdev_create_qconf(PCI_TYPE_VF, hdr->src);
+ if (entry) {
+ int rsp_offset =
+ sizeof(struct mbox_msg_hdr)/sizeof(u32);
+ m->raw[rsp_offset] = entry->qmax;
+ m->raw[rsp_offset + 1] = entry->qbase;
+ pr_debug("qmax = %u qbase = %u m_resp->raw[%d] = %u, m_resp->raw[%d] = %u\n",
+ entry->qmax, entry->qbase,
+ rsp_offset, m->raw[rsp_offset],
+ rsp_offset + 1, m->raw[rsp_offset + 1]);
+ }
+ }
+ break;
+ case MBOX_OP_FMAP:
+ {
+ struct mbox_msg_fmap *fmap = &m->fmap;
+ int ret = xdev_set_qmax(PCI_TYPE_VF, hdr->src,
+ fmap->qmax, &fmap->qbase);
+ if (ret < 0) {
+ rv = ret;
+ pr_err("VF qmax set failed %s: set 0x%x QCONF, Q 0x%x+0x%x. ret = %d\n",
+ xdev->conf.name, hdr->src,
+ fmap->qbase, fmap->qmax, ret);
+ } else {
+ pr_debug("VF qmax set success %s: set 0x%x QCONF, Q 0x%x+0x%x.\n",
+ xdev->conf.name, hdr->src,
+ fmap->qbase, fmap->qmax);
+
+ pr_debug("%s: set 0x%x FMAP, Q 0x%x+0x%x.\n",
+ xdev->conf.name, hdr->src, fmap->qbase,
+ fmap->qmax);
+
+ hw_set_fmap(xdev, hdr->src, fmap->qbase, fmap->qmax);
+
+ xdev_sriov_vf_fmap(xdev, hdr->src, fmap->qbase,
+ fmap->qmax);
+ }
+ }
+ break;
+ case MBOX_OP_QCONF:
+ {
+ struct mbox_msg_fmap *fmap = &m->fmap;
+
+ int ret = xdev_set_qmax(PCI_TYPE_VF, hdr->src,
+ fmap->qmax, &fmap->qbase);
+ if (ret < 0) {
+ rv = ret;
+ pr_err("VF qmax set failed %s: set 0x%x QCONF, Q 0x%x+0x%x.\n",
+ xdev->conf.name, hdr->src,
+ fmap->qbase, fmap->qmax);
+ } else {
+ pr_debug("VF qmax set success %s: set 0x%x QCONF, Q 0x%x+0x%x.\n",
+ xdev->conf.name, hdr->src,
+ fmap->qbase, fmap->qmax);
+ }
+ }
+ break;
+
+ case MBOX_OP_CSR:
+ {
+ struct mbox_msg_csr *csr = &m->csr;
+
+ rv = qdma_csr_read(xdev, &csr->csr_info, 0);
+ }
+ break;
+ case MBOX_OP_INTR_CTXT:
+ {
+ struct mbox_msg_intr_ctxt *ictxt = &m->intr_ctxt;
+
+ pr_debug("%s, rcv 0x%x INTR_CTXT, programing the context\n",
+ xdev->conf.name, hdr->src);
+ rv = qdma_prog_intr_context(xdev, ictxt);
+ }
+ break;
+ case MBOX_OP_QCTXT_CLR:
+ {
+ struct mbox_msg_qctxt *qctxt = &m->qctxt;
+
+ rv = qdma_descq_context_clear(xdev, qctxt->qid,
+ qctxt->st, qctxt->c2h, 0);
+ }
+ break;
+ case MBOX_OP_QCTXT_RD:
+ {
+ struct mbox_msg_qctxt *qctxt = &m->qctxt;
+
+ rv = qdma_descq_context_read(xdev, qctxt->qid,
+ qctxt->st, qctxt->c2h,
+ &qctxt->context);
+ }
+ break;
+ case MBOX_OP_QCTXT_WRT:
+ {
+ struct mbox_msg_qctxt *qctxt = &m->qctxt;
+
+ pr_debug("%s, rcv 0x%x QCTXT_WRT, qid 0x%x.\n",
+ xdev->conf.name, hdr->src, qctxt->qid);
+
+ /* always clear the context first */
+ rv = qdma_descq_context_clear(xdev, qctxt->qid,
+ qctxt->st, qctxt->c2h, 1);
+ if (rv < 0) {
+ pr_err("%s, 0x%x QCTXT_WRT, qid 0x%x, clr failed %d.\n",
+ xdev->conf.name, hdr->src, qctxt->qid,
+ rv);
+ break;
+ }
+
+ rv = qdma_descq_context_program(xdev, qctxt->qid,
+ qctxt->st, qctxt->c2h,
+ &qctxt->context);
+ }
+ break;
+ default:
+ pr_info("%s: rcv mbox UNKNOWN op 0x%x.\n",
+ xdev->conf.name, hdr->op);
+ print_hex_dump(KERN_INFO, "mbox rcv: ",
+ DUMP_PREFIX_OFFSET, 16, 1, (void *)hdr,
+ 64, false);
+ return -EINVAL;
+ break;
+ }
+
+ /* respond */
+ m_resp = qdma_mbox_msg_alloc(xdev, op);
+ if (!m_resp)
+ return -ENOMEM;
+
+ hdr_resp = &m_resp->hdr;
+
+ memcpy(m_resp->raw, m->raw, sizeof(u32) * MBOX_MSG_REG_MAX);
+
+ hdr_resp->op |= MBOX_MSG_OP_PF_MASK;
+ hdr_resp->dst = hdr->src;
+ hdr_resp->src = xdev->func_id;
+
+ hdr_resp->status = rv ? -1 : 0;
+
+ qdma_mbox_msg_send(xdev, m_resp, 0, 0, 0);
+
+ return rv;
+}
+#endif
+
+/*
+ * tx & rx workqueue handler
+ */
+static void mbox_tx_work(struct work_struct *work)
+{
+ struct qdma_mbox *mbox = container_of(work, struct qdma_mbox, tx_work);
+
+ spin_lock_bh(&mbox->list_lock);
+ while (!list_empty(&mbox->tx_todo_list)) {
+ struct mbox_msg *m = list_first_entry(&mbox->tx_todo_list,
+ struct mbox_msg, list);
+ if (mbox_hw_send(mbox, m) == 0) {
+ /* message sent */
+ list_del(&m->list);
+
+ /* response needed */
+ if (m->wait_resp)
+ list_add_tail(&m->list, &mbox->rx_pend_list);
+ else
+ qdma_mbox_msg_free(m);
+ } else
+ break;
+ }
+ spin_unlock_bh(&mbox->list_lock);
+}
+
+static inline void mbox_timer_stop(struct qdma_mbox *mbox)
+{
+ del_timer(&mbox->timer);
+}
+
+static inline void mbox_timer_start(struct qdma_mbox *mbox)
+{
+ struct timer_list *timer = &mbox->timer;
+
+ qdma_timer_start(timer, MBOX_TIMER_INTERVAL);
+}
+
+static void mbox_rx_work(struct work_struct *work)
+{
+ struct qdma_mbox *mbox = container_of(work, struct qdma_mbox, rx_work);
+ struct xlnx_dma_dev *xdev = mbox->xdev;
+ struct mbox_msg *m = &mbox->rx;
+ int rv;
+
+#ifndef __QDMA_VF__
+ /* clear the ack status */
+ mbox_pf_hw_clear_ack(mbox->xdev);
+#endif
+
+ rv = mbox_hw_rcv(mbox, m);
+ while (rv == 0) {
+ if (unlikely(xlnx_dma_device_flag_check(xdev,
+ XDEV_FLAG_OFFLINE)))
+ break;
+ else if (mbox_rcv_one_msg(mbox) == -EINVAL)
+ break;
+
+ rv = mbox_hw_rcv(mbox, m);
+ }
+
+ if (rv == -EPIPE) {
+ pr_info("%s: rcv'ed all zero mbox msg, status 0x%x=0x%x. disable mbox processing.\n",
+ xdev->conf.name, MBOX_BASE + MBOX_FN_STATUS,
+ __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS));
+ mbox_timer_stop(mbox);
+ } else if (xlnx_dma_device_flag_check(xdev, XDEV_FLAG_OFFLINE))
+ mbox_timer_stop(mbox);
+ else
+ mbox_timer_start(mbox);
+}
+
+/*
+ * non-interrupt mode: use timer for periodic checking of new messages
+ */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)
+static void mbox_rx_timer_handler(struct timer_list *t)
+#else
+static void mbox_rx_timer_handler(unsigned long arg)
+#endif
+{
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)
+ struct qdma_mbox *mbox = from_timer(mbox, t, timer);
+#else
+ struct qdma_mbox *mbox = (struct qdma_mbox *)arg;
+#endif
+
+ queue_work(mbox->workq, &mbox->rx_work);
+}
+
+/*
+ * mailbox initialization and cleanup
+ */
+void qdma_mbox_stop(struct xlnx_dma_dev *xdev)
+{
+ mbox_timer_stop(&xdev->mbox);
+}
+
+void qdma_mbox_start(struct xlnx_dma_dev *xdev)
+{
+ mbox_timer_start(&xdev->mbox);
+}
+
+void qdma_mbox_cleanup(struct xlnx_dma_dev *xdev)
+{
+ struct qdma_mbox *mbox = &xdev->mbox;
+
+ mbox_timer_stop(mbox);
+
+ if (mbox->workq) {
+ flush_workqueue(mbox->workq);
+ destroy_workqueue(mbox->workq);
+ }
+}
+
+int qdma_mbox_init(struct xlnx_dma_dev *xdev)
+{
+ struct qdma_mbox *mbox = &xdev->mbox;
+ struct timer_list *timer = &mbox->timer;
+ struct mbox_msg m;
+ char name[80];
+
+ /* ack any received messages in the Q */
+#ifdef __QDMA_VF__
+ u32 v;
+
+ v = __read_reg(xdev, MBOX_BASE + MBOX_FN_STATUS);
+ if (!(v & M_MBOX_FN_STATUS_IN_MSG))
+ __write_reg(xdev, MBOX_BASE + MBOX_FN_CMD, F_MBOX_FN_CMD_RCV);
+#elif defined(CONFIG_PCI_IOV)
+ mbox_pf_hw_clear_ack(xdev);
+#endif
+
+ mbox->xdev = xdev;
+
+ spin_lock_init(&mbox->lock);
+ spin_lock_init(&mbox->list_lock);
+ INIT_LIST_HEAD(&mbox->tx_todo_list);
+ INIT_LIST_HEAD(&mbox->rx_pend_list);
+ INIT_WORK(&mbox->tx_work, mbox_tx_work);
+ INIT_WORK(&mbox->rx_work, mbox_rx_work);
+
+ snprintf(name, 80, "%s_mbox_wq", xdev->conf.name);
+ mbox->workq = create_singlethread_workqueue(name);
+
+ if (!mbox->workq) {
+ pr_info("%s OOM, mbox workqueue.\n", xdev->conf.name);
+ goto cleanup;
+ }
+
+ /* read & discard whatever in the incoming message buffer */
+#ifdef __QDMA_VF__
+ mbox_hw_rcv(mbox, &m);
+#else
+{
+ int i;
+
+ for (i = 0; i < 256; i++)
+ mbox_hw_rcv(mbox, &m);
+}
+#endif
+
+ qdma_timer_setup(timer, mbox_rx_timer_handler, mbox);
+
+ return 0;
+
+cleanup:
+ qdma_mbox_cleanup(xdev);
+ return -ENOMEM;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_mbox.h b/QDMA/linux-kernel/libqdma/qdma_mbox.h
new file mode 100644
index 0000000000000000000000000000000000000000..fbd2035fcb70bee49904e5c256b70eac13cee1fb
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_mbox.h
@@ -0,0 +1,415 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_MBOX_H__
+#define __QDMA_MBOX_H__
+
+#include "qdma_compat.h"
+#include "qdma_device.h"
+
+/**
+ * @file
+ * @brief This file contains the declarations for qdma mailbox apis
+ *
+ */
+/**
+ * mailbox registers
+ */
+#ifdef __QDMA_VF__
+#define MBOX_BASE 0x1000
+#else
+#define MBOX_BASE 0x2400
+#endif
+
+/** mailbox function status */
+#define MBOX_FN_STATUS 0x0
+/** shift value for mailbox function status in msg */
+#define S_MBOX_FN_STATUS_IN_MSG 0
+/** mask value for mailbox function status in msg*/
+#define M_MBOX_FN_STATUS_IN_MSG 0x1
+/** face value for mailbox function status in msg */
+#define F_MBOX_FN_STATUS_IN_MSG 0x1
+
+/** shift value for out msg */
+#define S_MBOX_FN_STATUS_OUT_MSG 1
+/** mask value for out msg */
+#define M_MBOX_FN_STATUS_OUT_MSG 0x1
+/** face value for out msg */
+#define F_MBOX_FN_STATUS_OUT_MSG (1 << S_MBOX_FN_STATUS_OUT_MSG)
+/** shift value for status ack */
+#define S_MBOX_FN_STATUS_ACK 2 /* PF only, ack status */
+/** mask value for status ack */
+#define M_MBOX_FN_STATUS_ACK 0x1
+/** face value for status ack */
+#define F_MBOX_FN_STATUS_ACK (1 << S_MBOX_FN_STATUS_ACK)
+/** shift value for status src */
+#define S_MBOX_FN_STATUS_SRC 4 /* PF only, source func.*/
+/** mask value for status src */
+#define M_MBOX_FN_STATUS_SRC 0xFFF
+/** face value for status src */
+#define G_MBOX_FN_STATUS_SRC(x) \
+ (((x) >> S_MBOX_FN_STATUS_SRC) & M_MBOX_FN_STATUS_SRC)
+/** face value for mailbox function status */
+#define MBOX_FN_STATUS_MASK \
+ (F_MBOX_FN_STATUS_IN_MSG | \
+ F_MBOX_FN_STATUS_OUT_MSG | \
+ F_MBOX_FN_STATUS_ACK)
+
+/** mailbox function commands register */
+#define MBOX_FN_CMD 0x4
+/** shift value for send command */
+#define S_MBOX_FN_CMD_SND 0
+/** mask value for send command */
+#define M_MBOX_FN_CMD_SND 0x1
+/** face value for send command */
+#define F_MBOX_FN_CMD_SND (1 << S_MBOX_FN_CMD_SND)
+/** shift value for receive command */
+#define S_MBOX_FN_CMD_RCV 1
+/** mask value for receive command */
+#define M_MBOX_FN_CMD_RCV 0x1
+/** face value for receive command */
+#define F_MBOX_FN_CMD_RCV (1 << S_MBOX_FN_CMD_RCV)
+/** shift value for vf reset */
+#define S_MBOX_FN_CMD_VF_RESET 3 /* TBD PF only: reset VF */
+/** mask value for vf reset */
+#define M_MBOX_FN_CMD_VF_RESET 0x1
+/** mailbox isr vector register */
+#define MBOX_ISR_VEC 0x8
+/** shift value for isr vector */
+#define S_MBOX_ISR_VEC 0
+/** mask value for isr vector */
+#define M_MBOX_ISR_VEC 0x1F
+/** face value for isr vector */
+#define V_MBOX_ISR_VEC(x) ((x) & M_MBOX_ISR_VEC)
+/** mailbox FN target register */
+#define MBOX_FN_TARGET 0xC
+/** shift value for FN target id */
+#define S_MBOX_FN_TARGET_ID 0
+/** mask value for FN target id */
+#define M_MBOX_FN_TARGET_ID 0xFFF
+/** face value for FN target id */
+#define V_MBOX_FN_TARGET_ID(x) ((x) & M_MBOX_FN_TARGET_ID)
+/** mailbox isr enable register */
+#define MBOX_ISR_EN 0x10
+/** shift value for isr enable */
+#define S_MBOX_ISR_EN 0
+/** mask value for isr enable */
+#define M_MBOX_ISR_EN 0x1
+/** face value for isr enable */
+#define F_MBOX_ISR_EN 0x1
+/** pf acknowledge base */
+#define MBOX_PF_ACK_BASE 0x20
+/** pf acknowledge step */
+#define MBOX_PF_ACK_STEP 4
+/** pf acknowledge count */
+#define MBOX_PF_ACK_COUNT 8
+/** mailbox incoming msg base */
+#define MBOX_IN_MSG_BASE 0x800
+/** mailbox outgoing msg base */
+#define MBOX_OUT_MSG_BASE 0xc00
+/** mailbox msg step */
+#define MBOX_MSG_STEP 4
+/** mailbox register max */
+#define MBOX_MSG_REG_MAX 32
+
+/**
+ * @struct - hw_descq_context
+ * @brief queue context information
+ */
+struct hw_descq_context {
+ /** software descriptor context data: 4 data words */
+ u32 sw[5];
+ /** prefetch context data: 2 data words */
+ u32 prefetch[2];
+ /** queue completion context data: 4 data words */
+ u32 cmpt[5];
+ /** hardware descriptor context data: 2 data words */
+ u32 hw[2]; /* for retrieve only */
+ /** C2H or H2C context: 1 data word */
+ u32 cr[1]; /* for retrieve only */
+ /** FMAP context data */
+ u32 fmap[2];
+};
+
+/**
+ * @struct - stm_descq_context
+ * @brief queue stm information
+ */
+struct stm_descq_context {
+ /** STM data: 6 data words */
+ u32 stm[6];
+};
+
+/**
+ * mailbox messages
+ *
+ * NOTE: make sure the total message length is <= 128 bytes:
+ * mbox_msg_hdr: 4 bytes
+ * body: <= (128 - hdr) bytes
+ *
+ */
+
+/**
+ * mbox_msg_op - mailbox messages opcode: 1 ~ 0x1F
+ */
+#define MBOX_MSG_OP_PF_MASK 0x10
+enum mbox_msg_op {
+ /* 0x00 */ MBOX_OP_NOOP,
+ /** VF -> PF, request */
+ /* 0x01 */ MBOX_OP_BYE, /** vf offline */
+ /* 0x02 */ MBOX_OP_HELLO, /** vf online */
+
+ /* 0x03 */ MBOX_OP_FMAP, /** FMAP programming request */
+ /* 0x04 */ MBOX_OP_CSR, /** global CSR registers request */
+ /* 0x05 */ MBOX_OP_INTR_CTXT, /** interrupt context programming */
+ /* 0x06 */ MBOX_OP_QCTXT_WRT, /** queue context programming */
+ /* 0x07 */ MBOX_OP_QCTXT_RD, /** queue context read */
+ /* 0x08 */ MBOX_OP_QCTXT_CLR, /** queue context clear */
+ /* 0x09 */ MBOX_OP_QCTXT_INV, /** queue context invalidate */
+ /* 0x0a */ MBOX_OP_QCONF, /** queue context invalidate */
+
+ /** PF->VF: response */
+ /* 0x12 */ MBOX_OP_HELLO_RESP = 0x12,/** vf online */
+ /* 0x13 */ MBOX_OP_FMAP_RESP, /** FMAP programming */
+ /* 0x14 */ MBOX_OP_CSR_RESP, /** global CSR read */
+ /* 0x15 */ MBOX_OP_INTR_CTXT_RESP, /** interrupt context programming */
+ /* 0x16 */ MBOX_OP_QCTXT_WRT_RESP, /** queue context programming */
+ /* 0x17 */ MBOX_OP_QCTXT_RD_RESP, /** queue context read */
+ /* 0x18 */ MBOX_OP_QCTXT_CLR_RESP, /** queue context clear */
+ /* 0x19 */ MBOX_OP_QCTXT_INV_RESP, /** queue context invalidate */
+ /* 0x1a */ MBOX_OP_QCONF_RESP, /** queue context invalidate */
+
+ MBOX_OP_MAX
+};
+
+#define mbox_invalidate_msg(m) { (m)->hdr.op = MBOX_OP_NOOP; }
+
+/**
+ * @struct - mbox_msg_hdr
+ * @brief mailbox message header
+ */
+struct mbox_msg_hdr {
+ u8 op; /** opcode */
+ u16 src; /** src function */
+ u16 dst; /** dst function */
+ char status; /** execution status */
+};
+
+/**
+ * @struct - mbox_msg_fmap
+ * @brief FMAP programming command
+ */
+struct mbox_msg_fmap {
+ /** mailbox message header */
+ struct mbox_msg_hdr hdr;
+ /** start queue number in the queue range */
+ unsigned int qbase;
+ /** max queue number in the queue range(0-2k) */
+ unsigned int qmax;
+};
+
+/**
+ * @struct - mbox_msg_csr
+ * @brief mailbox csr reading message
+ */
+struct mbox_msg_csr {
+ /** mailbox message header*/
+ struct mbox_msg_hdr hdr;
+ struct qdma_csr_info csr_info;
+};
+
+/**
+ * @struct - mbox_msg_intr_ctxt
+ * @brief interrupt context mailbox message
+ */
+
+
+struct mbox_msg_intr_ctxt {
+ /** mailbox message header*/
+ struct mbox_msg_hdr hdr;
+ /** flag to indicate clear interrupt context*/
+ u16 clear:1;
+ /** filler variable*/
+ u16 filler:15;
+ /** start vector number*/
+ u8 vec_base; /* 0 ~ 7 */
+ /** number of intr context rings be assigned for virtual function*/
+ u8 num_rings; /* 1 ~ 8 */
+ /** ring index associated for each vector */
+ u32 ring_index_list[QDMA_NUM_DATA_VEC_FOR_INTR_CXT];
+ /** interrupt context data for all rings*/
+ u32 w[QDMA_NUM_DATA_VEC_FOR_INTR_CXT * 3];
+};
+
+/**
+ * @struct - mbox_msg_qctxt
+ * @brief queue context mailbox message header
+ */
+struct mbox_msg_qctxt {
+ /** mailbox message header*/
+ struct mbox_msg_hdr hdr;
+ /** flag to indicate to clear the queue context */
+ u8 clear:1;
+ /** flag to indicate to verify the queue context */
+ u8 verify:1;
+ /** queue direction */
+ u8 c2h:1;
+ /** queue mode */
+ u8 st:1;
+ /** flag to indicate to enable the interrupts */
+ u8 intr_en:1;
+ /** interrupt id */
+ u8 intr_id;
+ /** queue id */
+ unsigned short qid;
+ /** complete hw context */
+ struct hw_descq_context context;
+};
+
+/**
+ * @struct - mbox_msg
+ * @brief mailbox message
+ */
+struct mbox_msg {
+ struct work_struct work; /** workqueue item */
+ struct list_head list; /** message list */
+ qdma_wait_queue waitq;
+ struct kref refcnt;
+ u8 wait_resp;
+ u8 wait_op;
+ u8 rsvd[2];
+
+ union {
+ /** mailbox message header*/
+ struct mbox_msg_hdr hdr;
+ /** fmap mailbox message */
+ struct mbox_msg_fmap fmap;
+ /** interrupt context mailbox message */
+ struct mbox_msg_intr_ctxt intr_ctxt;
+ /** queue context mailbox message */
+ struct mbox_msg_qctxt qctxt;
+ /** global csr mailbox message */
+ struct mbox_msg_csr csr;
+ /** buffer to hold raw data between pf and vf */
+ u32 raw[MBOX_MSG_REG_MAX];
+ };
+};
+
+/**
+ * forward declaration of xlnx_dma_dev
+ */
+struct xlnx_dma_dev;
+/**
+ * @struct - qdma_mbox book keeping
+ * @brief mailbox book keeping structure
+ */
+struct qdma_mbox {
+ /** common lock */
+ spinlock_t lock;
+ /** tx lock */
+ spinlock_t hw_tx_lock;
+ /** rx lock */
+ spinlock_t hw_rx_lock;
+ /** work queue */
+ struct workqueue_struct *workq;
+ /** pointer to device data */
+ struct xlnx_dma_dev *xdev;
+
+ /** tx work_struct to pass data to tx work queue */
+ struct work_struct tx_work;
+ /** rx work_struct to pass data to rx work queue */
+ struct work_struct rx_work;
+ /** mbox rx message */
+ struct mbox_msg rx;
+ /** list lock */
+ spinlock_t list_lock;
+ /** list of messages waiting to be sent */
+ struct list_head tx_todo_list;
+ /** list of messages waiting for response */
+ struct list_head rx_pend_list;
+
+ /** timer list */
+ struct timer_list timer;
+
+};
+
+#define QDMA_MBOX_MSG_TIMEOUT_MS 10000 /* 10 sec*/
+/*****************************************************************************/
+/**
+ * qdma_mbox_init() - initialize qdma mailbox
+ *
+ * @param xdev: pointer to xlnx_dma_dev
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_mbox_init(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * qdma_mbox_cleanup() - cleanup resources of qdma mailbox
+ * qdma_mbox_stop() - stop mailbox processing
+ * qdma_mbox_start() - start mailbox processing
+ *
+ * @param xdev: pointer to xlnx_dma_dev
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_mbox_cleanup(struct xlnx_dma_dev *xdev);
+void qdma_mbox_stop(struct xlnx_dma_dev *xdev);
+void qdma_mbox_start(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * qdma_mbox_msg_send() - handler to send a mailbox message
+ *
+ * @param xdev: pointer to xlnx_dma_dev
+ * @param m: mailbox message
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_mbox_msg_send(struct xlnx_dma_dev *xdev, struct mbox_msg *m,
+ bool wait_resp, u8 resp_op, unsigned int timeout_ms);
+
+/*****************************************************************************/
+/**
+ * qdma_mbox_msg_alloc() - allocate a mailbox message
+ *
+ * @param xdev: pointer to xlnx_dma_dev
+ *
+ * @return 0: success
+ * @return NULL: failure
+ *****************************************************************************/
+struct mbox_msg *qdma_mbox_msg_alloc(struct xlnx_dma_dev *xdev,
+ enum mbox_msg_op op);
+
+/*****************************************************************************/
+/**
+ * __qdma_mbox_msg_free() - free the mailbox message
+ *
+ * @param fname: function name
+ * @param m: mailbox message
+ *
+ * @return none
+ *****************************************************************************/
+void __qdma_mbox_msg_free(const char *fname, struct mbox_msg *m);
+#define qdma_mbox_msg_free(m) __qdma_mbox_msg_free(__func__, m)
+
+#endif /* #ifndef __QDMA_MBOX_H__ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.c b/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.c
new file mode 100644
index 0000000000000000000000000000000000000000..e58af8cf4359df6075f81a8736ccf294cf1d8338
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.c
@@ -0,0 +1,608 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#include <linux/gfp.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include "libqdma_config.h"
+#include "qdma_qconf_mgr.h"
+
+/** for holding the Q configuration list */
+static struct qconf_entry_head qconf_list;
+/** mutex for protecting the qconf_list */
+static DEFINE_MUTEX(qconf_mutex);
+
+/* insert new to the position specified by previous */
+static inline void list_insert(struct list_head *new, struct list_head *prev)
+{
+ new->next = prev;
+ new->prev = prev->prev;
+ prev->prev->next = new;
+ prev->prev = new;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_dump_qconf- dump the q configuration
+ * @param[in] xdev: device type to be used PF/VF
+ *
+ * @return
+ *****************************************************************************/
+void xdev_dump_qconf(u32 xdev)
+{
+ struct qconf_entry *_qconf = NULL;
+ struct qconf_entry *_tmp = NULL;
+ struct list_head *listhead;
+ const char *list_str[][8] = {{"vf"}, {"pf"}, {"vf-free"} };
+ int end = 0;
+
+ if (xdev == PCI_TYPE_PF)
+ listhead = &qconf_list.pf_list;
+ else
+ listhead = &qconf_list.vf_list;
+
+ pr_info("Dumping %s list\n", *list_str[xdev]);
+
+ list_for_each_entry_safe(_qconf, _tmp, listhead, list_head) {
+ end = ((int)(_qconf->qbase + _qconf->qmax)) - 1;
+ if (end < 0)
+ end = 0;
+ pr_info("func_id = %u, qmax = %u, qbase = %u~%u, cfg_state = %c\n",
+ _qconf->func_id, _qconf->qmax, _qconf->qbase,
+ end,
+ _qconf->cfg_state ? 'c' : 'i');
+ }
+}
+
+/*****************************************************************************/
+/**
+ * xdev_dump_freelist - dump the freelist
+ *
+ * @return
+ *****************************************************************************/
+void xdev_dump_freelist(void)
+{
+ struct free_entry *_free = NULL;
+ struct free_entry *_tmp = NULL;
+ struct list_head *listhead = &qconf_list.vf_free_list;
+
+ pr_info("Dumping free list\n");
+
+ list_for_each_entry_safe(_free, _tmp, listhead, list_head) {
+ pr_info("free-cnt = %u, qbase-next = %d\n",
+ _free->free, _free->next_qbase);
+ }
+}
+
+static struct qconf_entry *find_func_qentry(u32 xdev, u8 func_id)
+{
+ struct qconf_entry *_qconf = NULL;
+ struct qconf_entry *_tmp = NULL;
+ struct list_head *listhead = &qconf_list.vf_list;
+
+ if (xdev)
+ listhead = &qconf_list.pf_list;
+
+ list_for_each_entry_safe(_qconf, _tmp, listhead, list_head) {
+ if (_qconf->func_id == func_id)
+ return _qconf;
+ }
+
+ return NULL;
+}
+
+struct free_entry *create_free_entry(u32 qmax, u32 qbase)
+{
+ struct free_entry *_free_entry = NULL;
+
+ _free_entry = (struct free_entry *)kzalloc(sizeof(struct free_entry),
+ GFP_KERNEL);
+ if (!_free_entry)
+ return _free_entry;
+ _free_entry->free = qmax;
+ _free_entry->next_qbase = qbase;
+
+ return _free_entry;
+}
+
+static void cleanup_free_list(void)
+{
+ struct free_entry *_free_entry = NULL;
+ struct free_entry *_free_tmp = NULL;
+
+ mutex_lock(&qconf_mutex);
+ list_for_each_entry_safe(_free_entry, _free_tmp,
+ &qconf_list.vf_free_list, list_head) {
+ list_del(&_free_entry->list_head);
+ kfree(_free_entry);
+ }
+ mutex_unlock(&qconf_mutex);
+
+}
+
+/** initialize the free list */
+static int init_vf_free_list(void)
+{
+ struct free_entry *_free_entry = NULL;
+ u32 free = 0;
+ u32 next_qbase = 0;
+
+ free = qconf_list.vf_qmax;
+ next_qbase = qconf_list.vf_qbase;
+
+ cleanup_free_list();
+
+ _free_entry = create_free_entry(free, next_qbase);
+ mutex_lock(&qconf_mutex);
+ list_add_tail(&_free_entry->list_head, &qconf_list.vf_free_list);
+ mutex_unlock(&qconf_mutex);
+
+ return 0;
+}
+
+/** finds a first fit to accomodate the qmax.
+ * This function returns the qbase of the requested qmax
+ * if found
+ */
+static int find_first_fit(u32 qmax, u32 *qbase)
+{
+ struct free_entry *_free_entry = NULL;
+ struct free_entry *_tmp = NULL;
+ int err = 0;
+ u8 found = 0;
+
+ pr_debug("Get first fit %u\n", qmax);
+ mutex_lock(&qconf_mutex);
+ list_for_each_entry_safe(_free_entry, _tmp,
+ &qconf_list.vf_free_list, list_head) {
+ if (_free_entry->free >= qmax) {
+ *qbase = _free_entry->next_qbase;
+ _free_entry->free -= qmax;
+ _free_entry->next_qbase += qmax;
+ /** reduce cfgd_free */
+ qconf_list.qcnt_cfgd_free -= qmax;
+ found = 1;
+ break;
+ }
+ }
+ mutex_unlock(&qconf_mutex);
+
+ if (!found) {
+ pr_warn("No free slot found free/qmax = %u/%u\n",
+ _free_entry->free, qmax);
+ err = -EINVAL;
+ }
+
+ return err;
+}
+
+/** To defragment the free list if any. if it finds a continous
+ * range of qbases, merge both the entries into one.
+ */
+static int defrag_free_list(void)
+{
+ struct free_entry *_free = NULL;
+ struct free_entry *_tmp = NULL;
+ struct free_entry *_prev = NULL;
+ struct list_head *listhead = &qconf_list.vf_free_list;
+ int defrag_cnt = 0;
+
+ pr_debug("Defragmenting free list\n");
+ mutex_lock(&qconf_mutex);
+ list_for_each_entry_safe(_free, _tmp, listhead, list_head) {
+ if (_prev) {
+ /** is it a continous range?? */
+ if ((_prev->next_qbase + _prev->free)
+ == _free->next_qbase) {
+ _free->free += _prev->free;
+ _free->next_qbase = _prev->next_qbase;
+ list_del(&_prev->list_head);
+ kfree(_prev);
+ defrag_cnt++;
+ }
+ }
+ _prev = _free;
+ }
+ mutex_unlock(&qconf_mutex);
+
+ if (defrag_cnt)
+ pr_debug("Defragmented %d entries\n", defrag_cnt);
+
+ return defrag_cnt;
+}
+
+/**
+ * function to update the free list. First find the place to
+ * fit the free entry by comparing the qbases. Once found,
+ * insert the new free entry and insert it before the found one
+ */
+static int update_free_list(struct qconf_entry *entry)
+{
+ struct free_entry *_free_entry = NULL;
+ struct free_entry *_free_new = NULL;
+ struct free_entry *_tmp = NULL;
+ struct list_head *listhead = &qconf_list.vf_free_list;
+
+ /** consider only user configured values */
+ if (entry->cfg_state != Q_CONFIGURED)
+ return 0;
+
+ mutex_lock(&qconf_mutex);
+ list_for_each_entry_safe(_free_entry, _tmp, listhead, list_head) {
+ /** add it before the next bigger qbase */
+ if (entry->qbase < _free_entry->next_qbase) {
+ _free_new = create_free_entry(entry->qmax,
+ entry->qbase);
+ list_insert(&_free_new->list_head,
+ &_free_entry->list_head);
+ /** update the cfgd_free */
+ qconf_list.qcnt_cfgd_free += entry->qmax;
+ break;
+ }
+ }
+ mutex_unlock(&qconf_mutex);
+
+ defrag_free_list();
+
+ return 0;
+}
+
+/** grab from initial qconf, make qmax=0, qbase=0 cfg_state as zeroed*/
+static int grab_from_init_qconf(struct list_head *listhead, u8 func_id)
+{
+ struct qconf_entry *_qconf = NULL;
+ struct qconf_entry *_tmp = NULL;
+ int grab_cnt = 0;
+ int ret = 0;
+
+ mutex_lock(&qconf_mutex);
+ if (qconf_list.qcnt_init_used > qconf_list.qcnt_cfgd_free) {
+ grab_cnt = qconf_list.qcnt_init_used
+ - qconf_list.qcnt_cfgd_free;
+ qconf_list.qcnt_init_used -= grab_cnt;
+ }
+
+ ret = grab_cnt;
+
+ pr_debug("grab_cnt = %u init_used = %u cfgd_free = %u\n",
+ grab_cnt, qconf_list.qcnt_init_used,
+ qconf_list.qcnt_cfgd_free);
+
+ if (grab_cnt) {
+ list_for_each_entry_safe(_qconf, _tmp, listhead, list_head) {
+ /** already grabbed */
+ if (!_qconf->qmax || (_qconf->cfg_state != Q_INITIAL))
+ continue;
+ if (!grab_cnt--)
+ break;
+ _qconf->qmax = 0;
+ _qconf->qbase = 0;
+ /** if grabbing from the same func_id,
+ * reduce the grab count by 1
+ */
+ if (unlikely(_qconf->func_id == func_id))
+ ret -= 1;
+ }
+ }
+ mutex_unlock(&qconf_mutex);
+
+ return ret;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_set_qmax() - sets the qmax value of VF device from virtfn/qdma/qmax
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ * @param[in] qmax: qmax value to set
+ * @param[out] qbase: pointer to return qbase value
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_set_qmax(u32 xdev, u8 func_id, u32 qmax, u32 *qbase)
+{
+ struct qconf_entry *func_entry = NULL;
+ struct list_head *listhead = &qconf_list.vf_list;
+ u32 num_qs_free = qconf_list.qcnt_cfgd_free;
+ int err = 0;
+ u32 _qbase = 0;
+ int grab_cnt = 0;
+ u32 cur_qbase = 0;
+ u32 cur_qmax = 0;
+
+ pr_debug("Setting qmax func_id = %u, qmax = %u\n", func_id, qmax);
+
+ if (xdev)
+ listhead = &qconf_list.pf_list;
+
+ func_entry = find_func_qentry(xdev, func_id);
+ if (!func_entry) {
+ pr_err("Set qmax request on non available device %u\n",
+ func_id);
+ return -EINVAL;
+ }
+ cur_qmax = func_entry->qmax;
+ cur_qbase = func_entry->qbase;
+ if (func_entry->cfg_state == Q_CONFIGURED) {
+ update_free_list(func_entry);
+ num_qs_free = qconf_list.qcnt_cfgd_free;
+ }
+
+ if (qmax > num_qs_free) {
+ pr_warn("No free qs!, requested/free = %u/%u\n",
+ qmax, num_qs_free);
+ return -EINVAL;
+ }
+
+ func_entry->qbase = 0;
+ err = find_first_fit(qmax, &_qbase);
+ if (err < 0) {
+ if (qmax)
+ pr_info("Not able to find a fit for qmax = %u\n",
+ qmax);
+ else
+ pr_info("0 q size, func_id = %u\n", func_id);
+
+ if (func_entry->cfg_state == Q_CONFIGURED) {
+ /** if failed to get a qbase, revert it to old values */
+ func_entry->qmax = cur_qmax;
+ func_entry->qbase = cur_qbase;
+ }
+
+ return err;
+ }
+
+ func_entry->cfg_state = Q_CONFIGURED;
+ func_entry->qmax = qmax;
+ func_entry->qbase = _qbase;
+ *qbase = _qbase;
+ mutex_lock(&qconf_mutex);
+ list_del(&func_entry->list_head);
+ list_add_tail(&func_entry->list_head, listhead);
+ mutex_unlock(&qconf_mutex);
+ grab_cnt = grab_from_init_qconf(listhead, func_id);
+
+ return grab_cnt;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_del_qconf - delete the q configuration entry of funcid
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_del_qconf(u32 xdev, u8 func_id)
+{
+ struct qconf_entry *_qconf = NULL;
+
+ _qconf = find_func_qentry(xdev, func_id);
+ if (_qconf)
+ _qconf->cfg_state = Q_INITIAL;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_create_qconf - add the q configuration entry of funcid during hello
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+struct qconf_entry *xdev_create_qconf(u32 xdev, u8 func_id)
+{
+ struct qconf_entry *_qconf = NULL;
+ struct list_head *listhead = &qconf_list.vf_list;
+
+ pr_debug("Creating func_id = %u\n", func_id);
+ if (xdev)
+ listhead = &qconf_list.pf_list;
+
+ _qconf = (struct qconf_entry *)kzalloc(sizeof(struct qconf_entry),
+ GFP_KERNEL);
+ if (!_qconf)
+ return _qconf;
+
+ _qconf->func_id = func_id;
+ _qconf->cfg_state = Q_INITIAL;
+ if (qconf_list.qcnt_cfgd_free) {
+ /* _qconf->qmax = 1; */
+ /* _qconf->qbase = 2047 - qconf_list.qcnt_init_used; */
+ _qconf->qmax = 1;
+ _qconf->qbase = 0;
+ }
+
+ mutex_lock(&qconf_mutex);
+ if (qconf_list.qcnt_init_used < qconf_list.vf_qmax)
+ qconf_list.qcnt_init_used++;
+ list_add_tail(&_qconf->list_head, listhead);
+ mutex_unlock(&qconf_mutex);
+
+ atomic_inc(&qconf_list.vf_cnt);
+
+ return _qconf;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_destroy_qconf - destroys the qconf entry during bye
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_destroy_qconf(u32 xdev, u8 func_id)
+{
+ struct qconf_entry *_qconf = NULL;
+
+ pr_debug("Destroying func_id = %u\n", func_id);
+ _qconf = find_func_qentry(xdev, func_id);
+ if (_qconf) {
+ mutex_lock(&qconf_mutex);
+ list_del(&_qconf->list_head);
+ mutex_unlock(&qconf_mutex);
+ update_free_list(_qconf);
+ if (_qconf->cfg_state == Q_INITIAL)
+ qconf_list.qcnt_init_used -= _qconf->qmax;
+ kfree(_qconf);
+ atomic_dec(&qconf_list.vf_cnt);
+ }
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_qconf_init - initialize the q configuration structures
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_qconf_init(void)
+{
+ INIT_LIST_HEAD(&qconf_list.vf_list);
+ INIT_LIST_HEAD(&qconf_list.pf_list);
+ INIT_LIST_HEAD(&qconf_list.vf_free_list);
+ atomic_set(&qconf_list.vf_cnt, 0);
+ qconf_list.vf_qmax = TOTAL_VF_QS;
+ qconf_list.pf_qmax = TOTAL_QDMA_QS - qconf_list.vf_qmax;
+ qconf_list.vf_qbase = qconf_list.pf_qmax;
+ qconf_list.qcnt_cfgd_free = qconf_list.vf_qmax;
+ qconf_list.qcnt_init_free = qconf_list.vf_qmax;
+ qconf_list.qcnt_init_used = 0;
+ init_vf_free_list();
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_qconf_cleanup - cleanup the q configuration
+ *
+ * @return
+ *****************************************************************************/
+void xdev_qconf_cleanup(void)
+{
+ struct qconf_entry *_qconf = NULL;
+ struct qconf_entry *_tmp = NULL;
+ struct list_head *listhead = &qconf_list.vf_list;
+
+ mutex_lock(&qconf_mutex);
+ list_for_each_entry_safe(_qconf, _tmp,
+ listhead, list_head) {
+ list_del(&_qconf->list_head);
+ kfree(_qconf);
+ }
+
+ _qconf = NULL;
+ listhead = &qconf_list.pf_list;
+ list_for_each_entry_safe(_qconf, _tmp,
+ listhead, list_head) {
+ list_del(&_qconf->list_head);
+ kfree(_qconf);
+ }
+ mutex_unlock(&qconf_mutex);
+
+ cleanup_free_list();
+}
+
+/*****************************************************************************/
+/**
+ * qconf_get_qmax - get the current qmax value of VF/PF
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] card_id: for differntiation the qdma card
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qconf_get_qmax(u32 xdev, u32 card_id)
+{
+ int qmax = 0;
+
+ mutex_lock(&qconf_mutex);
+ if (xdev == PCI_TYPE_PF)
+ qmax = qconf_list.pf_qmax;
+ else if (xdev == PCI_TYPE_VF)
+ qmax = qconf_list.vf_qmax;
+ else
+ pr_err("Invalid q id type specified\n");
+ mutex_unlock(&qconf_mutex);
+
+ return qmax;
+}
+
+/*****************************************************************************/
+/**
+ * is_vfqmax_configurable - checks if the vfqmax is configurable.
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: if not configurable
+ * @return 1: if configurable
+ *****************************************************************************/
+int is_vfqmax_configurable(void)
+{
+ return (atomic_read(&qconf_list.vf_cnt) == 0);
+}
+
+/*****************************************************************************/
+/**
+ * set_vf_qmax - set the value for vf_qmax;
+ *
+ * @param[in] qmax: qmax value to set
+ *
+ * @return: last set qmax value before qmax.
+ *****************************************************************************/
+int set_vf_qmax(u32 qmax)
+{
+ u32 last_qmax = qconf_list.vf_qmax;
+
+ mutex_lock(&qconf_mutex);
+ qconf_list.vf_qmax = qmax;
+ qconf_list.pf_qmax = TOTAL_QDMA_QS - qconf_list.vf_qmax;
+ qconf_list.vf_qbase = qconf_list.pf_qmax;
+ qconf_list.qcnt_cfgd_free = qconf_list.vf_qmax;
+ qconf_list.qcnt_init_free = qconf_list.vf_qmax;
+ qconf_list.qcnt_init_used = 0;
+ mutex_unlock(&qconf_mutex);
+
+ pr_debug("vf_qmax/pf_qmax/vf_qbase/cfgd_free/init_free/init_used %u/%u/%u/%u/%u/%u\n",
+ qconf_list.vf_qmax, qconf_list.pf_qmax,
+ qconf_list.vf_qbase, qconf_list.qcnt_cfgd_free,
+ qconf_list.qcnt_init_free, qconf_list.qcnt_init_used);
+
+ init_vf_free_list();
+
+ return last_qmax;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.h b/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.h
new file mode 100644
index 0000000000000000000000000000000000000000..a56f15c4ed4fa228dce6a361faad96a90c63d121
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_qconf_mgr.h
@@ -0,0 +1,217 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef _Q_CONF_H_
+#define _Q_CONF_H_
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/atomic.h>
+
+enum pci_dev_type {
+ /** device type is VF */
+ PCI_TYPE_VF,
+ /** device type is PF */
+ PCI_TYPE_PF,
+ /** unknown device type */
+ PCI_TYPE_INVAL
+};
+
+enum q_cfg_state {
+ /** initial state no qmax cfg done */
+ Q_INITIAL,
+ /** configured state qmax cfg already done */
+ Q_CONFIGURED,
+ /** unknown configuration state */
+ Q_INVALID
+};
+
+/* for free entry management */
+struct free_entry {
+ /** to connect to free_list */
+ struct list_head list_head;
+ /** next available qbase */
+ u32 next_qbase;
+ /** free qs available in this entry */
+ u32 free;
+ /** index of the entry */
+ u32 index;
+};
+
+/** q configuration entry */
+struct qconf_entry {
+ /** to connect to list head of */
+ struct list_head list_head;
+ /** idx of the device */
+ u32 idx;
+ /** qbase for func_id */
+ u32 qbase;
+ /** qmax for func_id */
+ u32 qmax;
+ /** current configuration state */
+ enum q_cfg_state cfg_state;
+ /** device type PF/VF */
+ enum pci_dev_type type;
+ /** func_id of the device */
+ u8 func_id;
+};
+
+/** for hodling the qconf_entry structure */
+struct qconf_entry_head {
+ /** for holding vf qconf_entry */
+ struct list_head vf_list;
+ /** for holding vf free_entry */
+ struct list_head vf_free_list;
+ /** for holding pf qconf_entry */
+ struct list_head pf_list;
+ /** for maximum qs for vf */
+ u32 vf_qmax;
+ /** for maximum qs for pf */
+ u32 pf_qmax;
+ /** for holding vf qconf_entry */
+ u32 vf_qbase;
+ /** number of vfs attached to all pfs */
+ atomic_t vf_cnt;
+ /** free count of qs which can be configured */
+ u32 qcnt_cfgd_free;
+ /** number of qs free for initial cfg devices */
+ u32 qcnt_init_free;
+ /** used by INITIAL state devices */
+ u32 qcnt_init_used;
+};
+
+/*****************************************************************************/
+/**
+ * xdev_set_qmax() - sets the qmax value of VF device from virtfn/qdma/qmax
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ * @param[in] qmax: qmax value to set
+ * @param[out] qbase: pointer to return qbase value
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_set_qmax(u32 xdev, u8 func_id, u32 qmax, u32 *qbase);
+
+/*****************************************************************************/
+/**
+ * xdev_del_qconf - delete the q configuration entry of funcid
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_del_qconf(u32 xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_create_qconf - add the q configuration entry of funcid during hello
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+struct qconf_entry *xdev_create_qconf(u32 xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_destroy_qconf - destroys the qconf entry during bye
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_destroy_qconf(u32 xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_qconf_init - initialize the q configuration structures
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int xdev_qconf_init(void);
+
+/*****************************************************************************/
+/**
+ * xdev_qconf_cleanup - cleanup the q configuration
+ *
+ * @return
+ *****************************************************************************/
+void xdev_qconf_cleanup(void);
+
+/*****************************************************************************/
+/**
+ * xdev_dump_freelist - dump the freelist
+ *
+ * @return
+ *****************************************************************************/
+void xdev_dump_freelist(void);
+
+/*****************************************************************************/
+/**
+ * xdev_dump_qconf- dump the q configuration
+ * @param[in] xdev: device type to be used PF/VF
+ *
+ * @return
+ *****************************************************************************/
+void xdev_dump_qconf(u32 xdev);
+
+/*****************************************************************************/
+/**
+ * set_vf_qmax - set the value for vf_qmax;
+ *
+ * @param[in] qmax: qmax value to set
+ *
+ * @return: last set qmax value before qmax.
+ *****************************************************************************/
+int set_vf_qmax(u32 qmax);
+
+/*****************************************************************************/
+/**
+ * qconf_get_qmax - get the current qmax value of VF/PF
+ *
+ * @param[in] xdev: device type to be used PF/VF
+ * @param[in] card_id: for differntiation the qdma card
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qconf_get_qmax(u32 xdev, u32 card_id);
+
+/*****************************************************************************/
+/**
+ * is_vfqmax_configurable - checks if the vfqmax is configurable.
+ *
+ * @param[in] xdev: list type to be used PF/VF
+ * @param[in] func_id: func id to set the qmax
+ *
+ * @return 0: if not configurable
+ * @return 1: if configurable
+ *****************************************************************************/
+int is_vfqmax_configurable(void);
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_regs.c b/QDMA/linux-kernel/libqdma/qdma_regs.c
new file mode 100644
index 0000000000000000000000000000000000000000..6615cf78a64b9820f601b7fdba26b17f7142570d
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_regs.c
@@ -0,0 +1,1109 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_regs.h"
+
+#include <linux/version.h>
+#include <linux/delay.h>
+#include <linux/printk.h>
+#include <linux/stddef.h>
+#include <linux/string.h>
+
+#include "xdev.h"
+#include "qdma_device.h"
+#include "qdma_descq.h"
+#include "qdma_st_c2h.h"
+
+/*****************************************************************************/
+/**
+ * qdma_device_read_config_register() - read dma config. register
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] reg_addr: register address
+ *
+ * @return value of the config register
+ *****************************************************************************/
+unsigned int qdma_device_read_config_register(unsigned long dev_hndl,
+ unsigned int reg_addr)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!xdev)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ return readl(xdev->regs + reg_addr);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_write_config_register() - write dma config. register
+ *
+ * @param[in] dev_hndl: dev_hndl returned from qdma_device_open()
+ * @param[in] reg_addr: register address
+ * @param[in] value: register value to be writen
+ *
+ *****************************************************************************/
+void qdma_device_write_config_register(unsigned long dev_hndl,
+ unsigned int reg_addr, unsigned int val)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!xdev)
+ return;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return;
+
+ pr_debug("%s reg 0x%x, w 0x%08x.\n", xdev->conf.name, reg_addr, val);
+ writel(val, xdev->regs + reg_addr);
+}
+
+
+#ifdef __QDMA_VF__
+#include "qdma_mbox.h"
+
+int qdma_csr_read(struct xlnx_dma_dev *xdev, struct qdma_csr_info *csr_info,
+ unsigned int timeout_ms)
+{
+ struct mbox_msg *m = qdma_mbox_msg_alloc(xdev, MBOX_OP_CSR);
+ struct mbox_msg_hdr *hdr = m ? &m->hdr : NULL;
+ struct mbox_msg_csr *csr = m ? &m->csr : NULL;
+ int rv;
+
+ if (!m)
+ return -ENOMEM;
+
+ memcpy(&csr->csr_info, csr_info, sizeof(*csr_info));
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_CSR_RESP, timeout_ms);
+ if (rv < 0)
+ goto free_msg;
+
+ rv = hdr->status;
+ if (rv < 0)
+ goto free_msg;
+
+ memcpy(csr_info, &csr->csr_info, sizeof(*csr_info));
+
+free_msg:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+static int send_csr_array_msg(struct xlnx_dma_dev *xdev,
+ unsigned int timeout_ms, enum csr_type type,
+ unsigned int *v, unsigned int *cs_acc)
+{
+ struct mbox_msg *m = qdma_mbox_msg_alloc(xdev, MBOX_OP_CSR);
+ struct mbox_msg_csr *csr = m ? &m->csr : NULL;
+ int i;
+ int rv = 0;
+
+ if (!m)
+ return -ENOMEM;
+
+ csr->csr_info.type = type;
+ rv = qdma_mbox_msg_send(xdev, m, 1, MBOX_OP_CSR_RESP, timeout_ms);
+ if (rv < 0)
+ goto err_out;
+
+ if (m->hdr.status) {
+ rv = m->hdr.status;
+ goto err_out;
+ }
+
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++)
+ v[i] = csr->csr_info.array[i];
+ *cs_acc = csr->csr_info.cmpl_status_acc;
+
+err_out:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int qdma_global_csr_get(unsigned long dev_hndl, struct global_csr_conf *csr)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int rv;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ rv = send_csr_array_msg(xdev, QDMA_MBOX_MSG_TIMEOUT_MS,
+ QDMA_CSR_TYPE_RNGSZ, csr->ring_sz,
+ &csr->cmpl_status_acc);
+ if (rv < 0)
+ return rv;
+
+ rv = send_csr_array_msg(xdev, QDMA_MBOX_MSG_TIMEOUT_MS,
+ QDMA_CSR_TYPE_BUFSZ, csr->c2h_buf_sz,
+ &csr->cmpl_status_acc);
+ if (rv < 0)
+ return rv;
+
+ rv = send_csr_array_msg(xdev, QDMA_MBOX_MSG_TIMEOUT_MS,
+ QDMA_CSR_TYPE_TIMER_CNT, csr->c2h_timer_cnt,
+ &csr->cmpl_status_acc);
+ if (rv < 0)
+ return rv;
+
+ rv = send_csr_array_msg(xdev, QDMA_MBOX_MSG_TIMEOUT_MS,
+ QDMA_CSR_TYPE_CNT_TH, csr->c2h_cnt_th,
+ &csr->cmpl_status_acc);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+
+int qdma_global_csr_set(unsigned long dev_hndl, struct global_csr_conf *csr)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ pr_info("VF %d setting csr NOT allowed.\n", xdev->func_id);
+ return -EINVAL;
+}
+
+#else /* ifdef __QDMA_VF__ */
+
+void qdma_csr_read_cmpl_status_acc(struct xlnx_dma_dev *xdev,
+ unsigned int *cs_acc)
+{
+ *cs_acc = __read_reg(xdev, QDMA_REG_GLBL_DSC_CFG) &
+ QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK;
+}
+
+void qdma_csr_read_rngsz(struct xlnx_dma_dev *xdev, unsigned int *rngsz)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_GLBL_RNG_SZ_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ rngsz[i] = __read_reg(xdev, reg);
+}
+
+int qdma_csr_write_rngsz(struct xlnx_dma_dev *xdev, unsigned int *rngsz)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_GLBL_RNG_SZ_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, rngsz[i]);
+
+ /**
+ * Return Operation Successful
+ */
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+void qdma_csr_read_bufsz(struct xlnx_dma_dev *xdev, unsigned int *bufsz)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_BUF_SZ_BASE;
+ for (i = 0; i < QDMA_REG_C2H_BUF_SZ_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ bufsz[i] = __read_reg(xdev, reg);
+}
+
+int qdma_csr_write_bufsz(struct xlnx_dma_dev *xdev, unsigned int *bufsz)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_BUF_SZ_BASE;
+ for (i = 0; i < QDMA_REG_C2H_BUF_SZ_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, bufsz[i]);
+
+ /**
+ * Return Operation Successful
+ */
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+void qdma_csr_read_timer_cnt(struct xlnx_dma_dev *xdev, unsigned int *tmr_cnt)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_TIMER_CNT_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ tmr_cnt[i] = __read_reg(xdev, reg);
+}
+
+int qdma_csr_write_timer_cnt(struct xlnx_dma_dev *xdev, unsigned int *tmr_cnt)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_TIMER_CNT_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, tmr_cnt[i]);
+ /**
+ * Return Operation Successful
+ */
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+void qdma_csr_read_cnt_thresh(struct xlnx_dma_dev *xdev, unsigned int *cnt_th)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_CNT_TH_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ cnt_th[i] = __read_reg(xdev, reg);
+}
+
+int qdma_csr_read(struct xlnx_dma_dev *xdev, struct qdma_csr_info *csr_info,
+ unsigned int timeout_ms)
+{
+ if (csr_info->idx_rngsz >= QDMA_GLOBAL_CSR_ARRAY_SZ ||
+ csr_info->idx_bufsz >= QDMA_GLOBAL_CSR_ARRAY_SZ ||
+ csr_info->idx_timer_cnt >= QDMA_GLOBAL_CSR_ARRAY_SZ ||
+ csr_info->idx_cnt_th >= QDMA_GLOBAL_CSR_ARRAY_SZ)
+ goto err_out;
+
+ csr_info->rngsz = __read_reg(xdev, csr_info->idx_rngsz * 4 +
+ QDMA_REG_GLBL_RNG_SZ_BASE);
+ csr_info->bufsz = __read_reg(xdev, csr_info->idx_bufsz * 4 +
+ QDMA_REG_C2H_BUF_SZ_BASE);
+ csr_info->timer_cnt = __read_reg(xdev, csr_info->idx_timer_cnt * 4 +
+ QDMA_REG_C2H_TIMER_CNT_BASE);
+ csr_info->cnt_th = __read_reg(xdev, csr_info->idx_cnt_th * 4 +
+ QDMA_REG_C2H_CNT_TH_BASE);
+ csr_info->cmpl_status_acc = __read_reg(xdev,
+ QDMA_REG_GLBL_DSC_CFG) &
+ QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK;
+
+ switch (csr_info->type) {
+ case QDMA_CSR_TYPE_NONE:
+ break;
+ case QDMA_CSR_TYPE_RNGSZ:
+ qdma_csr_read_rngsz(xdev, csr_info->array);
+ break;
+ case QDMA_CSR_TYPE_BUFSZ:
+ qdma_csr_read_bufsz(xdev, csr_info->array);
+ break;
+ case QDMA_CSR_TYPE_TIMER_CNT:
+ qdma_csr_read_timer_cnt(xdev, csr_info->array);
+ break;
+ case QDMA_CSR_TYPE_CNT_TH:
+ qdma_csr_read_cnt_thresh(xdev, csr_info->array);
+ break;
+ default:
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ pr_info("%s, type/idx invalid: 0x%x, %u,%u,%u,%u.\n",
+ xdev->conf.name, csr_info->type, csr_info->idx_rngsz,
+ csr_info->idx_bufsz, csr_info->idx_timer_cnt,
+ csr_info->idx_cnt_th);
+ return -EINVAL;
+}
+
+int qdma_csr_write_cnt_thresh(struct xlnx_dma_dev *xdev, unsigned int *cnt_th)
+{
+ int i;
+ unsigned int reg;
+
+ reg = QDMA_REG_C2H_CNT_TH_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, cnt_th[i]);
+
+ /**
+ * Return Operation Successful
+ */
+ return QDMA_OPERATION_SUCCESSFUL;
+}
+
+int qdma_global_csr_get(unsigned long dev_hndl, struct global_csr_conf *csr)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ qdma_csr_read_cmpl_status_acc(xdev, &csr->cmpl_status_acc);
+ qdma_csr_read_rngsz(xdev, csr->ring_sz);
+ qdma_csr_read_bufsz(xdev, csr->c2h_buf_sz);
+ qdma_csr_read_timer_cnt(xdev, csr->c2h_timer_cnt);
+ qdma_csr_read_cnt_thresh(xdev, csr->c2h_cnt_th);
+
+ return 0;
+}
+
+int qdma_global_csr_set(unsigned long dev_hndl, struct global_csr_conf *csr)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int i;
+ unsigned int reg;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ if (xdev->func_id) {
+ pr_info("func_id %d, csr setting not allowed.\n",
+ xdev->func_id);
+ return -EINVAL;
+ }
+
+ reg = __read_reg(xdev, QDMA_REG_GLBL_DSC_CFG);
+ reg &= ~QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK;
+ reg |= csr->cmpl_status_acc;
+ __write_reg(xdev, QDMA_REG_GLBL_DSC_CFG, reg);
+
+ reg = QDMA_REG_GLBL_RNG_SZ_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, csr->ring_sz[i]);
+
+ reg = QDMA_REG_C2H_BUF_SZ_BASE;
+ for (i = 0; i < QDMA_REG_C2H_BUF_SZ_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, csr->c2h_buf_sz[i]);
+
+ reg = QDMA_REG_C2H_TIMER_CNT_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, csr->c2h_timer_cnt[i]);
+
+ reg = QDMA_REG_C2H_CNT_TH_BASE;
+ for (i = 0; i < QDMA_GLOBAL_CSR_ARRAY_SZ; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, csr->c2h_cnt_th[i]);
+
+ return 0;
+}
+#endif
+
+/*
+ * hw_monitor_reg() - polling a register repeatly until
+ * (the register value & mask) == val or time is up
+ *
+ * return -EBUSY if register value didn't match, 1 other wise
+ */
+int hw_monitor_reg(struct xlnx_dma_dev *xdev, unsigned int reg, u32 mask,
+ u32 val, unsigned int interval_us, unsigned int timeout_us)
+{
+ int count;
+ u32 v;
+
+ if (!interval_us)
+ interval_us = QDMA_REG_POLL_DFLT_INTERVAL_US;
+ if (!timeout_us)
+ timeout_us = QDMA_REG_POLL_DFLT_TIMEOUT_US;
+
+ count = timeout_us / interval_us;
+
+ do {
+ v = __read_reg(xdev, reg);
+ if ((v & mask) == val)
+ return 1;
+ udelay(interval_us);
+ } while (--count);
+
+ v = __read_reg(xdev, reg);
+ if ((v & mask) == val)
+ return 1;
+
+ pr_debug("%s, reg 0x%x, timed out %uus, 0x%x & 0x%x != 0x%x.\n",
+ xdev->conf.name, reg, timeout_us, v, mask, val);
+
+ return -EBUSY;
+}
+
+int qdma_device_flr_quirk_set(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!xdev->flr_prsnt) {
+ pr_info("FLR not present, therefore skipping FLR reset\n");
+ return 0;
+ }
+
+ if (!dev_hndl || xdev_check_hndl(__func__, pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ __write_reg(xdev, QDMA_REG_FLR_STATUS, 0x1);
+ return 0;
+}
+
+int qdma_device_flr_quirk_check(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int rv;
+
+ if (!xdev->flr_prsnt) {
+ pr_info("FLR not present, therefore skipping FLR reset status\n");
+ return 0;
+ }
+
+ if (!dev_hndl || xdev_check_hndl(__func__, pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ /* wait for it to become zero */
+ rv = hw_monitor_reg(xdev, QDMA_REG_FLR_STATUS, 0x1, 0, 500, 500*1000);
+ if (rv < 0)
+ pr_info("%s, flr status stuck 0x%x.\n", xdev->conf.name,
+ __read_reg(xdev, QDMA_REG_FLR_STATUS));
+
+ return 0;
+}
+
+int qdma_device_flr_quirk(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ int rv;
+
+ rv = qdma_device_flr_quirk_set(pdev, dev_hndl);
+ if (rv < 0)
+ return rv;
+
+ return qdma_device_flr_quirk_check(pdev, dev_hndl);
+}
+
+
+int qdma_device_version_info(unsigned long dev_hndl,
+ struct qdma_version_info *version_info)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int reg_val = 0;
+
+ if (!xdev || !version_info)
+ return -EINVAL;
+
+ reg_val = __read_reg(xdev, QDMA_VERSION_REG);
+
+ version_info->vivado_release_id =
+ (reg_val & M_VIVADO_RELEASE_ID_MASK) >>
+ S_VIVADO_RELEASE_ID_SHIFT;
+ version_info->rtl_version =
+ (reg_val & M_RTL_VERSION_MASK) >> S_RTL_VERSION_SHIFT;
+ version_info->everest_ip =
+ (reg_val & M_EVEREST_IP_MASK) >> S_EVEREST_IP_SHIFT;
+
+ switch (version_info->rtl_version) {
+ case RTL1_VERSION:
+ sprintf(version_info->rtl_version_str, "RTL1");
+ break;
+ default:
+ sprintf(version_info->rtl_version_str, "RTL2");
+ break;
+ }
+
+ switch (version_info->vivado_release_id) {
+ case VIVADO_RELEASE_2018_3:
+ sprintf(version_info->vivado_release_str, "2018.3");
+ break;
+ default:
+ sprintf(version_info->vivado_release_str, "2018.2");
+ break;
+ }
+
+ if (version_info->everest_ip)
+ sprintf(version_info->everest_ip_str, "HARD_IP");
+ else
+ sprintf(version_info->everest_ip_str, "SOFT_IP");
+
+ return 0;
+}
+
+#ifndef __QDMA_VF__
+static int qdma_device_num_pfs_get(struct xlnx_dma_dev *xdev)
+{
+ int count = 0;
+ int reg_val = 0;
+
+ reg_val = __read_reg(xdev, QDMA_REG_GLBL_PF_BARLITE_INT);
+
+ if ((reg_val & PF_BARLITE_INT_0_MASK) >> PF_BARLITE_INT_0_SHIFT)
+ count++;
+ if ((reg_val & PF_BARLITE_INT_1_MASK) >> PF_BARLITE_INT_1_SHIFT)
+ count++;
+ if ((reg_val & PF_BARLITE_INT_2_MASK) >> PF_BARLITE_INT_2_SHIFT)
+ count++;
+ if ((reg_val & PF_BARLITE_INT_3_MASK) >> PF_BARLITE_INT_3_SHIFT)
+ count++;
+
+ xdev->pf_count = count;
+ return count;
+}
+
+void qdma_device_attributes_get(struct xlnx_dma_dev *xdev)
+{
+ int mm_c2h_flag = 0;
+ int mm_h2c_flag = 0;
+ int st_c2h_flag = 0;
+ int st_h2c_flag = 0;
+ unsigned int v1 = __read_reg(xdev, QDMA_REG_GLBL_QMAX);
+ int v2 = qdma_device_num_pfs_get(xdev);
+
+ xdev->conf.qsets_max = v1 / v2;
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
+ v2 = pci_sriov_get_totalvfs(xdev->conf.pdev);
+#else
+ v2 = QDMA_VF_MAX;
+#endif
+ if ((v2 * QDMA_Q_PER_VF_MAX) > xdev->conf.qsets_max) {
+ pr_warn("%s, max vf %d, per vf Q %u, manual setting needed.\n",
+ xdev->conf.name, v2, QDMA_Q_PER_VF_MAX);
+ xdev->conf.qsets_max = 0;
+ } else
+ xdev->conf.qsets_max -= v2 * QDMA_Q_PER_VF_MAX;
+
+ /** changed to static allocation, VF qs are allocated
+ * at the bottom. Function used only during initial
+ * allocation, VFs is different for different pfs
+ * so allocation of qs not uniform.
+ */
+ if (xdev->conf.qsets_max != MAX_QS_PER_PF)
+ xdev->conf.qsets_max = MAX_QS_PER_PF;
+
+ /* TODO : __read_reg(xdev, QDMA_REG_GLBL_MM_ENGINES);*/
+ xdev->mm_channel_max = 1;
+ xdev->flr_prsnt = (__read_reg(xdev, QDMA_REG_GLBL_MISC_CAP) &
+ MISC_CAP_FLR_PRESENT_MASK) >>
+ MISC_CAP_FLR_PRESENT_SHIFT;
+
+ v1 = __read_reg(xdev, QDMA_REG_GLBL_MDMA_CHANNEL);
+ mm_c2h_flag = (v1 & MDMA_CHANNEL_MM_C2H_ENABLED_MASK) ? 1 : 0;
+ mm_h2c_flag = (v1 & MDMA_CHANNEL_MM_H2C_ENABLED_MASK) ? 1 : 0;
+ st_c2h_flag = (v1 & MDMA_CHANNEL_ST_C2H_ENABLED_MASK) ? 1 : 0;
+ st_h2c_flag = (v1 & MDMA_CHANNEL_ST_H2C_ENABLED_MASK) ? 1 : 0;
+
+ xdev->mm_mode_en = (mm_c2h_flag && mm_h2c_flag) ? 1 : 0;
+ xdev->st_mode_en = (st_c2h_flag && st_h2c_flag) ? 1 : 0;
+
+ if (xdev->stm_en)
+ xdev->conf.bar_num_stm = STM_BAR;
+
+ pr_info("%s: present flr %d, mm %d, st %d.\n",
+ xdev->conf.name, xdev->flr_prsnt, xdev->mm_mode_en,
+ xdev->st_mode_en);
+}
+
+void hw_set_global_csr(struct xlnx_dma_dev *xdev)
+{
+ int i;
+ unsigned int reg;
+
+ reg = ((0x03 << QDMA_REG_GLBL_DSC_CFG_MAX_DESC_FETCH_SHIFT) | 0x05);
+ __write_reg(xdev, QDMA_REG_GLBL_DSC_CFG, reg);
+
+ reg = QDMA_REG_GLBL_RNG_SZ_BASE;
+ for (i = 0; i < QDMA_REG_GLBL_RNG_SZ_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, (RNG_SZ_DFLT << i) + 1);
+
+ reg = QDMA_REG_C2H_BUF_SZ_BASE;
+ for (i = 0; i < QDMA_REG_C2H_BUF_SZ_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, C2H_BUF_SZ_DFLT);
+
+ reg = QDMA_REG_C2H_TIMER_CNT_BASE;
+ for (i = 0; i < QDMA_REG_C2H_TIMER_CNT_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, C2H_TIMER_CNT_DFLT << i);
+
+ reg = QDMA_REG_C2H_CNT_TH_BASE;
+ for (i = 0; i < QDMA_REG_C2H_CNT_TH_COUNT; i++,
+ reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, C2H_CNT_TH_DFLT << i);
+
+}
+
+int qdma_trq_c2h_config(struct xlnx_dma_dev *xdev)
+{
+ unsigned int reg;
+ unsigned int dsc_cache_depth;
+ unsigned int cpml_coal_depth;
+
+ dsc_cache_depth = __read_reg(xdev, QDMA_C2H_PFCH_CACHE_DEPTH);
+ dsc_cache_depth >>= 2;
+ reg = (0x100 << QDMA_C2H_PFCH_CFG_PFCH_FL_TH_SHIFT) |
+ (8 << QDMA_C2H_PFCH_CFG_NUM_PFCH_SHIFT) |
+ (dsc_cache_depth << QDMA_C2H_PFCH_CFG_PFCH_QCNT_SHIFT) |
+ ((dsc_cache_depth - 2) <<
+ QDMA_C2H_PFCH_CFG_EVT_QCNT_TH_SHIFT);
+ __write_reg(xdev, QDMA_C2H_PFCH_CFG, reg);
+
+ cpml_coal_depth = __read_reg(xdev, QDMA_C2H_CMPT_COAL_BUF_DEPTH);
+ reg = (cpml_coal_depth << QDMA_C2H_CMPT_COAL_CFG_MAX_BUF_SZ_SHIFT) |
+ (25 << QDMA_C2H_CMPT_COAL_CFG_TICK_VAL_SHIFT) |
+ (5 << QDMA_C2H_CMPT_COAL_CFG_TICK_CNT_SHIFT);
+ __write_reg(xdev, QDMA_C2H_CMPT_COAL_CFG, reg);
+ __write_reg(xdev, QDMA_C2H_INT_TIMER_TICK, 25);
+ __write_reg(xdev, QDMA_H2C_DATA_THRESHOLD, 0x14000);
+
+ return 0;
+}
+
+void hw_mm_channel_enable(struct xlnx_dma_dev *xdev, int channel, bool c2h)
+{
+ int reg = (c2h) ? QDMA_REG_H2C_MM_CONTROL_BASE :
+ QDMA_REG_C2H_MM_CONTROL_BASE;
+
+ __write_reg(xdev, reg + channel * QDMA_REG_MM_CONTROL_STEP,
+ QDMA_REG_MM_CONTROL_RUN);
+}
+
+void hw_mm_channel_disable(struct xlnx_dma_dev *xdev, int channel, bool c2h)
+{
+ int reg = (c2h) ? QDMA_REG_H2C_MM_CONTROL_BASE :
+ QDMA_REG_C2H_MM_CONTROL_BASE;
+
+ __write_reg(xdev, reg + channel * QDMA_REG_MM_CONTROL_STEP, 0U);
+}
+
+int hw_set_fmap(struct xlnx_dma_dev *xdev, u16 func_id, unsigned int qbase,
+ unsigned int qmax)
+{
+ struct hw_descq_context context;
+ int rv;
+ u32 *data = NULL;
+
+ memset(&context, 0, sizeof(context));
+
+ data = context.fmap;
+ data[1] = V_FMAP_W1_QMAX(qmax);
+ data[0] = V_FMAP_W0_QBASE(qbase);
+
+ pr_debug("%s, FMAP 0x%08x 0x%08x\n",
+ xdev->conf.name, data[1], data[0]);
+
+ rv = hw_indirect_ctext_prog(xdev, func_id, QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_SEL_FMAP, context.fmap, 2, 1);
+ if (rv < 0)
+ return rv;
+
+ return 0;
+}
+
+int hw_read_fmap(struct xlnx_dma_dev *xdev, u16 func_id, unsigned int *qbase,
+ unsigned int *qmax)
+{
+ struct hw_descq_context context;
+ int rv;
+
+ memset(&context, 0, sizeof(context));
+
+ rv = hw_indirect_ctext_prog(xdev, func_id, QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_SEL_FMAP, context.fmap, 2, 1);
+ if (rv < 0)
+ return rv;
+
+ *qbase = context.fmap[0];
+ *qmax = context.fmap[1];
+
+ pr_debug("%s, FMAP qbase %d qmax %d\n",
+ xdev->conf.name, *qbase, *qmax);
+
+ return 0;
+}
+
+int hw_indirect_stm_prog(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ u8 fid, enum ind_stm_cmd_op op,
+ enum ind_stm_addr addr, u32 *data,
+ unsigned int cnt, bool clear)
+{
+ unsigned int reg;
+ u32 v;
+ int i;
+ int rv = 0;
+
+ spin_lock(&xdev->hw_prg_lock);
+ pr_debug("qid_hw 0x%x, op 0x%x, addr 0x%x, data 0x%p,%u\n",
+ qid_hw, op, addr, data, cnt);
+
+ if ((op == STM_CSR_CMD_WR) || (op == STM_CSR_CMD_RD)) {
+ if (unlikely(!cnt || cnt > 6)) {
+ pr_warn("Q 0x%x, op 0x%x, addr 0x%x, cnt %u/%d.\n",
+ qid_hw, op, addr, cnt,
+ 6);
+ rv = -EINVAL;
+ goto fn_ret;
+ }
+
+ if (unlikely(!data)) {
+ pr_warn("Q 0x%x, op 0x%x, sel 0x%x, data NULL.\n",
+ qid_hw, op, addr);
+ rv = -EINVAL;
+ goto fn_ret;
+ }
+
+ if (op == STM_CSR_CMD_WR) {
+ switch (addr) {
+ case STM_IND_ADDR_Q_CTX_H2C:
+ reg = STM_REG_BASE + STM_REG_IND_CTXT_DATA_BASE;
+
+ for (i = 0; i < cnt; i++,
+ reg += QDMA_REG_SZ_IN_BYTES) {
+ pr_debug("%s: i = %d; reg = 0x%x; data[%d] = 0x%x\n",
+ __func__, i, reg, i, data[i]);
+ writel(data[i], xdev->stm_regs + reg);
+ }
+ pr_debug("%s: data[5] for h2c-ctxt is: 0x%x\n",
+ __func__, data[5]);
+ /* write context valid bit */
+ writel(data[5], xdev->stm_regs + STM_REG_BASE +
+ STM_REG_IND_CTXT_DATA5);
+ break;
+
+ case STM_IND_ADDR_Q_CTX_C2H:
+ reg = STM_REG_BASE + STM_REG_IND_CTXT_DATA3;
+
+ for (i = 0; i < cnt; i++,
+ reg += QDMA_REG_SZ_IN_BYTES) {
+ pr_debug("%s: i = %d; reg = 0x%x; data[%d] = 0x%x\n",
+ __func__, i, reg, i, data[i]);
+ writel(data[i], xdev->stm_regs + reg);
+ }
+ pr_debug("%s: data[5] for c2h-ctxt is: 0x%x\n",
+ __func__, data[5]);
+ /* write context valid bit */
+ writel(data[5], xdev->stm_regs + STM_REG_BASE +
+ STM_REG_IND_CTXT_DATA5);
+ break;
+
+ case STM_IND_ADDR_H2C_MAP:
+ reg = STM_REG_BASE +
+ STM_REG_IND_CTXT_DATA_BASE + (4 * 4);
+ pr_debug("%s: reg = 0x%x; data = 0x%x\n",
+ __func__, reg, qid_hw);
+ if (clear)
+ writel(0, xdev->stm_regs + reg);
+ else
+ writel(qid_hw, xdev->stm_regs + reg);
+ break;
+
+ case STM_IND_ADDR_C2H_MAP: {
+ u32 c2h_map;
+
+ reg = STM_REG_BASE + STM_REG_C2H_DATA8;
+ c2h_map = (clear ? 0 : qid_hw) |
+ (DESC_SZ_8B << 11);
+ pr_debug("%s: reg = 0x%x; data = 0x%x\n",
+ __func__, reg, c2h_map);
+ writel(c2h_map, xdev->stm_regs + reg);
+ break;
+ }
+
+ default:
+ pr_err("%s: not supported address..\n",
+ __func__);
+ rv = -EINVAL;
+ goto fn_ret;
+ }
+ }
+ }
+
+ v = (qid_hw << S_STM_CMD_QID) | (op << S_STM_CMD_OP) |
+ (addr << S_STM_CMD_ADDR) | (fid << S_STM_CMD_FID);
+
+ pr_debug("ctxt_cmd reg 0x%x, qid 0x%x, op 0x%x, fid 0x%x addr 0x%x -> 0x%08x.\n",
+ STM_REG_BASE + STM_REG_IND_CTXT_CMD, qid_hw, op, fid, addr, v);
+ writel(v, xdev->stm_regs + STM_REG_BASE + STM_REG_IND_CTXT_CMD);
+
+ if (op == STM_CSR_CMD_RD) {
+ switch (addr) {
+ case STM_IND_ADDR_Q_CTX_C2H:
+ case STM_IND_ADDR_Q_CTX_H2C:
+ case STM_IND_ADDR_FORCED_CAN:
+ case STM_IND_ADDR_H2C_MAP:
+ reg = STM_REG_BASE + STM_REG_IND_CTXT_DATA_BASE;
+ for (i = 0; i < cnt;
+ i++, reg += QDMA_REG_SZ_IN_BYTES) {
+ data[i] = readl(xdev->stm_regs + reg);
+
+ pr_debug("%s: reg = 0x%x; data[%d] = 0x%x\n",
+ __func__, reg, i, data[i]);
+ }
+ /* read context valid bits */
+ data[5] = readl(xdev->stm_regs + STM_REG_BASE +
+ STM_REG_IND_CTXT_DATA5);
+ break;
+
+ case STM_IND_ADDR_C2H_MAP:
+ reg = STM_REG_BASE + STM_REG_C2H_DATA8;
+ data[0] = readl(xdev->stm_regs + reg);
+ break;
+
+ default:
+ pr_err("%s: not supported address..\n",
+ __func__);
+ rv = -EINVAL;
+ }
+ }
+
+fn_ret:
+ spin_unlock(&xdev->hw_prg_lock);
+ return rv;
+}
+
+int hw_indirect_ctext_prog(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ enum ind_ctxt_cmd_op op,
+ enum ind_ctxt_cmd_sel sel, u32 *data,
+ unsigned int cnt, bool verify)
+{
+ unsigned int reg;
+ u32 rd[QDMA_REG_IND_CTXT_REG_COUNT] = {0};
+ u32 v;
+ int i;
+ int rv = 0;
+
+ pr_debug("qid_hw 0x%x, op 0x%x, sel 0x%x, data 0x%p,%u, verify %d.\n",
+ qid_hw, op, sel, data, cnt, verify);
+
+ spin_lock(&xdev->hw_prg_lock);
+ if ((op == QDMA_CTXT_CMD_WR) || (op == QDMA_CTXT_CMD_RD)) {
+ if (unlikely(!cnt || cnt > QDMA_REG_IND_CTXT_REG_COUNT)) {
+ pr_warn("Q 0x%x, op 0x%x, sel 0x%x, cnt %u/%d.\n",
+ qid_hw, op, sel, cnt,
+ QDMA_REG_IND_CTXT_REG_COUNT);
+ rv = -EINVAL;
+ goto fn_ret;
+ }
+
+ if (unlikely(!data)) {
+ pr_warn("Q 0x%x, op 0x%x, sel 0x%x, data NULL.\n",
+ qid_hw, op, sel);
+ rv = -EINVAL;
+ goto fn_ret;
+ }
+
+ reg = QDMA_REG_IND_CTXT_MASK_BASE;
+ for (i = 0; i < QDMA_REG_IND_CTXT_REG_COUNT;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, 0xFFFFFFFF);
+
+ if (op == QDMA_CTXT_CMD_WR) {
+ reg = QDMA_REG_IND_CTXT_DATA_BASE;
+ for (i = 0; i < cnt;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, data[i]);
+ for (; i < QDMA_REG_IND_CTXT_REG_COUNT;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ __write_reg(xdev, reg, 0);
+ }
+ }
+
+ v = (qid_hw << IND_CTXT_CMD_QID_SHIFT) | (op << IND_CTXT_CMD_OP_SHIFT) |
+ (sel << IND_CTXT_CMD_SEL_SHIFT);
+
+ pr_debug("ctxt_cmd reg 0x%x, qid 0x%x, op 0x%x, sel 0x%x -> 0x%08x.\n",
+ QDMA_REG_IND_CTXT_CMD, qid_hw, op, sel, v);
+
+ __write_reg(xdev, QDMA_REG_IND_CTXT_CMD, v);
+
+ rv = hw_monitor_reg(xdev, QDMA_REG_IND_CTXT_CMD,
+ IND_CTXT_CMD_BUSY_MASK, 0, 100, 500*1000);
+ if (rv < 0) {
+ pr_info("%s, Q 0x%x, op 0x%x, sel 0x%x, timeout.\n",
+ xdev->conf.name, qid_hw, op, sel);
+ rv = -EBUSY;
+ goto fn_ret;
+ }
+
+ if (op == QDMA_CTXT_CMD_RD) {
+ reg = QDMA_REG_IND_CTXT_DATA_BASE;
+ for (i = 0; i < cnt;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ data[i] = __read_reg(xdev, reg);
+
+ goto fn_ret;
+ }
+
+ if (!verify)
+ goto fn_ret;
+
+ v = (qid_hw << IND_CTXT_CMD_QID_SHIFT) |
+ (QDMA_CTXT_CMD_RD << IND_CTXT_CMD_OP_SHIFT) |
+ (sel << IND_CTXT_CMD_SEL_SHIFT);
+
+ pr_debug("reg 0x%x, Q 0x%x, RD, sel 0x%x -> 0x%08x.\n",
+ QDMA_REG_IND_CTXT_CMD, qid_hw, sel, v);
+
+ __write_reg(xdev, QDMA_REG_IND_CTXT_CMD, v);
+
+ rv = hw_monitor_reg(xdev, QDMA_REG_IND_CTXT_CMD,
+ IND_CTXT_CMD_BUSY_MASK, 0, 100, 500*1000);
+ if (rv < 0) {
+ pr_warn("%s, Q 0x%x, op 0x%x, sel 0x%x, readback busy.\n",
+ xdev->conf.name, qid_hw, op, sel);
+ goto fn_ret;
+ }
+
+ reg = QDMA_REG_IND_CTXT_DATA_BASE;
+ for (i = 0; i < cnt;
+ i++, reg += QDMA_REG_SZ_IN_BYTES)
+ rd[i] = __read_reg(xdev, reg);
+
+ v = QDMA_REG_SZ_IN_BYTES * cnt;
+ if (memcmp(data, rd, v)) {
+ pr_warn("%s, indirect write data mismatch:\n", xdev->conf.name);
+ print_hex_dump(KERN_INFO, "WR ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)data, v, false);
+ print_hex_dump(KERN_INFO, "RD ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)rd, v, false);
+
+ rv = -EBUSY;
+ }
+
+fn_ret:
+ spin_unlock(&xdev->hw_prg_lock);
+
+ return rv;
+}
+
+#endif
+
+int qdma_queue_cmpl_ctrl(unsigned long dev_hndl, unsigned long id,
+ struct qdma_cmpl_ctrl *cctrl, bool set)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, NULL, 0, 1);
+ u32 val;
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (set) {
+ lock_descq(descq);
+
+ descq->conf.cmpl_trig_mode = cctrl->trigger_mode;
+ descq->conf.cmpl_timer_idx = cctrl->timer_idx;
+ descq->conf.cmpl_cnt_th_idx = cctrl->cnt_th_idx;
+ descq->conf.irq_en = cctrl->cmpl_en_intr;
+ descq->conf.cmpl_stat_en = cctrl->en_stat_desc;
+
+ descq_cmpt_cidx_update(descq, descq->cidx_cmpt_pend);
+
+ unlock_descq(descq);
+
+ } else {
+ /* read the setting */
+ val = __read_reg(descq->xdev, QDMA_REG_CMPT_CIDX_BASE +
+ descq->conf.qidx * QDMA_REG_PIDX_STEP);
+
+ cctrl->trigger_mode = descq->conf.cmpl_trig_mode =
+ (val >> S_CMPT_CIDX_UPD_TRIG_MODE) &
+ M_CMPT_CIDX_UPD_TRIG_MODE;
+ cctrl->timer_idx = descq->conf.cmpl_timer_idx =
+ (val >> S_CMPT_CIDX_UPD_TIMER_IDX) &
+ M_CMPT_CIDX_UPD_TIMER_IDX;
+ cctrl->cnt_th_idx = descq->conf.cmpl_cnt_th_idx =
+ (val >> S_CMPT_CIDX_UPD_CNTER_IDX) &
+ M_CMPT_CIDX_UPD_CNTER_IDX;
+ cctrl->cmpl_en_intr = descq->conf.irq_en =
+ (val & (1 << S_CMPT_CIDX_UPD_EN_INT)) ? 1 : 0;
+ cctrl->en_stat_desc = descq->conf.cmpl_stat_en =
+ (val & (1 << S_CMPT_CIDX_UPD_EN_STAT_DESC)) ? 1 : 0;
+ }
+
+ return 0;
+}
+
+#ifndef __QDMA_VF__
+int hw_init_qctxt_memory(struct xlnx_dma_dev *xdev, unsigned int qbase,
+ unsigned int qmax)
+{
+ u32 data[QDMA_REG_IND_CTXT_REG_COUNT];
+ unsigned int i = qbase;
+
+ memset(data, 0, sizeof(u32) * QDMA_REG_IND_CTXT_REG_COUNT);
+
+ for (; i < qmax; i++) {
+ enum ind_ctxt_cmd_sel sel = QDMA_CTXT_SEL_SW_C2H;
+ int rv;
+
+ for ( ; sel <= QDMA_CTXT_SEL_PFTCH; sel++) {
+ /** if the st mode(h2c/c2h) not enabled in the design,
+ * then skip the PFTCH and CMPT context setup
+ */
+ if ((xdev->st_mode_en == 0) &&
+ (sel == QDMA_CTXT_SEL_PFTCH ||
+ sel == QDMA_CTXT_SEL_CMPT)) {
+ pr_debug("ST context is skipped: sel = %d",
+ sel);
+ continue;
+ }
+
+ rv = hw_indirect_ctext_prog(xdev,
+ i, QDMA_CTXT_CMD_CLR,
+ sel, NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+ }
+ }
+
+ return 0;
+}
+
+int hw_init_global_context_memory(struct xlnx_dma_dev *xdev)
+{
+ unsigned int i;
+ int rv = 0;
+
+ /* queue context memory */
+ rv = hw_init_qctxt_memory(xdev, 0, QDMA_QSET_MAX);
+ if (rv < 0)
+ return rv;
+
+ if (xdev->conf.qdma_drv_mode == AUTO_MODE ||
+ xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE) {
+ /* interrupt aggregation context */
+ for (i = 0; i < QDMA_INTR_RNG_MAX; i++) {
+ rv = hw_indirect_ctext_prog(xdev,
+ i, QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_SEL_COAL, NULL, 0, 0);
+ if (rv < 0)
+ return rv;
+ }
+ }
+
+ /* fmap */
+ for (i = 0; i < QDMA_FUNC_MAX; i++)
+ hw_set_fmap(xdev, i, 0, 0);
+
+ return 0;
+}
+
+#endif
diff --git a/QDMA/linux-kernel/libqdma/qdma_regs.h b/QDMA/linux-kernel/libqdma/qdma_regs.h
new file mode 100644
index 0000000000000000000000000000000000000000..8c539e86e4badc49e54ce305f41e9e97231e89cf
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_regs.h
@@ -0,0 +1,1006 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_REGS_H__
+#define __QDMA_REGS_H__
+
+#include <linux/types.h>
+#include "xdev.h"
+
+#define QDMA_REG_SZ_IN_BYTES 4
+
+/* polling a register */
+#define QDMA_REG_POLL_DFLT_INTERVAL_US 100 /* 100us per poll */
+#define QDMA_REG_POLL_DFLT_TIMEOUT_US (500*1000) /* 500ms */
+
+/* desc. Q default */
+#define RNG_SZ_DFLT 64
+#define CMPT_RNG_SZ_DFLT 64
+#define C2H_TIMER_CNT_DFLT 0x1
+#define C2H_CNT_TH_DFLT 0x1
+#define C2H_BUF_SZ_DFLT PAGE_SIZE
+
+/* QDMA IP limits */
+#define QDMA_QSET_MAX 2048 /* 2K queues */
+#define QDMA_FUNC_MAX 256 /* 256 functions */
+#define QDMA_INTR_RNG_MAX 256 /* 256 interrupt aggregation rings */
+
+#define DESC_SZ_64B 3
+#define DESC_SZ_8B_BYTES 8
+#define DESC_SZ_16B_BYTES 16
+#define DESC_SZ_32B_BYTES 32
+#define DESC_SZ_64B_BYTES 64
+
+
+/* Driver visible Attribute Space 0x100 */
+#define QDMA_REG_GLBL_PF_BARLITE_INT 0x104
+#define PF_BARLITE_INT_3_SHIFT 18
+#define PF_BARLITE_INT_3_MASK 0xFC0000
+#define PF_BARLITE_INT_2_SHIFT 12
+#define PF_BARLITE_INT_2_MASK 0x3F000
+#define PF_BARLITE_INT_1_SHIFT 6
+#define PF_BARLITE_INT_1_MASK 0xFC0
+#define PF_BARLITE_INT_0_SHIFT 0
+#define PF_BARLITE_INT_0_MASK 0x3F
+
+#define MAGIC_NUMBER 0x1FD30000
+#define M_CONFIG_BAR_IDENTIFIER_MASK 0xFFFF0000
+#ifndef __QDMA_VF__
+#define QDMA_REG_CONFIG_BLOCK_IDENTIFIER 0x0
+#else
+#define QDMA_REG_CONFIG_BLOCK_IDENTIFIER 0x1014
+#endif
+#define QDMA_REG_PF_USER_BAR_IDENTIFIER 0x10C
+#define QDMA_REG_VF_USER_BAR_IDENTIFIER 0x1018
+#define M_USER_BAR_ID_MASK 0x3F
+
+#define QDMA_REG_GLBL_QMAX 0x120
+/* TBD : needs to be changed once we get the register offset from the h/w */
+#define QDMA_REG_GLBL_MM_ENGINES 0xABCD
+#define QDMA_REG_GLBL_MISC_CAP 0x134
+#define MISC_CAP_FLR_PRESENT_SHIFT 1
+#define MISC_CAP_FLR_PRESENT_MASK 0x2
+#define QDMA_REG_GLBL_MDMA_CHANNEL 0x118
+#define MDMA_CHANNEL_ST_C2H_ENABLED_SHIFT 16
+#define MDMA_CHANNEL_ST_C2H_ENABLED_MASK 0x10000
+#define MDMA_CHANNEL_ST_H2C_ENABLED_SHIFT 17
+#define MDMA_CHANNEL_ST_H2C_ENABLED_MASK 0x20000
+#define MDMA_CHANNEL_MM_C2H_ENABLED_SHIFT 8
+#define MDMA_CHANNEL_MM_C2H_ENABLED_MASK 0x100
+#define MDMA_CHANNEL_MM_H2C_ENABLED_SHIFT 0
+#define MDMA_CHANNEL_MM_H2C_ENABLED_MASK 0x1
+
+#ifndef __QDMA_VF__
+/*
+ * PF only registers
+ */
+#define QDMA_REG_FUNC_ID 0x12C
+
+/* CSR space 0x200 */
+#define QDMA_REG_GLBL_RNG_SZ_BASE 0x204
+#define QDMA_REG_GLBL_RNG_SZ_COUNT 16
+
+#define QDMA_REG_GLBL_SCRATCH 0x244
+
+#define QDMA_REG_GLBL_DSC_CFG 0x250
+#define QDMA_REG_GLBL_DSC_CFG_CMPL_STATUS_ACC_MASK 0x07
+#define QDMA_REG_GLBL_DSC_CFG_MAX_DESC_FETCH_MASK 0x07
+#define QDMA_REG_GLBL_DSC_CFG_MAX_DESC_FETCH_SHIFT 0x03
+
+#define QDMA_GLBL_INTERRUPT_CFG 0x2C4
+#define QDMA_GLBL_INTERRUPT_CFG_EN_LGCY_INTR (1 << 0)
+#define QDMA_GLBL_INTERRUPT_LGCY_INTR_PEND (1 << 1)
+
+#define QDMA_REG_C2H_TIMER_CNT_BASE 0xA00
+#define QDMA_REG_C2H_TIMER_CNT_COUNT 16
+
+#define QDMA_REG_C2H_CNT_TH_BASE 0xA40
+#define QDMA_REG_C2H_CNT_TH_COUNT 16
+
+#define QDMA_REG_C2H_BUF_SZ_BASE 0xAB0
+#define QDMA_REG_C2H_BUF_SZ_COUNT 16
+
+void qdma_csr_read_cmpl_status_acc(struct xlnx_dma_dev *xdev,
+ unsigned int *cs_acc);
+void qdma_csr_read_rngsz(struct xlnx_dma_dev *xdev, unsigned int *rngsz);
+int qdma_csr_write_rngsz(struct xlnx_dma_dev *xdev, unsigned int *rngsz);
+void qdma_csr_read_bufsz(struct xlnx_dma_dev *xdev, unsigned int *bufsz);
+int qdma_csr_write_bufsz(struct xlnx_dma_dev *xdev, unsigned int *bufsz);
+void qdma_csr_read_timer_cnt(struct xlnx_dma_dev *xdev, unsigned int *tmr_cnt);
+int qdma_csr_write_timer_cnt(struct xlnx_dma_dev *xdev, unsigned int *tmr_cnt);
+void qdma_csr_read_cnt_thresh(struct xlnx_dma_dev *xdev, unsigned int *cnt_th);
+int qdma_csr_write_cnt_thresh(struct xlnx_dma_dev *xdev, unsigned int *cnt_th);
+
+/*
+ * Function registers
+ */
+
+#define QDMA_REG_TRQ_SEL_FMAP_BASE 0x400
+#define QDMA_REG_TRQ_SEL_FMAP_STEP 0x4
+#define QDMA_REG_TRQ_SEL_FMAP_COUNT 256
+
+#define SEL_FMAP_QID_BASE_SHIFT 0
+#define SEL_FMAP_QID_BASE_MASK 0x7FFU
+#define SEL_FMAP_QID_MAX_SHIFT 11
+#define SEL_FMAP_QID_MAX_MASK 0xFFFU
+
+/*
+ * Indirect Programming
+ */
+#define QDMA_REG_IND_CTXT_REG_COUNT 8
+#define QDMA_REG_IND_CTXT_DATA_BASE 0x804
+#define QDMA_REG_IND_CTXT_MASK_BASE 0x824
+
+#define QDMA_REG_IND_CTXT_CMD 0x844
+#define IND_CTXT_CMD_QID_SHIFT 7
+#define IND_CTXT_CMD_QID_MASK 0xFFFFFFU
+
+#define IND_CTXT_CMD_OP_SHIFT 5
+#define IND_CTXT_CMD_OP_MASK 0x3U
+
+#define IND_CTXT_CMD_SEL_SHIFT 1
+#define IND_CTXT_CMD_SEL_MASK 0xFU
+
+#define IND_CTXT_CMD_BUSY_SHIFT 0
+#define IND_CTXT_CMD_BUSY_MASK 0x1U
+
+/*
+ * Queue registers
+ */
+#define QDMA_REG_MM_CONTROL_RUN 0x1U
+#define QDMA_REG_MM_CONTROL_STEP 0x100
+
+#define QDMA_REG_C2H_MM_CONTROL_BASE 0x1004
+#define QDMA_REG_H2C_MM_CONTROL_BASE 0x1204
+
+/*
+ * TRQ Sel registers
+ */
+#define QDMA_C2H_PFCH_CFG 0xB08
+#define QDMA_C2H_PFCH_CFG_EVT_QCNT_TH_SHIFT 25
+#define QDMA_C2H_PFCH_CFG_PFCH_QCNT_SHIFT 18
+#define QDMA_C2H_PFCH_CFG_NUM_PFCH_SHIFT 9
+#define QDMA_C2H_PFCH_CFG_PFCH_FL_TH_SHIFT 0
+
+#define QDMA_C2H_CMPT_COAL_CFG 0xB50
+#define QDMA_C2H_CMPT_COAL_CFG_TICK_CNT_SHIFT 2
+#define QDMA_C2H_CMPT_COAL_CFG_TICK_VAL_SHIFT 14
+#define QDMA_C2H_CMPT_COAL_CFG_MAX_BUF_SZ_SHIFT 26
+
+#define QDMA_C2H_INT_TIMER_TICK 0xB0C
+
+/*
+ * monitor
+ */
+#define QDMA_REG_C2H_STAT_AXIS_PKG_CMP 0xA94
+
+#endif /* ifndef __QDMA_VF__ */
+
+/*
+ * FLR
+ */
+#ifdef __QDMA_VF__
+#define QDMA_REG_FLR_STATUS 0x1100
+#else
+#define QDMA_REG_FLR_STATUS 0x2500
+#endif
+
+/*
+ * desc. Q pdix/cidx update
+ */
+
+#define QDMA_REG_PIDX_STEP 0x10
+
+#ifdef __QDMA_VF__
+
+#define QDMA_REG_INT_CIDX_BASE 0x3000
+#define QDMA_REG_H2C_PIDX_BASE 0x3004
+#define QDMA_REG_C2H_PIDX_BASE 0x3008
+#define QDMA_REG_CMPT_CIDX_BASE 0x300C
+
+#else
+
+#define QDMA_REG_INT_CIDX_BASE 0x18000
+#define QDMA_REG_H2C_PIDX_BASE 0x18004
+#define QDMA_REG_C2H_PIDX_BASE 0x18008
+#define QDMA_REG_CMPT_CIDX_BASE 0x1800C
+
+#endif
+
+/*
+ * Q Context programming (indirect)
+ */
+enum ind_ctxt_cmd_op {
+ QDMA_CTXT_CMD_CLR,
+ QDMA_CTXT_CMD_WR,
+ QDMA_CTXT_CMD_RD,
+ QDMA_CTXT_CMD_INV
+};
+
+enum ind_ctxt_cmd_sel {
+ QDMA_CTXT_SEL_SW_C2H,
+ QDMA_CTXT_SEL_SW_H2C,
+ QDMA_CTXT_SEL_HW_C2H,
+ QDMA_CTXT_SEL_HW_H2C,
+ QDMA_CTXT_SEL_CR_C2H,
+ QDMA_CTXT_SEL_CR_H2C,
+ QDMA_CTXT_SEL_CMPT,
+ QDMA_CTXT_SEL_PFTCH,
+ QDMA_CTXT_SEL_COAL,
+ QDMA_CTXT_SEL_PASID_RAM_LOW,
+ QDMA_CTXT_SEL_PASID_RAM_HIGH,
+ QDMA_CTXT_SEL_TIMER,
+ QDMA_CTXT_SEL_FMAP,
+};
+
+#define QDMA_REG_IND_CTXT_WCNT_1 1
+#define QDMA_REG_IND_CTXT_WCNT_2 2
+#define QDMA_REG_IND_CTXT_WCNT_3 3
+#define QDMA_REG_IND_CTXT_WCNT_4 4
+#define QDMA_REG_IND_CTXT_WCNT_5 5
+#define QDMA_REG_IND_CTXT_WCNT_6 6
+#define QDMA_REG_IND_CTXT_WCNT_7 7
+#define QDMA_REG_IND_CTXT_WCNT_8 8
+
+/** Context: FMAP */
+#define S_FMAP_W0_QBASE 0
+#define M_FMAP_W0_QBASE 0xFFFFFFU
+#define V_FMAP_W0_QBASE(x) ((x) << S_FMAP_W0_QBASE)
+
+#define S_FMAP_W1_QMAX 0
+#define M_FMAP_W1_QMAX 0xFFFFFFU
+#define V_FMAP_W1_QMAX(x) ((x) << S_FMAP_W1_QMAX)
+
+/* Context: Software */
+#define S_DESC_CTXT_W0_PIDX 0
+#define M_DESC_CTXT_W0_PIDX 0xFFFFU
+#define L_DESC_CTXT_W0_PIDX 16
+#define V_DESC_CTXT_W0_PIDX(x) ((x) << S_DESC_CTXT_W0_PIDX)
+
+#define S_DESC_CTXT_W0_F_INTR_ARM 16
+
+#define S_DESC_CTXT_W0_FUNC_ID 17
+#define M_DESC_CTXT_W0_FUNC_ID 0xFFU
+#define L_DESC_CTXT_W0_FUNC_ID 8
+#define V_DESC_CTXT_W0_FUNC_ID(x) ((x) << S_DESC_CTXT_W0_FUNC_ID)
+
+#define S_DESC_CTXT_W1_F_QEN 0
+#define S_DESC_CTXT_W1_F_FCRD_EN 1
+#define S_DESC_CTXT_W1_F_CMPL_STATUS_PEND_CHK 2
+#define S_DESC_CTXT_W1_F_CMPL_STATUS_ACC_EN 3
+#define S_DESC_CTXT_W1_F_AT 4
+
+#define S_DESC_CTXT_W1_FETCH_MAX 5
+#define L_DESC_CTXT_W1_FETCH_MAX 3
+
+#define S_DESC_CTXT_W1_RNG_SZ 12
+#define L_DESC_CTXT_W1_RNG_SZ 4
+#define M_DESC_CTXT_W1_RNG_SZ 0xFU
+#define V_DESC_CTXT_W1_RNG_SZ(x) ((x) << S_DESC_CTXT_W1_RNG_SZ)
+
+#define S_DESC_CTXT_W1_DSC_SZ 16
+#define L_DESC_CTXT_W1_DSC_SZ 2
+#define M_DESC_CTXT_W1_DSC_SZ 0x3U
+#define V_DESC_CTXT_W1_DSC_SZ(x) ((x) << S_DESC_CTXT_W1_DSC_SZ)
+
+#define S_DESC_CTXT_W1_F_BYP 18
+#define S_DESC_CTXT_W1_F_MM_CHN 19
+#define S_DESC_CTXT_W1_F_CMPL_STATUS_EN 20
+#define S_DESC_CTXT_W1_F_IRQ_EN 21
+
+#define S_DESC_CTXT_W1_PORT_ID 22
+#define L_DESC_CTXT_W1_PORT_ID 3
+#define M_DESC_CTXT_W1_PORT_ID 0x7U
+#define V_DESC_CTXT_W1_PORT_ID(x) ((x) << S_DESC_CTXT_W1_PORT_ID)
+
+#define S_DESC_CTXT_W1_F_IRQ_NO_LAST 25
+
+#define S_DESC_CTXT_W1_ERR 26
+#define M_DESC_CTXT_W1_ERR 0x3U
+#define L_DESC_CTXT_W1_ERR 2
+#define V_DESC_CTXT_W1_ERR(x) ((x) << S_DESC_CTXT_W1_ERR)
+
+#define S_DESC_CTXT_W1_F_CMPL_STATUS_ERR_SNT 28
+#define S_DESC_CTXT_W1_F_IRQ_REQ 29
+#define S_DESC_CTXT_W1_F_MRKR_DIS 30
+#define S_DESC_CTXT_W1_F_IS_MM 31
+
+#define S_DESC_CTXT_W4_VEC 0
+#define M_DESC_CTXT_W4_VEC 0x7FF
+#define L_DESC_CTXT_W4_VEC 11
+#define V_DESC_CTXT_W4_VEC(x) ((x) << S_DESC_CTXT_W4_VEC)
+
+#define S_DESC_CTXT_W1_F_INTR_AGGR 11
+
+/* Context: C2H Completion Context */
+
+#define S_CMPT_CTXT_W0_F_EN_STAT_DESC 0
+#define S_CMPT_CTXT_W0_F_EN_INT 1
+
+#define S_CMPT_CTXT_W0_TRIG_MODE 2
+#define M_CMPT_CTXT_W0_TRIG_MODE 0x7U
+#define L_CMPT_CTXT_W0_TRIG_MODE 3
+#define V_CMPT_CTXT_W0_TRIG_MODE(x) ((x) << S_CMPT_CTXT_W0_TRIG_MODE)
+
+#define S_CMPT_CTXT_W0_FNC_ID 5
+#define M_CMPT_CTXT_W0_FNC_ID 0xFFFU
+#define L_CMPT_CTXT_W0_FNC_ID 12
+#define V_CMPT_CTXT_W0_FNC_ID(x) \
+ ((x & M_CMPT_CTXT_W0_FNC_ID) << S_CMPT_CTXT_W0_FNC_ID)
+
+#define S_CMPT_CTXT_W0_COUNTER_IDX 17
+#define M_CMPT_CTXT_W0_COUNTER_IDX 0xFU
+#define L_CMPT_CTXT_W0_COUNTER_IDX 4
+#define V_CMPT_CTXT_W0_COUNTER_IDX(x) ((x) << S_CMPT_CTXT_W0_COUNTER_IDX)
+
+#define S_CMPT_CTXT_W0_TIMER_IDX 21
+#define M_CMPT_CTXT_W0_TIMER_IDX 0xFU
+#define L_CMPT_CTXT_W0_TIMER_IDX 4
+#define V_CMPT_CTXT_W0_TIMER_IDX(x) ((x) << S_CMPT_CTXT_W0_TIMER_IDX)
+
+#define S_CMPT_CTXT_W0_INT_ST 25
+#define M_CMPT_CTXT_W0_INT_ST 0x3U
+#define L_CMPT_CTXT_W0_INT_ST 2
+#define V_CMPT_CTXT_W0_INT_ST(x) ((x) << S_CMPT_CTXT_W0_INT_ST)
+
+#define S_CMPT_CTXT_W0_F_COLOR 27
+
+#define S_CMPT_CTXT_W0_RNG_SZ 28
+#define M_CMPT_CTXT_W0_RNG_SZ 0xFU
+#define L_CMPT_CTXT_W0_RNG_SZ 4
+#define V_CMPT_CTXT_W0_RNG_SZ(x) ((x) << S_CMPT_CTXT_W0_RNG_SZ)
+
+#define S_CMPT_CTXT_W2_BADDR_64 0
+#define M_CMPT_CTXT_W2_BADDR_64 0x3FFFFFFU
+#define L_CMPT_CTXT_W2_BADDR_64 26
+#define V_CMPT_CTXT_W2_BADDR_64(x) \
+ (((x) & M_CMPT_CTXT_W2_BADDR_64) << S_CMPT_CTXT_W2_BADDR_64)
+
+#define S_CMPT_CTXT_W2_DESC_SIZE 26
+#define M_CMPT_CTXT_W2_DESC_SIZE 0x3U
+#define L_CMPT_CTXT_W2_DESC_SIZE 2
+#define V_CMPT_CTXT_W2_DESC_SIZE(x) ((x) << S_CMPT_CTXT_W2_DESC_SIZE)
+
+#define S_CMPT_CTXT_W2_PIDX_L 28
+#define M_CMPT_CTXT_W2_PIDX_L 0xFU
+#define L_CMPT_CTXT_W2_PIDX_L 4
+#define V_CMPT_CTXT_W2_PIDX_L(x) ((x) << S_CMPT_CTXT_W2_PIDX_L)
+
+#define S_CMPT_CTXT_W3_PIDX_H 0
+#define M_CMPT_CTXT_W3_PIDX_H 0xFFFU
+#define L_CMPT_CTXT_W3_PIDX_H 12
+#define V_CMPT_CTXT_W3_PIDX_H(x) ((x) << S_CMPT_CTXT_W3_PIDX_H)
+
+#define S_CMPT_CTXT_W3_CIDX 12
+#define M_CMPT_CTXT_W3_CIDX 0xFFFFU
+#define L_CMPT_CTXT_W3_CIDX 16
+#define V_CMPT_CTXT_W3_CIDX(x) ((x) << S_CMPT_CTXT_W3_CIDX)
+
+#define S_CMPT_CTXT_W3_F_VALID 28
+
+#define S_CMPT_CTXT_W3_ERR 29
+#define M_CMPT_CTXT_W3_ERR 0x3U
+#define L_CMPT_CTXT_W3_ERR 2
+#define V_CMPT_CTXT_W3_ERR(x) ((x) << S_CMPT_CTXT_W3_ERR)
+
+#define S_CMPT_CTXT_W3_F_TRIG_PEND 31
+
+#define S_CMPT_CTXT_W4_F_TMR_RUNNING 0
+#define S_CMPT_CTXT_W4_F_FULL_UPDATE 1
+#define S_CMPT_CTXT_W4_F_OVF_CHK_DIS 2
+#define S_CMPT_CTXT_W4_F_AT 3
+
+#define S_CMPT_CTXT_W4_VEC 4
+#define M_CMPT_CTXT_W4_VEC 0x7FF
+#define L_CMPT_CTXT_W4_VEC 11
+#define V_CMPT_CTXT_W4_VEC(x) ((x) << S_CMPT_CTXT_W4_VEC)
+
+#define S_CMPT_CTXT_W4_F_INTR_AGGR 15
+
+/* Context: C2H Prefetch */
+#define S_PFTCH_W0_F_BYPASS 0
+
+#define S_PFTCH_W0_BUF_SIZE_IDX 1
+#define M_PFTCH_W0_BUF_SIZE_IDX 0xFU
+#define L_PFTCH_W0_BUF_SIZE_IDX 4
+#define V_PFTCH_W0_BUF_SIZE_IDX(x) ((x) << S_PFTCH_W0_BUF_SIZE_IDX)
+
+#define S_PFTCH_W0_PORT_ID 5
+#define M_PFTCH_W0_PORT_ID 0x7U
+#define L_PFTCH_W0_PORT_ID 3
+#define V_PFTCH_W0_PORT_ID(x) ((x) << S_PFTCH_W0_PORT_ID)
+
+/* TBD: uncomment when introduced back in future RTLs
+ * #define S_PFTCH_W0_FUNC_ID 8
+ * #define M_PFTCH_W0_FUNC_ID 0xFFFU
+ * #define L_PFTCH_W0_FUNC_ID 12
+ * #define V_PFTCH_W0_FUNC_ID(x) ((x) << S_PFTCH_W0_FUNC_ID)
+ */
+
+#define S_PFTCH_W0_F_ERR 26
+#define S_PFTCH_W0_F_EN_PFTCH 27
+#define S_PFTCH_W0_F_Q_IN_PFTCH 28
+
+#define S_PFTCH_W0_SW_CRDT_L 29
+#define M_PFTCH_W0_SW_CRDT_L 0x7U
+#define L_PFTCH_W0_SW_CRDT_L 3
+#define V_PFTCH_W0_SW_CRDT_L(x) ((x) << S_PFTCH_W0_SW_CRDT_L)
+
+#define S_PFTCH_W1_SW_CRDT_H 0
+#define M_PFTCH_W1_SW_CRDT_H 0x1FFFU
+#define L_PFTCH_W1_SW_CRDT_H 13
+#define V_PFTCH_W1_SW_CRDT_H(x) ((x) << S_PFTCH_W1_SW_CRDT_H)
+
+#define S_PFTCH_W1_F_VALID 13
+
+/* Context: Interrupt Coalescing */
+
+#define S_INT_AGGR_W0_F_VALID 0
+
+#define S_INT_AGGR_W0_VEC_ID 1
+#define M_INT_AGGR_W0_VEC_ID 0x7FFU
+#define L_INT_AGGR_W0_VEC_ID 11
+#define V_INT_AGGR_W0_VEC_ID(x) ((x) << S_INT_AGGR_W0_VEC_ID)
+
+#define S_INT_AGGR_W0_F_INT_ST 13
+#define S_INT_AGGR_W0_F_COLOR 14
+
+#define S_INT_AGGR_W0_BADDR_64 15
+#define M_INT_AGGR_W0_BADDR_64 0x1FFFFU
+#define L_INT_AGGR_W0_BADDR_64 17
+#define V_INT_AGGR_W0_BADDR_64(x) \
+ (((x) & M_INT_AGGR_W0_BADDR_64) << S_INT_AGGR_W0_BADDR_64)
+
+#define S_INT_AGGR_W2_BADDR_64 0
+#define M_INT_AGGR_W2_BADDR_64 0x7U
+#define L_INT_AGGR_W2_BADDR_64 3
+#define V_INT_AGGR_W2_BADDR_64(x) \
+ (((x) & M_INT_AGGR_W2_BADDR_64) << S_INT_AGGR_W2_BADDR_64)
+
+#define S_INT_AGGR_W2_VEC_SIZE 3
+#define M_INT_AGGR_W2_VEC_SIZE 0x7U
+#define L_INT_AGGR_W2_VEC_SIZE 3
+#define V_INT_AGGR_W2_VEC_SIZE(x) ((x) << S_INT_AGGR_W2_VEC_SIZE)
+
+#define S_INT_AGGR_W2_PIDX 6
+#define M_INT_AGGR_W2_PIDX 0xFFFU
+#define L_INT_AGGR_W2_PIDX 12
+#define V_INT_AGGR_W2_PIDX ((x) << S_INT_AGGR_W2_PIDX)
+
+#define S_INT_AGGR_W2_F_AT 18
+
+
+/*
+ * PIDX/CIDX update
+ */
+
+#define S_INTR_CIDX_UPD_SW_CIDX 0
+#define M_INTR_CIDX_UPD_SW_CIDX 0xFFFFU
+#define V_INTR_CIDX_UPD_SW_CIDX(x) ((x) << S_INTR_CIDX_UPD_SW_CIDX)
+
+#define S_INTR_CIDX_UPD_RING_INDEX 16
+#define M_INTR_CIDX_UPD_RING_INDEX 0xFFU
+#define V_INTR_CIDX_UPD_RING_INDEX(x) ((x) << S_INTR_CIDX_UPD_RING_INDEX)
+
+#define S_CMPL_STATUS_PIDX_UPD_EN_INT 16
+
+/*
+ * CMPT CIDX update
+ */
+#define S_CMPT_CIDX_UPD_SW_CIDX 0
+#define M_CMPT_CIDX_UPD_SW_IDX 0xFFFFU
+#define V_CMPT_CIDX_UPD_SW_IDX(x) ((x) << S_CMPT_CIDX_UPD_SW_CIDX)
+
+#define S_CMPT_CIDX_UPD_CNTER_IDX 16
+#define M_CMPT_CIDX_UPD_CNTER_IDX 0xFU
+#define V_CMPT_CIDX_UPD_CNTER_IDX(x) ((x) << S_CMPT_CIDX_UPD_CNTER_IDX)
+
+#define S_CMPT_CIDX_UPD_TIMER_IDX 20
+#define M_CMPT_CIDX_UPD_TIMER_IDX 0xFU
+#define V_CMPT_CIDX_UPD_TIMER_IDX(x) ((x) << S_CMPT_CIDX_UPD_TIMER_IDX)
+
+#define S_CMPT_CIDX_UPD_TRIG_MODE 24
+#define M_CMPT_CIDX_UPD_TRIG_MODE 0x7U
+#define V_CMPT_CIDX_UPD_TRIG_MODE(x) ((x) << S_CMPT_CIDX_UPD_TRIG_MODE)
+
+#define S_CMPT_CIDX_UPD_EN_STAT_DESC 27
+#define S_CMPT_CIDX_UPD_EN_INT 28
+
+/*
+ * descriptor & writeback status
+ */
+/**
+ * @struct - qdma_mm_desc
+ * @brief memory mapped descriptor format
+ */
+struct qdma_mm_desc {
+ /** source address */
+ __be64 src_addr;
+ /** flags */
+ __be32 flag_len;
+ /** reserved 32 bits */
+ __be32 rsvd0;
+ /** destination address */
+ __be64 dst_addr;
+ /** reserved 64 bits */
+ __be64 rsvd1;
+};
+
+#define S_DESC_F_DV 28
+#define S_DESC_F_SOP 29
+#define S_DESC_F_EOP 30
+
+
+#define S_H2C_DESC_F_SOP 1
+#define S_H2C_DESC_F_EOP 2
+
+
+#define S_H2C_DESC_NUM_GL 0
+#define M_H2C_DESC_NUM_GL 0x7U
+#define V_H2C_DESC_NUM_GL(x) ((x) << S_H2C_DESC_NUM_GL)
+
+#define S_H2C_DESC_NUM_CDH 3
+#define M_H2C_DESC_NUM_CDH 0xFU
+#define V_H2C_DESC_NUM_CDH(x) ((x) << S_H2C_DESC_NUM_CDH)
+
+#define S_H2C_DESC_F_ZERO_CDH 13
+#define S_H2C_DESC_F_EOT 14
+#define S_H2C_DESC_F_REQ_CMPL_STATUS 15
+
+/* FIXME pld_len and flags members are part of custom descriptor format needed
+ * by example design for ST loopback and desc bypass
+ */
+/**
+ * @struct - qdma_h2c_desc
+ * @brief memory mapped descriptor format
+ */
+struct qdma_h2c_desc {
+ __be16 cdh_flags; /**< cdh flags */
+ __be16 pld_len; /**< current packet length */
+ __be16 len; /**< total packet length */
+ __be16 flags; /**< descriptor flags */
+ __be64 src_addr; /**< source address */
+};
+
+/**
+ * @struct - qdma_c2h_desc
+ * @brief qdma c2h descriptor
+ */
+struct qdma_c2h_desc {
+ __be64 dst_addr; /**< destination address */
+};
+
+/**
+ * @struct - qdma_desc_cmpl_status
+ * @brief qdma writeback descriptor
+ */
+struct qdma_desc_cmpl_status {
+ __be16 pidx; /**< producer index */
+ __be16 cidx; /**< consumer index */
+ __be32 rsvd; /**< reserved 32 bits */
+};
+
+#define S_C2H_CMPT_ENTRY_F_FORMAT 0
+#define F_C2H_CMPT_ENTRY_F_FORMAT (1 << S_C2H_CMPT_ENTRY_F_FORMAT)
+#define DFORMAT0_CMPL_MASK 0xF /* udd starts at bit 4 */
+#define DFORMAT1_CMPL_MASK 0xFFFFF /* udd starts at bit 20 */
+
+
+#define S_C2H_CMPT_ENTRY_F_COLOR 1
+#define F_C2H_CMPT_ENTRY_F_COLOR (1 << S_C2H_CMPT_ENTRY_F_COLOR)
+
+#define S_C2H_CMPT_ENTRY_F_ERR 2
+#define F_C2H_CMPT_ENTRY_F_ERR (1 << S_C2H_CMPT_ENTRY_F_ERR)
+
+#define S_C2H_CMPT_ENTRY_F_DESC_USED 3
+#define F_C2H_CMPT_ENTRY_F_DESC_USED (1 << S_C2H_CMPT_ENTRY_F_DESC_USED)
+
+#define S_C2H_CMPT_ENTRY_LENGTH 4
+#define M_C2H_CMPT_ENTRY_LENGTH 0xFFFFU
+#define L_C2H_CMPT_ENTRY_LENGTH 16
+#define V_C2H_CMPT_ENTRY_LENGTH(x) \
+ (((x) & M_C2H_CMPT_ENTRY_LENGTH) << S_C2H_CMPT_ENTRY_LENGTH)
+
+#define S_C2H_CMPT_ENTRY_F_EOT 20
+#define F_C2H_CMPT_ENTRY_F_EOT (1 << S_C2H_CMPT_ENTRY_F_EOT)
+
+#define S_C2H_CMPT_ENTRY_F_USET_INTR 21
+
+#define S_C2H_CMPT_USER_DEFINED 22
+#define V_C2H_CMPT_USER_DEFINED(x) ((x) << S_C2H_CMPT_USER_DEFINED)
+
+#define M_C2H_CMPT_ENTRY_DMA_INFO 0xFFFFFF
+#define L_C2H_CMPT_ENTRY_DMA_INFO 3 /* 20 bits */
+/**
+ * @struct - qdma_c2h_cmpt_cmpl_status
+ * @brief qdma completion data descriptor
+ */
+struct qdma_c2h_cmpt_cmpl_status {
+ __be16 pidx; /**< producer index */
+ __be16 cidx; /**< consumer index */
+ __be32 color_isr_status; /**< isr color and status */
+};
+#define S_C2H_CMPT_F_COLOR 0
+
+#define S_C2H_CMPT_INT_STATE 1
+#define M_C2H_CMPT_INT_STATE 0x3U
+
+#define STM_REG_BASE 0x02000000
+#define STM_REG_IND_CTXT_DATA_BASE 0x0
+#define STM_REG_IND_CTXT_DATA3 0xC
+#define STM_REG_IND_CTXT_CMD 0x14
+#define STM_REG_REV 0x18
+#define STM_REG_C2H_DATA8 0x20
+#define STM_REG_IND_CTXT_DATA5 0x24
+#define STM_REG_H2C_MODE 0x30
+#define STM_REG_IND_CTXT_REG_COUNT 5
+#define STM_SUPPORTED_REV 0x4
+#define STM_ENABLED_DEVICE 0x6aa0
+#define STM_MAX_SUPPORTED_QID 64
+#define STM_MAX_PKT_SIZE 4096
+#define STM_PORT_MAP 0xE1E1
+
+#define S_STM_H2C_CTXT_ENTRY_VALID 0
+#define F_STM_H2C_CTXT_ENTRY_VALID (1 << S_STM_H2C_CTXT_ENTRY_VALID)
+#define S_STM_C2H_CTXT_ENTRY_VALID 16
+#define F_STM_C2H_CTXT_ENTRY_VALID (1 << S_STM_C2H_CTXT_ENTRY_VALID)
+#define S_STM_EN_STMA_BKCHAN 15
+#define F_STM_EN_STMA_BKCHAN (1 << S_STM_EN_STMA_BKCHAN)
+#define S_STM_PORT_MAP 16
+
+enum ind_stm_addr {
+ STM_IND_ADDR_C2H_MAP = 0x2,
+ STM_IND_ADDR_FORCED_CAN = 0x8,
+ STM_IND_ADDR_Q_CTX_H2C,
+ STM_IND_ADDR_H2C_MAP,
+ STM_IND_ADDR_Q_CTX_C2H,
+};
+
+enum ind_stm_cmd_op {
+ STM_CSR_CMD_WR = 4,
+ STM_CSR_CMD_RD = 8,
+};
+
+#define S_STM_CTXT_QID 16
+#define S_STM_CTXT_C2H_SLR 8
+#define S_STM_CTXT_C2H_TDEST_H 0
+#define S_STM_CTXT_C2H_TDEST_L 24
+#define S_STM_CTXT_C2H_FID 16
+#define S_STM_CTXT_H2C_SLR 8
+#define S_STM_CTXT_H2C_TDEST_H 0
+#define S_STM_CTXT_H2C_TDEST_L 24
+#define S_STM_CTXT_H2C_FID 16
+#define S_STM_CTXT_PKT_LIM 8
+#define S_STM_CTXT_MAX_ASK 0
+#define S_STM_CTXT_DPPKT 24
+#define S_STM_CTXT_LOG2_DPPKT 18
+
+#define S_STM_CMD_QID 0
+#define S_STM_CMD_FID 12
+#define S_STM_CMD_ADDR 24
+#define S_STM_CMD_OP 28
+
+/*
+ * HW API
+ */
+
+#include "xdev.h"
+
+#define __read_reg(xdev, reg_addr) (readl(xdev->regs + reg_addr))
+#ifdef DEBUG__
+#define __write_reg(xdev, reg_addr, val) \
+ do { \
+ pr_debug("%s, reg 0x%x, val 0x%x.\n", \
+ xdev->conf.name, reg_addr, (u32)val); \
+ writel(val, xdev->regs + reg_addr); \
+ } while (0)
+#else
+#define __write_reg(xdev, reg_addr, val) (writel(val, xdev->regs + reg_addr))
+#endif /* #ifdef DEBUG__ */
+
+struct xlnx_dma_dev;
+int hw_monitor_reg(struct xlnx_dma_dev *xdev, unsigned int reg, u32 mask,
+ u32 val, unsigned int interval_us, unsigned int timeout_us);
+#ifndef __QDMA_VF__
+void qdma_device_attributes_get(struct xlnx_dma_dev *xdev);
+void hw_mm_channel_enable(struct xlnx_dma_dev *xdev, int channel, bool c2h);
+void hw_mm_channel_disable(struct xlnx_dma_dev *xdev, int channel, bool c2h);
+void hw_set_global_csr(struct xlnx_dma_dev *xdev);
+int hw_set_fmap(struct xlnx_dma_dev *xdev, u16 func_id, unsigned int qbase,
+ unsigned int qmax);
+int hw_read_fmap(struct xlnx_dma_dev *xdev, u16 func_id, unsigned int *qbase,
+ unsigned int *qmax);
+int hw_indirect_ctext_prog(struct xlnx_dma_dev *xdev, unsigned int qid,
+ enum ind_ctxt_cmd_op op,
+ enum ind_ctxt_cmd_sel sel, u32 *data,
+ unsigned int cnt, bool verify);
+int hw_init_global_context_memory(struct xlnx_dma_dev *xdev);
+int hw_init_qctxt_memory(struct xlnx_dma_dev *xdev, unsigned int qbase,
+ unsigned int qmax);
+int hw_indirect_stm_prog(struct xlnx_dma_dev *xdev, unsigned int qid_hw,
+ u8 fid, enum ind_stm_cmd_op op,
+ enum ind_stm_addr addr, u32 *data, unsigned int cnt,
+ bool clear);
+int qdma_trq_c2h_config(struct xlnx_dma_dev *xdev);
+
+#endif /* #ifndef __QDMA_VF__ */
+
+#ifndef __QDMA_VF__
+#define QDMA_VERSION_REG 0x134
+#define M_RTL_VERSION_MASK 0xFF0000
+#define S_RTL_VERSION_SHIFT 16
+#define M_VIVADO_RELEASE_ID_MASK 0x0F000000
+#define S_VIVADO_RELEASE_ID_SHIFT 24
+#define M_EVEREST_IP_MASK 0x10000000
+#define S_EVEREST_IP_SHIFT 28
+#else
+#define QDMA_VERSION_REG 0x1014
+#define M_RTL_VERSION_MASK 0xFF
+#define S_RTL_VERSION_SHIFT 0
+#define M_VIVADO_RELEASE_ID_MASK 0x0F00
+#define S_VIVADO_RELEASE_ID_SHIFT 8
+#define M_EVEREST_IP_MASK 0x1000
+#define S_EVEREST_IP_SHIFT 12
+#endif
+
+/* HW Error Registers */
+
+#define QDMA_C2H_ERR_INT 0x0B04
+#define S_QDMA_C2H_ERR_INT_FUNC 0
+#define M_QDMA_C2H_ERR_INT_FUNC 0xFFU
+#define V_QDMA_C2H_ERR_INT_FUNC(x) ((x) << S_QDMA_C2H_ERR_INT_FUNC)
+
+#define S_QDMA_C2H_ERR_INT_VEC 12
+#define M_QDMA_C2H_ERR_INT_VEC 0x7FFU
+#define V_QDMA_C2H_ERR_INT_VEC(x) ((x) << S_QDMA_C2H_ERR_INT_VEC)
+
+#define S_QDMA_C2H_ERR_INT_F_ERR_INT_ARM 24
+
+
+#define QDMA_REG_GLBL_ERR_STAT 0x248
+#define QDMA_REG_GLBL_ERR_MASK 0x24C
+#define QDMA_REG_GLBL_ERR_MASK_VALUE 0xFFFU
+#define QDMA_REG_GLBL_F_ERR_RAM_SBE 0
+#define QDMA_REG_GLBL_F_ERR_RAM_DBE 1
+#define QDMA_REG_GLBL_F_ERR_DSC 2
+#define QDMA_REG_GLBL_F_ERR_TRQ 3
+#define QDMA_REG_GLBL_F_ERR_H2C_MM_0 4
+#define QDMA_REG_GLBL_F_ERR_H2C_MM_1 5
+#define QDMA_REG_GLBL_F_ERR_C2H_MM_0 6
+#define QDMA_REG_GLBL_F_ERR_C2H_MM_1 7
+#define QDMA_REG_GLBL_F_ERR_C2H_ST 8
+#define QDMA_REG_GLBL_F_ERR_IND_CTXT_CMD 9
+#define QDMA_REG_GLBL_F_ERR_BDG 10
+#define QDMA_REG_GLBL_F_ERR_H2C_ST 11
+
+
+/* Global Descriptor Error */
+#define QDMA_GLBL_DSC_ERR_STS 0x254
+#define QDMA_GLBL_DSC_ERR_MSK 0x258
+#define QDMA_GLBL_DSC_ERR_MSK_VALUE 0x1F9023FU
+#define QDMA_GLBL_DSC_ERR_STS_A_F_HDR_POIS 0
+#define QDMA_GLBL_DSC_ERR_STS_A_F_UR_CA 1
+#define QDMA_GLBL_DSC_ERR_STS_A_F_PARAM_MISMATCH 2
+#define QDMA_GLBL_DSC_ERR_STS_A_F_UNEXP_ADDR 3
+#define QDMA_GLBL_DSC_ERR_STS_A_F_TAG 4
+#define QDMA_GLBL_DSC_ERR_STS_A_F_FLR 5
+#define QDMA_GLBL_DSC_ERR_STS_A_F_TIMEOUT 9
+#define QDMA_GLBL_DSC_ERR_STS_A_F_DATA_POIS 16
+#define QDMA_GLBL_DSC_ERR_STS_A_F_FLR_CANCEL 19
+#define QDMA_GLBL_DSC_ERR_STS_A_F_DMA 20
+#define QDMA_GLBL_DSC_ERR_STS_A_F_DSC 21
+#define QDMA_GLBL_DSC_ERR_STS_A_F_RQ_CHAN 22
+#define QDMA_GLBL_DSC_ERR_STS_A_F_RAM_DBE 23
+#define QDMA_GLBL_DSC_ERR_STS_A_F_RAM_SBE 24
+
+
+#define QDMA_GLBL_DSC_ERR_LOG0 0x25C
+#define QDMA_GLBL_DSC_ERR_LOG1 0x260
+
+
+
+#define QDMA_GLBL_TRQ_ERR_STS 0x264
+#define QDMA_GLBL_TRQ_ERR_STS_F_UN_MAPPED 0
+#define QDMA_GLBL_TRQ_ERR_STS_F_QID_RANGE 1
+#define QDMA_GLBL_TRQ_ERR_STS_F_VF_ACCESS 2
+#define QDMA_GLBL_TRQ_ERR_STS_F_TCP_TIMEOUT 3
+
+#define QDMA_GLBL_TRQ_ERR_MSK 0x268
+#define QDMA_GLBL_TRQ_ERR_MSK_VALUE 0xFU
+
+#define QDMA_GLBL_TRQ_ERR_LOG 0x26C
+#define S_QDMA_GLBL_TRQ_ERR_LOG_ADDR 0
+#define M_QDMA_GLBL_TRQ_ERR_LOG_ADDR 0xFFFFU
+#define V_QDMA_GLBL_TRQ_ERR_LOG_ADDR(x) \
+ ((x) << S_QDMA_GLBL_TRQ_ERR_LOG_ADDR)
+#define S_QDMA_GLBL_TRQ_ERR_LOG_FUNC 16
+#define M_QDMA_GLBL_TRQ_ERR_LOG_FUNC 0xFFFU
+#define V_QDMA_GLBL_TRQ_ERR_LOG_FUNC(x) \
+ ((x) << S_QDMA_GLBL_TRQ_ERR_LOG_FUNC)
+#define S_QDMA_GLBL_TRQ_ERR_LOG_TARGET 28
+#define M_QDMA_GLBL_TRQ_ERR_LOG_TARGET 0xFU
+#define V_QDMA_GLBL_TRQ_ERR_LOG_TARGET(x) \
+ ((x) << S_QDMA_GLBL_TRQ_ERR_LOG_TARGET)
+
+/* TRQ errors */
+/**
+ * trq_err_sel - possible trq errors
+ */
+enum trq_err_sel {
+ /**< trq errors being to global 1 registers*/
+ QDMA_TRQ_ERR_SEL_GLBL1 = 1,
+ /**< trq errors being to global 2 registers*/
+ QDMA_TRQ_ERR_SEL_GLBL2 = 2,
+ /**< trq errors being to global registers*/
+ QDMA_TRQ_ERR_SEL_GLBL,
+ /**< trq errors being to fmap registers*/
+ QDMA_TRQ_ERR_SEL_FMAP,
+ /**< trq errors being to indirect interrupt*/
+ QDMA_TRQ_ERR_SEL_IND,
+ /**< trq errors being to c2h registers*/
+ QDMA_TRQ_ERR_SEL_C2H,
+ /**< trq errors being to c2h mm0 registers*/
+ QDMA_TRQ_ERR_SEL_C2H_MM0 = 9,
+ /**< trq errors being to h2c mm0 registers*/
+ QDMA_TRQ_ERR_SEL_H2C_MM0 = 11,
+ /**< trq errors being to pf queue registers */
+ QDMA_TRQ_ERR_SEL_QUEUE_PF = 13,
+};
+
+/* C2H ERROR Status Registers */
+#define QDMA_REG_C2H_ERR_STAT 0xAF0
+#define QDMA_REG_C2H_ERR_MASK 0xAF4
+#define QDMA_REG_C2H_ERR_MASK_VALUE 0xFEDBU
+#define QDMA_REG_C2H_ERR_F_MTY_MISMATCH 0
+#define QDMA_REG_C2H_ERR_F_LEN_MISMATCH 1
+#define QDMA_REG_C2H_ERR_F_QID_MISMATCH 3
+#define QDMA_REG_C2H_ERR_F_DSC_RSP_ERR 4
+#define QDMA_REG_C2H_ERR_F_ENG_WPL_DATA_PAR 6
+#define QDMA_REG_C2H_ERR_F_MSI_INT_FAIL 7
+#define QDMA_REG_C2H_ERR_F_DESC_CNT 9
+#define QDMA_REG_C2H_ERR_F_PORT_ID_CTXT_MISMATCH 10
+#define QDMA_REG_C2H_ERR_F_PORT_ID_BYP_IN_MISMATCH 11
+#define QDMA_REG_C2H_ERR_F_CMPT_INV_Q 12
+#define QDMA_REG_C2H_ERR_F_CMPT_QFULL 13
+#define QDMA_REG_C2H_ERR_F_CMPT_CIDX 14
+#define QDMA_REG_C2H_ERR_F_CMPT_PRTY 15
+
+
+#define QDMA_C2H_FATAL_ERR_STAT 0xAF8
+#define QDMA_C2H_FATAL_ERR_MASK 0xAFC
+#define QDMA_C2H_FATAL_ERR_MASK_VALUE 0x7DF1BU
+#define QDMA_C2H_FATAL_ERR_STAT_MTY_MISMATCH 0
+#define QDMA_C2H_FATAL_ERR_STAT_LEN_MISMATCH 1
+#define QDMA_C2H_FATAL_ERR_STAT_QID_MISMATCH 3
+#define QDMA_C2H_FATAL_ERR_STAT_TIMER_FIFO_RAM_RDBE 4
+#define QDMA_C2H_FATAL_ERR_STAT_PFTCH_LL_RAM_RDBE 8
+#define QDMA_C2H_FATAL_ERR_STAT_CMPT_CTXT_RAM_RDBE 9
+#define QDMA_C2H_FATAL_ERR_STAT_PFTCH_CTXT_RAM_RDBE 10
+#define QDMA_C2H_FATAL_ERR_STAT_DESC_REQ_FIFO_RAM_RDBE 11
+#define QDMA_C2H_FATAL_ERR_STAT_INT_CTXT_RAM_RDBE 12
+#define QDMA_C2H_FATAL_ERR_STAT_CMPT_AGGR_DAT_RAM_DBE 14
+#define QDMA_C2H_FATAL_ERR_STAT_TUSER_FIFO_RAM_DBE 15
+#define QDMA_C2H_FATAL_ERR_STAT_QID_FIFO_RAM_DBE 16
+#define QDMA_C2H_FATAL_ERR_STAT_PLD_FIFO_RAM_DBE 17
+#define QDMA_C2H_FATAL_ERR_STAT_WPL_DAT_PAR 18
+
+#define QDMA_C2H_FATAL_ERR_ENABLE 0xB00
+#define QDMA_C2H_FATAL_ERR_ENABLE_F_EN_WRQ_DIS 0
+#define QDMA_C2H_FATAL_ERR_ENABLE_F_EN_WPL_PAR_INV 1
+
+
+#define QDMA_C2H_FIRST_ERR_QID 0xB30
+#define S_QDMA_C2H_FIRST_ERR_QID 0
+#define M_QDMA_C2H_FIRST_ERR_QID 0xFFFU
+#define V_QDMA_C2H_FIRST_ERR_QID(x) \
+ ((x) << S_QDMA_C2H_FIRST_ERR_QID)
+#define S_QDMA_C2H_FIRST_ERR_TYPE 16
+#define M_QDMA_C2H_FIRST_ERR_TYPE 0x1FU
+#define V_QDMA_C2H_FIRST_ERR_TYPE(x) \
+ ((x) << S_QDMA_C2H_FIRST_ERR_TYPE)
+
+#define QDMA_C2H_PFCH_CACHE_DEPTH 0xBE0
+#define QDMA_C2H_CMPT_COAL_BUF_DEPTH 0xBE4
+
+#define QDMA_H2C_ERR_STAT 0xE00
+#define QDMA_H2C_ERR_MASK 0xE04
+#define QDMA_H2C_ERR_MASK_VALUE 0x7U
+#define QDMA_H2C_ERR_ZERO_LEN_DSC 0
+#define QDMA_H2C_ERR_CMPL_STATUS_MOP 1
+#define QDMA_H2C_ERR_NO_DMA_DSC 2
+
+#define QDMA_H2C_DATA_THRESHOLD 0xE24
+
+/* TRQ errors */
+/**
+ * hw_err_type_sel - hw error types
+ */
+enum hw_err_type_sel {
+ GLBL_ERR = 0, /**< global errors*/
+ GLBL_DSC_ERR, /**< descriptor errors*/
+ GLBL_TRQ_ERR, /**< trq errors*/
+ C2H_ERR, /**< c2h errors*/
+ C2H_FATAL_ERR, /**< c2h fatal errors*/
+ H2C_ERR, /**< h2c errors*/
+ ECC_SB_ERR, /**< ECC single bit error */
+ ECC_DB_ERR, /**< ECC double bit error */
+ HW_ERRS /**< hardware errors*/
+};
+
+/** ECC Single bit errors */
+#define QDMA_RAM_SBE_MASK_A (0xF0)
+#define QDMA_RAM_SBE_STAT_A (0xF4)
+#define QDMA_RAM_SBE_MASK_VALUE (0xFFFFFF11U)
+#define QDMA_RAM_SBE_STAT_MI_H2C0_DAT_ERR 0
+#define QDMA_RAM_SBE_STAT_MI_C2H0_DAT_ERR 4
+#define QDMA_RAM_SBE_STAT_H2C_RD_BRG_DAT_ERR 9
+#define QDMA_RAM_SBE_STAT_H2C_WR_BRG_DAT_ERR 10
+#define QDMA_RAM_SBE_STAT_C2H_RD_BRG_DAT_ERR 11
+#define QDMA_RAM_SBE_STAT_C2H_WR_BRG_DAT_ERR 12
+#define QDMA_RAM_SBE_STAT_FUNC_MAP_ERR 13
+#define QDMA_RAM_SBE_STAT_DSC_HW_CTXT_ERR 14
+#define QDMA_RAM_SBE_STAT_DSC_CRD_RCV_ER 15
+#define QDMA_RAM_SBE_STAT_DSC_SW_CTXT_ERR 16
+#define QDMA_RAM_SBE_STAT_DSC_CPLI_ERR 17
+#define QDMA_RAM_SBE_STAT_DSC_CPLD_ERR 18
+#define QDMA_RAM_SBE_STAT_PASID_CTXT_RAM_ERR 19
+#define QDMA_RAM_SBE_STAT_TIMER_FIFO_RAM_ERR 20
+#define QDMA_RAM_SBE_STAT_PAYLOAD_FIFO_RAM_ERR 21
+#define QDMA_RAM_SBE_STAT_QID_FIFO_RAM_ERR 22
+#define QDMA_RAM_SBE_STAT_TUSER_FIFO_RAM_ERR 23
+#define QDMA_RAM_SBE_STAT_WRB_COAL_DATA_RAM_ERR 24
+#define QDMA_RAM_SBE_STAT_INT_QID2VEC_RAM_ERR 25
+#define QDMA_RAM_SBE_STAT_INT_CTXT_RAM_ERR 26
+#define QDMA_RAM_SBE_STAT_DESC_REQ_FIFO_RAM_ERR 27
+#define QDMA_RAM_SBE_STAT_PFCH_CTXT_RAM_ERR 28
+#define QDMA_RAM_SBE_STAT_WRB_CTXT_RAM_ERR 29
+#define QDMA_RAM_SBE_STAT_PFCH_LL_RAM_ERR 30
+#define QDMA_RAM_SBE_STAT_H2C_PEND_FIFO_ERR 31
+
+/** ECC Double Bit errors */
+#define QDMA_RAM_DBE_MASK_A (0xF8)
+#define QDMA_RAM_DBE_STAT_A (0xFc)
+#define QDMA_RAM_DBE_MASK_VALUE (0xFFFFFF11U)
+#define QDMA_RAM_DBE_STAT_MI_H2C0_DAT_ERR 31
+#define QDMA_RAM_DBE_STAT_MI_C2H0_DAT_ERR 30
+#define QDMA_RAM_DBE_STAT_H2C_RD_BRG_DAT_ERR 29
+#define QDMA_RAM_DBE_STAT_H2C_WR_BRG_DAT_ERR 28
+#define QDMA_RAM_DBE_STAT_C2H_RD_BRG_DAT_ERR 27
+#define QDMA_RAM_DBE_STAT_C2H_WR_BRG_DAT_ERR 26
+#define QDMA_RAM_DBE_STAT_FUNC_MAP_ERR 25
+#define QDMA_RAM_DBE_STAT_DSC_HW_CTXT_ERR 24
+#define QDMA_RAM_DBE_STAT_DSC_CRD_RCV_ERR 23
+#define QDMA_RAM_DBE_STAT_DSC_SW_CTXT_ERR 22
+#define QDMA_RAM_DBE_STAT_DSC_CPLI_ERR 21
+#define QDMA_RAM_DBE_STAT_DSC_CPLD_ERR 20
+#define QDMA_RAM_DBE_STAT_PASID_CTXT_RAM_ERR 19
+#define QDMA_RAM_DBE_STAT_TIMER_FIFO_RAM_ERR 18
+#define QDMA_RAM_DBE_STAT_PAYLOAD_FIFO_RAM_ERR 17
+#define QDMA_RAM_DBE_STAT_QID_FIFO_RAM_ERR 16
+#define QDMA_RAM_DBE_STAT_TUSER_FIFO_RAM_ERR 15
+#define QDMA_RAM_DBE_STAT_WRB_COAL_DATA_RAM_ERR 14
+#define QDMA_RAM_DBE_STAT_INT_QID2VEC_RAM_ERR 13
+#define QDMA_RAM_DBE_STAT_INT_CTXT_RAM_ERR 12
+#define QDMA_RAM_DBE_STAT_DESC_REQ_FIFO_RAM_ERR 11
+#define QDMA_RAM_DBE_STAT_PFCH_CTXT_RAM_ERR 10
+#define QDMA_RAM_DBE_STAT_WRB_CTXT_RAM_ERR 9
+#define QDMA_RAM_DBE_STAT_PFCH_LL_RAM_ERR 4
+#define QDMA_RAM_DBE_STAT_H2C_PEND_FIFO_ERR 0
+
+#endif /* ifndef __QDMA_REGS_H__ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_sriov.c b/QDMA/linux-kernel/libqdma/qdma_sriov.c
new file mode 100644
index 0000000000000000000000000000000000000000..c14c28b1d1029764953f166910a2038ed8cbaa1c
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_sriov.c
@@ -0,0 +1,213 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+
+#include "xdev.h"
+#include "qdma_mbox.h"
+
+#ifdef __QDMA_VF__
+int xdev_sriov_vf_offline(struct xlnx_dma_dev *xdev, u8 func_id)
+{
+ struct mbox_msg *m;
+ int rv;
+
+ m = qdma_mbox_msg_alloc(xdev, MBOX_OP_BYE);
+ if (!m)
+ return -ENOMEM;
+
+ rv = qdma_mbox_msg_send(xdev, m, 0, 0, 0);
+ if (rv < 0) {
+ pr_info("%s, send bye failed %d.\n", xdev->conf.name, rv);
+ return rv;
+ }
+
+ return 0;
+}
+
+int xdev_sriov_vf_online(struct xlnx_dma_dev *xdev, u8 func_id)
+{
+ struct mbox_msg *m;
+ int rv;
+
+ m = qdma_mbox_msg_alloc(xdev, MBOX_OP_HELLO);
+ if (!m)
+ return -ENOMEM;
+
+ rv = qdma_mbox_msg_send(xdev, m, 0, 0, 0);
+ if (rv < 0) {
+ pr_info("%s, send hello failed %d.\n", xdev->conf.name, rv);
+ return rv;
+ }
+
+ /* initial value should always be qmax =1, qbase=0 */
+ xdev->conf.qsets_max = 1;
+ xdev->conf.qsets_base = 0;
+ qdma_device_set_cfg_state((unsigned long)xdev, QMAX_CFG_INITIAL);
+
+ return 0;
+}
+
+#elif defined(CONFIG_PCI_IOV)
+
+struct qdma_vf_info {
+ unsigned short func_id;
+ unsigned short qbase;
+ unsigned short qmax;
+ unsigned short filler;
+};
+
+void xdev_sriov_disable(struct xlnx_dma_dev *xdev)
+{
+ struct pci_dev *pdev = xdev->conf.pdev;
+
+ pci_disable_sriov(pdev);
+
+ kfree(xdev->vf_info);
+ xdev->vf_info = NULL;
+ xdev->vf_count = 0;
+
+ qdma_mbox_stop(xdev);
+}
+
+int xdev_sriov_enable(struct xlnx_dma_dev *xdev, int num_vfs)
+{
+ struct pci_dev *pdev = xdev->conf.pdev;
+ int current_vfs = pci_num_vf(pdev);
+ struct qdma_vf_info *vf;
+ int i;
+ int rv;
+
+ if (current_vfs) {
+ dev_err(&pdev->dev, "%d VFs already enabled!n", current_vfs);
+ return current_vfs;
+ }
+
+ vf = kmalloc(num_vfs * (sizeof(struct qdma_vf_info)), GFP_KERNEL);
+ if (!vf) {
+ pr_info("%s OOM, %d * %ld.\n",
+ xdev->conf.name, num_vfs, sizeof(struct qdma_vf_info));
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < num_vfs; i++)
+ vf[i].func_id = QDMA_FUNC_ID_INVALID;
+
+ xdev->vf_count = num_vfs;
+ xdev->vf_info = vf;
+
+ pr_debug("%s: req %d, current %d, assigned %d.\n",
+ xdev->conf.name, num_vfs, current_vfs, pci_vfs_assigned(pdev));
+
+ rv = pci_enable_sriov(pdev, num_vfs);
+ if (rv) {
+ pr_info("%s, enable sriov %d failed %d.\n",
+ xdev->conf.name, num_vfs, rv);
+ xdev_sriov_disable(xdev);
+ return 0;
+ }
+
+ qdma_mbox_start(xdev);
+
+ pr_debug("%s: done, req %d, current %d, assigned %d.\n",
+ xdev->conf.name, num_vfs, pci_num_vf(pdev),
+ pci_vfs_assigned(pdev));
+
+ return num_vfs;
+}
+
+int qdma_device_sriov_config(struct pci_dev *pdev, unsigned long dev_hndl,
+ int num_vfs)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int rv;
+
+ if (!dev_hndl)
+ return -EINVAL;
+
+ rv = xdev_check_hndl(__func__, pdev, dev_hndl);
+ if (rv < 0)
+ return rv;
+
+ /* if zeror disable sriov */
+ if (!num_vfs) {
+ xdev_sriov_disable(xdev);
+ return 0;
+ }
+
+ rv = xdev_sriov_enable(xdev, num_vfs);
+ if (rv < 0)
+ return rv;
+
+ return xdev->vf_count;
+}
+
+void xdev_sriov_vf_offline(struct xlnx_dma_dev *xdev, u8 func_id)
+{
+ struct qdma_vf_info *vf = (struct qdma_vf_info *)xdev->vf_info;
+ int i;
+
+ for (i = 0; i < xdev->vf_count; i++, vf++) {
+ if (vf->func_id == func_id) {
+ vf->func_id = QDMA_FUNC_ID_INVALID;
+ vf->qbase = 0;
+ vf->qmax = 0;
+ }
+ }
+}
+
+int xdev_sriov_vf_online(struct xlnx_dma_dev *xdev, u8 func_id)
+{
+ struct qdma_vf_info *vf = (struct qdma_vf_info *)xdev->vf_info;
+ int i;
+
+ for (i = 0; i < xdev->vf_count; i++, vf++) {
+ if (vf->func_id == QDMA_FUNC_ID_INVALID) {
+ vf->func_id = func_id;
+ return 0;
+ }
+ }
+
+ pr_info("%s, func 0x%x, NO free slot.\n", xdev->conf.name, func_id);
+ return -EINVAL;
+}
+
+int xdev_sriov_vf_fmap(struct xlnx_dma_dev *xdev, u8 func_id,
+ unsigned short qbase, unsigned short qmax)
+{
+ struct qdma_vf_info *vf = (struct qdma_vf_info *)xdev->vf_info;
+ int i;
+
+ for (i = 0; i < xdev->vf_count; i++, vf++) {
+ if (vf->func_id == func_id) {
+ vf->qbase = qbase;
+ vf->qmax = qmax;
+ return 0;
+ }
+ }
+
+ pr_info("%s, func 0x%x, NO match.\n", xdev->conf.name, func_id);
+ return -EINVAL;
+}
+
+#endif /* if defined(CONFIG_PCI_IOV) && !defined(__QDMA_VF__) */
diff --git a/QDMA/linux-kernel/libqdma/qdma_st_c2h.c b/QDMA/linux-kernel/libqdma/qdma_st_c2h.c
new file mode 100644
index 0000000000000000000000000000000000000000..cc1dfe4c7f342f13365652c66e062a052efb117a
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_st_c2h.c
@@ -0,0 +1,699 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_descq.h"
+
+#include <linux/kernel.h>
+#include <linux/delay.h>
+
+#include "qdma_device.h"
+#include "qdma_intr.h"
+#include "qdma_regs.h"
+#include "qdma_thread.h"
+#include "qdma_context.h"
+#include "thread.h"
+#include "qdma_compat.h"
+#include "qdma_st_c2h.h"
+#include "version.h"
+
+struct cmpl_info {
+ /* cmpl entry stat bits */
+ union {
+ u8 fbits;
+ struct cmpl_flag {
+ u8 format:1;
+ u8 color:1;
+ u8 err:1;
+ u8 desc_used:1;
+ u8 eot:1;
+ u8 filler:3;
+ } f;
+ };
+ u8 rsvd;
+ u16 len;
+ /* for tracking */
+ unsigned int pidx;
+ __be64 *entry;
+};
+
+/*
+ * ST C2H descq (i.e., freelist) RX buffers
+ */
+
+static inline void flq_unmap_one(struct qdma_sw_sg *sdesc,
+ struct qdma_c2h_desc *desc, struct device *dev,
+ unsigned char pg_order)
+{
+ if (sdesc->dma_addr) {
+ desc->dst_addr = 0UL;
+ dma_unmap_page(dev, sdesc->dma_addr, PAGE_SIZE << pg_order,
+ DMA_FROM_DEVICE);
+ sdesc->dma_addr = 0UL;
+ }
+}
+
+static inline void flq_free_one(struct qdma_sw_sg *sdesc,
+ struct qdma_c2h_desc *desc, struct device *dev,
+ unsigned char pg_order)
+{
+ if (sdesc && sdesc->pg) {
+ flq_unmap_one(sdesc, desc, dev, pg_order);
+ __free_pages(sdesc->pg, pg_order);
+ sdesc->pg = NULL;
+ }
+}
+
+static inline int flq_fill_one(struct qdma_sw_sg *sdesc,
+ struct qdma_c2h_desc *desc, struct device *dev,
+ int node, unsigned char pg_order, gfp_t gfp)
+{
+ struct page *pg;
+ dma_addr_t mapping;
+
+ pg = alloc_pages_node(node, __GFP_COMP | gfp, pg_order);
+ if (unlikely(!pg)) {
+ pr_info("OOM, order %d.\n", pg_order);
+ return -ENOMEM;
+ }
+
+ mapping = dma_map_page(dev, pg, 0, PAGE_SIZE << pg_order,
+ PCI_DMA_FROMDEVICE);
+ if (unlikely(dma_mapping_error(dev, mapping))) {
+ dev_err(dev, "page 0x%p mapping error 0x%llx.\n",
+ pg, (unsigned long long)mapping);
+ __free_pages(pg, pg_order);
+ return -EINVAL;
+ }
+
+ sdesc->pg = pg;
+ sdesc->dma_addr = mapping;
+ sdesc->len = PAGE_SIZE << pg_order;
+ sdesc->offset = 0;
+
+ desc->dst_addr = sdesc->dma_addr;
+ return 0;
+}
+
+void descq_flq_free_resource(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ struct device *dev = &xdev->conf.pdev->dev;
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ struct qdma_sw_sg *sdesc = flq->sdesc;
+ struct qdma_c2h_desc *desc = flq->desc;
+ unsigned char pg_order = flq->pg_order;
+ int i;
+
+ for (i = 0; i < flq->size; i++, sdesc++, desc++) {
+ if (sdesc)
+ flq_free_one(sdesc, desc, dev, pg_order);
+ else
+ break;
+ }
+
+ if (flq->sdesc) {
+ kfree(flq->sdesc);
+ flq->sdesc = NULL;
+ flq->sdesc_info = NULL;
+ }
+ memset(flq, 0, sizeof(struct qdma_flq));
+}
+
+int descq_flq_alloc_resource(struct qdma_descq *descq)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ struct device *dev = &xdev->conf.pdev->dev;
+ int node = dev_to_node(dev);
+ struct qdma_sw_sg *sdesc, *prev = NULL;
+ struct qdma_sdesc_info *sinfo, *sprev = NULL;
+ struct qdma_c2h_desc *desc = flq->desc;
+ int i;
+ int rv = 0;
+
+ sdesc = kzalloc_node(flq->size * (sizeof(struct qdma_sw_sg) +
+ sizeof(struct qdma_sdesc_info)),
+ GFP_KERNEL, node);
+ if (!sdesc) {
+ pr_info("OOM, sz %u.\n", flq->size);
+ return -ENOMEM;
+ }
+ flq->sdesc = sdesc;
+ flq->sdesc_info = sinfo = (struct qdma_sdesc_info *)(sdesc + flq->size);
+
+ /* make the flq to be a linked list ring */
+ for (i = 0; i < flq->size; i++, prev = sdesc, sdesc++,
+ sprev = sinfo, sinfo++) {
+ if (prev)
+ prev->next = sdesc;
+ if (sprev)
+ sprev->next = sinfo;
+ }
+ /* last entry's next points to the first entry */
+ prev->next = flq->sdesc;
+ sprev->next = flq->sdesc_info;
+
+ for (sdesc = flq->sdesc, i = 0; i < flq->size; i++, sdesc++, desc++) {
+ rv = flq_fill_one(sdesc, desc, dev, node, flq->pg_order,
+ GFP_KERNEL);
+ if (rv < 0) {
+ descq_flq_free_resource(descq);
+ return rv;
+ }
+ }
+
+ descq->cidx_cmpt_pend = 0;
+return 0;
+}
+
+static int qdma_flq_refill(struct qdma_descq *descq, int idx, int count,
+ int recycle, gfp_t gfp)
+{
+ struct xlnx_dma_dev *xdev = descq->xdev;
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ struct qdma_sw_sg *sdesc = flq->sdesc + idx;
+ struct qdma_c2h_desc *desc = flq->desc + idx;
+ struct qdma_sdesc_info *sinfo = flq->sdesc_info + idx;
+ int order = flq->pg_order;
+ int i;
+
+ for (i = 0; i < count; i++, idx++, sdesc++, desc++, sinfo++) {
+
+ if (idx == flq->size) {
+ idx = 0;
+ sdesc = flq->sdesc;
+ desc = flq->desc;
+ sinfo = flq->sdesc_info;
+ }
+
+ if (recycle) {
+ sdesc->len = PAGE_SIZE << order;
+ sdesc->offset = 0;
+ } else {
+ struct device *dev = &xdev->conf.pdev->dev;
+ int node = dev_to_node(dev);
+ int rv;
+
+ flq_unmap_one(sdesc, desc, dev, order);
+ rv = flq_fill_one(sdesc, desc, dev, node, order, gfp);
+ if (unlikely(rv < 0)) {
+ if (rv == -ENOMEM)
+ flq->alloc_fail++;
+ else
+ flq->mapping_err++;
+
+ break;
+ }
+ }
+ sinfo->fbits = 0;
+ descq->avail++;
+ }
+ if (list_empty(&descq->work_list) &&
+ list_empty(&descq->pend_list)) {
+ descq->pend_list_empty = 1;
+ if (descq->q_stop_wait)
+ qdma_waitq_wakeup(&descq->pend_list_wq);
+ }
+
+ return i;
+}
+
+/*
+ *
+ */
+int descq_st_c2h_read(struct qdma_descq *descq, struct qdma_request *req,
+ bool update_pidx, bool refill)
+{
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ unsigned int pidx = flq->pidx_pend;
+ struct qdma_sw_sg *fsg = flq->sdesc + pidx;
+ struct qdma_sw_sg *tsg = req->sgl;
+ unsigned int fsgcnt = ring_idx_delta(descq->pidx, pidx, flq->size);
+ unsigned int tsgoff = cb->sg_offset;
+ unsigned int foff = 0;
+ int i = 0, j = 0;
+ unsigned int copied = 0;
+
+ if (!fsgcnt)
+ return 0;
+
+ if (cb->sg_idx) {
+ for ( ; tsg && j < cb->sg_idx; j++)
+ tsg = tsg->next;
+
+ if (!tsg) {
+ pr_err("tsg error, index %u/%u.\n",
+ cb->sg_idx, req->sgcnt);
+ return -EINVAL;
+ }
+ }
+
+ while ((i < fsgcnt) && tsg) {
+ unsigned int flen = fsg->len;
+ unsigned char *faddr = page_address(fsg->pg) + fsg->offset;
+
+ foff = 0;
+
+ while (flen && tsg) {
+ unsigned int toff = tsg->offset + tsgoff;
+ unsigned int copy = min_t(unsigned int, flen,
+ tsg->len - tsgoff);
+
+ if (!req->no_memcpy)
+ memcpy(page_address(tsg->pg) + toff,
+ faddr, copy);
+
+ faddr += copy;
+ flen -= copy;
+ foff += copy;
+ tsgoff += copy;
+ copied += copy;
+
+ if (tsgoff == tsg->len) {
+ tsg = tsg->next;
+ tsgoff = 0;
+ j++;
+ }
+ }
+
+ if (foff == fsg->len) {
+ struct qdma_sdesc_info *sinfo = flq->sdesc_info + pidx;
+
+ if (sinfo->f.eop)
+ descq->cidx_cmpt_pend = sinfo->cidx;
+
+ pidx = ring_idx_incr(pidx, 1, descq->conf.rngsz);
+
+ i++;
+ foff = 0;
+ fsg = fsg->next;
+ }
+ }
+
+ incr_cmpl_desc_cnt(descq, i);
+
+ if (refill && i)
+ qdma_flq_refill(descq, flq->pidx_pend, i, 1, GFP_ATOMIC);
+
+ flq->pidx_pend = ring_idx_incr(flq->pidx_pend, i, flq->size);
+ if (foff) {
+ fsg->offset += foff;
+ fsg->len -= foff;
+ }
+
+ if (i && update_pidx) {
+ i = ring_idx_decr(flq->pidx_pend, 1, flq->size);
+ descq_c2h_pidx_update(descq, i);
+ }
+
+ cb->sg_idx = j;
+ cb->sg_offset = tsgoff;
+ cb->left -= copied;
+
+ flq->pkt_dlen -= copied;
+
+ return copied;
+}
+
+static int qdma_c2h_packets_proc_dflt(struct qdma_descq *descq)
+{
+ struct qdma_sgt_req_cb *cb, *tmp;
+
+ list_for_each_entry_safe(cb, tmp, &descq->pend_list, list) {
+ int rv;
+
+ /* check for zero length dma */
+ if (!cb->left) {
+ pr_debug("%s, cb 0x%p pending, zero len.\n",
+ descq->conf.name, cb);
+
+ qdma_sgt_req_done(descq, cb, 0);
+ return 0;
+ }
+
+ rv = descq_st_c2h_read(descq, (struct qdma_request *)cb, 0, 0);
+ if (rv < 0) {
+ pr_info("req 0x%p, error %d.\n", cb, rv);
+ qdma_sgt_req_done(descq, cb, rv);
+ continue;
+ }
+
+ if (!cb->left)
+ qdma_sgt_req_done(descq, cb, 0);
+ else
+ break;
+ }
+
+ return 0;
+}
+
+static inline void cmpt_next(struct qdma_descq *descq)
+{
+ u8 *desc_cmpt_cur = descq->desc_cmpt_cur + descq->cmpt_entry_len;
+
+ descq->desc_cmpt_cur = desc_cmpt_cur;
+ if (unlikely(++descq->cidx_cmpt == descq->conf.rngsz_cmpt)) {
+ descq->cidx_cmpt = 0;
+ descq->color ^= 1;
+ descq->desc_cmpt_cur = descq->desc_cmpt;
+ }
+}
+
+static inline bool is_new_cmpl_entry(struct qdma_descq *descq,
+ struct cmpl_info *cmpl)
+{
+ return cmpl->f.color == descq->color;
+}
+
+static int parse_cmpl_entry(struct qdma_descq *descq, struct cmpl_info *cmpl)
+{
+ __be64 *cmpt = (__be64 *)descq->desc_cmpt_cur;
+
+ dma_rmb();
+
+#if 0
+ print_hex_dump(KERN_INFO, "cmpl entry ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)cmpt, descq->cmpt_entry_len,
+ false);
+#endif
+
+ cmpl->entry = cmpt;
+ cmpl->f.format = cmpt[0] & F_C2H_CMPT_ENTRY_F_FORMAT ? 1 : 0;
+ cmpl->f.color = cmpt[0] & F_C2H_CMPT_ENTRY_F_COLOR ? 1 : 0;
+ cmpl->f.err = cmpt[0] & F_C2H_CMPT_ENTRY_F_ERR ? 1 : 0;
+ cmpl->f.eot = cmpt[0] & F_C2H_CMPT_ENTRY_F_EOT ? 1 : 0;
+ cmpl->f.desc_used = cmpt[0] & F_C2H_CMPT_ENTRY_F_DESC_USED ? 1 : 0;
+ if (!cmpl->f.format && cmpl->f.desc_used) {
+ cmpl->len = (cmpt[0] >> S_C2H_CMPT_ENTRY_LENGTH) &
+ M_C2H_CMPT_ENTRY_LENGTH;
+ /* zero length transfer allowed */
+ } else
+ cmpl->len = 0;
+
+ if (unlikely(cmpl->f.err)) {
+ pr_warn("%s, ERR compl entry %u error set\n",
+ descq->conf.name, descq->cidx_cmpt);
+ goto err_out;
+ }
+
+ /*
+ * format = 1 does not have length field, so the driver cannot
+ * figure out how many descriptor is used
+ */
+ if (unlikely(cmpl->f.format)) {
+ pr_err("%s: ERR cmpl. entry %u format=1.\n",
+ descq->conf.name, descq->cidx_cmpt);
+ goto err_out;
+ }
+
+ if (unlikely(!cmpl->f.desc_used && !descq->conf.cmpl_udd_en)) {
+ pr_warn("%s, ERR cmpl entry %u, desc_used 0, udd_en 0.\n",
+ descq->conf.name, descq->cidx_cmpt);
+ goto err_out;
+ }
+
+ return 0;
+
+err_out:
+ descq->err = 1;
+ print_hex_dump(KERN_INFO, "cmpl entry: ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)cmpt, descq->cmpt_entry_len,
+ false);
+ return -EINVAL;
+}
+
+static int rcv_pkt(struct qdma_descq *descq, struct cmpl_info *cmpl,
+ unsigned int len)
+{
+ unsigned int pidx = cmpl->pidx;
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ unsigned int pg_shift = flq->pg_shift;
+ unsigned int pg_mask = (1 << pg_shift) - 1;
+ unsigned int rngsz = descq->conf.rngsz;
+ /* zero length still uses one descriptor */
+ int fl_nr = len ? (len + pg_mask) >> pg_shift : 1;
+ unsigned int last = ring_idx_incr(cmpl->pidx, fl_nr - 1, rngsz);
+ unsigned int next = ring_idx_incr(last, 1, rngsz);
+ struct qdma_sw_sg *sdesc = flq->sdesc + last;
+ unsigned int cidx_next = ring_idx_incr(descq->cidx_cmpt, 1,
+ descq->conf.rngsz_cmpt);
+
+ if (descq->avail < fl_nr)
+ return -EBUSY;
+ descq->avail -= fl_nr;
+
+ if (len) {
+ unsigned int last_len = len & pg_mask;
+
+ if (last_len)
+ sdesc->len = last_len;
+ } else {
+ sdesc->len = 0;
+ }
+
+ if (descq->conf.fp_descq_c2h_packet) {
+ struct qdma_dev *qdev = xdev_2_qdev(descq->xdev);
+ int rv = descq->conf.fp_descq_c2h_packet(descq->conf.qidx +
+ (descq->conf.c2h ? qdev->qmax : 0),
+ descq->conf.quld, len, fl_nr, flq->sdesc + pidx,
+ descq->conf.cmpl_udd_en ?
+ (unsigned char *)cmpl->entry : NULL);
+
+ if (rv < 0)
+ return rv;
+
+ descq->cidx_cmpt_pend = cidx_next;
+ flq->pidx_pend = next;
+ } else {
+ int i;
+ struct qdma_sdesc_info *sinfo = flq->sdesc_info + pidx;
+
+ for (i = 0; i < fl_nr; i++, sinfo = sinfo->next) {
+ WARN_ON(sinfo->f.valid);
+ sinfo->f.valid = 1;
+ sinfo->cidx = cidx_next;
+ }
+
+ flq->sdesc_info[pidx].f.sop = 1;
+ flq->sdesc_info[last].f.eop = 1;
+
+ flq->pkt_dlen += len;
+ if (descq->conf.cmpl_udd_en)
+ flq->udd_cnt++;
+ }
+ cmpl->pidx = next;
+
+ return 0;
+}
+
+static int rcv_udd_only(struct qdma_descq *descq, struct cmpl_info *cmpl)
+{
+#ifdef XMP_DISABLE_ST_C2H_CMPL
+ __be64 cmpt_entry = cmpl->entry[0];
+#endif
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+
+ pr_debug("%s, rcv udd.\n", descq->conf.name);
+#if 0
+ print_hex_dump(KERN_INFO, "cmpl entry: ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)cmpl->entry, descq->cmpt_entry_len,
+ false);
+#endif
+
+ /* udd only: no descriptor used */
+ if (descq->conf.fp_descq_c2h_packet) {
+ struct qdma_dev *qdev = xdev_2_qdev(descq->xdev);
+
+ return descq->conf.fp_descq_c2h_packet(descq->conf.qidx +
+ (descq->conf.c2h ? qdev->qmax : 0),
+ descq->conf.quld, 0, 0, NULL,
+ (unsigned char *)cmpl->entry);
+ }
+#ifdef XMP_DISABLE_ST_C2H_CMPL
+ if ((cmpt_entry & (1 << 20)) > 0) {
+ __be16 pkt_cnt = (cmpt_entry >> 32) & 0xFFFF;
+ __be16 pkt_len = (cmpt_entry >> 48) & 0xFFFF;
+ int i;
+
+ pr_info("pkt %u * %u.\n", pkt_len, pkt_cnt);
+ for (i = 0; i < pkt_cnt; i++) {
+ int rv = rcv_pkt(descq, cmpl, pkt_len);
+
+ if (rv < 0)
+ break;
+ }
+ }
+#endif
+ flq->udd_cnt++;
+
+ return 0;
+}
+
+int descq_process_completion_st_c2h(struct qdma_descq *descq, int budget,
+ bool upd_cmpl)
+{
+ struct qdma_c2h_cmpt_cmpl_status *cs =
+ (struct qdma_c2h_cmpt_cmpl_status *)
+ descq->desc_cmpt_cmpl_status;
+ unsigned int rngsz_cmpt = descq->conf.rngsz_cmpt;
+ unsigned int pidx = descq->pidx;
+ unsigned int cidx_cmpt = descq->cidx_cmpt;
+ unsigned int pidx_cmpt = cs->pidx;
+ struct qdma_flq *flq = (struct qdma_flq *)descq->flq;
+ unsigned int pidx_pend = flq->pidx_pend;
+ bool uld_handler = descq->conf.fp_descq_c2h_packet ? true : false;
+ int pend;
+ int proc_cnt = 0;
+
+ /* once an error happens, stop processing of the Q */
+ if (descq->err) {
+ pr_info("%s: err.\n", descq->conf.name);
+ return 0;
+ }
+
+ dma_rmb();
+
+ pend = ring_idx_delta(pidx_cmpt, cidx_cmpt, rngsz_cmpt);
+ if (!pend) {
+ /* SW work around where next interrupt could be missed when
+ * there are no entries as of now
+ */
+ if ((descq->xdev->conf.qdma_drv_mode == DIRECT_INTR_MODE) ||
+ (descq->xdev->conf.qdma_drv_mode == INDIRECT_INTR_MODE))
+ descq_cmpt_cidx_update(descq, descq->cidx_cmpt);
+ return 0;
+ }
+
+#if 0
+ print_hex_dump(KERN_INFO, "cmpl status: ", DUMP_PREFIX_OFFSET,
+ 16, 1, (void *)cs, sizeof(*cs),
+ false);
+ pr_info("cmpl status: pidx 0x%x, cidx %x, color %d, int_state 0x%x.\n",
+ cs->pidx, cs->cidx, cs->color_isr_status & 0x1,
+ (cs->color_isr_status >> 1) & 0x3);
+#endif
+
+ if (!budget || budget > pend)
+ budget = pend;
+
+ while (likely(proc_cnt < budget)) {
+ struct cmpl_info cmpl;
+ int rv = parse_cmpl_entry(descq, &cmpl);
+
+ /* completion entry error, q is halted */
+ if (rv < 0)
+ return rv;
+
+ if (!is_new_cmpl_entry(descq, &cmpl))
+ break;
+
+ cmpl.pidx = pidx;
+
+ if (cmpl.f.desc_used) {
+ rv = rcv_pkt(descq, &cmpl, cmpl.len);
+ } else if (descq->conf.cmpl_udd_en) {
+ /* udd only: no descriptor used */
+ rv = rcv_udd_only(descq, &cmpl);
+ }
+
+ if (rv < 0) /* cannot process now, stop */
+ break;
+
+ pidx = cmpl.pidx;
+
+ cmpt_next(descq);
+ proc_cnt++;
+ }
+
+ if (proc_cnt) {
+ descq->pidx_cmpt = pidx_cmpt;
+ descq->pidx = pidx;
+
+ descq_cmpt_cidx_update(descq, descq->cidx_cmpt);
+ if (!descq->conf.fp_descq_c2h_packet)
+ qdma_c2h_packets_proc_dflt(descq);
+
+ /* some descq entries have been consumed */
+ if (flq->pidx_pend != pidx_pend) {
+ pend = ring_idx_delta(flq->pidx_pend, pidx_pend,
+ flq->size);
+ qdma_flq_refill(descq, pidx_pend, pend,
+ uld_handler ? 0 : 1, GFP_ATOMIC);
+
+ if (upd_cmpl && !descq->q_stop_wait) {
+ pend = ring_idx_decr(flq->pidx_pend, 1,
+ flq->size);
+ descq_c2h_pidx_update(descq, pend);
+ }
+ }
+ }
+
+ return 0;
+}
+
+int qdma_queue_c2h_peek(unsigned long dev_hndl, unsigned long id,
+ unsigned int *udd_cnt, unsigned int *pkt_cnt,
+ unsigned int *data_len)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, NULL, 0, 1);
+ struct qdma_flq *flq;
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ flq = (struct qdma_flq *)descq->flq;
+ *udd_cnt = flq->udd_cnt;
+ *pkt_cnt = flq->pkt_cnt;
+ *data_len = flq->pkt_dlen;
+
+ return 0;
+}
+
+int qdma_queue_packet_read(unsigned long dev_hndl, unsigned long id,
+ struct qdma_request *req, struct qdma_cmpl_ctrl *cctrl)
+{
+ struct qdma_descq *descq = qdma_device_get_descq_by_id(
+ (struct xlnx_dma_dev *)dev_hndl,
+ id, NULL, 0, 1);
+ struct qdma_sgt_req_cb *cb = qdma_req_cb_get(req);
+
+ if (!descq)
+ return QDMA_ERR_INVALID_QIDX;
+
+ if (!descq->conf.st || !descq->conf.c2h) {
+ pr_info("%s: st %d, c2h %d.\n",
+ descq->conf.name, descq->conf.st, descq->conf.c2h);
+ return -EINVAL;
+ }
+
+ memset(cb, 0, QDMA_REQ_OPAQUE_SIZE);
+
+ qdma_waitq_init(&cb->wq);
+
+ lock_descq(descq);
+ descq_st_c2h_read(descq, req, 1, 1);
+ unlock_descq(descq);
+
+ return req->count - cb->left;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_st_c2h.h b/QDMA/linux-kernel/libqdma/qdma_st_c2h.h
new file mode 100644
index 0000000000000000000000000000000000000000..1cbe8621d454331731b2c836cd0c1eeccbad2d8e
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_st_c2h.h
@@ -0,0 +1,232 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __QDMA_ST_C2H_H__
+#define __QDMA_ST_C2H_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma st c2h processing
+ *
+ */
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include "qdma_descq.h"
+#ifdef ERR_DEBUG
+#include "qdma_nl.h"
+#endif
+
+/**
+ * @struct - qdma_sdesc_info
+ * @brief qdma descriptor information
+ */
+struct qdma_sdesc_info {
+ /** pointer to next descriptor */
+ struct qdma_sdesc_info *next;
+ /**
+ * @union - desciptor flags
+ */
+ union {
+ /** 8 flag bits */
+ u8 fbits;
+ /**
+ * @struct - flags
+ * @brief desciptor flags
+ */
+ struct flags {
+ /** is descriptor valid */
+ u8 valid:1;
+ /** start of the packet */
+ u8 sop:1;
+ /** end of the packet */
+ u8 eop:1;
+ /** filler for 5 bits */
+ u8 filler:5;
+ } f;
+ };
+ /** reserved 3 bits */
+ u8 rsvd[3];
+ /** consumer index */
+ unsigned int cidx;
+};
+
+/**
+ * @struct - qdma_flq
+ * @brief qdma free list q page allocation book keeping
+ */
+struct qdma_flq {
+ /** RO: size of the decriptor */
+ unsigned int size;
+ /** RO: page order */
+ unsigned char pg_order;
+ /** RO: page shift */
+ unsigned char pg_shift;
+ /** RO: pointer to qdma c2h decriptor */
+ struct qdma_c2h_desc *desc;
+
+ /** RW: total # of udd outstanding */
+ unsigned int udd_cnt;
+ /** RW: total # of packet outstanding */
+ unsigned int pkt_cnt;
+ /** RW: total # of pkt payload length outstanding */
+ unsigned int pkt_dlen;
+ /** RW: # of available Rx buffers */
+ unsigned int avail;
+ /** RW: # of times buffer allocation failed */
+ unsigned long alloc_fail;
+ /** RW: # of RX Buffer DMA Mapping failures */
+ unsigned long mapping_err;
+ /** RW: consumer index */
+ unsigned int cidx;
+ /** RW: producer index */
+ unsigned int pidx;
+ /** RW: pending pidxes */
+ unsigned int pidx_pend;
+ /** RW: sw scatter gather list */
+ struct qdma_sw_sg *sdesc;
+ /** RW: sw descriptor info */
+ struct qdma_sdesc_info *sdesc_info;
+};
+
+/*****************************************************************************/
+/**
+ * qdma_descq_rxq_read() - read from the rx queue
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] req: queue request
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_rxq_read(struct qdma_descq *descq, struct qdma_request *req);
+
+/**
+ * qdma_descq_dump_cmpt() - dump the completion queue descriptors
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] start: start completion descriptor index
+ * @param[in] end: end completion descriptor index
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_descq_dump_cmpt(struct qdma_descq *descq, int start, int end,
+ char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * incr_cmpl_desc_cnt() - update the interrupt cidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] cnt: increment value
+ *
+ *****************************************************************************/
+void incr_cmpl_desc_cnt(struct qdma_descq *descq, unsigned int cnt);
+
+/*****************************************************************************/
+/**
+ * descq_flq_free_resource() - handler to free the pages for the request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void descq_flq_free_resource(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * descq_flq_alloc_resource() - handler to allocate the pages for the request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int descq_flq_alloc_resource(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * descq_process_completion_st_c2h() - handler to process the st c2h
+ * completion request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] budget: number of descriptors to process
+ * @param[in] upd_cmpl: if update completion required
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int descq_process_completion_st_c2h(struct qdma_descq *descq, int budget,
+ bool upd_cmpl);
+
+/*****************************************************************************/
+/**
+ * descq_st_c2h_read() - handler to process the st c2h read request
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] req: pointer to read request
+ * @param[in] update: flag to update the request
+ * @param[in] refill: flag to indicate whether to refill the flq
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int descq_st_c2h_read(struct qdma_descq *descq, struct qdma_request *req,
+ bool update, bool refill);
+
+/*****************************************************************************/
+/**
+ * descq_cmpt_cidx_update() - inline function to update the writeback cidx
+ *
+ * @param[in] descq: pointer to qdma_descq
+ * @param[in] cidx: cmpt consumer index
+ *
+ * @return none
+ *****************************************************************************/
+static inline void descq_cmpt_cidx_update(struct qdma_descq *descq,
+ unsigned int cidx)
+{
+#ifdef ERR_DEBUG
+ if (descq->induce_err & (1 << dsc)) {
+ cidx = descq->conf.rngsz;
+ pr_info("inducing error %d with pidx=%u cidx = %u", dsc,
+ descq->pidx, cidx);
+ }
+#endif
+ pr_debug("%s: cidx update 0x%x, reg 0x%x.\n", descq->conf.name, cidx,
+ QDMA_REG_CMPT_CIDX_BASE +
+ descq->conf.qidx * QDMA_REG_PIDX_STEP);
+
+ cidx |= (descq->conf.irq_en << S_CMPT_CIDX_UPD_EN_INT) |
+ (descq->conf.cmpl_stat_en << S_CMPT_CIDX_UPD_EN_STAT_DESC) |
+ (V_CMPT_CIDX_UPD_TRIG_MODE(descq->conf.cmpl_trig_mode)) |
+ (V_CMPT_CIDX_UPD_TIMER_IDX(descq->conf.cmpl_timer_idx)) |
+ (V_CMPT_CIDX_UPD_CNTER_IDX(descq->conf.cmpl_cnt_th_idx));
+
+ pr_debug("%s: cidx update 0x%x, reg 0x%x.\n", descq->conf.name, cidx,
+ QDMA_REG_CMPT_CIDX_BASE +
+ descq->conf.qidx * QDMA_REG_PIDX_STEP);
+
+ __write_reg(descq->xdev,
+ QDMA_REG_CMPT_CIDX_BASE + descq->conf.qidx * QDMA_REG_PIDX_STEP,
+ cidx);
+}
+
+#endif /* ifndef __QDMA_ST_C2H_H__ */
diff --git a/QDMA/linux-kernel/libqdma/qdma_thread.c b/QDMA/linux-kernel/libqdma/qdma_thread.c
new file mode 100644
index 0000000000000000000000000000000000000000..08e2197264269ffc811abeccab737624e6289fb4
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_thread.c
@@ -0,0 +1,236 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "qdma_thread.h"
+
+#include <linux/kernel.h>
+
+#include "qdma_descq.h"
+#include "thread.h"
+#include "xdev.h"
+
+/* ********************* global variables *********************************** */
+
+static unsigned int thread_cnt;
+/** completion status threads */
+static struct qdma_kthread *cs_threads;
+
+spinlock_t qcnt_lock;
+unsigned int cpu_count;
+static unsigned int *per_cpu_qcnt;
+
+/* ********************* static function declarations *********************** */
+
+static int qdma_thread_cmpl_status_pend(struct list_head *work_item);
+static int qdma_thread_cmpl_status_proc(struct list_head *work_item);
+
+/* ********************* static function definitions ************************ */
+static int qdma_thread_cmpl_status_pend(struct list_head *work_item)
+{
+ struct qdma_descq *descq = list_entry(work_item, struct qdma_descq,
+ cmplthp_list);
+ int pend = 0;
+
+ lock_descq(descq);
+ pend = !list_empty(&descq->pend_list) || !list_empty(&descq->work_list);
+ unlock_descq(descq);
+
+ return pend;
+}
+
+static int qdma_thread_cmpl_status_proc(struct list_head *work_item)
+{
+ struct qdma_descq *descq;
+
+ descq = list_entry(work_item, struct qdma_descq, cmplthp_list);
+ qdma_descq_service_cmpl_update(descq, 0, 1);
+ return 0;
+}
+
+/* ********************* public function definitions ************************ */
+
+void qdma_thread_remove_work(struct qdma_descq *descq)
+{
+ struct qdma_kthread *cmpl_thread;
+ int cpu_idx = cpu_count;
+
+
+ lock_descq(descq);
+ cmpl_thread = descq->cmplthp;
+ descq->cmplthp = NULL;
+
+ if (descq->cpu_assigned) {
+ descq->cpu_assigned = 0;
+ cpu_idx = descq->intr_work_cpu;
+ }
+
+ pr_debug("%s removing from thread %s, %u.\n",
+ descq->conf.name, cmpl_thread ? cmpl_thread->name : "?",
+ cpu_idx);
+
+ unlock_descq(descq);
+
+ if (cpu_idx < cpu_count) {
+ spin_lock(&qcnt_lock);
+ per_cpu_qcnt[cpu_idx]--;
+ spin_unlock(&qcnt_lock);
+ }
+
+ if (cmpl_thread) {
+ lock_thread(cmpl_thread);
+ list_del(&descq->cmplthp_list);
+ cmpl_thread->work_cnt--;
+ unlock_thread(cmpl_thread);
+ }
+}
+
+void qdma_thread_add_work(struct qdma_descq *descq)
+{
+ struct qdma_kthread *thp = cs_threads;
+ unsigned int v = 0;
+ int i, idx = thread_cnt;
+
+ if (descq->xdev->conf.qdma_drv_mode != POLL_MODE) {
+ spin_lock(&qcnt_lock);
+ idx = cpu_count - 1;
+ v = per_cpu_qcnt[idx];
+ for (i = idx - 1; i >= 0 && v; i--) {
+ if (per_cpu_qcnt[i] < v) {
+ idx = i;
+ v = per_cpu_qcnt[i];
+ }
+ }
+
+ per_cpu_qcnt[idx]++;
+ spin_unlock(&qcnt_lock);
+
+ lock_descq(descq);
+ descq->cpu_assigned = 1;
+ descq->intr_work_cpu = idx;
+ unlock_descq(descq);
+
+ pr_debug("%s 0x%p assigned to cpu %u.\n",
+ descq->conf.name, descq, idx);
+
+ return;
+ }
+
+ /* Polled mode only */
+ for (i = 0; i < thread_cnt; i++, thp++) {
+ lock_thread(thp);
+ if (idx == thread_cnt) {
+ v = thp->work_cnt;
+ idx = i;
+ } else if (!thp->work_cnt) {
+ idx = i;
+ unlock_thread(thp);
+ break;
+ } else if (thp->work_cnt < v)
+ idx = i;
+ unlock_thread(thp);
+ }
+
+ thp = cs_threads + idx;
+ lock_thread(thp);
+ list_add_tail(&descq->cmplthp_list, &thp->work_list);
+ descq->intr_work_cpu = idx;
+ thp->work_cnt++;
+ unlock_thread(thp);
+
+ pr_debug("%s 0x%p assigned to cmpl status thread %s,%u.\n",
+ descq->conf.name, descq, thp->name, thp->work_cnt);
+
+ lock_descq(descq);
+ descq->cmplthp = thp;
+ unlock_descq(descq);
+}
+
+int qdma_threads_create(unsigned int num_threads)
+{
+ struct qdma_kthread *thp;
+ int i;
+ int rv;
+
+ if (thread_cnt) {
+ pr_warn("threads already created!");
+ return 0;
+ }
+ spin_lock_init(&qcnt_lock);
+
+ cpu_count = num_online_cpus();
+ per_cpu_qcnt = kzalloc(cpu_count * sizeof(unsigned int), GFP_KERNEL);
+ if (!per_cpu_qcnt)
+ return -ENOMEM;
+
+ thread_cnt = (num_threads == 0) ? cpu_count : num_threads;
+
+ cs_threads = kzalloc(thread_cnt * sizeof(struct qdma_kthread),
+ GFP_KERNEL);
+ if (!cs_threads)
+ return -ENOMEM;
+
+ /* N dma writeback monitoring threads */
+ thp = cs_threads;
+ for (i = 0; i < thread_cnt; i++, thp++) {
+ thp->cpu = i;
+ thp->timeout = 0;
+ rv = qdma_kthread_start(thp, "qdma_cmpl_status_th", i);
+ if (rv < 0)
+ goto cleanup_threads;
+ thp->fproc = qdma_thread_cmpl_status_proc;
+ thp->fpending = qdma_thread_cmpl_status_pend;
+ }
+
+ return 0;
+
+cleanup_threads:
+ kfree(cs_threads);
+ cs_threads = NULL;
+ thread_cnt = 0;
+
+ return rv;
+}
+
+void qdma_threads_destroy(void)
+{
+ int i;
+ struct qdma_kthread *thp;
+
+ if (per_cpu_qcnt) {
+ spin_lock(&qcnt_lock);
+ kfree(per_cpu_qcnt);
+ per_cpu_qcnt = NULL;
+ spin_unlock(&qcnt_lock);
+ }
+
+ if (!thread_cnt)
+ return;
+
+ /* N dma writeback monitoring threads */
+ thp = cs_threads;
+ for (i = 0; i < thread_cnt; i++, thp++)
+ if (thp->fproc)
+ qdma_kthread_stop(thp);
+
+ kfree(cs_threads);
+ cs_threads = NULL;
+ thread_cnt = 0;
+}
diff --git a/QDMA/linux-kernel/libqdma/qdma_thread.h b/QDMA/linux-kernel/libqdma/qdma_thread.h
new file mode 100644
index 0000000000000000000000000000000000000000..d3fe3c07d1078b472f98749b9492652775c9687e
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/qdma_thread.h
@@ -0,0 +1,76 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef LIBQDMA_QDMA_THREAD_H_
+#define LIBQDMA_QDMA_THREAD_H_
+/**
+ * @file
+ * @brief This file contains the declarations for qdma thread handlers
+ *
+ */
+
+/** qdma_descq forward declaration */
+struct qdma_descq;
+
+/*****************************************************************************/
+/**
+ * qdma_threads_create() - create qdma threads
+ * This functions creates two threads for each cpu in the system or number of
+ * number of thread requested by param num_threads and assigns the
+ * thread handlers
+ * 1: queue processing thread
+ * 2: queue completion handler thread
+ *
+ * @param[in] num_threads - number of threads to be created
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_threads_create(unsigned int num_threads);
+
+/*****************************************************************************/
+/**
+ * qdma_threads_destroy() - destroy all the qdma threads created
+ * during system initialization
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_threads_destroy(void);
+
+/*****************************************************************************/
+/**
+ * qdma_thread_remove_work() - handler to remove the attached work thread
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_thread_remove_work(struct qdma_descq *descq);
+
+/*****************************************************************************/
+/**
+ * qdma_thread_add_work() - handler to add a work thread
+ *
+ * @param[in] descq: pointer to qdma_descq
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_thread_add_work(struct qdma_descq *descq);
+
+#endif /* LIBQDMA_QDMA_THREAD_H_ */
diff --git a/QDMA/linux-kernel/libqdma/thread.c b/QDMA/linux-kernel/libqdma/thread.c
new file mode 100644
index 0000000000000000000000000000000000000000..7bc98748c55d3f859679008d78e122944e8e9a91
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/thread.c
@@ -0,0 +1,193 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "thread.h"
+
+#include <linux/kernel.h>
+
+/*
+ * kernel thread function wrappers
+ */
+int qdma_kthread_dump(struct qdma_kthread *thp, char *buf, int buflen,
+ int detail)
+{
+ int len = 0;
+
+ if (!buf || !buflen)
+ return 0;
+
+ lock_thread(thp);
+ len += sprintf(buf + len, "%s, cpu %u, work %u.\n",
+ thp->name, thp->cpu, thp->work_cnt);
+
+ if (detail)
+ ;
+
+ unlock_thread(thp);
+
+ buf[len] = '\0';
+ return len;
+}
+
+static inline int xthread_work_pending(struct qdma_kthread *thp)
+{
+ struct list_head *work_item, *next;
+
+ /* any work items assigned to this thread? */
+ if (list_empty(&thp->work_list))
+ return 0;
+
+
+ /* any work item has pending work to do? */
+ list_for_each_safe(work_item, next, &thp->work_list) {
+ if (thp->fpending && thp->fpending(work_item))
+ return 1;
+
+ }
+ return 0;
+}
+
+static inline void xthread_reschedule(struct qdma_kthread *thp)
+{
+ if (thp->timeout) {
+ pr_debug_thread("%s rescheduling for %u seconds",
+ thp->name, thp->timeout);
+ qdma_waitq_wait_event_timeout(thp->waitq, thp->schedule,
+ msecs_to_jiffies(thp->timeout));
+ } else {
+ pr_debug_thread("%s rescheduling", thp->name);
+ qdma_waitq_wait_event(thp->waitq, thp->schedule);
+ }
+}
+
+static int xthread_main(void *data)
+{
+ struct qdma_kthread *thp = (struct qdma_kthread *)data;
+
+ pr_debug_thread("%s UP.\n", thp->name);
+
+ disallow_signal(SIGPIPE);
+
+ if (thp->finit)
+ thp->finit(thp);
+
+
+ while (!kthread_should_stop()) {
+
+ struct list_head *work_item, *next;
+
+ pr_debug_thread("%s interruptible\n", thp->name);
+
+ /* any work to do? */
+ lock_thread(thp);
+ if (!xthread_work_pending(thp)) {
+ unlock_thread(thp);
+ xthread_reschedule(thp);
+ lock_thread(thp);
+ }
+ thp->schedule = 0;
+
+ if (thp->work_cnt) {
+ pr_debug_thread("%s processing %u work items\n",
+ thp->name, thp->work_cnt);
+ /* do work */
+ list_for_each_safe(work_item, next, &thp->work_list) {
+ thp->fproc(work_item);
+ }
+ }
+ unlock_thread(thp);
+ schedule();
+ }
+
+ pr_debug_thread("%s, work done.\n", thp->name);
+
+
+ if (thp->fdone)
+ thp->fdone(thp);
+
+ pr_debug_thread("%s, exit.\n", thp->name);
+ return 0;
+}
+
+int qdma_kthread_start(struct qdma_kthread *thp, char *name, int id)
+{
+ int len;
+
+ if (thp->task) {
+ pr_warn("kthread %s task already running?\n", thp->name);
+ return -EINVAL;
+ }
+
+#ifdef __QDMA_VF__
+ len = snprintf(thp->name, sizeof(thp->name), "%s_vf_%d", name, id);
+ if (len < 0)
+ return -EINVAL;
+#else
+ len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id);
+ if (len < 0)
+ return -EINVAL;
+#endif
+ thp->id = id;
+
+ spin_lock_init(&thp->lock);
+ INIT_LIST_HEAD(&thp->work_list);
+ qdma_waitq_init(&thp->waitq);
+
+ thp->task = kthread_create_on_node(xthread_main, (void *)thp,
+ cpu_to_node(thp->cpu), "%s", thp->name);
+ if (IS_ERR(thp->task)) {
+ pr_err("kthread %s, create task failed: 0x%lx\n",
+ thp->name, (unsigned long)IS_ERR(thp->task));
+ thp->task = NULL;
+ return -EFAULT;
+ }
+
+ kthread_bind(thp->task, thp->cpu);
+
+ pr_debug_thread("kthread 0x%p, %s, cpu %u, task 0x%p.\n",
+ thp, thp->name, thp->cpu, thp->task);
+
+ wake_up_process(thp->task);
+ return 0;
+}
+
+int qdma_kthread_stop(struct qdma_kthread *thp)
+{
+ int rv;
+
+ if (!thp->task) {
+ pr_debug_thread("kthread %s, already stopped.\n", thp->name);
+ return 0;
+ }
+
+ thp->schedule = 1;
+ rv = kthread_stop(thp->task);
+ if (rv < 0) {
+ pr_warn("kthread %s, stop err %d.\n", thp->name, rv);
+ return rv;
+ }
+
+ pr_debug_thread("kthread %s, 0x%p, stopped.\n", thp->name, thp->task);
+ thp->task = NULL;
+
+ return 0;
+}
+
diff --git a/QDMA/linux-kernel/libqdma/thread.h b/QDMA/linux-kernel/libqdma/thread.h
new file mode 100644
index 0000000000000000000000000000000000000000..eea949a23b25f01653a2f742e55971b95cb8a21e
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/thread.h
@@ -0,0 +1,145 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __XDMA_KTHREAD_H__
+#define __XDMA_KTHREAD_H__
+/**
+ * @file
+ * @brief This file contains the declarations for qdma kernel threads
+ *
+ */
+#include <linux/version.h>
+#include <linux/spinlock.h>
+#include <linux/kthread.h>
+#include <linux/cpuset.h>
+#include <linux/signal.h>
+#include "qdma_compat.h"
+
+/**
+ * @struct - qdma_kthread
+ * @brief qdma thread book keeping parameters
+ */
+struct qdma_kthread {
+ /** thread lock*/
+ spinlock_t lock;
+ /** name of the thread */
+ char name[16];
+ /** cpu number for which the thread associated with */
+ unsigned short cpu;
+ /** thread id */
+ unsigned short id;
+ /** thread sleep timeout value */
+ unsigned int timeout;
+ /** flags for thread */
+ unsigned long flag;
+ /** thread wait queue */
+ qdma_wait_queue waitq;
+ /* flag to indicate scheduling of thread */
+ unsigned int schedule;
+ /** kernel task structure associated with thread*/
+ struct task_struct *task;
+ /** thread work list count */
+ unsigned int work_cnt;
+ /** thread work list count */
+ struct list_head work_list;
+ /** thread initialization handler */
+ int (*finit)(struct qdma_kthread *);
+ /** thread pending handler */
+ int (*fpending)(struct list_head *);
+ /** thread peocessing handler */
+ int (*fproc)(struct list_head *);
+ /** thread done handler */
+ int (*fdone)(struct qdma_kthread *);
+};
+
+/*****************************************************************************/
+/**
+ * qdma_kthread_dump() - handler to dump the thread information
+ *
+ * @param[in] thp: pointer to qdma_kthread
+ * @param[in] detail: flag to indicate whether details required or not
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return length of the buffer
+ *****************************************************************************/
+int qdma_kthread_dump(struct qdma_kthread *thp, char *buf, int buflen,
+ int detail);
+
+#ifdef DEBUG_THREADS
+#define lock_thread(thp) \
+ do { \
+ pr_debug("locking thp %s ...\n", (thp)->name); \
+ spin_lock(&(thp)->lock); \
+ } while (0)
+
+#define unlock_thread(thp) \
+ do { \
+ pr_debug("unlock thp %s ...\n", (thp)->name); \
+ spin_unlock(&(thp)->lock); \
+ } while (0)
+
+#define qdma_kthread_wakeup(thp) \
+ do { \
+ pr_debug("signaling thp %s ...\n", (thp)->name); \
+ wake_up_process((thp)->task); \
+ } while (0)
+
+#define pr_debug_thread(fmt, ...) pr_debug(fmt, __VA_ARGS__)
+
+#else
+/** lock thread macro */
+#define lock_thread(thp) spin_lock(&(thp)->lock)
+/** un lock thread macro */
+#define unlock_thread(thp) spin_unlock(&(thp)->lock)
+/** macro to wake up the qdma k thread */
+#define qdma_kthread_wakeup(thp) \
+ do { \
+ thp->schedule = 1; \
+ qdma_waitq_wakeup(&thp->waitq); \
+ } while (0)
+/** pr_debug_thread */
+#define pr_debug_thread(fmt, ...)
+#endif
+
+/*****************************************************************************/
+/**
+ * qdma_kthread_start() - handler to start the kernel thread
+ *
+ * @param[in] thp: pointer to qdma_kthread
+ * @param[in] name: name for the thread
+ * @param[in] id: thread id
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_kthread_start(struct qdma_kthread *thp, char *name, int id);
+
+/*****************************************************************************/
+/**
+ * qdma_kthread_stop() - handler to stop the kernel thread
+ *
+ * @param[in] thp: pointer to qdma_kthread
+ *
+ * @return 0: success
+ * @return <0: failure
+ *****************************************************************************/
+int qdma_kthread_stop(struct qdma_kthread *thp);
+
+#endif /* #ifndef __XDMA_KTHREAD_H__ */
diff --git a/QDMA/linux-kernel/libqdma/version.h b/QDMA/linux-kernel/libqdma/version.h
new file mode 100644
index 0000000000000000000000000000000000000000..7ba9dba245251aa1d529e231bfc760a13258c7d8
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/version.h
@@ -0,0 +1,33 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef __LIBQDMA_VERSION_H__
+#define __LIBQDMA_VERSION_H__
+
+#define LIBQDMA_MODULE_NAME "libqdma"
+#define LIBQDMA_MODULE_DESC "Xilinx QDMA Library"
+
+#define LIBQDMA_VERSION_MAJOR 2018
+#define LIBQDMA_VERSION_MINOR 3
+#define LIBQDMA_VERSION_PATCH 161
+
+#define LIBQDMA_VERSION_STR \
+ __stringify(LIBQDMA_VERSION_MAJOR) "." \
+ __stringify(LIBQDMA_VERSION_MINOR) "." \
+ __stringify(LIBQDMA_VERSION_PATCH)
+
+#define LIBQDMA_VERSION \
+ ((LIBQDMA_VERSION_MAJOR)*1000 + \
+ (LIBQDMA_VERSION_MINOR)*100 + \
+ LIBQDMA_VERSION_PATCH)
+
+#endif /* ifndef __LIBQDMA_VERSION_H__ */
diff --git a/QDMA/linux-kernel/libqdma/xdev.c b/QDMA/linux-kernel/libqdma/xdev.c
new file mode 100644
index 0000000000000000000000000000000000000000..f7159fb52e7b8d875643ec7de7a42baa7896959b
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/xdev.c
@@ -0,0 +1,1017 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+/**
+ * @file
+ * @brief This file contains the declarations for QDMA PCIe device
+ *
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/mm.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/vmalloc.h>
+
+#include "qdma_regs.h"
+#include "xdev.h"
+#include "qdma_mbox.h"
+#ifdef DEBUGFS
+#include "qdma_debugfs_dev.h"
+#endif
+
+#ifdef __LIST_NEXT_ENTRY__
+#define list_next_entry(pos, member) \
+ list_entry((pos)->member.next, typeof(*(pos)), member)
+#endif
+
+/**
+ * qdma device management
+ * maintains a list of the qdma devices
+ */
+static LIST_HEAD(xdev_list);
+
+/**
+ * mutex defined for qdma device management
+ */
+static DEFINE_MUTEX(xdev_mutex);
+
+#ifndef list_last_entry
+#define list_last_entry(ptr, type, member) \
+ list_entry((ptr)->prev, type, member)
+#endif
+
+/* extern declarations */
+void qdma_device_attributes_get(struct xlnx_dma_dev *xdev);
+int qdma_device_init(struct xlnx_dma_dev *xdev);
+void qdma_device_cleanup(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * xdev_list_first() - handler to return the first xdev entry from the list
+ *
+ * @return pointer to first xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_list_first(void)
+{
+ struct xlnx_dma_dev *xdev;
+
+ mutex_lock(&xdev_mutex);
+ xdev = list_first_entry(&xdev_list, struct xlnx_dma_dev, list_head);
+ mutex_unlock(&xdev_mutex);
+
+ return xdev;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_list_next() - handler to return the next xdev entry from the list
+ *
+ * @param[in] xdev: pointer to current xdev
+ *
+ * @return pointer to next xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_list_next(struct xlnx_dma_dev *xdev)
+{
+ struct xlnx_dma_dev *next;
+
+ mutex_lock(&xdev_mutex);
+ next = list_next_entry(xdev, list_head);
+ mutex_unlock(&xdev_mutex);
+
+ return next;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_list_dump() - list the dma device details
+ *
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return pointer to next xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+int xdev_list_dump(char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev, *tmp;
+ int len = 0;
+
+ mutex_lock(&xdev_mutex);
+ list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) {
+ len += sprintf(buf + len, "qdma%05x\t%02x:%02x.%02x\n",
+ xdev->conf.bdf, xdev->conf.pdev->bus->number,
+ PCI_SLOT(xdev->conf.pdev->devfn),
+ PCI_FUNC(xdev->conf.pdev->devfn));
+ if (len >= buflen)
+ break;
+ }
+ mutex_unlock(&xdev_mutex);
+
+ buf[len] = '\0';
+ return len;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_list_add() - add a new node to the xdma device lsit
+ *
+ * @param[in] xdev: pointer to current xdev
+ *
+ * @return none
+ *****************************************************************************/
+static inline void xdev_list_add(struct xlnx_dma_dev *xdev)
+{
+ u32 bdf = 0;
+ struct xlnx_dma_dev *_xdev, *tmp;
+ u32 last_bus = 0;
+ u32 last_dev = 0;
+
+ mutex_lock(&xdev_mutex);
+ bdf = ((xdev->conf.pdev->bus->number << PCI_SHIFT_BUS) |
+ (PCI_SLOT(xdev->conf.pdev->devfn) << PCI_SHIFT_DEV) |
+ PCI_FUNC(xdev->conf.pdev->devfn));
+ xdev->conf.bdf = bdf;
+ list_add_tail(&xdev->list_head, &xdev_list);
+
+ /*
+ * Iterate through the list of devices. Increment cfg_done, to
+ * get the mulitplier for initial configuration of queues. A
+ * '0' indicates queue is already configured. < 0, indicates
+ * config done using sysfs entry
+ */
+ list_for_each_entry_safe(_xdev, tmp, &xdev_list, list_head) {
+ /*are we dealing with a different card?*/
+#ifdef __QDMA_VF__
+ /** for VF check only bus number, as dev number can change
+ * in a single card
+ */
+ if (last_bus != _xdev->conf.pdev->bus->number)
+#else
+ if ((last_bus != _xdev->conf.pdev->bus->number) ||
+ (last_dev != PCI_SLOT(_xdev->conf.pdev->devfn)))
+#endif
+ xdev->conf.idx = 0;
+ xdev->conf.idx++;
+ last_bus = _xdev->conf.pdev->bus->number;
+ last_dev = PCI_SLOT(xdev->conf.pdev->devfn);
+ }
+ xdev->conf.cur_cfg_state = QMAX_CFG_UNCONFIGURED;
+ mutex_unlock(&xdev_mutex);
+}
+
+
+#undef list_last_entry
+/*****************************************************************************/
+/**
+ * xdev_list_add() - remove a node from the xdma device lsit
+ *
+ * @param[in] xdev: pointer to current xdev
+ *
+ * @return none
+ *****************************************************************************/
+static inline void xdev_list_remove(struct xlnx_dma_dev *xdev)
+{
+ mutex_lock(&xdev_mutex);
+ list_del(&xdev->list_head);
+ mutex_unlock(&xdev_mutex);
+}
+
+/*****************************************************************************/
+/**
+ * xdev_find_by_pdev() - find the xdev using struct pci_dev
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ *
+ * @return pointer to xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_find_by_pdev(struct pci_dev *pdev)
+{
+ struct xlnx_dma_dev *xdev, *tmp;
+
+ mutex_lock(&xdev_mutex);
+ list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) {
+ if (xdev->conf.pdev == pdev) {
+ mutex_unlock(&xdev_mutex);
+ return xdev;
+ }
+ }
+ mutex_unlock(&xdev_mutex);
+ return NULL;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_find_by_idx() - find the xdev using the index value
+ *
+ * @param[in] idx: index value in the xdev list
+ *
+ * @return pointer to xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_find_by_idx(int idx)
+{
+ struct xlnx_dma_dev *xdev, *tmp;
+
+ mutex_lock(&xdev_mutex);
+ list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) {
+ if (xdev->conf.bdf == idx) {
+ mutex_unlock(&xdev_mutex);
+ return xdev;
+ }
+ }
+ mutex_unlock(&xdev_mutex);
+ return NULL;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_check_hndl() - helper function to validate the device handle
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] hndl: device handle
+ *
+ * @return 0: success
+ * @return <0: on failure
+ *****************************************************************************/
+int xdev_check_hndl(const char *fname, struct pci_dev *pdev, unsigned long hndl)
+{
+ struct xlnx_dma_dev *xdev;
+
+ if (!pdev)
+ return -EINVAL;
+
+ xdev = xdev_find_by_pdev(pdev);
+ if (!xdev) {
+ pr_info("%s pdev 0x%p, hndl 0x%lx, NO match found!\n",
+ fname, pdev, hndl);
+ return -EINVAL;
+ }
+ if (((unsigned long)xdev) != hndl) {
+ pr_info("%s pdev 0x%p, hndl 0x%lx != 0x%p!\n",
+ fname, pdev, hndl, xdev);
+ return -EINVAL;
+ }
+
+ if (xdev->conf.pdev != pdev) {
+ pr_info("pci_dev(0x%lx) != pdev(0x%lx)\n",
+ (unsigned long)xdev->conf.pdev,
+ (unsigned long)pdev);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/**********************************************************************
+ * PCI-level Functions
+ **********************************************************************/
+
+/*****************************************************************************/
+/**
+ * xdev_unmap_bars() - Unmap the BAR regions that had been mapped
+ * earlier using map_bars()
+ *
+ * @param[in] xdev: pointer to current xdev
+ * @param[in] pdev: pointer to struct pci_dev
+ *
+ * @return none
+ *****************************************************************************/
+static void xdev_unmap_bars(struct xlnx_dma_dev *xdev, struct pci_dev *pdev)
+{
+ if (xdev->regs) {
+ /* unmap BAR */
+ pci_iounmap(pdev, xdev->regs);
+ /* mark as unmapped */
+ xdev->regs = NULL;
+ }
+
+ if (xdev->stm_regs) {
+ pci_iounmap(pdev, xdev->stm_regs);
+ xdev->stm_regs = NULL;
+ }
+}
+
+/*****************************************************************************/
+/**
+ * xdev_map_bars() - map device regions into kernel virtual address space
+ * earlier using map_bars()
+ *
+ * @param[in] xdev: pointer to current xdev
+ * @param[in] pdev: pointer to struct pci_dev
+ *
+ * Map the device memory regions into kernel virtual address space after
+ * verifying their sizes respect the minimum sizes needed
+ *
+ * @return length of the bar on success
+ * @return 0 on failure
+ *****************************************************************************/
+static int xdev_map_bars(struct xlnx_dma_dev *xdev, struct pci_dev *pdev)
+{
+ int map_len;
+
+ map_len = pci_resource_len(pdev, (int)xdev->conf.bar_num_config);
+ if (map_len > QDMA_MAX_BAR_LEN_MAPPED)
+ map_len = QDMA_MAX_BAR_LEN_MAPPED;
+
+ xdev->regs = pci_iomap(pdev, xdev->conf.bar_num_config, map_len);
+ if (!xdev->regs) {
+ pr_err("%s unable to map config bar %d.\n", xdev->conf.name,
+ xdev->conf.bar_num_config);
+ return -EINVAL;
+ }
+
+ if (pdev->device == STM_ENABLED_DEVICE) {
+ u32 rev;
+
+ map_len = pci_resource_len(pdev, STM_BAR);
+ xdev->stm_regs = pci_iomap(pdev, STM_BAR, map_len);
+ if (!xdev->stm_regs) {
+ pr_warn("%s unable to map bar %d.\n",
+ xdev->conf.name, STM_BAR);
+ return -EINVAL;
+ }
+
+ rev = readl(xdev->stm_regs + STM_REG_BASE + STM_REG_REV);
+ if (!(((rev >> 24) == 'S') && (((rev >> 16) & 0xFF) == 'T') &&
+ (((rev >> 8) & 0xFF) == 'M') &&
+ ((rev & 0xFF) == STM_SUPPORTED_REV))) {
+ pr_err("%s: Unsupported STM Rev found, rev 0x%x\n",
+ xdev->conf.name, rev);
+ xdev_unmap_bars(xdev, pdev);
+ return -EINVAL;
+ }
+ xdev->stm_en = 1;
+ xdev->stm_rev = rev & 0xFF;
+ } else {
+ xdev->stm_en = 0;
+ }
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_identify_bars() - verifies that the config bar passed from the user
+ * matches with the QDMA magic number. Also identifies
+ * the user bar and bypass bar
+ *
+ * @param[in] xdev: pointer to current xdev
+ * @param[in] pdev: pointer to struct pci_dev\
+ *
+ * @return 0 on success, -ve on failure
+ *****************************************************************************/
+static int xdev_identify_bars(struct xlnx_dma_dev *xdev, struct pci_dev *pdev)
+{
+ int bar_idx = 0;
+ int map_len = 0;
+ u32 id;
+ u8 num_bars_present = 0;
+ void __iomem *regs;
+ int bar_id_list[QDMA_BAR_NUM];
+ int bar_id_idx = 0;
+ u32 user_bar_id = 0;
+
+ /* Verify that the config bar passed from the user is correct or not */
+ map_len = pci_resource_len(pdev, (int)xdev->conf.bar_num_config);
+ if (!map_len) {
+ pr_err("%s pci_resource_len function failed for bar %d.\n",
+ xdev->conf.name, xdev->conf.bar_num_config);
+ return -EINVAL;
+ }
+
+ if (map_len > QDMA_MAX_BAR_LEN_MAPPED)
+ map_len = QDMA_MAX_BAR_LEN_MAPPED;
+
+ regs = pci_iomap(pdev, xdev->conf.bar_num_config, map_len);
+ if (!regs) {
+ pr_err("%s unable to map bar %d.\n",
+ xdev->conf.name, xdev->conf.bar_num_config);
+ return -EINVAL;
+ }
+
+ id = readl(regs + QDMA_REG_CONFIG_BLOCK_IDENTIFIER);
+ if ((id & M_CONFIG_BAR_IDENTIFIER_MASK) == MAGIC_NUMBER) {
+ pr_info("QDMA Config BAR passed by the user matches with the identifier register value(0x%08x)\n",
+ id);
+ } else {
+ pr_info("QDMA Config BAR passed by the user doesn't match the identifier register value(0x%08x)\n",
+ id);
+
+ /* unmap BAR we previously mapped */
+ pci_iounmap(pdev, regs);
+ return -EINVAL;
+ }
+
+ /* Find out the number of bars present in the design */
+ for (bar_idx = 0; bar_idx < QDMA_BAR_NUM; bar_idx++) {
+ map_len = pci_resource_len(pdev, bar_idx);
+ if (!map_len)
+ continue;
+
+ bar_id_list[bar_id_idx] = bar_idx;
+ bar_id_idx++;
+ num_bars_present++;
+ }
+
+ if (num_bars_present > 1) {
+ /* USER BAR IDENTIFICATION */
+#ifndef __QDMA_VF__
+ xdev->func_id = readl(regs + QDMA_REG_FUNC_ID);
+ user_bar_id = readl(regs + QDMA_REG_PF_USER_BAR_IDENTIFIER);
+ user_bar_id = (user_bar_id >> (6 * xdev->func_id)) &
+ M_USER_BAR_ID_MASK;
+#else
+ user_bar_id = readl(regs + QDMA_REG_VF_USER_BAR_IDENTIFIER);
+ user_bar_id = user_bar_id & M_USER_BAR_ID_MASK;
+#endif
+ for (bar_idx = 0; bar_idx < QDMA_BAR_NUM; bar_idx++) {
+ if (user_bar_id & (1 << bar_idx)) {
+ xdev->conf.bar_num_user = bar_idx;
+ pr_info("User BAR %d.\n",
+ xdev->conf.bar_num_user);
+ break;
+ }
+ }
+
+ /* BYPASS BAR IDENTIFICATION */
+ if (num_bars_present > 2) {
+ for (bar_idx = 0; bar_idx < num_bars_present;
+ bar_idx++) {
+ if ((bar_id_list[bar_idx] !=
+ xdev->conf.bar_num_user) &&
+ (bar_id_list[bar_idx] !=
+ xdev->conf.bar_num_config)) {
+ xdev->conf.bar_num_bypass =
+ bar_id_list[bar_idx];
+ pr_info("Bypass BAR %d.\n",
+ xdev->conf.bar_num_bypass);
+ break;
+ }
+ }
+ }
+ }
+ return 0;
+}
+/*****************************************************************************/
+/**
+ * xdev_map_bars() - allocate the dma device
+ *
+ * @param[in] conf: qdma device configuration
+ *
+ *
+ * @return pointer to dma device
+ * @return NULL on failure
+ *****************************************************************************/
+static struct xlnx_dma_dev *xdev_alloc(struct qdma_dev_conf *conf)
+{
+ struct xlnx_dma_dev *xdev;
+
+ /* allocate zeroed device book keeping structure */
+ xdev = kzalloc(sizeof(struct xlnx_dma_dev), GFP_KERNEL);
+ if (!xdev)
+ return NULL;
+
+ spin_lock_init(&xdev->hw_prg_lock);
+ spin_lock_init(&xdev->lock);
+
+ /* create a driver to device reference */
+ memcpy(&xdev->conf, conf, sizeof(*conf));
+
+ /* !! FIXME default to eanbled for everything */
+ xdev->flr_prsnt = 1;
+ xdev->st_mode_en = 1;
+ xdev->mm_mode_en = 1;
+ xdev->mm_channel_max = 1;
+
+ return xdev;
+}
+
+/*****************************************************************************/
+/**
+ * pci_dma_mask_set() - check the pci capability of the dma device
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ *
+ *
+ * @return 0: on success
+ * @return <0: on failure
+ *****************************************************************************/
+static int pci_dma_mask_set(struct pci_dev *pdev)
+{
+ /** 64-bit addressing capability for XDMA? */
+ if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
+ /** use 32-bit DMA for descriptors */
+ pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+ /** use 64-bit DMA, 32-bit for consistent */
+ } else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) {
+ pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+ /** use 32-bit DMA */
+ dev_info(&pdev->dev, "Using a 32-bit DMA mask.\n");
+ } else {
+ /** use 32-bit DMA */
+ dev_info(&pdev->dev, "No suitable DMA possible.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
+static void pci_enable_relaxed_ordering(struct pci_dev *pdev)
+{
+ pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
+}
+
+static void pci_enable_extended_tag(struct pci_dev *pdev)
+{
+ pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_EXT_TAG);
+}
+#else
+static void pci_enable_relaxed_ordering(struct pci_dev *pdev)
+{
+ u16 v;
+ int pos;
+
+ pos = pci_pcie_cap(pdev);
+ if (pos > 0) {
+ pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v);
+ v |= PCI_EXP_DEVCTL_RELAX_EN;
+ pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v);
+ }
+}
+
+static void pci_enable_extended_tag(struct pci_dev *pdev)
+{
+ u16 v;
+ int pos;
+
+ pos = pci_pcie_cap(pdev);
+ if (pos > 0) {
+ pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v);
+ v |= PCI_EXP_DEVCTL_EXT_TAG;
+ pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v);
+ }
+}
+#endif
+
+/*****************************************************************************/
+/**
+ * qdma_device_offline() - set the dma device in offline mode
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] dev_hndl: device handle
+ *
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_device_offline(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!dev_hndl)
+ return;
+
+ if (xdev_check_hndl(__func__, pdev, dev_hndl) < 0)
+ return;
+
+ if (xdev->conf.pdev != pdev) {
+ pr_info("pci_dev(0x%lx) != pdev(0x%lx)\n",
+ (unsigned long)xdev->conf.pdev, (unsigned long)pdev);
+ }
+
+ if (xlnx_dma_device_flag_check(xdev, XDEV_FLAG_OFFLINE))
+ return;
+ xdev_flag_set(xdev, XDEV_FLAG_OFFLINE);
+
+#ifdef __QDMA_VF__
+ xdev_sriov_vf_offline(xdev, 0);
+#elif defined(CONFIG_PCI_IOV)
+ xdev_sriov_disable(xdev);
+#endif
+
+ qdma_device_cleanup(xdev);
+
+ qdma_mbox_cleanup(xdev);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_online() - set the dma device in online mode
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] dev_hndl: device handle
+ *
+ *
+ * @return 0: on success
+ * @return <0: on failure
+ *****************************************************************************/
+int qdma_device_online(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+ int rv;
+
+ if (!dev_hndl)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ if (xdev->conf.pdev != pdev) {
+ pr_info("pci_dev(0x%lx) != pdev(0x%lx)\n",
+ (unsigned long)xdev->conf.pdev, (unsigned long)pdev);
+ }
+
+ rv = qdma_device_init(xdev);
+ if (rv < 0) {
+ pr_warn("qdma_init failed %d.\n", rv);
+ goto cleanup_qdma;
+ }
+ xdev_flag_clear(xdev, XDEV_FLAG_OFFLINE);
+ qdma_mbox_init(xdev);
+#ifdef __QDMA_VF__
+ /* PF mbox will start when vf > 0 */
+ qdma_mbox_start(xdev);
+ rv = xdev_sriov_vf_online(xdev, 0);
+ if (rv < 0)
+ goto cleanup_qdma;
+#elif defined(CONFIG_PCI_IOV)
+ if (xdev->conf.vf_max) {
+ rv = xdev_sriov_enable(xdev, xdev->conf.vf_max);
+ if (rv < 0)
+ goto cleanup_qdma;
+ }
+#endif
+ return 0;
+
+cleanup_qdma:
+ qdma_device_cleanup(xdev);
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_open() - open the dma device
+ *
+ * @param[in] mod_name: name of the dma device
+ * @param[in] conf: device configuration
+ * @param[in] dev_hndl: device handle
+ *
+ *
+ * @return 0: on success
+ * @return <0: on failure
+ *****************************************************************************/
+int qdma_device_open(const char *mod_name, struct qdma_dev_conf *conf,
+ unsigned long *dev_hndl)
+{
+ struct pci_dev *pdev = NULL;
+ struct xlnx_dma_dev *xdev = NULL;
+ int rv = 0;
+
+ *dev_hndl = 0UL;
+
+ if (!mod_name) {
+ pr_info("%s: mod_name is NULL.\n", __func__);
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+ }
+
+ if (!conf) {
+ pr_info("%s: queue_conf is NULL.\n", mod_name);
+ return QDMA_ERR_INVALID_INPUT_PARAM;
+ }
+
+ pdev = conf->pdev;
+
+ if (!pdev) {
+ pr_info("%s: pci device NULL.\n", mod_name);
+ return QDMA_ERR_INVALID_PCI_DEV;
+ }
+ conf->bar_num_stm = -1;
+
+ pr_info("%s, %02x:%02x.%02x, pdev 0x%p, 0x%x:0x%x.\n",
+ mod_name, pdev->bus->number, PCI_SLOT(pdev->devfn),
+ PCI_FUNC(pdev->devfn), pdev, pdev->vendor, pdev->device);
+
+ xdev = xdev_find_by_pdev(pdev);
+ if (xdev) {
+ pr_warn("%s, device %s already attached!\n",
+ mod_name, dev_name(&pdev->dev));
+ return QDMA_ERR_PCI_DEVICE_ALREADY_ATTACHED;
+ }
+
+ rv = pci_request_regions(pdev, mod_name);
+ if (rv) {
+ /* Just info, some other driver may have claimed the device. */
+ dev_info(&pdev->dev, "cannot obtain PCI resources\n");
+ return rv;
+ }
+
+ rv = pci_enable_device(pdev);
+ if (rv) {
+ dev_err(&pdev->dev, "cannot enable PCI device\n");
+ goto release_regions;
+ }
+
+ /* enable relaxed ordering */
+ pci_enable_relaxed_ordering(pdev);
+
+ /* enable extended tag */
+ pci_enable_extended_tag(pdev);
+
+ /* enable bus master capability */
+ pci_set_master(pdev);
+
+ rv = pci_dma_mask_set(pdev);
+ if (rv)
+ goto disable_device;
+
+ pcie_set_readrq(pdev, 512);
+
+ /* allocate zeroed device book keeping structure */
+ xdev = xdev_alloc(conf);
+ if (!xdev)
+ goto disable_device;
+
+ strncpy(xdev->mod_name, mod_name, QDMA_DEV_NAME_MAXLEN - 1);
+
+ xdev_flag_set(xdev, XDEV_FLAG_OFFLINE);
+ xdev_list_add(xdev);
+
+ rv = sprintf(xdev->conf.name, "qdma%05x-p%s",
+ xdev->conf.bdf, dev_name(&xdev->conf.pdev->dev));
+ xdev->conf.name[rv] = '\0';
+
+#ifdef DEBUGFS
+ /** time to clean debugfs */
+ dbgfs_dev_init(xdev);
+#endif
+
+ rv = xdev_identify_bars(xdev, pdev);
+ if (rv)
+ goto remove_xdev;
+
+ /* Mapping bars */
+ rv = xdev_map_bars(xdev, pdev);
+ if (rv)
+ goto unmap_bars;
+
+ /* program STM port map */
+ if (xdev->stm_en) {
+ u32 v = readl(xdev->stm_regs + STM_REG_BASE +
+ STM_REG_H2C_MODE);
+ v &= 0x0000FFFF;
+ v |= (STM_PORT_MAP << S_STM_PORT_MAP);
+ v |= F_STM_EN_STMA_BKCHAN;
+ writel(v, xdev->stm_regs + STM_REG_BASE + STM_REG_H2C_MODE);
+ }
+
+#ifndef __QDMA_VF__
+ /* get the device attributes */
+ qdma_device_attributes_get(xdev);
+
+ if (!xdev->mm_mode_en && !xdev->st_mode_en) {
+ pr_info("None of the modes ( ST or MM) are enabled\n");
+ rv = QDMA_ERR_INTERFACE_NOT_ENABLED_IN_DEVICE;
+ goto unmap_bars;
+ }
+#endif
+
+ memcpy(conf, &xdev->conf, sizeof(*conf));
+
+ rv = qdma_device_online(pdev, (unsigned long)xdev);
+ if (rv < 0)
+ goto cleanup_qdma;
+
+ pr_info("%s, %05x, pdev 0x%p, xdev 0x%p, ch %u, q %u, vf %u.\n",
+ dev_name(&pdev->dev), xdev->conf.bdf, pdev, xdev,
+ xdev->mm_channel_max, conf->qsets_max, conf->vf_max);
+
+ *dev_hndl = (unsigned long)xdev;
+
+ return QDMA_OPERATION_SUCCESSFUL;
+
+cleanup_qdma:
+ qdma_device_offline(pdev, (unsigned long)xdev);
+
+unmap_bars:
+ xdev_unmap_bars(xdev, pdev);
+remove_xdev:
+ xdev_list_remove(xdev);
+ kfree(xdev);
+
+disable_device:
+ pci_disable_device(pdev);
+
+release_regions:
+ pci_release_regions(pdev);
+
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_close() - close the dma device
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] dev_hndl: device handle
+ *
+ *
+ * @return none
+ *****************************************************************************/
+void qdma_device_close(struct pci_dev *pdev, unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!dev_hndl)
+ return;
+
+ if (xdev_check_hndl(__func__, pdev, dev_hndl) < 0)
+ return;
+
+ if (xdev->conf.pdev != pdev) {
+ pr_info("pci_dev(0x%lx) != pdev(0x%lx)\n",
+ (unsigned long)xdev->conf.pdev, (unsigned long)pdev);
+ }
+
+ qdma_device_offline(pdev, dev_hndl);
+
+ xdev_unmap_bars(xdev, pdev);
+
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+
+#ifdef DEBUGFS
+ /** time to clean debugfs */
+ dbgfs_dev_exit(xdev);
+#endif
+
+ xdev_list_remove(xdev);
+
+ kfree(xdev);
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_get_config() - get the device configuration
+ *
+ * @param[in] dev_hndl: device handle
+ * @param[out] conf: dma device configuration
+ * @param[out] buf, buflen:
+ * error message buffer, can be NULL/0 (i.e., optional)
+ *
+ *
+ * @return none
+ *****************************************************************************/
+int qdma_device_get_config(unsigned long dev_hndl, struct qdma_dev_conf *conf,
+ char *buf, int buflen)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ memcpy(conf, &xdev->conf, sizeof(*conf));
+
+ return 0;
+}
+
+int qdma_device_clear_stats(unsigned long dev_hndl)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *) dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ xdev->total_mm_h2c_pkts = 0;
+ xdev->total_mm_c2h_pkts = 0;
+ xdev->total_st_h2c_pkts = 0;
+ xdev->total_st_c2h_pkts = 0;
+
+ return 0;
+}
+
+int qdma_device_get_mmh2c_pkts(unsigned long dev_hndl,
+ unsigned long long *mmh2c_pkts)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *) dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ *mmh2c_pkts = xdev->total_mm_h2c_pkts;
+
+ return 0;
+}
+
+int qdma_device_get_mmc2h_pkts(unsigned long dev_hndl,
+ unsigned long long *mmc2h_pkts)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *) dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ *mmc2h_pkts = xdev->total_mm_c2h_pkts;
+
+ return 0;
+}
+
+int qdma_device_get_sth2c_pkts(unsigned long dev_hndl,
+ unsigned long long *sth2c_pkts)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *) dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ *sth2c_pkts = xdev->total_st_h2c_pkts;
+
+ return 0;
+}
+
+int qdma_device_get_stc2h_pkts(unsigned long dev_hndl,
+ unsigned long long *stc2h_pkts)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *) dev_hndl;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ *stc2h_pkts = xdev->total_st_c2h_pkts;
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_set_config() - set the device configuration
+ *
+ * @param[in] dev_hndl: device handle
+ * @param[in] conf: dma device configuration to set
+ *
+ * @return 0 on success ,<0 on failure
+ *****************************************************************************/
+int qdma_device_set_config(unsigned long dev_hndl, struct qdma_dev_conf *conf)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (!conf)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ memcpy(&xdev->conf, conf, sizeof(*conf));
+
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * qdma_device_set_cfg_state - set the device configuration state
+ *
+ * @param[in] dev_hndl: device handle
+ * @param[in] new_cfg_state: dma device conf state to set
+ *
+ *
+ * @return 0 on success ,<0 on failure
+ *****************************************************************************/
+
+int qdma_device_set_cfg_state(unsigned long dev_hndl,
+ enum qdma_dev_qmax_state new_cfg_state)
+{
+ struct xlnx_dma_dev *xdev = (struct xlnx_dma_dev *)dev_hndl;
+
+ if (new_cfg_state > QMAX_CFG_USER)
+ return -EINVAL;
+
+ if (xdev_check_hndl(__func__, xdev->conf.pdev, dev_hndl) < 0)
+ return -EINVAL;
+
+ xdev->conf.cur_cfg_state = new_cfg_state;
+
+ return 0;
+}
+
diff --git a/QDMA/linux-kernel/libqdma/xdev.h b/QDMA/linux-kernel/libqdma/xdev.h
new file mode 100644
index 0000000000000000000000000000000000000000..a875d020d7d532ffbf602ba46f7b08c4dbab6216
--- /dev/null
+++ b/QDMA/linux-kernel/libqdma/xdev.h
@@ -0,0 +1,531 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __XDEV_H__
+#define __XDEV_H__
+/**
+ * @file
+ * @brief This file contains the declarations for QDMA PCIe device
+ *
+ */
+#include <linux/types.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+
+#include "libqdma_export.h"
+#include "qdma_mbox.h"
+#include "qdma_qconf_mgr.h"
+#ifdef DEBUGFS
+#include "qdma_debugfs.h"
+#endif
+
+/**
+ * QDMA bars
+ */
+#define QDMA_BAR_NUM 6
+/**
+ * QDMA config bar size - 64MB
+ */
+#define QDMA_MAX_BAR_LEN_MAPPED 0x4000000
+
+/*
+ *module_param_array:
+ *config_bar=<bus_num(16bits)><pf0_config_bar(4bits)><pf1_config_bar(4bits)>
+ * <pf2_config_bar(4bits)><pf3_config_bar(4bits)>
+ *config_bar=<bus_num(16bits)><vf_pf0_config_bar(4bits)>
+ * <vf_pf1_config_bar(4bits)><vf_pf2_config_bar(4bits)>
+ * <vf_pf3_config_bar(4bits)>
+ *
+ */
+#define BUS_NUM_MASK 0xFFFF0000
+#define BUS_NUM_SHIFT 16
+
+#define PF_DEV_0_MASK 0x0000F000
+#define PF_DEV_0_SHIFT 12
+#define PF_DEV_1_MASK 0x00000F00
+#define PF_DEV_1_SHIFT 8
+#define PF_DEV_2_MASK 0x000000F0
+#define PF_DEV_2_SHIFT 4
+#define PF_DEV_3_MASK 0x0000000F
+#define PF_DEV_3_SHIFT 0
+
+#define VF_PF_IDENTIFIER_MASK 0xF
+#define VF_PF_IDENTIFIER_SHIFT 8
+
+enum qdma_pf_devices {
+ PF_DEVICE_0 = 0,
+ PF_DEVICE_1,
+ PF_DEVICE_2,
+ PF_DEVICE_3
+};
+/**
+ * number of bits to describe the DMA transfer descriptor
+ */
+#define QDMA_DESC_BLEN_BITS 28
+/**
+ * maximum size of a single DMA transfer descriptor
+ */
+#define QDMA_DESC_BLEN_MAX ((1 << (QDMA_DESC_BLEN_BITS)) - 1)
+
+/**
+ * obtain the 32 most significant (high) bits of a 32-bit or 64-bit address
+ */
+#define PCI_DMA_H(addr) ((addr >> 16) >> 16)
+/**
+ * obtain the 32 least significant (low) bits of a 32-bit or 64-bit address
+ */
+#define PCI_DMA_L(addr) (addr & 0xffffffffUL)
+
+/**
+ * Xiling DMA device forward declaration
+ */
+struct xlnx_dma_dev;
+
+/* XDMA PCIe device specific book-keeping */
+/**
+ * Flag for device offline
+ */
+#define XDEV_FLAG_OFFLINE 0x1
+/**
+ * Flag for IRQ
+ */
+#define XDEV_FLAG_IRQ 0x2
+/**
+ * Maximum number of interrupts supported per device
+ */
+#define XDEV_NUM_IRQ_MAX 8
+
+/**
+ * interrupt call back function handlers
+ */
+typedef irqreturn_t (*f_intr_handler)(int irq_index, int irq, void *dev_id);
+
+/**
+ * @struct - intr_coal_conf
+ * @brief interrut coalescing configuration
+ */
+struct intr_coal_conf {
+ /**< interrupt vector index */
+ u16 vec_id;
+ /**< number of entries in interrupt ring per vector */
+ u16 intr_rng_num_entries;
+ /**< interrupt ring base address */
+ dma_addr_t intr_ring_bus;
+ struct qdma_intr_ring *intr_ring_base;
+ /**< color value indicates the valid entry in the interrupt ring */
+ u8 color;
+ /**< interrupt ring consumer index */
+ unsigned int cidx;
+};
+
+/**
+ * Macros for Hardware Version info
+ */
+#define RTL1_VERSION 0
+#define RTL2_VERSION 1
+#define VIVADO_RELEASE_2018_3 0
+#define VIVADO_RELEASE_2018_2 1
+#define EVEREST_SOFT_IP 0
+#define EVEREST_HARD_IP 1
+
+/**
+ * intr_type_list - interrupt types
+ */
+enum intr_type_list {
+ INTR_TYPE_ERROR, /**< error interrupt */
+ INTR_TYPE_USER, /**< user interrupt */
+ INTR_TYPE_DATA, /**< data interrupt */
+ INTR_TYPE_MAX /**< max interrupt */
+};
+
+/**
+ * @struct - intr_vec_map_type
+ * @brief interrupt vector map details
+ */
+struct intr_vec_map_type {
+ enum intr_type_list intr_type; /**< interrupt type */
+ int intr_vec_index; /**< interrupt vector index */
+ f_intr_handler intr_handler; /**< interrupt handler */
+};
+
+/**< Interrupt info for MSI-X interrupt vectors per device */
+struct intr_info_t {
+ /**< msix_entry list for all vectors */
+ char msix_name[QDMA_DEV_NAME_MAXLEN + 16];
+ /**< queue list for each interrupt */
+ struct list_head intr_list;
+ /**< number of queues assigned for each interrupt */
+ int intr_list_cnt;
+ /**< interrupt vector map */
+ struct intr_vec_map_type intr_vec_map;
+};
+
+/**
+ * @struct - xlnx_dma_dev
+ * @brief Xilinx DMA device details
+ */
+struct xlnx_dma_dev {
+ /**< Xilinx DMA device name */
+ char mod_name[QDMA_DEV_NAME_MAXLEN];
+ /**< DMA device configuration */
+ struct qdma_dev_conf conf;
+ /**< DMA device list */
+ struct list_head list_head;
+ /**< DMA device lock to protects concurrent access */
+ spinlock_t lock;
+ /**< DMA device hardware program lock */
+ spinlock_t hw_prg_lock;
+ /**< device flags */
+ unsigned int flags;
+ u8 flr_prsnt:1;
+ /**< flag to indicate the FLR present status */
+ u8 st_mode_en:1;
+ /**< flag to indicate the streaming mode enabled status */
+ u8 mm_mode_en:1;
+ /**< flag to indicate the memory mapped mode enabled status */
+ u8 stm_en:1;
+ /**< flag to indicate the presence of STM */
+ /**< sriov info */
+ void *vf_info;
+ /**< number of virtual functions */
+ u8 vf_count;
+ /**< function id */
+ u16 func_id;
+#ifdef __QDMA_VF__
+ /**< parent function id, valid only for virtual function */
+ u8 func_id_parent;
+#else
+ /**< number of physical functions */
+ u8 pf_count;
+#endif
+ /**< max mm channels */
+ u8 mm_channel_max;
+ u8 stm_rev;
+ /**< PCIe config. bar */
+ void __iomem *regs;
+ /** PCIe Bar for STM config */
+ void __iomem *stm_regs;
+
+ /**< number of MSI-X interrupt vectors per device */
+ int num_vecs;
+ /**< msix_entry list for all MSIx vectors associated for device */
+ struct msix_entry *msix;
+ /**< interrupt info list for all MSIx vectors associated for device */
+ struct intr_info_t *dev_intr_info_list;
+ /**< data vector start index */
+ int dvec_start_idx;
+ /**< DMA private device to hold the qdma que details */
+ void *dev_priv;
+ /**< dsa configured max pkt size that STM can support */
+ u32 pipe_stm_max_pkt_size;
+ /**< list of interrupt coalescing configuration for each vector */
+ struct intr_coal_conf *intr_coal_list;
+ /**< legacy interrupt vector */
+ int vector_legacy;
+#ifdef ERR_DEBUG
+ /**< error lock */
+ spinlock_t err_lock;
+ /**< flag to indicate the error minitor status */
+ u8 err_mon_cancel;
+ /**< error minitor work handler */
+ struct delayed_work err_mon;
+#endif
+
+#ifdef DEBUGFS
+ /** debugfs device root */
+ struct dentry *dbgfs_dev_root;
+ /** debugfs queue root */
+ struct dentry *dbgfs_queues_root;
+ /* lock for creating qidx directory */
+ spinlock_t qidx_lock;
+#endif
+
+ /** number of packets processed in pf */
+ struct qdma_mbox mbox;
+ unsigned long long total_mm_h2c_pkts;
+ unsigned long long total_mm_c2h_pkts;
+ unsigned long long total_st_h2c_pkts;
+ unsigned long long total_st_c2h_pkts;
+ /**< for upper layer calling function */
+ unsigned int dev_ulf_extra[0];
+};
+
+/*****************************************************************************/
+/**
+ * xlnx_dma_device_flag_check() - helper function to check the flag status
+ *
+ * @param[in] xdev: pointer to xilinx dma device
+ * @param[in] f: flag value
+ *
+ *
+ * @return 1 if the flag is on
+ * @return 0 if the flag is off
+ *****************************************************************************/
+static inline int xlnx_dma_device_flag_check(struct xlnx_dma_dev *xdev,
+ unsigned int f)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ if (xdev->flags & f) {
+ spin_unlock_irqrestore(&xdev->lock, flags);
+ return 1;
+ }
+ spin_unlock_irqrestore(&xdev->lock, flags);
+ return 0;
+}
+
+/*****************************************************************************/
+/**
+ * xlnx_dma_device_flag_test_n_set() - helper function to test n set the flag
+ *
+ * @param[in] xdev: pointer to xilinx dma device
+ * @param[in] f: flag value
+ *
+ *
+ * @return 1 if the flag is already enabled
+ * @return 0 if the flag is off
+ *****************************************************************************/
+static inline int xlnx_dma_device_flag_test_n_set(struct xlnx_dma_dev *xdev,
+ unsigned int f)
+{
+ unsigned long flags;
+ int rv = 0;
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ if (xdev->flags & f) {
+ spin_unlock_irqrestore(&xdev->lock, flags);
+ rv = 1;
+ } else
+ xdev->flags |= f;
+ spin_unlock_irqrestore(&xdev->lock, flags);
+ return rv;
+}
+
+/*****************************************************************************/
+/**
+ * xdev_flag_set() - helper function to set the device flag
+ *
+ * @param[in] xdev: pointer to xilinx dma device
+ * @param[in] f: flag value
+ *
+ *
+ * @return none
+ *****************************************************************************/
+static inline void xdev_flag_set(struct xlnx_dma_dev *xdev, unsigned int f)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ xdev->flags |= f;
+ spin_unlock_irqrestore(&xdev->lock, flags);
+}
+
+/*****************************************************************************/
+/**
+ * xlnx_dma_device_flag_test_n_set() - helper function to clear the device flag
+ *
+ * @param[in] xdev: pointer to xilinx dma device
+ * @param[in] f: flag value
+ *
+ * @return none
+ *****************************************************************************/
+static inline void xdev_flag_clear(struct xlnx_dma_dev *xdev, unsigned int f)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&xdev->lock, flags);
+ xdev->flags &= ~f;
+ spin_unlock_irqrestore(&xdev->lock, flags);
+}
+
+/*****************************************************************************/
+/**
+ * xdev_find_by_pdev() - find the xdev using struct pci_dev
+ *
+ * @param[in] pdev: pointer to struct pci_dev
+ *
+ * @return pointer to xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_find_by_pdev(struct pci_dev *pdev);
+
+/*****************************************************************************/
+/**
+ * xdev_find_by_idx() - find the xdev using the index value
+ *
+ * @param[in] idx: index value in the xdev list
+ *
+ * @return pointer to xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_find_by_idx(int idx);
+
+/*****************************************************************************/
+/**
+ * xdev_list_first() - handler to return the first xdev entry from the list
+ *
+ * @return pointer to first xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_list_first(void);
+
+/*****************************************************************************/
+/**
+ * xdev_list_next() - handler to return the next xdev entry from the list
+ *
+ * @param[in] xdev: pointer to current xdev
+ *
+ * @return pointer to next xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+struct xlnx_dma_dev *xdev_list_next(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * xdev_list_dump() - list the dma device details
+ *
+ * @param[in] buflen: length of the input buffer
+ * @param[out] buf: message buffer
+ *
+ * @return pointer to next xlnx_dma_dev on success
+ * @return NULL on failure
+ *****************************************************************************/
+int xdev_list_dump(char *buf, int buflen);
+
+/*****************************************************************************/
+/**
+ * xdev_check_hndl() - helper function to validate the device handle
+ *
+ * @param[in] f: device name
+ * @param[in] pdev: pointer to struct pci_dev
+ * @param[in] hndl: device handle
+ *
+ * @return 0: success
+ * @return EINVAL: on failure
+ *****************************************************************************/
+int xdev_check_hndl(const char *f, struct pci_dev *pdev, unsigned long hndl);
+
+
+#ifdef __QDMA_VF__
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_offline() - API to set the virtual function to offline mode
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ *
+ * @return 0: success
+ * @return -1: on failure
+ *****************************************************************************/
+int xdev_sriov_vf_offline(struct xlnx_dma_dev *xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_online() - API to set the virtual function to online mode
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ *
+ * @return 0: success
+ * @return -1: on failure
+ *****************************************************************************/
+int xdev_sriov_vf_online(struct xlnx_dma_dev *xdev, u8 func_id);
+#elif defined(CONFIG_PCI_IOV)
+/* SR-IOV */
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_online() - API to disable the virtual function
+ *
+ * @param[in] xdev: pointer to xdev
+ *
+ * @return none
+ *****************************************************************************/
+void xdev_sriov_disable(struct xlnx_dma_dev *xdev);
+
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_online() - API to enable the virtual function
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ *
+ * @return number of vfs enabled on success
+ * @return -1: on failure
+ *****************************************************************************/
+int xdev_sriov_enable(struct xlnx_dma_dev *xdev, int num_vfs);
+
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_offline() - API to set the virtual function to offline mode
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ *
+ * @return none
+ *****************************************************************************/
+void xdev_sriov_vf_offline(struct xlnx_dma_dev *xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_offline() - API to set the virtual function to offline mode
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ *
+ * @return 0: success
+ * @return -1: on failure
+ *****************************************************************************/
+int xdev_sriov_vf_online(struct xlnx_dma_dev *xdev, u8 func_id);
+
+/*****************************************************************************/
+/**
+ * xdev_sriov_vf_offline() - API to configure the fmap for virtual function
+ *
+ * @param[in] xdev: pointer to xdev
+ * @param[in] func_id: function identifier
+ * @param[in] qbase: queue start
+ * @param[in] qmax: queue max
+ *
+ * @return 0: success
+ * @return -1: on failure
+ *****************************************************************************/
+int xdev_sriov_vf_fmap(struct xlnx_dma_dev *xdev, u8 func_id,
+ unsigned short qbase, unsigned short qmax);
+#else
+/** dummy declaration for xdev_sriov_disable()
+ * When virtual function is not enabled
+ */
+#define xdev_sriov_disable(xdev)
+/** dummy declaration for xdev_sriov_enable()
+ * When virtual function is not enabled
+ */
+#define xdev_sriov_enable(xdev, num_vfs)
+/** dummy declaration for xdev_sriov_vf_offline()
+ * When virtual function is not enabled
+ */
+#define xdev_sriov_vf_offline(xdev, func_id)
+/** dummy declaration for xdev_sriov_vf_online()
+ * When virtual function is not enabled
+ */
+#define xdev_sriov_vf_online(xdev, func_id)
+#endif
+
+#endif /* XDMA_LIB_H */
diff --git a/QDMA/linux-kernel/make/common_flags.mk b/QDMA/linux-kernel/make/common_flags.mk
new file mode 100644
index 0000000000000000000000000000000000000000..1b7c48c6c868c135cc9520770ec32913f32b0161
--- /dev/null
+++ b/QDMA/linux-kernel/make/common_flags.mk
@@ -0,0 +1,9 @@
+#Build flags common to xdma/qdma
+
+ifeq ($(VF),1)
+ EXTRA_FLAGS += -D__QDMA_VF__
+ PFVF_TYPE = _vf
+else
+ PFVF_TYPE =
+endif
+
diff --git a/QDMA/linux-kernel/make/distro_check.mk b/QDMA/linux-kernel/make/distro_check.mk
new file mode 100644
index 0000000000000000000000000000000000000000..ec339c7d9a7d920bba18dfa6f30efddee2677304
--- /dev/null
+++ b/QDMA/linux-kernel/make/distro_check.mk
@@ -0,0 +1,86 @@
+#
+# distro. checks
+# uses the same variables calculated from kernel_check.mk
+#
+
+distro :=
+dmajor :=
+dminor :=
+
+# kernel.org rc kernels
+#ifneq ($(shell echo $(kextraversion) | $(grep) -c 'git'),0)
+# distro := GIT
+# dmajor :=
+# dminor :=
+#endif # kernel.org
+
+ifneq ($(shell $(grep) -c 'RHEL' $(VERSION_H)),0)
+ distro := RHEL
+# distro_vmajor := $(shell $(grep) 'RHEL_MAJOR' $(VERSION_H) | cut -d' ' -f3)
+# distro_vminor := $(shell $(grep) 'RHEL_MINOR' $(VERSION_H) | cut -d' ' -f3)
+ dmajor := $(word 3, $(shell $(grep) 'RHEL_MAJOR' $(VERSION_H)))
+ dminor := $(word 3, $(shell $(grep) 'RHEL_MINOR' $(VERSION_H)))
+ ifeq ($(dmajor),)
+ ifeq ($(dminor),)
+ $(error ERROR! RHEL distro version NOT specified, check version.h.)
+ endif
+ endif
+ ifeq ($(shell [ $(dmajor) -lt 7 ] && echo 1),1)
+ $(error ERROR! Unsupported RHEL version $(dmajor).$(dminor).)
+ endif
+ ifeq ($(dmajor),7)
+ ifeq ($(shell [ $(dminor) -lt 1 ] && echo 1),1)
+ $(error ERROR! Unsupported RHEL version $(dmajor).$(dminor).)
+ endif
+ endif
+endif # RHEL
+
+# SLES does not provide any release macros like RHEL. So we are
+# setting Makefile flags for SLES releases based on the version
+# and patchlevel obtained from /etc/SuSE-release file
+ifneq ($(wildcard /etc/SuSE-release),)
+ distro := SLES
+ dmajor := $(shell cat /etc/SuSE-release | grep VERSION | awk '{print $$3}')
+ dminor := $(shell cat /etc/SuSE-release | grep PATCHLEVEL | awk '{print $$3}')
+endif
+
+$(info "distro=$(distro), dmajor=$(dmajor) dminor=$(dminor) ")
+
+# assume this is kernel.org kernels
+ifeq ($(distro),)
+ ifeq ($(kseries),2.6)
+ ifeq ($(shell [ $(ksublevel) -ge 32 ] && echo 1),1)
+ distro := GIT
+ else
+ $(error kernel version $(kbaseversion)$(kextraversion) NOT supported.)
+ $( kernel.org Requires >= 2.6.32.)
+ endif
+ endif
+
+ ifeq ($(kversion),3)
+ ifeq ($(shell [ $(kpatchlevel) -ge 1 ] && echo 1),1)
+ distro := GIT
+ else
+ $(error kernel version $(kbaseversion)$(kextraversion) NOT supported.)
+ $( kernel.org Requires >= 3.1.)
+ endif
+ endif
+
+ ifeq ($(kversion),4)
+ distro := GIT
+ endif
+endif # assume kernel.org kernels
+
+ifeq ($(distro),)
+ $(error kernel version $(kbaseversion)$(kextraversion) NOT supported.)
+ $( kernel.org Requires >= 2.6.35.)
+endif
+
+FLAGS += -D$(distro)$(dmajor)SP$(dminor)
+FLAGS += -D$(distro)$(dmajor)
+
+$(info $(kbaseversion)$(kextraversion): $(distro),$(dmajor),$(dminor), $(FLAGS))
+
+export distro
+export dmajor
+export dminor
diff --git a/QDMA/linux-kernel/make/kernel_check.mk b/QDMA/linux-kernel/make/kernel_check.mk
new file mode 100644
index 0000000000000000000000000000000000000000..bce232c984d693a7b3b4510d6328f16e221b5efd
--- /dev/null
+++ b/QDMA/linux-kernel/make/kernel_check.mk
@@ -0,0 +1,332 @@
+# Kernel directories.
+KERNELRELEASE := $(shell uname -r)
+
+# If KSRC=<path> is specified on the command line, KOBJ=<path> must
+# also be specified. This is to avoid mixups if the kernel object path
+# differs from the source path. A shortcut (KSRC=KOBJ) is to use KDIR.
+ifeq ($(KDIR),)
+ ifeq ($(KSRC),)
+ ifneq ($(KOBJ),)
+ $(warning When using KOBJ=<path>, the KSRC=<path> must also be defined.)
+ $(warning Use KDIR=<path> when KSRC and KOBJ are the same.)
+ $(error ERROR: kernel source path not specified)
+ endif
+ else
+ ifeq ($(KOBJ),)
+ $(warning When using KSRC=<path>, the KOBJ=<path> must also be defined.)
+ $(warning Use KDIR=<path> when KSRC and KOBJ are the same.)
+ $(error ERROR: KOBJ path not specified)
+ endif
+ endif
+else
+ override KSRC := $(KDIR)
+ override KOBJ := $(KDIR)
+endif
+
+# Only if KSRC/KOBJ were not defined on the command line.
+KSRC ?= $(wildcard /lib/modules/$(KERNELRELEASE)/source)
+KOBJ ?= $(wildcard /lib/modules/$(KERNELRELEASE)/build)
+KINC = $(KSRC)/include
+
+#$(warning KDIR: $(KDIR).)
+
+ifeq ($(KSRC),)
+ KSRC = $(KOBJ)
+endif
+
+# Define kernel files.
+VERSION_H := $(KOBJ)/include/linux/version.h
+AUTOCONF_H := $(KOBJ)/include/generated/autoconf.h
+UTSRELEASE_H := $(KOBJ)/include/generated/utsrelease.h
+
+ifeq ($(wildcard $(VERSION_H)),)
+ VERSION_H := $(KOBJ)/include/generated/uapi/linux/version.h
+endif
+ifeq ($(wildcard $(AUTOCONF_H)),)
+ AUTOCONF_H := $(KOBJ)/include/linux/autoconf.h
+endif
+ifeq ($(wildcard $(UTSRELEASE_H)),)
+ UTSRELEASE_H := $(KOBJ)/include/linux/utsrelease.h
+endif
+#ifeq ($(wildcard $(UTSRELEASE_H)),)
+# $(error NO utsrelease)
+#endif
+
+# Define architecture and target(for RPM).
+ARCH := $(shell uname -m)
+target := $(ARCH)
+override ARCH := $(shell echo $(ARCH) | sed 's/i.86/i386/')
+ifeq ($(USER_ARCH),)
+
+ ifeq ($(ARCH),ppc64le)
+ ifeq ($(wildcard $(KOBJ)/arch/$(ARCH)),)
+ override ARCH := powerpc
+ endif
+ endif
+
+ ifeq ($(ARCH),ppc64)
+ # Check if the kernel wants ppc64 or powerpc.
+ ifeq ($(wildcard $(KOBJ)/arch/$(ARCH)),)
+ override ARCH := powerpc
+ endif
+ endif
+else
+ # Honor the value of ARCH if specified by user.
+ override ARCH := $(USER_ARCH)
+endif
+
+# Functions.
+define path_check
+$(if $(wildcard $(1)),$(1),)
+endef
+define reverse_sort
+$(shell echo -e `echo "$(strip $(1))" |\
+ sed 's/[[:space:]]/\\\n/g'` | sort -r)
+endef
+define version_code
+$(shell let x=`sed '/^\#define[[:space:]]*LINUX_VERSION_CODE/!d;\
+ s/.*LINUX_VERSION_CODE[[:space:]]*//' < $(1)\
+ 2>/dev/null`;\
+ let a="$$x >> 16";\
+ let x="$$x - ($$a << 16)";\
+ let b="$$x >> 8";\
+ let x="$$x - ($$b << 8)";\
+ echo "$$a $$b $$x")
+endef
+
+# Checks for kernel source and object directories.
+ifeq ($(call path_check,$(KSRC)),)
+ $(warning Be sure the kernel source is properly installed or \
+ try specifying the kernel source tree using 'make KSRC=<path>')
+ $(error ERROR: missing kernel source)
+endif
+ifeq ($(call path_check,$(KOBJ)),)
+ $(warning Try specifying the kernel build tree using 'make KOBJ=<path>'.)
+ $(error ERROR: missing kernel build)
+endif
+
+# Check kernel source and build directories are somewhat likely to be correct.
+ifneq ($(notdir $(wildcard $(KSRC)/Makefile)),Makefile)
+ $(warning There seems to be a problem with the kernel \
+ source [$(KSRC)] directory.)
+ $(error ERROR: missing kernel Makefile)
+endif
+ifneq ($(notdir $(wildcard $(KOBJ)/Makefile)),Makefile)
+ $(warning There seems to be a problem with the kernel \
+ build [$(KOBJ)] directory.)
+ $(error ERROR: missing kernel Makefile)
+endif
+
+# Get kernel version code info.
+KERNELVERSION := $(strip $(call version_code,$(VERSION_H)))
+ifneq ($(words $(KERNELVERSION)), 3)
+ $(error ERROR: unexpected kernel version \
+ '$(shell echo $(KERNELVERSION) | sed 's/[[:space:]]/./g')')
+endif
+
+# Define kernel version details.
+kversion := $(word 1, $(KERNELVERSION))
+kpatchlevel := $(word 2, $(KERNELVERSION))
+ksublevel := $(word 3, $(KERNELVERSION))
+
+# The kernel base version, excluding the EXTRAVERSION string.
+kbaseversion := $(kversion).$(kpatchlevel).$(ksublevel)
+
+# The kernel series version.
+kseries := $(kversion).$(kpatchlevel)
+
+# Fix for variation of Module.symvers naming (thanks 2.6.17!).
+# I need to know the file name of the module symver generated by the kernel
+# during an external module build (MODPOST). Also used for kernels that don't
+# automatically generate the module symver file during MODPOST (2.6.0-2.6.17?).
+ifeq ($(shell $(grep) -c '^modulesymfile[[:space:]]*:\?=' \
+ $(KSRC)/scripts/Makefile.modpost),1)
+ modulesymfile := $(shell $(grep) '^modulesymfile[[:space:]]*:\?=' \
+ $(KSRC)/scripts/Makefile.modpost)
+ kernelsymfile := $(shell $(grep) '^kernelsymfile[[:space:]]*:\?=' \
+ $(KSRC)/scripts/Makefile.modpost)
+else
+ ifeq ($(shell $(grep) -c '^symverfile[[:space:]]*:\?=' \
+ $(KSRC)/scripts/Makefile.modpost),1)
+ symverfile := $(shell $(grep) '^symverfile[[:space:]]*:\?=' \
+ $(KSRC)/scripts/Makefile.modpost)
+ kernelsymfile := $(subst symverfile,kernelsymfile,$(symverfile))
+ endif
+endif
+modulesymfile ?= $(symverfile)
+ifeq ($(modulesymfile),)
+ $(warning The parsing of $(KSRC)/scripts/Makefile.modpost \
+ is not making sense.)
+ $(error ERROR cannot determine module symvers file)
+endif
+
+# Gnu make (3.80) bug #1516, $(eval ...) inside conditionals causes errors.
+# This is fixed in v3.81 and some v3.80 (RHEL4/5) but not on SLES10.
+# Workaround: include a separate makefile that does the eval.
+ifeq ($(shell echo '$(modulesymfile)' | $(grep) -c '^[[:alnum:]_]\+[[:space:]]*:\?=[[:space:]]*.\+'),1)
+ $(shell echo '$$(eval $$(modulesymfile))' > eval.mak)
+ include eval.mak
+else
+ modulesymfile =
+endif
+ifeq ($(shell echo '$(kernelsymfile)' | $(grep) -c '^[[:alnum:]_]\+[[:space:]]*:\?=[[:space:]]*.\+'),1)
+ $(shell echo '$$(eval $$(kernelsymfile))' > eval.mak)
+ include eval.mak
+else
+ kernelsymfile =
+endif
+modulesymfile := $(notdir $(modulesymfile))
+kernelsymfile := $(notdir $(kernelsymfile))
+$(shell [ -f eval.mak ] && /bin/rm -f eval.mak)
+
+ifneq ($(words $(modulesymfile)),1)
+ $(warning The parsing of $(KSRC)/scripts/Makefile.modpost \
+ is not making sense.)
+ $(warning You can try passing 'modulesymfile=Module.symvers' or \
+ similar to make.)
+ $(error ERROR cannot determine module symvers file)
+endif
+
+
+# Check for configured kernel.
+ifeq ($(wildcard $(AUTOCONF_H)),)
+ $(warning The kernel is not properly configured, try running \
+ 'make menuconfig' on your kernel.)
+ $(error ERROR: kernel missing autoconf.h)
+endif
+# Check for built kernel.
+ifeq ($(wildcard $(VERSION_H)),)
+ $(warning The kernel has not been compiled. Try building your kernel \
+ before building this driver.)
+ $(error ERROR: kernel missing version.h)
+endif
+
+# Check that kernel supports modules.
+ifneq ($(shell $(grep) -c '^\#define[[:space:]]\+CONFIG_MODULES[[:space:]]\+1' $(AUTOCONF_H)),1)
+ $(warning The kernel has not been configured for module support.)
+ $(warning Try configuring the kernel to allow external modules and \
+ recompile.)
+ $(error ERROR: kernel CONFIG_MODULES not defined)
+endif
+
+# Get kernel UTS_RELEASE info.
+ifneq ($(wildcard $(UTSRELEASE_H)),)
+ ifneq ($(shell $(grep) -c '^\#define[[:space:]]\+UTS_RELEASE' \
+ $(UTSRELEASE_H)),0)
+ utsrelease := $(UTSRELEASE_H)
+ endif
+else
+ ifneq ($(wildcard $(KOBJ)/include/linux/version.h),)
+ ifneq ($(shell $(grep) -c '^\#define[[:space:]]\+UTS_RELEASE' \
+ $(KOBJ)/include/linux/version.h),0)
+ utsrelease := $(KOBJ)/include/linux/version.h
+ endif
+ endif
+endif
+ifeq ($(utsrelease),)
+ $(error ERROR: cannot locate kernel UTS_RELEASE)
+endif
+# Getting the UTS_RELEASE on RHEL3 had problems due to the multiple defines
+# within the file. I can run this file through the C pre-processor and get
+# the actual UTS_RELEASE definition. This has only been tested on gcc, other
+# compilers may not work.
+utsrelease := $(strip $(shell $(CC) -E -dM -I $(KSRC)/include $(utsrelease) \
+ 2>/dev/null| sed '/^\#define[[:space:]]*UTS_RELEASE/!d;\
+ s/^\#define UTS_RELEASE[[:space:]]*"//;\
+ s/"//g'))
+
+# The kernel local version string if defined in config.
+klocalversion := $(shell sed '/^CONFIG_LOCALVERSION=/!d;\
+ s/^CONFIG_LOCALVERSION="//;s/"//g'\
+ 2>/dev/null < $(KOBJ)/.config)
+# The complete kernel EXTRAVERSION string.
+kextraversion := $(subst $(kbaseversion),,$(utsrelease))
+# The full kernel version should be the same as uts_release.
+kernelversion := $(utsrelease)
+
+# The kernel EXTRAVERSION creates a unique problem, especially since
+# kernel versioning extended into the EXTRAVERSION and distributions
+# add strings such as smp, largesmp, xen, etc or when additional minor
+# version numbers are appended.
+# Some code that we supply is dependent on the kernel version and
+# parts of the EXTRAVERSION, but not dependent on some of the additional
+# flags. This requires that I have a list of kernel version strings that
+# could map to the source version we require. For example, if the
+# kernel version is 2.6.9-67.ELsmp, we only care about the "2.6.9-67"
+# part, therefore, I need to split the EXTRAVERSION accordingly.
+# Another problem is when a user builds their own kernel, say 2.6.21.4
+# and adds an additional string to EXTRAVERSION. The EXTRAVERSION is
+# now ".4-custom" and I have to parse this with hopes of extracting
+# only the ".4" part, resulting in the needed "2.6.21.4" version.
+# Adding a BUGFIX version (int) field would be very helpfull!
+
+# EXTRAVERSION as defined only in the Makefile.
+extraversion1 := $(strip $(shell sed '/^EXTRAVERSION/!d;\
+ s/^EXTRAVERSION[[:space:]]*=//;s/"//g'\
+ < $(KSRC)/Makefile 2>/dev/null))
+# SLES9 likes to put make code in their EXTRAVERSION define. Let the
+# variables expand out to nothing, because the code will cause problems.
+extraversion1 := $(shell echo $(extraversion1))
+# EXTRAVERSION without local version.
+extraversion2 := $(strip $(subst $(klocalversion),,$(kextraversion)))
+# EXTRAVERSION with only the kernel .version (hopefully).
+extraversion3 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(^\.[0-9]*\).*/\1/'))
+# EXTRAVERSION without the Redhat EL tag.
+extraversion4 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\.EL.*//i'))
+# EXTRAVERSION with the Redhat EL tag, but nothing else after.
+extraversion5 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(\.EL\).*/\1/i'))
+# EXTRAVERSION with the Redhat EL tag, including a number (el5).
+extraversion6 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(\.EL[[:digit:]]*\).*/\1/i'))
+# EXTRAVERSION without the Redhat hotfix/update kernel version number.
+extraversion7 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(.*\-[[:digit:]]*\)\..*\(\.EL\).*/\1\2/i'))
+# EXTRAVERSION without the Redhat hotfix/update kernel version number with Redhat EL tag, including the number (el5).
+extraversion8 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(.*\-[[:digit:]]*\)\..*\(\.EL[[:digit:]]\).*/\1\2/i'))
+# EXTRAVERSION with only the RHEL distro version
+extraversion9 := $(strip $(shell echo $(kextraversion) |\
+ sed 's/\(.*\-[[:digit:]]*\)\..*\.EL[[:digit:]].*/\1/i'))
+
+# All known EXTRAVERSION strings, duplicates removed.
+extraversions := $(strip $(sort $(kextraversion) \
+ $(extraversion1) \
+ $(extraversion2) \
+ $(extraversion3) \
+ $(extraversion4) \
+ $(extraversion5) \
+ $(extraversion6) \
+ $(extraversion7) \
+ $(extraversion8) \
+ $(extraversion9)))
+
+# List of all possible kernel version names for target kernel.
+all_kernels := $(sort $(kbaseversion) \
+ $(foreach a,$(extraversions),$(kbaseversion)$(a)))
+
+# A reverse ordered list. This is used primarily to search source code
+# directory names to match the target kernel version.
+kversions := $(call reverse_sort, $(all_kernels))
+
+# Special cases for 2.4 series kernels.
+ifeq ($(kseries),2.4)
+ $(error ERROR: 2.4 kernel NOT supported.)
+endif
+# Note: Define only FLAGS here. These will convert to CFLAGS in the sub-make.
+# If the environment variable FLAGS is defined with make, things will break,
+# use CFLAGS instead.
+# General compiler flags.
+# Kernel version 3.3+ moved include/asm to include/generated, which breaks
+# outbox kernel drivers. Fix to include the new generated without code change.
+ifeq ($(kversion),3)
+ ifeq ($(shell [ $(kpatchlevel) -gt 2 ] && echo 1),1)
+ override CARCH := $(ARCH)
+ ifeq ($(CARCH),x86_64)
+ override CARCH := x86
+ endif
+ FLAGS += -I$(KSRC)/arch/$(CARCH)/include/generated
+ endif
+endif
diff --git a/QDMA/linux-kernel/scripts/datafile_16bit_pattern.bin b/QDMA/linux-kernel/scripts/datafile_16bit_pattern.bin
new file mode 100755
index 0000000000000000000000000000000000000000..1b50cdd57bcc0fb5c0e6ae027560cd03062b5434
Binary files /dev/null and b/QDMA/linux-kernel/scripts/datafile_16bit_pattern.bin differ
diff --git a/QDMA/linux-kernel/scripts/qdma_run_stress_test.sh b/QDMA/linux-kernel/scripts/qdma_run_stress_test.sh
new file mode 100755
index 0000000000000000000000000000000000000000..0f035e9706121c92a53282cfa432b6aee693b2c2
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/qdma_run_stress_test.sh
@@ -0,0 +1,678 @@
+#!/bin/bash
+#
+# This script is designed for general as well as stress testing. VF distruction can be done by appropriately
+# configuring pf_nvf_lst.
+#
+# AXI-MM Transfer
+# First H2C operation is performed for 1KBytes, this will write 1Kbytes of data to BRAM on card side.
+# Then C2H operation is performed for 1KBytes. DMA reads data from BRAM and will transfer
+# to local file 'out_mm0_0', which will be compared to original file for correctness.
+#
+# AXI-ST H2C Transfer
+# for H2C Streaming transfer data needs to be a per-defined data. 16 bit incremental data.
+# Data file is provided with the script.
+# H2C operation is performed, Data is read from Host memory and send to Card side. There is a data checker
+# on the card side which will check the data for correctness and will log the result in a register.
+# Script then read the register to check for results.
+#
+#
+# AXI-ST C2H Transfer
+# For C2H operation there is a data generator on the Card side which needs to be setup to generate data.
+# Qid, transfer length and number of paket are written before C2H transfer. Then
+# C2H transfer is started by writing to register. C2H operation is completed and the data is written to 'out_st0_0"
+# file which then is compared to a per-defined data file. The data generator will only generate pre-defined
+# data, so data comparison will need to be done with 'datafile_16bit_pattern.bin' file only.
+#
+#
+
+
+function print_help() {
+ echo ""
+echo "Usage : $0 <bdf> <num_qs> <total_time> <overide_pf_nvf_lst> <log_file_suffix>"
+ echo "Ex : $0 81000 256 00:01:00 0"
+ echo "<bdf> : PF Bus device function in bbddf format ex:06000"
+ echo ""
+ echo "<num_qs> : number of queue from qid_start"
+ echo " Default - 04 "
+ echo ""
+ echo "<total_time> : execution time in hh:mm:ss format"
+ echo " Default - 00:01:00 "
+ echo ""
+ echo "<overide_pf_nvf_lst> : override the default pf_nvf_lst by reading sriov_totalvfs"
+ echo " Default - 0 "
+ echo ""
+ #echo "<desc_byapss_en> : Enable desc bypass"
+ #echo " Default - 0 "
+ #echo ""
+ #echo "<pftch_en> : Enable prefetch"
+ #echo " Default - 0 "
+ #echo ""
+ #echo "<pftch_bypass_en> : Enable prefetch bypass"
+ #echo " Default - 0 "
+ #echo ""
+ echo ""
+ exit 1
+}
+
+if [ $# -lt 1 ]; then
+ echo "Invalid arguements."
+ print_help
+ exit;
+fi;
+
+log_file_suffix=default
+total_time="00:01:00"
+pf=$1
+num_qs=2048
+desc_byp=0
+pftch=0
+pftch_byp=0
+declare -a pf_nvf_lst=(1 1 1 1)
+overide_pf_nvf_lst=0
+max_q_func=512
+qmax_vfs=0
+total_funcs=4
+num_pfs=4
+declare -a bdf_array=()
+declare -a mm_q_lst=()
+declare -a st_q_lst=()
+declare -a qbase_array=()
+declare -a num_queue_per_pf=()
+declare -a full_bdf_array=()
+
+if [ ! -z $2 ]; then
+ num_qs=$2
+fi
+if [ ! -z $3 ]; then
+ total_time=$3
+fi
+if [ ! -z $4 ]; then
+ overide_pf_nvf_lst=$4
+fi
+if [ ! -z $5 ]; then
+ log_file_suffix=$5
+fi
+if [ ! -z $6 ]; then #if arg4 is there byp enable
+ desc_byp=$6
+fi
+
+if [ ! -z $7 ]; then #if arg5 is there pfetch enable
+ pftch=$7
+fi
+
+if [ ! -z $8 ]; then #if arg6 is there pfetch byp enable
+ pftch_byp=$8
+fi
+thrs=$((${total_time:0:2}))
+tmins=$((${total_time:3:2}))
+tsecs=$((${total_time:6:2}))
+total_exec_time=$(( (60*60*thrs) + (60*tmins) + tsecs ))
+
+infile='./datafile_16bit_pattern.bin'
+stress_test_exit=0
+
+function get_dev() {
+ bdf_array=()
+ qbase_array=()
+ num_queue_per_pf=()
+ full_bdf_array=()
+ local dev_list="$(dmactl dev list | grep qdma)"
+ while read -r line; do
+ IFS=$'\t ,~' read -r -a array <<< "$line"
+ qdmadev=${array[0]}
+ if [ "${qdmadev:0:6}" == "qdmavf" ]; then
+ qdmavfbdf=${qdmadev:6:5}
+ bdf_array+=("${qdmavfbdf}")
+ else
+ qdmabdf=${qdmadev:4:5}
+ bdf_array+=("${qdmabdf}")
+ fi
+ full_bdf_array+=("${array[1]}")
+ num_queue_per_pf+=("${array[4]}")
+ qbase_array+=("${array[5]}")
+ done <<< "$dev_list"
+}
+
+function is_vf_bdf() {
+ local lbdf=$1
+ local lpci_device=$((0x${lbdf:2:2}))
+ local lpci_func=$((0x${lbdf:4:1}))
+ if [ ${lpci_device} -gt 0 -o ${lpci_func} -gt 3 ]; then
+ echo 1
+ else
+ echo 0
+ fi
+}
+
+function get_base_qid() {
+ num_devs=${#bdf_array[@]}
+ lq_base==-1
+ for ((i = 0; i < num_devs; i++))
+ do
+ if [ ${bdf_array[i]} == $1 ]; then
+ lq_base=$((${qbase_array[${i}]}))
+ break
+ fi
+ done
+ echo ${lq_base}
+}
+
+function queue_start() {
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $1)
+ if [ $is_vf -eq 1 ]; then
+ qdmadev="qdmavf"
+ fi
+ echo -ne "setting up ${qdmadev}$1-$3-$2\033[0K\r"
+ dmactl ${qdmadev}$1 q add idx $2 mode $3 dir bi >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ if [ $? -lt 0 ]; then
+ echo "q add failed for ${qdmadev}$1-$3-$2"
+ return
+ fi
+
+ if [ $desc_byp -eq 1 ] && [ $pftch -eq 0 ]; then
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl ${qdmadev}$1 q start idx $2 dir bi desc_bypass_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ dmactl ${qdmadev}$1 q start idx $2 dir bi desc_bypass_en pfetch_bypass_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ elif [ $desc_byp -eq 1 ] && [ $pftch -eq 1 ]; then
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl ${qdmadev}$1 q start idx $2 dir bi desc_bypass_en pfetch_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ dmactl ${qdmadev}$1 q start idx $2 dir bi desc_bypass_en pfetch_en pfetch_bypass_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ elif [ $desc_byp -eq 0 ] && [ $pftch -eq 1 ] ; then
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl ${qdmadev}$1 q start idx $2 dir bi pfetch_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ dmactl ${qdmadev}$1 q start idx $2 dir bi pfetch_en pfetch_bypass_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ else
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl ${qdmadev}$1 q start idx $2 dir bi>> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ dmactl ${qdmadev}$1 q start idx $2 dir bi pfetch_bypass_en >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ fi
+ if [ $? -lt 0 ]; then
+ echo "q start failed for ${qdmadev}$1-$3-$2"
+ dmactl ${qdmadev}$1 q del idx $2 dir bi >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ return $?
+ fi
+ if [ $3 == mm ]; then
+ mm_q_lst+=("$1-$2")
+ else
+ st_q_lst+=("$1-$2")
+ fi
+ return 0
+}
+
+function cleanup_queue() {
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $1)
+ if [ $is_vf -eq 1 ]; then
+ qdmadev="qdmavf"
+ fi
+ echo -ne "cleaning up ${qdmadev}$1-$2\033[0K\r"
+ dmactl ${qdmadev}$1 q stop idx $2 dir bi >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ dmactl ${qdmadev}$1 q del idx $2 dir bi >> ./run_stress_test_${log_file_suffix}.log 2>&1
+
+}
+
+
+# Find user bar
+function get_user_bar() {
+ bdf=$1
+ tmp=`dmactl qdma${bdf} reg read bar 0 0x10C | grep "0x10c" | cut -d '=' -f2 | cut -d 'x' -f2 | cut -d '.' -f1`
+ bar_ext=$(printf '%x\n' "$(( 0x$tmp & 0x00000f ))")
+
+ if [ $bar_ext -eq 2 ]; then
+ usr_bar=1
+ elif [ $bar_ext -eq 4 ];then
+ usr_bar=2
+ fi
+}
+
+function get_env() {
+ if [ ${overide_pf_nvf_lst} -ne 0 ]; then
+ pf_nvf_lst=()
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ nvf=$(cat /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/sriov_totalvfs)
+ pf_nvf_lst+=("${nvf}")
+ done
+ fi
+ total_funcs=0
+ for (( i = 0; i < ${num_pfs}; i++))
+ do
+ num_vf=${pf_nvf_lst[$i]}
+ total_funcs=$((total_funcs+num_vf))
+ done
+ if [ $total_funcs -gt $num_qs ]; then
+ echo "ERROR: num_qs less than total functions. Min queues required are $total_funcs"
+ exit 1
+ fi
+ max_q_func=$((num_qs/total_funcs))
+ for (( i = 0; i < ${num_pfs}; i++))
+ do
+ num_vf=${pf_nvf_lst[$i]}
+ qmax_vfs=$((qmax_vfs+(max_q_func*num_vf)))
+ done
+
+}
+
+function start_vfs() {
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+
+ #set qmaxvfs
+ if [ ${pci_func} == 0 ]; then
+ echo "echo ${qmax_vfs} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/qdma/qmax_vfs"
+ echo ${qmax_vfs} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/qdma/qmax_vfs
+ if [ $? -lt 0 ]; then
+ echo "set qmax_vfs failed for 0000\:${pci_bus}\:${pci_device}.${pci_func}"
+ fi
+ fi
+ num_vfs=${pf_nvf_lst[${pci_func}]}
+ echo "echo ${num_vfs} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/sriov_numvfs"
+ echo ${num_vfs} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/sriov_numvfs
+ if [ $? -ne 0 ]; then
+ echo "enabling vfs failed for 0000\:${pci_bus}\:${pci_device}.${pci_func}/virtfn${i}"
+ fi
+ done
+ sleep 1 # allow all probes to finish
+}
+
+function set_pf_qmax() {
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ num_vfs=${pf_nvf_lst[${pci_func}]}
+ if (( num_vfs > 0 )); then
+ echo -ne "echo ${max_q_func} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/qdma/qmax\033[0K\r"
+ echo ${max_q_func} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/qdma/qmax
+ if [ $? -lt 0 ]; then
+ echo "set_qmax failed for 0000\:${pci_bus}\:${pci_device}.${pci_func}"
+ fi
+ sleep .1
+ fi
+ done
+ echo ""
+}
+
+function set_vf_qmax() {
+ total_funcs=${#bdf_array[@]}
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ num_vfs=${pf_nvf_lst[${pci_func}]}
+ for (( i = 0; i < num_vfs; i++ ))
+ do
+ echo -ne "echo ${max_q_func} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/virtfn${i}/qdma/qmax\033[0K\r"
+ echo ${max_q_func} > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/virtfn${i}/qdma/qmax
+ if [ $? -lt 0 ]; then
+ echo "set_qmax failed for 0000\:${pci_bus}\:${pci_device}.${pci_func}/virtfn${i}"
+ fi
+ sleep .1
+ done
+ done
+ echo ""
+}
+
+function queue_init() {
+ mode=mm
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ for (( qid = 0; qid < max_q_func; qid++ ))
+ do
+ if [ $mode == mm ]; then
+ mode=st
+ else
+ mode=mm
+ fi
+ queue_start $bdf $qid $mode
+ done
+ done
+ echo ""
+}
+
+function queue_exit() {
+ echo "exit sequence..."
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ for (( qid = 0; qid < max_q_func; qid++ ))
+ do
+ cleanup_queue $bdf $qid
+ done
+ done
+ echo ""
+ echo "Device Stats:" > ./run_stress_test_stat_${log_file_suffix}.log
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $bdf)
+ if [ $is_vf -ne 0 ]; then
+ qdmadev="qdmavf"
+ fi
+ echo "${qdmadev}${bdf} Stats:" >> ./run_stress_test_stat_${log_file_suffix}.log
+ dmactl ${qdmadev}${bdf} stat >> ./run_stress_test_stat_${log_file_suffix}.log
+ echo "---------------------------------------------------------" >> ./run_stress_test_stat_${log_file_suffix}.log
+ done
+ for bdf in ${bdf_array[@]}
+ do
+ pci_bus=${bdf:0:2}
+ pci_device=${bdf:2:2}
+ pci_func=${bdf:4:1}
+ is_vf=$(is_vf_bdf $bdf)
+ if [ $is_vf -eq 0 ]; then
+ echo "echo 0 > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/sriov_numvfs"
+ echo 0 > /sys/bus/pci/devices/0000\:${pci_bus}\:${pci_device}.${pci_func}/sriov_numvfs
+ fi
+ done
+}
+
+function setup_env() {
+ get_dev
+ num_pfs=${#bdf_array[@]}
+ get_env
+ start_vfs
+ set_pf_qmax
+ set_vf_qmax
+ get_dev
+ dmactl dev list
+ get_user_bar $pf
+}
+
+function monitor_exec_time() {
+ flag=1
+ MON_STARTTIME=$(date +%s)
+ monelapsed_time=0
+ while [ true ]; do
+ if [ $flag -eq 1 ]; then
+ echo -ne "elapsed time: $monelapsed_time secs...\033[0K\r"
+ flag=0
+ else
+ echo -ne "elapsed time: $monelapsed_time secs..|\033[0K\r"
+ flag=1
+ fi
+ MON_CURTIME=$(date +%s)
+ monelapsed_time=$((MON_CURTIME - MON_STARTTIME))
+ if [ $monelapsed_time -gt $total_exec_time ]; then
+ stress_test_exit=1
+ return
+ else
+ sleep 1
+ fi
+ done
+ echo ""
+ echo "$1-completed"
+}
+
+function run_mm_h2c_c2h() {
+ MM_STARTTIME=$(date +%s)
+ while [ true ]
+ do
+ echo "***********************************************" 2>&1 >> ./run_mm_stress_test_${log_file_suffix}.log
+ local size=$(( ($RANDOM % 262144) +1 ))
+ total_qs=${#mm_q_lst[@]}
+ for ((i=0; i < total_qs; i++)); do
+ mm_q_lst_entry=${mm_q_lst[${i}]}
+ bdf=${mm_q_lst_entry:0:5}
+ strlen=$(echo -n $mm_q_lst_entry | wc -m)
+ end_idx=$((strlen-1))
+ qid=${mm_q_lst_entry:6:${end_idx}}
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $bdf)
+ if [ $is_vf -eq 1 ]; then
+ qdmadev="qdmavf"
+ fi
+
+ echo "AXI-MM for Func ${bdf} Start" 2>&1 >> ./run_mm_stress_test_${log_file_suffix}.log
+ # Setup for Queues
+ dev_mm_c2h="/dev/${qdmadev}${bdf}-MM-$qid"
+ dev_mm_h2c="/dev/${qdmadev}${bdf}-MM-$qid"
+
+ out_mm="/tmp/out_mm${bdf}_$qid"
+ if [ $desc_byp -eq 1 ]; then
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x090 3 >> ./run_mm_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ # H2C transfer
+ dma_to_device -d $dev_mm_h2c -f $infile -s $size >> ./run_mm_stress_test_${log_file_suffix}.log 2>&1
+
+ # C2H transfer
+ dma_from_device -d $dev_mm_c2h -f $out_mm -s $size >> ./run_mm_stress_test_${log_file_suffix}.log 2>&1
+
+ # Compare file for correctness
+ cmp $out_mm $infile -n $size
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue ${qdmadev}${bdf}-$qid data did not match - iosz - ${size} ####" >> ./run_mm_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} q dump idx $qid >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} reg dump >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ echo "**** Test pass. Queue ${qdmadev}${bdf}-$qid" >> ./run_mm_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ echo "AXI-MM for Func ${bdf} End" 2>&1 >> ./run_mm_stress_test_${log_file_suffix}.log
+ MM_CURTIME=$(date +%s)
+ mmelapsed_time=$((MM_CURTIME - MM_STARTTIME))
+ if [ $mmelapsed_time -gt $total_exec_time ]; then
+ echo "exiting run_mm_h2c_c2h()"
+ return
+ fi
+ done
+ echo "***********************************************" 2>&1 >> ./run_mm_stress_test_${log_file_suffix}.log
+ done
+}
+
+
+
+function run_st_h2c() {
+ # AXI-ST H2C transfer
+ STH2C_STARTTIME=$(date +%s)
+ while [ true ]
+ do
+ echo "***********************************************" 2>&1 >> ./run_st_h2c_stress_test_${log_file_suffix}.log
+ local size=$(( ($RANDOM % 28672) +1 ))
+ total_qs=${#st_q_lst[@]}
+ for ((i=0; i < total_qs; i++)); do
+ st_q_lst_entry=${st_q_lst[${i}]}
+ bdf=${st_q_lst_entry:0:5}
+ strlen=$(echo -n $st_q_lst_entry | wc -m)
+ end_idx=$((strlen-1))
+ qid=${st_q_lst_entry:6:${end_idx}}
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $bdf)
+ if [ $is_vf -eq 1 ]; then
+ qdmadev="qdmavf"
+ fi
+ dev_st_h2c="/dev/${qdmadev}${bdf}-ST-$qid"
+ echo "AXI-ST H2C for Func ${bdf}-${qid} Start" 2>&1 >> ./run_st_h2c_stress_test_${log_file_suffix}.log
+
+ # Clear H2C match from previous runs. this register is in card side.
+ # MAtch clear register is on offset 0x0C
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x0C 0x1 >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1 # clear h2c Match register.
+
+ if [ $desc_byp -eq 1 ]; then
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x090 3 >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+ fi
+
+ # do H2C Transfer
+ dma_to_device -d $dev_st_h2c -f $infile -s $size >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+
+ # check for H2C data match. MAtch register is in offset 0x10.
+ pass=`dmactl ${qdmadev}${bdf} reg read bar $usr_bar 0x10 | grep "0x10" | cut -d '=' -f2 | cut -d 'x' -f2 | cut -d '.' -f1`
+ # convert hex to bin
+ code=`echo $pass | tr 'a-z' 'A-Z'`
+ val=`echo "obase=2; ibase=16; $code" | bc`
+ check=1
+ if [ $(($val & $check)) -eq 1 ];then
+ echo "*** Test passed for Queue $qid" >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+ else
+ echo "#### ERROR Test failed for ${qdmadev}${bdf}-${qid}. pattern did not match - iosz - ${size} ####" >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} q dump idx $qid >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} reg dump >> ./run_st_h2c_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ echo "AXI-ST H2C for Func ${bdf}-${qid} End" 2>&1 >> ./run_st_h2c_stress_test_${log_file_suffix}.log
+ STH2C_CURTIME=$(date +%s)
+ sth2celapsed_time=$((STH2C_CURTIME - STH2C_STARTTIME))
+ if [ $sth2celapsed_time -gt $total_exec_time ]; then
+ echo "exiting run_st_h2c()"
+ return
+ fi
+ done
+ echo "***********************************************" 2>&1 >> ./run_st_h2c_stress_test_${log_file_suffix}.log
+ done
+
+}
+
+function run_st_c2h() {
+ STC2H_STARTTIME=$(date +%s)
+ while [ true ]
+ do
+ echo "***********************************************" 2>&1 >> ./run_st_c2h_stress_test_${log_file_suffix}.log
+ local size=$(( ($RANDOM % 28672) +1 ))
+ local num_pkt=$(( ($RANDOM % 7) +1 ))
+ size=$((size+(size %2)))
+ total_qs=${#st_q_lst[@]}
+ for ((i = 0; i < total_qs; i++)); do
+ STC2H_CURTIME=$(date +%s)
+ stc2helapsed_time=$((STC2H_CURTIME - STC2H_STARTTIME))
+ if [ $stc2helapsed_time -gt $total_exec_time ]; then
+ echo "exiting run_st_c2h()"
+ return
+ fi
+ st_q_lst_entry=${st_q_lst[${i}]}
+ bdf=${st_q_lst_entry:0:5}
+ strlen=$(echo -n $st_q_lst_entry | wc -m)
+ end_idx=$((strlen-1))
+ qid=${st_q_lst_entry:6:${end_idx}}
+ qdmadev="qdma"
+ is_vf=$(is_vf_bdf $bdf)
+ if [ $is_vf -eq 1 ]; then
+ qdmadev="qdmavf"
+ fi
+ echo "AXI-ST C2H for Func ${bdf} Start" 2>&1 >> ./run_st_c2h_stress_test_${log_file_suffix}.log
+ q_base=`get_base_qid ${bdf}`
+ echo "q_base=${q_base}">> ./run_st_c2h_stress_test_${log_file_suffix}.log
+ if [ ${q_base} -lt 0 ]; then
+ echo "run_st_c2h: could not find q base for $bdf"
+ continue
+ fi
+ hw_qid=$((qid + q_base))
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x0 ${hw_qid} >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+
+ dev_st_c2h="/dev/${qdmadev}${bdf}-ST-$qid"
+ out_st="/tmp/out_st${bdf}_$qid"
+ let "tsize= $size*$num_pkt" # if more packets are requested.
+
+ #cache bypass
+ if [ $desc_byp -eq 1 ] && [ $pftch_byp -eq 0 ] ; then
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x090 2 >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ fi
+
+ #simple bypass
+ if [ $desc_byp -eq 1 ] && [ $pftch_byp -eq 1 ] ; then
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x090 4 >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ fi
+
+ #no bypass
+ if [ $desc_byp -eq 0 ] && [ $pftch_byp -eq 0 ] ; then
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x090 0 >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ fi
+
+ # Write transfer size to offset 0x04
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x4 $size >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+
+ # Write number of packets to offset 0x20
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x20 $num_pkt >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+
+ # Write to offset 0x80 bit [1] to trigger C2H data generator.
+ dmactl ${qdmadev}${bdf} reg write bar $usr_bar 0x08 2 >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+
+ # do C2H transfe
+ dma_from_device -d $dev_st_c2h -f $out_st -s $tsize >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+
+ cmp $out_st $infile -n $tsize>> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue ${qdmadev}${bdf}-$qid data did not match - iosz/numpkt - ${size}/${num_pkt} ####" >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} q dump idx $qid dir c2h >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ #dmactl ${qdmadev}${bdf} reg dump >> ./run_stress_test_${log_file_suffix}.log 2>&1
+ else
+ echo "**** Test pass. Queue $qid" >> ./run_st_c2h_stress_test_${log_file_suffix}.log 2>&1
+ fi
+ echo "AXI-ST C2H for Func ${bdf} End" 2>&1 >> ./run_st_c2h_stress_test_${log_file_suffix}.log
+ done
+ echo "***********************************************" 2>&1 >> ./run_st_c2h_stress_test_${log_file_suffix}.log
+ done
+}
+
+trap queue_exit EXIT
+echo "setting up environment"
+setup_env
+queue_init
+
+echo "transfer tests will run for $total_exec_time secs"
+echo "starting transfers"
+
+echo "###############################################################" > "run_mm_stress_test_${log_file_suffix}.log"
+echo "QDMA MM Stress Test on All Functions Starts" >> "run_mm_stress_test_${log_file_suffix}.log"
+echo "###############################################################" >> "run_mm_stress_test_${log_file_suffix}.log"
+run_mm_h2c_c2h &
+
+echo "###############################################################" > "run_st_h2c_stress_test_${log_file_suffix}.log"
+echo "QDMA ST H2C Stress Test on All Functions Starts" >> "run_st_h2c_stress_test_${log_file_suffix}.log"
+echo "###############################################################" >> "run_st_h2c_stress_test_${log_file_suffix}.log"
+run_st_h2c &
+
+echo "###############################################################" > "run_st_c2h_stress_test_${log_file_suffix}.log"
+echo "QDMA ST C2H Stress Test on All Functions Starts" >> "run_st_c2h_stress_test_${log_file_suffix}.log"
+echo "###############################################################" >> "run_st_c2h_stress_test_${log_file_suffix}.log"
+run_st_c2h &
+
+monitor_exec_time
+wait
+
+function print_errs() {
+ while read -r line; do
+ echo $line
+ done <<< "$1"
+}
+
+mm_errs=$(cat run_mm_stress_test_${log_file_suffix}.log | grep "Error\|ERROR\|FAIL\|Fail")
+st_h2c_errs=$(cat run_st_h2c_stress_test_${log_file_suffix}.log | grep "Error\|ERROR\|FAIL\|Fail")
+st_c2h_errs=$(cat run_st_c2h_stress_test_${log_file_suffix}.log | grep "Error\|ERROR\|FAIL\|Fail")
+if [ "${mm_errs}" != "" ]; then
+ echo "MM Errors:"
+fi
+print_errs "${mm_errs}"
+if [ "${st_h2c_errs}" != "" ]; then
+ echo "ST H2C Errors:"
+fi
+print_errs "${st_h2c_errs}"
+if [ "${st_c2h_errs}" != "" ]; then
+ echo "ST C2H Errors:"
+fi
+print_errs "${st_c2h_errs}"
+
+echo ""
+
+exit 0
\ No newline at end of file
diff --git a/QDMA/linux-kernel/scripts/qdma_run_test_mm_vf.sh b/QDMA/linux-kernel/scripts/qdma_run_test_mm_vf.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a2bebe26f17a377b3f9017f776107ee7bf2b3c9d
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/qdma_run_test_mm_vf.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+#
+# Simple run script to test QDMA in VF AXI-MM mode.
+#
+# VF AXI-MM Transfer
+# - H2C operation is performed to send data to BRAM on the FPGA.
+# - C2H operation is performed to reads data from BRAM.
+# - C2H data is stored in a local file 'out_mm0_$qid', which will be compared to original file for correctness.
+
+################################################
+# User Configurable Parameters
+################################################
+iteration=$1 # [Optional] Iterations
+size=$2 # [Optional] Size per payload packet
+
+################################################
+# Hard Coded Parameters
+################################################
+q_per_vf=1
+vf=00080
+size_max=4096
+host_adr_high=0
+infile='./datafile_16bit_pattern.bin'
+
+
+################################################
+# Input check
+################################################
+if [ "$1" == "-h" ]; then
+ echo "Example: qdma_run_test_st_vf.sh [iteration] [size(in byte)]"
+ echo "Example: qdma_run_test_st_vf.sh This will run VF MM test in random mode"
+ exit
+fi
+
+if [ -z $2 ] || [ $# -eq 0 ] ; then
+ echo "Run VF MM test in random mode"
+ sleep 3
+fi
+
+if [ -z $iteration ]; then
+ iteration=1
+fi
+
+if [ ! -z $size ]; then
+ f_size=1
+else
+ f_size=0
+fi
+
+#################################################
+# Helper Functions
+################################################
+
+function randomize_tx_params() {
+ #random host address
+ if [ $host_adr_high -ne 0 ]; then
+ hst_adr1=$RANDOM
+ hst_adr1=$((hst_adr1 % host_adr_high))
+ else
+ hst_adr1=0
+ fi
+
+ # byte size
+ size=$RANDOM
+ if [ $size -eq 0 ]; then
+ size=$(($RANDOM % 64 + 1)) ## for random number between 1 and 64
+ else
+ size=$((size % $size_max))
+ fi
+
+ # Correct if size is odd
+ even=$((size%2))
+ if [ $even -eq 1 ];then
+ size=$((size+1))
+ fi
+}
+
+
+function queue_start() {
+ echo "---- Queue Start $2 ----"
+ dmactl qdma$1 q add idx $2 mode $3 dir bi
+ dmactl qdma$1 q start idx $2 dir bi
+}
+
+function cleanup_queue() {
+ echo "---- Queue Clean up $2 ----"
+ dmactl qdma$1 q stop idx $2 dir bi
+ dmactl qdma$1 q del idx $2 dir bi
+}
+
+vfs=`dmactl dev list | grep qdmavf | cut -d' ' -f1`;
+
+echo "**** AXI-MM Start ****"
+for vfsdev in $vfs;do
+ vf="${vfsdev#*f}"
+ q_per_vf="$(dmactl dev list |grep qdmavf$vf | cut -d ' ' -f 3 | cut -d ',' -f 1 | xargs)"
+
+ for ((i=0; i< $q_per_vf; i++)) do
+ # Setup for Queues
+ qid=$i
+ dev_mm_c2h="/dev/qdmavf$vf-MM-$qid"
+ dev_mm_h2c="/dev/qdmavf$vf-MM-$qid"
+ loop=1
+
+ out_mm="out_mm0_"$qid
+ # Open the Queue for AXI-MM streaming interface.
+ queue_start vf$vf $qid mm
+
+ while [ "$loop" -le $iteration ]
+ do
+ # Determine if DMA is targeted @ random host address
+ if [ $f_size -eq 1 ]; then
+ hst_adr1=0
+ else
+ randomize_tx_params
+ fi
+
+ # H2C transfer
+ dma_to_device -d $dev_mm_h2c -f $infile -s $size -o $hst_adr1
+
+ # C2H transfer
+ dma_from_device -d $dev_mm_c2h -f $out_mm -s $size -o $hst_adr1
+
+ # Compare file for correctness
+ cmp $out_mm $infile -n $size
+
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue $qid data did not match ####"
+ exit 1
+ else
+ echo "**** Test pass. Queue $qid"
+ fi
+
+ wait
+
+ ((loop++))
+ done
+ # Close the Queues
+ cleanup_queue vf$vf $qid
+ done
+done
+echo "**** AXI-MM completed ****"
diff --git a/QDMA/linux-kernel/scripts/qdma_run_test_pf.sh b/QDMA/linux-kernel/scripts/qdma_run_test_pf.sh
new file mode 100755
index 0000000000000000000000000000000000000000..8e1ca2b99baf1f1989363401f60d5eac98c6eb8b
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/qdma_run_test_pf.sh
@@ -0,0 +1,453 @@
+#!/bin/bash
+#
+# Simple run script to test QDMA in AXI-MM and AXI-St mode.
+#
+# AXI-MM Transfer
+# First H2C operation is performed for 1KBytes, this will write 1Kbytes of data to BRAM on card side.
+# Then C2H operation is performed for 1KBytes. DMA reads data from BRAM and will transfer
+# to local file 'out_mm0_0', which will be compared to original file for correctness.
+#
+# AXI-ST H2C Transfer
+# for H2C Streaming transfer data needs to be a per-defined data. 16 bit incremental data.
+# Data file is provided with the script.
+# H2C operation is performed, Data is read from Host memory and send to Card side. There is a data checker
+# on the card side which will check the data for correctness and will log the result in a register.
+# Script then read the register to check for results.
+#
+#
+# AXI-ST C2H Transfer
+# For C2H operation there is a data generator on the Card side which needs to be setup to generate data.
+# Qid, transfer length and number of paket are written before C2H transfer. Then
+# C2H transfer is started by writing to register. C2H operation is completed and the data is written to 'out_st0_0"
+# file which then is compared to a per-defined data file. The data generator will only generate pre-defined
+# data, so data comparison will need to be done with 'datafile_16bit_pattern.bin' file only.
+#
+#
+
+
+function print_help() {
+ echo ""
+ echo "Usage : $0 <bdf> <qid_start> <num_qs> <desc_bypass_en> <pftch_en> <pfetch_bypass_en>"
+ echo "Ex : $0 81000 0 4 1 1 1"
+ echo "<bdf> : PF Bus device function in bbddf format ex:06000"
+ echo ""
+ echo "<qid_start> : qid start"
+ echo ""
+ echo "<num_qs> : number of queue from qid_start"
+ echo " Default - 04 "
+ echo ""
+ echo "<desc_byapss_en> : Enable desc bypass"
+ echo " Default - 0 "
+ echo ""
+ echo "<pftch_en> : Enable prefetch"
+ echo " Default - 0 "
+ echo ""
+ echo "<pftch_bypass_en> : Enable prefetch bypass"
+ echo " Default - 0 "
+ echo ""
+ echo ""
+ exit 1
+}
+
+if [ $# -lt 2 ]; then
+ echo "Invalid arguements."
+ print_help
+ exit;
+fi;
+
+pf=$1
+qid_start=$2
+num_qs=4
+desc_byp=0
+pftch=0
+pftch_byp=0
+
+if [ ! -z $3 ]; then
+ num_qs=$3
+fi
+
+if [ ! -z $4 ]; then #if arg4 is there byp enable
+ desc_byp=$4
+fi
+
+if [ ! -z $5 ]; then #if arg5 is there pfetch enable
+ pftch=$5
+fi
+
+if [ ! -z $6 ]; then #if arg6 is there pfetch byp enable
+ pftch_byp=$6
+fi
+
+echo "$pf $qid_start $num_qs $desc_byp $pftch $pftch_byp"
+size=1024
+num_pkt=1 #number of packets not more then 64
+infile='./datafile_16bit_pattern.bin'
+declare -a bypass_mode_lst=(NO_BYPASS_MODE DESC_BYPASS_MODE CACHE_BYPASS_MODE SIMPLE_BYPASS_MODE)
+
+
+function get_dev () {
+ pf_list="$(dmactl dev list | grep qdma)"
+ echo "$pf_list"
+ while read -r line; do
+ IFS=$'\t ,~' read -r -a array <<< "$line"
+ qdmabdf=${array[0]}
+ bdf_array+=("${qdmabdf#*a}")
+ full_bdf_array+=("${array[1]}")
+ num_queue_per_pf+=("${array[4]}")
+ qbase_array+=("${array[5]}")
+ done <<< "$pf_list"
+
+}
+
+function set_bypass_mode() {
+ dev=$1
+ mode=$2
+ dir=$3
+ bypass=$4
+ local reg_val=0x00;
+
+ if [ $mode == mm ]; then
+ case $dir in
+ h2c)
+ if [ $bypass == DESC_BYPASS_MODE ]; then
+ echo "setting DESC_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x1;
+ else
+ reg_val=0x0;
+ fi
+
+ ;;
+ c2h)
+ if [ $bypass == DESC_BYPASS_MODE ]; then
+ echo "setting DESC_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x2;
+ else
+ reg_val=0x0;
+ fi
+
+ ;;
+ bi)
+ if [ $bypass == DESC_BYPASS_MODE ]; then
+ echo "setting DESC_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x3;
+ else
+ reg_val=0x0;
+ fi
+ ;;
+ esac
+ else
+ case $dir in
+ h2c)
+ case $bypass in
+ CACHE_BYPASS_MODE)
+ echo "setting CACHE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x1;
+ ;;
+ SIMPLE_BYPASS_MODE)
+ echo "setting SIMPLE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x1;
+ ;;
+ *)
+ echo "setting NO_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x00;
+ ;;
+ esac
+ ;;
+ c2h)
+ case $bypass in
+ CACHE_BYPASS_MODE)
+ echo "setting CACHE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x2;
+ ;;
+ SIMPLE_BYPASS_MODE)
+ echo "setting SIMPLE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x4;
+ ;;
+ *)
+ echo "setting NO_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x00;
+ ;;
+ esac
+ ;;
+ bi)
+ case $bypass in
+ CACHE_BYPASS_MODE)
+ echo "setting CACHE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x3;
+ ;;
+ SIMPLE_BYPASS_MODE)
+ echo "setting SIMPLE_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x5;
+ ;;
+ *)
+ echo "setting NO_BYPASS_MODE for ${mode}-$dir"
+ reg_val=0x00;
+ ;;
+ esac
+ ;;
+ esac
+ fi
+ dmactl qdma$dev reg write bar 2 0x90 $reg_val
+}
+
+function get_bypass_mode() {
+ byp_mode=0
+ if [ $1 == mm ]; then
+ if [ $desc_byp -eq 1 ]; then
+ byp_mode=1
+ fi
+ else
+ if [ $desc_byp -eq 1 ] && [ $pftch_byp -eq 0 ]; then
+ byp_mode=2
+ elif [ $desc_byp -eq 1 ] && [ $pftch_byp -eq 1 ]; then
+ byp_mode=3
+ fi
+ fi
+ echo $byp_mode
+}
+
+function queue_start() {
+ echo "setting up qdma$1-$3-$2"
+ dmactl qdma$1 q add idx $2 mode $3 dir $4 >> ./run_pf.log 2>&1
+ if [ $? -ne 0 ]; then
+ echo "q add failed for qdma$1-$3-$2"
+ return
+ fi
+ bypass_mode=$(get_bypass_mode $3 $4)
+ set_bypass_mode $1 $3 $4 ${bypass_mode_lst[${bypass_mode}]}
+ if [ $3 == mm -o $4 == h2c ]; then
+ if [ $desc_byp -eq 1 ]; then
+ dmactl qdma$1 q start idx $2 dir $4 desc_bypass_en >> ./run_pf.log 2>&1
+ else
+ dmactl qdma$1 q start idx $2 dir $4 >> ./run_pf.log 2>&1
+ fi
+ else
+ if [ $desc_byp -eq 1 ] && [ $pftch -eq 0 ]; then
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl qdma$1 q start idx $2 dir $4 desc_bypass_en >> ./run_pf.log 2>&1
+ else
+ dmactl qdma$1 q start idx $2 dir $4 desc_bypass_en pfetch_bypass_en >> ./run_pf.log 2>&1
+ fi
+ elif [ $desc_byp -eq 1 ] && [ $pftch -eq 1 ]; then
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl qdma$1 q start idx $2 dir $4 desc_bypass_en pfetch_en >> ./run_pf.log 2>&1
+ else
+ dmactl qdma$1 q start idx $2 dir $4 desc_bypass_en pfetch_en pfetch_bypass_en >> ./run_pf.log 2>&1
+ fi
+ elif [ $desc_byp -eq 0 ] && [ $pftch -eq 1 ] ; then #
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl qdma$1 q start idx $2 dir $4 pfetch_en >> ./run_pf.log 2>&1
+ else
+ echo "Invalid case of bypass mode" >> ./run_pf.log 2>&1
+ dmactl qdma$1 q del idx $2 dir bi >> ./run_pf.log 2>&1
+ return 1
+ fi
+ else
+ if [ $pftch_byp -eq 0 ]; then
+ dmactl qdma$1 q start idx $2 dir $4>> ./run_pf.log 2>&1
+ else
+ echo "Invalid case of bypass mode" >> ./run_pf.log 2>&1
+ dmactl qdma$1 q del idx $2 dir bi >> ./run_pf.log 2>&1
+ return 1
+ fi
+ fi
+ fi
+ if [ $? -ne 0 ]; then
+ echo "q start failed for qdma$1-$3-$2-$4"
+ dmactl qdma$1 q del idx $2 dir bi >> ./run_pf.log 2>&1
+ return $?
+ fi
+
+
+ return 0
+}
+
+function cleanup_queue() {
+ echo "cleaning up qdma$1-$3-$2"
+ dmactl qdma$1 q stop idx $2 dir $4 >> ./run_pf.log 2>&1
+ dmactl qdma$1 q del idx $2 dir $4 >> ./run_pf.log 2>&1
+
+}
+
+
+# Find user bar
+function get_user_bar () {
+ local pf_bdf=$1
+ tmp=`dmactl qdma$pf_bdf reg read bar 0 0x10C | grep "0x10c" | cut -d '=' -f2 | cut -d 'x' -f2 | cut -d '.' -f1`
+ bar_ext=$(printf '%x\n' "$(( 0x$tmp & 0x00000f ))")
+
+ if [ $bar_ext -eq 2 ]; then
+ usr_bar=1
+ elif [ $bar_ext -eq 4 ];then
+ usr_bar=2
+ fi
+}
+
+
+function run_mm_h2c_c2h () {
+ for pf_bdf in ${bdf_array[@]}; do
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ echo "AXI-MM for Func $pf_bdf Start" 2>&1 | tee -a ./run_pf.log
+ get_user_bar $pf_bdf
+ for ((i=$qid_start; i < (($qid_start + $num_qs)); i++)); do
+ # Setup for Queues
+ qid=$i
+ dev_mm_c2h="/dev/qdma$pf_bdf-MM-$qid"
+ dev_mm_h2c="/dev/qdma$pf_bdf-MM-$qid"
+
+ out_mm="/tmp/out_mm"$pf_bdf"_"$qid
+ # Open the Queue for AXI-MM streaming interface.
+ queue_start $pf_bdf $qid mm bi
+ if [ $? -ne 0 ]; then
+ echo "q setup for qdma$pf_bdf-MM-$qid failed"
+ continue
+ fi
+ echo "setup for qdma$pf_bdf-MM-$qid done"
+ # H2C transfer
+ dma_to_device -d $dev_mm_h2c -f $infile -s $size >> ./run_pf.log 2>&1
+
+ # C2H transfer
+ dma_from_device -d $dev_mm_c2h -f $out_mm -s $size >> ./run_pf.log 2>&1
+
+ # Compare file for correctness
+ cmp $out_mm $infile -n $size
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue $qid data did not match ####"
+ dmactl qdma$pf_bdf q dump idx $qid >> ./run_pf.log 2>&1
+ dmactl qdma$pf_bdf reg dump >> ./run_pf.log 2>&1
+ else
+ echo "**** Test pass. Queue $qid"
+ fi
+ # Close the Queues
+ cleanup_queue $pf_bdf $qid st bi
+ echo "-----------------------------------------------"
+ done
+ echo "AXI-MM for Func $pf_bdf End" 2>&1 | tee -a ./run_pf.log
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ done
+}
+
+
+
+function run_st_h2c () {
+
+ # AXI-ST H2C transfer
+ for pf_bdf in "${bdf_array[@]}"; do
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ echo "AXI-ST H2C for Func $pf_bdf Start" 2>&1 | tee -a ./run_pf.log
+ get_user_bar $pf_bdf
+ for ((i=$qid_start; i < (($qid_start + $num_qs)); i++)); do
+ # Setup for Queues
+ qid=$i
+ queue_start $pf_bdf $qid st h2c # open the Queue for AXI-ST streaming interface.
+
+ dev_st_h2c="/dev/qdma$pf_bdf-ST-$qid"
+
+ # Clear H2C match from previous runs. this register is in card side.
+ # MAtch clear register is on offset 0x0C
+ dmactl qdma$pf_bdf reg write bar $usr_bar 0x0C 0x1 >> ./run_pf.log 2>&1 # clear h2c Match register.
+
+ # do H2C Transfer
+ dma_to_device -d $dev_st_h2c -f $infile -s $size >> ./run_pf.log 2>&1
+
+ if [ $? -ne 0 ]; then
+ echo "#### ERROR Test failed. Transfer failed ####"
+ cleanup_queue $pf_bdf $qid st h2c
+ continue
+ fi
+ # check for H2C data match. MAtch register is in offset 0x10.
+ pass=`dmactl qdma$pf_bdf reg read bar $usr_bar 0x10 | grep "0x10" | cut -d '=' -f2 | cut -d 'x' -f2 | cut -d '.' -f1`
+ # convert hex to bin
+ code=`echo $pass | tr 'a-z' 'A-Z'`
+ val=`echo "obase=2; ibase=16; $code" | bc`
+ check=1
+ if [ $(($val & $check)) -eq 1 ];then
+ echo "*** Test passed for Queue $qid"
+ else
+ echo "#### ERROR Test failed. pattern did not match ####"
+ dmactl qdma$pf_bdf q dump idx $qid >> ./run_pf.log 2>&1
+ dmactl qdma$pf_bdf reg dump >> ./run_pf.log 2>&1
+ fi
+ cleanup_queue $pf_bdf $qid st h2c
+ echo "-----------------------------------------------"
+ done
+ echo "AXI-ST H2C for Func $pf_bdf End" 2>&1 | tee -a ./run_pf.log
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ done
+
+}
+
+function run_st_c2h () {
+ local pf=0
+
+ for pf_bdf in "${bdf_array[@]}"; do
+
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ echo "AXI-ST C2H for Func $pf_bdf Start" 2>&1 | tee -a ./run_pf.log
+
+ get_user_bar $pf_bdf
+
+ for ((i=$qid_start; i < (($qid_start + $num_qs)); i++)); do
+ # Setup for Queues
+ qid=$i
+ out_st="/tmp/out_st"$pf_bdf"_"$qid
+
+ # Each PF is assigned with 32 Queues. PF0 has queue 0 to 31, PF1 has 32 to 63
+ # Write QID in offset 0x00
+ hw_qid=$(($qid + ${qbase_array[$pf]} ))
+ dmactl qdma$pf_bdf reg write bar $usr_bar 0x0 $hw_qid >> ./run_pf.log 2>&1
+
+ # open the Queue for AXI-ST streaming interface.
+ queue_start $pf_bdf $qid st c2h
+
+ dev_st_c2h="/dev/qdma$pf_bdf-ST-$qid"
+ let "tsize= $size*$num_pkt" # if more packets are requested.
+
+ # Write transfer size to offset 0x04
+ dmactl qdma$pf_bdf reg write bar $usr_bar 0x4 $size >> ./run_pf.log 2>&1
+
+ # Write number of packets to offset 0x20
+ dmactl qdma$pf_bdf reg write bar $usr_bar 0x20 $num_pkt >> ./run_pf.log 2>&1
+
+ # Write to offset 0x80 bit [1] to trigger C2H data generator.
+ dmactl qdma$pf_bdf reg write bar $usr_bar 0x08 2 >> ./run_pf.log 2>&1
+
+ # do C2H transfer
+ dma_from_device -d $dev_st_c2h -f $out_st -s $tsize >> ./run_pf.log 2>&1
+ if [ $? -ne 0 ]; then
+ echo "#### ERROR Test failed. Transfer failed ####"
+ cleanup_queue $pf_bdf $qid st c2h
+ continue
+ fi
+
+ cmp $out_st $infile -n $tsize
+ if [ $? -ne 0 ]; then
+ echo "#### Test ERROR. Queue $2 data did not match ####"
+ dmactl qdma$pf_bdf q dump idx $qid dir c2h >> ./run_pf.log 2>&1
+ dmactl qdma$pf_bdf reg dump >> ./run_pf.log 2>&1
+ else
+ echo "**** Test pass. Queue $qid"
+ fi
+ # Close the Queues
+ cleanup_queue $pf_bdf $qid st c2h
+ echo "-----------------------------------------------"
+ done
+ pf=$((pf+1));
+ echo "AXI-ST C2H for Func $pf_bdf End" 2>&1 | tee -a ./run_pf.log
+ echo "***********************************************" 2>&1 | tee -a ./run_pf.log
+ done
+}
+
+
+echo "###############################################################" > "run_pf.log"
+echo "QDMA Test on All PFs Starts" >> "run_pf.log"
+echo "###############################################################" >> "run_pf.log"
+
+get_dev
+run_mm_h2c_c2h
+run_st_h2c
+run_st_c2h
+
+exit 0
+
+
diff --git a/QDMA/linux-kernel/scripts/qdma_run_test_st_vf.sh b/QDMA/linux-kernel/scripts/qdma_run_test_st_vf.sh
new file mode 100755
index 0000000000000000000000000000000000000000..7acce1143a45487bed3f1fbead7e687c2f8d716a
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/qdma_run_test_st_vf.sh
@@ -0,0 +1,209 @@
+#!/bin/bash
+# Simple run script to test QDMA in VF AXI-ST mode.
+#
+# VF AXI-ST Transfer
+# - H2C operation is performed to send data to BRAM on the FPGA.
+# - C2H operation is performed to reads data from BRAM.
+# - C2H data is stored in a local file 'out_st_$qid', which will be compared to original file for correctness.
+
+
+################################################
+# User Configurable Parameters
+################################################
+
+iteration=$1 # [Optional] iterations of C2H tests
+size=$2 # [Optional] Size per payload packet
+num_pkt=$3 # [Optional] number of payload packet
+
+################################################
+# Hard Coded Parameters
+################################################
+usr_bar=2 # For VF, DMA BAR is bar 0. User BAR is bar 2.
+vf=00080
+size_max=4096
+host_adr_high=0
+infile='./datafile_16bit_pattern.bin'
+logfile="loopback$1_$2.log"
+
+################################################
+# Input check
+################################################
+if [ -z $iteration ]; then
+ iteration=1
+fi
+
+if [ ! -z $size ]; then
+ f_size=1
+else
+ f_size=0
+fi
+
+if [ -z $num_pkt ]; then
+ num_pkt=1
+fi
+
+#################################################
+# Helper Functions
+################################################
+function randomize_tx_params() {
+ #random host address
+ if [ $host_adr_high -ne 0 ]; then
+ hst_adr1=$RANDOM
+ hst_adr1=$((hst_adr1 % host_adr_high))
+ else
+ hst_adr1=0
+ fi
+
+ # byte size
+ size=$RANDOM
+ if [ $size -eq 0 ]; then
+ size=$(($RANDOM % 64 + 1)) ## for random number between 1 and 64
+ else
+ size=$((size % $size_max))
+ fi
+
+ # Correct if size is odd
+ even=$((size%2))
+ if [ $even -eq 1 ];then
+ size=$((size+1))
+ fi
+}
+
+function queue_start() {
+ echo "---- Queue Start $2 ----"
+ dmactl qdma$1 q add idx $2 dir h2c mode $3
+ dmactl qdma$1 q start idx $2 dir h2c
+ dmactl qdma$1 q add idx $2 dir c2h mode $3
+ dmactl qdma$1 q start idx $2 dir c2h
+}
+
+function cleanup_queue() {
+ echo "---- Queue Clean up $2 ----"
+ dmactl qdma$1 q stop idx $2 dir h2c
+ dmactl qdma$1 q del idx $2 dir h2c
+ dmactl qdma$1 q stop idx $2 dir c2h
+ dmactl qdma$1 q del idx $2 dir c2h
+}
+
+# Get a list of available devices
+vfs="$(dmactl dev list | grep qdmavf | cut -d ' ' -f1)"
+echo "############################# AXI-ST Start #################################"
+
+for vfsdev in $vfs; do
+
+ vf="${vfsdev#*f}"
+ q_per_vf="$(dmactl dev list |grep qdmavf$vf | cut -d ' ' -f 3 | cut -d ',' -f 1 | xargs)"
+ hw_qbase="$(dmactl dev list |grep qdmavf$vf|cut -d',' -f 2 | cut -d '~' -f 1|xargs)"
+
+ for ((i=0; i<$q_per_vf; i++)) do
+
+ # Setup for Queues
+ qid=$i
+ hw_qid=$((qid+hw_qbase))
+ dev_st_c2h="/dev/qdmavf$vf-ST-$qid"
+ dev_st_h2c="/dev/qdmavf$vf-ST-$qid"
+ out="out_st_$qid"
+ loop=1
+
+ # Open the Queue for AXI-ST streaming interface.
+ queue_start vf$vf $qid st > /dev/null
+
+ while [ "$loop" -le $iteration ]
+ do
+ # Determine if DMA is targeted @ random host address
+ if [ $f_size -eq 1 ]; then
+ hst_adr1=0
+ else
+ randomize_tx_params
+ fi
+
+ # if more packets are requested.
+ let "tsize= $size*$num_pkt"
+
+ echo ""
+ echo "########################################################################################"
+ echo "############# H2C ST LOOP $loop : dev=$dev_st_h2c qid=$qid hw_qid=$hw_qid"
+ echo "############# transfer_size=$tsize pkt_size=$size pkt_count=$num_pkt hst_adr=$hst_adr1"
+ echo "########################################################################################"
+
+ #clear match bit before each H2C ST transfer
+ dmactl qdmavf$vf reg write bar $usr_bar 0x0c 0x01
+
+ # H2C transfer
+ dma_to_device -d $dev_st_h2c -f $infile -s $tsize -o $hst_adr1 &
+ re=$?
+
+ wait
+
+ # Check match bit and QID
+ hwqid_match=$(dmactl qdmavf$vf reg read bar $usr_bar 0x10 | grep "0x10" | cut -d '=' -f2 | cut -d 'x' -f2 | cut -d '.' -f1)
+ code=`echo $hwqid_match | tr 'a-z' 'A-Z'`
+ val=`echo "obase=2; ibase=16; $code" | bc`
+ if [ $(($val & 0x1)) -ne 1 ];then
+ echo "### ERROR: QID MATCH is $hwqid_match "$hw_qid_hex"1"
+ re=-1
+ fi
+
+ if [ $re == 0 ]; then
+ echo "######################################################"
+ echo "############## VF H2C ST PASS QID $qid ################"
+ echo "######################################################"
+ else
+ echo "#### ERROR: VF H2C ST FAIL"
+ fi
+
+ echo ""
+ echo "########################################################################################"
+ echo "############# C2H ST LOOP $loop : dev=$dev_st_c2h qid=$qid hw_qid=$hw_qid"
+ echo "############# transfer_size=$tsize pkt_size=$size pkt_count=$num_pkt hst_adr=$hst_adr1"
+ echo "########################################################################################"
+
+ dmactl qdmavf$vf reg write bar $usr_bar 0x0 $hw_qid # for Queue 0
+ dmactl qdmavf$vf reg write bar $usr_bar 0x4 $size
+ dmactl qdmavf$vf reg write bar $usr_bar 0x20 $num_pkt #number of packets
+ dmactl qdmavf$vf reg write bar $usr_bar 0x08 2 # Must set C2H start before starting transfer
+
+ dma_from_device -d $dev_st_c2h -f $out -o $hst_adr1 -s $tsize &
+
+ wait
+
+ #Check if files is there.
+ if [ ! -f $out ]; then
+ echo " #### ERROR: Queue $qid output file does not exists ####"
+ echo " #### ERROR: Queue $qid output file does not exists ####" >> $logfile
+ cleanup_queue vf$vf $qid st
+ exit -1
+ fi
+
+ # check files size
+ filesize=$(stat -c%s "$out")
+ if [ $filesize -gt $tsize ]; then
+ echo "#### ERROR: Queue $qid output file size does not match, filesize= $filesize ####"
+ echo "#### ERROR: Queue $qid output file size does not match, filesize= $filesize ####" >> $logfile
+ cleanup_queue vf$vf $qid st
+ exit -1
+ fi
+
+ #compare file
+ cmp $out $infile -n $tsize
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue $qid data did not match ####"
+ echo "#### Test ERROR. Queue $qid data did not match ####" >> $logfile
+ dmactl qdmavf$vf q dump idx $qid mode st dir c2h
+ dmactl qdmavf$vf reg dump
+ cleanup_queue vf$vf $qid st
+ exit -1
+ else
+ echo "######################################################"
+ echo "############## VF C2H ST PASS QID $qid ################"
+ echo "######################################################"
+ fi
+ wait
+ ((loop++))
+ done
+ cleanup_queue vf$vf $qid st > /dev/null
+ done
+done
+echo "########################## AXI-ST completed ###################################"
+exit 0
+
diff --git a/QDMA/linux-kernel/scripts/qdma_vf_auto_tst.sh b/QDMA/linux-kernel/scripts/qdma_vf_auto_tst.sh
new file mode 100644
index 0000000000000000000000000000000000000000..95d92743ee6735520860650ce9a0199b9af87727
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/qdma_vf_auto_tst.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Simple script to automate the VF regression testing using gtest framework
+#
+# This script will insert the module, configure the qmax_vfs, and instantiate
+# vfs on each PF one at a time. After instantiating the VFs, it will trigger
+# qdma_test regression test suite for VF only.
+#
+# Inputs:
+# MODULE_TOP_DIR= Top directory for qdma linux module
+# GTEST_TOP_DIR= Top directory for gtest regression test suite
+# MASTER_PF_PCI_BDF= BDF For master PF.
+
+MODULE_DIR=$1
+GTEST_DIR=$2
+qdma_mod=$MODULE_DIR/build/qdma.ko
+qdma_mod_vf=$MODULE_DIR/build/qdma_vf.ko
+gtest_bin=$GTEST_DIR/build/src/qdma_test
+#busdev="0000:05:00"
+master_pf=0000:$3
+busdev=${master_pf%.*}
+mod_param_arr=("mode=1" "mode=2" "mode=3")
+vfs_cnt=(64 60 60 68)
+qmax_vfs=1024
+
+RED='\033[0;31m'
+NC='\033[0m'
+
+prep_test_env () {
+ rmmod qdma
+ rmmod qdma_vf
+ sleep 2
+ insmod $qdma_mod $1
+ insmod $qdma_mod_vf
+ sleep 2
+ if [ ! -f /sys/bus/pci/devices/$master_pf/qdma/qmax_vfs ];then
+ echo -e "${RED} Aborting, no /sys/bus/pci/devices/$master_pf/qdma/qmax_vfs found.${NC}"
+ exit -4
+ fi
+ echo $qmax_vfs > /sys/bus/pci/devices/$master_pf/qdma/qmax_vfs
+ echo $3 > /sys/bus/pci/devices/$busdev.$2/sriov_numvfs
+ ret=$?
+ echo
+ echo "**********************************"
+ echo "******Doing QDMA VF Tests *******"
+ echo "**********************************"
+ echo
+ echo PF=$2, VF=$3, qmax_vfs=$qmax_vfs
+ echo module_param=$1
+ echo
+ dmactl dev list
+
+ return $ret
+}
+
+if [ $# -lt 3 ];then
+ echo -e "${RED}Aborting. Invalid usgae. Try $0 <MODULE_TOP_DIR> <GTEST_TOP_DIR> <MASTER_PF_PCI_BDF>. ${NC}"
+ exit -1
+fi
+
+if [ ! -f $qdma_mod ] || [ ! -f $qdma_mod_vf ];then
+ echo -e "${RED}Aborting. Missing qdma drivers at $MODULE_DIR${NC}"
+ exit -2
+fi
+
+if [ ! -f $gtest_bin ];then
+ echo -e "${RED}Aborting. Missing qdma_test at $GTEST_DIR${NC}"
+ exit -3
+fi
+
+for fn in `seq 0 3`;do
+ for params in "${mod_param_arr[@]}";do
+ prep_test_env "$params" $fn ${vfs_cnt[$fn]}
+ ret=$?
+ if [ $ret -ne 0 ];then
+ echo -e "${RED}FAILED VF tests, Aborting${NC}"
+ exit -1
+ fi
+ sleep 3
+ $gtest_bin --gtest_filter="*VF*"
+ done
+done
diff --git a/QDMA/linux-kernel/scripts/stress_test_top.sh b/QDMA/linux-kernel/scripts/stress_test_top.sh
new file mode 100755
index 0000000000000000000000000000000000000000..150bbc5895ac3f9eeeb72107b89102605d56bf7a
--- /dev/null
+++ b/QDMA/linux-kernel/scripts/stress_test_top.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ $# -lt 1 ]; then
+ echo "Invalid arguements."
+ echo "$0 <bbddf> [<test mask>] [<total execution time>]"
+ exit;
+fi;
+
+bdf=01000
+if [ ! -z $1 ]; then
+ bdf=$1
+fi
+test_mask=0x0f
+if [ ! -z $2 ]; then
+ test_mask=$2
+fi
+run_time="04:00:00"
+if [ ! -z $3 ]; then
+ run_time=$3
+fi
+
+cd ..
+make clean
+make
+make install-user
+
+insmod build/qdma.ko
+echo ${bdf}
+dmactl qdma${bdf} reg write bar 2 0xA0 0x01
+rmmod qdma
+cd -
+
+
+function cleanup_env()
+{
+ rmmod qdma_vf
+ rmmod qdma
+}
+
+function run_stress_test()
+{
+ drv_mode=0
+ if [ $1 == poll ]; then
+ drv_mode=1
+ elif [ $1 == intr ]; then
+ drv_mode=2
+ elif [ $1 == aggr ]; then
+ drv_mode=3
+ else
+ drv_mode=0
+ fi
+ cd ..
+ insmod build/qdma.ko mode=${drv_mode}
+ insmod build/qdma_vf.ko mode=${drv_mode}
+ cd -
+ chmod +x qdma_run_stress_test.sh
+ ./qdma_run_stress_test.sh $bdf 2048 $run_time 1 $1
+ dmactl qdma$bdf reg write bar 2 0xA0 0x01
+ cleanup_env
+}
+
+trap cleanup_env EXIT
+
+if (( test_mask & 0x02 )); then
+ run_stress_test poll
+fi
+if (( test_mask & 0x04 )); then
+ run_stress_test intr
+fi
+if (( test_mask & 0x08 )); then
+ run_stress_test aggr
+fi
+if (( test_mask & 0x01 )); then
+ run_stress_test auto
+fi
diff --git a/QDMA/linux-kernel/tools/Makefile b/QDMA/linux-kernel/tools/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..157cd163537d67d043befbe8cba2fb1ffbfb62dc
--- /dev/null
+++ b/QDMA/linux-kernel/tools/Makefile
@@ -0,0 +1,21 @@
+CC ?= gcc
+all: dma_to_device dma_from_device dma_from_device_w_udd dmautils
+
+dma_to_device: dma_to_device.o
+ $(CC) -lrt -o $@ $< -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D_LARGE_FILE_SOURCE
+dma_from_device: dma_from_device.o
+ $(CC) -lrt -o $@ $< -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D_LARGE_FILE_SOURCE
+dma_from_device_w_udd: dma_from_device_w_udd.o
+ $(CC) -lrt -o $@ $< -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D_LARGE_FILE_SOURCE
+
+aioutility: aioutility.o posixaio_engine.o libaio_engine.o
+ $(CC) -g -pthread -lrt -laio -o $@ $^ -D_GNU_SOURCE -D_LARGE_FILE_SOURCE -D_AIO_AIX_SOURCE
+
+dmautils: dmautils.o
+ $(CC) -pthread -lrt -o $@ $< -laio -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D_LARGE_FILE_SOURCE
+
+%.o: %.c
+ $(CC) -g -c -std=c99 -o $@ $< -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D_LARGE_FILE_SOURCE -D_AIO_AIX_SOURCE
+
+clean:
+ rm -rf *.o *.bin dma_to_device dma_from_device dma_from_device_w_udd dmautils
diff --git a/QDMA/linux-kernel/tools/aio_example.aio b/QDMA/linux-kernel/tools/aio_example.aio
new file mode 100644
index 0000000000000000000000000000000000000000..cbd4146c3e76030cb9cc5a6fcd2a8e1d494895ed
--- /dev/null
+++ b/QDMA/linux-kernel/tools/aio_example.aio
@@ -0,0 +1,178 @@
+; anything in [] is a Job. Global Job is for Global environment setup.
+[global]
+;setup user/dmactl qdma0 reg write bar 1 0x30 0x11110000
+;setup user/dmactl qdma0 reg write bar 1 0x34 0x33332222
+;setup user/dmactl qdma0 reg write bar 1 0x38 0x55554444
+;setup user/dmactl qdma0 reg write bar 1 0x3c 0x77776666
+;setup user/dmactl qdma0 reg write bar 1 0x40 0x99998888
+;setup user/dmactl qdma0 reg write bar 1 0x44 0xbbbbaaaa
+;setup user/dmactl qdma0 reg write bar 1 0x48 0xddddcccc
+;setup user/dmactl qdma0 reg write bar 1 0x4c 0xffffeeee
+
+; Global environment setup and cleanup commands
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 4
+setup user/dmactl qdma0 q start idx 0 dir c2h
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+
+[IO2]
+;select io engine to use
+ioengine=libaio
+; IO environment setup and cleanup commands
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 3
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x50 0x00
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x2
+setup user/dmactl qdma0 reg write bar 1 0x08 0x2
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+; file on which IO needs to be performed
+filename=/dev/qdma0-ST-C2H-0
+; Tyep of IO opeartion
+iotype=read
+;Number of bytes for IO operations
+numread=4096
+; Block size limit for each read
+block=4096
+;Number of iterations to do for this IO
+numio=2
+; verify using a reference file or pattern
+verify=file
+; file to use for verification
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_cmptsz_rsv]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 3
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x2
+setup user/dmactl qdma0 reg write bar 1 0x50 0x0
+setup user/dmactl qdma0 reg write bar 1 0x08 0x2
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=2
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_cmptsz_8]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 0
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x0
+setup user/dmactl qdma0 reg write bar 1 0x08 0x2
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_cmptsz_16]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 1
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x1
+setup user/dmactl qdma0 reg write bar 1 0x08 0x2
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_cmptsz_32]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 2
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x2
+setup user/dmactl qdma0 reg write bar 1 0x08 0x2
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_w_udd_cmptsz_8]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 0
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x0
+setup user/dmactl qdma0 reg write bar 1 0x08 0x6
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_w_udd_cmptsz_16]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 1
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x1
+setup user/dmactl qdma0 reg write bar 1 0x08 0x6
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
+[dma_from_dev_w_udd_cmptsz_32]
+ioengine=posixaio
+setup user/dmactl qdma0 q add idx 0 mode st dir c2h cmptsz 2
+setup user/dmactl qdma0 q start idx 0 dir c2h
+setup user/dmactl qdma0 reg write bar 1 0x0 0
+setup user/dmactl qdma0 reg write bar 1 0x4 4096
+setup user/dmactl qdma0 reg write bar 1 0x20 0x1
+setup user/dmactl qdma0 reg write bar 1 0x50 0x2
+setup user/dmactl qdma0 reg write bar 1 0x08 0x6
+clean user/dmactl qdma0 q stop idx 0 dir c2h
+clean user/dmactl qdma0 q del idx 0 dir c2h
+filename=/dev/qdma0-ST-C2H-0
+iotype=read
+numread=4096
+block=4096
+numio=1
+verify=file
+verify_file=datafile_16bit_pattern.bin
+
diff --git a/QDMA/linux-kernel/tools/aioutility.c b/QDMA/linux-kernel/tools/aioutility.c
new file mode 100644
index 0000000000000000000000000000000000000000..cad2db9f55fe9b36cd3f383e9944c72529ea03d2
--- /dev/null
+++ b/QDMA/linux-kernel/tools/aioutility.c
@@ -0,0 +1,950 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#define _XOPEN_SOURCE 500
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <signal.h>
+
+#include <sys/mman.h>
+#include <sys/time.h>
+
+#include "aioutility.h"
+
+
+#define DEFAULT_BLOCK_SIZE 4096
+#define SEC2NSEC 1000000000
+#define SEC2USEC 1000000
+#define DEFAULT_PAGE_SIZE 4096
+
+//#define AIO_DEBUG
+#define USE_LINUX_NATIVE_AIO 1
+
+enum env_cmd_type {
+ ENV_CMD_GLOBAL_SETUP,
+ ENV_CMD_GLOBAL_CLEANUP,
+ ENV_CMD_IO_SETUP,
+ ENV_CMD_IO_CLEANUP,
+};
+
+sem_t prev_io;
+
+static struct env_cmds *glb_setup_cmd_head = NULL;
+static struct env_cmds *glb_setup_cmd_tail = NULL;
+static struct env_cmds *glb_clean_cmd_head = NULL;
+static struct env_cmds *glb_clean_cmd_tail = NULL;
+static struct io_jobs *io_job_head = NULL;
+static struct io_jobs *io_job_tail = NULL;
+struct aio_block *aio_block_head = NULL;
+struct aio_block *aio_block_tail = NULL;
+struct aio_job *aio_task_head = NULL;
+static unsigned int num_io_jobs = 0;
+#define ENV_GLOBAL_CMD_HEAD(cmd_type) (ENV_CMD_GLOBAL_SETUP == cmd_type)? &glb_setup_cmd_head:&glb_clean_cmd_head
+#define ENV_GLOBAL_CMD_TAIL(cmd_type) (ENV_CMD_GLOBAL_SETUP == cmd_type)? &glb_setup_cmd_tail:&glb_clean_cmd_tail
+#define ENV_IO_CMD_HEAD(cmd_type) (ENV_CMD_IO_SETUP == cmd_type)? &(iojob->setup_head):&(iojob->cleanup_head)
+#define ENV_IO_CMD_TAIL(cmd_type) (ENV_CMD_IO_SETUP == cmd_type)? &(iojob->setup_tail):&(iojob->cleanup_tail)
+
+static struct option const long_opts[] = {
+ {"config", required_argument, NULL, 'c'},
+ {0, 0, 0, 0}
+};
+
+static void usage(const char *name)
+{
+ int i = 0;
+ fprintf(stdout, "%s\n\n", name);
+ fprintf(stdout, "usage: %s [OPTIONS]\n\n", name);
+
+ fprintf(stdout, " -%c (--%s) config file that has IO jobs to be executed\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+}
+
+static int timespec_check(struct timespec *t)
+{
+ if ((t->tv_nsec < 0) || (t->tv_nsec >= SEC2NSEC))
+ return -1;
+ return 0;
+
+}
+
+void timespec_sub(struct timespec *t1, struct timespec *t2)
+{
+ if (timespec_check(t1) < 0) {
+ fprintf(stderr, "invalid time #1: %lld.%.9ld.\n",
+ (long long)t1->tv_sec, t1->tv_nsec);
+ return;
+ }
+ if (timespec_check(t2) < 0) {
+ fprintf(stderr, "invalid time #2: %lld.%.9ld.\n",
+ (long long)t2->tv_sec, t2->tv_nsec);
+ return;
+ }
+ t1->tv_sec -= t2->tv_sec;
+ t1->tv_nsec -= t2->tv_nsec;
+ if (t1->tv_nsec >= 1000000000) {
+ t1->tv_sec++;
+ t1->tv_nsec -= 1000000000;
+ } else if (t1->tv_nsec < 0) {
+ t1->tv_sec--;
+ t1->tv_nsec += 1000000000;
+ }
+}
+
+static void enqueue_global_env_cmd(enum env_cmd_type cmd_type,
+ struct env_cmds *gcmd)
+{
+ struct env_cmds **gcmd_head = ENV_GLOBAL_CMD_HEAD(cmd_type);
+ struct env_cmds **gcmd_tail = ENV_GLOBAL_CMD_TAIL(cmd_type);
+
+ gcmd->prev = NULL;
+ if (NULL == *gcmd_head) {
+ *gcmd_head = gcmd;
+ (*gcmd_head)->next = NULL;
+ *gcmd_tail = *gcmd_head;
+ } else {
+ gcmd->next = *gcmd_head;
+ (*gcmd_head)->prev = gcmd;
+ *gcmd_head = gcmd;
+ }
+}
+
+static struct env_cmds * dequeue_global_env_cmd(enum env_cmd_type cmd_type)
+{
+ struct env_cmds **gcmd_head = ENV_GLOBAL_CMD_HEAD(cmd_type);
+ struct env_cmds **gcmd_tail = ENV_GLOBAL_CMD_TAIL(cmd_type);
+ struct env_cmds *gcmd = *gcmd_head;
+
+ if (*gcmd_tail == *gcmd_head) {
+ *gcmd_tail = NULL;
+ *gcmd_head = NULL;
+ } else {
+ *gcmd_head = (*gcmd_head)->next;
+ (*gcmd_head)->prev = NULL;
+ }
+
+ return gcmd;
+}
+
+static void enqueue_io_env_cmd(enum env_cmd_type cmd_type,
+ struct io_jobs *iojob,
+ struct env_cmds *gcmd)
+{
+ struct env_cmds **gcmd_head = ENV_IO_CMD_HEAD(cmd_type);
+ struct env_cmds **gcmd_tail = ENV_IO_CMD_TAIL(cmd_type);
+
+ gcmd->prev = NULL;
+ if (NULL == *gcmd_head) {
+ *gcmd_head = gcmd;
+ (*gcmd_head)->next = NULL;
+ *gcmd_tail = *gcmd_head;
+ } else {
+ gcmd->next = *gcmd_head;
+ (*gcmd_head)->prev = gcmd;
+ *gcmd_head = gcmd;
+ }
+}
+
+static struct env_cmds * dequeue_io_env_cmd(enum env_cmd_type cmd_type,
+ struct io_jobs *iojob)
+{
+ struct env_cmds **gcmd_head = ENV_IO_CMD_HEAD(cmd_type);
+ struct env_cmds **gcmd_tail = ENV_IO_CMD_TAIL(cmd_type);
+ struct env_cmds *gcmd = *gcmd_head;
+
+ if (*gcmd_tail == *gcmd_head) {
+ *gcmd_tail = NULL;
+ *gcmd_head = NULL;
+ } else {
+ *gcmd_head = (*gcmd_head)->next;
+ (*gcmd_head)->prev = NULL;
+ }
+
+ return gcmd;
+}
+
+void enqueue_io_job(struct io_jobs *iojob)
+{
+ iojob->prev = NULL;
+ if (NULL == io_job_head) {
+ io_job_head = iojob;
+ io_job_head->next = NULL;
+ io_job_tail = io_job_head;
+ } else {
+ iojob->next = io_job_head;
+ io_job_head->prev = iojob;
+ io_job_head = iojob;
+ }
+ num_io_jobs++;
+}
+
+static struct io_jobs * dequeue_io_job(void)
+{
+ struct io_jobs *iojob = io_job_head;
+
+ if (io_job_tail == io_job_head) {
+ io_job_tail = NULL;
+ io_job_head = NULL;
+ } else {
+ io_job_head = io_job_head->next;
+ io_job_head->prev = NULL;
+ }
+
+ return iojob;
+}
+
+void enqueue_aio_block(struct aio_job *aiojob, struct aio_block *aioblock)
+{
+ if (NULL == aiojob->aio_block_head) {
+ aiojob->aio_block_head = aioblock;
+ aiojob->aio_block_head->next = NULL;
+ aiojob->aio_block_tail = aiojob->aio_block_head;
+ } else {
+ aioblock->next = aiojob->aio_block_head;
+ aiojob->aio_block_head = aioblock;
+ }
+}
+
+static struct aio_block * dequeue_aio_block(struct aio_job *aiojob)
+{
+ struct aio_block *aioblock = aiojob->aio_block_head;
+
+ if (aiojob->aio_block_tail == aiojob->aio_block_head) {
+ aiojob->aio_block_head = NULL;
+ aiojob->aio_block_tail = NULL;
+ } else
+ aiojob->aio_block_head = aiojob->aio_block_head->next;
+
+ return aioblock;
+}
+
+void enqueue_aio_task(struct aio_job *aiotask)
+{
+ if (NULL == aio_task_head) {
+ aio_task_head = aiotask;
+ aio_task_head->next = NULL;
+ } else {
+ aiotask->next = aio_task_head;
+ aio_task_head = aiotask;
+ }
+}
+
+struct aio_block * get_aio_block(void *_aioblock)
+{
+ struct aio_job *aiotask = aio_task_head;
+ struct aio_block *aioblock;
+
+ while (aiotask) {
+ aioblock = aiotask->aio_block_head;
+ while (aioblock) {
+ if (aioblock->aio_entry == _aioblock)
+ break;
+ aioblock = aioblock->next;
+ }
+ aiotask = aiotask->next;
+ }
+
+ return aioblock;
+}
+
+
+static char * strip_blanks(char *word, long unsigned int *banlks)
+{
+ char *p = word;
+ unsigned int i = 0;
+
+ while (isblank(p[0])) {
+ p++;
+ i++;
+ }
+ *banlks = i;
+
+ return p;
+}
+
+static bool get_io_param_attrs(char *linebuf, size_t linelen,
+ unsigned int *lhspos, unsigned int *rhspos)
+{
+ long unsigned int numblanks = 0;
+ char *p = strip_blanks(linebuf, &numblanks);
+ size_t i = 0;
+
+ while (i < (linelen - numblanks)) {
+ if ('=' != p[i])
+ i++;
+ else
+ break;
+ }
+ if (i == (linelen - numblanks)) {
+ printf("Error: assignment operator missing in below line:\n %s\n",
+ linebuf);
+ return false;
+ } else {
+ *lhspos = numblanks;
+ p = strip_blanks(p + i + 1,&numblanks);
+ *rhspos = i + 1 + numblanks;
+ }
+
+ return true;
+}
+
+static void add_env_cmd(enum env_cmd_type cmd_type,
+ char *linebuf, size_t linelen, void *iojob)
+{
+ struct env_cmds *gcmd = malloc(sizeof(struct env_cmds));
+ size_t numblanks;
+ char *sh_cmd = strip_blanks(linebuf, &numblanks);
+ unsigned int sh_cmd_len = linelen - numblanks;
+
+ gcmd->cmd = malloc(sh_cmd_len);
+ memcpy(gcmd->cmd, sh_cmd, sh_cmd_len);
+
+
+ if ((ENV_CMD_GLOBAL_SETUP == cmd_type) ||
+ (ENV_CMD_GLOBAL_CLEANUP == cmd_type))
+ enqueue_global_env_cmd(cmd_type, gcmd);
+ else
+ enqueue_io_env_cmd(cmd_type, (struct io_jobs *)iojob, gcmd);
+}
+
+static bool parse_global_cmd_info(FILE *fp,
+ unsigned int *linenum)
+{
+ char *linebuf = NULL;
+ char *realbuf;
+ size_t linelen = 0;
+ size_t numread;
+ size_t numblanks;
+ fpos_t prev_pos;
+
+ fgetpos(fp, &prev_pos);
+ while ((numread = getline(&linebuf, &linelen, fp)) != -1) {
+ numread--;
+ *linenum = *linenum + 1;
+ realbuf = strip_blanks(linebuf, &numblanks);
+ numread -= numblanks;
+ if (0 == numread)
+ continue;
+ else if ((realbuf[0] == ';') || (realbuf[0] == '#'))
+ continue;
+ else if (realbuf[0] == '[') { /* new job - return */
+ fsetpos(fp, &prev_pos);
+ break;
+ } else {
+ if (0 == strncmp(&realbuf[0], "setup", 5)) { /* enqueue env setup job */
+ add_env_cmd(ENV_CMD_GLOBAL_SETUP,
+ &realbuf[5],
+ numread - 5,
+ NULL);
+ } else if (0 == strncmp(&realbuf[0], "clean", 5)) { /* enqueue env cleanup job */
+ add_env_cmd(ENV_CMD_GLOBAL_CLEANUP,
+ &realbuf[5],
+ numread - 5,
+ NULL);
+ } else {
+ printf ("Error: unknown global command\n");
+ return false;
+ }
+ }
+ fgetpos(fp, &prev_pos);
+ }
+
+ return true;
+}
+
+static bool parse_io_job_attrs(struct io_jobs *iojob, char *linebuf, size_t numread)
+{
+ unsigned int lhspos = 0, rhspos = 0;
+ char *lhs, *rhs;
+
+ if (0 == strncmp(linebuf, "setup", 5)) { /* enqueue env setup job */
+ add_env_cmd(ENV_CMD_IO_SETUP,
+ &linebuf[5],
+ numread - 5,
+ iojob);
+ } else if (0 == strncmp(linebuf, "clean", 5)) { /* enqueue env cleanup job */
+ add_env_cmd(ENV_CMD_IO_CLEANUP,
+ &linebuf[5],
+ numread - 5,
+ iojob);
+ }else if (get_io_param_attrs(linebuf, numread, &lhspos, &rhspos)) {
+ lhs = linebuf + lhspos;
+ rhs = linebuf + rhspos;
+ if (0 == strncmp(lhs, "ioengine", 8)) {
+ if (0 == strncmp(rhs, "libaio", 6)) {
+ iojob->io_type = IO_ENGINE_LIBAIO;
+ iojob->commit_io = submit_naio_job;
+ iojob->cancel_io = cancel_naio_job;
+ }
+ else if (0 == strncmp(rhs, "posixaio", 8)) {
+ iojob->io_type = IO_ENGINE_POSIXAIO;
+ iojob->commit_io = submit_paio_job;
+ iojob->cancel_io = cancel_paio_job;
+ }
+ else {
+ printf ("Error: Invalid IO engine specified for %s job\n",
+ iojob->job_name);
+ return false;
+ }
+ } else if (0 == strncmp(lhs, "filename", 8)) {
+ iojob->node = malloc(numread - rhspos);
+ memcpy(iojob->node, rhs, numread - rhspos);
+ } else if (0 == strncmp(lhs, "iotype", 6)) {
+ iojob->iotype = malloc(numread - rhspos);
+ memcpy(iojob->iotype, rhs, numread - rhspos);
+ } else if ((0 == strncmp(lhs, "numwrite", 8)) ||
+ (0 == strncmp(lhs, "numread", 7))) {
+ sscanf(rhs, "%u", &iojob->numbytes);
+ } else if (0 == strncmp(lhs, "infile", 6)) {
+ int fbuf;
+ size_t datsize = 0;
+ struct stat infile_stat;
+ ssize_t numrd = 0;
+ char *wrfilename = malloc(numread - rhspos);
+
+ memcpy(wrfilename, rhs, numread - rhspos);
+
+ fbuf = open(wrfilename, O_RDONLY);
+
+ if (0 > fbuf) {
+ printf("Error: opening %s failed\n", wrfilename);
+ return false;
+ } else {
+ if (0 > fstat(fbuf, &infile_stat)) {
+ printf("Error: File stat did not work for %s\n", wrfilename);
+ return false;
+ }
+ datsize = infile_stat.st_size;
+ if (0 > lseek(fbuf, 0, SEEK_SET)) {
+ printf("Error: Setting file descriptor to beginning for %s failed\n",
+ wrfilename);
+ return false;
+ }
+ iojob->buf = malloc(datsize);
+
+ while (datsize) {
+ numrd = read(fbuf, iojob->buf, datsize);
+ if (0 < numrd) {
+ datsize -= numrd;
+ } else
+ break;
+ }
+ close(fbuf);
+ }
+ free(wrfilename);
+ } else if (0 == strncmp(lhs, "block", 5)) {
+ sscanf(rhs, "%u", &iojob->block_size);
+ } else if (0 == strncmp(lhs, "numio", 5)) {
+ sscanf(rhs, "%u", &iojob->numitr);
+ } else if (0 == strncmp(lhs, "verify_file", 11)) {
+ if (iojob->verify_read != READ_VERIFY_FILE) {
+ printf("Error: verify method not matching with input for %s\n",
+ iojob->job_name);
+ return false;
+ }
+ iojob->verify_file_name = malloc(numread - rhspos);
+ memcpy(iojob->verify_file_name, rhs, numread - rhspos);
+ } else if (0 == strncmp(lhs, "verify_pattern", 14)) {
+ if (iojob->verify_read != READ_VERIFY_PATTERN) {
+ printf("Error: verify method not matching with input for %s\n",
+ iojob->job_name);
+ return false;
+ }
+ iojob->verify_pattern = malloc(numread - rhspos);
+ memcpy(iojob->verify_pattern, rhs, numread - rhspos);
+ } else if (0 == strncmp(lhs, "verify", 6)) {
+ if (0 == strncmp(rhs, "file", 4)) {
+ iojob->verify_read = READ_VERIFY_FILE;
+ } else if (0 == strncmp(rhs, "pattern", 7)) {
+ iojob->verify_read = READ_VERIFY_PATTERN;
+ } else {
+ printf("Error: unknown read verify type in %s\n",
+ linebuf);
+ return false;
+ }
+ } else {
+ printf("Error: Unknown IO option\n");
+ }
+ } else
+ return false;
+
+ return true;
+}
+
+static bool parse_io_job_info(struct io_jobs *iojob, FILE *fp,
+ unsigned int *linenum)
+{
+ char *linebuf = NULL;
+ char *realbuf;
+ size_t linelen = 0;
+ size_t numread;
+ size_t numblanks;
+ fpos_t prev_pos;
+
+ fgetpos(fp, &prev_pos);
+ while ((numread = getline(&linebuf, &linelen, fp)) != -1) {
+ numread--;
+ (*linenum)++;
+ realbuf = strip_blanks(linebuf, &numblanks);
+ numread -= numblanks;
+ if (0 == numread)
+ continue;
+ else if ((realbuf[0] == ';') || (realbuf[0] == '#'))
+ continue;
+ else if (realbuf[0] == '[') { /* new job - return */
+ fsetpos(fp, &prev_pos);
+ break;
+ } else {
+ if (false == parse_io_job_attrs(iojob, linebuf, numread))
+ return false;
+ }
+ fgetpos(fp, &prev_pos);
+ }
+
+ return true;
+}
+
+static void parse_config_file(const char *cfg_fname)
+{
+ char *linebuf = NULL;
+ char *realbuf;
+ FILE *fp;
+ size_t linelen = 0;
+ size_t numread;
+ size_t numblanks;
+ unsigned int linenum = 0;
+
+ fp = fopen(cfg_fname, "r");
+ if (fp == NULL)
+ exit(EXIT_FAILURE);
+
+ while ((numread = getline(&linebuf, &linelen, fp)) != -1) {
+ numread--;
+ linenum++;
+ realbuf = strip_blanks(linebuf, &numblanks);
+ linelen -= numblanks;
+ if (0 == linelen)
+ continue;
+ else if ((';' == realbuf[0]) || ('#' == realbuf[0]))
+ continue;
+ else if (realbuf[0] == '[') { /* extract job name */
+ unsigned char job_eidx = 1;
+ char *jobname;
+
+ while (']' != realbuf[job_eidx])
+ job_eidx++;
+ if (0 == strncmp(&realbuf[1], "global", job_eidx - 1)) {
+ if (!parse_global_cmd_info(fp, &linenum)) {
+ fclose(fp);
+ exit(1);
+ }
+ } else { /* enqueue io job */
+ struct io_jobs *iojob = calloc(1, sizeof(struct io_jobs));
+
+ memset(iojob, 0 ,sizeof(struct io_jobs));
+ iojob->job_name = malloc(job_eidx - 1);
+ iojob->verify_read = READ_VERIFY_NONE;
+ memcpy(iojob->job_name, &realbuf[1], job_eidx - 1);
+ iojob->block_size = DEFAULT_BLOCK_SIZE;
+ if (!parse_io_job_info(iojob, fp, &linenum)) {
+ free(iojob->job_name);
+ free(iojob);
+ fclose(fp);
+ exit(1);
+ }
+ if (0 == strncmp(iojob->iotype, "read", 4)) {
+ if (0 == iojob->numbytes) {
+ printf ("Error:Number of bytes to read not provided for %s\n",
+ iojob->job_name);
+ exit(0);
+ } else {
+ unsigned int num_blocks = get_io_job_num_blocks(iojob);
+
+ if (0 != posix_memalign((void **)&iojob->buf,
+ DEFAULT_PAGE_SIZE /*alignment */ ,
+ (iojob->numitr *
+ num_blocks *
+ iojob->block_size) +
+ DEFAULT_PAGE_SIZE)) {
+ printf("Error: Memory could not be allocated\n");
+ exit(0);
+ }
+ }
+ }
+ enqueue_io_job(iojob);
+ }
+ }
+ }
+ fclose(fp);
+}
+
+void run_env_cmds(struct env_cmds *gcmd)
+{
+ while (NULL != gcmd) {
+ printf("Running cmd: %s\n", gcmd->cmd);
+ system(gcmd->cmd);
+ gcmd = gcmd->prev;
+ }
+}
+
+void verify_read(struct io_jobs *iojob)
+{
+ char *expected;
+ int rd_diff;
+ unsigned int i, j;
+ unsigned int total_jobs = get_io_job_num_blocks(iojob);
+
+ if (iojob->verify_read == READ_VERIFY_FILE) {
+ int fd;
+ unsigned int datsize;
+ struct stat infile_stat;
+ ssize_t numrd = 0;
+ int rc;
+
+ fd = open(iojob->verify_file_name, O_RDONLY);
+ if (0 > fstat(fd, &infile_stat)) {
+ printf("Error: File stat did not work for %s\n", iojob->verify_file_name);
+ return;
+ }
+ datsize = infile_stat.st_size;
+ if (0 > lseek(fd, 0, SEEK_SET)) {
+ printf("Error: Setting file descriptor to beginning for %s failed\n",
+ iojob->verify_file_name);
+ return;
+ }
+ expected = malloc(datsize);
+ while (datsize) {
+ numrd = read(fd, expected, datsize);
+ if (0 < numrd) {
+ datsize -= numrd;
+ } else
+ break;
+ }
+ close(fd);
+
+ for (i = 0; i < iojob->numitr; i++) {
+ unsigned int buf_offset = (i * total_jobs * iojob->block_size);
+ unsigned int io_numbytes = iojob->numbytes;
+
+ rd_diff = memcmp(expected,
+ &iojob->buf[buf_offset],
+ io_numbytes);
+ if (0 != rd_diff) {
+
+ printf("Error: read verify failed for %s itr %d at %d\n",
+ iojob->job_name, i, rd_diff);
+
+ } else
+ printf("Read validated for %s -> itr %u successfully\n", iojob->job_name, i);
+ }
+ free(expected);
+ } else if (iojob->verify_read == READ_VERIFY_PATTERN) {
+ unsigned int pat_len = strlen(iojob->verify_pattern);
+ unsigned int offset = 0;
+
+ unsigned int buf_offset = (i * iojob->block_size);
+ while (offset < iojob->numbytes) {
+ rd_diff = memcmp(iojob->verify_pattern,
+ &iojob->buf[offset],
+ pat_len);
+ if (0 != rd_diff)
+ printf("Error: read verify failed for %s at %d\n",
+ iojob->job_name, offset + rd_diff);
+ offset += pat_len;
+ }
+ }
+}
+
+unsigned int get_io_job_num_blocks(struct io_jobs *iojob)
+{
+ unsigned int num_blks;
+ unsigned int extra_block = (iojob->numbytes % iojob->block_size)? 1 : 0;
+
+ num_blks = (iojob->numbytes / iojob->block_size) + extra_block;
+
+ return num_blks;
+}
+
+
+#ifdef AIO_DEBUG
+static void dump_iojob(struct io_jobs *iojob)
+{
+ struct env_cmds *gcmd;
+
+ printf("%s:\n", iojob->job_name);
+ gcmd = iojob->setup_tail;
+ while (NULL != gcmd) {
+ printf("%s\n", gcmd->cmd);
+ gcmd = gcmd->prev;
+ }
+ if (NULL == iojob->setup_tail)
+ printf("No IO setup cmds listed\n");
+ if (NULL == iojob->cleanup_tail)
+ printf("No IO cleanup cmds listed\n");
+ gcmd = iojob->cleanup_tail;
+ while (NULL != gcmd) {
+ printf("%s\n", gcmd->cmd);
+ gcmd = gcmd->prev;
+ }
+ printf("working on %s:\n", iojob->node);
+ printf("IO Operation: %s\n", iojob->iotype);
+ printf("verification type: %d\n", iojob->verify_read);
+ printf("verify_file_name: %s\n", iojob->verify_file_name);
+ printf("verify_pattern: %s\n", iojob->verify_pattern);
+ printf("numbytes: %d\n", iojob->numbytes);
+ printf("block_size: %d\n", iojob->block_size);
+ printf("numitr: %d\n", iojob->numitr);
+}
+#endif
+
+static void run_io_job(unsigned int jobnum)
+{
+ struct io_jobs *iojob = io_job_tail;
+
+ if (jobnum > num_io_jobs) {
+ printf("Error: Invalid IO job number %d\n", jobnum);
+ return;
+ }
+
+ if (0 == jobnum) {
+ while (iojob) {
+#ifdef AIO_DEBUG
+ dump_iojob(iojob);
+#endif
+ iojob->commit_io(iojob);
+ iojob = iojob->prev;
+ }
+ } else {
+ while (--jobnum) {
+ iojob = iojob->prev;
+ }
+
+#ifdef AIO_DEBUG
+ dump_iojob(iojob);
+#endif
+ iojob->commit_io(iojob);
+ }
+}
+
+static void cancel_outstanding_reqs(void)
+{
+ struct aio_block *aioblock;
+ struct aio_job *aiojob = aio_task_head;
+ struct aiocb *iocb;
+ int rc;
+
+ while (NULL != aiojob) {
+ aiojob->iojob->cancel_io(aiojob);
+ aiojob = aiojob->next;
+ }
+}
+
+bool cleanup_aiojob(struct aio_job *aiojob, bool force)
+{
+ struct aio_block *aioblock;
+ void *aio_list;
+ struct timespec ts_cur;
+ int rc;
+ struct io_jobs *iojob = aiojob->iojob;
+
+ aioblock = aiojob->aio_block_head;
+ while (force) {
+ if (NULL != aioblock) {
+ if (aiojob->mask_completed & aioblock->mask)
+ goto aioblock_next;
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_cur);
+ ts_cur.tv_nsec += SEC2USEC;
+ printf("waiting for %s iter %d block %d to cancel\n",
+ iojob->job_name, aioblock->jobitr,
+ aioblock->block_num);
+ sem_timedwait(&aioblock->lock, &ts_cur);
+ sem_destroy(&aioblock->lock);
+ aiojob->mask_completed |= aioblock->mask;
+ } else
+ break;
+aioblock_next:
+ aioblock = aioblock->next;
+ }
+ if (aiojob->mask_completed != aiojob->mask_expected) {
+ if (force)
+ printf("Invalid path in aio task cleanup\n");
+ return false;
+ }
+ run_env_cmds(iojob->cleanup_tail);
+ while (1) {
+ aioblock = dequeue_aio_block(aiojob);
+ if (NULL == aioblock) break;
+ if (NULL != aioblock->aio_entry) {
+ free(aioblock->aio_entry);
+ aioblock->aio_entry = NULL;
+ }
+ free(aioblock);
+ aioblock = NULL;
+ }
+ close(aiojob->fd);
+ aio_list = aiojob->aio_list;
+ aio_task_head = aio_task_head->next;
+ aiojob->aio_list = NULL;
+ free(aiojob);
+ aiojob = NULL;
+ free(aio_list);
+
+ sem_post(&prev_io);
+
+ return true;
+}
+
+void cleanup_aio_jobs(bool force)
+{
+ struct aio_job *aiojob = aio_task_head;
+
+ while (NULL != aiojob) {
+ if (false == cleanup_aiojob(aiojob, force))
+ break; /* need to clean up sequentially in reverse */
+ else
+ aiojob = aio_task_head;
+ }
+}
+
+static void cleanup(void)
+{
+ struct io_jobs *iojob;
+ struct env_cmds *gcmd;
+
+ cancel_outstanding_reqs();
+ cleanup_aio_jobs(true);
+
+ while (1) {
+ iojob = dequeue_io_job();
+ if (NULL == iojob) break;
+ if (NULL != iojob->verify_file_name)
+ free(iojob->verify_file_name);
+ if (NULL != iojob->verify_pattern)
+ free(iojob->verify_pattern);
+ free(iojob->iotype);
+ free(iojob->buf);
+ free(iojob->node);
+ free(iojob->job_name);
+ while (iojob->cleanup_head) {
+ gcmd = dequeue_io_env_cmd(ENV_CMD_IO_CLEANUP, iojob);
+ free(gcmd->cmd);
+ free(gcmd);
+ }
+ while (iojob->setup_head) {
+ gcmd = dequeue_io_env_cmd(ENV_CMD_IO_SETUP, iojob);
+ free(gcmd->cmd);
+ free(gcmd);
+ }
+ free(iojob);
+ }
+ while (1) {
+ gcmd = dequeue_global_env_cmd(ENV_CMD_GLOBAL_CLEANUP);
+ if (NULL == gcmd) break;
+ free(gcmd->cmd);
+ free(gcmd);
+ }
+ while (1) {
+ gcmd = dequeue_global_env_cmd(ENV_CMD_GLOBAL_SETUP);
+ if (NULL == gcmd) break;
+ free(gcmd->cmd);
+ free(gcmd);
+ }
+}
+
+int main(int argc, char *argv[])
+{
+ int cmd_opt;
+ char *cfg_fname;
+ unsigned int i;
+ struct io_jobs *iojob;
+
+ while ((cmd_opt = getopt_long(argc, argv, "vhxc:c:", long_opts,
+ NULL)) != -1) {
+ switch (cmd_opt) {
+ case 0:
+ /* long option */
+ break;
+ case 'c':
+ /* condig file name */
+ cfg_fname = strdup(optarg);
+ break;
+ default:
+ usage(argv[0]);
+ exit(0);
+ break;
+ }
+ }
+
+ parse_config_file(cfg_fname);
+ sem_init(&prev_io, 0, 1);
+ atexit(cleanup);
+#ifdef AIO_DEBUG
+ {
+ struct env_cmds *gcmd;
+
+ gcmd = glb_setup_cmd_tail;
+ while (NULL != gcmd) {
+ printf("%s\n", gcmd->cmd);
+ gcmd = gcmd->prev;
+ }
+ gcmd = glb_clean_cmd_tail;
+ while (NULL != gcmd) {
+ printf("%s\n", gcmd->cmd);
+ gcmd = gcmd->prev;
+ }
+ }
+#endif
+
+ while (1) {
+ unsigned int j = 0;
+
+ printf("\nSelect the Job to run:\n");
+ printf("0. exit\n");
+ printf("1. global setup\n");
+ printf("2. global cleanup\n");
+ printf("3. All IO jobs\n");
+ i = 4;
+ iojob = io_job_tail;
+ while (iojob) {
+ printf("%d. %s\n", i, iojob->job_name);
+ iojob = iojob->prev;
+ i++;
+ }
+ scanf("%u", &j);
+ sem_wait(&prev_io);
+ if (0 == j) {
+ exit(0);
+ } else if (1 == j) {
+ run_env_cmds(glb_setup_cmd_tail);
+ sem_post(&prev_io);
+ } else if (2 == j) {
+ run_env_cmds(glb_clean_cmd_tail);
+ sem_post(&prev_io);
+ } else if (3 == j) {
+ run_io_job(0);
+ } else if (j < i) {
+ printf("Running IO Job %d\n", j-3);
+ run_io_job(j - 3);
+ } else {
+ printf ("Error: Invalid option\n");
+ }
+ sem_wait(&prev_io);
+ exit(0);
+ }
+
+ return 0;
+}
+
diff --git a/QDMA/linux-kernel/tools/aioutility.h b/QDMA/linux-kernel/tools/aioutility.h
new file mode 100644
index 0000000000000000000000000000000000000000..f9f674b57e274af63bac3cda5d1f644541255346
--- /dev/null
+++ b/QDMA/linux-kernel/tools/aioutility.h
@@ -0,0 +1,110 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#ifndef __AIOUTILITY_H__
+#define __AIOUTILITY_H__
+
+#include <semaphore.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <linux/types.h>
+
+enum read_verify_type {
+ READ_VERIFY_NONE,
+ READ_VERIFY_FILE,
+ READ_VERIFY_PATTERN
+};
+
+enum io_engine {
+ IO_ENGINE_LIBAIO,
+ IO_ENGINE_POSIXAIO
+};
+
+struct env_cmds {
+ char *cmd; /* commands to be run */
+ struct env_cmds *next;
+ struct env_cmds *prev;
+};
+
+typedef void (*submit_aio_job)(void *);
+typedef void (*cancel_aio_job)(void *);
+
+struct io_jobs {
+ char *job_name;
+ char *node;
+ char *iotype;
+ char *buf;
+ char *verify_file_name;
+ char *verify_pattern;
+ unsigned int numbytes;
+ unsigned int block_size;
+ unsigned int numitr;
+ enum read_verify_type verify_read;
+ enum io_engine io_type;
+ void *aiojob;
+ struct env_cmds *setup_head;
+ struct env_cmds *setup_tail;
+ struct env_cmds *cleanup_head;
+ struct env_cmds *cleanup_tail;
+ submit_aio_job commit_io;
+ cancel_aio_job cancel_io;
+ struct io_jobs *next;
+ struct io_jobs *prev;
+};
+
+struct aio_block {
+ sem_t lock;
+ __u64 mask;
+ void *aio_entry;
+ unsigned int jobitr;
+ unsigned int num_bytes;
+ unsigned int block_num;
+ unsigned int num_blocks_completed;
+ struct aio_job *parent;
+ void *next;
+};
+
+struct aio_job {
+ struct io_jobs *iojob;
+ int fd;
+ __u64 mask_expected;
+ __u64 mask_completed;
+ struct timespec ts_start;
+ void *aio_list;
+ void *naio_ctxt;
+ void *events;
+ struct aio_block *aio_block_head;
+ struct aio_block *aio_block_tail;
+ struct aio_job *next;
+};
+
+void submit_naio_job(void *iojob);
+void submit_paio_job(void *iojob);
+void cancel_naio_job(void *aiojob);
+void cancel_paio_job(void *aiojob);
+
+unsigned int get_io_job_num_blocks(struct io_jobs *iojob);
+void verify_read(struct io_jobs *iojob);
+bool cleanup_aiojob(struct aio_job *aiojob, bool force);
+void cleanup_aio_jobs(bool);
+void enqueue_io_job(struct io_jobs *iojob);
+void enqueue_aio_block(struct aio_job *aiojob, struct aio_block *aioblock);
+void enqueue_aio_task(struct aio_job *aiotask);
+struct aio_block * get_aio_block(void *_aioblock);
+
+void run_env_cmds(struct env_cmds *gcmd);
+
+void timespec_sub(struct timespec *t1, struct timespec *t2);
+
+#endif
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/mm-bi.zip b/QDMA/linux-kernel/tools/config/dmautils_config/mm-bi.zip
new file mode 100644
index 0000000000000000000000000000000000000000..548e1c8804b13ea7d67fb91e0fb3694e856f291e
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/mm-bi.zip differ
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/mm-c2h.zip b/QDMA/linux-kernel/tools/config/dmautils_config/mm-c2h.zip
new file mode 100644
index 0000000000000000000000000000000000000000..64f01c3f65065b11324fcbd04798e243c226efba
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/mm-c2h.zip differ
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/mm-h2c.zip b/QDMA/linux-kernel/tools/config/dmautils_config/mm-h2c.zip
new file mode 100644
index 0000000000000000000000000000000000000000..2ae62697b21ecaa3af4f58b59de729d20471f8d1
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/mm-h2c.zip differ
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/st-bi.zip b/QDMA/linux-kernel/tools/config/dmautils_config/st-bi.zip
new file mode 100644
index 0000000000000000000000000000000000000000..d7463397e04e20c4751af0b759ccf73336ee52e6
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/st-bi.zip differ
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/st-c2h-pfetch1.zip b/QDMA/linux-kernel/tools/config/dmautils_config/st-c2h-pfetch1.zip
new file mode 100644
index 0000000000000000000000000000000000000000..47a9e2bf22af5d74892799394bf2a083759ce543
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/st-c2h-pfetch1.zip differ
diff --git a/QDMA/linux-kernel/tools/config/dmautils_config/st-h2c.zip b/QDMA/linux-kernel/tools/config/dmautils_config/st-h2c.zip
new file mode 100644
index 0000000000000000000000000000000000000000..bd5d571360ae445644cae508235359bb653dc278
Binary files /dev/null and b/QDMA/linux-kernel/tools/config/dmautils_config/st-h2c.zip differ
diff --git a/QDMA/linux-kernel/tools/dma_from_device.c b/QDMA/linux-kernel/tools/dma_from_device.c
new file mode 100644
index 0000000000000000000000000000000000000000..23930689b5e9ebd72af5546efd6d216bc58a5ad3
--- /dev/null
+++ b/QDMA/linux-kernel/tools/dma_from_device.c
@@ -0,0 +1,233 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#define _DEFAULT_SOURCE
+#define _XOPEN_SOURCE 500
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "dma_xfer_utils.c"
+
+#define DEVICE_NAME_DEFAULT "/dev/qdma01000-MM-0"
+#define SIZE_DEFAULT (32)
+#define COUNT_DEFAULT (1)
+
+static struct option const long_opts[] = {
+ {"device", required_argument, NULL, 'd'},
+ {"address", required_argument, NULL, 'a'},
+ {"size", required_argument, NULL, 's'},
+ {"offset", required_argument, NULL, 'o'},
+ {"count", required_argument, NULL, 'c'},
+ {"file", required_argument, NULL, 'f'},
+ {"help", no_argument, NULL, 'h'},
+ {"verbose", no_argument, NULL, 'v'},
+ {0, 0, 0, 0}
+};
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *ofname);
+static int no_write = 0;
+
+static void usage(const char *name)
+{
+ int i = 0;
+ fprintf(stdout, "%s\n\n", name);
+ fprintf(stdout, "usage: %s [OPTIONS]\n\n", name);
+ fprintf(stdout, "Read via SGDMA, optionally save output to a file\n\n");
+
+ fprintf(stdout, " -%c (--%s) device (defaults to %s)\n",
+ long_opts[i].val, long_opts[i].name, DEVICE_NAME_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) the start address on the AXI bus\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) size of a single transfer in bytes, default %d.\n",
+ long_opts[i].val, long_opts[i].name, SIZE_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) page offset of transfer\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) number of transfers, default is %d.\n",
+ long_opts[i].val, long_opts[i].name, COUNT_DEFAULT);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) file to write the data of the transfers\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) print usage help and exit\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) verbose output\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+}
+
+int main(int argc, char *argv[])
+{
+ int cmd_opt;
+ char *device = DEVICE_NAME_DEFAULT;
+ uint64_t address = 0;
+ uint64_t size = SIZE_DEFAULT;
+ uint64_t offset = 0;
+ uint64_t count = COUNT_DEFAULT;
+ char *ofname = NULL;
+
+ while ((cmd_opt = getopt_long(argc, argv, "vhxc:f:d:a:s:o:", long_opts,
+ NULL)) != -1) {
+ switch (cmd_opt) {
+ case 0:
+ /* long option */
+ break;
+ case 'd':
+ /* device node name */
+ device = strdup(optarg);
+ break;
+ case 'a':
+ /* RAM address on the AXI bus in bytes */
+ address = getopt_integer(optarg);
+ break;
+ /* RAM size in bytes */
+ case 's':
+ size = getopt_integer(optarg);
+ break;
+ case 'o':
+ offset = getopt_integer(optarg) & 4095;
+ break;
+ /* count */
+ case 'c':
+ count = getopt_integer(optarg);
+ break;
+ /* count */
+ case 'f':
+ ofname = strdup(optarg);
+ break;
+ /* print usage help and exit */
+ case 'x':
+ no_write++;
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ exit(0);
+ break;
+ }
+ }
+ if (verbose)
+ fprintf(stdout,
+ "dev %s, addr 0x%lx, size 0x%lx, offset 0x%lx, count %lu\n",
+ device, address, size, offset, count);
+
+ return test_dma(device, address, size, offset, count, ofname);
+}
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *ofname)
+{
+ ssize_t rc;
+ uint64_t i;
+ char *buffer = NULL;
+ char *allocated = NULL;
+ struct timespec ts_start, ts_end;
+ int out_fd = -1;
+ int fpga_fd = open(devname, O_RDWR | O_NONBLOCK);
+ long total_time = 0;
+ float result;
+ float avg_time = 0;
+
+ if (fpga_fd < 0) {
+ fprintf(stderr, "unable to open device %s, %d.\n",
+ devname, fpga_fd);
+ perror("open device");
+ return -EINVAL;
+ }
+
+ /* create file to write data to */
+ if (ofname) {
+ out_fd = open(ofname, O_RDWR | O_CREAT | O_TRUNC | O_SYNC,
+ 0666);
+ if (out_fd < 0) {
+ fprintf(stderr, "unable to open output file %s, %d.\n",
+ ofname, out_fd);
+ perror("open output file");
+ rc = -EINVAL;
+ goto out;
+ }
+ }
+
+ posix_memalign((void **)&allocated, 4096 /*alignment */ , size + 4096);
+ if (!allocated) {
+ fprintf(stderr, "OOM %lu.\n", size + 4096);
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ buffer = allocated + offset;
+ if (verbose)
+ fprintf(stdout, "host buffer 0x%lx, %p.\n", size + 4096, buffer);
+
+ for (i = 0; i < count; i++) {
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_start);
+ /* lseek & read data from AXI MM into buffer using SGDMA */
+ rc = read_to_buffer(devname, fpga_fd, buffer, size, addr);
+ if (rc < 0)
+ goto out;
+ clock_gettime(CLOCK_MONOTONIC, &ts_end);
+
+ /* subtract the start time from the end time */
+ timespec_sub(&ts_end, &ts_start);
+ total_time += ts_end.tv_nsec;
+ /* a bit less accurate but side-effects are accounted for */
+ if (verbose)
+ fprintf(stdout,
+ "#%lu: CLOCK_MONOTONIC %ld.%09ld sec. read %ld bytes\n",
+ i, ts_end.tv_sec, ts_end.tv_nsec, size);
+
+ /* file argument given? */
+ if ((out_fd >= 0) & (no_write == 0)) {
+ rc = write_from_buffer(ofname, out_fd, buffer,
+ size, i*size);
+ if (rc < 0)
+ goto out;
+ }
+ }
+ avg_time = (float)total_time/(float)count;
+ result = ((float)size)*1000/avg_time;
+ if (verbose)
+ printf("** Avg time device %s, total time %ld nsec, avg_time = %f, size = %lu, BW = %f \n",
+ devname, total_time, avg_time, size, result);
+ printf("** Average BW = %lu, %f\n", size, result);
+ rc = 0;
+
+out:
+ close(fpga_fd);
+ if (out_fd >= 0)
+ close(out_fd);
+ free(allocated);
+
+ return rc;
+}
diff --git a/QDMA/linux-kernel/tools/dma_from_device_w_udd.c b/QDMA/linux-kernel/tools/dma_from_device_w_udd.c
new file mode 100644
index 0000000000000000000000000000000000000000..2fb6fcdab45a8d042ba7ae98c601455c269e8b81
--- /dev/null
+++ b/QDMA/linux-kernel/tools/dma_from_device_w_udd.c
@@ -0,0 +1,266 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#define _DEFAULT_SOURCE
+#define _XOPEN_SOURCE 500
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "dma_xfer_utils.c"
+
+#define DEVICE_NAME_DEFAULT "/dev/xdma0_c2h_0"
+#define SIZE_DEFAULT (32)
+#define COUNT_DEFAULT (1)
+
+static struct option const long_opts[] = {
+ {"device", required_argument, NULL, 'd'},
+ {"address", required_argument, NULL, 'a'},
+ {"size", required_argument, NULL, 's'},
+ {"offset", required_argument, NULL, 'o'},
+ {"count", required_argument, NULL, 'c'},
+ {"file", required_argument, NULL, 'f'},
+ {"udd_file", required_argument, NULL, 'u'},
+ {"help", no_argument, NULL, 'h'},
+ {"verbose", no_argument, NULL, 'v'},
+ {0, 0, 0, 0}
+};
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *ofname, char *uddofname);
+static int no_write = 0;
+
+static void usage(const char *name)
+{
+ int i = 0;
+ fprintf(stdout, "%s\n\n", name);
+ fprintf(stdout, "usage: %s [OPTIONS]\n\n", name);
+ fprintf(stdout, "Read via SGDMA, optionally save output to a file\n\n");
+
+ fprintf(stdout, " -%c (--%s) device (defaults to %s)\n",
+ long_opts[i].val, long_opts[i].name, DEVICE_NAME_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) the start address on the AXI bus\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) size of a single packet in bytes, default %d.\n",
+ long_opts[i].val, long_opts[i].name, SIZE_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) page offset of transfer\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) number of transfers, default is %d.\n",
+ long_opts[i].val, long_opts[i].name, COUNT_DEFAULT);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) file to write the data of the transfers\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) file to write the UDD data from the transfers\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) print usage help and exit\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) verbose output\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+}
+
+int main(int argc, char *argv[])
+{
+ int cmd_opt;
+ char *device = DEVICE_NAME_DEFAULT;
+ uint64_t address = 0;
+ uint64_t size = SIZE_DEFAULT;
+ uint64_t offset = 0;
+ uint64_t count = COUNT_DEFAULT;
+ uint64_t num_pkts = 1;
+ char *ofname = NULL;
+ char *uddofname = NULL;
+
+ while ((cmd_opt = getopt_long(argc, argv, "vhxc:f:d:a:s:o:u:c:", long_opts,
+ NULL)) != -1) {
+ switch (cmd_opt) {
+ case 0:
+ /* long option */
+ break;
+ case 'd':
+ /* device node name */
+ device = strdup(optarg);
+ break;
+ case 'a':
+ /* RAM address on the AXI bus in bytes */
+ address = getopt_integer(optarg);
+ break;
+ /* RAM size in bytes */
+ case 's':
+ size = getopt_integer(optarg);
+ break;
+ case 'o':
+ offset = getopt_integer(optarg) & 4095;
+ break;
+ /* count */
+ case 'c':
+ count = getopt_integer(optarg);
+ break;
+ /* count */
+ case 'f':
+ ofname = strdup(optarg);
+ break;
+ case 'u':
+ uddofname = strdup(optarg);
+ break;
+ /* print usage help and exit */
+ case 'x':
+ no_write++;
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ exit(0);
+ break;
+ }
+ }
+ if (verbose)
+ fprintf(stdout,
+ "dev %s, addr 0x%lx, size 0x%lx, offset 0x%lx, count %lu\n",
+ device, address, size, offset, count);
+
+ return test_dma(device, address, size, offset, count, ofname, uddofname);
+}
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *ofname, char *uddofname)
+{
+ ssize_t rc;
+ uint64_t i;
+ char *buffer = NULL;
+ char *allocated = NULL;
+ struct timespec ts_start, ts_end;
+ int out_fd = -1;
+ int out_udd_fd = -1;
+ int fpga_fd = open(devname, O_RDWR | O_NONBLOCK);
+ long total_time = 0;
+ float result;
+ float avg_time = 0;
+ uint64_t num_pgs = 0;
+
+ if (fpga_fd < 0) {
+ fprintf(stderr, "unable to open device %s, %d.\n",
+ devname, fpga_fd);
+ perror("open device");
+ return -EINVAL;
+ }
+
+ /* create file to write data to */
+ if (ofname) {
+ out_fd = open(ofname, O_RDWR | O_CREAT | O_TRUNC | O_SYNC,
+ 0666);
+ if (out_fd < 0) {
+ fprintf(stderr, "unable to open output file %s, %d.\n",
+ ofname, out_fd);
+ perror("open output file");
+ rc = -EINVAL;
+ goto out;
+ }
+ out_udd_fd = open(uddofname, O_RDWR | O_CREAT | O_TRUNC | O_SYNC,
+ 0666);
+ if (out_udd_fd < 0) {
+ fprintf(stderr, "unable to open output file %s, %d.\n",
+ uddofname, out_udd_fd);
+ perror("open output file");
+ rc = -EINVAL;
+ goto out;
+ }
+ }
+
+ num_pgs = (((size + offset) * count) >> 12) + 1;
+ posix_memalign((void **)&allocated, 4096 /*alignment */ , num_pgs * 4096);
+ if (!allocated) {
+ fprintf(stderr, "OOM %lu.\n", num_pgs * 4096);
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ if (verbose)
+ fprintf(stdout, "host buffer 0x%lx, %p.\n", size + 4096, buffer);
+
+ buffer = allocated;
+ rc = read_to_buffer(devname, fpga_fd, buffer, ((size + offset) * count), addr);
+ if (rc < 0)
+ goto out;
+ for (i = 0; i < count; i++) {
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_start);
+ /* lseek & read data from AXI MM into buffer using SGDMA */
+ clock_gettime(CLOCK_MONOTONIC, &ts_end);
+
+ /* subtract the start time from the end time */
+ timespec_sub(&ts_end, &ts_start);
+ total_time += ts_end.tv_nsec;
+ /* a bit less accurate but side-effects are accounted for */
+ if (verbose)
+ fprintf(stdout,
+ "#%lu: CLOCK_MONOTONIC %ld.%09ld sec. read %ld bytes\n",
+ i, ts_end.tv_sec, ts_end.tv_nsec, size);
+
+ /* file argument given? */
+ if ((out_udd_fd >= 0) & (no_write == 0)) {
+ rc = write_from_buffer(uddofname, out_udd_fd, buffer,
+ offset, i*offset);
+ if (rc < 0)
+ goto out;
+ }
+ buffer += offset; /* point to actual data */
+ /* file argument given? */
+ if ((out_fd >= 0) & (no_write == 0)) {
+ rc = write_from_buffer(ofname, out_fd, buffer,
+ size, i*size);
+ if (rc < 0)
+ goto out;
+ }
+ buffer += size; /* point to next packet */
+ }
+ avg_time = (float)total_time/(float)count;
+ result = ((float)size)*1000/avg_time;
+ if (verbose)
+ printf("** Avg time device %s, total time %ld nsec, avg_time = %f, size = %lu, BW = %f \n",
+ devname, total_time, avg_time, size, result);
+ printf("** Average BW = %lu, %f\n", size, result);
+ rc = 0;
+
+out:
+ close(fpga_fd);
+ if (out_fd >= 0)
+ close(out_fd);
+ if (out_udd_fd >= 0)
+ close(out_udd_fd);
+ free(allocated);
+
+ return rc;
+}
diff --git a/QDMA/linux-kernel/tools/dma_to_device.c b/QDMA/linux-kernel/tools/dma_to_device.c
new file mode 100644
index 0000000000000000000000000000000000000000..b8710f87613a758c80ff45c1d8b5cc029d00b387
--- /dev/null
+++ b/QDMA/linux-kernel/tools/dma_to_device.c
@@ -0,0 +1,266 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#define _DEFAULT_SOURCE
+#define _XOPEN_SOURCE 500
+#include <assert.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "dma_xfer_utils.c"
+
+static struct option const long_opts[] = {
+ {"device", required_argument, NULL, 'd'},
+ {"address", required_argument, NULL, 'a'},
+ {"size", required_argument, NULL, 's'},
+ {"offset", required_argument, NULL, 'o'},
+ {"count", required_argument, NULL, 'c'},
+ {"data infile", required_argument, NULL, 'f'},
+ {"data outfile", required_argument, NULL, 'w'},
+ {"help", no_argument, NULL, 'h'},
+ {"verbose", no_argument, NULL, 'v'},
+ {0, 0, 0, 0}
+};
+
+#define DEVICE_NAME_DEFAULT "/dev/qdma01000-MM-0"
+#define SIZE_DEFAULT (32)
+#define COUNT_DEFAULT (1)
+
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *filename, char *);
+
+static void usage(const char *name)
+{
+ int i = 0;
+
+ fprintf(stdout, "%s\n\n", name);
+ fprintf(stdout, "usage: %s [OPTIONS]\n\n", name);
+ fprintf(stdout,
+ "Write via SGDMA, optionally read input from a file.\n\n");
+
+ fprintf(stdout, " -%c (--%s) device (defaults to %s)\n",
+ long_opts[i].val, long_opts[i].name, DEVICE_NAME_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) the start address on the AXI bus\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) size of a single transfer in bytes, default %d,\n",
+ long_opts[i].val, long_opts[i].name, SIZE_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) page offset of transfer\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) number of transfers, default %d\n",
+ long_opts[i].val, long_opts[i].name, COUNT_DEFAULT);
+ i++;
+ fprintf(stdout, " -%c (--%s) filename to read the data from.\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout,
+ " -%c (--%s) filename to write the data of the transfers\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) print usage help and exit\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+ fprintf(stdout, " -%c (--%s) verbose output\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+}
+
+int main(int argc, char *argv[])
+{
+ int cmd_opt;
+ char *device = DEVICE_NAME_DEFAULT;
+ uint64_t address = 0;
+ uint64_t size = SIZE_DEFAULT;
+ uint64_t offset = 0;
+ uint64_t count = COUNT_DEFAULT;
+ char *infname = NULL;
+ char *ofname = NULL;
+
+ while ((cmd_opt =
+ getopt_long(argc, argv, "vhc:f:d:a:s:o:w:", long_opts,
+ NULL)) != -1) {
+ switch (cmd_opt) {
+ case 0:
+ /* long option */
+ break;
+ case 'd':
+ /* device node name */
+ //fprintf(stdout, "'%s'\n", optarg);
+ device = strdup(optarg);
+ break;
+ case 'a':
+ /* RAM address on the AXI bus in bytes */
+ address = getopt_integer(optarg);
+ break;
+ case 's':
+ /* size in bytes */
+ size = getopt_integer(optarg);
+ break;
+ case 'o':
+ offset = getopt_integer(optarg) & 4095;
+ break;
+ /* count */
+ case 'c':
+ count = getopt_integer(optarg);
+ break;
+ /* count */
+ case 'f':
+ infname = strdup(optarg);
+ break;
+ case 'w':
+ ofname = strdup(optarg);
+ break;
+ /* print usage help and exit */
+ case 'v':
+ verbose = 1;
+ break;
+ case 'h':
+ default:
+ usage(argv[0]);
+ exit(0);
+ break;
+ }
+ }
+
+ if (verbose)
+ fprintf(stdout,
+ "dev %s, address 0x%lx, size 0x%lx, offset 0x%lx, count %lu\n",
+ device, address, size, offset, count);
+
+ return test_dma(device, address, size, offset, count, infname, ofname);
+}
+
+static int test_dma(char *devname, uint64_t addr, uint64_t size,
+ uint64_t offset, uint64_t count, char *infname,
+ char *ofname)
+{
+ uint64_t i;
+ ssize_t rc;
+ char *buffer = NULL;
+ char *allocated = NULL;
+ struct timespec ts_start, ts_end;
+ int infile_fd = -1;
+ int outfile_fd = -1;
+ int fpga_fd = open(devname, O_RDWR);
+ long total_time = 0;
+ float result;
+ float avg_time = 0;
+
+ if (fpga_fd < 0) {
+ fprintf(stderr, "unable to open device %s, %d.\n",
+ devname, fpga_fd);
+ perror("open device");
+ return -EINVAL;
+ }
+
+ if (infname) {
+ infile_fd = open(infname, O_RDONLY);
+ if (infile_fd < 0) {
+ fprintf(stderr, "unable to open input file %s, %d.\n",
+ infname, infile_fd);
+ perror("open input file");
+ rc = -EINVAL;
+ goto out;
+ }
+ }
+
+ if (ofname) {
+ outfile_fd =
+ open(ofname, O_RDWR | O_CREAT | O_TRUNC | O_SYNC,
+ 0666);
+ if (outfile_fd < 0) {
+ fprintf(stderr, "unable to open output file %s, %d.\n",
+ ofname, outfile_fd);
+ perror("open output file");
+ rc = -EINVAL;
+ goto out;
+ }
+ }
+
+ posix_memalign((void **)&allocated, 4096 /*alignment */ , size + 4096);
+ if (!allocated) {
+ fprintf(stderr, "OOM %lu.\n", size + 4096);
+ rc = -ENOMEM;
+ goto out;
+ }
+ buffer = allocated + offset;
+ if (verbose)
+ fprintf(stdout, "host buffer 0x%lx = %p\n",
+ size + 4096, buffer);
+
+ if (infile_fd >= 0) {
+ rc = read_to_buffer(infname, infile_fd, buffer, size, 0);
+ if (rc < 0)
+ goto out;
+ }
+
+ for (i = 0; i < count; i++) {
+ /* write buffer to AXI MM address using SGDMA */
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_start);
+
+ rc = write_from_buffer(devname, fpga_fd, buffer, size, addr);
+ if (rc < 0)
+ goto out;
+
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_end);
+ /* subtract the start time from the end time */
+ timespec_sub(&ts_end, &ts_start);
+ total_time += ts_end.tv_nsec;
+ /* a bit less accurate but side-effects are accounted for */
+ if (verbose)
+ fprintf(stdout,
+ "#%lu: CLOCK_MONOTONIC %ld.%09ld sec. write %ld bytes\n",
+ i, ts_end.tv_sec, ts_end.tv_nsec, size);
+
+ if (outfile_fd >= 0) {
+ rc = write_from_buffer(ofname, outfile_fd, buffer,
+ size, i * size);
+ if (rc < 0)
+ goto out;
+ }
+ }
+ avg_time = (float)total_time/(float)count;
+ result = ((float)size)*1000/avg_time;
+ if (verbose)
+ printf("** Avg time device %s, total time %ld nsec, avg_time = %f, size = %lu, BW = %f \n",
+ devname, total_time, avg_time, size, result);
+
+ printf("** Average BW = %lu, %f\n",size, result);
+ rc = 0;
+
+out:
+ close(fpga_fd);
+ if (infile_fd >= 0)
+ close(infile_fd);
+ if (outfile_fd >= 0)
+ close(outfile_fd);
+ free(allocated);
+
+ return rc;
+}
diff --git a/QDMA/linux-kernel/tools/dma_xfer_utils.c b/QDMA/linux-kernel/tools/dma_xfer_utils.c
new file mode 100644
index 0000000000000000000000000000000000000000..ceff6d9ac184b9286fbac96e7eeb4c0b77d239f8
--- /dev/null
+++ b/QDMA/linux-kernel/tools/dma_xfer_utils.c
@@ -0,0 +1,198 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <time.h>
+#include <errno.h>
+#include <sys/types.h>
+
+/*
+ * man 2 write:
+ * On Linux, write() (and similar system calls) will transfer at most
+ * 0x7ffff000 (2,147,479,552) bytes, returning the number of bytes
+ * actually transferred. (This is true on both 32-bit and 64-bit
+ * systems.)
+ */
+
+#define RW_MAX_SIZE 0x7ffff000
+
+int verbose = 0;
+
+uint64_t getopt_integer(char *optarg)
+{
+ int rc;
+ uint64_t value;
+
+ rc = sscanf(optarg, "0x%lx", &value);
+ if (rc <= 0)
+ rc = sscanf(optarg, "%lu", &value);
+ //printf("sscanf() = %d, value = 0x%lx\n", rc, value);
+
+ return value;
+}
+
+ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
+ uint64_t base)
+{
+ ssize_t rc;
+ uint64_t count = 0;
+ char *buf = buffer;
+ off_t offset = base;
+
+ do { /* Support zero byte transfer */
+ uint64_t bytes = size - count;
+
+ if (bytes > RW_MAX_SIZE)
+ bytes = RW_MAX_SIZE;
+
+ if (offset) {
+ rc = lseek(fd, offset, SEEK_SET);
+ if (rc < 0) {
+ fprintf(stderr,
+ "%s, seek off 0x%lx failed %zd.\n",
+ fname, offset, rc);
+ perror("seek file");
+ return -EIO;
+ }
+ if (rc != offset) {
+ fprintf(stderr,
+ "%s, seek off 0x%lx != 0x%lx.\n",
+ fname, rc, offset);
+ return -EIO;
+ }
+ }
+
+ /* read data from file into memory buffer */
+ rc = read(fd, buf, bytes);
+ if (rc < 0) {
+ fprintf(stderr,
+ "%s, read off 0x%lx + 0x%lx failed %zd.\n",
+ fname, offset, bytes, rc);
+ perror("read file");
+ return -EIO;
+ }
+ if (rc != bytes) {
+ fprintf(stderr,
+ "%s, R off 0x%lx, 0x%lx != 0x%lx.\n",
+ fname, count, rc, bytes);
+ return -EIO;
+ }
+
+ count += bytes;
+ buf += bytes;
+ offset += bytes;
+ } while (count < size);
+
+ if (count != size) {
+ fprintf(stderr, "%s, R failed 0x%lx != 0x%lx.\n",
+ fname, count, size);
+ return -EIO;
+ }
+ return count;
+}
+
+ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
+ uint64_t base)
+{
+ ssize_t rc;
+ uint64_t count = 0;
+ char *buf = buffer;
+ off_t offset = base;
+
+ do { /* Support zero byte transfer */
+ uint64_t bytes = size - count;
+
+ if (bytes > RW_MAX_SIZE)
+ bytes = RW_MAX_SIZE;
+
+ if (offset) {
+ rc = lseek(fd, offset, SEEK_SET);
+ if (rc < 0) {
+ fprintf(stderr,
+ "%s, seek off 0x%lx failed %zd.\n",
+ fname, offset, rc);
+ perror("seek file");
+ return -EIO;
+ }
+ if (rc != offset) {
+ fprintf(stderr,
+ "%s, seek off 0x%lx != 0x%lx.\n",
+ fname, rc, offset);
+ return -EIO;
+ }
+ }
+
+ /* write data to file from memory buffer */
+ rc = write(fd, buf, bytes);
+ if (rc < 0) {
+ fprintf(stderr, "%s, W off 0x%lx, 0x%lx failed %zd.\n",
+ fname, offset, bytes, rc);
+ perror("write file");
+ return -EIO;
+ }
+ if (rc != bytes) {
+ fprintf(stderr, "%s, W off 0x%lx, 0x%lx != 0x%lx.\n",
+ fname, offset, rc, bytes);
+ return -EIO;
+ }
+
+ count += bytes;
+ buf += bytes;
+ offset += bytes;
+ } while (count < size);
+
+ if (count != size) {
+ fprintf(stderr, "%s, R failed 0x%lx != 0x%lx.\n",
+ fname, count, size);
+ return -EIO;
+ }
+ return count;
+}
+
+
+/* Subtract timespec t2 from t1
+ *
+ * Both t1 and t2 must already be normalized
+ * i.e. 0 <= nsec < 1000000000
+ */
+static int timespec_check(struct timespec *t)
+{
+ if ((t->tv_nsec < 0) || (t->tv_nsec >= 1000000000))
+ return -1;
+ return 0;
+
+}
+
+void timespec_sub(struct timespec *t1, struct timespec *t2)
+{
+ if (timespec_check(t1) < 0) {
+ fprintf(stderr, "invalid time #1: %lld.%.9ld.\n",
+ (long long)t1->tv_sec, t1->tv_nsec);
+ return;
+ }
+ if (timespec_check(t2) < 0) {
+ fprintf(stderr, "invalid time #2: %lld.%.9ld.\n",
+ (long long)t2->tv_sec, t2->tv_nsec);
+ return;
+ }
+ t1->tv_sec -= t2->tv_sec;
+ t1->tv_nsec -= t2->tv_nsec;
+ if (t1->tv_nsec >= 1000000000) {
+ t1->tv_sec++;
+ t1->tv_nsec -= 1000000000;
+ } else if (t1->tv_nsec < 0) {
+ t1->tv_sec--;
+ t1->tv_nsec += 1000000000;
+ }
+}
+
diff --git a/QDMA/linux-kernel/tools/dmautils.c b/QDMA/linux-kernel/tools/dmautils.c
new file mode 100644
index 0000000000000000000000000000000000000000..df813566c2d280f84f90907287c4b415fe88137c
--- /dev/null
+++ b/QDMA/linux-kernel/tools/dmautils.c
@@ -0,0 +1,1489 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <semaphore.h>
+#include <time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/shm.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <stdbool.h>
+#include <linux/types.h>
+#include <getopt.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <signal.h>
+#include <stddef.h>
+#include <errno.h>
+#include <error.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <sys/ioctl.h>
+#include </usr/include/pthread.h>
+#include <libaio.h>
+#include <sys/sysinfo.h>
+
+#define SEC2NSEC 1000000000
+#define SEC2USEC 1000000
+#define DEFAULT_PAGE_SIZE 4096
+#define PAGE_SHIFT 12
+
+#define DATA_VALIDATION 0
+
+static struct option const long_opts[] = {
+ {"config", required_argument, NULL, 'c'},
+ {0, 0, 0, 0}
+};
+
+static void prep_reg_dump(void);
+static void prep_pci_dump(void);
+
+static void usage(const char *name)
+{
+ int i = 0;
+ fprintf(stdout, "%s\n\n", name);
+ fprintf(stdout, "usage: %s [OPTIONS]\n\n", name);
+
+ fprintf(stdout, " -%c (--%s) config file that has configration for IO\n",
+ long_opts[i].val, long_opts[i].name);
+ i++;
+}
+
+static char * strip_blanks(char *word, long unsigned int *banlks)
+{
+ char *p = word;
+ unsigned int i = 0;
+
+ while (isblank(p[0])) {
+ p++;
+ i++;
+ }
+ *banlks = i;
+
+ return p;
+}
+
+static unsigned int copy_value(char *src, char *dst, unsigned int max_len)
+{
+ char *p = src;
+ unsigned int i = 0;
+
+ while (max_len && !isspace(p[0])) {
+ dst[i] = p[0];
+ p++;
+ i++;
+ max_len--;
+ }
+
+ return i;
+}
+
+static char * strip_comments(char *word)
+{
+ char *p = strtok(word, "#");
+
+ return p;
+}
+
+#define MSEC2NSEC 1000000
+#define Q_ADD_CMD_LEN 100
+#define Q_START_CMD_LEN 500
+#define Q_STOP_CMD_LEN 200
+#define Q_DEL_CMD_LEN 200
+#define Q_DUMP_CMD_LEN 200
+#define REG_DUMP_CMD_LEN 200
+#define CMPL_STATUS_ACC_CMD_LEN 200
+#define PCI_DUMP_CMD_LEN 100
+
+#define QDMA_UL_IMM_DUMP_C2H_DATA (1 << 17)
+#define QDMA_UL_STOP_C2H_TRANSFER (1 << 18)
+#define QDMA_UL_DROP_ENABLE (1 << 19)
+#define QDMA_UL_IMM_DUMP_CMPT_FIFO (1 << 20)
+#define QDMA_UL_STOP_CMPT_TRANSFER (1 << 21)
+
+//#define DEBUG
+
+enum q_mode {
+ Q_MODE_MM,
+ Q_MODE_ST,
+ Q_MODES
+};
+
+enum q_dir {
+ Q_DIR_H2C,
+ Q_DIR_C2H,
+ Q_DIR_BI,
+ Q_DIRS
+};
+
+#define THREADS_SET_CPU_AFFINITY 0
+
+struct io_info {
+ unsigned int num_req_submitted;
+ unsigned int num_req_completed;
+ struct list_head *head;
+ struct list_head *tail;
+ sem_t llock;
+ int pid;
+ pthread_t evt_id;
+ char q_name[20];
+ char q_add[Q_ADD_CMD_LEN];
+ char q_start[Q_START_CMD_LEN];
+ char q_stop[Q_STOP_CMD_LEN];
+ char q_del[Q_DEL_CMD_LEN];
+ char q_dump[Q_DUMP_CMD_LEN];
+ char trig_mode[10];
+ unsigned char q_ctrl;
+ unsigned int q_added;
+ unsigned int q_started;
+ unsigned int q_wait_for_stop;
+ int fd;
+ unsigned int pf;
+ unsigned int qid;
+ enum q_mode mode;
+ enum q_dir dir;
+ unsigned int idx_tmr;
+ unsigned int idx_cnt;
+ unsigned int idx_rngsz;
+ unsigned int pfetch_en;
+ unsigned int pkt_burst;
+ unsigned int pkt_sz;
+ unsigned int cmptsz;
+#ifdef DEBUG
+ unsigned long long total_nodes;
+ unsigned long long freed_nodes;
+#endif
+ unsigned int thread_id;
+#if THREADS_SET_CPU_AFFINITY
+ int cpu;
+#endif
+};
+
+struct list_head {
+ struct list_head *next;
+ unsigned int max_events;
+ unsigned int completed_events;
+ io_context_t ctxt;
+};
+
+#define container_of(ptr, type, member) ({ \
+ const struct iocb *__mptr = (ptr); \
+ (type *)( (char *)__mptr - offsetof(type,member) );})
+
+static unsigned int io_exit = 0;
+static unsigned int force_exit = 0;
+static unsigned int num_q = 0;
+static unsigned int pkt_sz = 0;
+static unsigned int num_pkts;
+static unsigned int tsecs = 0;
+struct io_info *info = NULL;
+static char cfg_name[20];
+static unsigned int pci_bus = 0;
+static unsigned int pci_dev = 0;
+unsigned int num_thrds = 0;
+unsigned int num_thrds_per_q = 1;
+int shmid;
+int base_pid;
+enum q_mode mode;
+enum q_dir dir;
+unsigned int num_pf = 0;
+unsigned int pf_start = 0;
+unsigned int q_start = 0;
+unsigned int idx_rngsz = 0;
+unsigned int idx_tmr = 0;
+unsigned int idx_cnt = 0;
+unsigned int pfetch_en = 0;
+unsigned int cmptsz = 0;
+unsigned int no_memcpy = 1;
+char trigmode[10];
+char reg_dump[REG_DUMP_CMD_LEN];
+char pci_dump[PCI_DUMP_CMD_LEN];
+unsigned int dump_en = 0;
+static struct timespec g_ts_start;
+static unsigned char *q_lst_stop = NULL;
+int q_lst_stop_mid;
+int *child_pid_lst = NULL;
+#if THREADS_SET_CPU_AFFINITY
+unsigned int num_processors = 1;
+#endif
+#if DATA_VALIDATION
+unsigned short valid_data[2*1024];
+#endif
+
+static void clear_events(struct io_info *_info, struct list_head *node);
+static int setup_thrd_env(struct io_info *_info, unsigned char is_new_fd);
+static int setup_pf_env(struct io_info *_info);
+
+static int arg_read_int(char *s, uint32_t *v)
+{
+ char *p = NULL;
+
+
+ *v = strtoul(s, &p, 0);
+ if (*p && (*p != '\n') && !isblank(*p)) {
+ printf("Error:something not right%s %s %s",s, p, isblank(*p)? "true": "false");
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static void dump_thrd_info(struct io_info *_info) {
+ unsigned int i;
+
+ printf("q_name = %s\n", info->q_name);
+ printf("dir = %u\n", _info->dir);
+ printf("mode = %u\n", _info->mode);
+ printf("idx_cnt = %u\n", _info->idx_cnt);
+ printf("idx_rngsz = %u\n", _info->idx_rngsz);
+ printf("idx_tmr = %u\n", _info->idx_tmr);
+ printf("pf = %x\n", _info->pf);
+ printf("qid = %u\n", _info->qid);
+ printf("fd = %u\n", _info->fd);
+ printf("trig_mode = %s\n", _info->trig_mode);
+ printf("q_ctrl = %u\n", _info->q_ctrl);
+ printf("q_added = %u\n", _info->q_added);
+ printf("q_started = %u\n", _info->q_started);
+#if THREADS_SET_CPU_AFFINITY
+ printf("cpu = %u\n", _info->cpu);
+#endif
+}
+
+struct dma_meminfo {
+ void *memptr;
+ unsigned int num_blks;
+};
+
+#define USE_MEMPOOL
+
+struct mempool_handle {
+ void *mempool;
+ unsigned int mempool_blkidx;
+ unsigned int mempool_blksz;
+ unsigned int total_memblks;
+ struct dma_meminfo *mempool_info;
+#ifdef DEBUG
+ unsigned int id;
+ unsigned int loop;
+#endif
+};
+
+static struct mempool_handle ctxhandle;
+static struct mempool_handle iocbhandle;
+static struct mempool_handle datahandle;
+
+static void mempool_create(struct mempool_handle *mpool, unsigned int entry_size, unsigned int max_entries)
+{
+#ifdef USE_MEMPOOL
+ if (posix_memalign((void **)&mpool->mempool, DEFAULT_PAGE_SIZE,
+ max_entries * (entry_size + sizeof(struct dma_meminfo)))) {
+ printf("OOM\n");
+ exit(1);
+ }
+ mpool->mempool_info = (struct dma_meminfo *)(((char *)mpool->mempool) + (max_entries * entry_size));
+#endif
+ mpool->mempool_blksz = entry_size;
+ mpool->total_memblks = max_entries;
+ mpool->mempool_blkidx = 0;
+}
+
+static void mempool_free(struct mempool_handle *mpool)
+{
+#ifdef USE_MEMPOOL
+ free(mpool->mempool);
+ mpool->mempool = NULL;
+#endif
+}
+
+static void *dma_memalloc(struct mempool_handle *mpool, unsigned int num_blks)
+{
+ unsigned int _mempool_blkidx = mpool->mempool_blkidx;
+ unsigned int tmp_blkidx = _mempool_blkidx;
+ unsigned int max_blkcnt = tmp_blkidx + num_blks;
+ unsigned int i, avail = 0;
+ void *memptr = NULL;
+ char *mempool = mpool->mempool;
+ struct dma_meminfo *_mempool_info = mpool->mempool_info;
+ unsigned int _total_memblks = mpool->total_memblks;
+
+#ifdef USE_MEMPOOL
+ if (max_blkcnt > _total_memblks) {
+ tmp_blkidx = 0;
+ max_blkcnt = num_blks;
+ }
+ for (i = tmp_blkidx; (i < _total_memblks) && (i < max_blkcnt); i++) {
+ if (_mempool_info[i].memptr) { /* occupied blks ahead */
+ i += _mempool_info[i].num_blks;
+ max_blkcnt = i + num_blks;
+ avail = 0;
+ tmp_blkidx = i;
+ } else
+ avail++;
+ if (max_blkcnt > _total_memblks) { /* reached the end of mempool. circle through*/
+ if (num_blks > _mempool_blkidx) return NULL; /* Continuous num_blks not available */
+ i = 0;
+ avail = 0;
+ max_blkcnt = num_blks;
+ tmp_blkidx = 0;
+ }
+ }
+ if (avail < num_blks) { /* no required available blocks */
+ return NULL;
+ }
+
+ memptr = &(mempool[tmp_blkidx * mpool->mempool_blksz]);
+ _mempool_info[tmp_blkidx].memptr = memptr;
+ _mempool_info[tmp_blkidx].num_blks = num_blks;
+ mpool->mempool_blkidx = tmp_blkidx + num_blks;
+#else
+ memptr = calloc(num_blks, mpool->mempool_blksz);
+#endif
+
+ return memptr;
+}
+
+static void dma_free(struct mempool_handle *mpool, void *memptr)
+{
+#ifdef USE_MEMPOOL
+ struct dma_meminfo *_meminfo = mpool->mempool_info;
+ unsigned int _total_memblks = mpool->total_memblks;
+ unsigned int idx;
+
+ if (!memptr) return;
+
+ idx = (memptr - mpool->mempool)/mpool->mempool_blksz;
+#ifdef DEBUG
+ if (idx >= _total_memblks) {
+ printf("Asserting: %u:Invalid memory index %u acquired\n", mpool->id, idx);
+ while(1);
+ }
+#endif
+
+ _meminfo[idx].num_blks = 0;
+ _meminfo[idx].memptr = NULL;
+#else
+ free(memptr);
+#endif
+}
+
+static void create_thread_info(void)
+{
+ unsigned int base = 0;
+ unsigned int dir_factor = 1;
+ unsigned int q_ctrl = 1;
+ unsigned int i, j, k;
+ struct io_info *_info;
+ int last_fd = -1;
+ int s;
+ unsigned char is_new_fd = 1;
+ char reg_cmd[100] = {'\0'};
+#if THREADS_SET_CPU_AFFINITY
+ int h2c_cpu = 0;
+ int max_h2c_cpu = (num_processors / 2);
+ int c2h_cpu = max_h2c_cpu;
+ int max_c2h_cpu = (num_processors / 2) + ((num_processors % 2) ? 1 : 0);
+#endif
+
+ if (dir == Q_DIR_BI)
+ dir_factor = 2;
+ if ((shmid = shmget(IPC_PRIVATE, num_thrds * sizeof(struct io_info), IPC_CREAT | 0666)) < 0)
+ {
+ perror("smget returned -1\n");
+ error(-1, errno, " ");
+ exit(-1);
+ }
+ if ((q_lst_stop_mid = shmget(IPC_PRIVATE, dir_factor * num_q * num_pf, IPC_CREAT | 0666)) < 0)
+ {
+ perror("smget returned -1\n");
+ error(-1, errno, " ");
+ exit(-1);
+ }
+ if ((q_lst_stop = (unsigned char *) shmat(q_lst_stop_mid, NULL, 0)) == (unsigned char *) -1) {
+ perror("Process shmat returned NULL\n");
+ error(-1, errno, " ");
+ exit(1);
+ }
+ memset(q_lst_stop, 0 , num_q);
+ if (shmdt(q_lst_stop) == -1) {
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ if ((_info = (struct io_info *) shmat(shmid, NULL, 0)) == (struct io_info *) -1) {
+ perror("Process shmat returned NULL\n");
+ error(-1, errno, " ");
+ }
+ prep_pci_dump();
+ prep_reg_dump();
+ if ((mode == Q_MODE_ST) && (dir != Q_DIR_H2C)) {
+ snprintf(reg_cmd, 100, "dmactl qdma%05x reg write bar 2 0x%x %d",
+ (pci_bus << 12) | (pci_dev << 4) | pf_start, 0x50,
+ cmptsz/* | QDMA_UL_DROP_ENABLE*/);
+ system(reg_cmd);
+ memset(reg_cmd, '\0', 100);
+ snprintf(reg_cmd, 100, "dmactl qdma%05x reg write bar 2 0x%x %d",
+ (pci_bus << 12) | (pci_dev << 4) | pf_start, 0x90, pkt_sz);
+ printf("%s\n", reg_cmd);
+ system(reg_cmd);
+ usleep(1000);
+ memset(reg_cmd, '\0', 100);
+ }
+
+ base = 0;
+ for (k = 0; k < num_pf; k++) {
+ for (i = 0 ; i < num_q; i++) {
+ q_ctrl = 1;
+#if THREADS_SET_CPU_AFFINITY
+ if (h2c_cpu >= max_h2c_cpu)
+ h2c_cpu = 0;
+ if (c2h_cpu >= (max_h2c_cpu + max_h2c_cpu))
+ c2h_cpu = max_h2c_cpu;
+#endif
+ for (j = 0; j < num_thrds_per_q; j++) {
+ is_new_fd = 1;
+ if ((dir == Q_DIR_H2C) || (dir == Q_DIR_BI)) {
+ snprintf(_info[base].q_name, 20, "qdma%02x%02x%01x-%s-%d", pci_bus, pci_dev,
+ pf_start+k, (mode == Q_MODE_MM) ? "MM" : "ST", q_start + i);
+ _info[base].dir = Q_DIR_H2C;
+ _info[base].mode = mode;
+ _info[base].idx_rngsz = idx_rngsz;
+ _info[base].pf = (pci_bus << 12) | (pci_dev << 4) | (pf_start + k);
+ _info[base].qid = q_start + i;
+ _info[base].q_ctrl = q_ctrl;
+ _info[base].fd = last_fd;
+ _info[base].pkt_burst = num_pkts;
+ _info[base].pkt_sz = pkt_sz;
+#if THREADS_SET_CPU_AFFINITY
+ _info[base].cpu = h2c_cpu;
+#endif
+ sem_init(&_info[base].llock, 0, 1);
+ if (q_ctrl != 0) {
+ last_fd = setup_thrd_env(&_info[base], is_new_fd);
+ }
+ _info[base].thread_id = base;
+// dump_thrd_info(&_info[base]);
+ base++;
+ is_new_fd = 0;
+ }
+ if (dir != Q_DIR_H2C)
+ {
+ snprintf(_info[base].q_name, 20, "qdma%02x%02x%01x-%s-%d", pci_bus, pci_dev,
+ pf_start+k, (mode == Q_MODE_MM) ? "MM" : "ST", q_start + i);
+ _info[base].dir = Q_DIR_C2H;
+ _info[base].mode = mode;
+ _info[base].idx_rngsz = idx_rngsz;
+ _info[base].pf = (pci_bus << 12) | (pci_dev << 4) | (pf_start + k);
+ _info[base].qid = q_start + i;
+ _info[base].q_ctrl = q_ctrl;
+ _info[base].pkt_burst = num_pkts;
+ _info[base].pkt_sz = pkt_sz;
+#if THREADS_SET_CPU_AFFINITY
+ _info[base].cpu = c2h_cpu;
+#endif
+ if (_info[base].mode == Q_MODE_ST) {
+ _info[base].pfetch_en = pfetch_en;
+ _info[base].idx_cnt = idx_cnt;
+ _info[base].idx_tmr = idx_tmr;
+ _info[base].cmptsz = cmptsz;
+ strncpy(_info[base].trig_mode, trigmode, 10);
+ }
+ sem_init(&_info[base].llock, 0, 1);
+ _info[base].fd = last_fd;
+ if (q_ctrl != 0) {
+ last_fd = setup_thrd_env(&_info[base], is_new_fd);
+ }
+ _info[base].thread_id = base;
+// dump_thrd_info(&_info[base]);
+ base++;
+ }
+ q_ctrl = 0;
+ }
+#if THREADS_SET_CPU_AFFINITY
+ h2c_cpu++;
+ c2h_cpu++;
+#endif
+ }
+ }
+ if ((mode == Q_MODE_ST) && (dir != Q_DIR_H2C)) {
+ snprintf(reg_cmd, 100, "dmactl qdma%05x reg write bar 2 0x%x %d",
+ (pci_bus << 12) | (pci_dev << 4) | pf_start, 0x08,
+ QDMA_UL_IMM_DUMP_C2H_DATA | QDMA_UL_IMM_DUMP_CMPT_FIFO/* | QDMA_UL_DROP_ENABLE*/);
+ system(reg_cmd);
+ memset(reg_cmd, '\0', 100);
+ usleep(1000);
+ }
+ if (shmdt(_info) == -1) {
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+}
+
+static void parse_config_file(const char *cfg_fname)
+{
+ char *linebuf = NULL;
+ char *realbuf;
+ FILE *fp;
+ size_t linelen = 0;
+ size_t numread;
+ size_t numblanks;
+ unsigned int linenum = 0;
+ char *config, *value;
+ unsigned int dir_factor = 1;
+
+ fp = fopen(cfg_fname, "r");
+ if (fp == NULL)
+ exit(EXIT_FAILURE);
+
+ while ((numread = getline(&linebuf, &linelen, fp)) != -1) {
+ numread--;
+ linenum++;
+ linebuf = strip_comments(linebuf);
+ realbuf = strip_blanks(linebuf, &numblanks);
+ linelen -= numblanks;
+ if (0 == linelen)
+ continue;
+ config = strtok(realbuf, "=");
+ value = strtok(NULL, "=");
+ if (!strncmp(config, "mode", 4)) {
+ if (!strncmp(value, "mm", 2))
+ mode = Q_MODE_MM;
+ else if(!strncmp(value, "st", 2))
+ mode = Q_MODE_ST;
+ else {
+ printf("Error: Unkown mode");
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "dir", 3)) {
+ if (!strncmp(value, "h2c", 3))
+ dir = Q_DIR_H2C;
+ else if(!strncmp(value, "c2h", 3))
+ dir = Q_DIR_C2H;
+ else if(!strncmp(value, "bi", 2))
+ dir = Q_DIR_BI;
+ else {
+ printf("Error: Unkown dir");
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "name", 3)) {
+ copy_value(value, cfg_name, 20);
+ } else if (!strncmp(config, "pf_range", 8)) {
+ char *pf_range_start = strtok(value, ":");
+ char *pf_range_end = strtok(NULL, ":");
+ unsigned int start;
+ unsigned int end;
+ if (arg_read_int(pf_range_start, &start)) {
+ printf("Error: Invalid pf range start:%s\n", pf_range_start);
+ goto prase_cleanup;
+ }
+ if (arg_read_int(pf_range_end, &end)) {
+ printf("Error: Invalid pf range end:%s\n", pf_range_end);
+ goto prase_cleanup;
+ }
+
+ pf_start = start;
+ num_pf = end - start + 1;
+ } else if (!strncmp(config, "q_range", 7)) {
+ char *q_range_start = strtok(value, ":");
+ char *q_range_end = strtok(NULL, ":");
+ unsigned int start;
+ unsigned int end;
+ if (arg_read_int(q_range_start, &start)) {
+ printf("Error: Invalid q range start:%s\n", q_range_start);
+ goto prase_cleanup;
+ }
+ if (arg_read_int(q_range_end, &end)) {
+ printf("Error: Invalid q range end:%s\n", q_range_end);
+ goto prase_cleanup;
+ }
+
+ q_start = start;
+ num_q = end - start + 1;
+ } else if (!strncmp(config, "rngidx", 6)) {
+ if (arg_read_int(value, &idx_rngsz)) {
+ printf("Error: Invalid idx_rngsz:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "tmr_idx", 7)) {
+ if (arg_read_int(value, &idx_tmr)) {
+ printf("Error: Invalid idx_tmr:%s\n", value);
+ goto prase_cleanup;
+ }
+ }
+ if (!strncmp(config, "cntr_idx", 8)) {
+ if (arg_read_int(value, &idx_cnt)) {
+ printf("Error: Invalid idx_cnt:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "pfetch_en", 9)) {
+ if (arg_read_int(value, &pfetch_en)) {
+ printf("Error: Invalid pfetch_en:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "cmptsz", 5)) {
+ if (arg_read_int(value, &cmptsz)) {
+ printf("Error: Invalid cmptsz:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "dump_en", 5)) {
+ if (arg_read_int(value, &dump_en)) {
+ printf("Error: Invalid dump_en:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "trig_mode", 9)) {
+ copy_value(value, trigmode, 10);
+ } else if (!strncmp(config, "runtime", 9)) {
+ if (arg_read_int(value, &tsecs)) {
+ printf("Error: Invalid tsecs:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "num_threads", 11)) {
+ if (arg_read_int(value, &num_thrds_per_q)) {
+ printf("Error: Invalid num_threads:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "pkt_sz", 6)) {
+ if (arg_read_int(value, &pkt_sz)) {
+ printf("Error: Invalid pkt_sz:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "num_pkt", 7)) {
+ if (arg_read_int(value, &num_pkts)) {
+ printf("Error: Invalid io_sz:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "no_memcpy", 9)) {
+ if (arg_read_int(value, &no_memcpy)) {
+ printf("Error: Invalid io_sz:%s\n", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "pci_bus", 7)) {
+ char *p;
+
+ pci_bus = strtoul(value, &p, 16);
+ if (*p && (*p != '\n')) {
+ printf("Error: bad parameter \"%s\", integer expected", value);
+ goto prase_cleanup;
+ }
+ } else if (!strncmp(config, "pci_dev", 7)) {
+ char *p;
+
+ pci_dev = strtoul(value, &p, 16);
+ if (*p && (*p != '\n')) {
+ printf("Error: bad parameter \"%s\", integer expected", value);
+ goto prase_cleanup;
+ }
+ }
+ }
+ fclose(fp);
+
+ if (!pci_bus && !pci_dev) {
+ printf("Error: PCI bus information not provided\n");
+ exit(1);
+ }
+ if (dir == Q_DIR_BI)
+ dir_factor = 2;
+ num_thrds = num_pf * num_q * dir_factor * num_thrds_per_q;
+ create_thread_info();
+ return;
+ /*dump_thrd_info();
+ exit(1);*/
+prase_cleanup:
+ fclose(fp);
+}
+
+#define MAX_AIO_EVENTS 65536
+
+static void list_add_tail(struct io_info *_info, struct list_head *node)
+{
+ sem_wait(&_info->llock);
+ if (_info->head == NULL) {
+ _info->head = node;
+ _info->tail = node;
+ } else {
+ _info->tail->next = node;
+ _info->tail = node;
+ }
+#ifdef DEBUG
+ _info->total_nodes++;
+#endif
+ sem_post( &_info->llock);
+}
+
+static void list_add_head(struct io_info *_info, struct list_head *node)
+{
+ sem_wait(&_info->llock);
+ node->next = _info->head;
+ if (_info->head == NULL) {
+ _info->tail = node;
+ }
+ _info->head = node;
+ sem_post( &_info->llock);
+}
+
+static struct list_head *list_pop(struct io_info *_info)
+{
+ struct list_head *node = NULL;
+
+ sem_wait(&_info->llock);
+ node = _info->head;
+ if (_info->head == _info->tail)
+ _info->tail = NULL;
+
+ if (node)
+ _info->head = node->next;
+
+ sem_post(&_info->llock);
+
+ return node;
+}
+
+static void list_free(struct io_info *_info)
+{
+ struct list_head *node = NULL;
+ struct list_head *prev_node = NULL;
+ unsigned int i;
+
+ sem_wait(&_info->llock);
+#ifdef DEBUG
+ printf("Need to free %llu nodes in thrd%u\n", _info->total_nodes - _info->freed_nodes, _info->thread_id);
+#endif
+ node = _info->head;
+
+ while (node != NULL) {
+ clear_events(_info, node);
+ io_destroy(node->ctxt);
+ prev_node = node;
+ node = node->next;
+ dma_free(&ctxhandle, prev_node);
+#ifdef DEBUG
+ _info->freed_nodes++;
+#endif
+ }
+ sem_post(&_info->llock);
+}
+
+/* Subtract timespec t2 from t1
+ *
+ * Both t1 and t2 must already be normalized
+ * i.e. 0 <= nsec < 1000000000
+ */
+static int timespec_check(struct timespec *t)
+{
+ if ((t->tv_nsec < 0) || (t->tv_nsec >= 1000000000))
+ return -1;
+ return 0;
+
+}
+
+void timespec_sub(struct timespec *t1, struct timespec *t2)
+{
+ if (timespec_check(t1) < 0) {
+ fprintf(stderr, "invalid time #1: %lld.%.9ld.\n",
+ (long long)t1->tv_sec, t1->tv_nsec);
+ return;
+ }
+ if (timespec_check(t2) < 0) {
+ fprintf(stderr, "invalid time #2: %lld.%.9ld.\n",
+ (long long)t2->tv_sec, t2->tv_nsec);
+ return;
+ }
+ t1->tv_sec -= t2->tv_sec;
+ t1->tv_nsec -= t2->tv_nsec;
+ if (t1->tv_nsec >= 1000000000) {
+ t1->tv_sec++;
+ t1->tv_nsec -= 1000000000;
+ } else if (t1->tv_nsec < 0) {
+ t1->tv_sec--;
+ t1->tv_nsec += 1000000000;
+ }
+}
+
+static void clear_events(struct io_info *_info, struct list_head *node) {
+ struct io_event *events = NULL;
+ int num_events = 0;
+ unsigned int i, j, bufcnt;
+ struct timespec ts_cur = {1, 0};
+
+ if (node->max_events <= node->completed_events)
+ return;
+#ifdef DEBUG
+ printf("Thrd%u: Need to clear %u/%u events in node %p\n",
+ _info->thread_id, node->max_events - node->completed_events,
+ node->max_events, node);
+#endif
+ events = calloc(node->max_events - node->completed_events, sizeof(struct io_event));
+ if (events == NULL) {
+ printf("OOM\n");
+ return;
+ }
+ do {
+ num_events = io_getevents(node->ctxt, 1,
+ node->max_events - node->completed_events, events,
+ &ts_cur);
+ for (j = 0; (num_events > 0) && (j < num_events); j++) {
+ struct iocb *iocb = (struct iocb *)events[j].obj;
+ struct iovec *iov;
+
+ node->completed_events++;
+ if (!iocb) {
+ printf("Error: Invalid IOCB from events\n");
+ continue;
+ }
+
+ iov = (struct iovec *)iocb->u.c.buf;
+
+ for (bufcnt = 0; bufcnt < iocb->u.c.nbytes; bufcnt++)
+ dma_free(&datahandle, iov[bufcnt].iov_base);
+ dma_free(&iocbhandle, iocb);
+ }
+ } while ((num_events > 0) && (node->max_events > node->completed_events));
+
+ free(events);
+}
+
+static void *event_mon(void *argp)
+{
+ struct io_info *_info = (struct io_info *)argp;
+ unsigned int i, j, bufcnt;
+ struct io_event *events = NULL;
+ int num_events = 0;
+ int ret;
+ struct timespec ts_cur = {0, 0};
+#if DATA_VALIDATION
+ unsigned short *rcv_data;
+ unsigned int k;
+#endif
+
+ events = calloc(MAX_AIO_EVENTS, sizeof(struct io_event));
+ if (events == NULL) {
+ printf("OOM\n");
+ exit(1);
+ }
+ while ((0 == io_exit) && (0 == force_exit)) {
+ struct list_head *node = list_pop(_info);
+
+ if (!node)
+ continue;
+
+ memset(events, 0, MAX_AIO_EVENTS * sizeof(struct io_event));
+ do {
+ num_events = io_getevents(node->ctxt, 1,
+ node->max_events - node->completed_events, events,
+ &ts_cur);
+ for (j = 0; (num_events > 0) && (j < num_events); j++) {
+ struct iocb *iocb = (struct iocb *)events[j].obj;
+ struct iovec *iov = NULL;
+
+ if (!iocb) {
+ printf("Error: Invalid IOCB from events\n");
+ continue;
+ }
+ _info->num_req_completed += events[j].res;
+
+ iov = (struct iovec *)(iocb->u.c.buf);
+ if (!iov) {
+ printf("invalid buffer\n");
+ continue;
+ }
+#if DATA_VALIDATION
+ rcv_data = iov[0].iov_base;
+ for (k = 0; k < (iov[0].iov_len/2) && events[j].res && !(events[j].res2); k += 8) {
+ printf("%04x: %04x %04x %04x %04x %04x %04x %04x %04x\n", k,
+ rcv_data[k], rcv_data[k+1], rcv_data[k+2],
+ rcv_data[k+3], rcv_data[k+4], rcv_data[k+5],
+ rcv_data[k+6], rcv_data[k+7]);
+ }
+#endif
+ for (bufcnt = 0; (bufcnt < iocb->u.c.nbytes) && iov; bufcnt++)
+ dma_free(&datahandle, iov[bufcnt].iov_base);
+ dma_free(&iocbhandle, iocb);
+ }
+ if (num_events > 0)
+ node->completed_events += num_events;
+ if (node->completed_events >= node->max_events) {
+ io_destroy(node->ctxt);
+ dma_free(&ctxhandle, node);
+ break;
+ }
+ } while ((0 == io_exit) && (0 == force_exit));
+
+ if (node->completed_events < node->max_events)
+ list_add_head(_info, node);
+ }
+ free(events);
+#ifdef DEBUG
+ printf("Exiting evt_thrd: %d\n", _info->thread_id);
+#endif
+
+ return NULL;
+}
+
+static void io_proc_cleanup(struct io_info *_info)
+{
+ unsigned int i, j = 0;
+ int s;
+ unsigned int q_offset;
+ unsigned int dir_factor = (dir == Q_DIR_BI) ? 1 : 2;
+ unsigned int q_lst_idx_base;
+ unsigned char is_q_stop = (_info->q_ctrl && _info->q_started);
+ char reg_cmd[100] = {'\0'};
+
+ io_exit = 1;
+ pthread_join(_info->evt_id, NULL);
+
+ q_offset = (_info->dir == Q_DIR_H2C)? 0 : num_q;
+ if (dir != Q_DIR_BI)
+ q_offset = 0;
+ q_lst_idx_base = ((((_info->pf & 0x0000F) - pf_start) * num_q * dir_factor) + q_offset);
+ _info->q_wait_for_stop = 1;
+
+ while (is_q_stop) {
+ j = 0;
+ for (i = 0; i < num_thrds; i++) {
+ if ((info[i].pf != _info->pf) ||
+ (info[i].qid != _info->qid) ||
+ (info[i].dir != _info->dir))
+ continue;
+ if (info[i].q_wait_for_stop)
+ j++;
+ else
+ break;
+ }
+ if (j != num_thrds_per_q)
+ sched_yield();
+ else
+ break;
+ }
+ if ((mode == Q_MODE_ST) && (dir != Q_DIR_H2C)) {
+ do {
+ for (i = 0; i < num_thrds; i++) {
+ if (!info[i].q_wait_for_stop)
+ break;
+ }
+ if (i == num_thrds)
+ break;
+ } while (1);
+ }
+ if (is_q_stop) {
+ if (dump_en) {
+ s = system(_info->q_dump);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n",
+ _info->q_dump, s);
+ }
+ }
+ printf("%s\n", _info->q_stop);
+ s = system(_info->q_stop);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n",_info->q_stop, s);
+ }
+ snprintf(reg_cmd, 100, "dmactl qdma%05x reg write bar 2 0x%x %d",
+ _info->pf, 0x08, QDMA_UL_STOP_C2H_TRANSFER | QDMA_UL_STOP_CMPT_TRANSFER);
+ system(reg_cmd);
+ _info->q_started = 0;
+ q_lst_stop[q_lst_idx_base + _info->qid - q_start] = 1;
+ }
+ while (!(q_lst_stop[q_lst_idx_base + _info->qid - q_start])) {
+ sched_yield();
+ }
+ if (shmdt(q_lst_stop) == -1) {
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ list_free(_info);
+
+ mempool_free(&iocbhandle);
+ mempool_free(&ctxhandle);
+ mempool_free(&datahandle);
+}
+
+static void *io_thread(void *argp)
+{
+ struct io_info *_info = (struct io_info *)argp;
+ struct iocb *iocb;
+ int ret;
+ int s;
+ unsigned int max_io = MAX_AIO_EVENTS;
+ unsigned int cnt = 0;
+ pthread_attr_t attr;
+ unsigned int io_sz = _info->pkt_sz;
+ unsigned int burst_cnt = _info->pkt_burst;
+ unsigned int num_desc;
+ unsigned int max_reqs;
+ struct iocb *io_list[1];
+
+ if ((_info->mode == Q_MODE_ST) && (_info->dir == Q_DIR_C2H)) {
+ io_sz = _info->pkt_burst * _info->pkt_sz;
+ burst_cnt = 1;
+ }
+ num_desc = (io_sz + DEFAULT_PAGE_SIZE - 1) >> PAGE_SHIFT;
+ max_reqs = (64 << idx_rngsz);
+ mempool_create(&datahandle, num_desc*DEFAULT_PAGE_SIZE, max_reqs + (burst_cnt * num_desc));
+ mempool_create(&ctxhandle, sizeof(struct list_head), max_reqs);
+ mempool_create(&iocbhandle, sizeof(struct iocb) + (burst_cnt * sizeof(struct iovec)), max_reqs + (burst_cnt * num_desc));
+#ifdef DEBUG
+ ctxhandle.id = 1;
+ datahandle.id = 0;
+ iocbhandle.id = 2;
+#endif
+ s = pthread_attr_init(&attr);
+ if (s != 0)
+ printf("pthread_attr_init failed\n");
+ if (pthread_create(&_info->evt_id, &attr, event_mon, _info))
+ exit(1);
+
+ do {
+ struct list_head *node = NULL;
+ struct timespec ts_cur;
+
+ if (tsecs) {
+ ret = clock_gettime(CLOCK_MONOTONIC, &ts_cur);
+ timespec_sub(&ts_cur, &g_ts_start);
+ if (ts_cur.tv_sec >= tsecs)
+ break;
+ }
+ node = dma_memalloc(&ctxhandle, 1);
+ if (!node) {
+ continue;
+ }
+ ret = io_queue_init(max_io, &node->ctxt);
+ if (ret != 0) {
+ printf("Error: io_setup error %d on %u\n", ret, _info->thread_id);
+ dma_free(&ctxhandle, node);
+ sched_yield();
+ continue;
+ }
+ cnt = 0;
+ node->max_events = max_io;
+ list_add_tail(_info, node);
+ do {
+ struct iovec *iov = NULL;
+ unsigned int iovcnt;
+ if (tsecs) {
+ ret = clock_gettime(CLOCK_MONOTONIC, &ts_cur);
+ timespec_sub(&ts_cur, &g_ts_start);
+ if (ts_cur.tv_sec >= tsecs) {
+ node->max_events = cnt;
+ break;
+ }
+ }
+
+ if (((_info->num_req_submitted - _info->num_req_completed) *
+ num_desc) > max_reqs) {
+ sched_yield();
+ continue;
+ }
+
+ io_list[0] = dma_memalloc(&iocbhandle, 1);
+ if (io_list[0] == NULL) {
+ if (cnt) {
+ node->max_events = cnt;
+ break;
+ }
+ else {
+ sched_yield();
+ continue;
+ }
+ }
+ iov = (struct iovec *)(io_list[0] + 1);
+ for (iovcnt = 0; iovcnt < burst_cnt; iovcnt++) {
+ iov[iovcnt].iov_base = dma_memalloc(&datahandle, 1);
+ if (iov[iovcnt].iov_base == NULL)
+ break;
+ iov[iovcnt].iov_len = io_sz;
+ }
+ if (iovcnt == 0) {
+ dma_free(&iocbhandle, io_list[0]);
+ continue;
+ }
+ if (_info->dir == Q_DIR_H2C) {
+ io_prep_pwritev(io_list[0],
+ _info->fd,
+ iov,
+ iovcnt,
+ 0);
+ } else {
+ io_prep_preadv(io_list[0],
+ _info->fd,
+ iov,
+ iovcnt,
+ 0);
+ }
+
+ ret = io_submit(node->ctxt, 1, io_list);
+ if(ret != 1) {
+ printf("Error: io_submit error:%d on %s for %u\n", ret, _info->q_name, cnt);
+ for (; iovcnt > 0; iovcnt--)
+ dma_free(&datahandle, iov[iovcnt].iov_base);
+ dma_free(&iocbhandle, io_list[0]);
+ node->max_events = cnt;
+ break;
+ } else {
+ cnt++;
+ _info->num_req_submitted += iovcnt;
+ }
+ } while (tsecs && !force_exit && (cnt < max_io));
+ } while (tsecs && !force_exit);
+
+ io_proc_cleanup(_info);
+
+ return NULL;
+}
+
+static void prep_q_add(struct io_info *_info)
+{
+ memset(_info->q_add, '\0', Q_ADD_CMD_LEN);
+ snprintf(_info->q_add, Q_ADD_CMD_LEN, "dmactl qdma%05x q add idx %d mode %s dir %s",
+ _info->pf, _info->qid, (_info->mode == Q_MODE_MM) ? "mm" : "st",
+ (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h");
+}
+
+static void prep_q_start(struct io_info *_info)
+{
+ int nwr;
+ memset(_info->q_start, '\0', Q_START_CMD_LEN);
+ if ((_info->dir == Q_DIR_C2H) && (_info->mode == Q_MODE_ST)) {
+ nwr = snprintf(_info->q_start, Q_START_CMD_LEN, "dmactl qdma%05x q start idx %d dir %s idx_ringsz %d cmptsz %u idx_tmr %d idx_cntr %d trigmode %s",
+ _info->pf, _info->qid, (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h",
+ _info->idx_rngsz, _info->cmptsz, _info->idx_tmr, _info->idx_cnt, _info->trig_mode);
+ if (_info->pfetch_en)
+ nwr = sprintf(_info->q_start + nwr, " pfetch_en");
+ } else {
+ snprintf(_info->q_start, Q_START_CMD_LEN, "dmactl qdma%05x q start idx %d dir %s idx_ringsz %d",
+ _info->pf, _info->qid, (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h",
+ _info->idx_rngsz);
+ }
+}
+
+static void prep_q_stop(struct io_info *_info)
+{
+ memset(_info->q_stop, '\0', Q_STOP_CMD_LEN);
+ snprintf(_info->q_stop, Q_STOP_CMD_LEN, "dmactl qdma%05x q stop idx %d dir %s",
+ _info->pf, _info->qid, (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h");
+}
+
+static void prep_q_del(struct io_info *_info)
+{
+ memset(_info->q_del, '\0', Q_DEL_CMD_LEN);
+ snprintf(_info->q_del, Q_DEL_CMD_LEN, "dmactl qdma%05x q del idx %d dir %s",
+ _info->pf, _info->qid, (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h");
+}
+
+static void prep_q_dump(struct io_info *_info)
+{
+ memset(_info->q_dump, '\0', Q_DEL_CMD_LEN);
+ snprintf(_info->q_dump, Q_DEL_CMD_LEN, "dmactl qdma%05x q dump idx %d dir %s",
+ _info->pf, _info->qid, (_info->dir == Q_DIR_H2C) ? "h2c" : "c2h");
+}
+
+static void prep_reg_dump(void)
+{
+ memset(reg_dump, '\0', REG_DUMP_CMD_LEN);
+ snprintf(reg_dump, REG_DUMP_CMD_LEN, "dmactl qdma%02x%02x%01x reg dump", pci_bus, pci_dev, pf_start);
+}
+
+static void prep_pci_dump(void)
+{
+ memset(pci_dump, '\0', PCI_DUMP_CMD_LEN);
+ snprintf(pci_dump, PCI_DUMP_CMD_LEN, "lspci -s %02x:%02x.%01x -vvv", pci_bus, pci_dev, pf_start);
+}
+
+static int setup_thrd_env(struct io_info *_info, unsigned char is_new_fd)
+{
+ int s;
+ int last_fd = -1;
+
+ char node[25] = {'\0'};
+ char thrd_name[50] = {'\0'};
+
+ prep_q_add(_info);
+ prep_q_start(_info);
+ prep_q_stop(_info);
+ prep_q_del(_info);
+ prep_q_dump(_info);
+
+ /* add queue */
+ printf("%s\n", _info->q_add);
+ s = system(_info->q_add);
+ if (s != 0) {
+ exit(1);
+ }
+ _info->q_added++;
+
+ /* start queue */
+ printf("%s\n", _info->q_start);
+ s= system(_info->q_start);
+ if (s != 0) {
+ exit(1);
+ }
+ _info->q_started++;
+
+ if (is_new_fd) {
+ snprintf(node, 25, "/dev/%s", _info->q_name);
+ _info->fd = open(node, O_RDWR);
+ if (_info->fd < 0) {
+ printf("Error: Cannot find %s\n", node);
+ exit(1);
+ }
+ }
+
+ s = ioctl(_info->fd, 0, &no_memcpy);
+ if (s != 0) {
+ printf("failed to set non memcpy\n");
+ exit(1);
+ }
+
+ return _info->fd;
+}
+
+static void dump_result(unsigned long long total_io_sz)
+{
+ unsigned long long gig_div = ((unsigned long long)tsecs * 1000000000);
+ unsigned long long meg_div = ((unsigned long long)tsecs * 1000000);
+ unsigned long long kil_div = ((unsigned long long)tsecs * 1000);
+ unsigned long long byt_div = ((unsigned long long)tsecs);
+
+ if ((total_io_sz/gig_div)) {
+ printf("BW = %llu.%llu GB/sec\n", (total_io_sz/gig_div),
+ ((total_io_sz % gig_div)/meg_div));
+ } else if ((total_io_sz/meg_div)) {
+ printf("BW = %llu.%llu MB/sec\n", (total_io_sz/meg_div),
+ ((total_io_sz % meg_div)/kil_div));
+ } else if ((total_io_sz/kil_div)) {
+ printf("BW = %llu.%llu KB/sec\n", (total_io_sz/kil_div),
+ ((total_io_sz % kil_div)/byt_div));
+ } else
+ printf("BW = %llu Bytes/sec\n", (total_io_sz/byt_div));
+}
+
+int is_valid_fd(int fd)
+{
+ return fcntl(fd, F_GETFL) != -1 || errno != EBADF;
+}
+
+static void cleanup(void)
+{
+ int i;
+ unsigned long long total_num_h2c_ios = 0;
+ unsigned long long total_num_c2h_ios = 0;
+ unsigned long long total_io_sz = 0;
+ int s;
+
+ if ((io_exit == 0)) {
+ printf("force exit: cleaning up\n");
+ force_exit = 1;
+ if (getpid() != base_pid) {
+ if (shmdt(info) == -1){
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ if ((shmdt(q_lst_stop) == -1)) {
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ } else
+ for (i = 1; i < num_thrds; i++)
+ wait(NULL);
+ }/* else
+ printf("normal exit: cleaning up\n");*/
+
+ if (getpid() != base_pid) return;
+
+ if (child_pid_lst != NULL)
+ free(child_pid_lst);
+ if (shmctl(q_lst_stop_mid, IPC_RMID, NULL) == -1) {
+ perror("shmctl returned -1\n");
+ error(-1, errno, " ");
+ }
+ if (info == NULL) return;
+
+ if (dump_en) {
+ s = system(pci_dump);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n", pci_dump, s);
+ }
+ s = system(reg_dump);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n", reg_dump, s);
+ }
+ }
+ for (i = 0; i < num_thrds; i++) {
+ if (info[i].q_ctrl && info[i].q_started) {
+ if (dump_en) {
+ s = system(info[i].q_dump);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n",
+ info[i].q_dump, s);
+ }
+ }
+ printf("%s\n", info[i].q_stop);
+ s = system(info[i].q_stop);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n",info[i].q_stop, s);
+ }
+ info[i].q_started = 0;
+ }
+ if ((info[i].q_ctrl != 0) && (info[i].fd > 0) && is_valid_fd(info[i].fd))
+ close(info[i].fd);
+ if (info[i].q_ctrl && info[i].q_added) {
+ printf("%s\n", info[i].q_del);
+ s = system(info[i].q_del);
+ if (s != 0) {
+ printf("Failed: %s\nerrcode = %d\n",info[i].q_del, s);
+ }
+ info[i].q_added = 0;
+ }
+ }
+
+ /* accumulate the statistics */
+ for (i = 0; i < num_thrds; i++) {
+ if (info[i].dir == Q_DIR_H2C)
+ total_num_h2c_ios += info[i].num_req_completed;
+ else {
+ if (info[i].mode == Q_MODE_ST)
+ info[i].num_req_completed *= info[i].pkt_burst;
+ total_num_c2h_ios += info[i].num_req_completed;
+ }
+ }
+ if (shmdt(info) == -1){
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ if (shmctl(shmid, IPC_RMID, NULL) == -1) {
+ perror("shmctl returned -1\n");
+ error(-1, errno, " ");
+ }
+
+ if (tsecs == 0) tsecs = 1;
+ if (0 != total_num_h2c_ios) {
+ total_io_sz = (total_num_h2c_ios*pkt_sz);
+ printf("WRITE: total pps = %llu", total_num_h2c_ios/tsecs);
+ printf(" ");
+ dump_result(total_io_sz);
+ }
+ if (0 != total_num_c2h_ios) {
+ total_io_sz = (total_num_c2h_ios*pkt_sz);
+ printf("READ: total pps = %llu", total_num_c2h_ios/tsecs);
+ printf(" ");
+ dump_result(total_io_sz);
+ }
+ if ((0 == total_num_h2c_ios) && (0 == total_num_c2h_ios))
+ printf("No IOs happened\n");
+}
+
+int main(int argc, char *argv[])
+{
+ int cmd_opt;
+ char *cfg_fname = NULL;
+ unsigned int i, j;
+ int s;
+ int last_fd = -1;
+ unsigned int aio_max_nr = 0xFFFFFFFF;
+ char aio_max_nr_cmd[100] = {'\0'};
+ int pid;
+#if THREADS_SET_CPU_AFFINITY
+ cpu_set_t set;
+#endif
+
+ while ((cmd_opt = getopt_long(argc, argv, "vhxc:c:", long_opts,
+ NULL)) != -1) {
+ switch (cmd_opt) {
+ case 0:
+ /* long option */
+ break;
+ case 'c':
+ /* config file name */
+ cfg_fname = strdup(optarg);
+ break;
+ default:
+ usage(argv[0]);
+ exit(0);
+ break;
+ }
+ }
+ if (cfg_fname == NULL)
+ return 1;
+
+#if THREADS_SET_CPU_AFFINITY
+ num_processors = get_nprocs_conf();
+ CPU_ZERO(&set);
+#endif
+#if DATA_VALIDATION
+ for (i = 0; i < 2*1024; i++)
+ valid_data[i] = i;
+#endif
+ parse_config_file(cfg_fname);
+ atexit(cleanup);
+
+ snprintf(aio_max_nr_cmd, 100, "echo %u > /proc/sys/fs/aio-max-nr", aio_max_nr);
+ system(aio_max_nr_cmd);
+
+ printf("dmautils(%u) threads\n", num_thrds);
+ child_pid_lst = calloc(num_thrds, sizeof(int));
+ base_pid = getpid();
+ child_pid_lst[0] = base_pid;
+ for (i = 1; i < num_thrds; i++) {
+ if (getpid() == base_pid)
+ child_pid_lst[i] = fork();
+ else
+ break;
+ }
+ if ((info = (struct io_info *) shmat(shmid, NULL, 0)) == (struct io_info *) -1) {
+ perror("Process shmat returned NULL\n");
+ error(-1, errno, " ");
+ exit(1);
+ }
+ if ((q_lst_stop = (unsigned char *) shmat(q_lst_stop_mid, NULL, 0)) == (unsigned char *) -1) {
+ perror("Process shmat returned NULL\n");
+ error(-1, errno, " ");
+ exit(1);
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &g_ts_start);
+ if (getpid() == base_pid) {
+ io_thread(&info[0]);
+#if THREADS_SET_CPU_AFFINITY
+ for (j = 0; j < num_processors; j++) {
+ if (j != info[0].cpu)
+ CPU_SET(j, &set);
+ }
+ if (sched_setaffinity(base_pid, sizeof(set), &set) == -1)
+ printf("setaffinity for thrd%u failed\n", info[i].thread_id);
+#endif
+ for(i = 1; i < num_thrds; i++) {
+ waitpid(child_pid_lst[i], NULL, 0);
+ }
+ free(child_pid_lst);
+ child_pid_lst = NULL;
+ } else {
+ info[i].pid = getpid();
+#if THREADS_SET_CPU_AFFINITY
+ for (j = 0; j < num_processors; j++) {
+ if (j != info[i].cpu)
+ CPU_SET(j, &set);
+ }
+ if (sched_setaffinity(getpid(), sizeof(set), &set) == -1)
+ printf("setaffinity for thrd%u failed\n", info[i].thread_id);
+#endif
+ io_thread(&info[i - 1]);
+ if ((shmdt(info) == -1)) {
+ perror("shmdt returned -1\n");
+ error(-1, errno, " ");
+ }
+ }
+
+ io_exit = 1;
+ return 0;
+}
+
diff --git a/QDMA/linux-kernel/tools/libaio_engine.c b/QDMA/linux-kernel/tools/libaio_engine.c
new file mode 100644
index 0000000000000000000000000000000000000000..51d8ec5332a530fa69eaae16ec6d76a44ef1943e
--- /dev/null
+++ b/QDMA/linux-kernel/tools/libaio_engine.c
@@ -0,0 +1,249 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <ctype.h>
+
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <string.h>
+#include "aioutility.h"
+
+#include <libaio.h>
+
+#define MSEC2NSEC 1000000
+
+static void aio_completion_handler(io_context_t io_ctx, struct iocb *iocb, long res, long res2)
+{
+ struct aio_block *aioblock;
+ struct aio_job *aiojob;
+ struct io_jobs *iojob;
+
+ aioblock = get_aio_block(iocb);
+ if (NULL == aioblock) return;
+ aiojob = aioblock->parent;
+ iojob = aiojob->iojob;
+
+ /* Did the request complete? */
+ aiojob->mask_completed |= aioblock->mask;
+ if (0 == res2) {
+ ssize_t rc;
+ struct timespec ts_end;
+ long total_time = 0;
+ float result;
+ float avg_time = 0;
+
+ if (0 <= res) {
+ printf("%s iteration %d block %d completed successfully\n",
+ iojob->job_name, aioblock->jobitr, aioblock->block_num);
+ if (aiojob->mask_expected == aiojob->mask_completed) {
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_end);
+ timespec_sub(&ts_end, &(aiojob->ts_start));
+ total_time += ts_end.tv_nsec;
+ result = ((float)(iojob->numbytes * iojob->numitr))*1000/total_time;
+ printf("** %s Job, total time %ld nsec, for bytes = %u, BW = %fMB/s \n",
+ iojob->job_name, total_time,
+ (iojob->numbytes * iojob->numitr),
+ result);
+ close(aiojob->fd);
+ verify_read(iojob);
+ } else
+ printf("Expected mask = %llx vs complete mask = %llx\n", aiojob->mask_expected, aiojob->mask_completed);
+ } else
+ printf("Error: %s job itr %d block %d completed with error\n",
+ iojob->job_name, aioblock->jobitr, aioblock->block_num);
+ } else
+ printf("Error: %s job itr %d block %d failed with error %ld\n",
+ iojob->job_name, aioblock->jobitr,
+ aioblock->block_num, res2);
+
+ sem_post(&aioblock->lock); /* ready for destruction */
+}
+
+static bool create_naio_job(struct iocb **io_list, struct io_jobs *iojob)
+{
+ int fd;
+ unsigned int i;
+ struct aio_block *aioblock; /* one per block */
+ struct aio_job *aiojob; /* one per job */
+ ssize_t rc;
+ unsigned int blk_idx = 0;
+ unsigned int blk_num;
+
+ fd = open(iojob->node, O_RDWR);
+ if (0 > fd) {
+ printf("Error: Opening %s node\n", iojob->node);
+ return false;
+ }
+
+ aiojob = calloc(1, sizeof(struct aio_job));
+
+ iojob->aiojob = aiojob;
+ blk_num = get_io_job_num_blocks(iojob);
+ aiojob->mask_completed = 0;
+ aiojob->mask_expected = 0;
+ aiojob->iojob = iojob;
+
+ aiojob->aio_list = (void *)io_list;
+
+ for (i = 0; i < iojob->numitr; i++) {
+ unsigned int j;
+
+ for (j = 0; j < blk_num; j++) {
+ struct iocb *aio_entry;
+ unsigned int buf_off = (i * blk_num * iojob->block_size) + (j * iojob->block_size);
+ unsigned int io_numbytes = iojob->block_size;
+
+ if (j == (blk_num - 1) &&
+ (iojob->numbytes % iojob->block_size))
+ io_numbytes = (iojob->numbytes % iojob->block_size);
+
+ aioblock = calloc(1, sizeof(struct aio_block));
+ aioblock->parent = aiojob;
+ sem_init(&aioblock->lock, 0, 1); /* to track completion */
+ aioblock->jobitr = i;
+ aioblock->block_num = j;
+ io_list[blk_idx] = calloc(1, sizeof(struct iocb));
+ aio_entry = io_list[blk_idx];
+ aioblock->aio_entry = io_list[blk_idx];
+
+ if (0 == strncmp(iojob->iotype, "read", 5)) {
+ io_prep_pread(aio_entry,
+ fd,
+ iojob->buf + buf_off,
+ io_numbytes,
+ 0);
+ } else if (0 == strncmp(iojob->iotype, "write", 4)) {
+ io_prep_pwrite(aio_entry,
+ fd,
+ iojob->buf + buf_off,
+ io_numbytes,
+ 0);
+ } else {
+ printf("Error: Invalid IO operation specified for %s IO job\n",
+ iojob->job_name);
+ goto ret_false;
+ }
+
+ io_set_callback(aio_entry, aio_completion_handler);
+
+ aioblock->mask = (((__u64)1) << blk_idx);
+ aiojob->mask_expected |= aioblock->mask;
+ sem_wait(&aioblock->lock);
+ enqueue_aio_block(aiojob, aioblock);
+ blk_idx++;
+ }
+ /* since all IOs are submitted at once, start time shall almost be same */
+ rc = clock_gettime(CLOCK_MONOTONIC, &(aiojob->ts_start));
+ }
+ enqueue_aio_task(aiojob);
+ return true;
+ret_false:
+ if (aiojob) {
+ aiojob->mask_completed = aiojob->mask_expected;
+ cleanup_aiojob(aiojob, false);
+ }
+
+ return false;
+}
+
+void submit_naio_job(void *_iojob)
+{
+ struct io_jobs *iojob = (struct io_jobs *)_iojob;
+ struct iocb **io_list;
+ unsigned int numb_blks = get_io_job_num_blocks(iojob);
+ unsigned int total_jobs = (iojob->numitr * numb_blks);
+ struct io_event *events;
+ io_context_t io_ctxt;
+ int ret;
+ struct timespec ts_cur;
+ int i;
+
+ /* setup IO job env */
+ run_env_cmds(iojob->setup_tail);
+
+ io_list = (struct iocb **)calloc(total_jobs,
+ sizeof(struct iocb *));
+
+ events = calloc(total_jobs, sizeof(struct io_event));
+
+ if (false == create_naio_job(io_list, iojob)) {
+ printf("Error: Failed to create IO job %s\n", iojob->job_name);
+ run_env_cmds(iojob->cleanup_tail);
+ return;
+ }
+
+ memset(&io_ctxt, 0, sizeof(io_ctxt));
+ if(io_queue_init(total_jobs, &io_ctxt)!=0){//init
+ printf("Error: io_setup error\n");
+ return;
+ }
+ ret = io_submit(io_ctxt, total_jobs, io_list);
+ if(ret != total_jobs) {
+ printf("Error: io_submit error:%d\n", ret);
+ return;
+ }
+
+ ret = clock_gettime(CLOCK_MONOTONIC, &ts_cur);
+ for (i = 0; i < total_jobs; i++) {
+ if (ts_cur.tv_nsec > (ts_cur.tv_nsec + (50 * MSEC2NSEC))) {
+ ts_cur.tv_sec += 1;
+ }
+ ts_cur.tv_nsec += (50 * MSEC2NSEC); /*50 msec/io*/
+ }
+ if (io_getevents(io_ctxt,
+ total_jobs,
+ total_jobs,
+ events,
+ &ts_cur) == total_jobs) {
+ for (i = 0; i < total_jobs; i++) {
+ ((io_callback_t)(events[i].data))(io_ctxt,
+ events[i].obj,
+ events[i].res,
+ events[i].res2);
+ }
+ } else {
+ printf("AIO completion not received\n");
+ }
+ io_destroy(io_ctxt);
+ cleanup_aio_jobs(true);
+}
+
+void cancel_naio_job(void *_aiojob)
+{
+ struct aio_job *aiojob = (struct aio_job *)_aiojob;
+ int rc;
+ struct iocb *iocb;
+ struct aio_block *aioblock;
+
+ aioblock = aiojob->aio_block_head;
+ while (1) {
+ if (NULL == aioblock) break;
+ if (aioblock->mask != (aiojob->mask_expected &
+ aioblock->mask)) {
+ iocb = (struct iocb *)aioblock->aio_entry;
+
+ rc = io_cancel((io_context_t)aiojob->naio_ctxt,
+ iocb,
+ (struct io_event *)(aiojob->events));
+ if (0 != rc)
+ printf("Warning: %s job block %d itr %d not cancelled. Will wait until cancelled\n",
+ aiojob->iojob->job_name,
+ aioblock->block_num, aioblock->jobitr);
+ sem_post(&aioblock->lock); /* ready for destruction */
+ }
+ aioblock = aioblock->next;
+ }
+}
+
diff --git a/QDMA/linux-kernel/tools/posixaio_engine.c b/QDMA/linux-kernel/tools/posixaio_engine.c
new file mode 100644
index 0000000000000000000000000000000000000000..39d54c8dac15db04b72e6b00b91938df60ac84da
--- /dev/null
+++ b/QDMA/linux-kernel/tools/posixaio_engine.c
@@ -0,0 +1,224 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#define _XOPEN_SOURCE 600
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <signal.h>
+#include <string.h>
+
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <aio.h>
+#include "aioutility.h"
+
+
+static void aio_completion_handler( sigval_t sigval )
+{
+ struct aiocb *req;
+ struct io_jobs *iojob;
+ struct aio_block *aioblock;
+ struct aio_job *aiojob;
+ ssize_t ret;
+ int req_err;
+
+ aioblock = (struct aio_block *)(sigval.sival_ptr);
+ if (NULL == aioblock) return;
+ req = (struct aiocb *)aioblock->aio_entry;
+ if (NULL == aioblock->aio_entry) return;
+ aiojob = aioblock->parent;
+ iojob = aiojob->iojob;
+
+ /* Did the request complete? */
+ req_err = aio_error( req );
+ aiojob->mask_completed |= aioblock->mask;
+ if (0 == req_err) {
+ ssize_t rc;
+ struct timespec ts_end;
+ long total_time = 0;
+ float result;
+ float avg_time = 0;
+
+ ret = aio_return( req );
+
+ if (0 <= ret) {
+ printf("%s iteration %d block %d completed successfully\n",
+ iojob->job_name, aioblock->jobitr, aioblock->block_num);
+ if (aiojob->mask_expected == aiojob->mask_completed) {
+ rc = clock_gettime(CLOCK_MONOTONIC, &ts_end);
+ timespec_sub(&ts_end, &(aiojob->ts_start));
+ total_time += ts_end.tv_nsec;
+ result = ((float)(iojob->numbytes * iojob->numitr))*1000/total_time;
+ printf("** %s Job, total time %ld nsec, for bytes = %u, BW = %fMB/s \n",
+ iojob->job_name, total_time,
+ (iojob->numbytes * iojob->numitr),
+ result);
+ verify_read(iojob);
+ } else
+ printf("Expected mask = %llx vs complete mask = %llx\n", aiojob->mask_expected, aiojob->mask_completed);
+ } else
+ printf("Error: %s job itr %d block %d completed with error\n",
+ iojob->job_name, aioblock->jobitr, aioblock->block_num);
+ } else
+ printf("Error: %s job itr %d block %d failed with error %d\n",
+ iojob->job_name, aioblock->jobitr,
+ aioblock->block_num, req_err);
+
+ sem_post(&aioblock->lock); /* ready for destruction */
+ cleanup_aio_jobs(false);
+}
+
+static bool create_paio_job(struct aiocb **aio_list, struct io_jobs *iojob)
+{
+ int fd;
+ unsigned int i;
+ struct aio_block *aioblock; /* one per block */
+ struct aio_job *aiojob; /* one per job */
+ ssize_t rc;
+ unsigned int blk_idx = 0;
+ unsigned int blk_num;
+
+ fd = open(iojob->node, O_RDWR);
+ if (0 > fd) {
+ printf("Error: Opening %s node\n", iojob->node);
+ return false;
+ }
+ aiojob = calloc(1, sizeof(struct aio_job));
+
+ iojob->aiojob = aiojob;
+ blk_num = get_io_job_num_blocks(iojob);
+ aiojob->mask_completed = 0;
+ aiojob->mask_expected = 0;
+ aiojob->iojob = iojob;
+
+ aiojob->aio_list = (void *)aio_list;
+
+ for (i = 0; i < iojob->numitr; i++) {
+ unsigned int j;
+
+ for (j = 0; j < blk_num; j++) {
+ struct aiocb *aio_entry;
+ unsigned int buf_off = (i * blk_num * iojob->block_size) + (j * iojob->block_size);
+ unsigned int io_numbytes = iojob->block_size;
+
+ aioblock = calloc(1, sizeof(struct aio_block));
+ aioblock->parent = aiojob;
+ sem_init(&aioblock->lock, 0, 1); /* to track completion */
+ aioblock->jobitr = i;
+ aioblock->block_num = j;
+ aio_list[blk_idx] = calloc(1, sizeof(struct aiocb));
+ aio_entry = aio_list[blk_idx];
+
+ aioblock->aio_entry = aio_list[blk_idx];
+ /* Set up the AIO request */
+ aio_entry->aio_fildes = fd;
+ aio_entry->aio_buf = iojob->buf + buf_off;
+ aio_entry->aio_nbytes = io_numbytes;
+
+ /* Link the AIO request with a thread callback */
+ aio_entry->aio_sigevent.sigev_notify = SIGEV_THREAD;
+ aio_entry->aio_sigevent._sigev_un._sigev_thread._function = aio_completion_handler;
+ aio_entry->aio_sigevent._sigev_un._sigev_thread._attribute = NULL;
+ aio_entry->aio_sigevent.sigev_value.sival_ptr = aioblock;
+
+ if (0 == strncmp(iojob->iotype, "write", 5)) {
+ aio_entry->aio_lio_opcode = LIO_WRITE;
+ } else if (0 == strncmp(iojob->iotype, "read", 4)) {
+ aio_entry->aio_lio_opcode = LIO_READ;
+ } else {
+ printf("Error: Invalid IO operation specified for %s IO job\n",
+ iojob->job_name);
+ goto ret_false;
+ }
+ sem_wait(&aioblock->lock);
+ aioblock->mask = (((__u64)1) << blk_idx);
+ aiojob->mask_expected |= aioblock->mask;
+ enqueue_aio_block(aiojob, aioblock);
+ blk_idx++;
+ }
+ /* since all IOs are submitted at once, start time shall almost be same */
+ rc = clock_gettime(CLOCK_MONOTONIC, &(aiojob->ts_start));
+ }
+ enqueue_aio_task(aiojob);
+ return true;
+ret_false:
+ if (aiojob) {
+ aiojob->mask_completed = aiojob->mask_expected;
+ cleanup_aiojob(aiojob, false);
+ }
+
+ return false;
+}
+
+void submit_paio_job(void *_iojob)
+{
+ struct io_jobs *iojob = (struct io_jobs *)_iojob;
+ struct aiocb **aio_list;
+ unsigned int numb_blks = get_io_job_num_blocks(iojob);
+ unsigned int total_jobs = (iojob->numitr * numb_blks);
+
+ printf("setting up environment for %s\n", iojob->job_name);
+ /* setup IO job env */
+ run_env_cmds(iojob->setup_tail);
+
+ aio_list = (struct aiocb **)calloc(total_jobs,
+ sizeof(struct aiocb *));
+ if (false == create_paio_job(aio_list, iojob)) {
+ printf("\nError: Could not create %s IO job. Skipping.\n", iojob->job_name);
+ run_env_cmds(iojob->cleanup_tail);
+ }
+ else {
+ int s = 0;
+ int i;
+
+ /* submit all jobs at once */
+ s = lio_listio(LIO_WAIT, aio_list, total_jobs, NULL);
+ if (0 <= s)
+ printf("\nSuccessfully submitted %s IO job.\n", iojob->job_name);
+ else {
+ printf("\nError: Could not submit %s IO job.\n",
+ iojob->job_name);
+ for (i = 0; i < total_jobs; i++)
+ printf(" block %d - error %d.\n",
+ i, aio_error(aio_list[i]));
+ free(aio_list);
+ aio_list = NULL;
+ }
+ }
+}
+
+void cancel_paio_job(void *_aiojob)
+{
+ struct aio_job *aiojob = (struct aio_job *)_aiojob;
+ int rc;
+ struct aiocb *iocb;
+ struct aio_block *aioblock;
+
+ aioblock = aiojob->aio_block_head;
+ while (1) {
+ if (NULL == aioblock) break;
+ if (aioblock->mask !=
+ (aiojob->mask_expected &
+ aioblock->mask)) {
+ iocb = (struct aiocb *)aioblock->aio_entry;
+
+ rc = aio_cancel(aiojob->fd, iocb);
+ if (AIO_NOTCANCELED == rc)
+ printf("Warning: %s job block %d itr %d not cancelled. Will wait until cancelled\n",
+ aiojob->iojob->job_name,
+ aioblock->block_num, aioblock->jobitr);
+ }
+ aioblock = aioblock->next;
+ }
+}
diff --git a/QDMA/linux-kernel/user/Makefile b/QDMA/linux-kernel/user/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..5a3ceb9ce32a93d39bc58e8c2bbde3d4ad0dbf20
--- /dev/null
+++ b/QDMA/linux-kernel/user/Makefile
@@ -0,0 +1,19 @@
+SHELL = /bin/bash
+
+CFLAGS += -g
+#CFLAGS += -O2 -fno-inline -Wall -Wstrict-prototypes
+CFLAGS += -I. -I../include
+CFLAGS += $(EXTRA_FLAGS)
+
+DMACTL = dmactl
+DMACTL_OBJS := $(patsubst %.c,%.o,$(wildcard cli/*.c))
+
+all: clean dmactl
+
+dmactl: $(DMACTL_OBJS)
+ @mkdir -p -m 755 $(build_dir)
+ $(CC) $^ -o $(DMACTL)
+ @cp -f $(DMACTL) $(build_dir)
+
+clean:
+ @rm -f *.o */*.o $(DMACTL) $(build_dir)/$(DMACTL)*
diff --git a/QDMA/linux-kernel/user/bash/decode_hw_ctxt.sh b/QDMA/linux-kernel/user/bash/decode_hw_ctxt.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e82ebc5306f2e670ae24efcdb397c067953cc3ec
--- /dev/null
+++ b/QDMA/linux-kernel/user/bash/decode_hw_ctxt.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+ echo "$0: <w1> <w0>"
+ exit
+fi
+
+w1=$1
+w0=$2
+
+let "w0 = $w0 + 0"
+let "w1 = $w1 + 0"
+
+printf '\n0x%08x 0x%08x\n\n' $w1 $w0
+
+printf '\nW1 0x%08x:\n' $w1
+
+let "v = ($w1 >> 15) & 0x1"
+printf '[47] (W1[15]) rsvd 0x%x\n' $v
+
+let "v = ($w1 >> 11) & 0xF"
+printf '[46:43] (W1[14:11]) fetch_pnd 0x%x\n' $v
+
+let "v = ($w1 >> 10) & 0x1"
+printf '[42] (W1[10]) evt_pnd 0x%x\n' $v
+
+let "v = ($w1 >> 9) & 0x1"
+printf '[41] (W1[9]) idl_stp_b 0x%x\n' $v
+
+let "v = ($w1 >> 8) & 0x1"
+printf '[40] (W1[8]) dsc_pnd 0x%x\n' $v
+
+let "v = $w1 & 0xFF"
+printf '[39:32] (W1[7:0]) rsvd 0x%x\n' $v
+
+printf '\nW0 0x%08x:\n' $w0
+
+let "v = ($w0 >> 16) & 0xFFFF"
+printf '[31:16] (W0[31:16] crd_use 0x%x\n' $v
+
+let "v = $w0 & 0xFFFF"
+printf '[15:0] (W0[15:0] cidx 0x%x\n' $v
diff --git a/QDMA/linux-kernel/user/bash/decode_preftch_ctxt.sh b/QDMA/linux-kernel/user/bash/decode_preftch_ctxt.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5a2cf5d589d4bea72d2f08791ec3e1a8059f9371
--- /dev/null
+++ b/QDMA/linux-kernel/user/bash/decode_preftch_ctxt.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+if [ $# -lt 2 ]; then
+ echo "$0: <w1> <w0>"
+ exit
+fi
+
+w1=$1
+w0=$2
+
+let "w0 = $w0 + 0"
+let "w1 = $w1 + 0"
+
+printf '\n0x%08x 0x%08x\n\n' $w1 $w0
+
+let "v = ($w1 >> 13) & 0x1"
+printf '[45] (W1[13]) valid 0x%x\n' $v
+
+let "v = $w1 & 0x1FFF"
+let "v1 = ($w0 >> 29) & 0x7"
+let "v2 = ($v << 3) | $v1"
+printf '[44:29] (W1[12:0] W0[31:29]) sw_crdt 0x%x\n' $v2
+
+let "v = ($w0 >> 28) & 0x1"
+printf '[28] (W0[28] q_is_in_pftch 0x%x\n' $v
+
+let "v = ($w0 >> 27) & 0x1"
+printf '[27] (W0[27] en_pftch 0x%x\n' $v
+
+let "v = ($w0 >> 8) & 0x3FFFF"
+printf '[25:8] (W0[25:8] rsvd 0x%x\n' $v
+
+let "v = ($w0 >> 5) & 0x7"
+printf '[7:5] (W0[7:5] port_id 0x%x\n' $v
+
+let "v = ($w0 >> 1) & 0xF"
+printf '[4:1] (W0[4:1] buf_size_idx 0x%x\n' $v
+
+let "v = $w0 & 0x1"
+printf '[0] (W0[0] bypass 0x%x\n' $v
diff --git a/QDMA/linux-kernel/user/bash/decode_sw_ctxt.sh b/QDMA/linux-kernel/user/bash/decode_sw_ctxt.sh
new file mode 100755
index 0000000000000000000000000000000000000000..028c2bc6b21809a439c8e1589c0f7d807bd87b6a
--- /dev/null
+++ b/QDMA/linux-kernel/user/bash/decode_sw_ctxt.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+if [ $# -lt 5 ]; then
+ echo "$0: <w4> <w3> <w2> <w1> <w0>"
+ exit
+fi
+
+w4=$1
+w3=$2
+w2=$3
+w1=$4
+w0=$5
+
+let "w0 = $w0 + 0"
+let "w1 = $w1 + 0"
+let "w2 = $w2 + 0"
+let "w3 = $w3 + 0"
+let "w4 = $w4 + 0"
+
+printf '\n0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n\n' $w4 $w3 $w2 $w1 $w0
+
+printf '\nW4 0x%08x:\n' $w4
+
+let "v = ($w4 >> 11) & 0x1"
+printf '[139] (W4[11]) int_aggr 0x%x\n' $v
+
+let "v = ($w4 >> 0) & 0x7FF"
+printf '[138:128] (W3[10:0] dsc_base_h 0x%x\n' $v
+
+printf '\n[127:64] (W3,W2) dsc_base 0x%08x%08x\n' $w3 $w2
+
+printf '\nW1 0x%08x:\n' $w1
+
+let "v = ($w1 >> 31) & 0x1"
+printf '[63] (W1[31]) is_mm 0x%x\n' $v
+
+let "v = ($w1 >> 30) & 0x1"
+printf '[62] (W1[30]) mrkr_dis 0x%x\n' $v
+
+let "v = ($w1 >> 29) & 0x1"
+printf '[61] (W1[29]) irq_req 0x%x\n' $v
+
+let "v = ($w1 >> 28) & 0x1"
+printf '[60] (W1[28]) err_cmpl_status_sent 0x%x\n' $v
+
+let "v = ($w1 >> 26) & 0x3"
+printf '[59:58] (W1[27:26] err 0x%x\n' $v
+
+let "v = ($w1 >> 25) & 0x1"
+printf '[57] (W1[25]) irq_no_last 0x%x\n' $v
+
+let "v = ($w1 >> 22) & 0x7"
+printf '[56:54] (W1[23:22] port_id 0x%x\n' $v
+
+let "v = ($w1 >> 21) & 0x1"
+printf '[53] (W1[21]) irq_en 0x%x\n' $v
+
+let "v = ($w1 >> 20) & 0x1"
+printf '[52] (W1[20]) cmpl_status_en 0x%x\n' $v
+
+let "v = ($w1 >> 19) & 0x1"
+printf '[51] (W1[19]) mm_chn 0x%x\n' $v
+
+let "v = ($w1 >> 18) & 0x1"
+printf '[50] (W1[18]) byp 0x%x\n' $v
+
+let "v = ($w1 >> 16) & 0x3"
+printf '[49:48] (W1[17:16] dsc_sz 0x%x\n' $v
+
+let "v = ($w1 >> 12) & 0xF"
+printf '[47:44] (W1[15:12] rng_sz 0x%x\n' $v
+
+let "v = ($w1 >> 8) & 0xF"
+printf '[43:40] (W1[11:8] rsvd 0x%x\n' $v
+
+let "v = ($w1 >> 5) & 0x7"
+printf '[39:37] (W1[7:5] fetch_max 0x%x\n' $v
+
+let "v = ($w1 >> 4) & 0x1"
+printf '[36] (W1[4] at 0x%x\n' $v
+
+let "v = ($w1 >> 3) & 0x1"
+printf '[35] (W1[3]) cmpl_status_acc_en 0x%x\n' $v
+
+let "v = ($w1 >> 2) & 0x1"
+printf '[34] (W1[2]) cmpl_status_pend_chk 0x%x\n' $v
+
+let "v = ($w1 >> 1) & 0x1"
+printf '[33] (W1[1]) fcrd_en 0x%x\n' $v
+
+let "v = $w1 & 0x1"
+printf '[32] (W1[0]) qen 0x%x\n' $v
+
+printf '\nW0 0x%08x:\n' $w0
+
+let "v = ($w0 >> 25) & 0x7F"
+printf '[31:25] (W0[31:25] reserved 0x%x\n' $v
+
+let "v = ($w0 >> 17) & 0xFF"
+printf '[24:17] (W0[24:17] fnc_id 0x%x\n' $v
+
+let "v = ($w0 >> 16) & 0x1"
+printf '[16] (W0[16]) irq_arm 0x%x\n' $v
+
+let "v = $w0 & 0xFF"
+printf '[15:0] (W0[15:0] pidx 0x%x\n' $v
diff --git a/QDMA/linux-kernel/user/bash/decode_wrb_ctxt.sh b/QDMA/linux-kernel/user/bash/decode_wrb_ctxt.sh
new file mode 100755
index 0000000000000000000000000000000000000000..763a9433383b3fcfed5c50bb982f1dbd092a0771
--- /dev/null
+++ b/QDMA/linux-kernel/user/bash/decode_wrb_ctxt.sh
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+if [ $# -lt 4 ]; then
+ echo "$0: <w3> <w2> <w1> <w0>"
+ exit
+fi
+
+w4=$1
+w3=$2
+w2=$3
+w1=$4
+w0=$5
+
+let "w0 = $w0 + 0"
+let "w1 = $w1 + 0"
+let "w2 = $w2 + 0"
+let "w3 = $w3 + 0"
+let "w4 = $w4 + 0"
+
+printf '\n0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n\n' $w4 $w3 $w2 $w1 $w0
+
+let "v = ($w4 >> 3) & 0x1"
+printf '[131] (W4[3]) at 0x%x\n' $v
+
+let "v = ($w4 >> 2) & 0x1"
+printf '[130] (W4[2]) ovf_chk_dis 0x%x\n' $v
+
+let "v = ($w4 >> 1) & 0x1"
+printf '[129] (W4[1]) full_upd 0x%x\n' $v
+
+let "v = ($w4 >> 0) & 0x1"
+printf '[128] (W4[0]) tmr_runnig 0x%x\n' $v
+
+let "v = ($w3 >> 31) & 0x1"
+printf '[127] (W3[31]) user_trig_pend 0x%x\n' $v
+
+let "v = ($w3 >> 29) & 0xFFFF"
+printf '[126:125] (W3[30:29]) err 0x%x\n' $v
+
+let "v = ($w3 >> 28) & 0x1"
+printf '[124] (W3[28]) valid 0x%x\n' $v
+
+let "v = ($w3 >> 12) & 0xFFFF"
+printf '[123:108] (W3[27:12]) cidx 0x%x\n' $v
+
+let "v = $w3 & 0xFFF"
+let "v1 = ($w2 >> 28) & 0xF"
+let "v2 = ($v << 4) | $v1"
+printf '[107:92] (W3[11:0] W2[31:28]) pidx 0x%x\n' $v2
+
+let "v = ($w2 >> 26) & 0x3"
+printf '[91:90] (W2[27:26]) desc_size 0x%x\n' $v
+
+let "v2 = $w2 & 0x3FFFFFF"
+let "v1 = ($w1 >> 6) & 0x3FFFFFF"
+printf '[89:38] (W2[25:0] W1[31:6]) baddr_64 '
+printf '0x%08x ' $v2
+printf '0x%08x \n' $v1
+
+let "v = ($w1 >> 0) & 0x3F"
+printf '[37:32] (W0[5:0]) rsvd 0x%x\n' $v
+
+let "v = ($w0 >> 28) & 0xF"
+printf '[31:28] (W0[31:28]) rng_sz 0x%x\n' $v
+
+let "v = ($w0 >> 27) & 0x1"
+printf '[27] (W0[27]) color 0x%x\n' $v
+
+let "v = ($w0 >> 25) & 0x3"
+printf '[26:25] (W0[26:25]) int_st 0x%x\n' $v
+
+let "v = ($w0 >> 21) & 0xF"
+printf '[24:21] (W0[24:21]) timer_idx 0x%x\n' $v
+
+let "v = ($w0 >> 17) & 0xF"
+printf '[20:17] (W0[20:17]) counter_idx 0x%x\n' $v
+
+let "v = ($w0 >> 13) & 0xF"
+printf '[16:13] (W0[16:`3]) rsvd 0x%x\n' $v
+
+let "v = ($w0 >> 5) & 0xF"
+printf '[12:5] (W0[12:5]) fnc_id 0x%x\n' $v
+
+let "v = ($w0 >> 2) & 0x7"
+printf '[4:2] (W0[4:2]) trig_mode 0x%x\n' $v
+
+let "v = ($w0 >> 1) & 0x1"
+printf '[1] (W0[1]) en_int 0x%x\n' $v
+
+let "v = $w0 & 0x1"
+printf '[0] (W0[0]) en_stat_desc 0x%x\n' $v
diff --git a/QDMA/linux-kernel/user/bash/qdma_run_test_st_vf.sh b/QDMA/linux-kernel/user/bash/qdma_run_test_st_vf.sh
new file mode 100755
index 0000000000000000000000000000000000000000..88a95abecee1ffbbc0c06e63791b210eac8e3579
--- /dev/null
+++ b/QDMA/linux-kernel/user/bash/qdma_run_test_st_vf.sh
@@ -0,0 +1,198 @@
+#!/bin/bash
+
+if [ -z $1 ] || [ $1 == '-h' ]
+then
+ echo "ERROR: Invalid Command Line"
+ echo "Example: run_st_c2h_vf.sh pfn vfn size(in byte) num_pkt iteration"
+ exit
+fi
+
+################################################
+# User Configurable Parameters
+################################################
+
+pfn=$1 # PF number
+vfn=$2 # VF number
+size=$3 # Size per payload packet
+num_pkt=$4 # number of payload packet
+iteration=$5 # iterations of C2H tests
+num_qs=4
+logfile="loopback$1_$2.log"
+infile='/root/Desktop/datafile_16bit_pattern.bin'
+
+if [ -z $iteration ]; then
+ iteration=1
+fi
+
+if [ ! -z $size ]; then
+ f_size=1
+else
+ f_size=0
+fi
+
+#################################################
+# Helper Functions
+################################################
+
+usr_bar=1
+vf=0
+host_adr_high=4094
+mem_div=16384
+s_high=$mem_div
+
+function randomize_tx_params() {
+ #random host address between 0 to 3094
+ hst_adr1=$random
+ let "hst_adr1 %= $host_adr_high"
+ hst_adr1=0;
+ # byte size
+ f1=$random
+ let "f1 %= 2"
+ if [ $f1 -eq 0 ]; then
+ size=$(( ($random %64) +1 )) ## for random number between 1 and 64
+ else
+ size=$(( ($random<<15)|($random+1) ))
+ fi
+ let "size %= $s_high"
+ echo "size = $size"
+ even=$size
+ let "even %= 2"
+ if [ $even -eq 1 ];then
+ let "size = $size+1"
+ fi
+
+ num_pkt=$(( ($RANDOM % 9) +1))
+ echo " num_pkt = $num_pkt"
+
+}
+
+function queue_start() {
+ echo "---- Queue Start $2 ----"
+ dmactl qdma$1 q add idx $2 mode $3 dir h2c
+ dmactl qdma$1 q start idx $2 dir h2c mode $3
+ dmactl qdma$1 q add idx $2 mode $3 dir c2h
+ dmactl qdma$1 q start idx $2 dir c2h mode $3
+}
+
+function cleanup_queue() {
+ echo "---- Queue Clean up $2 ----"
+ dmactl qdma$1 q stop idx $2 dir h2c mode $3
+ dmactl qdma$1 q del idx $2 dir h2c mode $3
+ dmactl qdma$1 q stop idx $2 dir c2h mode $3
+ dmactl qdma$1 q del idx $2 dir c2h mode $3
+}
+
+
+echo "############################# AXI-ST Start #################################"
+
+
+for ((i=0; i< $num_qs; i++)) do
+ # Setup for Queues
+ qid=$i
+ hw_qid=$(($qid + $(($vfn*8)) + 128))
+ dev_st_c2h="/dev/qdmavf0-ST-C2H-$qid"
+ dev_st_h2c="/dev/qdmavf0-ST-H2C-$qid"
+ out="out$qid"
+ loop=1
+
+ # Open the Queue for AXI-ST streaming interface.
+ queue_start vf$vf $qid st > /dev/null
+
+ while [ "$loop" -le $iteration ]
+ do
+ # Determine if DMA is targeted @ random host address
+ if [ $f_size -eq 1 ]; then
+ hst_adr1=0
+ else
+ randomize_tx_params
+ fi
+
+ # if more packets are requested.
+ let "tsize= $size*$num_pkt"
+
+ echo ""
+ echo "########################################################################################"
+ echo "############# H2C ST LOOP $loop : dev=$dev_st_h2c pfn=$pfn vfn=$vfn qid=$qid hw_qid=$hw_qid"
+ echo "############# transfer_size=$tsize pkt_size=$size pkt_count=$num_pkt hst_adr=$hst_adr1"
+ echo "########################################################################################"
+
+ #clear match bit before each H2C ST transfer
+ dmactl qdmavf0 reg write bar $usr_bar 0x0c 0x01
+
+
+ # H2C transfer
+ dma_to_device -d $dev_st_h2c -f $infile -s $tsize &
+ re=$?
+
+ wait
+
+ # Check match bit and QID
+ hwqid_match=$(dmactl qdmavf0 reg read bar $usr_bar 0x10 | grep -o '0x[0-9][0-9][0-9]')
+ hw_qid_hex=$(printf '%x' $hw_qid)
+ if [ $hwqid_match != 0x$hw_qid_hex'1' ]; then
+ echo "#### ERROR: QID MATCH is $hwqid_match"
+ re=-1
+ fi
+
+ if [ $re == 0 ]; then
+ echo "######################################################"
+ echo "############## VF H2C ST PASS QID $qid ################"
+ echo "######################################################"
+ else
+ echo "#### ERROR: VF H2C ST FAIL"
+ fi
+
+ echo ""
+ echo "########################################################################################"
+ echo "############# C2H ST LOOP $loop : dev=$dev_st_c2h pfn=$pfn vfn=$vfn qid=$qid hw_qid=$hw_qid"
+ echo "############# transfer_size=$tsize pkt_size=$size pkt_count=$num_pkt hst_adr=$hst_adr1"
+ echo "########################################################################################"
+
+ dmactl qdmavf0 reg write bar $usr_bar 0x0 $hw_qid # for Queue 0
+ dmactl qdmavf0 reg write bar $usr_bar 0x4 $size
+ dmactl qdmavf0 reg write bar $usr_bar 0x20 $num_pkt #number of packets
+ dmactl qdmavf0 reg write bar $usr_bar 0x08 2 # Must set C2H start before starting transfer
+
+ dma_from_device -d $dev_st_c2h -f $out -o $hst_adr1 -s $tsize &
+
+ wait
+
+ #Check if files is there.
+ if [ ! -f $out ]; then
+ echo " #### ERROR: Queue $qid output file does not exists ####"
+ echo " #### ERROR: Queue $qid output file does not exists ####" >> $logfile
+ cleanup_queue vf$vf $qid st
+ exit -1
+ fi
+
+ # check files size
+ filesize=$(stat -c%s "$out")
+ if [ $filesize -gt $tsize ]; then
+ echo "#### ERROR: Queue $qid output file size does not match, filesize= $filesize ####"
+ echo "#### ERROR: Queue $qid output file size does not match, filesize= $filesize ####" >> $logfile
+ cleanup_queue vf$vf $qid st
+ exit -1
+ fi
+
+ #compare file
+ cmp $out $infile -n $tsize
+ if [ $? -eq 1 ]; then
+ echo "#### Test ERROR. Queue $qid data did not match ####"
+ echo "#### Test ERROR. Queue $qid data did not match ####" >> $logfile
+ dmactl qdmavf0 q dump idx $qid mode st dir c2h
+ dmactl qdmavf0 reg dump
+ cleanup_queue vf$vf $qid st
+ exit -1
+ else
+ echo "######################################################"
+ echo "############## VF C2H ST PASS QID $qid ################"
+ echo "######################################################"
+ fi
+ wait
+ ((loop++))
+ done
+ cleanup_queue vf$vf $qid st > /dev/null
+done
+echo "########################## AXI-ST completed ###################################"
+exit 0
+
diff --git a/QDMA/linux-kernel/user/cli/cmd_parse.c b/QDMA/linux-kernel/user/cli/cmd_parse.c
new file mode 100644
index 0000000000000000000000000000000000000000..88d123d259615003a0ab5bde02eb5a5858268702
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/cmd_parse.c
@@ -0,0 +1,1201 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cmd_parse.h"
+#include "nl_user.h"
+#include "version.h"
+
+static int read_range(int argc, char *argv[], int i, unsigned int *v1,
+ unsigned int *v2);
+
+static const char *progname;
+
+#define Q_ADD_ATTR_IGNORE_MASK ~((1 << QPARM_IDX) | \
+ (1 << QPARM_MODE) | \
+ (1 << QPARM_DIR))
+#define Q_START_ATTR_IGNORE_MASK ((1 << QPARM_MODE) | \
+ (1 << QPARM_DESC) | \
+ (1 << QPARM_CMPT))
+#define Q_STOP_ATTR_IGNORE_MASK ~((1 << QPARM_IDX) | \
+ (1 << QPARM_DIR))
+#define Q_DEL_ATTR_IGNORE_MASK ~((1 << QPARM_IDX) | \
+ (1 << QPARM_DIR))
+#define Q_DUMP_ATTR_IGNORE_MASK ~((1 << QPARM_IDX) | \
+ (1 << QPARM_DIR) | \
+ (1 << QPARM_DESC) | \
+ (1 << QPARM_CMPT))
+#define Q_DUMP_PKT_ATTR_IGNORE_MASK ~((1 << QPARM_IDX) | \
+ (1 << QPARM_DIR))
+#define Q_H2C_ATTR_IGNORE_MASK ((1 << QPARM_C2H_BUFSZ_IDX) | \
+ (1 << QPARM_CMPT_TMR_IDX) | \
+ (1 << QPARM_CMPT_CNTR_IDX) | \
+ (1 << QPARM_CMPT_TRIG_MODE) | \
+ (1 << QPARM_CMPTSZ))
+
+#define Q_ADD_FLAG_IGNORE_MASK ~(XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM | \
+ XNL_F_QDIR_BOTH)
+#define Q_START_FLAG_IGNORE_MASK (XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM)
+#define Q_STOP_FLAG_IGNORE_MASK ~(XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM | \
+ XNL_F_QDIR_BOTH)
+#define Q_DEL_FLAG_IGNORE_MASK ~(XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM | \
+ XNL_F_QDIR_BOTH)
+#define Q_DUMP_FLAG_IGNORE_MASK ~(XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM | \
+ XNL_F_QDIR_BOTH)
+#define Q_DUMP_PKT_FLAG_IGNORE_MASK ~(XNL_F_QMODE_ST | \
+ XNL_F_QMODE_MM | \
+ XNL_F_QDIR_C2H)
+#define Q_H2C_FLAG_IGNORE_MASK (XNL_F_C2H_CMPL_INTR_EN | \
+ XNL_F_CMPL_UDD_EN)
+
+#ifdef ERR_DEBUG
+char *qdma_err_str[qdma_errs] = {
+ "err_ram_sbe",
+ "err_ram_dbe",
+ "err_dsc",
+ "err_trq",
+ "err_h2c_mm_0",
+ "err_h2c_mm_1",
+ "err_c2h_mm_0",
+ "err_c2h_mm_1",
+ "err_c2h_st",
+ "ind_ctxt_cmd_err",
+ "err_bdg",
+ "err_h2c_st",
+ "poison",
+ "ur_ca",
+ "param",
+ "addr",
+ "tag",
+ "flr",
+ "timeout",
+ "dat_poison",
+ "flr_cancel",
+ "dma",
+ "dsc",
+ "rq_cancel",
+ "dbe",
+ "sbe",
+ "unmapped",
+ "qid_range",
+ "vf_access_err",
+ "tcp_timeout",
+ "mty_mismatch",
+ "len_mismatch",
+ "qid_mismatch",
+ "desc_rsp_err",
+ "eng_wpl_data_par_err",
+ "msi_int_fail",
+ "err_desc_cnt",
+ "portid_ctxt_mismatch",
+ "portid_byp_in_mismatch",
+ "cmpt_inv_q_err",
+ "cmpt_qfull_err",
+ "cmpt_cidx_err",
+ "cmpt_prty_err",
+ "fatal_mty_mismatch",
+ "fatal_len_mismatch",
+ "fatal_qid_mismatch",
+ "timer_fifo_ram_rdbe",
+ "fatal_eng_wpl_data_par_err",
+ "pfch_II_ram_rdbe",
+ "cmpt_ctxt_ram_rdbe",
+ "pfch_ctxt_ram_rdbe",
+ "desc_req_fifo_ram_rdbe",
+ "int_ctxt_ram_rdbe",
+ "cmpt_coal_data_ram_rdbe",
+ "tuser_fifo_ram_rdbe",
+ "qid_fifo_ram_rdbe",
+ "payload_fifo_ram_rdbe",
+ "wpl_data_par_err",
+ "zero_len_desc_err",
+ "csi_mop_err",
+ "no_dma_dsc_err",
+ "sb_mi_h2c0_dat",
+ "sb_mi_c2h0_dat",
+ "sb_h2c_rd_brg_dat",
+ "sb_h2c_wr_brg_dat",
+ "sb_c2h_rd_brg_dat",
+ "sb_c2h_wr_brg_dat",
+ "sb_func_map",
+ "sb_dsc_hw_ctxt",
+ "sb_dsc_crd_rcv",
+ "sb_dsc_sw_ctxt",
+ "sb_dsc_cpli",
+ "sb_dsc_cpld",
+ "sb_pasid_ctxt_ram",
+ "sb_timer_fifo_ram",
+ "sb_payload_fifo_ram",
+ "sb_qid_fifo_ram",
+ "sb_tuser_fifo_ram",
+ "sb_wrb_coal_data_ram",
+ "sb_int_qid2vec_ram",
+ "sb_int_ctxt_ram",
+ "sb_desc_req_fifo_ram",
+ "sb_pfch_ctxt_ram",
+ "sb_wrb_ctxt_ram",
+ "sb_pfch_ll_ram",
+ "sb_h2c_pend_fifo",
+ "db_mi_h2c0_dat",
+ "db_mi_c2h0_dat",
+ "db_h2c_rd_brg_dat",
+ "db_h2c_wr_brg_dat",
+ "db_c2h_rd_brg_dat",
+ "db_c2h_wr_brg_dat",
+ "db_func_map",
+ "db_dsc_hw_ctxt",
+ "db_dsc_crd_rcv",
+ "db_dsc_sw_ctxt",
+ "db_dsc_cpli",
+ "db_dsc_cpld",
+ "db_pasid_ctxt_ram",
+ "db_timer_fifo_ram",
+ "db_payload_fifo_ram",
+ "db_qid_fifo_ram",
+ "db_tuser_fifo_ram",
+ "db_wrb_coal_data_ram",
+ "db_int_qid2vec_ram",
+ "db_int_ctxt_ram",
+ "db_desc_req_fifo_ram",
+ "db_pfch_ctxt_ram",
+ "db_wrb_ctxt_ram",
+ "db_pfch_ll_ram",
+ "db_h2c_pend_fifo",
+};
+#endif
+
+static void __attribute__((noreturn)) usage(FILE *fp)
+{
+ fprintf(fp, "Usage: %s [dev|qdma[vf]<N>] [operation] \n", progname);
+ fprintf(fp, "\tdev [operation]: system wide FPGA operations\n");
+ fprintf(fp,
+ "\t\tlist list all qdma functions\n");
+ fprintf(fp,
+ "\tqdma[N] [operation]: per QDMA FPGA operations\n");
+ fprintf(fp,
+ "\t\tversion lists the Hardware and Software version\n"
+ "\t\tstat statistics of qdma[N] device\n"
+ "\t\tstat clear clear all statistics data of qdma[N} device\n"
+ "\t\tq list list all queues\n"
+ "\t\tq add idx <N> [mode <mm|st>] [dir <h2c|c2h|bi>] - add a queue\n"
+ "\t\t *mode default to mm\n"
+ "\t\t *dir default to h2c\n"
+ "\t\tq add list <start_idx> <num_Qs> [mode <mm|st>] [dir <h2c|c2h|bi>] - add multiple queues at once\n"
+ "\t\tq start idx <N> [dir <h2c|c2h|bi>] [idx_ringsz <0:15>] [idx_bufsz <0:15>] [idx_tmr <0:15>]\n"
+ " [idx_cntr <0:15>] [trigmode <every|usr_cnt|usr|usr_tmr|dis>] [cmptsz <0|1|2|3>] [sw_desc_sz <3>]\n"
+ " [desc_bypass_en] [pfetch_en] [pfetch_bypass_en] [dis_cmpl_status]\n"
+ " [dis_cmpl_status_acc] [dis_cmpl_status_pend_chk] [c2h_udd_en]\n"
+ " [cmpl_ovf_dis] [dis_fetch_credit] [dis_cmpl_status] [c2h_cmpl_intr_en] - start a single queue\n"
+ "\t\tq start list <start_idx> <num_Qs> [dir <h2c|c2h|bi>] [idx_bufsz <0:15>] [idx_tmr <0:15>]\n"
+ " [idx_cntr <0:15>] [trigmode <every|usr_cnt|usr|usr_tmr|dis>] [cmptsz <0|1|2|3>] [sw_desc_sz <3>]\n"
+ " [desc_bypass_en] [pfetch_en] [pfetch_bypass_en] [dis_cmpl_status]\n"
+ " [dis_cmpl_status_acc] [dis_cmpl_status_pend_chk] [cmpl_ovf_dis]\n"
+ " [dis_fetch_credit] [dis_cmpl_status] [c2h_cmpl_intr_en] - start multiple queues at once\n"
+ "\t\tq stop idx <N> dir [<h2c|c2h|bi>] - stop a single queue\n"
+ "\t\tq stop list <start_idx> <num_Qs> dir [<h2c|c2h|bi>] - stop list of queues at once\n"
+ "\t\tq del idx <N> dir [<h2c|c2h|bi>] - delete a queue\n"
+ "\t\tq del list <start_idx> <num_Qs> dir [<h2c|c2h|bi>] - delete list of queues at once\n"
+ "\t\tq dump idx <N> dir [<h2c|c2h|bi>] dump queue param\n"
+ "\t\tq dump list <start_idx> <num_Qs> dir [<h2c|c2h|bi>] - dump queue param\n"
+ "\t\tq dump idx <N> dir [<h2c|c2h|bi>] desc <x> <y> - dump desc ring entry x ~ y\n"
+ "\t\tq dump list <start_idx> <num_Qs> dir [<h2c|c2h|bi>] desc <x> <y> - dump desc ring entry x ~ y\n"
+ "\t\tq dump idx <N> dir [<h2c|c2h|bi>] cmpt <x> <y> - dump cmpt ring entry x ~ y\n"
+ "\t\tq dump list <start_idx> <num_Qs> dir [<h2c|c2h|bi>] cmpt <x> <y> - dump cmpt ring entry x ~ y\n"
+#ifdef ERR_DEBUG
+ "\t\tq err help - help to induce errors \n"
+ "\t\tq err idx <N> [<err <[1|0]>>] dir <[h2c|c2h|bi]> - induce errors on q idx <N> \n"
+#endif
+ );
+ fprintf(fp,
+ "\t\treg dump [dmap <Q> <N>] - register dump. Only dump dmap registers if dmap is specified.\n"
+ "\t\t specify dmap range to dump: Q=queue, N=num of queues\n"
+ "\t\treg read [bar <N>] <addr> - read a register\n"
+ "\t\treg write [bar <N>] <addr> <val> - write a register\n");
+ fprintf(fp,
+ "\t\tintring dump vector <N> <start_idx> <end_idx> - interrupt ring dump for vector number <N> \n"
+ "\t\t for intrrupt entries :<start_idx> --- <end_idx>\n");
+
+ exit(fp == stderr ? 1 : 0);
+}
+
+static int arg_read_int(char *s, uint32_t *v)
+{
+ char *p;
+
+ *v = strtoul(s, &p, 0);
+ if (*p) {
+ warnx("bad parameter \"%s\", integer expected", s);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int parse_ifname(char *name, struct xcmd_info *xcmd)
+{
+ int rv;
+ int len = strlen(name);
+ int pos, i;
+ uint32_t v;
+ char *p;
+
+ /* qdmaN of qdmavfN*/
+ if (len > 11) {
+ warnx("interface name %s too long, expect qdma<N>.\n", name);
+ return -EINVAL;
+ }
+ if (strncmp(name, "qdma", 4)) {
+ warnx("bad interface name %s, expect qdma<N>.\n", name);
+ return -EINVAL;
+ }
+ if (name[4] == 'v' && name[5] == 'f') {
+ xcmd->vf = 1;
+ pos = 6;
+ } else {
+ xcmd->vf = 0;
+ pos = 4;
+ }
+ for (i = pos; i < len; i++) {
+ if (!isxdigit(name[i])) {
+ warnx("%s unexpected <qdmaN>, %d.\n", name, i);
+ return -EINVAL;
+ }
+ }
+
+ v = strtoul(name + pos, &p, 16);
+ if (*p) {
+ warnx("bad parameter \"%s\", integer expected", name + pos);
+ return -EINVAL;
+ }
+
+ xcmd->if_bdf = v;
+ return 0;
+}
+
+#define get_next_arg(argc, argv, i) \
+ if (++(*i) >= argc) { \
+ warnx("%s missing parameter after \"%s\".\n", __FUNCTION__, argv[--(*i)]); \
+ return -EINVAL; \
+ }
+
+#define __get_next_arg(argc, argv, i) \
+ if (++i >= argc) { \
+ warnx("%s missing parameter aft \"%s\".\n", __FUNCTION__, argv[--i]); \
+ return -EINVAL; \
+ }
+
+static int next_arg_read_int(int argc, char *argv[], int *i, unsigned int *v)
+{
+ get_next_arg(argc, argv, i);
+ return arg_read_int(argv[*i], v);
+}
+
+static int next_arg_read_pair(int argc, char *argv[], int *start, char *s,
+ unsigned int *v, int optional)
+{
+ int rv;
+ int i = *start;
+
+ /* string followed by an int */
+ if ((i + 2) >= argc) {
+ if (optional) {
+ warnx("No optional parm %s after \"%s\".\n", s, argv[i]);
+ return 0;
+ } else {
+ warnx("missing parameter after \"%s\".\n", argv[i]);
+ return -EINVAL;
+ }
+ }
+
+ __get_next_arg(argc, argv, i);
+
+ if (!strcmp(s, argv[i])) {
+ get_next_arg(argc, argv, &i);
+ *start = i;
+ return arg_read_int(argv[i], v);
+ }
+
+ warnx("bad parameter, \"%s\".\n", argv[i]);
+ return -EINVAL;
+}
+
+static int validate_regcmd(enum xnl_op_t qcmd, struct xcmd_reg *regcmd)
+{
+ int invalid = 0;
+
+ switch(qcmd) {
+ case XNL_CMD_REG_DUMP:
+ break;
+ case XNL_CMD_REG_RD:
+ case XNL_CMD_REG_WRT:
+ if ((regcmd->bar != 0) && (regcmd->bar != 2)) {
+ printf("dmactl: bar %d number out of range\n",
+ regcmd->bar);
+ invalid = -EINVAL;
+ break;
+ }
+#if 0
+ if ((regcmd->bar == 0) &&
+ !is_valid_addr(regcmd->bar, regcmd->reg)) {
+ printf("dmactl: Invalid address 0x%x in bar %u\n",
+ regcmd->reg, regcmd->bar);
+ invalid = -EINVAL;
+ }
+#endif
+ break;
+ default:
+ invalid = -EINVAL;
+ break;
+ }
+
+ return invalid;
+}
+
+static int parse_reg_cmd(int argc, char *argv[], int i, struct xcmd_info *xcmd)
+{
+ struct xcmd_reg *regcmd = &xcmd->u.reg;
+ int rv;
+ int args_valid;
+
+ /*
+ * reg dump
+ * reg read [bar <N>] <addr>
+ * reg write [bar <N>] <addr> <val>
+ */
+
+ memset(regcmd, 0, sizeof(struct xcmd_reg));
+ if (!strcmp(argv[i], "dump")) {
+ xcmd->op = XNL_CMD_REG_DUMP;
+ i++;
+
+ if (i < argc) {
+ if (!strcmp(argv[i], "dmap")) {
+ get_next_arg(argc, argv, &i);
+ rv = read_range(argc, argv, i, ®cmd->range_start,
+ ®cmd->range_end);
+ if (rv < 0)
+ return rv;
+ i = rv;
+ }
+ }
+
+ } else if (!strcmp(argv[i], "read")) {
+ xcmd->op = XNL_CMD_REG_RD;
+
+ get_next_arg(argc, argv, &i);
+ if (!strcmp(argv[i], "bar")) {
+ rv = next_arg_read_int(argc, argv, &i, ®cmd->bar);
+ if (rv < 0)
+ return rv;
+ regcmd->sflags |= XCMD_REG_F_BAR_SET;
+ get_next_arg(argc, argv, &i);
+ }
+ rv = arg_read_int(argv[i], ®cmd->reg);
+ if (rv < 0)
+ return rv;
+ regcmd->sflags |= XCMD_REG_F_REG_SET;
+
+ i++;
+
+ } else if (!strcmp(argv[i], "write")) {
+ xcmd->op = XNL_CMD_REG_WRT;
+
+ get_next_arg(argc, argv, &i);
+ if (!strcmp(argv[i], "bar")) {
+ rv = next_arg_read_int(argc, argv, &i, ®cmd->bar);
+ if (rv < 0)
+ return rv;
+ regcmd->sflags |= XCMD_REG_F_BAR_SET;
+ get_next_arg(argc, argv, &i);
+ }
+ rv = arg_read_int(argv[i], &xcmd->u.reg.reg);
+ if (rv < 0)
+ return rv;
+ regcmd->sflags |= XCMD_REG_F_REG_SET;
+
+ rv = next_arg_read_int(argc, argv, &i, &xcmd->u.reg.val);
+ if (rv < 0)
+ return rv;
+ regcmd->sflags |= XCMD_REG_F_VAL_SET;
+
+ i++;
+ }
+
+ args_valid = validate_regcmd(xcmd->op, regcmd);
+
+ return (args_valid == 0) ? i : args_valid;
+}
+
+static int read_range(int argc, char *argv[], int i, unsigned int *v1,
+ unsigned int *v2)
+{
+ int rv;
+
+ /* range */
+ rv = arg_read_int(argv[i], v1);
+ if (rv < 0)
+ return rv;
+
+ get_next_arg(argc, argv, &i);
+ rv = arg_read_int(argv[i], v2);
+ if (rv < 0)
+ return rv;
+
+ if (v2 < v1) {
+ warnx("invalid range %u ~ %u.\n", *v1, *v2);
+ return -EINVAL;
+ }
+
+ return ++i;
+}
+
+static char *qparm_type_str[QPARM_MAX] = {
+ "idx",
+ "mode",
+ "dir",
+ "desc",
+ "cmpt",
+ "cmptsz",
+ "sw_desc_sz",
+ "idx_ringsz",
+ "idx_bufsz",
+ "idx_tmr",
+ "idx_cntr",
+ "trigmode",
+ "pip_gl_max",
+ "pip_flow_id",
+ "pipe_slr_id",
+ "pipe_tdest",
+#ifdef ERR_DEBUG
+ "err_no"
+#endif
+};
+
+static char *qflag_type_str[MAX_QFLAGS] = {
+ "mode",
+ "mode",
+ "dir",
+ "dir",
+ "pfetch_en",
+ "bypass",
+ "fetch_credit",
+ "dis_fetch_credit",
+ "dis_cmpl_status_acc",
+ "dis_cmpl_status",
+ "dis_cmpl_status_pend_chk",
+ "dis_cmpl_status",
+ "c2h_cmpl_intr_en",
+ "c2h_udd_en",
+ "cmpl_ovf_dis"
+};
+
+#define IS_SIZE_IDX_VALID(x) (x < 16)
+
+static void print_ignored_params(uint32_t ignore_mask, uint8_t isflag)
+{
+ unsigned int maxcount = isflag ? MAX_QFLAGS : QPARM_MAX;
+ char **qparam = isflag ? qflag_type_str : qparm_type_str;
+ char *pdesc = isflag ? "flag" : "attr";
+ unsigned int i;
+
+ for (i = 0; ignore_mask && (i < maxcount); i++) {
+ if (ignore_mask & 0x01)
+ warnx("Warn: Ignoring %s: %s", pdesc, qparam[i]);
+ ignore_mask >>= 1;
+ }
+}
+
+static int validate_qcmd(enum xnl_op_t qcmd, struct xcmd_q_parm *qparm)
+{
+ int invalid = 0;
+ switch(qcmd) {
+ case XNL_CMD_Q_LIST:
+ if (qparm->sflags)
+ warnx("Warn: Ignoring all attributes and flags");
+ break;
+ case XNL_CMD_Q_ADD:
+ print_ignored_params(qparm->sflags &
+ Q_ADD_ATTR_IGNORE_MASK, 0);
+ print_ignored_params(qparm->flags &
+ Q_ADD_FLAG_IGNORE_MASK, 1);
+ break;
+ case XNL_CMD_Q_START:
+ if (!IS_SIZE_IDX_VALID(qparm->c2h_bufsz_idx)) {
+ warnx("dmactl: C2H Buf index out of range");
+ invalid = -EINVAL;
+ }
+ if (!IS_SIZE_IDX_VALID(qparm->qrngsz_idx)) {
+ warnx("dmactl: Queue ring size index out of range");
+ invalid = -EINVAL;
+ }
+ if (!IS_SIZE_IDX_VALID(qparm->cmpt_cntr_idx)) {
+ warnx("dmactl: CMPT counter index out of range");
+ invalid = -EINVAL;
+ }
+ if (!IS_SIZE_IDX_VALID(qparm->cmpt_tmr_idx)) {
+ warnx("dmactl: CMPT timer index out of range");
+ invalid = -EINVAL;
+ }
+ if (qparm->cmpt_entry_size >=
+ XNL_ST_C2H_NUM_CMPT_DESC_SIZES) {
+ warnx("dmactl: CMPT entry size out of range");
+ invalid = -EINVAL;
+ }
+ if (qparm->flags & XNL_F_PFETCH_BYPASS_EN) {
+ if (!(qparm->flags & XNL_F_DESC_BYPASS_EN)) {
+ printf("Error:desc bypass enable must be enabled for configuring pfetch bypass enable\n");
+ invalid = -EINVAL;
+ break;
+ }
+ }
+ if (qparm->flags & XNL_F_CMPL_UDD_EN) {
+ if (!(qparm->sflags & (1 << QPARM_CMPTSZ))) {
+ printf("Error: cmptsz required for enabling c2h udd packet\n");
+ invalid = -EINVAL;
+ break;
+ }
+ }
+ if (qparm->flags & XNL_F_QDIR_H2C) {
+ print_ignored_params(qparm->sflags &
+ (Q_START_ATTR_IGNORE_MASK |
+ Q_H2C_ATTR_IGNORE_MASK), 0);
+ print_ignored_params(qparm->flags &
+ (Q_START_FLAG_IGNORE_MASK |
+ Q_H2C_FLAG_IGNORE_MASK), 1);
+ }
+ if ((qparm->sflags & (1 << QPARM_SW_DESC_SZ))) {
+ /* TODO: in 2018.3 RTL1 , only 64B sw_desc_size is supported */
+ if (qparm->sw_desc_sz != DESC_SIZE_64B) {
+ warnx("dmactl: desc size out of range");
+ invalid = -EINVAL;
+ }
+ }
+
+ break;
+ case XNL_CMD_Q_STOP:
+ print_ignored_params(qparm->sflags &
+ Q_STOP_ATTR_IGNORE_MASK, 0);
+ print_ignored_params(qparm->flags &
+ Q_STOP_FLAG_IGNORE_MASK, 1);
+ break;
+ case XNL_CMD_Q_DEL:
+ print_ignored_params(qparm->sflags &
+ Q_DEL_ATTR_IGNORE_MASK, 0);
+ print_ignored_params(qparm->flags &
+ Q_DEL_FLAG_IGNORE_MASK, 1);
+ break;
+ case XNL_CMD_Q_CMPT:
+ case XNL_CMD_Q_DUMP:
+ if ((qparm->sflags & ((1 << QPARM_DESC) |
+ (1 << QPARM_CMPT))) == ((1 << QPARM_DESC) |
+ (1 << QPARM_CMPT))) {
+ invalid = -EINVAL;
+ printf("Error: Both desc and cmpt attr cannot be taken for Q DUMP\n");
+ break;
+ }
+ case XNL_CMD_Q_DESC:
+ print_ignored_params(qparm->sflags &
+ Q_DUMP_ATTR_IGNORE_MASK, 0);
+ print_ignored_params(qparm->flags &
+ Q_DUMP_FLAG_IGNORE_MASK, 1);
+ break;
+ case XNL_CMD_Q_RX_PKT:
+ if (qparm->flags & XNL_F_QDIR_H2C) {
+ printf("Rx dump packet is st c2h only command\n");
+ invalid = -EINVAL;
+ break;
+ }
+ print_ignored_params(qparm->sflags &
+ Q_DUMP_PKT_ATTR_IGNORE_MASK, 0);
+ print_ignored_params(qparm->flags &
+ Q_DUMP_PKT_FLAG_IGNORE_MASK, 1);
+ break;
+#ifdef ERR_DEBUG
+ case XNL_CMD_Q_ERR_INDUCE:
+ break;
+#endif
+ default:
+ invalid = -EINVAL;
+ break;
+ }
+
+ return invalid;
+}
+
+#ifdef ERR_DEBUG
+static unsigned char get_err_num(char *err)
+{
+ uint32_t i;
+
+ for (i = 0; i < qdma_errs; i++)
+ if (!strcmp(err, qdma_err_str[i]))
+ break;
+
+ return i;
+}
+#endif
+
+static int read_qparm(int argc, char *argv[], int i, struct xcmd_q_parm *qparm,
+ unsigned int f_arg_required)
+{
+ int rv;
+ uint32_t v1, v2;
+ unsigned int f_arg_set = 0;;
+ unsigned int mask;
+
+ /*
+ * idx <val>
+ * list <start_idx> <num_q>
+ * ringsz <val>
+ * bufsz <val>
+ * mode <mm|st>
+ * dir <h2c|c2h|bi>
+ * cdev <0|1>
+ * bypass <0|1>
+ * desc <x> <y>
+ * cmpt <x> <y>
+ * cmptsz <0|1|2|3>
+ */
+
+ qparm->idx = XNL_QIDX_INVALID;
+
+ while (i < argc) {
+#ifdef ERR_DEBUG
+ if ((f_arg_required & (1 << QPARAM_ERR_NO)) &&
+ (!strcmp(argv[i], "help"))) {
+ uint32_t j;
+
+ fprintf(stdout, "q err idx <N> <num_errs> [list of <err <[1|0]>>] dir <[h2c|c2h|bi]>\n");
+ fprintf(stdout, "Supported errors:\n");
+ for (j = 0; j < qdma_errs; j++)
+ fprintf(stdout, "\t%s\n", qdma_err_str[j]);
+
+ return argc;
+ }
+#endif
+ if (!strcmp(argv[i], "idx")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->idx = v1;
+ f_arg_set |= 1 << QPARM_IDX;
+ qparm->num_q = 1;
+#ifdef ERR_DEBUG
+ if (f_arg_required & (1 << QPARAM_ERR_NO)) {
+ unsigned char err_no;
+
+ get_next_arg(argc, argv, &i);
+
+ err_no = get_err_num(argv[i]);
+ if (err_no >= qdma_errs) {
+ fprintf(stderr,
+ "unknown err %s.\n",
+ argv[i]);
+ return -EINVAL;
+ }
+
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->err.en = v1 ? 1 : 0;
+ qparm->err.err_no = err_no;
+ printf("%s-%u: err_no/en: %u/%u\n", argv[i], qparm->idx, qparm->err.err_no, qparm->err.en);
+ i++;
+ f_arg_set |= 1 << QPARAM_ERR_NO;
+ } else
+ i++;
+#else
+ i++;
+#endif
+
+ } else if (!strcmp(argv[i], "list")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->idx = v1;
+ f_arg_set |= 1 << QPARM_IDX;
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->num_q = v1;
+ i++;
+
+ } else if (!strcmp(argv[i], "mode")) {
+ get_next_arg(argc, argv, (&i));
+
+ if (!strcmp(argv[i], "mm")) {
+ qparm->flags |= XNL_F_QMODE_MM;
+ } else if (!strcmp(argv[i], "st")) {
+ qparm->flags |= XNL_F_QMODE_ST;
+ } else {
+ warnx("unknown q mode %s.\n", argv[i]);
+ return -EINVAL;
+ }
+ f_arg_set |= 1 << QPARM_MODE;
+ i++;
+
+ } else if (!strcmp(argv[i], "dir")) {
+ get_next_arg(argc, argv, (&i));
+
+ if (!strcmp(argv[i], "h2c")) {
+ qparm->flags |= XNL_F_QDIR_H2C;
+ } else if (!strcmp(argv[i], "c2h")) {
+ qparm->flags |= XNL_F_QDIR_C2H;
+ } else if (!strcmp(argv[i], "bi")) {
+ qparm->flags |= XNL_F_QDIR_BOTH;
+ } else {
+ warnx("unknown q dir %s.\n", argv[i]);
+ return -EINVAL;
+ }
+ f_arg_set |= 1 << QPARM_DIR;
+ i++;
+
+ } else if (!strcmp(argv[i], "idx_bufsz")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->c2h_bufsz_idx = v1;
+
+ f_arg_set |= 1 << QPARM_C2H_BUFSZ_IDX;
+ i++;
+
+ } else if (!strcmp(argv[i], "idx_ringsz")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->qrngsz_idx = v1;
+ f_arg_set |= 1 << QPARM_RNGSZ_IDX;
+ i++;
+ } else if (!strcmp(argv[i], "idx_tmr")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->cmpt_tmr_idx = v1;
+ f_arg_set |= 1 << QPARM_CMPT_TMR_IDX;
+ i++;
+ } else if (!strcmp(argv[i], "idx_cntr")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+
+ qparm->cmpt_cntr_idx = v1;
+ f_arg_set |= 1 << QPARM_CMPT_CNTR_IDX;
+ i++;
+ } else if (!strcmp(argv[i], "pipe_gl_max")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+ qparm->pipe_gl_max = v1;
+ f_arg_set |= 1 << QPARM_PIPE_GL_MAX;
+ i++;
+
+
+ } else if (!strcmp(argv[i], "pipe_flow_id")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+ qparm->pipe_flow_id = v1;
+ f_arg_set |= 1 << QPARM_PIPE_FLOW_ID;
+ i++;
+ } else if (!strcmp(argv[i], "pipe_slr_id")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+ qparm->pipe_slr_id = v1;
+ f_arg_set |= 1 << QPARM_PIPE_SLR_ID;
+ i++;
+ } else if (!strcmp(argv[i], "pipe_tdest")) {
+ rv = next_arg_read_int(argc, argv, &i, &v1);
+ if (rv < 0)
+ return rv;
+ qparm->pipe_tdest = v1;
+ f_arg_set |= 1 << QPARM_PIPE_TDEST;
+ i++;
+ } else if (!strcmp(argv[i], "cmpl_ovf_dis")) {
+ qparm->flags |= XNL_F_CMPT_OVF_CHK_DIS;
+ i++;
+ } else if (!strcmp(argv[i], "trigmode")) {
+ get_next_arg(argc, argv, (&i));
+
+ if (!strcmp(argv[i], "every")) {
+ v1 = 1;
+ } else if (!strcmp(argv[i], "usr_cnt")) {
+ v1 = 2;
+ } else if (!strcmp(argv[i], "usr")) {
+ v1 = 3;
+ } else if (!strcmp(argv[i], "usr_tmr")) {
+ v1=4;
+ } else if (!strcmp(argv[i], "cntr_tmr")) {
+ v1=5;
+ } else if (!strcmp(argv[i], "dis")) {
+ v1 = 0;
+ } else {
+ warnx("unknown q trigmode %s.\n", argv[i]);
+ return -EINVAL;
+ }
+
+ qparm->cmpt_trig_mode = v1;
+ f_arg_set |= 1 << QPARM_CMPT_TRIG_MODE;
+ i++;
+ } else if (!strcmp(argv[i], "desc")) {
+ get_next_arg(argc, argv, &i);
+ rv = read_range(argc, argv, i, &qparm->range_start,
+ &qparm->range_end);
+ if (rv < 0)
+ return rv;
+ i = rv;
+ f_arg_set |= 1 << QPARM_DESC;
+
+ } else if (!strcmp(argv[i], "cmpt")) {
+ get_next_arg(argc, argv, &i);
+ rv = read_range(argc, argv, i, &qparm->range_start,
+ &qparm->range_end);
+ if (rv < 0)
+ return rv;
+ i = rv;
+ f_arg_set |= 1 << QPARM_CMPT;
+
+ } else if (!strcmp(argv[i], "cmptsz")) {
+ get_next_arg(argc, argv, &i);
+ sscanf(argv[i], "%hhu", &qparm->cmpt_entry_size);
+ f_arg_set |= 1 << QPARM_CMPTSZ;
+ i++;
+ } else if (!strcmp(argv[i], "sw_desc_sz")) {
+ get_next_arg(argc, argv, &i);
+ sscanf(argv[i], "%hhu", &qparm->sw_desc_sz);
+ f_arg_set |= 1 << QPARM_SW_DESC_SZ;
+ i++;
+ } else if (!strcmp(argv[i], "pfetch_en")) {
+ qparm->flags |= XNL_F_PFETCH_EN;
+ i++;
+ } else if (!strcmp(argv[i], "pfetch_bypass_en")) {
+ qparm->flags |= XNL_F_PFETCH_BYPASS_EN;
+ i++;
+ } else if (!strcmp(argv[i], "desc_bypass_en")) {
+ qparm->flags |= XNL_F_DESC_BYPASS_EN;
+ i++;
+ } else if (!strcmp(argv[i], "c2h_cmpl_intr_en")) {
+ qparm->flags |= XNL_F_C2H_CMPL_INTR_EN;
+ i++;
+ } else if (!strcmp(argv[i], "dis_cmpl_status")) {
+ qparm->flags &= ~XNL_F_CMPL_STATUS_EN;
+ i++;
+ } else if (!strcmp(argv[i], "dis_cmpl_status_acc")) {
+ qparm->flags &= ~XNL_F_CMPL_STATUS_ACC_EN;
+ i++;
+ } else if (!strcmp(argv[i], "dis_cmpl_status_pend_chk")) {
+ qparm->flags &= ~XNL_F_CMPL_STATUS_PEND_CHK;
+ i++;
+ } else if (!strcmp(argv[i], "dis_fetch_credit")) {
+ qparm->flags &= ~XNL_F_FETCH_CREDIT;
+ i++;
+ } else if (!strcmp(argv[i], "dis_cmpl_status")) {
+ qparm->flags &= ~XNL_F_CMPL_STATUS_DESC_EN;
+ i++;
+ } else if (!strcmp(argv[i], "c2h_udd_en")) {
+ qparm->flags |= XNL_F_CMPL_UDD_EN;
+ i++;
+ } else {
+ warnx("unknown q parameter %s.\n", argv[i]);
+ return -EINVAL;
+ }
+ }
+ if ((f_arg_required & (1 << QPARM_RNGSZ_IDX)) &&
+ !(f_arg_set & (1 << QPARM_RNGSZ_IDX))) {
+ warnx("Info: Default ring size set to 2048");
+ qparm->qrngsz_idx = 5;
+ f_arg_set |= 1 << QPARM_RNGSZ_IDX;
+ }
+ /* check for any missing mandatory parameters */
+ mask = f_arg_set & f_arg_required;
+ if (mask != f_arg_required) {
+ int i;
+ unsigned int bit_mask = 1;
+
+ mask = (mask ^ f_arg_required) & f_arg_required;
+
+ for (i = 0; i < QPARM_MAX; i++, bit_mask <<= 1) {
+ if (!(bit_mask & f_arg_required))
+ continue;
+ warnx("missing q parameter %s.\n", qparm_type_str[i]);
+ return -EINVAL;
+ }
+ }
+
+ if (!(f_arg_set & 1 << QPARM_DIR)) {
+ /* default to H2C */
+ warnx("Warn: Default dir set to \'h2c\'");
+ f_arg_set |= 1 << QPARM_DIR;
+ qparm->flags |= XNL_F_QDIR_H2C;
+ }
+
+ qparm->sflags = f_arg_set;
+
+ return argc;
+}
+
+static int parse_q_cmd(int argc, char *argv[], int i, struct xcmd_info *xcmd)
+{
+ struct xcmd_q_parm *qparm = &xcmd->u.qparm;
+ int rv;
+ int args_valid;
+
+ /*
+ * q list
+ * q add idx <N> mode <mm|st> [dir <h2c|c2h|bi>] [cdev <0|1>] [cmptsz <0|1|2|3>]
+ * q start idx <N> dir <h2c|c2h|bi>
+ * q stop idx <N> dir <h2c|c2h|bi>
+ * q del idx <N> dir <h2c|c2h|bi>
+ * q dump idx <N> dir <h2c|c2h|bi>
+ * q dump idx <N> dir <h2c|c2h|bi> desc <x> <y>
+ * q dump idx <N> dir <h2c|c2h|bi> cmpt <x> <y>
+ * q pkt idx <N>
+ */
+
+ if (!strcmp(argv[i], "list")) {
+ xcmd->op = XNL_CMD_Q_LIST;
+ return ++i;
+ } else if (!strcmp(argv[i], "add")) {
+ unsigned int mask;
+
+ xcmd->op = XNL_CMD_Q_ADD;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, (1 << QPARM_IDX));
+ /* error checking of parameter values */
+ if (qparm->sflags & (1 << QPARM_MODE)) {
+ mask = XNL_F_QMODE_MM | XNL_F_QMODE_ST;
+ if ((qparm->flags & mask) == mask) {
+ warnx("mode mm/st cannot be combined.\n");
+ return -EINVAL;
+ }
+ } else {
+ /* default to MM */
+ warnx("Warn: Default mode set to \'mm\'");
+ qparm->sflags |= 1 << QPARM_MODE;
+ qparm->flags |= XNL_F_QMODE_MM;
+ }
+
+ } else if (!strcmp(argv[i], "start")) {
+ xcmd->op = XNL_CMD_Q_START;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, ((1 << QPARM_IDX) |
+ (1 << QPARM_RNGSZ_IDX)));
+ qparm->flags |= (XNL_F_CMPL_STATUS_EN | XNL_F_CMPL_STATUS_ACC_EN |
+ XNL_F_CMPL_STATUS_PEND_CHK | XNL_F_CMPL_STATUS_DESC_EN |
+ XNL_F_FETCH_CREDIT);
+ if (qparm->flags & (XNL_F_QDIR_C2H | XNL_F_QMODE_ST) ==
+ (XNL_F_QDIR_C2H | XNL_F_QMODE_ST)) {
+ if (!(qparm->sflags & (1 << QPARM_CMPTSZ))) {
+ /* default to 8B */
+ qparm->cmpt_entry_size = XNL_ST_C2H_CMPT_DESC_SIZE_8B;
+ qparm->sflags |=
+ (1 << QPARM_CMPTSZ);
+ }
+ }
+ } else if (!strcmp(argv[i], "stop")) {
+ xcmd->op = XNL_CMD_Q_STOP;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, (1 << QPARM_IDX));
+
+ } else if (!strcmp(argv[i], "del")) {
+ xcmd->op = XNL_CMD_Q_DEL;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, (1 << QPARM_IDX));
+
+ } else if (!strcmp(argv[i], "dump")) {
+ xcmd->op = XNL_CMD_Q_DUMP;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, (1 << QPARM_IDX));
+ } else if (!strcmp(argv[i], "pkt")) {
+ xcmd->op = XNL_CMD_Q_RX_PKT;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, (1 << QPARM_IDX));
+#ifdef ERR_DEBUG
+ } else if (!strcmp(argv[i], "err")) {
+ xcmd->op = XNL_CMD_Q_ERR_INDUCE;
+ get_next_arg(argc, argv, &i);
+ rv = read_qparm(argc, argv, i, qparm, ((1 << QPARM_IDX) |
+ (1 << QPARAM_ERR_NO)));
+#endif
+ } else {
+ printf("Error: Unknown q command\n");
+ return -EINVAL;
+ }
+
+ if (rv < 0)
+ return rv;
+ i = rv;
+
+ if (xcmd->op == XNL_CMD_Q_DUMP) {
+ unsigned int mask = (1 << QPARM_DESC) | (1 << QPARM_CMPT);
+
+ if ((qparm->sflags & mask) == mask) {
+ warnx("dump cmpt/desc cannot be combined.\n");
+ return -EINVAL;
+ }
+ if ((qparm->sflags & (1 << QPARM_DESC)))
+ xcmd->op = XNL_CMD_Q_DESC;
+ else if ((qparm->sflags & (1 << QPARM_CMPT)))
+ xcmd->op = XNL_CMD_Q_CMPT;
+ }
+
+ args_valid = validate_qcmd(xcmd->op, qparm);
+
+ return (args_valid == 0) ? i : args_valid;
+
+ return i;
+}
+
+static int parse_dev_cmd(int argc, char *argv[], int i, struct xcmd_info *xcmd)
+{
+ if (!strcmp(argv[i], "list")) {
+ xcmd->op = XNL_CMD_DEV_LIST;
+ i++;
+ }
+
+ return i;
+}
+
+static int parse_stat_cmd(int argc, char *argv[], int i, struct xcmd_info *xcmd)
+{
+ int rv;
+
+ xcmd->op = XNL_CMD_DEV_STAT;
+ if (i >= argc)
+ return i;
+ if (!strcmp(argv[i], "clear")) {
+ xcmd->op = XNL_CMD_DEV_STAT_CLEAR;
+ i++;
+ }
+ return i;
+}
+
+static int parse_intr_cmd(int argc, char *argv[], int i, struct xcmd_info *xcmd)
+{
+ struct xcmd_intr *intrcmd = &xcmd->u.intr;
+ int rv;
+
+ /*
+ * intr dump vector <N>
+ */
+
+ memset(intrcmd, 0, sizeof(struct xcmd_intr));
+ if (!strcmp(argv[i], "dump")) {
+ xcmd->op = XNL_CMD_INTR_RING_DUMP;
+
+ get_next_arg(argc, argv, &i);
+ if (!strcmp(argv[i], "vector")) {
+ rv = next_arg_read_int(argc, argv, &i, &intrcmd->vector);
+ if (rv < 0)
+ return rv;
+ }
+ rv = next_arg_read_int(argc, argv, &i, &intrcmd->start_idx);
+ if (rv < 0) {
+ intrcmd->start_idx = 0;
+ intrcmd->end_idx = QDMA_MAX_INT_RING_ENTRIES - 1;
+ goto func_ret;
+ }
+ rv = next_arg_read_int(argc, argv, &i, &intrcmd->end_idx);
+ if (rv < 0)
+ intrcmd->end_idx = QDMA_MAX_INT_RING_ENTRIES - 1;
+ }
+func_ret:
+ i++;
+ return i;
+}
+
+int parse_cmd(int argc, char *argv[], struct xcmd_info *xcmd)
+{
+ char *ifname;
+ int i;
+ int rv;
+
+ memset(xcmd, 0, sizeof(struct xcmd_info));
+
+ progname = argv[0];
+
+ if (argc == 1)
+ usage(stderr);
+
+ if (argc == 2) {
+ if (!strcmp(argv[1], "?") || !strcmp(argv[1], "-h") ||
+ !strcmp(argv[1], "help") || !strcmp(argv[1], "--help"))
+ usage(stdout);
+
+ if (!strcmp(argv[1], "-v") || !strcmp(argv[1], "--version")) {
+ printf("%s version %s\n", PROGNAME, VERSION);
+ printf("%s\n", COPYRIGHT);
+ exit(0);
+ }
+ }
+
+ if (!strcmp(argv[1], "dev")) {
+ rv = parse_dev_cmd(argc, argv, 2, xcmd);
+ goto done;
+ }
+
+ /* which dma fpga */
+ ifname = argv[1];
+ rv = parse_ifname(ifname, xcmd);
+ if (rv < 0)
+ return rv;
+
+ if (argc == 2) {
+ rv = 2;
+ xcmd->op = XNL_CMD_DEV_INFO;
+ goto done;
+ }
+
+ i = 3;
+ if (!strcmp(argv[2], "reg")) {
+ rv = parse_reg_cmd(argc, argv, i, xcmd);
+ } else if (!strcmp(argv[2], "stat")) {
+ rv = parse_stat_cmd(argc, argv, i, xcmd);
+ } else if (!strcmp(argv[2], "q")) {
+ rv = parse_q_cmd(argc, argv, i, xcmd);
+ } else if (!strcmp(argv[2], "intring")){
+ rv = parse_intr_cmd(argc, argv, i, xcmd);
+ } else if (!strcmp(argv[2], "version")){
+ rv = 3;
+ xcmd->op = XNL_CMD_VERSION;
+ } else {
+ warnx("bad parameter \"%s\".\n", argv[2]);
+ return -EINVAL;
+ }
+
+done:
+ if (rv < 0)
+ return rv;
+ i = rv;
+
+ if (i < argc) {
+ warnx("unexpected parameter \"%s\".\n", argv[i]);
+ return -EINVAL;
+ }
+ return 0;
+}
diff --git a/QDMA/linux-kernel/user/cli/cmd_parse.h b/QDMA/linux-kernel/user/cli/cmd_parse.h
new file mode 100644
index 0000000000000000000000000000000000000000..3a4e3e5ad54774c889f08d738e3c342e70a6143e
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/cmd_parse.h
@@ -0,0 +1,40 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#ifndef USER_CLI_CMD_PARSE_H_
+#define USER_CLI_CMD_PARSE_H_
+
+#include "qdma_nl.h"
+#include "reg_cmd.h"
+#include "nl_user.h"
+
+struct xcmd_info {
+ unsigned char vf:1;
+ unsigned char op:7;
+ unsigned char config_bar;
+ unsigned char user_bar;
+ unsigned short qmax;
+ char stm_bar;
+ char ifname[8];
+ union {
+ struct xcmd_intr intr;
+ struct xcmd_reg reg;
+ struct xcmd_q_parm qparm;
+ } u;
+ uint32_t if_bdf;
+ uint32_t attr_mask;
+ uint32_t attrs[XNL_ATTR_MAX];
+ char drv_str[128];
+};
+
+int parse_cmd(int argc, char *argv[], struct xcmd_info *xcmd);
+
+#endif /* USER_CLI_CMD_PARSE_H_ */
diff --git a/QDMA/linux-kernel/user/cli/main.c b/QDMA/linux-kernel/user/cli/main.c
new file mode 100644
index 0000000000000000000000000000000000000000..0da10b1b593a73bd76b781eac88d38ee7dce7165
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/main.c
@@ -0,0 +1,76 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <string.h>
+
+#include "nl_user.h"
+#include "cmd_parse.h"
+#include "reg_cmd.h"
+
+int main(int argc, char *argv[])
+{
+ struct xnl_cb cb;
+ struct xcmd_info xcmd;
+ unsigned char op;
+ int rv = 0;
+
+ memset(&xcmd, 0, sizeof(xcmd));
+
+ rv = parse_cmd(argc, argv, &xcmd);
+ if (rv < 0)
+ return rv;
+
+#if 0
+ printf("cmd op %s, 0x%x, ifname %s.\n",
+ xnl_op_str[xcmd.op], xcmd.op, xcmd.ifname);
+#endif
+
+ memset(&cb, 0, sizeof(struct xnl_cb));
+
+ if (xcmd.op == XNL_CMD_DEV_LIST) {
+ /* try pf nl server */
+ rv = xnl_connect(&cb, 0);
+ if (!rv)
+ rv = xnl_send_cmd(&cb, &xcmd);
+ xnl_close(&cb);
+
+ /* try vf nl server */
+ memset(&cb, 0, sizeof(struct xnl_cb));
+ rv = xnl_connect(&cb, 1);
+ if (!rv)
+ rv = xnl_send_cmd(&cb, &xcmd);
+ xnl_close(&cb);
+
+ goto close;
+ }
+
+ /* for all other command, query the target device info first */
+ rv = xnl_connect(&cb, xcmd.vf);
+ if (rv < 0)
+ goto close;
+
+ op = xcmd.op;
+ xcmd.op = XNL_CMD_DEV_INFO;
+ rv = xnl_send_cmd(&cb, &xcmd);
+ xcmd.op = op;
+ if (rv < 0)
+ goto close;
+
+ if ((xcmd.op == XNL_CMD_REG_DUMP) || (xcmd.op == XNL_CMD_REG_RD) ||
+ (xcmd.op == XNL_CMD_REG_WRT))
+ rv = proc_reg_cmd(&xcmd);
+ else
+ rv = xnl_send_cmd(&cb, &xcmd);
+
+close:
+ xnl_close(&cb);
+ return rv;
+}
diff --git a/QDMA/linux-kernel/user/cli/nl_user.c b/QDMA/linux-kernel/user/cli/nl_user.c
new file mode 100644
index 0000000000000000000000000000000000000000..24746dfc0a3eb7e3b636b9a98468ebb772c67da2
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/nl_user.c
@@ -0,0 +1,575 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <ctype.h>
+#include <string.h>
+#include <unistd.h>
+#include <err.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <linux/genetlink.h>
+#include <qdma_nl.h>
+
+#include "nl_user.h"
+#include "cmd_parse.h"
+
+/* Generic macros for dealing with netlink sockets. Might be duplicated
+ * elsewhere. It is recommended that commercial grade applications use
+ * libnl or libnetlink and use the interfaces provided by the library
+ */
+
+/*
+ * netlink message
+ */
+struct xnl_hdr {
+ struct nlmsghdr n;
+ struct genlmsghdr g;
+};
+
+struct xnl_gen_msg {
+ struct xnl_hdr hdr;
+ char data[0];
+};
+
+void xnl_close(struct xnl_cb *cb)
+{
+ close(cb->fd);
+}
+
+static int xnl_send(struct xnl_cb *cb, struct xnl_hdr *hdr)
+{
+ int rv;
+ struct sockaddr_nl addr = {
+ .nl_family = AF_NETLINK,
+ };
+
+ hdr->n.nlmsg_seq = cb->snd_seq;
+ cb->snd_seq++;
+
+ rv = sendto(cb->fd, (char *)hdr, hdr->n.nlmsg_len, 0,
+ (struct sockaddr *)&addr, sizeof(addr));
+ if (rv != hdr->n.nlmsg_len) {
+ perror("nl send err");
+ return -1;
+ }
+
+ return 0;
+}
+
+static int xnl_recv(struct xnl_cb *cb, struct xnl_hdr *hdr, int dlen, int print)
+{
+ int rv;
+ struct sockaddr_nl addr = {
+ .nl_family = AF_NETLINK,
+ };
+
+ memset(hdr, 0, sizeof(struct xnl_gen_msg) + dlen);
+
+ rv = recv(cb->fd, hdr, dlen, 0);
+ if (rv < 0) {
+ perror("nl recv err");
+ return -1;
+ }
+ /* as long as there is attribute, even if it is shorter than expected */
+ if (!NLMSG_OK((&hdr->n), rv) && (rv <= sizeof(struct xnl_hdr))) {
+ if (print)
+ fprintf(stderr,
+ "nl recv:, invalid message, cmd 0x%x, %d,%d.\n",
+ hdr->g.cmd, dlen, rv);
+ return -1;
+ }
+
+ if (hdr->n.nlmsg_type == NLMSG_ERROR) {
+ if (print)
+ fprintf(stderr, "nl recv, msg error, cmd 0x%x\n",
+ hdr->g.cmd);
+ return -1;
+ }
+
+ return 0;
+}
+
+static inline struct xnl_gen_msg *xnl_msg_alloc(int dlen)
+{
+ struct xnl_gen_msg *msg;
+
+ msg = malloc(sizeof(struct xnl_gen_msg) + dlen);
+ if (!msg) {
+ fprintf(stderr, "%s: OOM, %d.\n", __FUNCTION__, dlen);
+ return NULL;
+ }
+
+ memset(msg, 0, sizeof(struct xnl_gen_msg) + dlen);
+ return msg;
+}
+
+int xnl_connect(struct xnl_cb *cb, int vf)
+{
+ int fd;
+ struct sockaddr_nl addr;
+ struct xnl_gen_msg *msg = xnl_msg_alloc(XNL_RESP_BUFLEN_MIN);
+ struct xnl_hdr *hdr;
+ struct nlattr *attr;
+ int rv = -1;
+
+ if (!msg) {
+ fprintf(stderr, "%s, msg OOM.\n", __FUNCTION__);
+ return -ENOMEM;
+ }
+ hdr = (struct xnl_hdr *)msg;
+
+ fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+ if (fd < 0) {
+ perror("nl socket err");
+ rv = fd;
+ goto out;
+ }
+ cb->fd = fd;
+
+ memset(&addr, 0, sizeof(struct sockaddr_nl));
+ addr.nl_family = AF_NETLINK;
+ rv = bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_nl));
+ if (rv < 0) {
+ perror("nl bind err");
+ goto out;
+ }
+
+ hdr->n.nlmsg_type = GENL_ID_CTRL;
+ hdr->n.nlmsg_flags = NLM_F_REQUEST;
+ hdr->n.nlmsg_pid = getpid();
+ hdr->n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+
+ hdr->g.cmd = CTRL_CMD_GETFAMILY;
+ hdr->g.version = XNL_VERSION;
+
+ attr = (struct nlattr *)(msg->data);
+ attr->nla_type = CTRL_ATTR_FAMILY_NAME;
+
+ if (vf) {
+ attr->nla_len = strlen(XNL_NAME_VF) + 1 + NLA_HDRLEN;
+ strcpy((char *)(attr + 1), XNL_NAME_VF);
+
+ } else {
+ attr->nla_len = strlen(XNL_NAME_PF) + 1 + NLA_HDRLEN;
+ strcpy((char *)(attr + 1), XNL_NAME_PF);
+ }
+ hdr->n.nlmsg_len += NLMSG_ALIGN(attr->nla_len);
+
+ rv = xnl_send(cb, (struct xnl_hdr *)hdr);
+ if (rv < 0)
+ goto out;
+
+ rv = xnl_recv(cb, hdr, XNL_RESP_BUFLEN_MIN, 0);
+ if (rv < 0)
+ goto out;
+
+#if 0
+ /* family name */
+ if (attr->nla_type == CTRL_ATTR_FAMILY_NAME)
+ printf("family name: %s.\n", (char *)(attr + 1));
+#endif
+
+ attr = (struct nlattr *)((char *)attr + NLA_ALIGN(attr->nla_len));
+ /* family ID */
+ if (attr->nla_type == CTRL_ATTR_FAMILY_ID) {
+ cb->family = *(__u16 *)(attr + 1);
+// printf("family id: 0x%x.\n", cb->family);
+ }
+
+ rv = 0;
+
+out:
+ free(msg);
+ return rv;
+}
+
+static void xnl_msg_set_hdr(struct xnl_hdr *hdr, int family, int op)
+{
+ hdr->n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
+ hdr->n.nlmsg_type = family;
+ hdr->n.nlmsg_flags = NLM_F_REQUEST;
+ hdr->n.nlmsg_pid = getpid();
+
+ hdr->g.cmd = op;
+}
+
+static int xnl_send_op(struct xnl_cb *cb, int op)
+{
+ struct xnl_hdr req;
+ int rv;
+
+ memset(&req, 0, sizeof(struct xnl_hdr));
+
+ xnl_msg_set_hdr(&req, cb->family, op);
+
+ rv = xnl_send(cb, &req);
+
+ return rv;
+}
+
+static int xnl_msg_add_int_attr(struct xnl_hdr *hdr, enum xnl_attr_t type,
+ unsigned int v)
+{
+ struct nlattr *attr = (struct nlattr *)((char *)hdr + hdr->n.nlmsg_len);
+
+ attr->nla_type = (__u16)type;
+ attr->nla_len = sizeof(__u32) + NLA_HDRLEN;
+ *(__u32 *)(attr+ 1) = v;
+
+ hdr->n.nlmsg_len += NLMSG_ALIGN(attr->nla_len);
+ return 0;
+}
+
+static int xnl_msg_add_str_attr(struct xnl_hdr *hdr, enum xnl_attr_t type,
+ char *s)
+{
+ struct nlattr *attr = (struct nlattr *)((char *)hdr + hdr->n.nlmsg_len);
+ int len = strlen(s);
+
+ attr->nla_type = (__u16)type;
+ attr->nla_len = len + 1 + NLA_HDRLEN;
+
+ strcpy((char *)(attr + 1), s);
+
+ hdr->n.nlmsg_len += NLMSG_ALIGN(attr->nla_len);
+ return 0;
+}
+
+static int recv_attrs(struct xnl_hdr *hdr, struct xcmd_info *xcmd)
+{
+ unsigned char *p = (unsigned char *)(hdr + 1);
+ int maxlen = hdr->n.nlmsg_len - NLMSG_LENGTH(GENL_HDRLEN);
+
+#if 0
+ printf("nl recv, hdr len %d, data %d, gen op 0x%x, %s, ver 0x%x.\n",
+ hdr->n.nlmsg_len, maxlen, hdr->g.cmd, xnl_op_str[hdr->g.cmd],
+ hdr->g.version);
+#endif
+
+ xcmd->attr_mask = 0;
+ while (maxlen > 0) {
+ struct nlattr *na = (struct nlattr *)p;
+ int len = NLA_ALIGN(na->nla_len);
+
+ if (na->nla_type >= XNL_ATTR_MAX) {
+ fprintf(stderr, "unknown attr type %d, len %d.\n",
+ na->nla_type, na->nla_len);
+ return -EINVAL;
+ }
+
+ xcmd->attr_mask |= 1 << na->nla_type;
+
+ if (na->nla_type == XNL_ATTR_GENMSG) {
+ printf("\n%s\n", (char *)(na + 1));
+
+ } else if (na->nla_type == XNL_ATTR_DRV_INFO) {
+ strncpy(xcmd->drv_str, (char *)(na + 1), 128);
+ } else {
+ xcmd->attrs[na->nla_type] = *(uint32_t *)(na + 1);
+ }
+
+ p += len;
+ maxlen -= len;
+ }
+
+ return 0;
+}
+
+static void get_dev_stat(struct xcmd_info *xcmd)
+{
+ unsigned long long mmh2c_pkts;
+ unsigned long long mmc2h_pkts;
+ unsigned long long sth2c_pkts;
+ unsigned long long stc2h_pkts;
+ unsigned int pkts;
+
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_MMH2C_PKTS1];
+ mmh2c_pkts = pkts;
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_MMH2C_PKTS2];
+ mmh2c_pkts |= (((unsigned long long)pkts) << 32);
+
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_MMC2H_PKTS1];
+ mmc2h_pkts = pkts;
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_MMC2H_PKTS2];
+ mmc2h_pkts |= (((unsigned long long)pkts) << 32);
+
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_STH2C_PKTS1];
+ sth2c_pkts = pkts;
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_STH2C_PKTS2];
+ sth2c_pkts |= (((unsigned long long)pkts) << 32);
+
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_STC2H_PKTS1];
+ stc2h_pkts = pkts;
+ pkts = xcmd->attrs[XNL_ATTR_DEV_STAT_STC2H_PKTS2];
+ stc2h_pkts |= (((unsigned long long)pkts) << 32);
+
+ printf("qdma%s%05x:statistics\n", xcmd->vf ? "vf" : "", xcmd->if_bdf);
+ printf("Total MM H2C packets processed = %llu\n", mmh2c_pkts);
+ printf("Total MM C2H packets processed = %llu\n", mmc2h_pkts);
+ printf("Total ST H2C packets processed = %llu\n", sth2c_pkts);
+ printf("Total ST C2H packets processed = %llu\n", stc2h_pkts);
+}
+
+static int recv_nl_msg(struct xnl_hdr *hdr, struct xcmd_info *xcmd)
+{
+ unsigned int op = hdr->g.cmd;
+ unsigned int usr_bar;
+
+ recv_attrs(hdr, xcmd);
+
+ switch(op) {
+ case XNL_CMD_DEV_LIST:
+ break;
+ case XNL_CMD_DEV_INFO:
+ xcmd->config_bar = xcmd->attrs[XNL_ATTR_DEV_CFG_BAR];
+ usr_bar = (int)xcmd->attrs[XNL_ATTR_DEV_USR_BAR];
+ xcmd->qmax = xcmd->attrs[XNL_ATTR_DEV_QSET_MAX];
+ xcmd->stm_bar = xcmd->attrs[XNL_ATTR_DEV_STM_BAR];
+
+ if (usr_bar+1 == 0)
+ xcmd->user_bar = 2;
+ else
+ xcmd->user_bar = usr_bar;
+
+#ifdef DEBUG
+ printf("qdma%s%05x:\t%02x:%02x.%02x\t",
+ xcmd->vf ? "vf" : "", xcmd->if_bdf,
+ xcmd->attrs[XNL_ATTR_PCI_BUS],
+ xcmd->attrs[XNL_ATTR_PCI_DEV],
+ xcmd->attrs[XNL_ATTR_PCI_FUNC]);
+ printf("config bar: %d, user bar: %d, max #. QP: %d\n",
+ xcmd->config_bar, xcmd->user_bar, xcmd->qmax);
+#endif
+ break;
+ case XNL_CMD_DEV_STAT:
+ get_dev_stat(xcmd);
+ break;
+ case XNL_CMD_DEV_STAT_CLEAR:
+ break;
+ case XNL_CMD_VERSION:
+ break;
+ case XNL_CMD_Q_LIST:
+ break;
+ case XNL_CMD_Q_ADD:
+ break;
+ case XNL_CMD_Q_START:
+ break;
+ case XNL_CMD_Q_STOP:
+ break;
+ case XNL_CMD_Q_DEL:
+ break;
+ case XNL_CMD_REG_RD:
+ xcmd->u.reg.val = xcmd->attrs[XNL_ATTR_REG_VAL];
+ break;
+ case XNL_CMD_REG_WRT:
+ xcmd->u.reg.val = xcmd->attrs[XNL_ATTR_REG_VAL];
+ break;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static void xnl_msg_add_extra_config_attrs(struct xnl_hdr *hdr,
+ struct xcmd_info *xcmd)
+{
+ if (xcmd->u.qparm.sflags & (1 << QPARM_RNGSZ_IDX))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QRNGSZ_IDX,
+ xcmd->u.qparm.qrngsz_idx);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_C2H_BUFSZ_IDX))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_C2H_BUFSZ_IDX,
+ xcmd->u.qparm.c2h_bufsz_idx);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_CMPTSZ))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_CMPT_DESC_SIZE,
+ xcmd->u.qparm.cmpt_entry_size);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_SW_DESC_SZ))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_SW_DESC_SIZE,
+ xcmd->u.qparm.sw_desc_sz);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_CMPT_TMR_IDX))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_CMPT_TIMER_IDX,
+ xcmd->u.qparm.cmpt_tmr_idx);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_CMPT_CNTR_IDX))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_CMPT_CNTR_IDX,
+ xcmd->u.qparm.cmpt_cntr_idx);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_CMPT_TRIG_MODE))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_CMPT_TRIG_MODE,
+ xcmd->u.qparm.cmpt_trig_mode);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_PIPE_GL_MAX))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_PIPE_GL_MAX,
+ xcmd->u.qparm.pipe_gl_max);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_PIPE_FLOW_ID))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_PIPE_FLOW_ID,
+ xcmd->u.qparm.pipe_flow_id);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_PIPE_SLR_ID))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_PIPE_SLR_ID,
+ xcmd->u.qparm.pipe_slr_id);
+ if (xcmd->u.qparm.sflags & (1 << QPARM_PIPE_TDEST))
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_PIPE_TDEST,
+ xcmd->u.qparm.pipe_tdest);
+}
+
+static int get_cmd_resp_buf_len(struct xcmd_info *xcmd)
+{
+ int buf_len = XNL_RESP_BUFLEN_MAX;
+ unsigned int row_len = 50;
+
+ switch (xcmd->op) {
+ case XNL_CMD_Q_DESC:
+ row_len *= 2;
+ case XNL_CMD_Q_CMPT:
+ buf_len += ((xcmd->u.qparm.range_end -
+ xcmd->u.qparm.range_start)*row_len);
+ break;
+ case XNL_CMD_INTR_RING_DUMP:
+ buf_len += ((xcmd->u.intr.end_idx -
+ xcmd->u.intr.start_idx)*row_len);
+ break;
+ case XNL_CMD_DEV_LIST:
+ case XNL_CMD_Q_START:
+ case XNL_CMD_Q_STOP:
+ case XNL_CMD_Q_DEL:
+ return buf_len;
+ case XNL_CMD_Q_LIST:
+ case XNL_CMD_Q_DUMP:
+ break;
+ default:
+ buf_len = XNL_RESP_BUFLEN_MIN;
+ break;
+ }
+ if ((xcmd->u.qparm.flags & XNL_F_QDIR_BOTH) == XNL_F_QDIR_BOTH)
+ buf_len *= 2;
+ if (xcmd->u.qparm.flags & XNL_F_QDIR_BOTH)
+ buf_len *= xcmd->u.qparm.num_q;
+
+ return buf_len;
+}
+
+int xnl_send_cmd(struct xnl_cb *cb, struct xcmd_info *xcmd)
+{
+ struct xnl_gen_msg *msg;
+ struct xnl_hdr *hdr;
+ struct nlattr *attr;
+ int dlen = get_cmd_resp_buf_len(xcmd);
+ int i;
+ int rv;
+ enum xnl_st_c2h_cmpt_desc_size cmpt_desc_size;
+
+#if 0
+ printf("%s: op %s, 0x%x, ifname %s.\n", __FUNCTION__,
+ xnl_op_str[xcmd->op], xcmd->op, xcmd->ifname);
+#endif
+
+ msg = xnl_msg_alloc(dlen);
+ if (!msg) {
+ fprintf(stderr, "%s: OOM, %s, op %s,0x%x.\n", __FUNCTION__,
+ xcmd->ifname, xnl_op_str[xcmd->op], xcmd->op);
+ return -ENOMEM;
+ }
+
+ hdr = (struct xnl_hdr *)msg;
+ attr = (struct nlattr *)(msg->data);
+
+ xnl_msg_set_hdr(hdr, cb->family, xcmd->op);
+
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_DEV_IDX, xcmd->if_bdf);
+
+ switch(xcmd->op) {
+ case XNL_CMD_DEV_LIST:
+ case XNL_CMD_DEV_INFO:
+ case XNL_CMD_DEV_STAT:
+ case XNL_CMD_DEV_STAT_CLEAR:
+ case XNL_CMD_Q_LIST:
+ /* no parameter */
+ break;
+ case XNL_CMD_Q_ADD:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QIDX, xcmd->u.qparm.idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_NUM_Q, xcmd->u.qparm.num_q);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QFLAG, xcmd->u.qparm.flags);
+ break;
+ case XNL_CMD_Q_START:
+ xnl_msg_add_extra_config_attrs(hdr, xcmd);
+ case XNL_CMD_Q_STOP:
+ case XNL_CMD_Q_DEL:
+ case XNL_CMD_Q_DUMP:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QIDX, xcmd->u.qparm.idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_NUM_Q, xcmd->u.qparm.num_q);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QFLAG, xcmd->u.qparm.flags);
+ break;
+ case XNL_CMD_Q_DESC:
+ case XNL_CMD_Q_CMPT:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QIDX, xcmd->u.qparm.idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_NUM_Q, xcmd->u.qparm.num_q);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QFLAG, xcmd->u.qparm.flags);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_RANGE_START,
+ xcmd->u.qparm.range_start);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_RANGE_END,
+ xcmd->u.qparm.range_end);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_RSP_BUF_LEN, dlen);
+ break;
+ case XNL_CMD_Q_RX_PKT:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QIDX, xcmd->u.qparm.idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_NUM_Q, xcmd->u.qparm.num_q);
+ /* hard coded to C2H */
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QFLAG, XNL_F_QDIR_C2H);
+ break;
+ case XNL_CMD_INTR_RING_DUMP:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_INTR_VECTOR_IDX,
+ xcmd->u.intr.vector);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_INTR_VECTOR_START_IDX,
+ xcmd->u.intr.start_idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_INTR_VECTOR_END_IDX,
+ xcmd->u.intr.end_idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_RSP_BUF_LEN, dlen);
+ break;
+ case XNL_CMD_REG_RD:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_REG_BAR_NUM,
+ xcmd->u.reg.bar);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_REG_ADDR,
+ xcmd->u.reg.reg);
+ break;
+ case XNL_CMD_REG_WRT:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_REG_BAR_NUM,
+ xcmd->u.reg.bar);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_REG_ADDR,
+ xcmd->u.reg.reg);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_REG_VAL,
+ xcmd->u.reg.val);
+ break;
+#ifdef ERR_DEBUG
+ case XNL_CMD_Q_ERR_INDUCE:
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QIDX, xcmd->u.qparm.idx);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QFLAG, xcmd->u.qparm.flags);
+ xnl_msg_add_int_attr(hdr, XNL_ATTR_QPARAM_ERR_INFO,
+ xcmd->u.qparm.err_info);
+ break;
+#endif
+ default:
+ break;
+ }
+
+ rv = xnl_send(cb, hdr);
+ if (rv < 0)
+ goto out;
+
+ rv = xnl_recv(cb, hdr, dlen, 1);
+ if (rv < 0)
+ goto out;
+
+ rv = recv_nl_msg(hdr, xcmd);
+out:
+ free(msg);
+ return rv;
+}
diff --git a/QDMA/linux-kernel/user/cli/nl_user.h b/QDMA/linux-kernel/user/cli/nl_user.h
new file mode 100644
index 0000000000000000000000000000000000000000..69fd122ba990d0887afd1f1659855d71f8c52a70
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/nl_user.h
@@ -0,0 +1,95 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#ifndef __NL_USER_H__
+#define __NL_USER_H__
+
+#include <stdint.h>
+#ifdef ERR_DEBUG
+#include "qdma_nl.h"
+#endif
+
+#define DESC_SIZE_64B 3
+
+struct xcmd_info;
+
+enum q_parm_type {
+ QPARM_IDX,
+ QPARM_MODE,
+ QPARM_DIR,
+ QPARM_DESC,
+ QPARM_CMPT,
+ QPARM_CMPTSZ,
+ QPARM_SW_DESC_SZ,
+ QPARM_RNGSZ_IDX,
+ QPARM_C2H_BUFSZ_IDX,
+ QPARM_CMPT_TMR_IDX,
+ QPARM_CMPT_CNTR_IDX,
+ QPARM_CMPT_TRIG_MODE,
+ QPARM_PIPE_GL_MAX,
+ QPARM_PIPE_FLOW_ID,
+ QPARM_PIPE_SLR_ID,
+ QPARM_PIPE_TDEST,
+#ifdef ERR_DEBUG
+ QPARAM_ERR_NO,
+#endif
+
+ QPARM_MAX,
+};
+
+struct xcmd_q_parm {
+ unsigned int sflags;
+ uint32_t flags;
+ uint32_t idx;
+ uint32_t num_q;
+ uint32_t range_start;
+ uint32_t range_end;
+ unsigned char sw_desc_sz;
+ unsigned char cmpt_entry_size;
+ unsigned char qrngsz_idx;
+ unsigned char c2h_bufsz_idx;
+ unsigned char cmpt_tmr_idx;
+ unsigned char cmpt_cntr_idx;
+ unsigned char cmpt_trig_mode;
+ unsigned char is_qp;
+ uint8_t pipe_gl_max;
+ uint8_t pipe_flow_id;
+ uint8_t pipe_slr_id;
+ uint16_t pipe_tdest;
+#ifdef ERR_DEBUG
+ union {
+ unsigned int err_info;
+ struct {
+ unsigned int err_no:31;
+ unsigned int en:1;
+ } err;
+ };
+#endif
+};
+
+struct xcmd_intr {
+ unsigned int vector;
+ int start_idx;
+ int end_idx;
+};
+
+struct xnl_cb {
+ int fd;
+ unsigned short family;
+ unsigned int snd_seq;
+ unsigned int rcv_seq;
+};
+
+int xnl_connect(struct xnl_cb *cb, int vf);
+void xnl_close(struct xnl_cb *cb);
+int xnl_send_cmd(struct xnl_cb *cb, struct xcmd_info *xcmd);
+
+#endif /* ifndef __NL_USER_H__ */
diff --git a/QDMA/linux-kernel/user/cli/reg_cmd.c b/QDMA/linux-kernel/user/cli/reg_cmd.c
new file mode 100644
index 0000000000000000000000000000000000000000..c54e8aed76cdbfd04551f366f8259d9410e01685
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/reg_cmd.c
@@ -0,0 +1,401 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#include <endian.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "reg_cmd.h"
+#include "cmd_parse.h"
+#include "xdev_regs.h"
+
+#define QDMA_CFG_BAR_SIZE 0xB400
+#define QDMA_USR_BAR_SIZE 0x100
+#define STM_BAR_SIZE 0x0200003C
+
+struct xdev_info {
+ unsigned char bus;
+ unsigned char dev;
+ unsigned char func;
+ unsigned char config_bar;
+ unsigned char user_bar;
+ char stm_bar;
+};
+
+static struct xreg_info qdma_dmap_regs[] = {
+/* QDMA_TRQ_SEL_QUEUE_PF (0x6400) */
+ {"DMAP_SEL_INT_CIDX", 0x6400, 512, 0x10, 0, 0,},
+ {"DMAP_SEL_H2C_DSC_PIDX", 0x6404, 512, 0x10, 0, 0,},
+ {"DMAP_SEL_C2H_DSC_PIDX", 0x6408, 512, 0x10, 0, 0,},
+ {"DMAP_SEL_CMPT_CIDX", 0x640C, 512, 0x10, 0, 0,},
+ {"", 0, 0, 0 }
+};
+
+/*
+ * INTERNAL: for debug testing only
+ */
+static struct xreg_info stm_regs[] = {
+ {"H2C_DATA_0", 0x02000000, 0, 0, 0, 0,},
+ {"H2C_DATA_1", 0x02000004, 0, 0, 0, 0,},
+ {"H2C_DATA_2", 0x02000008, 0, 0, 0, 0,},
+ {"H2C_DATA_3", 0x0200000C, 0, 0, 0, 0,},
+ {"H2C_DATA_4", 0x02000010, 0, 0, 0, 0,},
+ {"IND_CTXT_CMD", 0x02000014, 0, 0, 0, 0,},
+ {"STM_REV", 0x02000018, 0, 0, 0, 0,},
+ {"CAM_CLR_STATUS", 0x0200001C, 0, 0, 0, 0,},
+ {"C2H_DATA8", 0x02000020, 0, 0, 0, 0,},
+ {"H2C_DATA_5", 0x02000024, 0, 0, 0, 0,},
+ {"STATUS", 0x0200002C, 0, 0, 0, 0,},
+ {"H2C_MODE", 0x02000030, 0, 0, 0, 0,},
+ {"H2C_STATUS", 0x02000034, 0, 0, 0, 0,},
+ {"C2H_MODE", 0x02000038, 0, 0, 0, 0,},
+ {"C2H_STATUS", 0x0200003C, 0, 0, 0, 0,},
+ {"H2C_PORT0_DEBUG0", 0x02000040, 0, 0, 0, 0,},
+ {"H2C_PORT1_DEBUG0", 0x02000044, 0, 0, 0, 0,},
+ {"H2C_PORT2_DEBUG0", 0x02000048, 0, 0, 0, 0,},
+ {"C2H_PORT0_DEBUG0", 0x02000050, 0, 0, 0, 0,},
+ {"C2H_PORT1_DEBUG0", 0x02000054, 0, 0, 0, 0,},
+ {"C2H_PORT2_DEBUG0", 0x02000058, 0, 0, 0, 0,},
+ {"H2C_DEBUG0", 0x02000060, 0, 0, 0, 0,},
+ {"H2C_DEBUG1", 0x02000064, 0, 0, 0, 0,},
+ {"AWERR", 0x02000068, 0, 0, 0, 0,},
+ {"ARERR", 0x0200006C, 0, 0, 0, 0,},
+ {"H2C_PORT0_DEBUG1", 0x02000070, 0, 0, 0, 0,},
+ {"H2C_PORT1_DEBUG1", 0x02000074, 0, 0, 0, 0,},
+ {"H2C_PORT2_DEBUG1", 0x02000078, 0, 0, 0, 0,},
+ {"", 0, 0, 0 }
+};
+
+
+/*
+ * Register I/O through mmap of BAR0.
+ */
+
+/* /sys/bus/pci/devices/0000:<bus>:<dev>.<func>/resource<bar#> */
+#define get_syspath_bar_mmap(s, bus,dev,func,bar) \
+ snprintf(s, sizeof(s), \
+ "/sys/bus/pci/devices/0000:%02x:%02x.%x/resource%u", \
+ bus, dev, func, bar)
+
+static uint32_t *mmap_bar(char *fname, size_t len, int prot)
+{
+ int fd;
+ uint32_t *bar;
+
+ fd = open(fname, (prot & PROT_WRITE) ? O_RDWR : O_RDONLY);
+ if (fd < 0)
+ return NULL;
+
+ bar = mmap(NULL, len, prot, MAP_SHARED, fd, 0);
+ close(fd);
+
+ return bar == MAP_FAILED ? NULL : bar;
+}
+
+static int32_t reg_read_mmap(struct xdev_info *xdev,
+ unsigned char barno,
+ struct xcmd_info *xcmd,
+ unsigned int *val)
+{
+ uint32_t *bar;
+ struct xnl_cb cb;
+ char fname[256];
+ int rv = 0;
+
+ memset(&cb, 0, sizeof(struct xnl_cb));
+
+ get_syspath_bar_mmap(fname, xdev->bus, xdev->dev, xdev->func, barno);
+
+ bar = mmap_bar(fname, xcmd->u.reg.reg + 4, PROT_READ);
+ if (!bar) {
+ if (xcmd->op == XNL_CMD_REG_WRT)
+ xcmd->op = XNL_CMD_REG_RD;
+
+ rv = xnl_connect(&cb, 0);
+ if (rv < 0)
+ return -1;
+ rv = xnl_send_cmd(&cb, xcmd);
+ if(rv != 0)
+ return -1;
+ xnl_close(&cb);
+
+ *val = le32toh(xcmd->u.reg.val);
+ return rv;
+ }
+
+ *val = le32toh(bar[xcmd->u.reg.reg / 4]);
+ munmap(bar, xcmd->u.reg.reg + 4);
+
+ return rv;
+}
+
+static int32_t reg_write_mmap(struct xdev_info *xdev,
+ unsigned char barno,
+ struct xcmd_info *xcmd)
+{
+ uint32_t *bar;
+ struct xnl_cb cb;
+ char fname[256];
+ int rv = 0;
+
+ memset(&cb, 0, sizeof(struct xnl_cb));
+ get_syspath_bar_mmap(fname, xdev->bus, xdev->dev, xdev->func, barno);
+
+ bar = mmap_bar(fname, xcmd->u.reg.reg + 4, PROT_WRITE);
+ if (!bar) {
+ rv = xnl_connect(&cb, 0);
+ if (rv < 0)
+ return -1;
+ rv = xnl_send_cmd(&cb, xcmd);
+ if (rv != 0)
+ return -1;
+ xnl_close(&cb);
+
+ return 0;
+ }
+
+ bar[xcmd->u.reg.reg / 4] = htole32(xcmd->u.reg.val);
+ munmap(bar, xcmd->u.reg.reg + 4);
+ return 0;
+}
+
+static void print_repeated_reg(uint32_t *bar, struct xreg_info *xreg,
+ unsigned start, unsigned limit, struct xdev_info *xdev, struct xcmd_info *xcmd)
+{
+ int i;
+ int end = start + limit;
+ int step = xreg->step ? xreg->step : 4;
+ uint32_t val;
+ int32_t rv = 0;
+
+ for (i = start; i < end; i++) {
+ uint32_t addr = xreg->addr + (i * step);
+ char name[40];
+ int l = sprintf(name, "%s_%d",
+ xreg->name, i);
+
+ if (xcmd == NULL) {
+ val = le32toh(bar[addr / 4]);
+ } else {
+ xcmd->u.reg.reg = addr;
+ rv = reg_read_mmap(xdev, xcmd->u.reg.bar, xcmd, &val);
+ if (rv < 0) {
+ printf("\n");
+ continue;
+ }
+ }
+ printf("[%#7x] %-47s %#-10x %u\n",
+ addr, name, val, val);
+ }
+}
+
+static void dump_regs(uint32_t *bar, struct xreg_info *reg_list, struct xdev_info *xdev, struct xcmd_info *xcmd)
+{
+ struct xreg_info *xreg = reg_list;
+ uint32_t val;
+ int32_t rv = 0;
+
+ for (xreg = reg_list; strlen(xreg->name); xreg++) {
+ if (!xreg->len) {
+ if (xreg->repeat) {
+ if (xcmd == NULL)
+ print_repeated_reg(bar, xreg, 0, xreg->repeat, NULL, NULL);
+ else
+ print_repeated_reg(NULL, xreg, 0, xreg->repeat, xdev, xcmd);
+ } else {
+ uint32_t addr = xreg->addr;
+ if (xcmd == NULL) {
+ val = le32toh(bar[addr / 4]);
+ } else {
+ xcmd->u.reg.reg = addr;
+ rv = reg_read_mmap(xdev, xcmd->u.reg.bar, xcmd, &val);
+ if (rv < 0)
+ continue;
+ }
+ printf("[%#7x] %-47s %#-10x %u\n",
+ addr, xreg->name, val, val);
+ }
+ } else {
+ uint32_t addr = xreg->addr;
+ if (xcmd == NULL) {
+ uint32_t val = le32toh(bar[addr / 4]);
+ } else {
+ xcmd->u.reg.reg = addr;
+ rv = reg_read_mmap(xdev, xcmd->u.reg.bar, xcmd, &val);
+ if (rv < 0)
+ continue;
+ }
+
+ uint32_t v = (val >> xreg->shift) &
+ ((1 << xreg->len) - 1);
+
+ printf(" %*u:%u %-47s %#-10x %u\n",
+ xreg->shift < 10 ? 3 : 2,
+ xreg->shift + xreg->len - 1,
+ xreg->shift, xreg->name, v, v);
+ }
+ }
+}
+
+static void reg_dump_mmap(struct xdev_info *xdev, unsigned char barno,
+ struct xreg_info *reg_list, unsigned int max, struct xcmd_info *xcmd)
+{
+ uint32_t *bar;
+ char fname[256];
+
+ get_syspath_bar_mmap(fname, xdev->bus, xdev->dev, xdev->func, barno);
+
+ bar = mmap_bar(fname, max, PROT_READ);
+ if (!bar) {
+ xcmd->op = XNL_CMD_REG_RD;
+ xcmd->u.reg.bar = barno;
+ dump_regs(NULL, reg_list, xdev, xcmd);
+ return;
+ }
+
+ dump_regs(bar, reg_list, NULL, NULL);
+ munmap(bar, max);
+}
+
+static void reg_dump_range(struct xdev_info *xdev, unsigned char barno,
+ unsigned int max, struct xreg_info *reg_list,
+ unsigned int start, unsigned int limit, struct xcmd_info *xcmd)
+{
+ struct xreg_info *xreg = reg_list;
+ uint32_t *bar;
+ char fname[256];
+
+ get_syspath_bar_mmap(fname, xdev->bus, xdev->dev, xdev->func, barno);
+
+ bar = mmap_bar(fname, max, PROT_READ);
+ if (!bar) {
+ xcmd->op = XNL_CMD_REG_RD;
+ xcmd->u.reg.bar = barno;
+ for (xreg = reg_list; strlen(xreg->name); xreg++) {
+ print_repeated_reg(NULL, xreg, start, limit, xdev, xcmd);
+ }
+ return;
+ }
+
+ for (xreg = reg_list; strlen(xreg->name); xreg++) {
+ print_repeated_reg(bar, xreg, start, limit, NULL, NULL);
+ }
+
+ munmap(bar, max);
+}
+
+static inline void print_seperator(void)
+{
+ char buffer[81];
+
+ memset(buffer, '#', 80);
+ buffer[80] = '\0';
+
+ fprintf(stdout, "%s\n", buffer);
+}
+
+int is_valid_addr(unsigned char bar_no, unsigned int reg_addr)
+{
+ struct xreg_info *rinfo = (bar_no == 0) ? qdma_config_regs:
+ qdma_user_regs;
+ unsigned int i;
+ unsigned int size = (bar_no == 0) ?
+ (sizeof(qdma_config_regs) / sizeof(struct xreg_info)):
+ (sizeof(qdma_user_regs) / sizeof(struct xreg_info));
+
+ for (i = 0; i < size; i++)
+ if (reg_addr == rinfo[i].addr)
+ return 1;
+
+ return 0;
+}
+
+int proc_reg_cmd(struct xcmd_info *xcmd)
+{
+ struct xcmd_reg *regcmd = &xcmd->u.reg;
+ struct xdev_info xdev;
+ int32_t rv = 0;
+ unsigned int mask = (1 << XNL_ATTR_PCI_BUS) | (1 << XNL_ATTR_PCI_DEV) |
+ (1 << XNL_ATTR_PCI_FUNC) | (1 << XNL_ATTR_DEV_CFG_BAR) |
+ (1 << XNL_ATTR_DEV_USR_BAR);
+ unsigned int barno;
+ int32_t v;
+
+ if ((xcmd->attr_mask & mask) != mask) {
+ fprintf(stderr, "%s: device info missing, 0x%x/0x%x.\n",
+ __FUNCTION__, xcmd->attr_mask, mask);
+ return -EINVAL;
+ }
+
+ memset(&xdev, 0, sizeof(struct xdev_info));
+ xdev.bus = xcmd->attrs[XNL_ATTR_PCI_BUS];
+ xdev.dev = xcmd->attrs[XNL_ATTR_PCI_DEV];
+ xdev.func = xcmd->attrs[XNL_ATTR_PCI_FUNC];
+ xdev.config_bar = xcmd->attrs[XNL_ATTR_DEV_CFG_BAR];
+ xdev.user_bar = xcmd->attrs[XNL_ATTR_DEV_USR_BAR];
+ xdev.stm_bar = xcmd->attrs[XNL_ATTR_DEV_STM_BAR];
+
+ barno = (regcmd->sflags & XCMD_REG_F_BAR_SET) ?
+ regcmd->bar : xdev.config_bar;
+
+ switch (xcmd->op) {
+ case XNL_CMD_REG_RD:
+ rv = reg_read_mmap(&xdev, barno, xcmd, &v);
+ if (rv < 0)
+ return -1;
+ fprintf(stdout, "qdma%05x, %02x:%02x.%02x, bar#%u, 0x%x = 0x%x.\n",
+ xcmd->if_bdf, xdev.bus, xdev.dev, xdev.func, barno,
+ regcmd->reg, v);
+ break;
+ case XNL_CMD_REG_WRT:
+ v = reg_write_mmap(&xdev, barno, xcmd);
+ if (v < 0)
+ return -1;
+ rv = reg_read_mmap(&xdev, barno, xcmd, &v);
+ if (rv < 0)
+ return -1;
+ fprintf(stdout, "qdma%05x, %02x:%02x.%02x, bar#%u, reg 0x%x -> 0x%x, read back 0x%x.\n",
+ xcmd->if_bdf, xdev.bus, xdev.dev, xdev.func, barno,
+ regcmd->reg, regcmd->val, v);
+ break;
+ case XNL_CMD_REG_DUMP:
+ print_seperator();
+ fprintf(stdout, "###\t\tqdma%05x, pci %02x:%02x.%02x, reg dump\n",
+ xcmd->if_bdf, xdev.bus, xdev.dev, xdev.func);
+ print_seperator();
+
+ fprintf(stdout, "\nUSER BAR #%d\n", xdev.user_bar);
+ reg_dump_mmap(&xdev, xdev.user_bar, qdma_user_regs,
+ QDMA_USR_BAR_SIZE, xcmd);
+
+ if (xdev.stm_bar != -1) {
+ fprintf(stdout, "\nSTM BAR #%d\n", xdev.stm_bar);
+ reg_dump_mmap(&xdev, xdev.stm_bar, stm_regs,
+ STM_BAR_SIZE, xcmd);
+ }
+
+ fprintf(stdout, "\nCONFIG BAR #%d\n", xdev.config_bar);
+ reg_dump_mmap(&xdev, xdev.config_bar, qdma_config_regs,
+ QDMA_CFG_BAR_SIZE, xcmd);
+ reg_dump_range(&xdev, xdev.config_bar, QDMA_CFG_BAR_SIZE,
+ qdma_dmap_regs, regcmd->range_start,
+ regcmd->range_end, xcmd);
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
diff --git a/QDMA/linux-kernel/user/cli/reg_cmd.h b/QDMA/linux-kernel/user/cli/reg_cmd.h
new file mode 100644
index 0000000000000000000000000000000000000000..bdbf487f4ed9e81de455e5e528d4dc776996c7d1
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/reg_cmd.h
@@ -0,0 +1,34 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#ifndef USER_CLI_REG_CMD_H_
+#define USER_CLI_REG_CMD_H_
+
+#include <stdint.h>
+
+struct xcmd_info;
+
+struct xcmd_reg {
+ unsigned int sflags;
+#define XCMD_REG_F_BAR_SET 0x1
+#define XCMD_REG_F_REG_SET 0x2
+#define XCMD_REG_F_VAL_SET 0x4
+ unsigned int bar;
+ unsigned int reg;
+ unsigned int val;
+ unsigned int range_start;
+ unsigned int range_end;
+};
+
+int proc_reg_cmd(struct xcmd_info *xcmd);
+int is_valid_addr(unsigned char bar_no, unsigned int reg_addr);
+
+#endif /* USER_CLI_REG_CMD_H_ */
diff --git a/QDMA/linux-kernel/user/cli/version.h b/QDMA/linux-kernel/user/cli/version.h
new file mode 100644
index 0000000000000000000000000000000000000000..521d80e9ba3a58146018a359196fbffe015f46b3
--- /dev/null
+++ b/QDMA/linux-kernel/user/cli/version.h
@@ -0,0 +1,19 @@
+/*
+ * This file is part of the QDMA userspace application
+ * to enable the user to execute the QDMA functionality
+ *
+ * Copyright (c) 2018-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree)
+ */
+
+#ifndef __XDMATOOL_VERSION_H
+#define __XDMATOOL_VERSION_H
+
+#define PROGNAME "dmatool"
+#define VERSION "1.0.13"
+#define COPYRIGHT "Copyright (c) 2018 Xilinx Inc."
+
+#endif
diff --git a/README.md b/README.md
index 4435ad53e91183e6ed803c6314f8e6a3221ea853..1c8a4269ce853f75c03a090252b84d0fd165ae40 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,18 @@
-# dma_ip_drivers
\ No newline at end of file
+==============
+Xilinx DMA IP Reference drivers
+==============
+
+-------------------------------------------------------------------------------
+The Xilinx PCI Express Multi Queue DMA (QDMA) IP provides high-performance direct memory access (DMA) via PCI Express. The PCIe QDMA can be implemented in UltraScale devices.
+
+Both the linux kernel driver and the DPDK driver can be run on a PCI Express root port host PC to interact with the QDMA endpoint IP via PCI Express.
+
+-------------------------------------------------------------------------------
+
+`QDMA DPDK Reference Driver User Guide <QDMA/DPDK/docs/DPDK_qdma_driver_user_guide.pdf>`_
+
+-------------------------------------------------------------------------------
+
+`QDMA Linux Kernel Reference Driver User Guide <QDMA/linux-kernel/docs/linux_qdma_driver_user_guide.pdf>`_
+
+-------------------------------------------------------------------------------