* For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
* irrelevant.
*/
- #include <asm-generic/io-64-nonatomic-hi-lo.h>
+ #include <linux/io-64-nonatomic-hi-lo.h>
static bool force_enable_dimms;
module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save_fail " : "",
flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore_fail " : "",
flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush_fail " : "",
- flags & ACPI_NFIT_MEM_ARMED ? "not_armed " : "",
+ flags & ACPI_NFIT_MEM_NOT_ARMED ? "not_armed " : "",
flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart_event " : "");
}
static DEVICE_ATTR_RO(flags);
flags |= NDD_ALIASING;
mem_flags = __to_nfit_memdev(nfit_mem)->flags;
- if (mem_flags & ACPI_NFIT_MEM_ARMED)
+ if (mem_flags & ACPI_NFIT_MEM_NOT_ARMED)
flags |= NDD_UNARMED;
rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? " save_fail" : "",
mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? " restore_fail":"",
mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? " flush_fail" : "",
- mem_flags & ACPI_NFIT_MEM_ARMED ? " not_armed" : "");
+ mem_flags & ACPI_NFIT_MEM_NOT_ARMED ? " not_armed" : "");
}
#include <asm/io.h>
#include <asm/uaccess.h>
- #include <asm-generic/io-64-nonatomic-lo-hi.h>
+ #include <linux/io-64-nonatomic-lo-hi.h>
#include "internal.h"
/* stuff for debugger support */
int acpi_in_debugger;
EXPORT_SYMBOL(acpi_in_debugger);
-
-extern char line_buf[80];
#endif /*ENABLE_DEBUGGER */
static int (*__acpi_os_prepare_sleep)(u8 sleep_state, u32 pm1a_ctrl,
static struct workqueue_struct *kacpi_notify_wq;
static struct workqueue_struct *kacpi_hotplug_wq;
static bool acpi_os_initialized;
+unsigned int acpi_sci_irq = INVALID_ACPI_IRQ;
/*
* This list of permanent mappings is for memory that may be accessed from
acpi_irq_handler = NULL;
return AE_NOT_ACQUIRED;
}
+ acpi_sci_irq = irq;
return AE_OK;
}
-acpi_status acpi_os_remove_interrupt_handler(u32 irq, acpi_osd_handler handler)
+acpi_status acpi_os_remove_interrupt_handler(u32 gsi, acpi_osd_handler handler)
{
- if (irq != acpi_gbl_FADT.sci_interrupt)
+ if (gsi != acpi_gbl_FADT.sci_interrupt || !acpi_sci_irq_valid())
return AE_BAD_PARAMETER;
- free_irq(irq, acpi_irq);
+ free_irq(acpi_sci_irq, acpi_irq);
acpi_irq_handler = NULL;
+ acpi_sci_irq = INVALID_ACPI_IRQ;
return AE_OK;
}
* Make sure the GPE handler or the fixed event handler is not used
* on another CPU after removal.
*/
- if (acpi_irq_handler)
- synchronize_hardirq(acpi_gbl_FADT.sci_interrupt);
+ if (acpi_sci_irq_valid())
+ synchronize_hardirq(acpi_sci_irq);
flush_workqueue(kacpid_wq);
flush_workqueue(kacpi_notify_wq);
}
return AE_OK;
}
-#ifdef ACPI_FUTURE_USAGE
-u32 acpi_os_get_line(char *buffer)
+acpi_status acpi_os_get_line(char *buffer, u32 buffer_length, u32 *bytes_read)
{
-
#ifdef ENABLE_DEBUGGER
if (acpi_in_debugger) {
u32 chars;
- kdb_read(buffer, sizeof(line_buf));
+ kdb_read(buffer, buffer_length);
/* remove the CR kdb includes */
chars = strlen(buffer) - 1;
}
#endif
- return 0;
+ return AE_OK;
}
-#endif /* ACPI_FUTURE_USAGE */
acpi_status acpi_os_signal(u32 function, void *info)
{
#include <linux/pm_qos.h>
#include <linux/pm_runtime.h>
#include <linux/seq_file.h>
-
- #include <asm-generic/io-64-nonatomic-lo-hi.h>
++#include <linux/io-64-nonatomic-lo-hi.h>
#include "intel-lpss.h"
#define LPSS_PRIV_SSP_REG 0x20
#define LPSS_PRIV_SSP_REG_DIS_DMA_FIN BIT(0)
-#define LPSS_PRIV_REMAP_ADDR_LO 0x40
-#define LPSS_PRIV_REMAP_ADDR_HI 0x44
+#define LPSS_PRIV_REMAP_ADDR 0x40
#define LPSS_PRIV_CAPS 0xfc
#define LPSS_PRIV_CAPS_NO_IDMA BIT(8)
{
resource_size_t addr = lpss->info->mem->start;
- writel(addr, lpss->priv + LPSS_PRIV_REMAP_ADDR_LO);
-#if BITS_PER_LONG > 32
- writel(addr >> 32, lpss->priv + LPSS_PRIV_REMAP_ADDR_HI);
-#else
- writel(0, lpss->priv + LPSS_PRIV_REMAP_ADDR_HI);
-#endif
+ lo_hi_writeq(addr, lpss->priv + LPSS_PRIV_REMAP_ADDR);
}
static void intel_lpss_deassert_reset(const struct intel_lpss *lpss)
* SOFTWARE.
*/
- #include <asm-generic/kmap_types.h>
+ #include <linux/highmem.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/pci.h>
pr_debug("\n");
}
+enum {
+ MLX5_DRIVER_STATUS_ABORTED = 0xfe,
+ MLX5_DRIVER_SYND = 0xbadd00de,
+};
+
+static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
+ u32 *synd, u8 *status)
+{
+ *synd = 0;
+ *status = 0;
+
+ switch (op) {
+ case MLX5_CMD_OP_TEARDOWN_HCA:
+ case MLX5_CMD_OP_DISABLE_HCA:
+ case MLX5_CMD_OP_MANAGE_PAGES:
+ case MLX5_CMD_OP_DESTROY_MKEY:
+ case MLX5_CMD_OP_DESTROY_EQ:
+ case MLX5_CMD_OP_DESTROY_CQ:
+ case MLX5_CMD_OP_DESTROY_QP:
+ case MLX5_CMD_OP_DESTROY_PSV:
+ case MLX5_CMD_OP_DESTROY_SRQ:
+ case MLX5_CMD_OP_DESTROY_XRC_SRQ:
+ case MLX5_CMD_OP_DESTROY_DCT:
+ case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
+ case MLX5_CMD_OP_DEALLOC_PD:
+ case MLX5_CMD_OP_DEALLOC_UAR:
+ case MLX5_CMD_OP_DETTACH_FROM_MCG:
+ case MLX5_CMD_OP_DEALLOC_XRCD:
+ case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
+ case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
+ case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
+ case MLX5_CMD_OP_DESTROY_TIR:
+ case MLX5_CMD_OP_DESTROY_SQ:
+ case MLX5_CMD_OP_DESTROY_RQ:
+ case MLX5_CMD_OP_DESTROY_RMP:
+ case MLX5_CMD_OP_DESTROY_TIS:
+ case MLX5_CMD_OP_DESTROY_RQT:
+ case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
+ case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
+ case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
+ return MLX5_CMD_STAT_OK;
+
+ case MLX5_CMD_OP_QUERY_HCA_CAP:
+ case MLX5_CMD_OP_QUERY_ADAPTER:
+ case MLX5_CMD_OP_INIT_HCA:
+ case MLX5_CMD_OP_ENABLE_HCA:
+ case MLX5_CMD_OP_QUERY_PAGES:
+ case MLX5_CMD_OP_SET_HCA_CAP:
+ case MLX5_CMD_OP_QUERY_ISSI:
+ case MLX5_CMD_OP_SET_ISSI:
+ case MLX5_CMD_OP_CREATE_MKEY:
+ case MLX5_CMD_OP_QUERY_MKEY:
+ case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
+ case MLX5_CMD_OP_PAGE_FAULT_RESUME:
+ case MLX5_CMD_OP_CREATE_EQ:
+ case MLX5_CMD_OP_QUERY_EQ:
+ case MLX5_CMD_OP_GEN_EQE:
+ case MLX5_CMD_OP_CREATE_CQ:
+ case MLX5_CMD_OP_QUERY_CQ:
+ case MLX5_CMD_OP_MODIFY_CQ:
+ case MLX5_CMD_OP_CREATE_QP:
+ case MLX5_CMD_OP_RST2INIT_QP:
+ case MLX5_CMD_OP_INIT2RTR_QP:
+ case MLX5_CMD_OP_RTR2RTS_QP:
+ case MLX5_CMD_OP_RTS2RTS_QP:
+ case MLX5_CMD_OP_SQERR2RTS_QP:
+ case MLX5_CMD_OP_2ERR_QP:
+ case MLX5_CMD_OP_2RST_QP:
+ case MLX5_CMD_OP_QUERY_QP:
+ case MLX5_CMD_OP_SQD_RTS_QP:
+ case MLX5_CMD_OP_INIT2INIT_QP:
+ case MLX5_CMD_OP_CREATE_PSV:
+ case MLX5_CMD_OP_CREATE_SRQ:
+ case MLX5_CMD_OP_QUERY_SRQ:
+ case MLX5_CMD_OP_ARM_RQ:
+ case MLX5_CMD_OP_CREATE_XRC_SRQ:
+ case MLX5_CMD_OP_QUERY_XRC_SRQ:
+ case MLX5_CMD_OP_ARM_XRC_SRQ:
+ case MLX5_CMD_OP_CREATE_DCT:
+ case MLX5_CMD_OP_DRAIN_DCT:
+ case MLX5_CMD_OP_QUERY_DCT:
+ case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
+ case MLX5_CMD_OP_QUERY_VPORT_STATE:
+ case MLX5_CMD_OP_MODIFY_VPORT_STATE:
+ case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
+ case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
+ case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
+ case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
+ case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
+ case MLX5_CMD_OP_SET_ROCE_ADDRESS:
+ case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
+ case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
+ case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
+ case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
+ case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
+ case MLX5_CMD_OP_ALLOC_Q_COUNTER:
+ case MLX5_CMD_OP_QUERY_Q_COUNTER:
+ case MLX5_CMD_OP_ALLOC_PD:
+ case MLX5_CMD_OP_ALLOC_UAR:
+ case MLX5_CMD_OP_CONFIG_INT_MODERATION:
+ case MLX5_CMD_OP_ACCESS_REG:
+ case MLX5_CMD_OP_ATTACH_TO_MCG:
+ case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
+ case MLX5_CMD_OP_MAD_IFC:
+ case MLX5_CMD_OP_QUERY_MAD_DEMUX:
+ case MLX5_CMD_OP_SET_MAD_DEMUX:
+ case MLX5_CMD_OP_NOP:
+ case MLX5_CMD_OP_ALLOC_XRCD:
+ case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
+ case MLX5_CMD_OP_QUERY_CONG_STATUS:
+ case MLX5_CMD_OP_MODIFY_CONG_STATUS:
+ case MLX5_CMD_OP_QUERY_CONG_PARAMS:
+ case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
+ case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
+ case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
+ case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
+ case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
+ case MLX5_CMD_OP_CREATE_TIR:
+ case MLX5_CMD_OP_MODIFY_TIR:
+ case MLX5_CMD_OP_QUERY_TIR:
+ case MLX5_CMD_OP_CREATE_SQ:
+ case MLX5_CMD_OP_MODIFY_SQ:
+ case MLX5_CMD_OP_QUERY_SQ:
+ case MLX5_CMD_OP_CREATE_RQ:
+ case MLX5_CMD_OP_MODIFY_RQ:
+ case MLX5_CMD_OP_QUERY_RQ:
+ case MLX5_CMD_OP_CREATE_RMP:
+ case MLX5_CMD_OP_MODIFY_RMP:
+ case MLX5_CMD_OP_QUERY_RMP:
+ case MLX5_CMD_OP_CREATE_TIS:
+ case MLX5_CMD_OP_MODIFY_TIS:
+ case MLX5_CMD_OP_QUERY_TIS:
+ case MLX5_CMD_OP_CREATE_RQT:
+ case MLX5_CMD_OP_MODIFY_RQT:
+ case MLX5_CMD_OP_QUERY_RQT:
+ case MLX5_CMD_OP_CREATE_FLOW_TABLE:
+ case MLX5_CMD_OP_QUERY_FLOW_TABLE:
+ case MLX5_CMD_OP_CREATE_FLOW_GROUP:
+ case MLX5_CMD_OP_QUERY_FLOW_GROUP:
+ case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
+ case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
+ *status = MLX5_DRIVER_STATUS_ABORTED;
+ *synd = MLX5_DRIVER_SYND;
+ return -EIO;
+ default:
+ mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
+ return -EINVAL;
+ }
+}
+
const char *mlx5_command_str(int command)
{
switch (command) {
struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd);
struct mlx5_cmd_layout *lay;
struct semaphore *sem;
+ unsigned long flags;
sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
down(sem);
}
} else {
ent->idx = cmd->max_reg_cmds;
+ spin_lock_irqsave(&cmd->alloc_lock, flags);
+ clear_bit(ent->idx, &cmd->bitmask);
+ spin_unlock_irqrestore(&cmd->alloc_lock, flags);
}
ent->token = alloc_token(cmd);
return err;
}
+static __be32 *get_synd_ptr(struct mlx5_outbox_hdr *out)
+{
+ return &out->syndrome;
+}
+
+static u8 *get_status_ptr(struct mlx5_outbox_hdr *out)
+{
+ return &out->status;
+}
+
/* Notes:
* 1. Callback functions may not sleep
* 2. page queue commands do not support asynchrous completion
}
}
-void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector)
+void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec)
{
struct mlx5_cmd *cmd = &dev->cmd;
struct mlx5_cmd_work_ent *ent;
s64 ds;
struct mlx5_cmd_stats *stats;
unsigned long flags;
+ unsigned long vector;
+ /* there can be at most 32 command queues */
+ vector = vec & 0xffffffff;
for (i = 0; i < (1 << cmd->log_sz); i++) {
if (test_bit(i, &vector)) {
struct semaphore *sem;
ent->ret = verify_signature(ent);
else
ent->ret = 0;
- ent->status = ent->lay->status_own >> 1;
+ if (vec & MLX5_TRIGGERED_CMD_COMP)
+ ent->status = MLX5_DRIVER_STATUS_ABORTED;
+ else
+ ent->status = ent->lay->status_own >> 1;
+
mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
ent->ret, deliv_status_to_str(ent->status), ent->status);
}
free_ent(cmd, ent->idx);
+
if (ent->callback) {
ds = ent->ts2 - ent->ts1;
if (ent->op < ARRAY_SIZE(cmd->stats)) {
mlx5_free_cmd_msg(dev, ent->out);
free_msg(dev, ent->in);
+ err = err ? err : ent->status;
free_cmd(ent);
callback(err, context);
} else {
return msg;
}
+static u16 opcode_from_in(struct mlx5_inbox_hdr *in)
+{
+ return be16_to_cpu(in->opcode);
+}
+
static int is_manage_pages(struct mlx5_inbox_hdr *in)
{
return be16_to_cpu(in->opcode) == MLX5_CMD_OP_MANAGE_PAGES;
gfp_t gfp;
int err;
u8 status = 0;
+ u32 drv_synd;
+
+ if (pci_channel_offline(dev->pdev) ||
+ dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
+ err = mlx5_internal_err_ret_value(dev, opcode_from_in(in), &drv_synd, &status);
+ *get_synd_ptr(out) = cpu_to_be32(drv_synd);
+ *get_status_ptr(out) = status;
+ return err;
+ }
pages_queue = is_manage_pages(in);
gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
int err;
int i;
+ memset(cmd, 0, sizeof(*cmd));
cmd_if_rev = cmdif_rev(dev);
if (cmd_if_rev != CMD_IF_REV) {
dev_err(&dev->pdev->dev,
* SOFTWARE.
*/
- #include <asm-generic/kmap_types.h>
+ #include <linux/highmem.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/io-mapping.h>
#include <linux/interrupt.h>
+#include <linux/delay.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cq.h>
#include <linux/mlx5/qp.h>
#include <linux/mlx5/srq.h>
#include <linux/debugfs.h>
#include <linux/kmod.h>
+#include <linux/delay.h>
#include <linux/mlx5/mlx5_ifc.h>
#include "mlx5_core.h"
module_param_named(prof_sel, prof_sel, int, 0444);
MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
-struct workqueue_struct *mlx5_core_wq;
static LIST_HEAD(intf_list);
static LIST_HEAD(dev_list);
static DEFINE_MUTEX(intf_mutex);
},
};
+#define FW_INIT_TIMEOUT_MILI 2000
+#define FW_INIT_WAIT_MS 2
+
+static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili)
+{
+ unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
+ int err = 0;
+
+ while (fw_initializing(dev)) {
+ if (time_after(jiffies, end)) {
+ err = -EBUSY;
+ break;
+ }
+ msleep(FW_INIT_WAIT_MS);
+ }
+
+ return err;
+}
+
static int set_dma_caps(struct pci_dev *pdev)
{
int err;
return err;
}
+static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
+{
+ struct pci_dev *pdev = dev->pdev;
+ int err = 0;
+
+ mutex_lock(&dev->pci_status_mutex);
+ if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
+ err = pci_enable_device(pdev);
+ if (!err)
+ dev->pci_status = MLX5_PCI_STATUS_ENABLED;
+ }
+ mutex_unlock(&dev->pci_status_mutex);
+
+ return err;
+}
+
+static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
+{
+ struct pci_dev *pdev = dev->pdev;
+
+ mutex_lock(&dev->pci_status_mutex);
+ if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
+ pci_disable_device(pdev);
+ dev->pci_status = MLX5_PCI_STATUS_DISABLED;
+ }
+ mutex_unlock(&dev->pci_status_mutex);
+}
+
static int request_bar(struct pci_dev *pdev)
{
int err = 0;
io_mapping_free(dev->priv.bf_mapping);
}
-static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
+static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+ struct mlx5_device_context *dev_ctx;
+ struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+ dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
+ if (!dev_ctx)
+ return;
+
+ dev_ctx->intf = intf;
+ dev_ctx->context = intf->add(dev);
+
+ if (dev_ctx->context) {
+ spin_lock_irq(&priv->ctx_lock);
+ list_add_tail(&dev_ctx->list, &priv->ctx_list);
+ spin_unlock_irq(&priv->ctx_lock);
+ } else {
+ kfree(dev_ctx);
+ }
+}
+
+static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+ struct mlx5_device_context *dev_ctx;
+ struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+ list_for_each_entry(dev_ctx, &priv->ctx_list, list)
+ if (dev_ctx->intf == intf) {
+ spin_lock_irq(&priv->ctx_lock);
+ list_del(&dev_ctx->list);
+ spin_unlock_irq(&priv->ctx_lock);
+
+ intf->remove(dev, dev_ctx->context);
+ kfree(dev_ctx);
+ return;
+ }
+}
+
+static int mlx5_register_device(struct mlx5_core_dev *dev)
{
struct mlx5_priv *priv = &dev->priv;
- int err;
+ struct mlx5_interface *intf;
+
+ mutex_lock(&intf_mutex);
+ list_add_tail(&priv->dev_list, &dev_list);
+ list_for_each_entry(intf, &intf_list, list)
+ mlx5_add_device(intf, priv);
+ mutex_unlock(&intf_mutex);
+
+ return 0;
+}
+
+static void mlx5_unregister_device(struct mlx5_core_dev *dev)
+{
+ struct mlx5_priv *priv = &dev->priv;
+ struct mlx5_interface *intf;
+
+ mutex_lock(&intf_mutex);
+ list_for_each_entry(intf, &intf_list, list)
+ mlx5_remove_device(intf, priv);
+ list_del(&priv->dev_list);
+ mutex_unlock(&intf_mutex);
+}
+
+int mlx5_register_interface(struct mlx5_interface *intf)
+{
+ struct mlx5_priv *priv;
+
+ if (!intf->add || !intf->remove)
+ return -EINVAL;
+
+ mutex_lock(&intf_mutex);
+ list_add_tail(&intf->list, &intf_list);
+ list_for_each_entry(priv, &dev_list, dev_list)
+ mlx5_add_device(intf, priv);
+ mutex_unlock(&intf_mutex);
+
+ return 0;
+}
+EXPORT_SYMBOL(mlx5_register_interface);
+
+void mlx5_unregister_interface(struct mlx5_interface *intf)
+{
+ struct mlx5_priv *priv;
+
+ mutex_lock(&intf_mutex);
+ list_for_each_entry(priv, &dev_list, dev_list)
+ mlx5_remove_device(intf, priv);
+ list_del(&intf->list);
+ mutex_unlock(&intf_mutex);
+}
+EXPORT_SYMBOL(mlx5_unregister_interface);
+
+void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
+{
+ struct mlx5_priv *priv = &mdev->priv;
+ struct mlx5_device_context *dev_ctx;
+ unsigned long flags;
+ void *result = NULL;
+
+ spin_lock_irqsave(&priv->ctx_lock, flags);
+
+ list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
+ if ((dev_ctx->intf->protocol == protocol) &&
+ dev_ctx->intf->get_dev) {
+ result = dev_ctx->intf->get_dev(dev_ctx->context);
+ break;
+ }
+
+ spin_unlock_irqrestore(&priv->ctx_lock, flags);
+
+ return result;
+}
+EXPORT_SYMBOL(mlx5_get_protocol_dev);
+
+static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+{
+ struct pci_dev *pdev = dev->pdev;
+ int err = 0;
- dev->pdev = pdev;
pci_set_drvdata(dev->pdev, dev);
strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
if (!priv->dbg_root)
return -ENOMEM;
- err = pci_enable_device(pdev);
+ err = mlx5_pci_enable_device(dev);
if (err) {
dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
goto err_dbg;
dev_err(&pdev->dev, "Failed mapping initialization segment, aborting\n");
goto err_clr_master;
}
+
+ return 0;
+
+err_clr_master:
+ pci_clear_master(dev->pdev);
+ release_bar(dev->pdev);
+err_disable:
+ mlx5_pci_disable_device(dev);
+
+err_dbg:
+ debugfs_remove(priv->dbg_root);
+ return err;
+}
+
+static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+{
+ iounmap(dev->iseg);
+ pci_clear_master(dev->pdev);
+ release_bar(dev->pdev);
+ mlx5_pci_disable_device(dev);
+ debugfs_remove(priv->dbg_root);
+}
+
+#define MLX5_IB_MOD "mlx5_ib"
+static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
+{
+ struct pci_dev *pdev = dev->pdev;
+ int err;
+
+ mutex_lock(&dev->intf_state_mutex);
+ if (dev->interface_state == MLX5_INTERFACE_STATE_UP) {
+ dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n",
+ __func__);
+ goto out;
+ }
+
dev_info(&pdev->dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
fw_rev_min(dev), fw_rev_sub(dev));
+ /* on load removing any previous indication of internal error, device is
+ * up
+ */
+ dev->state = MLX5_DEVICE_STATE_UP;
+
err = mlx5_cmd_init(dev);
if (err) {
dev_err(&pdev->dev, "Failed initializing command interface, aborting\n");
- goto err_unmap;
+ goto out_err;
+ }
+
+ err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
+ if (err) {
+ dev_err(&dev->pdev->dev, "Firmware over %d MS in initializing state, aborting\n",
+ FW_INIT_TIMEOUT_MILI);
+ goto out_err;
}
mlx5_pagealloc_init(dev);
mlx5_init_srq_table(dev);
mlx5_init_mr_table(dev);
+ err = mlx5_register_device(dev);
+ if (err) {
+ dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
+ goto err_reg_dev;
+ }
+
+ err = request_module_nowait(MLX5_IB_MOD);
+ if (err)
+ pr_info("failed request module on %s\n", MLX5_IB_MOD);
+
+ dev->interface_state = MLX5_INTERFACE_STATE_UP;
+out:
+ mutex_unlock(&dev->intf_state_mutex);
+
return 0;
+err_reg_dev:
+ mlx5_cleanup_mr_table(dev);
+ mlx5_cleanup_srq_table(dev);
+ mlx5_cleanup_qp_table(dev);
+ mlx5_cleanup_cq_table(dev);
+ mlx5_irq_clear_affinity_hints(dev);
+
err_unmap_bf_area:
unmap_bf_area(dev);
mlx5_stop_health_poll(dev);
if (mlx5_cmd_teardown_hca(dev)) {
dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
- return err;
+ goto out_err;
}
err_pagealloc_stop:
mlx5_pagealloc_cleanup(dev);
mlx5_cmd_cleanup(dev);
-err_unmap:
- iounmap(dev->iseg);
-
-err_clr_master:
- pci_clear_master(dev->pdev);
- release_bar(dev->pdev);
-
-err_disable:
- pci_disable_device(dev->pdev);
+out_err:
+ dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
+ mutex_unlock(&dev->intf_state_mutex);
-err_dbg:
- debugfs_remove(priv->dbg_root);
return err;
}
-static void mlx5_dev_cleanup(struct mlx5_core_dev *dev)
+static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
{
- struct mlx5_priv *priv = &dev->priv;
+ int err = 0;
+ mutex_lock(&dev->intf_state_mutex);
+ if (dev->interface_state == MLX5_INTERFACE_STATE_DOWN) {
+ dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n",
+ __func__);
+ goto out;
+ }
+ mlx5_unregister_device(dev);
+ mlx5_cleanup_mr_table(dev);
mlx5_cleanup_srq_table(dev);
mlx5_cleanup_qp_table(dev);
mlx5_cleanup_cq_table(dev);
mlx5_eq_cleanup(dev);
mlx5_disable_msix(dev);
mlx5_stop_health_poll(dev);
- if (mlx5_cmd_teardown_hca(dev)) {
+ err = mlx5_cmd_teardown_hca(dev);
+ if (err) {
dev_err(&dev->pdev->dev, "tear_down_hca failed, skip cleanup\n");
- return;
+ goto out;
}
mlx5_pagealloc_stop(dev);
mlx5_reclaim_startup_pages(dev);
mlx5_core_disable_hca(dev);
mlx5_pagealloc_cleanup(dev);
mlx5_cmd_cleanup(dev);
- iounmap(dev->iseg);
- pci_clear_master(dev->pdev);
- release_bar(dev->pdev);
- pci_disable_device(dev->pdev);
- debugfs_remove(priv->dbg_root);
-}
-
-static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
-{
- struct mlx5_device_context *dev_ctx;
- struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
-
- dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
- if (!dev_ctx) {
- pr_warn("mlx5_add_device: alloc context failed\n");
- return;
- }
-
- dev_ctx->intf = intf;
- dev_ctx->context = intf->add(dev);
-
- if (dev_ctx->context) {
- spin_lock_irq(&priv->ctx_lock);
- list_add_tail(&dev_ctx->list, &priv->ctx_list);
- spin_unlock_irq(&priv->ctx_lock);
- } else {
- kfree(dev_ctx);
- }
-}
-
-static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
-{
- struct mlx5_device_context *dev_ctx;
- struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
-
- list_for_each_entry(dev_ctx, &priv->ctx_list, list)
- if (dev_ctx->intf == intf) {
- spin_lock_irq(&priv->ctx_lock);
- list_del(&dev_ctx->list);
- spin_unlock_irq(&priv->ctx_lock);
-
- intf->remove(dev, dev_ctx->context);
- kfree(dev_ctx);
- return;
- }
-}
-static int mlx5_register_device(struct mlx5_core_dev *dev)
-{
- struct mlx5_priv *priv = &dev->priv;
- struct mlx5_interface *intf;
-
- mutex_lock(&intf_mutex);
- list_add_tail(&priv->dev_list, &dev_list);
- list_for_each_entry(intf, &intf_list, list)
- mlx5_add_device(intf, priv);
- mutex_unlock(&intf_mutex);
-
- return 0;
-}
-static void mlx5_unregister_device(struct mlx5_core_dev *dev)
-{
- struct mlx5_priv *priv = &dev->priv;
- struct mlx5_interface *intf;
-
- mutex_lock(&intf_mutex);
- list_for_each_entry(intf, &intf_list, list)
- mlx5_remove_device(intf, priv);
- list_del(&priv->dev_list);
- mutex_unlock(&intf_mutex);
-}
-
-int mlx5_register_interface(struct mlx5_interface *intf)
-{
- struct mlx5_priv *priv;
-
- if (!intf->add || !intf->remove)
- return -EINVAL;
-
- mutex_lock(&intf_mutex);
- list_add_tail(&intf->list, &intf_list);
- list_for_each_entry(priv, &dev_list, dev_list)
- mlx5_add_device(intf, priv);
- mutex_unlock(&intf_mutex);
-
- return 0;
-}
-EXPORT_SYMBOL(mlx5_register_interface);
-
-void mlx5_unregister_interface(struct mlx5_interface *intf)
-{
- struct mlx5_priv *priv;
-
- mutex_lock(&intf_mutex);
- list_for_each_entry(priv, &dev_list, dev_list)
- mlx5_remove_device(intf, priv);
- list_del(&intf->list);
- mutex_unlock(&intf_mutex);
-}
-EXPORT_SYMBOL(mlx5_unregister_interface);
-
-void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
-{
- struct mlx5_priv *priv = &mdev->priv;
- struct mlx5_device_context *dev_ctx;
- unsigned long flags;
- void *result = NULL;
-
- spin_lock_irqsave(&priv->ctx_lock, flags);
-
- list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
- if ((dev_ctx->intf->protocol == protocol) &&
- dev_ctx->intf->get_dev) {
- result = dev_ctx->intf->get_dev(dev_ctx->context);
- break;
- }
- spin_unlock_irqrestore(&priv->ctx_lock, flags);
-
- return result;
+out:
+ dev->interface_state = MLX5_INTERFACE_STATE_DOWN;
+ mutex_unlock(&dev->intf_state_mutex);
+ return err;
}
-EXPORT_SYMBOL(mlx5_get_protocol_dev);
-static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
- unsigned long param)
+void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
+ unsigned long param)
{
struct mlx5_priv *priv = &dev->priv;
struct mlx5_device_context *dev_ctx;
void *data);
};
-#define MLX5_IB_MOD "mlx5_ib"
static int init_one(struct pci_dev *pdev,
const struct pci_device_id *id)
prof_sel = MLX5_DEFAULT_PROF;
}
dev->profile = &profile[prof_sel];
+ dev->pdev = pdev;
dev->event = mlx5_core_event;
INIT_LIST_HEAD(&priv->ctx_list);
spin_lock_init(&priv->ctx_lock);
- err = mlx5_dev_init(dev, pdev);
+ mutex_init(&dev->pci_status_mutex);
+ mutex_init(&dev->intf_state_mutex);
+ err = mlx5_pci_init(dev, priv);
if (err) {
- dev_err(&pdev->dev, "mlx5_dev_init failed %d\n", err);
- goto out;
+ dev_err(&pdev->dev, "mlx5_pci_init failed with error code %d\n", err);
+ goto clean_dev;
}
- err = mlx5_register_device(dev);
+ err = mlx5_health_init(dev);
if (err) {
- dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
- goto out_init;
+ dev_err(&pdev->dev, "mlx5_health_init failed with error code %d\n", err);
+ goto close_pci;
}
- err = request_module_nowait(MLX5_IB_MOD);
- if (err)
- pr_info("failed request module on %s\n", MLX5_IB_MOD);
+ err = mlx5_load_one(dev, priv);
+ if (err) {
+ dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err);
+ goto clean_health;
+ }
return 0;
-out_init:
- mlx5_dev_cleanup(dev);
-out:
+clean_health:
+ mlx5_health_cleanup(dev);
+close_pci:
+ mlx5_pci_close(dev, priv);
+clean_dev:
+ pci_set_drvdata(pdev, NULL);
kfree(dev);
+
return err;
}
+
static void remove_one(struct pci_dev *pdev)
{
struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+ struct mlx5_priv *priv = &dev->priv;
- mlx5_unregister_device(dev);
- mlx5_dev_cleanup(dev);
+ if (mlx5_unload_one(dev, priv)) {
+ dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
+ mlx5_health_cleanup(dev);
+ return;
+ }
+ mlx5_health_cleanup(dev);
+ mlx5_pci_close(dev, priv);
+ pci_set_drvdata(pdev, NULL);
kfree(dev);
}
+static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
+ pci_channel_state_t state)
+{
+ struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+ struct mlx5_priv *priv = &dev->priv;
+
+ dev_info(&pdev->dev, "%s was called\n", __func__);
+ mlx5_enter_error_state(dev);
+ mlx5_unload_one(dev, priv);
+ mlx5_pci_disable_device(dev);
+ return state == pci_channel_io_perm_failure ?
+ PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
+{
+ struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+ int err = 0;
+
+ dev_info(&pdev->dev, "%s was called\n", __func__);
+
+ err = mlx5_pci_enable_device(dev);
+ if (err) {
+ dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n"
+ , __func__, err);
+ return PCI_ERS_RESULT_DISCONNECT;
+ }
+ pci_set_master(pdev);
+ pci_set_power_state(pdev, PCI_D0);
+ pci_restore_state(pdev);
+
+ return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
+}
+
+void mlx5_disable_device(struct mlx5_core_dev *dev)
+{
+ mlx5_pci_err_detected(dev->pdev, 0);
+}
+
+/* wait for the device to show vital signs. For now we check
+ * that we can read the device ID and that the health buffer
+ * shows a non zero value which is different than 0xffffffff
+ */
+static void wait_vital(struct pci_dev *pdev)
+{
+ struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+ struct mlx5_core_health *health = &dev->priv.health;
+ const int niter = 100;
+ u32 count;
+ u16 did;
+ int i;
+
+ /* Wait for firmware to be ready after reset */
+ msleep(1000);
+ for (i = 0; i < niter; i++) {
+ if (pci_read_config_word(pdev, 2, &did)) {
+ dev_warn(&pdev->dev, "failed reading config word\n");
+ break;
+ }
+ if (did == pdev->device) {
+ dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i);
+ break;
+ }
+ msleep(50);
+ }
+ if (i == niter)
+ dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
+
+ for (i = 0; i < niter; i++) {
+ count = ioread32be(health->health_counter);
+ if (count && count != 0xffffffff) {
+ dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i);
+ break;
+ }
+ msleep(50);
+ }
+
+ if (i == niter)
+ dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
+}
+
+static void mlx5_pci_resume(struct pci_dev *pdev)
+{
+ struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
+ struct mlx5_priv *priv = &dev->priv;
+ int err;
+
+ dev_info(&pdev->dev, "%s was called\n", __func__);
+
+ pci_save_state(pdev);
+ wait_vital(pdev);
+
+ err = mlx5_load_one(dev, priv);
+ if (err)
+ dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n"
+ , __func__, err);
+ else
+ dev_info(&pdev->dev, "%s: device recovered\n", __func__);
+}
+
+static const struct pci_error_handlers mlx5_err_handler = {
+ .error_detected = mlx5_pci_err_detected,
+ .slot_reset = mlx5_pci_slot_reset,
+ .resume = mlx5_pci_resume
+};
+
static const struct pci_device_id mlx5_core_pci_table[] = {
{ PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */
{ PCI_VDEVICE(MELLANOX, 0x1012) }, /* Connect-IB VF */
.name = DRIVER_NAME,
.id_table = mlx5_core_pci_table,
.probe = init_one,
- .remove = remove_one
+ .remove = remove_one,
+ .err_handler = &mlx5_err_handler
};
static int __init init(void)
int err;
mlx5_register_debugfs();
- mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq");
- if (!mlx5_core_wq) {
- err = -ENOMEM;
- goto err_debug;
- }
- mlx5_health_init();
err = pci_register_driver(&mlx5_core_driver);
if (err)
- goto err_health;
+ goto err_debug;
#ifdef CONFIG_MLX5_CORE_EN
mlx5e_init();
return 0;
-err_health:
- mlx5_health_cleanup();
- destroy_workqueue(mlx5_core_wq);
err_debug:
mlx5_unregister_debugfs();
return err;
mlx5e_cleanup();
#endif
pci_unregister_driver(&mlx5_core_driver);
- mlx5_health_cleanup();
- destroy_workqueue(mlx5_core_wq);
mlx5_unregister_debugfs();
}
* SOFTWARE.
*/
- #include <asm-generic/kmap_types.h>
+ #include <linux/highmem.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mlx5/driver.h>
return err;
}
+
+static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id)
+{
+ struct mlx5_manage_pages_inbox *in;
+ struct mlx5_manage_pages_outbox out;
+ int err;
+
+ in = kzalloc(sizeof(*in), GFP_KERNEL);
+ if (!in)
+ return;
+
+ memset(&out, 0, sizeof(out));
+ in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
+ in->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
+ in->func_id = cpu_to_be16(func_id);
+ err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
+ if (!err)
+ err = mlx5_cmd_status_to_err(&out.hdr);
+
+ if (err)
+ mlx5_core_warn(dev, "page notify failed\n");
+
+ kfree(in);
+}
+
static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
int notify_fail)
{
struct mlx5_manage_pages_inbox *in;
struct mlx5_manage_pages_outbox out;
- struct mlx5_manage_pages_inbox *nin;
int inlen;
u64 addr;
int err;
inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
in = mlx5_vzalloc(inlen);
if (!in) {
+ err = -ENOMEM;
mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
- return -ENOMEM;
+ goto out_free;
}
memset(&out, 0, sizeof(out));
if (err) {
mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
func_id, npages, err);
- goto out_alloc;
+ goto out_4k;
}
dev->priv.fw_pages += npages;
- if (out.hdr.status) {
- err = mlx5_cmd_status_to_err(&out.hdr);
- if (err) {
- mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n",
- func_id, npages, out.hdr.status);
- goto out_alloc;
- }
+ err = mlx5_cmd_status_to_err(&out.hdr);
+ if (err) {
+ mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n",
+ func_id, npages, out.hdr.status);
+ goto out_4k;
}
mlx5_core_dbg(dev, "err %d\n", err);
- goto out_free;
-
-out_alloc:
- if (notify_fail) {
- nin = kzalloc(sizeof(*nin), GFP_KERNEL);
- if (!nin) {
- mlx5_core_warn(dev, "allocation failed\n");
- goto out_4k;
- }
- memset(&out, 0, sizeof(out));
- nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
- nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
- if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out)))
- mlx5_core_warn(dev, "page notify failed\n");
- kfree(nin);
- }
+ kvfree(in);
+ return 0;
out_4k:
for (i--; i >= 0; i--)
free_4k(dev, be64_to_cpu(in->pas[i]));
out_free:
kvfree(in);
+ if (notify_fail)
+ page_notify_fail(dev, func_id);
return err;
}
struct fw_page *fwp;
struct rb_node *p;
int nclaimed = 0;
- int err;
+ int err = 0;
do {
p = rb_first(&dev->priv.page_root);
if (p) {
fwp = rb_entry(p, struct fw_page, rb_node);
- err = reclaim_pages(dev, fwp->func_id,
- optimal_reclaimed_pages(),
- &nclaimed);
+ if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
+ free_4k(dev, fwp->addr);
+ nclaimed = 1;
+ } else {
+ err = reclaim_pages(dev, fwp->func_id,
+ optimal_reclaimed_pages(),
+ &nclaimed);
+ }
if (err) {
mlx5_core_warn(dev, "failed reclaiming pages (%d)\n",
err);
#include <net/ip_fib.h>
#include <net/netevent.h>
#include <net/arp.h>
- #include <asm-generic/io-64-nonatomic-lo-hi.h>
+ #include <linux/io-64-nonatomic-lo-hi.h>
#include <generated/utsrelease.h>
#include "rocker.h"
struct hlist_node entry;
u32 key_crc32; /* key */
bool learned;
+ unsigned long touched;
struct rocker_fdb_tbl_key {
- u32 pport;
+ struct rocker_port *rocker_port;
u8 addr[ETH_ALEN];
__be16 vlan_id;
} key;
__be16 internal_vlan_id;
int stp_state;
u32 brport_flags;
+ unsigned long ageing_time;
bool ctrls[ROCKER_CTRL_MAX];
unsigned long vlan_bitmap[ROCKER_VLAN_BITMAP_LEN];
struct napi_struct napi_tx;
struct napi_struct napi_rx;
struct rocker_dma_ring_info tx_ring;
struct rocker_dma_ring_info rx_ring;
- struct list_head trans_mem;
};
struct rocker {
u64 flow_tbl_next_cookie;
DECLARE_HASHTABLE(group_tbl, 16);
spinlock_t group_tbl_lock; /* for group tbl accesses */
+ struct timer_list fdb_cleanup_timer;
DECLARE_HASHTABLE(fdb_tbl, 16);
spinlock_t fdb_tbl_lock; /* for fdb tbl accesses */
unsigned long internal_vlan_bitmap[ROCKER_INTERNAL_VLAN_BITMAP_LEN];
#define ROCKER_OP_FLAG_REFRESH BIT(3)
static void *__rocker_port_mem_alloc(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
size_t size)
{
- struct list_head *elem = NULL;
+ struct switchdev_trans_item *elem = NULL;
gfp_t gfp_flags = (flags & ROCKER_OP_FLAG_NOWAIT) ?
GFP_ATOMIC : GFP_KERNEL;
/* If in transaction prepare phase, allocate the memory
- * and enqueue it on a per-port list. If in transaction
- * commit phase, dequeue the memory from the per-port list
+ * and enqueue it on a transaction. If in transaction
+ * commit phase, dequeue the memory from the transaction
* rather than re-allocating the memory. The idea is the
* driver code paths for prepare and commit are identical
* so the memory allocated in the prepare phase is the
* memory used in the commit phase.
*/
- switch (trans) {
- case SWITCHDEV_TRANS_PREPARE:
+ if (!trans) {
+ elem = kzalloc(size + sizeof(*elem), gfp_flags);
+ } else if (switchdev_trans_ph_prepare(trans)) {
elem = kzalloc(size + sizeof(*elem), gfp_flags);
if (!elem)
return NULL;
- list_add_tail(elem, &rocker_port->trans_mem);
- break;
- case SWITCHDEV_TRANS_COMMIT:
- BUG_ON(list_empty(&rocker_port->trans_mem));
- elem = rocker_port->trans_mem.next;
- list_del_init(elem);
- break;
- case SWITCHDEV_TRANS_NONE:
- elem = kzalloc(size + sizeof(*elem), gfp_flags);
- if (elem)
- INIT_LIST_HEAD(elem);
- break;
- default:
- break;
+ switchdev_trans_item_enqueue(trans, elem, kfree, elem);
+ } else {
+ elem = switchdev_trans_item_dequeue(trans);
}
return elem ? elem + 1 : NULL;
}
static void *rocker_port_kzalloc(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
size_t size)
{
return __rocker_port_mem_alloc(rocker_port, trans, flags, size);
}
static void *rocker_port_kcalloc(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
size_t n, size_t size)
{
return __rocker_port_mem_alloc(rocker_port, trans, flags, n * size);
}
-static void rocker_port_kfree(enum switchdev_trans trans, const void *mem)
+static void rocker_port_kfree(struct switchdev_trans *trans, const void *mem)
{
- struct list_head *elem;
+ struct switchdev_trans_item *elem;
/* Frees are ignored if in transaction prepare phase. The
* memory remains on the per-port list until freed in the
* commit phase.
*/
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
return;
- elem = (struct list_head *)mem - 1;
- BUG_ON(!list_empty(elem));
+ elem = (struct switchdev_trans_item *) mem - 1;
kfree(elem);
}
}
static struct rocker_wait *rocker_wait_create(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
int flags)
{
struct rocker_wait *wait;
return wait;
}
-static void rocker_wait_destroy(enum switchdev_trans trans,
+static void rocker_wait_destroy(struct switchdev_trans *trans,
struct rocker_wait *wait)
{
rocker_port_kfree(trans, wait);
wait = rocker_desc_cookie_ptr_get(desc_info);
if (wait->nowait) {
rocker_desc_gen_clear(desc_info);
- rocker_wait_destroy(SWITCHDEV_TRANS_NONE, wait);
+ rocker_wait_destroy(NULL, wait);
} else {
rocker_wait_wake_up(wait);
}
}
static int rocker_port_fdb(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
const unsigned char *addr,
__be16 vlan_id, int flags);
rocker_port->stp_state != BR_STATE_FORWARDING)
return 0;
- return rocker_port_fdb(rocker_port, SWITCHDEV_TRANS_NONE,
- addr, vlan_id, flags);
+ return rocker_port_fdb(rocker_port, NULL, addr, vlan_id, flags);
}
static int rocker_event_process(const struct rocker *rocker,
void *priv);
static int rocker_cmd_exec(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
rocker_cmd_prep_cb_t prepare, void *prepare_priv,
rocker_cmd_proc_cb_t process, void *process_priv)
{
rocker_desc_cookie_ptr_set(desc_info, wait);
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);
spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
if (nowait)
return 0;
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
if (!rocker_wait_event_timeout(wait, HZ / 10))
return -EIO;
static int rocker_cmd_get_port_settings_ethtool(struct rocker_port *rocker_port,
struct ethtool_cmd *ecmd)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_get_port_settings_prep, NULL,
rocker_cmd_get_port_settings_ethtool_proc,
ecmd);
static int rocker_cmd_get_port_settings_macaddr(struct rocker_port *rocker_port,
unsigned char *macaddr)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_get_port_settings_prep, NULL,
rocker_cmd_get_port_settings_macaddr_proc,
macaddr);
static int rocker_cmd_set_port_settings_ethtool(struct rocker_port *rocker_port,
struct ethtool_cmd *ecmd)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_set_port_settings_ethtool_prep,
ecmd, NULL, NULL);
}
static int rocker_cmd_set_port_settings_macaddr(struct rocker_port *rocker_port,
unsigned char *macaddr)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_set_port_settings_macaddr_prep,
macaddr, NULL, NULL);
}
static int rocker_cmd_set_port_settings_mtu(struct rocker_port *rocker_port,
int mtu)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_set_port_settings_mtu_prep,
&mtu, NULL, NULL);
}
static int rocker_port_set_learning(struct rocker_port *rocker_port,
- enum switchdev_trans trans)
+ struct switchdev_trans *trans)
{
return rocker_cmd_exec(rocker_port, trans, 0,
rocker_cmd_set_port_learning_prep,
}
static int rocker_flow_tbl_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_flow_tbl_entry *match)
{
struct rocker *rocker = rocker_port->rocker;
if (found) {
match->cookie = found->cookie;
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_del(&found->entry);
rocker_port_kfree(trans, found);
found = match;
found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD;
}
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_add(rocker->flow_tbl, &found->entry, found->key_crc32);
spin_unlock_irqrestore(&rocker->flow_tbl_lock, lock_flags);
}
static int rocker_flow_tbl_del(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_flow_tbl_entry *match)
{
struct rocker *rocker = rocker_port->rocker;
found = rocker_flow_tbl_find(rocker, match);
if (found) {
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_del(&found->entry);
found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL;
}
}
static int rocker_flow_tbl_do(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_flow_tbl_entry *entry)
{
if (flags & ROCKER_OP_FLAG_REMOVE)
}
static int rocker_flow_tbl_ig_port(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
u32 in_pport, u32 in_pport_mask,
enum rocker_of_dpa_table_id goto_tbl)
{
}
static int rocker_flow_tbl_vlan(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
u32 in_pport, __be16 vlan_id,
__be16 vlan_id_mask,
enum rocker_of_dpa_table_id goto_tbl,
}
static int rocker_flow_tbl_term_mac(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
u32 in_pport, u32 in_pport_mask,
__be16 eth_type, const u8 *eth_dst,
const u8 *eth_dst_mask, __be16 vlan_id,
}
static int rocker_flow_tbl_bridge(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const u8 *eth_dst, const u8 *eth_dst_mask,
__be16 vlan_id, u32 tunnel_id,
enum rocker_of_dpa_table_id goto_tbl,
}
static int rocker_flow_tbl_ucast4_routing(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
__be16 eth_type, __be32 dst,
__be32 dst_mask, u32 priority,
enum rocker_of_dpa_table_id goto_tbl,
}
static int rocker_flow_tbl_acl(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
u32 in_pport, u32 in_pport_mask,
const u8 *eth_src, const u8 *eth_src_mask,
const u8 *eth_dst, const u8 *eth_dst_mask,
return NULL;
}
-static void rocker_group_tbl_entry_free(enum switchdev_trans trans,
+static void rocker_group_tbl_entry_free(struct switchdev_trans *trans,
struct rocker_group_tbl_entry *entry)
{
switch (ROCKER_GROUP_TYPE_GET(entry->group_id)) {
}
static int rocker_group_tbl_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_group_tbl_entry *match)
{
struct rocker *rocker = rocker_port->rocker;
found = rocker_group_tbl_find(rocker, match);
if (found) {
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_del(&found->entry);
rocker_group_tbl_entry_free(trans, found);
found = match;
found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD;
}
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_add(rocker->group_tbl, &found->entry, found->group_id);
spin_unlock_irqrestore(&rocker->group_tbl_lock, lock_flags);
}
static int rocker_group_tbl_del(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_group_tbl_entry *match)
{
struct rocker *rocker = rocker_port->rocker;
found = rocker_group_tbl_find(rocker, match);
if (found) {
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_del(&found->entry);
found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL;
}
}
static int rocker_group_tbl_do(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
struct rocker_group_tbl_entry *entry)
{
if (flags & ROCKER_OP_FLAG_REMOVE)
}
static int rocker_group_l2_interface(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be16 vlan_id, u32 out_pport,
int pop_vlan)
{
}
static int rocker_group_l2_fan_out(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
int flags, u8 group_count,
const u32 *group_ids, u32 group_id)
{
}
static int rocker_group_l2_flood(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be16 vlan_id, u8 group_count,
const u32 *group_ids, u32 group_id)
{
}
static int rocker_group_l3_unicast(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
u32 index, const u8 *src_mac, const u8 *dst_mac,
__be16 vlan_id, bool ttl_check, u32 pport)
{
}
static void _rocker_neigh_add(struct rocker *rocker,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
struct rocker_neigh_tbl_entry *entry)
{
- if (trans != SWITCHDEV_TRANS_COMMIT)
+ if (!switchdev_trans_ph_commit(trans))
entry->index = rocker->neigh_tbl_next_index++;
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
return;
entry->ref_count++;
hash_add(rocker->neigh_tbl, &entry->entry,
be32_to_cpu(entry->ip_addr));
}
-static void _rocker_neigh_del(enum switchdev_trans trans,
+static void _rocker_neigh_del(struct switchdev_trans *trans,
struct rocker_neigh_tbl_entry *entry)
{
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
return;
if (--entry->ref_count == 0) {
hash_del(&entry->entry);
}
static void _rocker_neigh_update(struct rocker_neigh_tbl_entry *entry,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
const u8 *eth_dst, bool ttl_check)
{
if (eth_dst) {
ether_addr_copy(entry->eth_dst, eth_dst);
entry->ttl_check = ttl_check;
- } else if (trans != SWITCHDEV_TRANS_PREPARE) {
+ } else if (!switchdev_trans_ph_prepare(trans)) {
entry->ref_count++;
}
}
static int rocker_port_ipv4_neigh(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
int flags, __be32 ip_addr, const u8 *eth_dst)
{
struct rocker *rocker = rocker_port->rocker;
}
static int rocker_port_ipv4_resolve(struct rocker_port *rocker_port,
- enum switchdev_trans trans, __be32 ip_addr)
+ struct switchdev_trans *trans,
+ __be32 ip_addr)
{
struct net_device *dev = rocker_port->dev;
struct neighbour *n = __ipv4_neigh_lookup(dev, (__force u32)ip_addr);
}
static int rocker_port_ipv4_nh(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be32 ip_addr, u32 *index)
{
struct rocker *rocker = rocker_port->rocker;
}
static int rocker_port_vlan_flood_group(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
int flags, __be16 vlan_id)
{
struct rocker_port *p;
}
static int rocker_port_vlan_l2_groups(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be16 vlan_id, bool pop_vlan)
{
const struct rocker *rocker = rocker_port->rocker;
};
static int rocker_port_ctrl_vlan_acl(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const struct rocker_ctrl *ctrl, __be16 vlan_id)
{
u32 in_pport = rocker_port->pport;
}
static int rocker_port_ctrl_vlan_bridge(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans,
+ int flags,
const struct rocker_ctrl *ctrl,
__be16 vlan_id)
{
}
static int rocker_port_ctrl_vlan_term(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const struct rocker_ctrl *ctrl, __be16 vlan_id)
{
u32 in_pport_mask = 0xffffffff;
}
static int rocker_port_ctrl_vlan(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const struct rocker_ctrl *ctrl, __be16 vlan_id)
{
if (ctrl->acl)
}
static int rocker_port_ctrl_vlan_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be16 vlan_id)
{
int err = 0;
}
static int rocker_port_ctrl(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const struct rocker_ctrl *ctrl)
{
u16 vid;
}
static int rocker_port_vlan(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags, u16 vid)
+ struct switchdev_trans *trans, int flags, u16 vid)
{
enum rocker_of_dpa_table_id goto_tbl =
ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC;
"Error (%d) port VLAN table\n", err);
err_out:
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
change_bit(ntohs(internal_vlan_id), rocker_port->vlan_bitmap);
return err;
}
static int rocker_port_ig_tbl(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags)
+ struct switchdev_trans *trans, int flags)
{
enum rocker_of_dpa_table_id goto_tbl;
u32 in_pport;
struct rocker_fdb_learn_work {
struct work_struct work;
struct rocker_port *rocker_port;
- enum switchdev_trans trans;
+ struct switchdev_trans *trans;
int flags;
u8 addr[ETH_ALEN];
u16 vid;
}
static int rocker_port_fdb_learn(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
const u8 *addr, __be16 vlan_id)
{
struct rocker_fdb_learn_work *lw;
ether_addr_copy(lw->addr, addr);
lw->vid = rocker_port_vlan_to_vid(rocker_port, vlan_id);
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
rocker_port_kfree(trans, lw);
else
schedule_work(&lw->work);
}
static int rocker_port_fdb(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
const unsigned char *addr,
__be16 vlan_id, int flags)
{
return -ENOMEM;
fdb->learned = (flags & ROCKER_OP_FLAG_LEARNED);
- fdb->key.pport = rocker_port->pport;
+ fdb->touched = jiffies;
+ fdb->key.rocker_port = rocker_port;
ether_addr_copy(fdb->key.addr, addr);
fdb->key.vlan_id = vlan_id;
fdb->key_crc32 = crc32(~0, &fdb->key, sizeof(fdb->key));
found = rocker_fdb_tbl_find(rocker, fdb);
- if (removing && found) {
- rocker_port_kfree(trans, fdb);
- if (trans != SWITCHDEV_TRANS_PREPARE)
- hash_del(&found->entry);
- } else if (!removing && !found) {
- if (trans != SWITCHDEV_TRANS_PREPARE)
- hash_add(rocker->fdb_tbl, &fdb->entry, fdb->key_crc32);
+ if (found) {
+ found->touched = jiffies;
+ if (removing) {
+ rocker_port_kfree(trans, fdb);
+ if (!switchdev_trans_ph_prepare(trans))
+ hash_del(&found->entry);
+ }
+ } else if (!removing) {
+ if (!switchdev_trans_ph_prepare(trans))
+ hash_add(rocker->fdb_tbl, &fdb->entry,
+ fdb->key_crc32);
}
spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
}
static int rocker_port_fdb_flush(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags)
+ struct switchdev_trans *trans, int flags)
{
struct rocker *rocker = rocker_port->rocker;
struct rocker_fdb_tbl_entry *found;
rocker_port->stp_state == BR_STATE_FORWARDING)
return 0;
- flags |= ROCKER_OP_FLAG_REMOVE;
+ flags |= ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_REMOVE;
spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, found, entry) {
- if (found->key.pport != rocker_port->pport)
+ if (found->key.rocker_port != rocker_port)
continue;
if (!found->learned)
continue;
found->key.vlan_id);
if (err)
goto err_out;
- if (trans != SWITCHDEV_TRANS_PREPARE)
+ if (!switchdev_trans_ph_prepare(trans))
hash_del(&found->entry);
}
return err;
}
+static void rocker_fdb_cleanup(unsigned long data)
+{
+ struct rocker *rocker = (struct rocker *)data;
+ struct rocker_port *rocker_port;
+ struct rocker_fdb_tbl_entry *entry;
+ struct hlist_node *tmp;
+ unsigned long next_timer = jiffies + BR_MIN_AGEING_TIME;
+ unsigned long expires;
+ unsigned long lock_flags;
+ int flags = ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_REMOVE |
+ ROCKER_OP_FLAG_LEARNED;
+ int bkt;
+
+ spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
+
+ hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, entry, entry) {
+ if (!entry->learned)
+ continue;
+ rocker_port = entry->key.rocker_port;
+ expires = entry->touched + rocker_port->ageing_time;
+ if (time_before_eq(expires, jiffies)) {
+ rocker_port_fdb_learn(rocker_port, NULL,
+ flags, entry->key.addr,
+ entry->key.vlan_id);
+ hash_del(&entry->entry);
+ } else if (time_before(expires, next_timer)) {
+ next_timer = expires;
+ }
+ }
+
+ spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
+
+ mod_timer(&rocker->fdb_cleanup_timer, round_jiffies_up(next_timer));
+}
+
static int rocker_port_router_mac(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
__be16 vlan_id)
{
u32 in_pport_mask = 0xffffffff;
}
static int rocker_port_fwding(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags)
+ struct switchdev_trans *trans, int flags)
{
bool pop_vlan;
u32 out_pport;
}
static int rocker_port_stp_update(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags,
+ struct switchdev_trans *trans, int flags,
u8 state)
{
bool want[ROCKER_CTRL_MAX] = { 0, };
bool prev_ctrls[ROCKER_CTRL_MAX];
- u8 prev_state;
+ u8 uninitialized_var(prev_state);
int err;
int i;
- if (trans == SWITCHDEV_TRANS_PREPARE) {
+ if (switchdev_trans_ph_prepare(trans)) {
memcpy(prev_ctrls, rocker_port->ctrls, sizeof(prev_ctrls));
prev_state = rocker_port->stp_state;
}
err = rocker_port_fwding(rocker_port, trans, flags);
err_out:
- if (trans == SWITCHDEV_TRANS_PREPARE) {
+ if (switchdev_trans_ph_prepare(trans)) {
memcpy(rocker_port->ctrls, prev_ctrls, sizeof(prev_ctrls));
rocker_port->stp_state = prev_state;
}
}
static int rocker_port_fwd_enable(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags)
+ struct switchdev_trans *trans, int flags)
{
if (rocker_port_is_bridged(rocker_port))
/* bridge STP will enable port */
}
static int rocker_port_fwd_disable(struct rocker_port *rocker_port,
- enum switchdev_trans trans, int flags)
+ struct switchdev_trans *trans, int flags)
{
if (rocker_port_is_bridged(rocker_port))
/* bridge STP will disable port */
}
static int rocker_port_fib_ipv4(struct rocker_port *rocker_port,
- enum switchdev_trans trans, __be32 dst,
+ struct switchdev_trans *trans, __be32 dst,
int dst_len, const struct fib_info *fi,
u32 tb_id, int flags)
{
goto err_request_rx_irq;
}
- err = rocker_port_fwd_enable(rocker_port, SWITCHDEV_TRANS_NONE, 0);
+ err = rocker_port_fwd_enable(rocker_port, NULL, 0);
if (err)
goto err_fwd_enable;
rocker_port_set_enable(rocker_port, false);
napi_disable(&rocker_port->napi_rx);
napi_disable(&rocker_port->napi_tx);
- rocker_port_fwd_disable(rocker_port, SWITCHDEV_TRANS_NONE,
+ rocker_port_fwd_disable(rocker_port, NULL,
ROCKER_OP_FLAG_NOWAIT);
free_irq(rocker_msix_rx_vector(rocker_port), rocker_port);
free_irq(rocker_msix_tx_vector(rocker_port), rocker_port);
struct port_name name = { .buf = buf, .len = len };
int err;
- err = rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ err = rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_get_port_settings_prep, NULL,
rocker_cmd_get_port_settings_phys_name_proc,
&name);
int flags = ROCKER_OP_FLAG_REMOVE | ROCKER_OP_FLAG_NOWAIT;
__be32 ip_addr = *(__be32 *)n->primary_key;
- rocker_port_ipv4_neigh(rocker_port, SWITCHDEV_TRANS_NONE,
+ rocker_port_ipv4_neigh(rocker_port, NULL,
flags, ip_addr, n->ha);
}
const struct rocker *rocker = rocker_port->rocker;
switch (attr->id) {
- case SWITCHDEV_ATTR_PORT_PARENT_ID:
+ case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
attr->u.ppid.id_len = sizeof(rocker->hw.id);
memcpy(&attr->u.ppid.id, &rocker->hw.id, attr->u.ppid.id_len);
break;
- case SWITCHDEV_ATTR_PORT_BRIDGE_FLAGS:
+ case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
attr->u.brport_flags = rocker_port->brport_flags;
break;
default:
return 0;
}
-static void rocker_port_trans_abort(const struct rocker_port *rocker_port)
-{
- struct list_head *mem, *tmp;
-
- list_for_each_safe(mem, tmp, &rocker_port->trans_mem) {
- list_del(mem);
- kfree(mem);
- }
-}
-
static int rocker_port_brport_flags_set(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
+ struct switchdev_trans *trans,
unsigned long brport_flags)
{
unsigned long orig_flags;
if ((orig_flags ^ rocker_port->brport_flags) & BR_LEARNING)
err = rocker_port_set_learning(rocker_port, trans);
- if (trans == SWITCHDEV_TRANS_PREPARE)
+ if (switchdev_trans_ph_prepare(trans))
rocker_port->brport_flags = orig_flags;
return err;
}
+static int rocker_port_bridge_ageing_time(struct rocker_port *rocker_port,
+ struct switchdev_trans *trans,
+ u32 ageing_time)
+{
+ if (!switchdev_trans_ph_prepare(trans)) {
+ rocker_port->ageing_time = clock_t_to_jiffies(ageing_time);
+ mod_timer(&rocker_port->rocker->fdb_cleanup_timer, jiffies);
+ }
+
+ return 0;
+}
+
static int rocker_port_attr_set(struct net_device *dev,
- struct switchdev_attr *attr)
+ const struct switchdev_attr *attr,
+ struct switchdev_trans *trans)
{
struct rocker_port *rocker_port = netdev_priv(dev);
int err = 0;
- switch (attr->trans) {
- case SWITCHDEV_TRANS_PREPARE:
- BUG_ON(!list_empty(&rocker_port->trans_mem));
- break;
- case SWITCHDEV_TRANS_ABORT:
- rocker_port_trans_abort(rocker_port);
- return 0;
- default:
- break;
- }
-
switch (attr->id) {
- case SWITCHDEV_ATTR_PORT_STP_STATE:
- err = rocker_port_stp_update(rocker_port, attr->trans,
- ROCKER_OP_FLAG_NOWAIT,
+ case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
+ err = rocker_port_stp_update(rocker_port, trans, 0,
attr->u.stp_state);
break;
- case SWITCHDEV_ATTR_PORT_BRIDGE_FLAGS:
- err = rocker_port_brport_flags_set(rocker_port, attr->trans,
+ case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
+ err = rocker_port_brport_flags_set(rocker_port, trans,
attr->u.brport_flags);
break;
+ case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
+ err = rocker_port_bridge_ageing_time(rocker_port, trans,
+ attr->u.ageing_time);
+ break;
default:
err = -EOPNOTSUPP;
break;
}
static int rocker_port_vlan_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans, u16 vid, u16 flags)
+ struct switchdev_trans *trans,
+ u16 vid, u16 flags)
{
int err;
}
static int rocker_port_vlans_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
- const struct switchdev_obj_vlan *vlan)
+ struct switchdev_trans *trans,
+ const struct switchdev_obj_port_vlan *vlan)
{
u16 vid;
int err;
}
static int rocker_port_fdb_add(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
- const struct switchdev_obj_fdb *fdb)
+ struct switchdev_trans *trans,
+ const struct switchdev_obj_port_fdb *fdb)
{
__be16 vlan_id = rocker_port_vid_to_vlan(rocker_port, fdb->vid, NULL);
int flags = 0;
}
static int rocker_port_obj_add(struct net_device *dev,
- struct switchdev_obj *obj)
+ const struct switchdev_obj *obj,
+ struct switchdev_trans *trans)
{
struct rocker_port *rocker_port = netdev_priv(dev);
const struct switchdev_obj_ipv4_fib *fib4;
int err = 0;
- switch (obj->trans) {
- case SWITCHDEV_TRANS_PREPARE:
- BUG_ON(!list_empty(&rocker_port->trans_mem));
- break;
- case SWITCHDEV_TRANS_ABORT:
- rocker_port_trans_abort(rocker_port);
- return 0;
- default:
- break;
- }
-
switch (obj->id) {
- case SWITCHDEV_OBJ_PORT_VLAN:
- err = rocker_port_vlans_add(rocker_port, obj->trans,
- &obj->u.vlan);
+ case SWITCHDEV_OBJ_ID_PORT_VLAN:
+ err = rocker_port_vlans_add(rocker_port, trans,
+ SWITCHDEV_OBJ_PORT_VLAN(obj));
break;
- case SWITCHDEV_OBJ_IPV4_FIB:
- fib4 = &obj->u.ipv4_fib;
- err = rocker_port_fib_ipv4(rocker_port, obj->trans,
+ case SWITCHDEV_OBJ_ID_IPV4_FIB:
+ fib4 = SWITCHDEV_OBJ_IPV4_FIB(obj);
+ err = rocker_port_fib_ipv4(rocker_port, trans,
htonl(fib4->dst), fib4->dst_len,
- fib4->fi, fib4->tb_id, 0);
+ &fib4->fi, fib4->tb_id, 0);
break;
- case SWITCHDEV_OBJ_PORT_FDB:
- err = rocker_port_fdb_add(rocker_port, obj->trans, &obj->u.fdb);
+ case SWITCHDEV_OBJ_ID_PORT_FDB:
+ err = rocker_port_fdb_add(rocker_port, trans,
+ SWITCHDEV_OBJ_PORT_FDB(obj));
break;
default:
err = -EOPNOTSUPP;
{
int err;
- err = rocker_port_router_mac(rocker_port, SWITCHDEV_TRANS_NONE,
+ err = rocker_port_router_mac(rocker_port, NULL,
ROCKER_OP_FLAG_REMOVE, htons(vid));
if (err)
return err;
- return rocker_port_vlan(rocker_port, SWITCHDEV_TRANS_NONE,
+ return rocker_port_vlan(rocker_port, NULL,
ROCKER_OP_FLAG_REMOVE, vid);
}
static int rocker_port_vlans_del(struct rocker_port *rocker_port,
- const struct switchdev_obj_vlan *vlan)
+ const struct switchdev_obj_port_vlan *vlan)
{
u16 vid;
int err;
}
static int rocker_port_fdb_del(struct rocker_port *rocker_port,
- enum switchdev_trans trans,
- const struct switchdev_obj_fdb *fdb)
+ struct switchdev_trans *trans,
+ const struct switchdev_obj_port_fdb *fdb)
{
__be16 vlan_id = rocker_port_vid_to_vlan(rocker_port, fdb->vid, NULL);
- int flags = ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_REMOVE;
+ int flags = ROCKER_OP_FLAG_REMOVE;
if (!rocker_port_is_bridged(rocker_port))
return -EINVAL;
}
static int rocker_port_obj_del(struct net_device *dev,
- struct switchdev_obj *obj)
+ const struct switchdev_obj *obj)
{
struct rocker_port *rocker_port = netdev_priv(dev);
const struct switchdev_obj_ipv4_fib *fib4;
int err = 0;
switch (obj->id) {
- case SWITCHDEV_OBJ_PORT_VLAN:
- err = rocker_port_vlans_del(rocker_port, &obj->u.vlan);
+ case SWITCHDEV_OBJ_ID_PORT_VLAN:
+ err = rocker_port_vlans_del(rocker_port,
+ SWITCHDEV_OBJ_PORT_VLAN(obj));
break;
- case SWITCHDEV_OBJ_IPV4_FIB:
- fib4 = &obj->u.ipv4_fib;
- err = rocker_port_fib_ipv4(rocker_port, SWITCHDEV_TRANS_NONE,
+ case SWITCHDEV_OBJ_ID_IPV4_FIB:
+ fib4 = SWITCHDEV_OBJ_IPV4_FIB(obj);
+ err = rocker_port_fib_ipv4(rocker_port, NULL,
htonl(fib4->dst), fib4->dst_len,
- fib4->fi, fib4->tb_id,
+ &fib4->fi, fib4->tb_id,
ROCKER_OP_FLAG_REMOVE);
break;
- case SWITCHDEV_OBJ_PORT_FDB:
- err = rocker_port_fdb_del(rocker_port, obj->trans, &obj->u.fdb);
+ case SWITCHDEV_OBJ_ID_PORT_FDB:
+ err = rocker_port_fdb_del(rocker_port, NULL,
+ SWITCHDEV_OBJ_PORT_FDB(obj));
break;
default:
err = -EOPNOTSUPP;
}
static int rocker_port_fdb_dump(const struct rocker_port *rocker_port,
- struct switchdev_obj *obj)
+ struct switchdev_obj_port_fdb *fdb,
+ switchdev_obj_dump_cb_t *cb)
{
struct rocker *rocker = rocker_port->rocker;
- struct switchdev_obj_fdb *fdb = &obj->u.fdb;
struct rocker_fdb_tbl_entry *found;
struct hlist_node *tmp;
unsigned long lock_flags;
spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, found, entry) {
- if (found->key.pport != rocker_port->pport)
+ if (found->key.rocker_port != rocker_port)
continue;
- fdb->addr = found->key.addr;
+ ether_addr_copy(fdb->addr, found->key.addr);
fdb->ndm_state = NUD_REACHABLE;
fdb->vid = rocker_port_vlan_to_vid(rocker_port,
found->key.vlan_id);
- err = obj->cb(rocker_port->dev, obj);
+ err = cb(&fdb->obj);
if (err)
break;
}
}
static int rocker_port_vlan_dump(const struct rocker_port *rocker_port,
- struct switchdev_obj *obj)
+ struct switchdev_obj_port_vlan *vlan,
+ switchdev_obj_dump_cb_t *cb)
{
- struct switchdev_obj_vlan *vlan = &obj->u.vlan;
u16 vid;
int err = 0;
if (rocker_vlan_id_is_internal(htons(vid)))
vlan->flags |= BRIDGE_VLAN_INFO_PVID;
vlan->vid_begin = vlan->vid_end = vid;
- err = obj->cb(rocker_port->dev, obj);
+ err = cb(&vlan->obj);
if (err)
break;
}
}
static int rocker_port_obj_dump(struct net_device *dev,
- struct switchdev_obj *obj)
+ struct switchdev_obj *obj,
+ switchdev_obj_dump_cb_t *cb)
{
const struct rocker_port *rocker_port = netdev_priv(dev);
int err = 0;
switch (obj->id) {
- case SWITCHDEV_OBJ_PORT_FDB:
- err = rocker_port_fdb_dump(rocker_port, obj);
+ case SWITCHDEV_OBJ_ID_PORT_FDB:
+ err = rocker_port_fdb_dump(rocker_port,
+ SWITCHDEV_OBJ_PORT_FDB(obj), cb);
break;
- case SWITCHDEV_OBJ_PORT_VLAN:
- err = rocker_port_vlan_dump(rocker_port, obj);
+ case SWITCHDEV_OBJ_ID_PORT_VLAN:
+ err = rocker_port_vlan_dump(rocker_port,
+ SWITCHDEV_OBJ_PORT_VLAN(obj), cb);
break;
default:
err = -EOPNOTSUPP;
static int rocker_cmd_get_port_stats_ethtool(struct rocker_port *rocker_port,
void *priv)
{
- return rocker_cmd_exec(rocker_port, SWITCHDEV_TRANS_NONE, 0,
+ return rocker_cmd_exec(rocker_port, NULL, 0,
rocker_cmd_get_port_stats_prep, NULL,
rocker_cmd_get_port_stats_ethtool_proc,
priv);
rocker_port = rocker->ports[i];
if (!rocker_port)
continue;
- rocker_port_ig_tbl(rocker_port, SWITCHDEV_TRANS_NONE,
- ROCKER_OP_FLAG_REMOVE);
+ rocker_port_ig_tbl(rocker_port, NULL, ROCKER_OP_FLAG_REMOVE);
unregister_netdev(rocker_port->dev);
free_netdev(rocker_port->dev);
}
rocker_port->port_number = port_number;
rocker_port->pport = port_number + 1;
rocker_port->brport_flags = BR_LEARNING | BR_LEARNING_SYNC;
- INIT_LIST_HEAD(&rocker_port->trans_mem);
+ rocker_port->ageing_time = BR_DEFAULT_AGEING_TIME;
rocker_port_dev_addr_init(rocker_port);
dev->netdev_ops = &rocker_port_netdev_ops;
switchdev_port_fwd_mark_set(rocker_port->dev, NULL, false);
- rocker_port_set_learning(rocker_port, SWITCHDEV_TRANS_NONE);
+ rocker_port_set_learning(rocker_port, NULL);
- err = rocker_port_ig_tbl(rocker_port, SWITCHDEV_TRANS_NONE, 0);
+ err = rocker_port_ig_tbl(rocker_port, NULL, 0);
if (err) {
netdev_err(rocker_port->dev, "install ig port table failed\n");
goto err_port_ig_tbl;
rocker_port->internal_vlan_id =
rocker_port_internal_vlan_id_get(rocker_port, dev->ifindex);
- err = rocker_port_vlan_add(rocker_port, SWITCHDEV_TRANS_NONE,
- untagged_vid, 0);
+ err = rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
if (err) {
netdev_err(rocker_port->dev, "install untagged VLAN failed\n");
goto err_untagged_vlan;
return 0;
err_untagged_vlan:
- rocker_port_ig_tbl(rocker_port, SWITCHDEV_TRANS_NONE,
- ROCKER_OP_FLAG_REMOVE);
+ rocker_port_ig_tbl(rocker_port, NULL, ROCKER_OP_FLAG_REMOVE);
err_port_ig_tbl:
rocker->ports[port_number] = NULL;
unregister_netdev(dev);
goto err_init_tbls;
}
+ setup_timer(&rocker->fdb_cleanup_timer, rocker_fdb_cleanup,
+ (unsigned long) rocker);
+ mod_timer(&rocker->fdb_cleanup_timer, jiffies);
+
err = rocker_probe_ports(rocker);
if (err) {
dev_err(&pdev->dev, "failed to probe ports\n");
return 0;
err_probe_ports:
+ del_timer_sync(&rocker->fdb_cleanup_timer);
rocker_free_tbls(rocker);
err_init_tbls:
free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
{
struct rocker *rocker = pci_get_drvdata(pdev);
+ del_timer_sync(&rocker->fdb_cleanup_timer);
rocker_free_tbls(rocker);
rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
rocker_remove_ports(rocker);
rocker_port->bridge_dev = bridge;
switchdev_port_fwd_mark_set(rocker_port->dev, bridge, true);
- return rocker_port_vlan_add(rocker_port, SWITCHDEV_TRANS_NONE,
- untagged_vid, 0);
+ return rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
}
static int rocker_port_bridge_leave(struct rocker_port *rocker_port)
false);
rocker_port->bridge_dev = NULL;
- err = rocker_port_vlan_add(rocker_port, SWITCHDEV_TRANS_NONE,
- untagged_vid, 0);
+ err = rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
if (err)
return err;
if (rocker_port->dev->flags & IFF_UP)
- err = rocker_port_fwd_enable(rocker_port,
- SWITCHDEV_TRANS_NONE, 0);
+ err = rocker_port_fwd_enable(rocker_port, NULL, 0);
return err;
}
rocker_port->bridge_dev = master;
- err = rocker_port_fwd_disable(rocker_port, SWITCHDEV_TRANS_NONE, 0);
+ err = rocker_port_fwd_disable(rocker_port, NULL, 0);
if (err)
return err;
- err = rocker_port_fwd_enable(rocker_port, SWITCHDEV_TRANS_NONE, 0);
+ err = rocker_port_fwd_enable(rocker_port, NULL, 0);
return err;
}
ROCKER_OP_FLAG_NOWAIT;
__be32 ip_addr = *(__be32 *)n->primary_key;
- return rocker_port_ipv4_neigh(rocker_port, SWITCHDEV_TRANS_NONE,
- flags, ip_addr, n->ha);
+ return rocker_port_ipv4_neigh(rocker_port, NULL, flags, ip_addr, n->ha);
}
static int rocker_netevent_event(struct notifier_block *unused,
* more details.
*/
-#include <linux/nvme.h>
#include <linux/bitops.h>
#include <linux/blkdev.h>
#include <linux/blk-mq.h>
#include <linux/slab.h>
#include <linux/t10-pi.h>
#include <linux/types.h>
+#include <linux/pr.h>
#include <scsi/sg.h>
- #include <asm-generic/io-64-nonatomic-lo-hi.h>
+ #include <linux/io-64-nonatomic-lo-hi.h>
+#include <asm/unaligned.h>
+
+#include <uapi/linux/nvme_ioctl.h>
+#include "nvme.h"
#define NVME_MINORS (1U << MINORBITS)
#define NVME_Q_DEPTH 1024
static struct class *nvme_class;
-static void nvme_reset_failed_dev(struct work_struct *ws);
+static int __nvme_reset(struct nvme_dev *dev);
static int nvme_reset(struct nvme_dev *dev);
static int nvme_process_cq(struct nvme_queue *nvmeq);
+static void nvme_dead_ctrl(struct nvme_dev *dev);
struct async_cmd_info {
struct kthread_work work;
virt = bip_get_seed(bip);
phys = nvme_block_nr(ns, blk_rq_pos(req));
nlb = (blk_rq_bytes(req) >> ns->lba_shift);
- ts = ns->disk->integrity->tuple_size;
+ ts = ns->disk->queue->integrity.tuple_size;
for (i = 0; i < nlb; i++, virt++, phys++) {
pi = (struct t10_pi_tuple *)p;
kunmap_atomic(pmap);
}
-static int nvme_noop_verify(struct blk_integrity_iter *iter)
-{
- return 0;
-}
-
-static int nvme_noop_generate(struct blk_integrity_iter *iter)
-{
- return 0;
-}
-
-struct blk_integrity nvme_meta_noop = {
- .name = "NVME_META_NOOP",
- .generate_fn = nvme_noop_generate,
- .verify_fn = nvme_noop_verify,
-};
-
static void nvme_init_integrity(struct nvme_ns *ns)
{
struct blk_integrity integrity;
switch (ns->pi_type) {
case NVME_NS_DPS_PI_TYPE3:
- integrity = t10_pi_type3_crc;
+ integrity.profile = &t10_pi_type3_crc;
break;
case NVME_NS_DPS_PI_TYPE1:
case NVME_NS_DPS_PI_TYPE2:
- integrity = t10_pi_type1_crc;
+ integrity.profile = &t10_pi_type1_crc;
break;
default:
- integrity = nvme_meta_noop;
+ integrity.profile = NULL;
break;
}
integrity.tuple_size = ns->ms;
struct nvme_iod *iod = ctx;
struct request *req = iod_get_private(iod);
struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
-
u16 status = le16_to_cpup(&cqe->status) >> 1;
+ bool requeue = false;
+ int error = 0;
if (unlikely(status)) {
if (!(status & NVME_SC_DNR || blk_noretry_request(req))
&& (jiffies - req->start_time) < req->timeout) {
unsigned long flags;
+ requeue = true;
blk_mq_requeue_request(req);
spin_lock_irqsave(req->q->queue_lock, flags);
if (!blk_queue_stopped(req->q))
blk_mq_kick_requeue_list(req->q);
spin_unlock_irqrestore(req->q->queue_lock, flags);
- return;
+ goto release_iod;
}
if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
if (cmd_rq->ctx == CMD_CTX_CANCELLED)
- status = -EINTR;
+ error = -EINTR;
+ else
+ error = status;
} else {
- status = nvme_error_status(status);
+ error = nvme_error_status(status);
}
}
if (cmd_rq->aborted)
dev_warn(nvmeq->dev->dev,
"completing aborted command with status:%04x\n",
- status);
+ error);
+release_iod:
if (iod->nents) {
dma_unmap_sg(nvmeq->dev->dev, iod->sg, iod->nents,
rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
}
nvme_free_iod(nvmeq->dev, iod);
- blk_mq_complete_request(req, status);
+ if (likely(!requeue))
+ blk_mq_complete_request(req, error);
}
/* length is in bytes. gfp flags indicates whether we may sleep. */
struct nvme_command cmd;
if (!nvmeq->qid || cmd_rq->aborted) {
- unsigned long flags;
-
- spin_lock_irqsave(&dev_list_lock, flags);
- if (work_busy(&dev->reset_work))
- goto out;
- list_del_init(&dev->node);
- dev_warn(dev->dev, "I/O %d QID %d timeout, reset controller\n",
- req->tag, nvmeq->qid);
- dev->reset_workfn = nvme_reset_failed_dev;
- queue_work(nvme_workq, &dev->reset_work);
- out:
- spin_unlock_irqrestore(&dev_list_lock, flags);
+ spin_lock(&dev_list_lock);
+ if (!__nvme_reset(dev)) {
+ dev_warn(dev->dev,
+ "I/O %d QID %d timeout, reset controller\n",
+ req->tag, nvmeq->qid);
+ }
+ spin_unlock(&dev_list_lock);
return;
}
length = (io.nblocks + 1) << ns->lba_shift;
meta_len = (io.nblocks + 1) * ns->ms;
- metadata = (void __user *)(unsigned long)io.metadata;
+ metadata = (void __user *)(uintptr_t)io.metadata;
write = io.opcode & 1;
if (ns->ext) {
c.rw.metadata = cpu_to_le64(meta_dma);
status = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
- (void __user *)io.addr, length, NULL, 0);
+ (void __user *)(uintptr_t)io.addr, length, NULL, 0);
unmap:
if (meta) {
if (status == NVME_SC_SUCCESS && !write) {
timeout = msecs_to_jiffies(cmd.timeout_ms);
status = __nvme_submit_sync_cmd(ns ? ns->queue : dev->admin_q, &c,
- NULL, (void __user *)cmd.addr, cmd.data_len,
+ NULL, (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
&cmd.result, timeout);
if (status >= 0) {
if (put_user(cmd.result, &ucmd->result))
#define nvme_compat_ioctl NULL
#endif
+static void nvme_free_dev(struct kref *kref);
+static void nvme_free_ns(struct kref *kref)
+{
+ struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
+
+ if (ns->type == NVME_NS_LIGHTNVM)
+ nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
+
+ spin_lock(&dev_list_lock);
+ ns->disk->private_data = NULL;
+ spin_unlock(&dev_list_lock);
+
+ kref_put(&ns->dev->kref, nvme_free_dev);
+ put_disk(ns->disk);
+ kfree(ns);
+}
+
static int nvme_open(struct block_device *bdev, fmode_t mode)
{
int ret = 0;
ns = bdev->bd_disk->private_data;
if (!ns)
ret = -ENXIO;
- else if (!kref_get_unless_zero(&ns->dev->kref))
+ else if (!kref_get_unless_zero(&ns->kref))
ret = -ENXIO;
spin_unlock(&dev_list_lock);
return ret;
}
-static void nvme_free_dev(struct kref *kref);
-
static void nvme_release(struct gendisk *disk, fmode_t mode)
{
struct nvme_ns *ns = disk->private_data;
- struct nvme_dev *dev = ns->dev;
-
- kref_put(&dev->kref, nvme_free_dev);
+ kref_put(&ns->kref, nvme_free_ns);
}
static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
return -ENODEV;
}
+ if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
+ if (nvme_nvm_register(ns->queue, disk->disk_name)) {
+ dev_warn(dev->dev,
+ "%s: LightNVM init failure\n", __func__);
+ kfree(id);
+ return -ENODEV;
+ }
+ ns->type = NVME_NS_LIGHTNVM;
+ }
+
old_ms = ns->ms;
lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
ns->lba_shift = id->lbaf[lbaf].ds;
pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
id->dps & NVME_NS_DPS_PI_MASK : 0;
+ blk_mq_freeze_queue(disk->queue);
if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
ns->ms != old_ms ||
bs != queue_logical_block_size(disk->queue) ||
ns->pi_type = pi_type;
blk_queue_logical_block_size(ns->queue, bs);
- if (ns->ms && !blk_get_integrity(disk) && (disk->flags & GENHD_FL_UP) &&
- !ns->ext)
+ if (ns->ms && !ns->ext)
nvme_init_integrity(ns);
- if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
+ if ((ns->ms && !(ns->ms == 8 && ns->pi_type) &&
+ !blk_get_integrity(disk)) ||
+ ns->type == NVME_NS_LIGHTNVM)
set_capacity(disk, 0);
else
set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
if (dev->oncs & NVME_CTRL_ONCS_DSM)
nvme_config_discard(ns);
+ blk_mq_unfreeze_queue(disk->queue);
kfree(id);
return 0;
}
+static char nvme_pr_type(enum pr_type type)
+{
+ switch (type) {
+ case PR_WRITE_EXCLUSIVE:
+ return 1;
+ case PR_EXCLUSIVE_ACCESS:
+ return 2;
+ case PR_WRITE_EXCLUSIVE_REG_ONLY:
+ return 3;
+ case PR_EXCLUSIVE_ACCESS_REG_ONLY:
+ return 4;
+ case PR_WRITE_EXCLUSIVE_ALL_REGS:
+ return 5;
+ case PR_EXCLUSIVE_ACCESS_ALL_REGS:
+ return 6;
+ default:
+ return 0;
+ }
+};
+
+static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
+ u64 key, u64 sa_key, u8 op)
+{
+ struct nvme_ns *ns = bdev->bd_disk->private_data;
+ struct nvme_command c;
+ u8 data[16] = { 0, };
+
+ put_unaligned_le64(key, &data[0]);
+ put_unaligned_le64(sa_key, &data[8]);
+
+ memset(&c, 0, sizeof(c));
+ c.common.opcode = op;
+ c.common.nsid = cpu_to_le32(ns->ns_id);
+ c.common.cdw10[0] = cpu_to_le32(cdw10);
+
+ return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
+}
+
+static int nvme_pr_register(struct block_device *bdev, u64 old,
+ u64 new, unsigned flags)
+{
+ u32 cdw10;
+
+ if (flags & ~PR_FL_IGNORE_KEY)
+ return -EOPNOTSUPP;
+
+ cdw10 = old ? 2 : 0;
+ cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
+ cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
+ return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_reserve(struct block_device *bdev, u64 key,
+ enum pr_type type, unsigned flags)
+{
+ u32 cdw10;
+
+ if (flags & ~PR_FL_IGNORE_KEY)
+ return -EOPNOTSUPP;
+
+ cdw10 = nvme_pr_type(type) << 8;
+ cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
+ enum pr_type type, bool abort)
+{
+ u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
+ return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
+}
+
+static int nvme_pr_clear(struct block_device *bdev, u64 key)
+{
+ u32 cdw10 = 1 | (key ? 1 << 3 : 0);
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
+}
+
+static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
+{
+ u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
+}
+
+static const struct pr_ops nvme_pr_ops = {
+ .pr_register = nvme_pr_register,
+ .pr_reserve = nvme_pr_reserve,
+ .pr_release = nvme_pr_release,
+ .pr_preempt = nvme_pr_preempt,
+ .pr_clear = nvme_pr_clear,
+};
+
static const struct block_device_operations nvme_fops = {
.owner = THIS_MODULE,
.ioctl = nvme_ioctl,
.release = nvme_release,
.getgeo = nvme_getgeo,
.revalidate_disk= nvme_revalidate_disk,
+ .pr_ops = &nvme_pr_ops,
};
static int nvme_kthread(void *data)
if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
csts & NVME_CSTS_CFS) {
- if (work_busy(&dev->reset_work))
- continue;
- list_del_init(&dev->node);
- dev_warn(dev->dev,
- "Failed status: %x, reset controller\n",
- readl(&dev->bar->csts));
- dev->reset_workfn = nvme_reset_failed_dev;
- queue_work(nvme_workq, &dev->reset_work);
+ if (!__nvme_reset(dev)) {
+ dev_warn(dev->dev,
+ "Failed status: %x, reset controller\n",
+ readl(&dev->bar->csts));
+ }
continue;
}
for (i = 0; i < dev->queue_count; i++) {
if (!disk)
goto out_free_queue;
+ kref_init(&ns->kref);
ns->ns_id = nsid;
ns->disk = disk;
ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
if (nvme_revalidate_disk(ns->disk))
goto out_free_disk;
- add_disk(ns->disk);
- if (ns->ms) {
- struct block_device *bd = bdget_disk(ns->disk, 0);
- if (!bd)
- return;
- if (blkdev_get(bd, FMODE_READ, NULL)) {
- bdput(bd);
- return;
+ kref_get(&dev->kref);
+ if (ns->type != NVME_NS_LIGHTNVM) {
+ add_disk(ns->disk);
+ if (ns->ms) {
+ struct block_device *bd = bdget_disk(ns->disk, 0);
+ if (!bd)
+ return;
+ if (blkdev_get(bd, FMODE_READ, NULL)) {
+ bdput(bd);
+ return;
+ }
+ blkdev_reread_part(bd);
+ blkdev_put(bd, FMODE_READ);
}
- blkdev_reread_part(bd);
- blkdev_put(bd, FMODE_READ);
}
return;
out_free_disk:
kfree(ns);
}
+/*
+ * Create I/O queues. Failing to create an I/O queue is not an issue,
+ * we can continue with less than the desired amount of queues, and
+ * even a controller without I/O queues an still be used to issue
+ * admin commands. This might be useful to upgrade a buggy firmware
+ * for example.
+ */
static void nvme_create_io_queues(struct nvme_dev *dev)
{
unsigned i;
break;
for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
- if (nvme_create_queue(dev->queues[i], i))
+ if (nvme_create_queue(dev->queues[i], i)) {
+ nvme_free_queues(dev, i);
break;
+ }
}
static int set_queue_count(struct nvme_dev *dev, int count)
return result;
}
-static void nvme_free_namespace(struct nvme_ns *ns)
-{
- list_del(&ns->list);
-
- spin_lock(&dev_list_lock);
- ns->disk->private_data = NULL;
- spin_unlock(&dev_list_lock);
-
- put_disk(ns->disk);
- kfree(ns);
-}
-
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
if (kill)
blk_set_queue_dying(ns->queue);
- if (ns->disk->flags & GENHD_FL_UP) {
- if (blk_get_integrity(ns->disk))
- blk_integrity_unregister(ns->disk);
+ if (ns->disk->flags & GENHD_FL_UP)
del_gendisk(ns->disk);
- }
if (kill || !blk_queue_dying(ns->queue)) {
blk_mq_abort_requeue_list(ns->queue);
blk_cleanup_queue(ns->queue);
- }
+ }
+ list_del_init(&ns->list);
+ kref_put(&ns->kref, nvme_free_ns);
}
static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
for (i = 1; i <= nn; i++) {
ns = nvme_find_ns(dev, i);
if (ns) {
- if (revalidate_disk(ns->disk)) {
+ if (revalidate_disk(ns->disk))
nvme_ns_remove(ns);
- nvme_free_namespace(ns);
- }
} else
nvme_alloc_ns(dev, i);
}
list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
- if (ns->ns_id > nn) {
+ if (ns->ns_id > nn)
nvme_ns_remove(ns);
- nvme_free_namespace(ns);
- }
}
list_sort(NULL, &dev->namespaces, ns_cmp);
}
static void nvme_dev_remove(struct nvme_dev *dev)
{
- struct nvme_ns *ns;
+ struct nvme_ns *ns, *next;
- list_for_each_entry(ns, &dev->namespaces, list)
+ list_for_each_entry_safe(ns, next, &dev->namespaces, list)
nvme_ns_remove(ns);
}
spin_unlock(&dev_list_lock);
}
-static void nvme_free_namespaces(struct nvme_dev *dev)
-{
- struct nvme_ns *ns, *next;
-
- list_for_each_entry_safe(ns, next, &dev->namespaces, list)
- nvme_free_namespace(ns);
-}
-
static void nvme_free_dev(struct kref *kref)
{
struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
put_device(dev->dev);
put_device(dev->device);
- nvme_free_namespaces(dev);
nvme_release_instance(dev);
if (dev->tagset.tags)
blk_mq_free_tag_set(&dev->tagset);
.compat_ioctl = nvme_dev_ioctl,
};
-static int nvme_dev_start(struct nvme_dev *dev)
+static void nvme_probe_work(struct work_struct *work)
{
- int result;
+ struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
bool start_thread = false;
+ int result;
result = nvme_dev_map(dev);
if (result)
- return result;
+ goto out;
result = nvme_configure_admin_queue(dev);
if (result)
goto free_tags;
dev->event_limit = 1;
- return result;
+
+ /*
+ * Keep the controller around but remove all namespaces if we don't have
+ * any working I/O queue.
+ */
+ if (dev->online_queues < 2) {
+ dev_warn(dev->dev, "IO queues not created\n");
+ nvme_dev_remove(dev);
+ } else {
+ nvme_unfreeze_queues(dev);
+ nvme_dev_add(dev);
+ }
+
+ return;
free_tags:
nvme_dev_remove_admin(dev);
nvme_dev_list_remove(dev);
unmap:
nvme_dev_unmap(dev);
- return result;
+ out:
+ if (!work_busy(&dev->reset_work))
+ nvme_dead_ctrl(dev);
}
static int nvme_remove_dead_ctrl(void *arg)
return 0;
}
-static void nvme_remove_disks(struct work_struct *ws)
-{
- struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
-
- nvme_free_queues(dev, 1);
- nvme_dev_remove(dev);
-}
-
-static int nvme_dev_resume(struct nvme_dev *dev)
-{
- int ret;
-
- ret = nvme_dev_start(dev);
- if (ret)
- return ret;
- if (dev->online_queues < 2) {
- spin_lock(&dev_list_lock);
- dev->reset_workfn = nvme_remove_disks;
- queue_work(nvme_workq, &dev->reset_work);
- spin_unlock(&dev_list_lock);
- } else {
- nvme_unfreeze_queues(dev);
- nvme_dev_add(dev);
- }
- return 0;
-}
-
static void nvme_dead_ctrl(struct nvme_dev *dev)
{
dev_warn(dev->dev, "Device failed to resume\n");
}
}
-static void nvme_dev_reset(struct nvme_dev *dev)
+static void nvme_reset_work(struct work_struct *ws)
{
+ struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
bool in_probe = work_busy(&dev->probe_work);
nvme_dev_shutdown(dev);
schedule_work(&dev->probe_work);
}
-static void nvme_reset_failed_dev(struct work_struct *ws)
+static int __nvme_reset(struct nvme_dev *dev)
{
- struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
- nvme_dev_reset(dev);
-}
-
-static void nvme_reset_workfn(struct work_struct *work)
-{
- struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
- dev->reset_workfn(work);
+ if (work_pending(&dev->reset_work))
+ return -EBUSY;
+ list_del_init(&dev->node);
+ queue_work(nvme_workq, &dev->reset_work);
+ return 0;
}
static int nvme_reset(struct nvme_dev *dev)
{
- int ret = -EBUSY;
+ int ret;
if (!dev->admin_q || blk_queue_dying(dev->admin_q))
return -ENODEV;
spin_lock(&dev_list_lock);
- if (!work_pending(&dev->reset_work)) {
- dev->reset_workfn = nvme_reset_failed_dev;
- queue_work(nvme_workq, &dev->reset_work);
- ret = 0;
- }
+ ret = __nvme_reset(dev);
spin_unlock(&dev_list_lock);
if (!ret) {
}
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
-static void nvme_async_probe(struct work_struct *work);
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
int node, result = -ENOMEM;
goto free;
INIT_LIST_HEAD(&dev->namespaces);
- dev->reset_workfn = nvme_reset_failed_dev;
- INIT_WORK(&dev->reset_work, nvme_reset_workfn);
+ INIT_WORK(&dev->reset_work, nvme_reset_work);
dev->dev = get_device(&pdev->dev);
pci_set_drvdata(pdev, dev);
result = nvme_set_instance(dev);
INIT_LIST_HEAD(&dev->node);
INIT_WORK(&dev->scan_work, nvme_dev_scan);
- INIT_WORK(&dev->probe_work, nvme_async_probe);
+ INIT_WORK(&dev->probe_work, nvme_probe_work);
schedule_work(&dev->probe_work);
return 0;
return result;
}
-static void nvme_async_probe(struct work_struct *work)
-{
- struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
-
- if (nvme_dev_resume(dev) && !work_busy(&dev->reset_work))
- nvme_dead_ctrl(dev);
-}
-
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
struct nvme_dev *dev = pci_get_drvdata(pdev);
if (prepare)
nvme_dev_shutdown(dev);
else
- nvme_dev_resume(dev);
+ schedule_work(&dev->probe_work);
}
static void nvme_shutdown(struct pci_dev *pdev)
struct pci_dev *pdev = to_pci_dev(dev);
struct nvme_dev *ndev = pci_get_drvdata(pdev);
- if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) {
- ndev->reset_workfn = nvme_reset_failed_dev;
- queue_work(nvme_workq, &ndev->reset_work);
- }
+ schedule_work(&ndev->probe_work);
return 0;
}
#endif
#include <linux/timer.h>
#include <linux/kernel.h>
#include <linux/usb/hcd.h>
-
- #include <asm-generic/io-64-nonatomic-lo-hi.h>
++#include <linux/io-64-nonatomic-lo-hi.h>
/* Code sharing between pci-quirks and xhci hcd */
#include "xhci-ext-caps.h"
* @hcc_params: HCCPARAMS - Capability Parameters
* @db_off: DBOFF - Doorbell array offset
* @run_regs_off: RTSOFF - Runtime register space offset
+ * @hcc_params2: HCCPARAMS2 Capability Parameters 2, xhci 1.1 only
*/
struct xhci_cap_regs {
__le32 hc_capbase;
__le32 hcc_params;
__le32 db_off;
__le32 run_regs_off;
+ __le32 hcc_params2; /* xhci 1.1 */
/* Reserved up to (CAPLENGTH - 0x1C) */
};
/* run_regs_off bitmask - bits 0:4 reserved */
#define RTSOFF_MASK (~0x1f)
+/* HCCPARAMS2 - hcc_params2 - bitmasks */
+/* true: HC supports U3 entry Capability */
+#define HCC2_U3C(p) ((p) & (1 << 0))
+/* true: HC supports Configure endpoint command Max exit latency too large */
+#define HCC2_CMC(p) ((p) & (1 << 1))
+/* true: HC supports Force Save context Capability */
+#define HCC2_FSC(p) ((p) & (1 << 2))
+/* true: HC supports Compliance Transition Capability */
+#define HCC2_CTC(p) ((p) & (1 << 3))
+/* true: HC support Large ESIT payload Capability > 48k */
+#define HCC2_LEC(p) ((p) & (1 << 4))
+/* true: HC support Configuration Information Capability */
+#define HCC2_CIC(p) ((p) & (1 << 5))
+/* true: HC support Extended TBC Capability, Isoc burst count > 65535 */
+#define HCC2_ETC(p) ((p) & (1 << 6))
/* Number of registers per port */
#define NUM_PORT_REGS 4
/* CONFIG - Configure Register - config_reg bitmasks */
/* bits 0:7 - maximum number of device slots enabled (NumSlotsEn) */
#define MAX_DEVS(p) ((p) & 0xff)
-/* bits 8:31 - reserved and should be preserved */
+/* bit 8: U3 Entry Enabled, assert PLC when root port enters U3, xhci 1.1 */
+#define CONFIG_U3E (1 << 8)
+/* bit 9: Configuration Information Enable, xhci 1.1 */
+#define CONFIG_CIE (1 << 9)
+/* bits 10:31 - reserved and should be preserved */
/* PORTSC - Port Status and Control Register - port_status_base bitmasks */
/* true: device connected */
#define XDEV_LS (0x2 << 10)
#define XDEV_HS (0x3 << 10)
#define XDEV_SS (0x4 << 10)
+#define XDEV_SSP (0x5 << 10)
#define DEV_UNDEFSPEED(p) (((p) & DEV_SPEED_MASK) == (0x0<<10))
#define DEV_FULLSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_FS)
#define DEV_LOWSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_LS)
#define DEV_HIGHSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_HS)
#define DEV_SUPERSPEED(p) (((p) & DEV_SPEED_MASK) == XDEV_SS)
+#define DEV_SUPERSPEEDPLUS(p) (((p) & DEV_SPEED_MASK) == XDEV_SSP)
+#define DEV_SUPERSPEED_ANY(p) (((p) & DEV_SPEED_MASK) >= XDEV_SS)
+#define DEV_PORT_SPEED(p) (((p) >> 10) & 0x0f)
+
/* Bits 20:23 in the Slot Context are the speed for the device */
#define SLOT_SPEED_FS (XDEV_FS << 10)
#define SLOT_SPEED_LS (XDEV_LS << 10)
#define PORT_L1DS(p) (((p) & 0xff) << 8)
#define PORT_HLE (1 << 16)
+/* USB3 Protocol PORTLI Port Link Information */
+#define PORT_RX_LANES(p) (((p) >> 16) & 0xf)
+#define PORT_TX_LANES(p) (((p) >> 20) & 0xf)
/* USB2 Protocol PORTHLPMC */
#define PORT_HIRDM(p)((p) & 3)
};
#define XHCI_EXT_PORT_MAJOR(x) (((x) >> 24) & 0xff)
+#define XHCI_EXT_PORT_MINOR(x) (((x) >> 16) & 0xff)
+#define XHCI_EXT_PORT_PSIC(x) (((x) >> 28) & 0x0f)
#define XHCI_EXT_PORT_OFF(x) ((x) & 0xff)
#define XHCI_EXT_PORT_COUNT(x) (((x) >> 8) & 0xff)
+#define XHCI_EXT_PORT_PSIV(x) (((x) >> 0) & 0x0f)
+#define XHCI_EXT_PORT_PSIE(x) (((x) >> 4) & 0x03)
+#define XHCI_EXT_PORT_PLT(x) (((x) >> 6) & 0x03)
+#define XHCI_EXT_PORT_PFD(x) (((x) >> 8) & 0x01)
+#define XHCI_EXT_PORT_LP(x) (((x) >> 14) & 0x03)
+#define XHCI_EXT_PORT_PSIM(x) (((x) >> 16) & 0xffff)
+
+#define PLT_MASK (0x03 << 6)
+#define PLT_SYM (0x00 << 6)
+#define PLT_ASYM_RX (0x02 << 6)
+#define PLT_ASYM_TX (0x03 << 6)
+
/**
* struct xhci_container_ctx
* @type: Type of context. Used to calculated offsets to contained contexts.
/* Normal TRB fields */
/* transfer_len bitmasks - bits 0:16 */
#define TRB_LEN(p) ((p) & 0x1ffff)
+/* TD Size, packets remaining in this TD, bits 21:17 (5 bits, so max 31) */
+#define TRB_TD_SIZE(p) (min((p), (u32)31) << 17)
/* Interrupter Target - which MSI-X vector to target the completion event at */
#define TRB_INTR_TARGET(p) (((p) & 0x3ff) << 22)
#define GET_INTR_TARGET(p) (((p) >> 22) & 0x3ff)
return 1;
}
+struct xhci_hub {
+ u8 maj_rev;
+ u8 min_rev;
+ u32 *psi; /* array of protocol speed ID entries */
+ u8 psi_count;
+ u8 psi_uid_count;
+};
+
/* There is one xhci_hcd structure per controller */
struct xhci_hcd {
struct usb_hcd *main_hcd;
__u32 hcs_params2;
__u32 hcs_params3;
__u32 hcc_params;
+ __u32 hcc_params2;
spinlock_t lock;
unsigned int num_usb3_ports;
/* Array of pointers to USB 2.0 PORTSC registers */
__le32 __iomem **usb2_ports;
+ struct xhci_hub usb2_rhub;
+ struct xhci_hub usb3_rhub;
unsigned int num_usb2_ports;
/* support xHCI 0.96 spec USB2 software LPM */
unsigned sw_lpm_support:1;
static inline u64 xhci_read_64(const struct xhci_hcd *xhci,
__le64 __iomem *regs)
{
- __u32 __iomem *ptr = (__u32 __iomem *) regs;
- u64 val_lo = readl(ptr);
- u64 val_hi = readl(ptr + 1);
- return val_lo + (val_hi << 32);
+ return lo_hi_readq(regs);
}
static inline void xhci_write_64(struct xhci_hcd *xhci,
const u64 val, __le64 __iomem *regs)
{
- __u32 __iomem *ptr = (__u32 __iomem *) regs;
- u32 val_lo = lower_32_bits(val);
- u32 val_hi = upper_32_bits(val);
-
- writel(val_lo, ptr);
- writel(val_hi, ptr + 1);
+ lo_hi_writeq(val, regs);
}
static inline int xhci_link_trb_quirk(struct xhci_hcd *xhci)
*/
#include <linux/bitops.h>
- #include <asm-generic/bitops/count_zeros.h>
+ #include <linux/count_zeros.h>
#include "mpi-internal.h"
#define MAX_EXTERN_MPI_BITS 16384
return 0;
}
EXPORT_SYMBOL_GPL(mpi_set_buffer);
+
+/**
+ * mpi_write_to_sgl() - Funnction exports MPI to an sgl (msb first)
+ *
+ * This function works in the same way as the mpi_read_buffer, but it
+ * takes an sgl instead of u8 * buf.
+ *
+ * @a: a multi precision integer
+ * @sgl: scatterlist to write to. Needs to be at least
+ * mpi_get_size(a) long.
+ * @nbytes: in/out param - it has the be set to the maximum number of
+ * bytes that can be written to sgl. This has to be at least
+ * the size of the integer a. On return it receives the actual
+ * length of the data written.
+ * @sign: if not NULL, it will be set to the sign of a.
+ *
+ * Return: 0 on success or error code in case of error
+ */
+int mpi_write_to_sgl(MPI a, struct scatterlist *sgl, unsigned *nbytes,
+ int *sign)
+{
+ u8 *p, *p2;
+ mpi_limb_t alimb, alimb2;
+ unsigned int n = mpi_get_size(a);
+ int i, x, y = 0, lzeros = 0, buf_len;
+
+ if (!nbytes || *nbytes < n)
+ return -EINVAL;
+
+ if (sign)
+ *sign = a->sign;
+
+ p = (void *)&a->d[a->nlimbs] - 1;
+
+ for (i = a->nlimbs * sizeof(alimb) - 1; i >= 0; i--, p--) {
+ if (!*p)
+ lzeros++;
+ else
+ break;
+ }
+
+ *nbytes = n - lzeros;
+ buf_len = sgl->length;
+ p2 = sg_virt(sgl);
+
+ for (i = a->nlimbs - 1; i >= 0; i--) {
+ alimb = a->d[i];
+ p = (u8 *)&alimb2;
+#if BYTES_PER_MPI_LIMB == 4
+ *p++ = alimb >> 24;
+ *p++ = alimb >> 16;
+ *p++ = alimb >> 8;
+ *p++ = alimb;
+#elif BYTES_PER_MPI_LIMB == 8
+ *p++ = alimb >> 56;
+ *p++ = alimb >> 48;
+ *p++ = alimb >> 40;
+ *p++ = alimb >> 32;
+ *p++ = alimb >> 24;
+ *p++ = alimb >> 16;
+ *p++ = alimb >> 8;
+ *p++ = alimb;
+#else
+#error please implement for this limb size.
+#endif
+ if (lzeros > 0) {
+ if (lzeros >= sizeof(alimb)) {
+ p -= sizeof(alimb);
+ continue;
+ } else {
+ mpi_limb_t *limb1 = (void *)p - sizeof(alimb);
+ mpi_limb_t *limb2 = (void *)p - sizeof(alimb)
+ + lzeros;
+ *limb1 = *limb2;
+ p -= lzeros;
+ y = lzeros;
+ }
+ lzeros -= sizeof(alimb);
+ }
+
+ p = p - (sizeof(alimb) - y);
+
+ for (x = 0; x < sizeof(alimb) - y; x++) {
+ if (!buf_len) {
+ sgl = sg_next(sgl);
+ if (!sgl)
+ return -EINVAL;
+ buf_len = sgl->length;
+ p2 = sg_virt(sgl);
+ }
+ *p2++ = *p++;
+ buf_len--;
+ }
+ y = 0;
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(mpi_write_to_sgl);
+
+/*
+ * mpi_read_raw_from_sgl() - Function allocates an MPI and populates it with
+ * data from the sgl
+ *
+ * This function works in the same way as the mpi_read_raw_data, but it
+ * takes an sgl instead of void * buffer. i.e. it allocates
+ * a new MPI and reads the content of the sgl to the MPI.
+ *
+ * @sgl: scatterlist to read from
+ * @len: number of bytes to read
+ *
+ * Return: Pointer to a new MPI or NULL on error
+ */
+MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int len)
+{
+ struct scatterlist *sg;
+ int x, i, j, z, lzeros, ents;
+ unsigned int nbits, nlimbs, nbytes;
+ mpi_limb_t a;
+ MPI val = NULL;
+
+ lzeros = 0;
+ ents = sg_nents(sgl);
+
+ for_each_sg(sgl, sg, ents, i) {
+ const u8 *buff = sg_virt(sg);
+ int len = sg->length;
+
+ while (len && !*buff) {
+ lzeros++;
+ len--;
+ buff++;
+ }
+
+ if (len && *buff)
+ break;
+
+ ents--;
+ lzeros = 0;
+ }
+
+ sgl = sg;
+
+ if (!ents)
+ nbytes = 0;
+ else
+ nbytes = len - lzeros;
+
+ nbits = nbytes * 8;
+ if (nbits > MAX_EXTERN_MPI_BITS) {
+ pr_info("MPI: mpi too large (%u bits)\n", nbits);
+ return NULL;
+ }
+
+ if (nbytes > 0)
+ nbits -= count_leading_zeros(*(u8 *)(sg_virt(sgl) + lzeros));
+ else
+ nbits = 0;
+
+ nlimbs = DIV_ROUND_UP(nbytes, BYTES_PER_MPI_LIMB);
+ val = mpi_alloc(nlimbs);
+ if (!val)
+ return NULL;
+
+ val->nbits = nbits;
+ val->sign = 0;
+ val->nlimbs = nlimbs;
+
+ if (nbytes == 0)
+ return val;
+
+ j = nlimbs - 1;
+ a = 0;
+ z = 0;
+ x = BYTES_PER_MPI_LIMB - nbytes % BYTES_PER_MPI_LIMB;
+ x %= BYTES_PER_MPI_LIMB;
+
+ for_each_sg(sgl, sg, ents, i) {
+ const u8 *buffer = sg_virt(sg) + lzeros;
+ int len = sg->length - lzeros;
+ int buf_shift = x;
+
+ if (sg_is_last(sg) && (len % BYTES_PER_MPI_LIMB))
+ len += BYTES_PER_MPI_LIMB - (len % BYTES_PER_MPI_LIMB);
+
+ for (; x < len + buf_shift; x++) {
+ a <<= 8;
+ a |= *buffer++;
+ if (((z + x + 1) % BYTES_PER_MPI_LIMB) == 0) {
+ val->d[j--] = a;
+ a = 0;
+ }
+ }
+ z += x;
+ x = 0;
+ lzeros = 0;
+ }
+ return val;
+}
+EXPORT_SYMBOL_GPL(mpi_read_raw_from_sgl);
projects / karo-tx-linux.git / commitdiff