/* common channel initialization */
void ioat_init_channel(struct ioatdma_device *device,
struct ioat_chan_common *chan, int idx,
- work_func_t work_fn, void (*tasklet)(unsigned long),
- unsigned long tasklet_data)
+ void (*timer_fn)(unsigned long),
+ void (*tasklet)(unsigned long),
+ unsigned long ioat)
{
struct dma_device *dma = &device->common;
chan->device = device;
chan->reg_base = device->reg_base + (0x80 * (idx + 1));
- INIT_DELAYED_WORK(&chan->work, work_fn);
spin_lock_init(&chan->cleanup_lock);
chan->common.device = dma;
list_add_tail(&chan->common.device_node, &dma->channels);
device->idx[idx] = chan;
- tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
+ init_timer(&chan->timer);
+ chan->timer.function = timer_fn;
+ chan->timer.data = ioat;
+ tasklet_init(&chan->cleanup_task, tasklet, ioat);
tasklet_disable(&chan->cleanup_task);
}
-static void ioat1_reset_part2(struct work_struct *work);
+static void ioat1_timer_event(unsigned long data);
/**
* ioat1_dma_enumerate_channels - find and initialize the device's channels
INIT_LIST_HEAD(&dma->channels);
dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+ dma->chancnt &= 0x1f; /* bits [4:0] valid */
+ if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+ dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+ dma->chancnt, ARRAY_SIZE(device->idx));
+ dma->chancnt = ARRAY_SIZE(device->idx);
+ }
xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+ xfercap_scale &= 0x1f; /* bits [4:0] valid */
xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
break;
ioat_init_channel(device, &ioat->base, i,
- ioat1_reset_part2,
+ ioat1_timer_event,
ioat1_cleanup_tasklet,
(unsigned long) ioat);
ioat->xfercap = xfercap;
}
}
-/**
- * ioat1_reset_part2 - reinit the channel after a reset
- */
-static void ioat1_reset_part2(struct work_struct *work)
-{
- struct ioat_chan_common *chan;
- struct ioat_dma_chan *ioat;
- struct ioat_desc_sw *desc;
- int dmacount;
- bool start_null = false;
-
- chan = container_of(work, struct ioat_chan_common, work.work);
- ioat = container_of(chan, struct ioat_dma_chan, base);
- spin_lock_bh(&chan->cleanup_lock);
- spin_lock_bh(&ioat->desc_lock);
-
- chan->completion_virt->low = 0;
- chan->completion_virt->high = 0;
- ioat->pending = 0;
-
- /* count the descriptors waiting */
- dmacount = 0;
- if (ioat->used_desc.prev) {
- desc = to_ioat_desc(ioat->used_desc.prev);
- do {
- dmacount++;
- desc = to_ioat_desc(desc->node.next);
- } while (&desc->node != ioat->used_desc.next);
- }
-
- if (dmacount) {
- /*
- * write the new starting descriptor address
- * this puts channel engine into ARMED state
- */
- desc = to_ioat_desc(ioat->used_desc.prev);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
- writeb(IOAT_CHANCMD_START, chan->reg_base
- + IOAT_CHANCMD_OFFSET(chan->device->version));
- } else
- start_null = true;
- spin_unlock_bh(&ioat->desc_lock);
- spin_unlock_bh(&chan->cleanup_lock);
-
- dev_err(to_dev(chan),
- "chan%d reset - %d descs waiting, %d total desc\n",
- chan_num(chan), dmacount, ioat->desccount);
-
- if (start_null)
- ioat1_dma_start_null_desc(ioat);
-}
-
/**
* ioat1_reset_channel - restart a channel
* @ioat: IOAT DMA channel handle
void __iomem *reg_base = chan->reg_base;
u32 chansts, chanerr;
- if (!ioat->used_desc.prev)
- return;
-
- dev_dbg(to_dev(chan), "%s\n", __func__);
+ dev_warn(to_dev(chan), "reset\n");
chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
- chansts = (chan->completion_virt->low
- & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
+ chansts = *chan->completion & IOAT_CHANSTS_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
* while we're waiting.
*/
- spin_lock_bh(&ioat->desc_lock);
ioat->pending = INT_MIN;
writeb(IOAT_CHANCMD_RESET,
reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
- spin_unlock_bh(&ioat->desc_lock);
-
- /* schedule the 2nd half instead of sleeping a long time */
- schedule_delayed_work(&chan->work, RESET_DELAY);
-}
-
-/**
- * ioat1_chan_watchdog - watch for stuck channels
- */
-static void ioat1_chan_watchdog(struct work_struct *work)
-{
- struct ioatdma_device *device =
- container_of(work, struct ioatdma_device, work.work);
- struct ioat_dma_chan *ioat;
- struct ioat_chan_common *chan;
- int i;
-
- union {
- u64 full;
- struct {
- u32 low;
- u32 high;
- };
- } completion_hw;
- unsigned long compl_desc_addr_hw;
-
- for (i = 0; i < device->common.chancnt; i++) {
- chan = ioat_chan_by_index(device, i);
- ioat = container_of(chan, struct ioat_dma_chan, base);
-
- if (/* have we started processing anything yet */
- chan->last_completion
- /* have we completed any since last watchdog cycle? */
- && (chan->last_completion == chan->watchdog_completion)
- /* has TCP stuck on one cookie since last watchdog? */
- && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
- && (chan->watchdog_tcp_cookie != chan->completed_cookie)
- /* is there something in the chain to be processed? */
- /* CB1 chain always has at least the last one processed */
- && (ioat->used_desc.prev != ioat->used_desc.next)
- && ioat->pending == 0) {
-
- /*
- * check CHANSTS register for completed
- * descriptor address.
- * if it is different than completion writeback,
- * it is not zero
- * and it has changed since the last watchdog
- * we can assume that channel
- * is still working correctly
- * and the problem is in completion writeback.
- * update completion writeback
- * with actual CHANSTS value
- * else
- * try resetting the channel
- */
-
- completion_hw.low = readl(chan->reg_base +
- IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
- completion_hw.high = readl(chan->reg_base +
- IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
-#if (BITS_PER_LONG == 64)
- compl_desc_addr_hw =
- completion_hw.full
- & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
- compl_desc_addr_hw =
- completion_hw.low & IOAT_LOW_COMPLETION_MASK;
-#endif
-
- if ((compl_desc_addr_hw != 0)
- && (compl_desc_addr_hw != chan->watchdog_completion)
- && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
- chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
- chan->completion_virt->low = completion_hw.low;
- chan->completion_virt->high = completion_hw.high;
- } else {
- ioat1_reset_channel(ioat);
- chan->watchdog_completion = 0;
- chan->last_compl_desc_addr_hw = 0;
- }
- } else {
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_completion = chan->last_completion;
- }
-
- chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
- }
-
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+ set_bit(IOAT_RESET_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + RESET_DELAY);
}
static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
struct dma_chan *c = tx->chan;
struct ioat_dma_chan *ioat = to_ioat_chan(c);
struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+ struct ioat_chan_common *chan = &ioat->base;
struct ioat_desc_sw *first;
struct ioat_desc_sw *chain_tail;
dma_cookie_t cookie;
dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
/* write address into NextDescriptor field of last desc in chain */
- first = to_ioat_desc(tx->tx_list.next);
+ first = to_ioat_desc(desc->tx_list.next);
chain_tail = to_ioat_desc(ioat->used_desc.prev);
/* make descriptor updates globally visible before chaining */
wmb();
chain_tail->hw->next = first->txd.phys;
- list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
+ list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
dump_desc_dbg(ioat, chain_tail);
dump_desc_dbg(ioat, first);
- ioat->pending += desc->tx_cnt;
+ if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+ ioat->active += desc->hw->tx_cnt;
+ ioat->pending += desc->hw->tx_cnt;
if (ioat->pending >= ioat_pending_level)
__ioat1_dma_memcpy_issue_pending(ioat);
spin_unlock_bh(&ioat->desc_lock);
memset(desc, 0, sizeof(*desc));
+ INIT_LIST_HEAD(&desc_sw->tx_list);
dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
desc_sw->txd.tx_submit = ioat1_tx_submit;
desc_sw->hw = desc;
struct ioat_dma_chan *ioat = to_ioat_chan(c);
struct ioat_chan_common *chan = &ioat->base;
struct ioat_desc_sw *desc;
- u16 chanctrl;
u32 chanerr;
int i;
LIST_HEAD(tmp_list);
return ioat->desccount;
/* Setup register to interrupt and write completion status on error */
- chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
- IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
- IOAT_CHANCTRL_ERR_COMPLETION_EN;
- writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+ writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
if (chanerr) {
/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
- chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
- GFP_KERNEL,
- &chan->completion_addr);
- memset(chan->completion_virt, 0,
- sizeof(*chan->completion_virt));
- writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
+ chan->completion = pci_pool_alloc(chan->device->completion_pool,
+ GFP_KERNEL, &chan->completion_dma);
+ memset(chan->completion, 0, sizeof(*chan->completion));
+ writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
- writel(((u64) chan->completion_addr) >> 32,
+ writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
tasklet_enable(&chan->cleanup_task);
return;
tasklet_disable(&chan->cleanup_task);
+ del_timer_sync(&chan->timer);
ioat1_cleanup(ioat);
/* Delay 100ms after reset to allow internal DMA logic to quiesce
spin_unlock_bh(&ioat->desc_lock);
pci_pool_free(ioatdma_device->completion_pool,
- chan->completion_virt,
- chan->completion_addr);
+ chan->completion,
+ chan->completion_dma);
/* one is ok since we left it on there on purpose */
if (in_use_descs > 1)
dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
in_use_descs - 1);
- chan->last_completion = chan->completion_addr = 0;
- chan->watchdog_completion = 0;
- chan->last_compl_desc_addr_hw = 0;
- chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
+ chan->last_completion = 0;
+ chan->completion_dma = 0;
ioat->pending = 0;
ioat->desccount = 0;
}
spin_unlock_bh(&ioat->desc_lock);
desc->txd.flags = flags;
- desc->tx_cnt = tx_cnt;
desc->len = total_len;
- list_splice(&chain, &desc->txd.tx_list);
+ list_splice(&chain, &desc->tx_list);
hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
hw->ctl_f.compl_write = 1;
+ hw->tx_cnt = tx_cnt;
dump_desc_dbg(ioat, desc);
return &desc->txd;
static void ioat1_cleanup_tasklet(unsigned long data)
{
struct ioat_dma_chan *chan = (void *)data;
- ioat1_cleanup(chan);
- writew(IOAT_CHANCTRL_INT_DISABLE,
- chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
-}
-static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
- int direction, enum dma_ctrl_flags flags, bool dst)
-{
- if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
- (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
- pci_unmap_single(pdev, addr, len, direction);
- else
- pci_unmap_page(pdev, addr, len, direction);
+ ioat1_cleanup(chan);
+ writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
}
-
void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
size_t len, struct ioat_dma_descriptor *hw)
{
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
{
unsigned long phys_complete;
+ u64 completion;
- /* The completion writeback can happen at any time,
- so reads by the driver need to be atomic operations
- The descriptor physical addresses are limited to 32-bits
- when the CPU can only do a 32-bit mov */
-
-#if (BITS_PER_LONG == 64)
- phys_complete =
- chan->completion_virt->full
- & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
- phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
-#endif
+ completion = *chan->completion;
+ phys_complete = ioat_chansts_to_addr(completion);
dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete);
- if ((chan->completion_virt->full
- & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
- IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
+ if (is_ioat_halted(completion)) {
+ u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
- readl(chan->reg_base + IOAT_CHANERR_OFFSET));
+ chanerr);
/* TODO do something to salvage the situation */
}
return phys_complete;
}
-/**
- * ioat1_cleanup - cleanup up finished descriptors
- * @chan: ioat channel to be cleaned up
- */
-static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+ unsigned long *phys_complete)
{
- struct ioat_chan_common *chan = &ioat->base;
- unsigned long phys_complete;
- struct ioat_desc_sw *desc, *_desc;
- dma_cookie_t cookie = 0;
- struct dma_async_tx_descriptor *tx;
-
- prefetch(chan->completion_virt);
+ *phys_complete = ioat_get_current_completion(chan);
+ if (*phys_complete == chan->last_completion)
+ return false;
+ clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
- if (!spin_trylock_bh(&chan->cleanup_lock))
- return;
-
- phys_complete = ioat_get_current_completion(chan);
- if (phys_complete == chan->last_completion) {
- spin_unlock_bh(&chan->cleanup_lock);
- /*
- * perhaps we're stuck so hard that the watchdog can't go off?
- * try to catch it after 2 seconds
- */
- if (time_after(jiffies,
- chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
- ioat1_chan_watchdog(&(chan->device->work.work));
- chan->last_completion_time = jiffies;
- }
- return;
- }
- chan->last_completion_time = jiffies;
+ return true;
+}
- cookie = 0;
- if (!spin_trylock_bh(&ioat->desc_lock)) {
- spin_unlock_bh(&chan->cleanup_lock);
- return;
- }
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ struct list_head *_desc, *n;
+ struct dma_async_tx_descriptor *tx;
dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
__func__, phys_complete);
- list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+ list_for_each_safe(_desc, n, &ioat->used_desc) {
+ struct ioat_desc_sw *desc;
+
+ prefetch(n);
+ desc = list_entry(_desc, typeof(*desc), node);
tx = &desc->txd;
/*
* Incoming DMA requests may use multiple descriptors,
*/
dump_desc_dbg(ioat, desc);
if (tx->cookie) {
- cookie = tx->cookie;
+ chan->completed_cookie = tx->cookie;
+ tx->cookie = 0;
ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+ ioat->active -= desc->hw->tx_cnt;
if (tx->callback) {
tx->callback(tx->callback_param);
tx->callback = NULL;
*/
if (async_tx_test_ack(tx))
list_move_tail(&desc->node, &ioat->free_desc);
- else
- tx->cookie = 0;
} else {
/*
* last used desc. Do not remove, so we can
- * append from it, but don't look at it next
- * time, either
+ * append from it.
*/
- tx->cookie = 0;
+
+ /* if nothing else is pending, cancel the
+ * completion timeout
+ */
+ if (n == &ioat->used_desc) {
+ dev_dbg(to_dev(chan),
+ "%s cancel completion timeout\n",
+ __func__);
+ clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ }
/* TODO check status bits? */
break;
}
}
+ chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+ struct ioat_chan_common *chan = &ioat->base;
+ unsigned long phys_complete;
+
+ prefetch(chan->completion);
+
+ if (!spin_trylock_bh(&chan->cleanup_lock))
+ return;
+
+ if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ if (!spin_trylock_bh(&ioat->desc_lock)) {
+ spin_unlock_bh(&chan->cleanup_lock);
+ return;
+ }
+
+ __cleanup(ioat, phys_complete);
+
spin_unlock_bh(&ioat->desc_lock);
+ spin_unlock_bh(&chan->cleanup_lock);
+}
- chan->last_completion = phys_complete;
- if (cookie != 0)
- chan->completed_cookie = cookie;
+static void ioat1_timer_event(unsigned long data)
+{
+ struct ioat_dma_chan *ioat = (void *) data;
+ struct ioat_chan_common *chan = &ioat->base;
+
+ dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+ spin_lock_bh(&chan->cleanup_lock);
+ if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+ struct ioat_desc_sw *desc;
+
+ spin_lock_bh(&ioat->desc_lock);
+
+ /* restart active descriptors */
+ desc = to_ioat_desc(ioat->used_desc.prev);
+ ioat_set_chainaddr(ioat, desc->txd.phys);
+ ioat_start(chan);
+ ioat->pending = 0;
+ set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ spin_unlock_bh(&ioat->desc_lock);
+ } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+ unsigned long phys_complete;
+
+ spin_lock_bh(&ioat->desc_lock);
+ /* if we haven't made progress and we have already
+ * acknowledged a pending completion once, then be more
+ * forceful with a restart
+ */
+ if (ioat_cleanup_preamble(chan, &phys_complete))
+ __cleanup(ioat, phys_complete);
+ else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+ ioat1_reset_channel(ioat);
+ else {
+ u64 status = ioat_chansts(chan);
+
+ /* manually update the last completion address */
+ if (ioat_chansts_to_addr(status) != 0)
+ *chan->completion = status;
+
+ set_bit(IOAT_COMPLETION_ACK, &chan->state);
+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+ }
+ spin_unlock_bh(&ioat->desc_lock);
+ }
spin_unlock_bh(&chan->cleanup_lock);
}
list_add_tail(&desc->node, &ioat->used_desc);
dump_desc_dbg(ioat, desc);
- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
- writel(((u64) desc->txd.phys) >> 32,
- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
- writeb(IOAT_CHANCMD_START, chan->reg_base
- + IOAT_CHANCMD_OFFSET(chan->device->version));
+ ioat_set_chainaddr(ioat, desc->txd.phys);
+ ioat_start(chan);
spin_unlock_bh(&ioat->desc_lock);
}
*/
#define IOAT_TEST_SIZE 2000
-static void ioat_dma_test_callback(void *dma_async_param)
+static void __devinit ioat_dma_test_callback(void *dma_async_param)
{
struct completion *cmp = dma_async_param;
* ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
* @device: device to be tested
*/
-static int ioat_dma_self_test(struct ioatdma_device *device)
+int __devinit ioat_dma_self_test(struct ioatdma_device *device)
{
int i;
u8 *src;
writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
}
-int ioat_probe(struct ioatdma_device *device)
+int __devinit ioat_probe(struct ioatdma_device *device)
{
int err = -ENODEV;
struct dma_device *dma = &device->common;
dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->dev = &pdev->dev;
- dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
- " %d channels, device version 0x%02x, driver version %s\n",
- dma->chancnt, device->version, IOAT_DMA_VERSION);
-
if (!dma->chancnt) {
- dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
- "zero channels detected\n");
+ dev_err(dev, "zero channels detected\n");
goto err_setup_interrupts;
}
if (err)
goto err_setup_interrupts;
- err = ioat_dma_self_test(device);
+ err = device->self_test(device);
if (err)
goto err_self_test;
return err;
}
-int ioat_register(struct ioatdma_device *device)
+int __devinit ioat_register(struct ioatdma_device *device)
{
int err = dma_async_device_register(&device->common);
pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
}
-int ioat1_dma_probe(struct ioatdma_device *device, int dca)
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+ struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+ return sprintf(page, "%d\n", ioat->desccount);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+ struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+ return sprintf(page, "%d\n", ioat->active);
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static ssize_t cap_show(struct dma_chan *c, char *page)
+{
+ struct dma_device *dma = c->device;
+
+ return sprintf(page, "copy%s%s%s%s%s%s\n",
+ dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
+ dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
+ dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
+ dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
+ dma_has_cap(DMA_MEMSET, dma->cap_mask) ? " fill" : "",
+ dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
+
+}
+struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
+
+static ssize_t version_show(struct dma_chan *c, char *page)
+{
+ struct dma_device *dma = c->device;
+ struct ioatdma_device *device = to_ioatdma_device(dma);
+
+ return sprintf(page, "%d.%d\n",
+ device->version >> 4, device->version & 0xf);
+}
+struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
+
+static struct attribute *ioat1_attrs[] = {
+ &ring_size_attr.attr,
+ &ring_active_attr.attr,
+ &ioat_cap_attr.attr,
+ &ioat_version_attr.attr,
+ NULL,
+};
+
+static ssize_t
+ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
+{
+ struct ioat_sysfs_entry *entry;
+ struct ioat_chan_common *chan;
+
+ entry = container_of(attr, struct ioat_sysfs_entry, attr);
+ chan = container_of(kobj, struct ioat_chan_common, kobj);
+
+ if (!entry->show)
+ return -EIO;
+ return entry->show(&chan->common, page);
+}
+
+struct sysfs_ops ioat_sysfs_ops = {
+ .show = ioat_attr_show,
+};
+
+static struct kobj_type ioat1_ktype = {
+ .sysfs_ops = &ioat_sysfs_ops,
+ .default_attrs = ioat1_attrs,
+};
+
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
+{
+ struct dma_device *dma = &device->common;
+ struct dma_chan *c;
+
+ list_for_each_entry(c, &dma->channels, device_node) {
+ struct ioat_chan_common *chan = to_chan_common(c);
+ struct kobject *parent = &c->dev->device.kobj;
+ int err;
+
+ err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
+ if (err) {
+ dev_warn(to_dev(chan),
+ "sysfs init error (%d), continuing...\n", err);
+ kobject_put(&chan->kobj);
+ set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
+ }
+ }
+}
+
+void ioat_kobject_del(struct ioatdma_device *device)
+{
+ struct dma_device *dma = &device->common;
+ struct dma_chan *c;
+
+ list_for_each_entry(c, &dma->channels, device_node) {
+ struct ioat_chan_common *chan = to_chan_common(c);
+
+ if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
+ kobject_del(&chan->kobj);
+ kobject_put(&chan->kobj);
+ }
+ }
+}
+
+int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
{
struct pci_dev *pdev = device->pdev;
struct dma_device *dma;
device->intr_quirk = ioat1_intr_quirk;
device->enumerate_channels = ioat1_enumerate_channels;
+ device->self_test = ioat_dma_self_test;
dma = &device->common;
dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
err = ioat_register(device);
if (err)
return err;
+ ioat_kobject_add(device, &ioat1_ktype);
+
if (dca)
device->dca = ioat_dca_init(pdev, device->reg_base);
- INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-
return err;
}
-void ioat_dma_remove(struct ioatdma_device *device)
+void __devexit ioat_dma_remove(struct ioatdma_device *device)
{
struct dma_device *dma = &device->common;
- if (device->version != IOAT_VER_3_0)
- cancel_delayed_work(&device->work);
-
ioat_disable_interrupts(device);
+ ioat_kobject_del(device);
+
dma_async_device_unregister(dma);
pci_pool_destroy(device->dma_pool);
projects / karo-tx-linux.git / blobdiff