drivers/misc/genwqe/card_ddcb.c

   1 /**
   2  * IBM Accelerator Family 'GenWQE'
   3  *
   4  * (C) Copyright IBM Corp. 2013
   5  *
   6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   8  * Author: Michael Jung <mijung@gmx.net>
   9  * Author: Michael Ruettger <michael@ibmra.de>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License (version 2 only)
  13  * as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 /*
  22  * Device Driver Control Block (DDCB) queue support. Definition of
  23  * interrupt handlers for queue support as well as triggering the
  24  * health monitor code in case of problems. The current hardware uses
  25  * an MSI interrupt which is shared between error handling and
  26  * functional code.
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/module.h>
  31 #include <linux/sched.h>
  32 #include <linux/wait.h>
  33 #include <linux/pci.h>
  34 #include <linux/string.h>
  35 #include <linux/dma-mapping.h>
  36 #include <linux/delay.h>
  37 #include <linux/module.h>
  38 #include <linux/interrupt.h>
  39 #include <linux/crc-itu-t.h>
  40
  41 #include "card_base.h"
  42 #include "card_ddcb.h"
  43
  44 /*
  45  * N: next DDCB, this is where the next DDCB will be put.
  46  * A: active DDCB, this is where the code will look for the next completion.
  47  * x: DDCB is enqueued, we are waiting for its completion.
  48
  49  * Situation (1): Empty queue
  50  *  +---+---+---+---+---+---+---+---+
  51  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  52  *  |   |   |   |   |   |   |   |   |
  53  *  +---+---+---+---+---+---+---+---+
  54  *           A/N
  55  *  enqueued_ddcbs = A - N = 2 - 2 = 0
  56  *
  57  * Situation (2): Wrapped, N > A
  58  *  +---+---+---+---+---+---+---+---+
  59  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  60  *  |   |   | x | x |   |   |   |   |
  61  *  +---+---+---+---+---+---+---+---+
  62  *            A       N
  63  *  enqueued_ddcbs = N - A = 4 - 2 = 2
  64  *
  65  * Situation (3): Queue wrapped, A > N
  66  *  +---+---+---+---+---+---+---+---+
  67  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  68  *  | x | x |   |   | x | x | x | x |
  69  *  +---+---+---+---+---+---+---+---+
  70  *            N       A
  71  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
  72  *
  73  * Situation (4a): Queue full N > A
  74  *  +---+---+---+---+---+---+---+---+
  75  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  76  *  | x | x | x | x | x | x | x |   |
  77  *  +---+---+---+---+---+---+---+---+
  78  *    A                           N
  79  *
  80  *  enqueued_ddcbs = N - A = 7 - 0 = 7
  81  *
  82  * Situation (4a): Queue full A > N
  83  *  +---+---+---+---+---+---+---+---+
  84  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  85  *  | x | x | x |   | x | x | x | x |
  86  *  +---+---+---+---+---+---+---+---+
  87  *                N   A
  88  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
  89  */
  90
  91 static int queue_empty(struct ddcb_queue *queue)
  92 {
  93         return queue->ddcb_next == queue->ddcb_act;
  94 }
  95
  96 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
  97 {
  98         if (queue->ddcb_next >= queue->ddcb_act)
  99                 return queue->ddcb_next - queue->ddcb_act;
 100
 101         return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
 102 }
 103
 104 static int queue_free_ddcbs(struct ddcb_queue *queue)
 105 {
 106         int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
 107
 108         if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
 109                 return 0;
 110         }
 111         return free_ddcbs;
 112 }
 113
 114 /*
 115  * Use of the PRIV field in the DDCB for queue debugging:
 116  *
 117  * (1) Trying to get rid of a DDCB which saw a timeout:
 118  *     pddcb->priv[6] = 0xcc;   # cleared
 119  *
 120  * (2) Append a DDCB via NEXT bit:
 121  *     pddcb->priv[7] = 0xaa;   # appended
 122  *
 123  * (3) DDCB needed tapping:
 124  *     pddcb->priv[7] = 0xbb;   # tapped
 125  *
 126  * (4) DDCB marked as correctly finished:
 127  *     pddcb->priv[6] = 0xff;   # finished
 128  */
 129
 130 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
 131 {
 132         pddcb->priv[7] = 0xbb;  /* tapped */
 133 }
 134
 135 static inline void ddcb_mark_appended(struct ddcb *pddcb)
 136 {
 137         pddcb->priv[7] = 0xaa;  /* appended */
 138 }
 139
 140 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
 141 {
 142         pddcb->priv[6] = 0xcc; /* cleared */
 143 }
 144
 145 static inline void ddcb_mark_finished(struct ddcb *pddcb)
 146 {
 147         pddcb->priv[6] = 0xff;  /* finished */
 148 }
 149
 150 static inline void ddcb_mark_unused(struct ddcb *pddcb)
 151 {
 152         pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
 153 }
 154
 155 /**
 156  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
 157  * @buff:       pointer to data buffer
 158  * @len:        length of data for calculation
 159  * @init:       initial crc (0xffff at start)
 160  *
 161  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
 162  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
 163  *          should result in a crc16 of 0x89c3
 164  *
 165  * Return: crc16 checksum in big endian format !
 166  */
 167 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
 168 {
 169         return crc_itu_t(init, buff, len);
 170 }
 171
 172 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
 173 {
 174         int i;
 175         struct ddcb *pddcb;
 176         unsigned long flags;
 177         struct pci_dev *pci_dev = cd->pci_dev;
 178
 179         spin_lock_irqsave(&cd->print_lock, flags);
 180
 181         dev_info(&pci_dev->dev,
 182                  "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
 183                  cd->card_idx, queue->ddcb_act, queue->ddcb_next);
 184
 185         pddcb = queue->ddcb_vaddr;
 186         for (i = 0; i < queue->ddcb_max; i++) {
 187                 dev_err(&pci_dev->dev,
 188                         "  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
 189                         i == queue->ddcb_act ? '>' : ' ',
 190                         i,
 191                         be16_to_cpu(pddcb->retc_16),
 192                         be16_to_cpu(pddcb->seqnum_16),
 193                         pddcb->hsi,
 194                         pddcb->shi,
 195                         be64_to_cpu(pddcb->priv_64),
 196                         pddcb->cmd);
 197                 pddcb++;
 198         }
 199         spin_unlock_irqrestore(&cd->print_lock, flags);
 200 }
 201
 202 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
 203 {
 204         struct ddcb_requ *req;
 205
 206         req = kzalloc(sizeof(*req), GFP_ATOMIC);
 207         if (!req)
 208                 return NULL;
 209
 210         return &req->cmd;
 211 }
 212
 213 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
 214 {
 215         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 216
 217         kfree(req);
 218 }
 219
 220 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
 221 {
 222         return req->req_state;
 223 }
 224
 225 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
 226                                        enum genwqe_requ_state new_state)
 227 {
 228         req->req_state = new_state;
 229 }
 230
 231 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
 232 {
 233         return req->cmd.ddata_addr != 0x0;
 234 }
 235
 236 /**
 237  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
 238  * @cd:          pointer to genwqe device descriptor
 239  * @req:         DDCB work request
 240  *
 241  * Status of ddcb_requ mirrors this hardware state, but is copied in
 242  * the ddcb_requ on interrupt/polling function. The lowlevel code
 243  * should check the hardware state directly, the higher level code
 244  * should check the copy.
 245  *
 246  * This function will also return true if the state of the queue is
 247  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
 248  * shutdown case.
 249  */
 250 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
 251 {
 252         return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
 253                 (cd->card_state != GENWQE_CARD_USED);
 254 }
 255
 256 /**
 257  * enqueue_ddcb() - Enqueue a DDCB
 258  * @cd:         pointer to genwqe device descriptor
 259  * @queue:      queue this operation should be done on
 260  * @ddcb_no:    pointer to ddcb number being tapped
 261  *
 262  * Start execution of DDCB by tapping or append to queue via NEXT
 263  * bit. This is done by an atomic 'compare and swap' instruction and
 264  * checking SHI and HSI of the previous DDCB.
 265  *
 266  * This function must only be called with ddcb_lock held.
 267  *
 268  * Return: 1 if new DDCB is appended to previous
 269  *         2 if DDCB queue is tapped via register/simulation
 270  */
 271 #define RET_DDCB_APPENDED 1
 272 #define RET_DDCB_TAPPED   2
 273
 274 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
 275                         struct ddcb *pddcb, int ddcb_no)
 276 {
 277         unsigned int try;
 278         int prev_no;
 279         struct ddcb *prev_ddcb;
 280         __be32 old, new, icrc_hsi_shi;
 281         u64 num;
 282
 283         /*
 284          * For performance checks a Dispatch Timestamp can be put into
 285          * DDCB It is supposed to use the SLU's free running counter,
 286          * but this requires PCIe cycles.
 287          */
 288         ddcb_mark_unused(pddcb);
 289
 290         /* check previous DDCB if already fetched */
 291         prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
 292         prev_ddcb = &queue->ddcb_vaddr[prev_no];
 293
 294         /*
 295          * It might have happened that the HSI.FETCHED bit is
 296          * set. Retry in this case. Therefore I expect maximum 2 times
 297          * trying.
 298          */
 299         ddcb_mark_appended(pddcb);
 300         for (try = 0; try < 2; try++) {
 301                 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
 302
 303                 /* try to append via NEXT bit if prev DDCB is not completed */
 304                 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
 305                         break;
 306
 307                 new = (old | DDCB_NEXT_BE32);
 308
 309                 wmb();          /* need to ensure write ordering */
 310                 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
 311
 312                 if (icrc_hsi_shi == old)
 313                         return RET_DDCB_APPENDED; /* appended to queue */
 314         }
 315
 316         /* Queue must be re-started by updating QUEUE_OFFSET */
 317         ddcb_mark_tapped(pddcb);
 318         num = (u64)ddcb_no << 8;
 319
 320         wmb();                  /* need to ensure write ordering */
 321         __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
 322
 323         return RET_DDCB_TAPPED;
 324 }
 325
 326 /**
 327  * copy_ddcb_results() - Copy output state from real DDCB to request
 328  *
 329  * Copy DDCB ASV to request struct. There is no endian
 330  * conversion made, since data structure in ASV is still
 331  * unknown here.
 332  *
 333  * This is needed by:
 334  *   - genwqe_purge_ddcb()
 335  *   - genwqe_check_ddcb_queue()
 336  */
 337 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
 338 {
 339         struct ddcb_queue *queue = req->queue;
 340         struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
 341
 342         memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
 343
 344         /* copy status flags of the variant part */
 345         req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
 346         req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
 347         req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
 348
 349         req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
 350         req->cmd.progress = be32_to_cpu(pddcb->progress_32);
 351         req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
 352
 353         if (ddcb_requ_collect_debug_data(req)) {
 354                 int prev_no = (ddcb_no == 0) ?
 355                         queue->ddcb_max - 1 : ddcb_no - 1;
 356                 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
 357
 358                 memcpy(&req->debug_data.ddcb_finished, pddcb,
 359                        sizeof(req->debug_data.ddcb_finished));
 360                 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
 361                        sizeof(req->debug_data.ddcb_prev));
 362         }
 363 }
 364
 365 /**
 366  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
 367  * @cd:         pointer to genwqe device descriptor
 368  *
 369  * Return: Number of DDCBs which were finished
 370  */
 371 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
 372                                    struct ddcb_queue *queue)
 373 {
 374         unsigned long flags;
 375         int ddcbs_finished = 0;
 376         struct pci_dev *pci_dev = cd->pci_dev;
 377
 378         spin_lock_irqsave(&queue->ddcb_lock, flags);
 379
 380         /* FIXME avoid soft locking CPU */
 381         while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
 382
 383                 struct ddcb *pddcb;
 384                 struct ddcb_requ *req;
 385                 u16 vcrc, vcrc_16, retc_16;
 386
 387                 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 388
 389                 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
 390                     0x00000000)
 391                         goto go_home; /* not completed, continue waiting */
 392
 393                 wmb();  /*  Add sync to decouple prev. read operations */
 394
 395                 /* Note: DDCB could be purged */
 396                 req = queue->ddcb_req[queue->ddcb_act];
 397                 if (req == NULL) {
 398                         /* this occurs if DDCB is purged, not an error */
 399                         /* Move active DDCB further; Nothing to do anymore. */
 400                         goto pick_next_one;
 401                 }
 402
 403                 /*
 404                  * HSI=0x44 (fetched and completed), but RETC is
 405                  * 0x101, or even worse 0x000.
 406                  *
 407                  * In case of seeing the queue in inconsistent state
 408                  * we read the errcnts and the queue status to provide
 409                  * a trigger for our PCIe analyzer stop capturing.
 410                  */
 411                 retc_16 = be16_to_cpu(pddcb->retc_16);
 412                 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
 413                         u64 errcnts, status;
 414                         u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
 415
 416                         errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
 417                         status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 418
 419                         dev_err(&pci_dev->dev,
 420                                 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
 421                                 __func__, be16_to_cpu(pddcb->seqnum_16),
 422                                 pddcb->hsi, retc_16, errcnts, status,
 423                                 queue->ddcb_daddr + ddcb_offs);
 424                 }
 425
 426                 copy_ddcb_results(req, queue->ddcb_act);
 427                 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
 428
 429                 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
 430                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 431
 432                 ddcb_mark_finished(pddcb);
 433
 434                 /* calculate CRC_16 to see if VCRC is correct */
 435                 vcrc = genwqe_crc16(pddcb->asv,
 436                                    VCRC_LENGTH(req->cmd.asv_length),
 437                                    0xffff);
 438                 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
 439                 if (vcrc != vcrc_16) {
 440                         printk_ratelimited(KERN_ERR
 441                                 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
 442                                 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 443                                 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
 444                                 vcrc, vcrc_16);
 445                 }
 446
 447                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 448                 queue->ddcbs_completed++;
 449                 queue->ddcbs_in_flight--;
 450
 451                 /* wake up process waiting for this DDCB, and
 452                    processes on the busy queue */
 453                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
 454                 wake_up_interruptible(&queue->busy_waitq);
 455
 456 pick_next_one:
 457                 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
 458                 ddcbs_finished++;
 459         }
 460
 461  go_home:
 462         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 463         return ddcbs_finished;
 464 }
 465
 466 /**
 467  * __genwqe_wait_ddcb(): Waits until DDCB is completed
 468  * @cd:         pointer to genwqe device descriptor
 469  * @req:        pointer to requsted DDCB parameters
 470  *
 471  * The Service Layer will update the RETC in DDCB when processing is
 472  * pending or done.
 473  *
 474  * Return: > 0 remaining jiffies, DDCB completed
 475  *           -ETIMEDOUT when timeout
 476  *           -ERESTARTSYS when ^C
 477  *           -EINVAL when unknown error condition
 478  *
 479  * When an error is returned the called needs to ensure that
 480  * purge_ddcb() is being called to get the &req removed from the
 481  * queue.
 482  */
 483 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 484 {
 485         int rc;
 486         unsigned int ddcb_no;
 487         struct ddcb_queue *queue;
 488         struct pci_dev *pci_dev = cd->pci_dev;
 489
 490         if (req == NULL)
 491                 return -EINVAL;
 492
 493         queue = req->queue;
 494         if (queue == NULL)
 495                 return -EINVAL;
 496
 497         ddcb_no = req->num;
 498         if (ddcb_no >= queue->ddcb_max)
 499                 return -EINVAL;
 500
 501         rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
 502                                 ddcb_requ_finished(cd, req),
 503                                 genwqe_ddcb_software_timeout * HZ);
 504
 505         /*
 506          * We need to distinguish 3 cases here:
 507          *   1. rc == 0              timeout occured
 508          *   2. rc == -ERESTARTSYS   signal received
 509          *   3. rc > 0               remaining jiffies condition is true
 510          */
 511         if (rc == 0) {
 512                 struct ddcb_queue *queue = req->queue;
 513                 struct ddcb *pddcb;
 514
 515                 /*
 516                  * Timeout may be caused by long task switching time.
 517                  * When timeout happens, check if the request has
 518                  * meanwhile completed.
 519                  */
 520                 genwqe_check_ddcb_queue(cd, req->queue);
 521                 if (ddcb_requ_finished(cd, req))
 522                         return rc;
 523
 524                 dev_err(&pci_dev->dev,
 525                         "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
 526                         __func__, req->num, rc, ddcb_requ_get_state(req),
 527                         req);
 528                 dev_err(&pci_dev->dev,
 529                         "[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
 530                         __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
 531
 532                 pddcb = &queue->ddcb_vaddr[req->num];
 533                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 534
 535                 print_ddcb_info(cd, req->queue);
 536                 return -ETIMEDOUT;
 537
 538         } else if (rc == -ERESTARTSYS) {
 539                 return rc;
 540                 /*
 541                  * EINTR:       Stops the application
 542                  * ERESTARTSYS: Restartable systemcall; called again
 543                  */
 544
 545         } else if (rc < 0) {
 546                 dev_err(&pci_dev->dev,
 547                         "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
 548                         __func__, req->num, rc, ddcb_requ_get_state(req));
 549                 return -EINVAL;
 550         }
 551
 552         /* Severe error occured. Driver is forced to stop operation */
 553         if (cd->card_state != GENWQE_CARD_USED) {
 554                 dev_err(&pci_dev->dev,
 555                         "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
 556                         __func__, req->num, rc);
 557                 return -EIO;
 558         }
 559         return rc;
 560 }
 561
 562 /**
 563  * get_next_ddcb() - Get next available DDCB
 564  * @cd:         pointer to genwqe device descriptor
 565  *
 566  * DDCB's content is completely cleared but presets for PRE and
 567  * SEQNUM. This function must only be called when ddcb_lock is held.
 568  *
 569  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
 570  */
 571 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
 572                                   struct ddcb_queue *queue,
 573                                   int *num)
 574 {
 575         u64 *pu64;
 576         struct ddcb *pddcb;
 577
 578         if (queue_free_ddcbs(queue) == 0) /* queue is  full */
 579                 return NULL;
 580
 581         /* find new ddcb */
 582         pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
 583
 584         /* if it is not completed, we are not allowed to use it */
 585         /* barrier(); */
 586         if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
 587                 return NULL;
 588
 589         *num = queue->ddcb_next;        /* internal DDCB number */
 590         queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
 591
 592         /* clear important DDCB fields */
 593         pu64 = (u64 *)pddcb;
 594         pu64[0] = 0ULL;         /* offs 0x00 (ICRC,HSI,SHI,...) */
 595         pu64[1] = 0ULL;         /* offs 0x01 (ACFUNC,CMD...) */
 596
 597         /* destroy previous results in ASV */
 598         pu64[0x80/8] = 0ULL;    /* offs 0x80 (ASV + 0) */
 599         pu64[0x88/8] = 0ULL;    /* offs 0x88 (ASV + 0x08) */
 600         pu64[0x90/8] = 0ULL;    /* offs 0x90 (ASV + 0x10) */
 601         pu64[0x98/8] = 0ULL;    /* offs 0x98 (ASV + 0x18) */
 602         pu64[0xd0/8] = 0ULL;    /* offs 0xd0 (RETC,ATTN...) */
 603
 604         pddcb->pre = DDCB_PRESET_PRE; /* 128 */
 605         pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
 606         return pddcb;
 607 }
 608
 609 /**
 610  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
 611  * @cd:         genwqe device descriptor
 612  * @req:        DDCB request
 613  *
 614  * This will fail when the request was already FETCHED. In this case
 615  * we need to wait until it is finished. Else the DDCB can be
 616  * reused. This function also ensures that the request data structure
 617  * is removed from ddcb_req[].
 618  *
 619  * Do not forget to call this function when genwqe_wait_ddcb() fails,
 620  * such that the request gets really removed from ddcb_req[].
 621  *
 622  * Return: 0 success
 623  */
 624 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 625 {
 626         struct ddcb *pddcb = NULL;
 627         unsigned int t;
 628         unsigned long flags;
 629         struct ddcb_queue *queue = req->queue;
 630         struct pci_dev *pci_dev = cd->pci_dev;
 631         u64 queue_status;
 632         __be32 icrc_hsi_shi = 0x0000;
 633         __be32 old, new;
 634
 635         /* unsigned long flags; */
 636         if (genwqe_ddcb_software_timeout <= 0) {
 637                 dev_err(&pci_dev->dev,
 638                         "[%s] err: software timeout is not set!\n", __func__);
 639                 return -EFAULT;
 640         }
 641
 642         pddcb = &queue->ddcb_vaddr[req->num];
 643
 644         for (t = 0; t < genwqe_ddcb_software_timeout * 10; t++) {
 645
 646                 spin_lock_irqsave(&queue->ddcb_lock, flags);
 647
 648                 /* Check if req was meanwhile finished */
 649                 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
 650                         goto go_home;
 651
 652                 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
 653                 old = pddcb->icrc_hsi_shi_32;   /* read SHI/HSI in BE32 */
 654                 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
 655
 656                         new = (old | DDCB_PURGE_BE32);
 657                         icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
 658                                                old, new);
 659                         if (icrc_hsi_shi == old)
 660                                 goto finish_ddcb;
 661                 }
 662
 663                 /* normal finish with HSI bit */
 664                 barrier();
 665                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 666                 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
 667                         goto finish_ddcb;
 668
 669                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 670
 671                 /*
 672                  * Here the check_ddcb() function will most likely
 673                  * discover this DDCB to be finished some point in
 674                  * time. It will mark the req finished and free it up
 675                  * in the list.
 676                  */
 677
 678                 copy_ddcb_results(req, req->num); /* for the failing case */
 679                 msleep(100); /* sleep for 1/10 second and try again */
 680                 continue;
 681
 682 finish_ddcb:
 683                 copy_ddcb_results(req, req->num);
 684                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 685                 queue->ddcbs_in_flight--;
 686                 queue->ddcb_req[req->num] = NULL; /* delete from array */
 687                 ddcb_mark_cleared(pddcb);
 688
 689                 /* Move active DDCB further; Nothing to do here anymore. */
 690
 691                 /*
 692                  * We need to ensure that there is at least one free
 693                  * DDCB in the queue. To do that, we must update
 694                  * ddcb_act only if the COMPLETED bit is set for the
 695                  * DDCB we are working on else we treat that DDCB even
 696                  * if we PURGED it as occupied (hardware is supposed
 697                  * to set the COMPLETED bit yet!).
 698                  */
 699                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 700                 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
 701                     (queue->ddcb_act == req->num)) {
 702                         queue->ddcb_act = ((queue->ddcb_act + 1) %
 703                                            queue->ddcb_max);
 704                 }
 705 go_home:
 706                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 707                 return 0;
 708         }
 709
 710         /*
 711          * If the card is dead and the queue is forced to stop, we
 712          * might see this in the queue status register.
 713          */
 714         queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 715
 716         dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
 717         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 718
 719         dev_err(&pci_dev->dev,
 720                 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
 721                 __func__, req->num, genwqe_ddcb_software_timeout,
 722                 queue_status);
 723
 724         print_ddcb_info(cd, req->queue);
 725
 726         return -EFAULT;
 727 }
 728
 729 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
 730 {
 731         int len;
 732         struct pci_dev *pci_dev = cd->pci_dev;
 733
 734         if (d == NULL) {
 735                 dev_err(&pci_dev->dev,
 736                         "[%s] err: invalid memory for debug data!\n",
 737                         __func__);
 738                 return -EFAULT;
 739         }
 740
 741         len  = sizeof(d->driver_version);
 742         snprintf(d->driver_version, len, "%s", DRV_VERSION);
 743         d->slu_unitcfg = cd->slu_unitcfg;
 744         d->app_unitcfg = cd->app_unitcfg;
 745         return 0;
 746 }
 747
 748 /**
 749  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
 750  * @cd:         pointer to genwqe device descriptor
 751  * @req:        pointer to DDCB execution request
 752  * @f_flags:    file mode: blocking, non-blocking
 753  *
 754  * Return: 0 if enqueuing succeeded
 755  *         -EIO if card is unusable/PCIe problems
 756  *         -EBUSY if enqueuing failed
 757  */
 758 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 759                           unsigned int f_flags)
 760 {
 761         struct ddcb *pddcb;
 762         unsigned long flags;
 763         struct ddcb_queue *queue;
 764         struct pci_dev *pci_dev = cd->pci_dev;
 765         u16 icrc;
 766
 767  retry:
 768         if (cd->card_state != GENWQE_CARD_USED) {
 769                 printk_ratelimited(KERN_ERR
 770                         "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
 771                         GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 772                         __func__, req->num);
 773                 return -EIO;
 774         }
 775
 776         queue = req->queue = &cd->queue;
 777
 778         /* FIXME circumvention to improve performance when no irq is
 779          * there.
 780          */
 781         if (genwqe_polling_enabled)
 782                 genwqe_check_ddcb_queue(cd, queue);
 783
 784         /*
 785          * It must be ensured to process all DDCBs in successive
 786          * order. Use a lock here in order to prevent nested DDCB
 787          * enqueuing.
 788          */
 789         spin_lock_irqsave(&queue->ddcb_lock, flags);
 790
 791         pddcb = get_next_ddcb(cd, queue, &req->num);    /* get ptr and num */
 792         if (pddcb == NULL) {
 793                 int rc;
 794
 795                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 796
 797                 if (f_flags & O_NONBLOCK) {
 798                         queue->return_on_busy++;
 799                         return -EBUSY;
 800                 }
 801
 802                 queue->wait_on_busy++;
 803                 rc = wait_event_interruptible(queue->busy_waitq,
 804                                               queue_free_ddcbs(queue) != 0);
 805                 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
 806                         __func__, rc);
 807                 if (rc == -ERESTARTSYS)
 808                         return rc;  /* interrupted by a signal */
 809
 810                 goto retry;
 811         }
 812
 813         if (queue->ddcb_req[req->num] != NULL) {
 814                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 815
 816                 dev_err(&pci_dev->dev,
 817                         "[%s] picked DDCB %d with req=%p still in use!!\n",
 818                         __func__, req->num, req);
 819                 return -EFAULT;
 820         }
 821         ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
 822         queue->ddcb_req[req->num] = req;
 823
 824         pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
 825         pddcb->cmd = req->cmd.cmd;
 826         pddcb->acfunc = req->cmd.acfunc;        /* functional unit */
 827
 828         /*
 829          * We know that we can get retc 0x104 with CRC error, do not
 830          * stop the queue in those cases for this command. XDIR = 1
 831          * does not work for old SLU versions.
 832          *
 833          * Last bitstream with the old XDIR behavior had SLU_ID
 834          * 0x34199.
 835          */
 836         if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
 837                 pddcb->xdir = 0x1;
 838         else
 839                 pddcb->xdir = 0x0;
 840
 841
 842         pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
 843                       ((req->cmd.asv_length  / 8)));
 844         pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
 845
 846         /*
 847          * If copying the whole DDCB_ASIV_LENGTH is impacting
 848          * performance we need to change it to
 849          * req->cmd.asiv_length. But simulation benefits from some
 850          * non-architectured bits behind the architectured content.
 851          *
 852          * How much data is copied depends on the availability of the
 853          * ATS field, which was introduced late. If the ATS field is
 854          * supported ASIV is 8 bytes shorter than it used to be. Since
 855          * the ATS field is copied too, the code should do exactly
 856          * what it did before, but I wanted to make copying of the ATS
 857          * field very explicit.
 858          */
 859         if (genwqe_get_slu_id(cd) <= 0x2) {
 860                 memcpy(&pddcb->__asiv[0],       /* destination */
 861                        &req->cmd.__asiv[0],     /* source */
 862                        DDCB_ASIV_LENGTH);       /* req->cmd.asiv_length */
 863         } else {
 864                 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
 865                 memcpy(&pddcb->n.asiv[0],       /* destination */
 866                         &req->cmd.asiv[0],      /* source */
 867                         DDCB_ASIV_LENGTH_ATS);  /* req->cmd.asiv_length */
 868         }
 869
 870         pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
 871
 872         /*
 873          * Calculate CRC_16 for corresponding range PSP(7:4). Include
 874          * empty 4 bytes prior to the data.
 875          */
 876         icrc = genwqe_crc16((const u8 *)pddcb,
 877                            ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
 878         pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
 879
 880         /* enable DDCB completion irq */
 881         if (!genwqe_polling_enabled)
 882                 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
 883
 884         dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
 885         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 886
 887         if (ddcb_requ_collect_debug_data(req)) {
 888                 /* use the kernel copy of debug data. copying back to
 889                    user buffer happens later */
 890
 891                 genwqe_init_debug_data(cd, &req->debug_data);
 892                 memcpy(&req->debug_data.ddcb_before, pddcb,
 893                        sizeof(req->debug_data.ddcb_before));
 894         }
 895
 896         enqueue_ddcb(cd, queue, pddcb, req->num);
 897         queue->ddcbs_in_flight++;
 898
 899         if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
 900                 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
 901
 902         ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
 903         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 904         wake_up_interruptible(&cd->queue_waitq);
 905
 906         return 0;
 907 }
 908
 909 /**
 910  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
 911  * @cd:         pointer to genwqe device descriptor
 912  * @req:        user provided DDCB request
 913  * @f_flags:    file mode: blocking, non-blocking
 914  */
 915 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
 916                               struct genwqe_ddcb_cmd *cmd,
 917                               unsigned int f_flags)
 918 {
 919         int rc = 0;
 920         struct pci_dev *pci_dev = cd->pci_dev;
 921         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 922
 923         if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
 924                 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
 925                         __func__, cmd->asiv_length);
 926                 return -EINVAL;
 927         }
 928         if (cmd->asv_length > DDCB_ASV_LENGTH) {
 929                 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
 930                         __func__, cmd->asiv_length);
 931                 return -EINVAL;
 932         }
 933         rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
 934         if (rc != 0)
 935                 return rc;
 936
 937         rc = __genwqe_wait_ddcb(cd, req);
 938         if (rc < 0)             /* error or signal interrupt */
 939                 goto err_exit;
 940
 941         if (ddcb_requ_collect_debug_data(req)) {
 942                 if (copy_to_user((struct genwqe_debug_data __user *)
 943                                  (unsigned long)cmd->ddata_addr,
 944                                  &req->debug_data,
 945                                  sizeof(struct genwqe_debug_data)))
 946                         return -EFAULT;
 947         }
 948
 949         /*
 950          * Higher values than 0x102 indicate completion with faults,
 951          * lower values than 0x102 indicate processing faults. Note
 952          * that DDCB might have been purged. E.g. Cntl+C.
 953          */
 954         if (cmd->retc != DDCB_RETC_COMPLETE) {
 955                 /* This might happen e.g. flash read, and needs to be
 956                    handled by the upper layer code. */
 957                 rc = -EBADMSG;  /* not processed/error retc */
 958         }
 959
 960         return rc;
 961
 962  err_exit:
 963         __genwqe_purge_ddcb(cd, req);
 964
 965         if (ddcb_requ_collect_debug_data(req)) {
 966                 if (copy_to_user((struct genwqe_debug_data __user *)
 967                                  (unsigned long)cmd->ddata_addr,
 968                                  &req->debug_data,
 969                                  sizeof(struct genwqe_debug_data)))
 970                         return -EFAULT;
 971         }
 972         return rc;
 973 }
 974
 975 /**
 976  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
 977  *
 978  * We use this as condition for our wait-queue code.
 979  */
 980 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
 981 {
 982         unsigned long flags;
 983         struct ddcb *pddcb;
 984         struct ddcb_queue *queue = &cd->queue;
 985
 986         spin_lock_irqsave(&queue->ddcb_lock, flags);
 987
 988         if (queue_empty(queue)) { /* emtpy queue */
 989                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 990                 return 0;
 991         }
 992
 993         pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 994         if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
 995                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 996                 return 1;
 997         }
 998
 999         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1000         return 0;
1001 }
1002
1003 /**
1004  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1005  *
1006  * Keep track on the number of DDCBs which ware currently in the
1007  * queue. This is needed for statistics as well as conditon if we want
1008  * to wait or better do polling in case of no interrupts available.
1009  */
1010 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1011 {
1012         unsigned long flags;
1013         int ddcbs_in_flight = 0;
1014         struct ddcb_queue *queue = &cd->queue;
1015
1016         spin_lock_irqsave(&queue->ddcb_lock, flags);
1017         ddcbs_in_flight += queue->ddcbs_in_flight;
1018         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1019
1020         return ddcbs_in_flight;
1021 }
1022
1023 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1024 {
1025         int rc, i;
1026         struct ddcb *pddcb;
1027         u64 val64;
1028         unsigned int queue_size;
1029         struct pci_dev *pci_dev = cd->pci_dev;
1030
1031         if (genwqe_ddcb_max < 2)
1032                 return -EINVAL;
1033
1034         queue_size = roundup(genwqe_ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1035
1036         queue->ddcbs_in_flight = 0;  /* statistics */
1037         queue->ddcbs_max_in_flight = 0;
1038         queue->ddcbs_completed = 0;
1039         queue->return_on_busy = 0;
1040         queue->wait_on_busy = 0;
1041
1042         queue->ddcb_seq   = 0x100; /* start sequence number */
1043         queue->ddcb_max   = genwqe_ddcb_max; /* module parameter */
1044         queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1045                                                 &queue->ddcb_daddr);
1046         if (queue->ddcb_vaddr == NULL) {
1047                 dev_err(&pci_dev->dev,
1048                         "[%s] **err: could not allocate DDCB **\n", __func__);
1049                 return -ENOMEM;
1050         }
1051         memset(queue->ddcb_vaddr, 0, queue_size);
1052
1053         queue->ddcb_req = kzalloc(sizeof(struct ddcb_requ *) *
1054                                   queue->ddcb_max, GFP_KERNEL);
1055         if (!queue->ddcb_req) {
1056                 rc = -ENOMEM;
1057                 goto free_ddcbs;
1058         }
1059
1060         queue->ddcb_waitqs = kzalloc(sizeof(wait_queue_head_t) *
1061                                      queue->ddcb_max, GFP_KERNEL);
1062         if (!queue->ddcb_waitqs) {
1063                 rc = -ENOMEM;
1064                 goto free_requs;
1065         }
1066
1067         for (i = 0; i < queue->ddcb_max; i++) {
1068                 pddcb = &queue->ddcb_vaddr[i];               /* DDCBs */
1069                 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1070                 pddcb->retc_16 = cpu_to_be16(0xfff);
1071
1072                 queue->ddcb_req[i] = NULL;                   /* requests */
1073                 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1074         }
1075
1076         queue->ddcb_act  = 0;
1077         queue->ddcb_next = 0;   /* queue is empty */
1078
1079         spin_lock_init(&queue->ddcb_lock);
1080         init_waitqueue_head(&queue->busy_waitq);
1081
1082         val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1083         __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1084         __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1085         __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1086         __genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1087         return 0;
1088
1089  free_requs:
1090         kfree(queue->ddcb_req);
1091         queue->ddcb_req = NULL;
1092  free_ddcbs:
1093         __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1094                                 queue->ddcb_daddr);
1095         queue->ddcb_vaddr = NULL;
1096         queue->ddcb_daddr = 0ull;
1097         return -ENODEV;
1098
1099 }
1100
1101 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1102 {
1103         return queue->ddcb_vaddr != NULL;
1104 }
1105
1106 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1107 {
1108         unsigned int queue_size;
1109
1110         queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1111
1112         kfree(queue->ddcb_req);
1113         queue->ddcb_req = NULL;
1114
1115         if (queue->ddcb_vaddr) {
1116                 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1117                                         queue->ddcb_daddr);
1118                 queue->ddcb_vaddr = NULL;
1119                 queue->ddcb_daddr = 0ull;
1120         }
1121 }
1122
1123 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1124 {
1125         u64 gfir;
1126         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1127         struct pci_dev *pci_dev = cd->pci_dev;
1128
1129         /*
1130          * In case of fatal FIR error the queue is stopped, such that
1131          * we can safely check it without risking anything.
1132          */
1133         cd->irqs_processed++;
1134         wake_up_interruptible(&cd->queue_waitq);
1135
1136         /*
1137          * Checking for errors before kicking the queue might be
1138          * safer, but slower for the good-case ... See above.
1139          */
1140         gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1141         if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1142             !pci_channel_offline(pci_dev)) {
1143
1144                 if (cd->use_platform_recovery) {
1145                         /*
1146                          * Since we use raw accessors, EEH errors won't be
1147                          * detected by the platform until we do a non-raw
1148                          * MMIO or config space read
1149                          */
1150                         readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1151
1152                         /* Don't do anything if the PCI channel is frozen */
1153                         if (pci_channel_offline(pci_dev))
1154                                 goto exit;
1155                 }
1156
1157                 wake_up_interruptible(&cd->health_waitq);
1158
1159                 /*
1160                  * By default GFIRs causes recovery actions. This
1161                  * count is just for debug when recovery is masked.
1162                  */
1163                 dev_err_ratelimited(&pci_dev->dev,
1164                                     "[%s] GFIR=%016llx\n",
1165                                     __func__, gfir);
1166         }
1167
1168  exit:
1169         return IRQ_HANDLED;
1170 }
1171
1172 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1173 {
1174         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1175
1176         cd->irqs_processed++;
1177         wake_up_interruptible(&cd->queue_waitq);
1178
1179         return IRQ_HANDLED;
1180 }
1181
1182 /**
1183  * genwqe_card_thread() - Work thread for the DDCB queue
1184  *
1185  * The idea is to check if there are DDCBs in processing. If there are
1186  * some finished DDCBs, we process them and wakeup the
1187  * requestors. Otherwise we give other processes time using
1188  * cond_resched().
1189  */
1190 static int genwqe_card_thread(void *data)
1191 {
1192         int should_stop = 0, rc = 0;
1193         struct genwqe_dev *cd = (struct genwqe_dev *)data;
1194
1195         while (!kthread_should_stop()) {
1196
1197                 genwqe_check_ddcb_queue(cd, &cd->queue);
1198
1199                 if (genwqe_polling_enabled) {
1200                         rc = wait_event_interruptible_timeout(
1201                                 cd->queue_waitq,
1202                                 genwqe_ddcbs_in_flight(cd) ||
1203                                 (should_stop = kthread_should_stop()), 1);
1204                 } else {
1205                         rc = wait_event_interruptible_timeout(
1206                                 cd->queue_waitq,
1207                                 genwqe_next_ddcb_ready(cd) ||
1208                                 (should_stop = kthread_should_stop()), HZ);
1209                 }
1210                 if (should_stop)
1211                         break;
1212
1213                 /*
1214                  * Avoid soft lockups on heavy loads; we do not want
1215                  * to disable our interrupts.
1216                  */
1217                 cond_resched();
1218         }
1219         return 0;
1220 }
1221
1222 /**
1223  * genwqe_setup_service_layer() - Setup DDCB queue
1224  * @cd:         pointer to genwqe device descriptor
1225  *
1226  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1227  *
1228  * Return: 0 success
1229  */
1230 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1231 {
1232         int rc;
1233         struct ddcb_queue *queue;
1234         struct pci_dev *pci_dev = cd->pci_dev;
1235
1236         if (genwqe_is_privileged(cd)) {
1237                 rc = genwqe_card_reset(cd);
1238                 if (rc < 0) {
1239                         dev_err(&pci_dev->dev,
1240                                 "[%s] err: reset failed.\n", __func__);
1241                         return rc;
1242                 }
1243                 genwqe_read_softreset(cd);
1244         }
1245
1246         queue = &cd->queue;
1247         queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1248         queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1249         queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1250         queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1251         queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1252         queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1253         queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1254         queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1255         queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1256
1257         rc = setup_ddcb_queue(cd, queue);
1258         if (rc != 0) {
1259                 rc = -ENODEV;
1260                 goto err_out;
1261         }
1262
1263         init_waitqueue_head(&cd->queue_waitq);
1264         cd->card_thread = kthread_run(genwqe_card_thread, cd,
1265                                       GENWQE_DEVNAME "%d_thread",
1266                                       cd->card_idx);
1267         if (IS_ERR(cd->card_thread)) {
1268                 rc = PTR_ERR(cd->card_thread);
1269                 cd->card_thread = NULL;
1270                 goto stop_free_queue;
1271         }
1272
1273         rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1274         if (rc)
1275                 goto stop_kthread;
1276
1277         /*
1278          * We must have all wait-queues initialized when we enable the
1279          * interrupts. Otherwise we might crash if we get an early
1280          * irq.
1281          */
1282         init_waitqueue_head(&cd->health_waitq);
1283
1284         if (genwqe_is_privileged(cd)) {
1285                 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1286                                  GENWQE_DEVNAME, cd);
1287         } else {
1288                 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1289                                  GENWQE_DEVNAME, cd);
1290         }
1291         if (rc < 0) {
1292                 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1293                 goto stop_irq_cap;
1294         }
1295
1296         cd->card_state = GENWQE_CARD_USED;
1297         return 0;
1298
1299  stop_irq_cap:
1300         genwqe_reset_interrupt_capability(cd);
1301  stop_kthread:
1302         kthread_stop(cd->card_thread);
1303         cd->card_thread = NULL;
1304  stop_free_queue:
1305         free_ddcb_queue(cd, queue);
1306  err_out:
1307         return rc;
1308 }
1309
1310 /**
1311  * queue_wake_up_all() - Handles fatal error case
1312  *
1313  * The PCI device got unusable and we have to stop all pending
1314  * requests as fast as we can. The code after this must purge the
1315  * DDCBs in question and ensure that all mappings are freed.
1316  */
1317 static int queue_wake_up_all(struct genwqe_dev *cd)
1318 {
1319         unsigned int i;
1320         unsigned long flags;
1321         struct ddcb_queue *queue = &cd->queue;
1322
1323         spin_lock_irqsave(&queue->ddcb_lock, flags);
1324
1325         for (i = 0; i < queue->ddcb_max; i++)
1326                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1327
1328         wake_up_interruptible(&queue->busy_waitq);
1329         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1330
1331         return 0;
1332 }
1333
1334 /**
1335  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1336  *
1337  * Relies on the pre-condition that there are no users of the card
1338  * device anymore e.g. with open file-descriptors.
1339  *
1340  * This function must be robust enough to be called twice.
1341  */
1342 int genwqe_finish_queue(struct genwqe_dev *cd)
1343 {
1344         int i, rc = 0, in_flight;
1345         int waitmax = genwqe_ddcb_software_timeout;
1346         struct pci_dev *pci_dev = cd->pci_dev;
1347         struct ddcb_queue *queue = &cd->queue;
1348
1349         if (!ddcb_queue_initialized(queue))
1350                 return 0;
1351
1352         /* Do not wipe out the error state. */
1353         if (cd->card_state == GENWQE_CARD_USED)
1354                 cd->card_state = GENWQE_CARD_UNUSED;
1355
1356         /* Wake up all requests in the DDCB queue such that they
1357            should be removed nicely. */
1358         queue_wake_up_all(cd);
1359
1360         /* We must wait to get rid of the DDCBs in flight */
1361         for (i = 0; i < waitmax; i++) {
1362                 in_flight = genwqe_ddcbs_in_flight(cd);
1363
1364                 if (in_flight == 0)
1365                         break;
1366
1367                 dev_dbg(&pci_dev->dev,
1368                         "  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1369                         i, waitmax, in_flight);
1370
1371                 /*
1372                  * Severe severe error situation: The card itself has
1373                  * 16 DDCB queues, each queue has e.g. 32 entries,
1374                  * each DDBC has a hardware timeout of currently 250
1375                  * msec but the PFs have a hardware timeout of 8 sec
1376                  * ... so I take something large.
1377                  */
1378                 msleep(1000);
1379         }
1380         if (i == waitmax) {
1381                 dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1382                         __func__);
1383                 rc = -EIO;
1384         }
1385         return rc;
1386 }
1387
1388 /**
1389  * genwqe_release_service_layer() - Shutdown DDCB queue
1390  * @cd:       genwqe device descriptor
1391  *
1392  * This function must be robust enough to be called twice.
1393  */
1394 int genwqe_release_service_layer(struct genwqe_dev *cd)
1395 {
1396         struct pci_dev *pci_dev = cd->pci_dev;
1397
1398         if (!ddcb_queue_initialized(&cd->queue))
1399                 return 1;
1400
1401         free_irq(pci_dev->irq, cd);
1402         genwqe_reset_interrupt_capability(cd);
1403
1404         if (cd->card_thread != NULL) {
1405                 kthread_stop(cd->card_thread);
1406                 cd->card_thread = NULL;
1407         }
1408
1409         free_ddcb_queue(cd, &cd->queue);
1410         return 0;
1411 }