drivers/block/skd_main.c

   1 /* Copyright 2012 STEC, Inc.
   2  *
   3  * This file is licensed under the terms of the 3-clause
   4  * BSD License (http://opensource.org/licenses/BSD-3-Clause)
   5  * or the GNU GPL-2.0 (http://www.gnu.org/licenses/gpl-2.0.html),
   6  * at your option. Both licenses are also available in the LICENSE file
   7  * distributed with this project. This file may not be copied, modified,
   8  * or distributed except in accordance with those terms.
   9  * Gordoni Waidhofer <gwaidhofer@stec-inc.com>
  10  * Initial Driver Design!
  11  * Thomas Swann <tswann@stec-inc.com>
  12  * Interrupt handling.
  13  * Ramprasad Chinthekindi <rchinthekindi@stec-inc.com>
  14  * biomode implementation.
  15  * Akhil Bhansali <abhansali@stec-inc.com>
  16  * Added support for DISCARD / FLUSH and FUA.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/init.h>
  22 #include <linux/pci.h>
  23 #include <linux/slab.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/blkdev.h>
  26 #include <linux/sched.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/compiler.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/bitops.h>
  31 #include <linux/delay.h>
  32 #include <linux/time.h>
  33 #include <linux/hdreg.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/completion.h>
  36 #include <linux/scatterlist.h>
  37 #include <linux/version.h>
  38 #include <linux/err.h>
  39 #include <linux/scatterlist.h>
  40 #include <linux/aer.h>
  41 #include <linux/ctype.h>
  42 #include <linux/wait.h>
  43 #include <linux/uio.h>
  44 #include <scsi/scsi.h>
  45 #include <scsi/scsi_host.h>
  46 #include <scsi/scsi_tcq.h>
  47 #include <scsi/scsi_cmnd.h>
  48 #include <scsi/sg.h>
  49 #include <linux/io.h>
  50 #include <linux/uaccess.h>
  51 #include <asm-generic/unaligned.h>
  52
  53 #include "skd_s1120.h"
  54
  55 static int skd_dbg_level;
  56 static int skd_isr_comp_limit = 4;
  57
  58 enum {
  59         STEC_LINK_2_5GTS = 0,
  60         STEC_LINK_5GTS = 1,
  61         STEC_LINK_8GTS = 2,
  62         STEC_LINK_UNKNOWN = 0xFF
  63 };
  64
  65 enum {
  66         SKD_FLUSH_INITIALIZER,
  67         SKD_FLUSH_ZERO_SIZE_FIRST,
  68         SKD_FLUSH_DATA_SECOND,
  69 };
  70
  71 #define DPRINTK(skdev, fmt, args ...) \
  72         do { \
  73                 if (unlikely((skdev)->dbg_level > 0)) { \
  74                         pr_err("%s:%s:%d " fmt, (skdev)->name,  \
  75                                __func__, __LINE__, ## args); \
  76                 } \
  77         } while (0)
  78
  79 #define SKD_ASSERT(expr) \
  80         do { \
  81                 if (unlikely(!(expr))) { \
  82                         pr_err("Assertion failed! %s,%s,%s,line=%d\n",  \
  83                                # expr, __FILE__, __func__, __LINE__); \
  84                 } \
  85         } while (0)
  86
  87 #define VPRINTK(skdev, fmt, args ...) \
  88         do { \
  89                 if (unlikely((skdev)->dbg_level > 1)) { \
  90                         pr_err("%s:%s:%d " fmt, (skdev)->name,  \
  91                                __func__, __LINE__, ## args); \
  92                 } \
  93         } while (0)
  94
  95
  96 #define DRV_NAME "skd"
  97 #define DRV_VERSION "2.2.1"
  98 #define DRV_BUILD_ID "0260"
  99 #define PFX DRV_NAME ": "
 100 #define DRV_BIN_VERSION 0x100
 101 #define DRV_VER_COMPL   "2.2.1." DRV_BUILD_ID
 102
 103 MODULE_AUTHOR("bug-reports: support@stec-inc.com");
 104 MODULE_LICENSE("Dual BSD/GPL");
 105
 106 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block/BIO driver (b" DRV_BUILD_ID ")");
 107 MODULE_VERSION(DRV_VERSION "-" DRV_BUILD_ID);
 108
 109 #define PCI_VENDOR_ID_STEC      0x1B39
 110 #define PCI_DEVICE_ID_S1120     0x0001
 111
 112 #define SKD_FUA_NV              (1 << 1)
 113 #define SKD_MINORS_PER_DEVICE   16
 114
 115 #define SKD_MAX_QUEUE_DEPTH     200u
 116
 117 #define SKD_PAUSE_TIMEOUT       (5 * 1000)
 118
 119 #define SKD_N_FITMSG_BYTES      (512u)
 120
 121 #define SKD_N_SPECIAL_CONTEXT   32u
 122 #define SKD_N_SPECIAL_FITMSG_BYTES      (128u)
 123
 124 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
 125  * 128KB limit.  That allows 4096*4K = 16M xfer size
 126  */
 127 #define SKD_N_SG_PER_REQ_DEFAULT 256u
 128 #define SKD_N_SG_PER_SPECIAL    256u
 129
 130 #define SKD_N_COMPLETION_ENTRY  256u
 131 #define SKD_N_READ_CAP_BYTES    (8u)
 132
 133 #define SKD_N_INTERNAL_BYTES    (512u)
 134
 135 /* 5 bits of uniqifier, 0xF800 */
 136 #define SKD_ID_INCR             (0x400)
 137 #define SKD_ID_TABLE_MASK       (3u << 8u)
 138 #define  SKD_ID_RW_REQUEST      (0u << 8u)
 139 #define  SKD_ID_INTERNAL        (1u << 8u)
 140 #define  SKD_ID_SPECIAL_REQUEST (2u << 8u)
 141 #define  SKD_ID_FIT_MSG         (3u << 8u)
 142 #define SKD_ID_SLOT_MASK        0x00FFu
 143 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
 144
 145 #define SKD_N_TIMEOUT_SLOT      4u
 146 #define SKD_TIMEOUT_SLOT_MASK   3u
 147
 148 #define SKD_N_MAX_SECTORS 2048u
 149
 150 #define SKD_MAX_RETRIES 2u
 151
 152 #define SKD_TIMER_SECONDS(seconds) (seconds)
 153 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
 154
 155 #define INQ_STD_NBYTES 36
 156 #define SKD_DISCARD_CDB_LENGTH  24
 157
 158 enum skd_drvr_state {
 159         SKD_DRVR_STATE_LOAD,
 160         SKD_DRVR_STATE_IDLE,
 161         SKD_DRVR_STATE_BUSY,
 162         SKD_DRVR_STATE_STARTING,
 163         SKD_DRVR_STATE_ONLINE,
 164         SKD_DRVR_STATE_PAUSING,
 165         SKD_DRVR_STATE_PAUSED,
 166         SKD_DRVR_STATE_DRAINING_TIMEOUT,
 167         SKD_DRVR_STATE_RESTARTING,
 168         SKD_DRVR_STATE_RESUMING,
 169         SKD_DRVR_STATE_STOPPING,
 170         SKD_DRVR_STATE_FAULT,
 171         SKD_DRVR_STATE_DISAPPEARED,
 172         SKD_DRVR_STATE_PROTOCOL_MISMATCH,
 173         SKD_DRVR_STATE_BUSY_ERASE,
 174         SKD_DRVR_STATE_BUSY_SANITIZE,
 175         SKD_DRVR_STATE_BUSY_IMMINENT,
 176         SKD_DRVR_STATE_WAIT_BOOT,
 177         SKD_DRVR_STATE_SYNCING,
 178 };
 179
 180 #define SKD_WAIT_BOOT_TIMO      SKD_TIMER_SECONDS(90u)
 181 #define SKD_STARTING_TIMO       SKD_TIMER_SECONDS(8u)
 182 #define SKD_RESTARTING_TIMO     SKD_TIMER_MINUTES(4u)
 183 #define SKD_DRAINING_TIMO       SKD_TIMER_SECONDS(6u)
 184 #define SKD_BUSY_TIMO           SKD_TIMER_MINUTES(20u)
 185 #define SKD_STARTED_BUSY_TIMO   SKD_TIMER_SECONDS(60u)
 186 #define SKD_START_WAIT_SECONDS  90u
 187
 188 enum skd_req_state {
 189         SKD_REQ_STATE_IDLE,
 190         SKD_REQ_STATE_SETUP,
 191         SKD_REQ_STATE_BUSY,
 192         SKD_REQ_STATE_COMPLETED,
 193         SKD_REQ_STATE_TIMEOUT,
 194         SKD_REQ_STATE_ABORTED,
 195 };
 196
 197 enum skd_fit_msg_state {
 198         SKD_MSG_STATE_IDLE,
 199         SKD_MSG_STATE_BUSY,
 200 };
 201
 202 enum skd_check_status_action {
 203         SKD_CHECK_STATUS_REPORT_GOOD,
 204         SKD_CHECK_STATUS_REPORT_SMART_ALERT,
 205         SKD_CHECK_STATUS_REQUEUE_REQUEST,
 206         SKD_CHECK_STATUS_REPORT_ERROR,
 207         SKD_CHECK_STATUS_BUSY_IMMINENT,
 208 };
 209
 210 struct skd_fitmsg_context {
 211         enum skd_fit_msg_state state;
 212
 213         struct skd_fitmsg_context *next;
 214
 215         u32 id;
 216         u16 outstanding;
 217
 218         u32 length;
 219         u32 offset;
 220
 221         u8 *msg_buf;
 222         dma_addr_t mb_dma_address;
 223 };
 224
 225 struct skd_request_context {
 226         enum skd_req_state state;
 227
 228         struct skd_request_context *next;
 229
 230         u16 id;
 231         u32 fitmsg_id;
 232
 233         struct request *req;
 234         struct bio *bio;
 235         unsigned long start_time;
 236         u8 flush_cmd;
 237         u8 discard_page;
 238
 239         u32 timeout_stamp;
 240         u8 sg_data_dir;
 241         struct scatterlist *sg;
 242         u32 n_sg;
 243         u32 sg_byte_count;
 244
 245         struct fit_sg_descriptor *sksg_list;
 246         dma_addr_t sksg_dma_address;
 247
 248         struct fit_completion_entry_v1 completion;
 249
 250         struct fit_comp_error_info err_info;
 251
 252 };
 253 #define SKD_DATA_DIR_HOST_TO_CARD       1
 254 #define SKD_DATA_DIR_CARD_TO_HOST       2
 255 #define SKD_DATA_DIR_NONE               3       /* especially for DISCARD requests. */
 256
 257 struct skd_special_context {
 258         struct skd_request_context req;
 259
 260         u8 orphaned;
 261
 262         void *data_buf;
 263         dma_addr_t db_dma_address;
 264
 265         u8 *msg_buf;
 266         dma_addr_t mb_dma_address;
 267 };
 268
 269 struct skd_sg_io {
 270         fmode_t mode;
 271         void __user *argp;
 272
 273         struct sg_io_hdr sg;
 274
 275         u8 cdb[16];
 276
 277         u32 dxfer_len;
 278         u32 iovcnt;
 279         struct sg_iovec *iov;
 280         struct sg_iovec no_iov_iov;
 281
 282         struct skd_special_context *skspcl;
 283 };
 284
 285 typedef enum skd_irq_type {
 286         SKD_IRQ_LEGACY,
 287         SKD_IRQ_MSI,
 288         SKD_IRQ_MSIX
 289 } skd_irq_type_t;
 290
 291 #define SKD_MAX_BARS                    2
 292
 293 struct skd_device {
 294         volatile void __iomem *mem_map[SKD_MAX_BARS];
 295         resource_size_t mem_phys[SKD_MAX_BARS];
 296         u32 mem_size[SKD_MAX_BARS];
 297
 298         skd_irq_type_t irq_type;
 299         u32 msix_count;
 300         struct skd_msix_entry *msix_entries;
 301
 302         struct pci_dev *pdev;
 303         int pcie_error_reporting_is_enabled;
 304
 305         spinlock_t lock;
 306         struct gendisk *disk;
 307         struct request_queue *queue;
 308         struct device *class_dev;
 309         int gendisk_on;
 310         int sync_done;
 311
 312         atomic_t device_count;
 313         u32 devno;
 314         u32 major;
 315         char name[32];
 316         char isr_name[30];
 317
 318         enum skd_drvr_state state;
 319         u32 drive_state;
 320
 321         u32 in_flight;
 322         u32 cur_max_queue_depth;
 323         u32 queue_low_water_mark;
 324         u32 dev_max_queue_depth;
 325
 326         u32 num_fitmsg_context;
 327         u32 num_req_context;
 328
 329         u32 timeout_slot[SKD_N_TIMEOUT_SLOT];
 330         u32 timeout_stamp;
 331         struct skd_fitmsg_context *skmsg_free_list;
 332         struct skd_fitmsg_context *skmsg_table;
 333
 334         struct skd_request_context *skreq_free_list;
 335         struct skd_request_context *skreq_table;
 336
 337         struct skd_special_context *skspcl_free_list;
 338         struct skd_special_context *skspcl_table;
 339
 340         struct skd_special_context internal_skspcl;
 341         u32 read_cap_blocksize;
 342         u32 read_cap_last_lba;
 343         int read_cap_is_valid;
 344         int inquiry_is_valid;
 345         u8 inq_serial_num[13];  /*12 chars plus null term */
 346         u8 id_str[80];          /* holds a composite name (pci + sernum) */
 347
 348         u8 skcomp_cycle;
 349         u32 skcomp_ix;
 350         struct fit_completion_entry_v1 *skcomp_table;
 351         struct fit_comp_error_info *skerr_table;
 352         dma_addr_t cq_dma_address;
 353
 354         wait_queue_head_t waitq;
 355
 356         struct timer_list timer;
 357         u32 timer_countdown;
 358         u32 timer_substate;
 359
 360         int n_special;
 361         int sgs_per_request;
 362         u32 last_mtd;
 363
 364         u32 proto_ver;
 365
 366         int dbg_level;
 367         u32 connect_time_stamp;
 368         int connect_retries;
 369 #define SKD_MAX_CONNECT_RETRIES 16
 370         u32 drive_jiffies;
 371
 372         u32 timo_slot;
 373
 374
 375         struct work_struct completion_worker;
 376
 377         struct bio_list bio_queue;
 378         int queue_stopped;
 379
 380         struct list_head flush_list;
 381 };
 382
 383 #define SKD_FLUSH_JOB   "skd-flush-jobs"
 384 struct kmem_cache *skd_flush_slab;
 385
 386 /*
 387  * These commands hold "nonzero size FLUSH bios",
 388  * which are enqueud in skdev->flush_list during
 389  * completion of "zero size FLUSH commands".
 390  * It will be active in biomode.
 391  */
 392 struct skd_flush_cmd {
 393         void *cmd;
 394         struct list_head flist;
 395 };
 396
 397 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
 398 #define SKD_READL(DEV, OFF)      skd_reg_read32(DEV, OFF)
 399 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
 400
 401 static inline u32 skd_reg_read32(struct skd_device *skdev, u32 offset)
 402 {
 403         u32 val;
 404
 405         if (likely(skdev->dbg_level < 2))
 406                 return readl(skdev->mem_map[1] + offset);
 407         else {
 408                 barrier();
 409                 val = readl(skdev->mem_map[1] + offset);
 410                 barrier();
 411                 VPRINTK(skdev, "offset %x = %x\n", offset, val);
 412                 return val;
 413         }
 414
 415 }
 416
 417 static inline void skd_reg_write32(struct skd_device *skdev, u32 val,
 418                                    u32 offset)
 419 {
 420         if (likely(skdev->dbg_level < 2)) {
 421                 writel(val, skdev->mem_map[1] + offset);
 422                 barrier();
 423                 readl(skdev->mem_map[1] + offset);
 424                 barrier();
 425         } else {
 426                 barrier();
 427                 writel(val, skdev->mem_map[1] + offset);
 428                 barrier();
 429                 readl(skdev->mem_map[1] + offset);
 430                 barrier();
 431                 VPRINTK(skdev, "offset %x = %x\n", offset, val);
 432         }
 433 }
 434
 435 static inline void skd_reg_write64(struct skd_device *skdev, u64 val,
 436                                    u32 offset)
 437 {
 438         if (likely(skdev->dbg_level < 2)) {
 439                 writeq(val, skdev->mem_map[1] + offset);
 440                 barrier();
 441                 readq(skdev->mem_map[1] + offset);
 442                 barrier();
 443         } else {
 444                 barrier();
 445                 writeq(val, skdev->mem_map[1] + offset);
 446                 barrier();
 447                 readq(skdev->mem_map[1] + offset);
 448                 barrier();
 449                 VPRINTK(skdev, "offset %x = %016llx\n", offset, val);
 450         }
 451 }
 452
 453
 454 #define SKD_IRQ_DEFAULT SKD_IRQ_MSI
 455 static int skd_isr_type = SKD_IRQ_DEFAULT;
 456
 457 module_param(skd_isr_type, int, 0444);
 458 MODULE_PARM_DESC(skd_isr_type, "Interrupt type capability."
 459                  " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
 460
 461 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
 462 static int skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
 463
 464 module_param(skd_max_req_per_msg, int, 0444);
 465 MODULE_PARM_DESC(skd_max_req_per_msg,
 466                  "Maximum SCSI requests packed in a single message."
 467                  " (1-14, default==1)");
 468
 469 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
 470 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
 471 static int skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
 472
 473 module_param(skd_max_queue_depth, int, 0444);
 474 MODULE_PARM_DESC(skd_max_queue_depth,
 475                  "Maximum SCSI requests issued to s1120."
 476                  " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR ")");
 477
 478 static int skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
 479 module_param(skd_sgs_per_request, int, 0444);
 480 MODULE_PARM_DESC(skd_sgs_per_request,
 481                  "Maximum SG elements per block request."
 482                  " (1-4096, default==256)");
 483
 484 static int skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
 485 module_param(skd_max_pass_thru, int, 0444);
 486 MODULE_PARM_DESC(skd_max_pass_thru,
 487                  "Maximum SCSI pass-thru at a time." " (1-50, default==32)");
 488
 489 module_param(skd_dbg_level, int, 0444);
 490 MODULE_PARM_DESC(skd_dbg_level, "s1120 debug level (0,1,2)");
 491
 492 module_param(skd_isr_comp_limit, int, 0444);
 493 MODULE_PARM_DESC(skd_isr_comp_limit, "s1120 isr comp limit (0=none) default=4");
 494
 495 static int skd_bio;
 496 module_param(skd_bio, int, 0444);
 497 MODULE_PARM_DESC(skd_bio,
 498                  "Register as a bio device instead of block (0, 1) default=0");
 499
 500 /* Major device number dynamically assigned. */
 501 static u32 skd_major;
 502
 503 static struct skd_device *skd_construct(struct pci_dev *pdev);
 504 static void skd_destruct(struct skd_device *skdev);
 505 static const struct block_device_operations skd_blockdev_ops;
 506 static void skd_send_fitmsg(struct skd_device *skdev,
 507                             struct skd_fitmsg_context *skmsg);
 508 static void skd_send_special_fitmsg(struct skd_device *skdev,
 509                                     struct skd_special_context *skspcl);
 510 static void skd_request_fn(struct request_queue *rq);
 511 static void skd_end_request(struct skd_device *skdev,
 512                             struct skd_request_context *skreq, int error);
 513 static int skd_preop_sg_list(struct skd_device *skdev,
 514                              struct skd_request_context *skreq);
 515 static void skd_postop_sg_list(struct skd_device *skdev,
 516                                struct skd_request_context *skreq);
 517
 518 static void skd_restart_device(struct skd_device *skdev);
 519 static int skd_quiesce_dev(struct skd_device *skdev);
 520 static int skd_unquiesce_dev(struct skd_device *skdev);
 521 static void skd_release_special(struct skd_device *skdev,
 522                                 struct skd_special_context *skspcl);
 523 static void skd_disable_interrupts(struct skd_device *skdev);
 524 static void skd_isr_fwstate(struct skd_device *skdev);
 525 static void skd_recover_requests(struct skd_device *skdev, int requeue);
 526 static void skd_soft_reset(struct skd_device *skdev);
 527
 528 static const char *skd_name(struct skd_device *skdev);
 529 const char *skd_drive_state_to_str(int state);
 530 const char *skd_skdev_state_to_str(enum skd_drvr_state state);
 531 static void skd_log_skdev(struct skd_device *skdev, const char *event);
 532 static void skd_log_skmsg(struct skd_device *skdev,
 533                           struct skd_fitmsg_context *skmsg, const char *event);
 534 static void skd_log_skreq(struct skd_device *skdev,
 535                           struct skd_request_context *skreq, const char *event);
 536
 537 /* FLUSH FUA flag handling. */
 538 static int skd_flush_cmd_enqueue(struct skd_device *, void *);
 539 static void *skd_flush_cmd_dequeue(struct skd_device *);
 540
 541
 542 /*
 543  *****************************************************************************
 544  * READ/WRITE REQUESTS
 545  *****************************************************************************
 546  */
 547 static void skd_stop_queue(struct skd_device *skdev)
 548 {
 549         if (!skd_bio)
 550                 blk_stop_queue(skdev->queue);
 551         else
 552                 skdev->queue_stopped = 1;
 553 }
 554
 555 static void skd_unstop_queue(struct skd_device *skdev)
 556 {
 557         if (!skd_bio)
 558                 queue_flag_clear(QUEUE_FLAG_STOPPED, skdev->queue);
 559         else
 560                 skdev->queue_stopped = 0;
 561 }
 562
 563 static void skd_start_queue(struct skd_device *skdev)
 564 {
 565         if (!skd_bio) {
 566                 blk_start_queue(skdev->queue);
 567         } else {
 568                 pr_err("(%s): Starting queue\n", skd_name(skdev));
 569                 skdev->queue_stopped = 0;
 570                 skd_request_fn(skdev->queue);
 571         }
 572 }
 573
 574 static int skd_queue_stopped(struct skd_device *skdev)
 575 {
 576         if (!skd_bio)
 577                 return blk_queue_stopped(skdev->queue);
 578         else
 579                 return skdev->queue_stopped;
 580 }
 581
 582 static void skd_fail_all_pending_blk(struct skd_device *skdev)
 583 {
 584         struct request_queue *q = skdev->queue;
 585         struct request *req;
 586
 587         for (;; ) {
 588                 req = blk_peek_request(q);
 589                 if (req == NULL)
 590                         break;
 591                 blk_start_request(req);
 592                 __blk_end_request_all(req, -EIO);
 593         }
 594 }
 595
 596 static void skd_fail_all_pending_bio(struct skd_device *skdev)
 597 {
 598         struct bio *bio;
 599         int error = -EIO;
 600
 601         for (;; ) {
 602                 bio = bio_list_pop(&skdev->bio_queue);
 603
 604                 if (bio == NULL)
 605                         break;
 606
 607                 bio_endio(bio, error);
 608         }
 609 }
 610
 611 static void skd_fail_all_pending(struct skd_device *skdev)
 612 {
 613         if (!skd_bio)
 614                 skd_fail_all_pending_blk(skdev);
 615         else
 616                 skd_fail_all_pending_bio(skdev);
 617 }
 618
 619 static void skd_make_request(struct request_queue *q, struct bio *bio)
 620 {
 621         struct skd_device *skdev = q->queuedata;
 622         unsigned long flags;
 623
 624         spin_lock_irqsave(&skdev->lock, flags);
 625
 626         bio_list_add(&skdev->bio_queue, bio);
 627         skd_request_fn(skdev->queue);
 628
 629         spin_unlock_irqrestore(&skdev->lock, flags);
 630 }
 631
 632 static void
 633 skd_prep_rw_cdb(struct skd_scsi_request *scsi_req,
 634                 int data_dir, unsigned lba,
 635                 unsigned count)
 636 {
 637         if (data_dir == READ)
 638                 scsi_req->cdb[0] = 0x28;
 639         else
 640                 scsi_req->cdb[0] = 0x2a;
 641
 642         scsi_req->cdb[1] = 0;
 643         scsi_req->cdb[2] = (lba & 0xff000000) >> 24;
 644         scsi_req->cdb[3] = (lba & 0xff0000) >> 16;
 645         scsi_req->cdb[4] = (lba & 0xff00) >> 8;
 646         scsi_req->cdb[5] = (lba & 0xff);
 647         scsi_req->cdb[6] = 0;
 648         scsi_req->cdb[7] = (count & 0xff00) >> 8;
 649         scsi_req->cdb[8] = count & 0xff;
 650         scsi_req->cdb[9] = 0;
 651 }
 652
 653 static void
 654 skd_prep_zerosize_flush_cdb(struct skd_scsi_request *scsi_req,
 655                         struct skd_request_context *skreq)
 656 {
 657         skreq->flush_cmd = 1;
 658
 659         scsi_req->cdb[0] = 0x35;
 660         scsi_req->cdb[1] = 0;
 661         scsi_req->cdb[2] = 0;
 662         scsi_req->cdb[3] = 0;
 663         scsi_req->cdb[4] = 0;
 664         scsi_req->cdb[5] = 0;
 665         scsi_req->cdb[6] = 0;
 666         scsi_req->cdb[7] = 0;
 667         scsi_req->cdb[8] = 0;
 668         scsi_req->cdb[9] = 0;
 669 }
 670
 671 static void
 672 skd_prep_discard_cdb(struct skd_scsi_request *scsi_req,
 673                         struct skd_request_context *skreq,
 674                         struct page *page,
 675                         u32 lba, u32 count)
 676 {
 677         char *buf;
 678         unsigned long len;
 679         struct request *req;
 680
 681         buf = page_address(page);
 682         len = SKD_DISCARD_CDB_LENGTH;
 683
 684         scsi_req->cdb[0] = UNMAP;
 685         scsi_req->cdb[8] = len;
 686
 687         put_unaligned_be16(6 + 16, &buf[0]);
 688         put_unaligned_be16(16, &buf[2]);
 689         put_unaligned_be64(lba, &buf[8]);
 690         put_unaligned_be32(count, &buf[16]);
 691
 692         if (!skd_bio) {
 693                 req = skreq->req;
 694                 blk_add_request_payload(req, page, len);
 695                 req->buffer = buf;
 696         } else {
 697                 skreq->bio->bi_io_vec->bv_page = page;
 698                 skreq->bio->bi_io_vec->bv_offset = 0;
 699                 skreq->bio->bi_io_vec->bv_len = len;
 700
 701                 skreq->bio->bi_vcnt = 1;
 702                 skreq->bio->bi_phys_segments = 1;
 703         }
 704 }
 705
 706 static void skd_request_fn_not_online(struct request_queue *q);
 707
 708 static void skd_request_fn(struct request_queue *q)
 709 {
 710         struct skd_device *skdev = q->queuedata;
 711         struct skd_fitmsg_context *skmsg = NULL;
 712         struct fit_msg_hdr *fmh = NULL;
 713         struct skd_request_context *skreq;
 714         struct request *req = NULL;
 715         struct bio *bio = NULL;
 716         struct skd_scsi_request *scsi_req;
 717         struct page *page;
 718         unsigned long io_flags;
 719         int error;
 720         u32 lba;
 721         u32 count;
 722         int data_dir;
 723         u32 be_lba;
 724         u32 be_count;
 725         u64 be_dmaa;
 726         u64 cmdctxt;
 727         u32 timo_slot;
 728         void *cmd_ptr;
 729         int flush, fua;
 730
 731         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
 732                 skd_request_fn_not_online(q);
 733                 return;
 734         }
 735
 736         if (skd_queue_stopped(skdev)) {
 737                 if (skdev->skmsg_free_list == NULL ||
 738                     skdev->skreq_free_list == NULL ||
 739                     skdev->in_flight >= skdev->queue_low_water_mark)
 740                         /* There is still some kind of shortage */
 741                         return;
 742
 743                 skd_unstop_queue(skdev);
 744         }
 745
 746         /*
 747          * Stop conditions:
 748          *  - There are no more native requests
 749          *  - There are already the maximum number of requests in progress
 750          *  - There are no more skd_request_context entries
 751          *  - There are no more FIT msg buffers
 752          */
 753         for (;; ) {
 754
 755                 flush = fua = 0;
 756
 757                 if (!skd_bio) {
 758                         req = blk_peek_request(q);
 759
 760                         /* Are there any native requests to start? */
 761                         if (req == NULL)
 762                                 break;
 763
 764                         lba = (u32)blk_rq_pos(req);
 765                         count = blk_rq_sectors(req);
 766                         data_dir = rq_data_dir(req);
 767                         io_flags = req->cmd_flags;
 768
 769                         if (io_flags & REQ_FLUSH)
 770                                 flush++;
 771
 772                         if (io_flags & REQ_FUA)
 773                                 fua++;
 774
 775                         VPRINTK(skdev,
 776                                 "new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
 777                                 req, lba, lba, count, count, data_dir);
 778                 } else {
 779                         if (!list_empty(&skdev->flush_list)) {
 780                                 /* Process data part of FLUSH request. */
 781                                 bio = (struct bio *)skd_flush_cmd_dequeue(skdev);
 782                                 flush++;
 783                                 VPRINTK(skdev, "processing FLUSH request with data.\n");
 784                         } else {
 785                                 /* peek at our bio queue */
 786                                 bio = bio_list_peek(&skdev->bio_queue);
 787                         }
 788
 789                         /* Are there any native requests to start? */
 790                         if (bio == NULL)
 791                                 break;
 792
 793                         lba = (u32)bio->bi_sector;
 794                         count = bio_sectors(bio);
 795                         data_dir = bio_data_dir(bio);
 796                         io_flags = bio->bi_rw;
 797
 798                         VPRINTK(skdev,
 799                                 "new bio=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
 800                                 bio, lba, lba, count, count, data_dir);
 801
 802                         if (io_flags & REQ_FLUSH)
 803                                 flush++;
 804
 805                         if (io_flags & REQ_FUA)
 806                                 fua++;
 807                 }
 808
 809                 /* At this point we know there is a request
 810                  * (from our bio q or req q depending on the way
 811                  * the driver is built do checks for resources.
 812                  */
 813
 814                 /* Are too many requets already in progress? */
 815                 if (skdev->in_flight >= skdev->cur_max_queue_depth) {
 816                         VPRINTK(skdev, "qdepth %d, limit %d\n",
 817                                 skdev->in_flight, skdev->cur_max_queue_depth);
 818                         break;
 819                 }
 820
 821                 /* Is a skd_request_context available? */
 822                 skreq = skdev->skreq_free_list;
 823                 if (skreq == NULL) {
 824                         VPRINTK(skdev, "Out of req=%p\n", q);
 825                         break;
 826                 }
 827                 SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);
 828                 SKD_ASSERT((skreq->id & SKD_ID_INCR) == 0);
 829
 830                 /* Now we check to see if we can get a fit msg */
 831                 if (skmsg == NULL) {
 832                         if (skdev->skmsg_free_list == NULL) {
 833                                 VPRINTK(skdev, "Out of msg\n");
 834                                 break;
 835                         }
 836                 }
 837
 838                 skreq->flush_cmd = 0;
 839                 skreq->n_sg = 0;
 840                 skreq->sg_byte_count = 0;
 841                 skreq->discard_page = 0;
 842
 843                 /*
 844                  * OK to now dequeue request from either bio or q.
 845                  *
 846                  * At this point we are comitted to either start or reject
 847                  * the native request. Note that skd_request_context is
 848                  * available but is still at the head of the free list.
 849                  */
 850                 if (!skd_bio) {
 851                         blk_start_request(req);
 852                         skreq->req = req;
 853                         skreq->fitmsg_id = 0;
 854                 } else {
 855                         if (unlikely(flush == SKD_FLUSH_DATA_SECOND)) {
 856                                 skreq->bio = bio;
 857                         } else {
 858                                 skreq->bio = bio_list_pop(&skdev->bio_queue);
 859                                 SKD_ASSERT(skreq->bio == bio);
 860                                 skreq->start_time = jiffies;
 861                                 part_inc_in_flight(&skdev->disk->part0,
 862                                                    bio_data_dir(bio));
 863                         }
 864
 865                         skreq->fitmsg_id = 0;
 866                 }
 867
 868                 /* Either a FIT msg is in progress or we have to start one. */
 869                 if (skmsg == NULL) {
 870                         /* Are there any FIT msg buffers available? */
 871                         skmsg = skdev->skmsg_free_list;
 872                         if (skmsg == NULL) {
 873                                 VPRINTK(skdev, "Out of msg skdev=%p\n", skdev);
 874                                 break;
 875                         }
 876                         SKD_ASSERT(skmsg->state == SKD_MSG_STATE_IDLE);
 877                         SKD_ASSERT((skmsg->id & SKD_ID_INCR) == 0);
 878
 879                         skdev->skmsg_free_list = skmsg->next;
 880
 881                         skmsg->state = SKD_MSG_STATE_BUSY;
 882                         skmsg->id += SKD_ID_INCR;
 883
 884                         /* Initialize the FIT msg header */
 885                         fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
 886                         memset(fmh, 0, sizeof(*fmh));
 887                         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
 888                         skmsg->length = sizeof(*fmh);
 889                 }
 890
 891                 skreq->fitmsg_id = skmsg->id;
 892
 893                 /*
 894                  * Note that a FIT msg may have just been started
 895                  * but contains no SoFIT requests yet.
 896                  */
 897
 898                 /*
 899                  * Transcode the request, checking as we go. The outcome of
 900                  * the transcoding is represented by the error variable.
 901                  */
 902                 cmd_ptr = &skmsg->msg_buf[skmsg->length];
 903                 memset(cmd_ptr, 0, 32);
 904
 905                 be_lba = cpu_to_be32(lba);
 906                 be_count = cpu_to_be32(count);
 907                 be_dmaa = cpu_to_be64((u64)skreq->sksg_dma_address);
 908                 cmdctxt = skreq->id + SKD_ID_INCR;
 909
 910                 scsi_req = cmd_ptr;
 911                 scsi_req->hdr.tag = cmdctxt;
 912                 scsi_req->hdr.sg_list_dma_address = be_dmaa;
 913
 914                 if (data_dir == READ)
 915                         skreq->sg_data_dir = SKD_DATA_DIR_CARD_TO_HOST;
 916                 else
 917                         skreq->sg_data_dir = SKD_DATA_DIR_HOST_TO_CARD;
 918
 919                 if (io_flags & REQ_DISCARD) {
 920                         page = alloc_page(GFP_ATOMIC | __GFP_ZERO);
 921                         if (!page) {
 922                                 pr_err("request_fn:Page allocation failed.\n");
 923                                 skd_end_request(skdev, skreq, -ENOMEM);
 924                                 break;
 925                         }
 926                         skreq->discard_page = 1;
 927                         skd_prep_discard_cdb(scsi_req, skreq, page, lba, count);
 928
 929                 } else if (flush == SKD_FLUSH_ZERO_SIZE_FIRST) {
 930                         skd_prep_zerosize_flush_cdb(scsi_req, skreq);
 931                         SKD_ASSERT(skreq->flush_cmd == 1);
 932
 933                 } else {
 934                         skd_prep_rw_cdb(scsi_req, data_dir, lba, count);
 935                 }
 936
 937                 if (fua)
 938                         scsi_req->cdb[1] |= SKD_FUA_NV;
 939
 940                 if ((!skd_bio && !req->bio) ||
 941                         (skd_bio && flush == SKD_FLUSH_ZERO_SIZE_FIRST))
 942                         goto skip_sg;
 943
 944                 error = skd_preop_sg_list(skdev, skreq);
 945
 946                 if (error != 0) {
 947                         /*
 948                          * Complete the native request with error.
 949                          * Note that the request context is still at the
 950                          * head of the free list, and that the SoFIT request
 951                          * was encoded into the FIT msg buffer but the FIT
 952                          * msg length has not been updated. In short, the
 953                          * only resource that has been allocated but might
 954                          * not be used is that the FIT msg could be empty.
 955                          */
 956                         DPRINTK(skdev, "error Out\n");
 957                         skd_end_request(skdev, skreq, error);
 958                         continue;
 959                 }
 960
 961 skip_sg:
 962                 scsi_req->hdr.sg_list_len_bytes =
 963                         cpu_to_be32(skreq->sg_byte_count);
 964
 965                 /* Complete resource allocations. */
 966                 skdev->skreq_free_list = skreq->next;
 967                 skreq->state = SKD_REQ_STATE_BUSY;
 968                 skreq->id += SKD_ID_INCR;
 969
 970                 skmsg->length += sizeof(struct skd_scsi_request);
 971                 fmh->num_protocol_cmds_coalesced++;
 972
 973                 /*
 974                  * Update the active request counts.
 975                  * Capture the timeout timestamp.
 976                  */
 977                 skreq->timeout_stamp = skdev->timeout_stamp;
 978                 timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
 979                 skdev->timeout_slot[timo_slot]++;
 980                 skdev->in_flight++;
 981                 VPRINTK(skdev, "req=0x%x busy=%d\n",
 982                         skreq->id, skdev->in_flight);
 983
 984                 /*
 985                  * If the FIT msg buffer is full send it.
 986                  */
 987                 if (skmsg->length >= SKD_N_FITMSG_BYTES ||
 988                     fmh->num_protocol_cmds_coalesced >= skd_max_req_per_msg) {
 989                         skd_send_fitmsg(skdev, skmsg);
 990                         skmsg = NULL;
 991                         fmh = NULL;
 992                 }
 993         }
 994
 995         /*
 996          * Is a FIT msg in progress? If it is empty put the buffer back
 997          * on the free list. If it is non-empty send what we got.
 998          * This minimizes latency when there are fewer requests than
 999          * what fits in a FIT msg.
1000          */
1001         if (skmsg != NULL) {
1002                 /* Bigger than just a FIT msg header? */
1003                 if (skmsg->length > sizeof(struct fit_msg_hdr)) {
1004                         VPRINTK(skdev, "sending msg=%p, len %d\n",
1005                                 skmsg, skmsg->length);
1006                         skd_send_fitmsg(skdev, skmsg);
1007                 } else {
1008                         /*
1009                          * The FIT msg is empty. It means we got started
1010                          * on the msg, but the requests were rejected.
1011                          */
1012                         skmsg->state = SKD_MSG_STATE_IDLE;
1013                         skmsg->id += SKD_ID_INCR;
1014                         skmsg->next = skdev->skmsg_free_list;
1015                         skdev->skmsg_free_list = skmsg;
1016                 }
1017                 skmsg = NULL;
1018                 fmh = NULL;
1019         }
1020
1021         /*
1022          * If req is non-NULL it means there is something to do but
1023          * we are out of a resource.
1024          */
1025         if (((!skd_bio) && req) ||
1026             ((skd_bio) && bio_list_peek(&skdev->bio_queue)))
1027                 skd_stop_queue(skdev);
1028 }
1029
1030 static void skd_end_request_blk(struct skd_device *skdev,
1031                                 struct skd_request_context *skreq, int error)
1032 {
1033         struct request *req = skreq->req;
1034         unsigned int io_flags = req->cmd_flags;
1035
1036         if ((io_flags & REQ_DISCARD) &&
1037                 (skreq->discard_page == 1)) {
1038                 VPRINTK(skdev, "skd_end_request_blk, free the page!");
1039                 free_page((unsigned long)req->buffer);
1040                 req->buffer = NULL;
1041         }
1042
1043         if (unlikely(error)) {
1044                 struct request *req = skreq->req;
1045                 char *cmd = (rq_data_dir(req) == READ) ? "read" : "write";
1046                 u32 lba = (u32)blk_rq_pos(req);
1047                 u32 count = blk_rq_sectors(req);
1048
1049                 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
1050                        skd_name(skdev), cmd, lba, count, skreq->id);
1051         } else
1052                 VPRINTK(skdev, "id=0x%x error=%d\n", skreq->id, error);
1053
1054         __blk_end_request_all(skreq->req, error);
1055 }
1056
1057 static int skd_preop_sg_list_blk(struct skd_device *skdev,
1058                                  struct skd_request_context *skreq)
1059 {
1060         struct request *req = skreq->req;
1061         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
1062         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
1063         struct scatterlist *sg = &skreq->sg[0];
1064         int n_sg;
1065         int i;
1066
1067         skreq->sg_byte_count = 0;
1068
1069         /* SKD_ASSERT(skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD ||
1070                    skreq->sg_data_dir == SKD_DATA_DIR_CARD_TO_HOST); */
1071
1072         n_sg = blk_rq_map_sg(skdev->queue, req, sg);
1073         if (n_sg <= 0)
1074                 return -EINVAL;
1075
1076         /*
1077          * Map scatterlist to PCI bus addresses.
1078          * Note PCI might change the number of entries.
1079          */
1080         n_sg = pci_map_sg(skdev->pdev, sg, n_sg, pci_dir);
1081         if (n_sg <= 0)
1082                 return -EINVAL;
1083
1084         SKD_ASSERT(n_sg <= skdev->sgs_per_request);
1085
1086         skreq->n_sg = n_sg;
1087
1088         for (i = 0; i < n_sg; i++) {
1089                 struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1090                 u32 cnt = sg_dma_len(&sg[i]);
1091                 uint64_t dma_addr = sg_dma_address(&sg[i]);
1092
1093                 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
1094                 sgd->byte_count = cnt;
1095                 skreq->sg_byte_count += cnt;
1096                 sgd->host_side_addr = dma_addr;
1097                 sgd->dev_side_addr = 0;
1098         }
1099
1100         skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
1101         skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
1102
1103         if (unlikely(skdev->dbg_level > 1)) {
1104                 VPRINTK(skdev, "skreq=%x sksg_list=%p sksg_dma=%llx\n",
1105                         skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
1106                 for (i = 0; i < n_sg; i++) {
1107                         struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1108                         VPRINTK(skdev, "  sg[%d] count=%u ctrl=0x%x "
1109                                 "addr=0x%llx next=0x%llx\n",
1110                                 i, sgd->byte_count, sgd->control,
1111                                 sgd->host_side_addr, sgd->next_desc_ptr);
1112                 }
1113         }
1114
1115         return 0;
1116 }
1117
1118 static void skd_postop_sg_list_blk(struct skd_device *skdev,
1119                                    struct skd_request_context *skreq)
1120 {
1121         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
1122         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
1123
1124         /*
1125          * restore the next ptr for next IO request so we
1126          * don't have to set it every time.
1127          */
1128         skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
1129                 skreq->sksg_dma_address +
1130                 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
1131         pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, pci_dir);
1132 }
1133
1134 static void skd_end_request_bio(struct skd_device *skdev,
1135                                 struct skd_request_context *skreq, int error)
1136 {
1137         struct bio *bio = skreq->bio;
1138         int rw = bio_data_dir(bio);
1139         unsigned long io_flags = bio->bi_rw;
1140
1141         if ((io_flags & REQ_DISCARD) &&
1142                 (skreq->discard_page == 1)) {
1143                 VPRINTK(skdev, "biomode: skd_end_request: freeing DISCARD page.\n");
1144                 free_page((unsigned long)page_address(bio->bi_io_vec->bv_page));
1145         }
1146
1147         if (unlikely(error)) {
1148                 u32 lba = (u32)skreq->bio->bi_sector;
1149                 u32 count = bio_sectors(skreq->bio);
1150                 char *cmd = (rw == WRITE) ? "write" : "read";
1151                 pr_err("(%s): Error cmd=%s sect=%u count=%u id=0x%x\n",
1152                        skd_name(skdev), cmd, lba, count, skreq->id);
1153         }
1154         {
1155                 int cpu = part_stat_lock();
1156
1157                 if (likely(!error)) {
1158                         part_stat_inc(cpu, &skdev->disk->part0, ios[rw]);
1159                         part_stat_add(cpu, &skdev->disk->part0, sectors[rw],
1160                                       bio_sectors(bio));
1161                 }
1162                 part_stat_add(cpu, &skdev->disk->part0, ticks[rw],
1163                               jiffies - skreq->start_time);
1164                 part_dec_in_flight(&skdev->disk->part0, rw);
1165                 part_stat_unlock();
1166         }
1167
1168         VPRINTK(skdev, "id=0x%x error=%d\n", skreq->id, error);
1169
1170         bio_endio(skreq->bio, error);
1171 }
1172
1173 static int skd_preop_sg_list_bio(struct skd_device *skdev,
1174                                  struct skd_request_context *skreq)
1175 {
1176         struct bio *bio = skreq->bio;
1177         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
1178         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
1179         int n_sg;
1180         int i;
1181         struct bio_vec *vec;
1182         struct fit_sg_descriptor *sgd;
1183         u64 dma_addr;
1184         u32 count;
1185         int errs = 0;
1186         unsigned int io_flags = 0;
1187         io_flags |= bio->bi_rw;
1188
1189         skreq->sg_byte_count = 0;
1190         n_sg = skreq->n_sg = skreq->bio->bi_vcnt;
1191
1192         if (n_sg <= 0)
1193                 return -EINVAL;
1194
1195         if (n_sg > skdev->sgs_per_request) {
1196                 pr_err("(%s): sg overflow n=%d\n",
1197                        skd_name(skdev), n_sg);
1198                 skreq->n_sg = 0;
1199                 return -EIO;
1200         }
1201
1202         for (i = 0; i < skreq->n_sg; i++) {
1203                 vec = bio_iovec_idx(bio, i);
1204                 dma_addr = pci_map_page(skdev->pdev,
1205                                         vec->bv_page,
1206                                         vec->bv_offset, vec->bv_len, pci_dir);
1207                 count = vec->bv_len;
1208
1209                 if (count == 0 || count > 64u * 1024u || (count & 3) != 0
1210                     || (dma_addr & 3) != 0) {
1211                         pr_err(
1212                                "(%s): Bad sg ix=%d count=%d addr=0x%llx\n",
1213                                skd_name(skdev), i, count, dma_addr);
1214                         errs++;
1215                 }
1216
1217                 sgd = &skreq->sksg_list[i];
1218
1219                 sgd->control = FIT_SGD_CONTROL_NOT_LAST;
1220                 sgd->byte_count = vec->bv_len;
1221                 skreq->sg_byte_count += vec->bv_len;
1222                 sgd->host_side_addr = dma_addr;
1223                 sgd->dev_side_addr = 0; /* not used */
1224         }
1225
1226         skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;
1227         skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;
1228
1229
1230          if (!(io_flags & REQ_DISCARD)) {
1231                 count = bio_sectors(bio) << 9u;
1232                 if (count != skreq->sg_byte_count) {
1233                         pr_err("(%s): mismatch count sg=%d req=%d\n",
1234                                skd_name(skdev), skreq->sg_byte_count, count);
1235                         errs++;
1236                 }
1237         }
1238
1239         if (unlikely(skdev->dbg_level > 1)) {
1240                 VPRINTK(skdev, "skreq=%x sksg_list=%p sksg_dma=%llx\n",
1241                         skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
1242                 for (i = 0; i < n_sg; i++) {
1243                         struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1244                         VPRINTK(skdev, "  sg[%d] count=%u ctrl=0x%x "
1245                                 "addr=0x%llx next=0x%llx\n",
1246                                 i, sgd->byte_count, sgd->control,
1247                                 sgd->host_side_addr, sgd->next_desc_ptr);
1248                 }
1249         }
1250
1251         if (errs != 0) {
1252                 skd_postop_sg_list(skdev, skreq);
1253                 skreq->n_sg = 0;
1254                 return -EIO;
1255         }
1256
1257         return 0;
1258 }
1259
1260 static int skd_preop_sg_list(struct skd_device *skdev,
1261                              struct skd_request_context *skreq)
1262 {
1263         if (!skd_bio)
1264                 return skd_preop_sg_list_blk(skdev, skreq);
1265         else
1266                 return skd_preop_sg_list_bio(skdev, skreq);
1267 }
1268
1269 static void skd_postop_sg_list_bio(struct skd_device *skdev,
1270                                    struct skd_request_context *skreq)
1271 {
1272         int writing = skreq->sg_data_dir == SKD_DATA_DIR_HOST_TO_CARD;
1273         int pci_dir = writing ? PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE;
1274         int i;
1275         struct fit_sg_descriptor *sgd;
1276
1277         /*
1278          * restore the next ptr for next IO request so we
1279          * don't have to set it every time.
1280          */
1281         skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =
1282                 skreq->sksg_dma_address +
1283                 ((skreq->n_sg) * sizeof(struct fit_sg_descriptor));
1284
1285         for (i = 0; i < skreq->n_sg; i++) {
1286                 sgd = &skreq->sksg_list[i];
1287                 pci_unmap_page(skdev->pdev, sgd->host_side_addr,
1288                                sgd->byte_count, pci_dir);
1289         }
1290 }
1291
1292 static void skd_postop_sg_list(struct skd_device *skdev,
1293                                struct skd_request_context *skreq)
1294 {
1295         if (!skd_bio)
1296                 skd_postop_sg_list_blk(skdev, skreq);
1297         else
1298                 skd_postop_sg_list_bio(skdev, skreq);
1299 }
1300
1301 static void skd_end_request(struct skd_device *skdev,
1302                             struct skd_request_context *skreq, int error)
1303 {
1304         if (likely(!skd_bio))
1305                 skd_end_request_blk(skdev, skreq, error);
1306         else
1307                 skd_end_request_bio(skdev, skreq, error);
1308 }
1309
1310 static void skd_request_fn_not_online(struct request_queue *q)
1311 {
1312         struct skd_device *skdev = q->queuedata;
1313         int error;
1314
1315         SKD_ASSERT(skdev->state != SKD_DRVR_STATE_ONLINE);
1316
1317         skd_log_skdev(skdev, "req_not_online");
1318         switch (skdev->state) {
1319         case SKD_DRVR_STATE_PAUSING:
1320         case SKD_DRVR_STATE_PAUSED:
1321         case SKD_DRVR_STATE_STARTING:
1322         case SKD_DRVR_STATE_RESTARTING:
1323         case SKD_DRVR_STATE_WAIT_BOOT:
1324         /* In case of starting, we haven't started the queue,
1325          * so we can't get here... but requests are
1326          * possibly hanging out waiting for us because we
1327          * reported the dev/skd0 already.  They'll wait
1328          * forever if connect doesn't complete.
1329          * What to do??? delay dev/skd0 ??
1330          */
1331         case SKD_DRVR_STATE_BUSY:
1332         case SKD_DRVR_STATE_BUSY_IMMINENT:
1333         case SKD_DRVR_STATE_BUSY_ERASE:
1334         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1335                 return;
1336
1337         case SKD_DRVR_STATE_BUSY_SANITIZE:
1338         case SKD_DRVR_STATE_STOPPING:
1339         case SKD_DRVR_STATE_SYNCING:
1340         case SKD_DRVR_STATE_FAULT:
1341         case SKD_DRVR_STATE_DISAPPEARED:
1342         default:
1343                 error = -EIO;
1344                 break;
1345         }
1346
1347         /* If we get here, terminate all pending block requeusts
1348          * with EIO and any scsi pass thru with appropriate sense
1349          */
1350
1351         skd_fail_all_pending(skdev);
1352 }
1353
1354 /*
1355  *****************************************************************************
1356  * TIMER
1357  *****************************************************************************
1358  */
1359
1360 static void skd_timer_tick_not_online(struct skd_device *skdev);
1361
1362 static void skd_timer_tick(ulong arg)
1363 {
1364         struct skd_device *skdev = (struct skd_device *)arg;
1365
1366         u32 timo_slot;
1367         u32 overdue_timestamp;
1368         unsigned long reqflags;
1369         u32 state;
1370
1371         if (skdev->state == SKD_DRVR_STATE_FAULT)
1372                 /* The driver has declared fault, and we want it to
1373                  * stay that way until driver is reloaded.
1374                  */
1375                 return;
1376
1377         spin_lock_irqsave(&skdev->lock, reqflags);
1378
1379         state = SKD_READL(skdev, FIT_STATUS);
1380         state &= FIT_SR_DRIVE_STATE_MASK;
1381         if (state != skdev->drive_state)
1382                 skd_isr_fwstate(skdev);
1383
1384         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
1385                 skd_timer_tick_not_online(skdev);
1386                 goto timer_func_out;
1387         }
1388         skdev->timeout_stamp++;
1389         timo_slot = skdev->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
1390
1391         /*
1392          * All requests that happened during the previous use of
1393          * this slot should be done by now. The previous use was
1394          * over 7 seconds ago.
1395          */
1396         if (skdev->timeout_slot[timo_slot] == 0)
1397                 goto timer_func_out;
1398
1399         /* Something is overdue */
1400         overdue_timestamp = skdev->timeout_stamp - SKD_N_TIMEOUT_SLOT;
1401
1402         DPRINTK(skdev, "found %d timeouts, draining busy=%d\n",
1403                 skdev->timeout_slot[timo_slot], skdev->in_flight);
1404         pr_err("(%s): Overdue IOs (%d), busy %d\n",
1405                skd_name(skdev), skdev->timeout_slot[timo_slot],
1406                skdev->in_flight);
1407
1408         skdev->timer_countdown = SKD_DRAINING_TIMO;
1409         skdev->state = SKD_DRVR_STATE_DRAINING_TIMEOUT;
1410         skdev->timo_slot = timo_slot;
1411         skd_stop_queue(skdev);
1412
1413 timer_func_out:
1414         mod_timer(&skdev->timer, (jiffies + HZ));
1415
1416         spin_unlock_irqrestore(&skdev->lock, reqflags);
1417 }
1418
1419 static void skd_timer_tick_not_online(struct skd_device *skdev)
1420 {
1421         switch (skdev->state) {
1422         case SKD_DRVR_STATE_IDLE:
1423         case SKD_DRVR_STATE_LOAD:
1424                 break;
1425         case SKD_DRVR_STATE_BUSY_SANITIZE:
1426                 VPRINTK(skdev, "drive busy sanitize[%x], driver[%x]\n",
1427                         skdev->drive_state, skdev->state);
1428                 /* If we've been in sanitize for 3 seconds, we figure we're not
1429                  * going to get anymore completions, so recover requests now
1430                  */
1431                 if (skdev->timer_countdown > 0) {
1432                         skdev->timer_countdown--;
1433                         return;
1434                 }
1435                 skd_recover_requests(skdev, 0);
1436                 break;
1437
1438         case SKD_DRVR_STATE_BUSY:
1439         case SKD_DRVR_STATE_BUSY_IMMINENT:
1440         case SKD_DRVR_STATE_BUSY_ERASE:
1441                 VPRINTK(skdev, "busy[%x], countdown=%d\n",
1442                         skdev->state, skdev->timer_countdown);
1443                 if (skdev->timer_countdown > 0) {
1444                         skdev->timer_countdown--;
1445                         return;
1446                 }
1447                 DPRINTK(skdev, "busy[%x], timedout=%d, restarting device.",
1448                         skdev->state, skdev->timer_countdown);
1449                 skd_restart_device(skdev);
1450                 break;
1451
1452         case SKD_DRVR_STATE_WAIT_BOOT:
1453         case SKD_DRVR_STATE_STARTING:
1454                 if (skdev->timer_countdown > 0) {
1455                         skdev->timer_countdown--;
1456                         return;
1457                 }
1458                 /* For now, we fault the drive.  Could attempt resets to
1459                  * revcover at some point. */
1460                 skdev->state = SKD_DRVR_STATE_FAULT;
1461
1462                 pr_err("(%s): DriveFault Connect Timeout (%x)\n",
1463                        skd_name(skdev), skdev->drive_state);
1464
1465                 /*start the queue so we can respond with error to requests */
1466                 /* wakeup anyone waiting for startup complete */
1467                 skd_start_queue(skdev);
1468                 skdev->gendisk_on = -1;
1469                 wake_up_interruptible(&skdev->waitq);
1470                 break;
1471
1472         case SKD_DRVR_STATE_ONLINE:
1473                 /* shouldn't get here. */
1474                 break;
1475
1476         case SKD_DRVR_STATE_PAUSING:
1477         case SKD_DRVR_STATE_PAUSED:
1478                 break;
1479
1480         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
1481                 DPRINTK(skdev,
1482                         "draining busy [%d] tick[%d] qdb[%d] tmls[%d]\n",
1483                         skdev->timo_slot,
1484                         skdev->timer_countdown,
1485                         skdev->in_flight,
1486                         skdev->timeout_slot[skdev->timo_slot]);
1487                 /* if the slot has cleared we can let the I/O continue */
1488                 if (skdev->timeout_slot[skdev->timo_slot] == 0) {
1489                         DPRINTK(skdev, "Slot drained, starting queue.\n");
1490                         skdev->state = SKD_DRVR_STATE_ONLINE;
1491                         skd_start_queue(skdev);
1492                         return;
1493                 }
1494                 if (skdev->timer_countdown > 0) {
1495                         skdev->timer_countdown--;
1496                         return;
1497                 }
1498                 skd_restart_device(skdev);
1499                 break;
1500
1501         case SKD_DRVR_STATE_RESTARTING:
1502                 if (skdev->timer_countdown > 0) {
1503                         skdev->timer_countdown--;
1504                         return;
1505                 }
1506                 /* For now, we fault the drive. Could attempt resets to
1507                  * revcover at some point. */
1508                 skdev->state = SKD_DRVR_STATE_FAULT;
1509                 pr_err("(%s): DriveFault Reconnect Timeout (%x)\n",
1510                        skd_name(skdev), skdev->drive_state);
1511
1512                 /*
1513                  * Recovering does two things:
1514                  * 1. completes IO with error
1515                  * 2. reclaims dma resources
1516                  * When is it safe to recover requests?
1517                  * - if the drive state is faulted
1518                  * - if the state is still soft reset after out timeout
1519                  * - if the drive registers are dead (state = FF)
1520                  * If it is "unsafe", we still need to recover, so we will
1521                  * disable pci bus mastering and disable our interrupts.
1522                  */
1523
1524                 if ((skdev->drive_state == FIT_SR_DRIVE_SOFT_RESET) ||
1525                     (skdev->drive_state == FIT_SR_DRIVE_FAULT) ||
1526                     (skdev->drive_state == FIT_SR_DRIVE_STATE_MASK))
1527                         /* It never came out of soft reset. Try to
1528                          * recover the requests and then let them
1529                          * fail. This is to mitigate hung processes. */
1530                         skd_recover_requests(skdev, 0);
1531                 else {
1532                         pr_err("(%s): Disable BusMaster (%x)\n",
1533                                skd_name(skdev), skdev->drive_state);
1534                         pci_disable_device(skdev->pdev);
1535                         skd_disable_interrupts(skdev);
1536                         skd_recover_requests(skdev, 0);
1537                 }
1538
1539                 /*start the queue so we can respond with error to requests */
1540                 /* wakeup anyone waiting for startup complete */
1541                 skd_start_queue(skdev);
1542                 skdev->gendisk_on = -1;
1543                 wake_up_interruptible(&skdev->waitq);
1544                 break;
1545
1546         case SKD_DRVR_STATE_RESUMING:
1547         case SKD_DRVR_STATE_STOPPING:
1548         case SKD_DRVR_STATE_SYNCING:
1549         case SKD_DRVR_STATE_FAULT:
1550         case SKD_DRVR_STATE_DISAPPEARED:
1551         default:
1552                 break;
1553         }
1554 }
1555
1556 static int skd_start_timer(struct skd_device *skdev)
1557 {
1558         int rc;
1559
1560         init_timer(&skdev->timer);
1561         setup_timer(&skdev->timer, skd_timer_tick, (ulong)skdev);
1562
1563         rc = mod_timer(&skdev->timer, (jiffies + HZ));
1564         if (rc)
1565                 pr_err("%s: failed to start timer %d\n",
1566                        __func__, rc);
1567         return rc;
1568 }
1569
1570 static void skd_kill_timer(struct skd_device *skdev)
1571 {
1572         del_timer_sync(&skdev->timer);
1573 }
1574
1575 /*
1576  *****************************************************************************
1577  * IOCTL
1578  *****************************************************************************
1579  */
1580 static int skd_ioctl_sg_io(struct skd_device *skdev,
1581                            fmode_t mode, void __user *argp);
1582 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1583                                         struct skd_sg_io *sksgio);
1584 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1585                                    struct skd_sg_io *sksgio);
1586 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1587                                     struct skd_sg_io *sksgio);
1588 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1589                                  struct skd_sg_io *sksgio, int dxfer_dir);
1590 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
1591                                  struct skd_sg_io *sksgio);
1592 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio);
1593 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
1594                                     struct skd_sg_io *sksgio);
1595 static int skd_sg_io_put_status(struct skd_device *skdev,
1596                                 struct skd_sg_io *sksgio);
1597
1598 static void skd_complete_special(struct skd_device *skdev,
1599                                  volatile struct fit_completion_entry_v1
1600                                  *skcomp,
1601                                  volatile struct fit_comp_error_info *skerr,
1602                                  struct skd_special_context *skspcl);
1603
1604 static int skd_bdev_ioctl(struct block_device *bdev, fmode_t mode,
1605                           uint cmd_in, ulong arg)
1606 {
1607         int rc = 0;
1608         struct gendisk *disk = bdev->bd_disk;
1609         struct skd_device *skdev = disk->private_data;
1610         void __user *p = (void *)arg;
1611
1612         DPRINTK(skdev, "%s: CMD[%s] ioctl  mode 0x%x, cmd 0x%x arg %0lx\n",
1613                 disk->disk_name, current->comm, mode, cmd_in, arg);
1614
1615         if (!capable(CAP_SYS_ADMIN))
1616                 return -EPERM;
1617
1618         switch (cmd_in) {
1619         case SG_SET_TIMEOUT:
1620         case SG_GET_TIMEOUT:
1621         case SG_GET_VERSION_NUM:
1622                 rc = scsi_cmd_ioctl(disk->queue, disk, mode, cmd_in, p);
1623                 break;
1624         case SG_IO:
1625                 rc = skd_ioctl_sg_io(skdev, mode, p);
1626                 break;
1627
1628         default:
1629                 rc = -ENOTTY;
1630                 break;
1631         }
1632
1633         DPRINTK(skdev, "%s:  completion rc %d\n", disk->disk_name, rc);
1634         return rc;
1635 }
1636
1637 static int skd_ioctl_sg_io(struct skd_device *skdev, fmode_t mode,
1638                            void __user *argp)
1639 {
1640         int rc;
1641         struct skd_sg_io sksgio;
1642
1643         memset(&sksgio, 0, sizeof(sksgio));
1644         sksgio.mode = mode;
1645         sksgio.argp = argp;
1646         sksgio.iov = &sksgio.no_iov_iov;
1647
1648         switch (skdev->state) {
1649         case SKD_DRVR_STATE_ONLINE:
1650         case SKD_DRVR_STATE_BUSY_IMMINENT:
1651                 break;
1652
1653         default:
1654                 DPRINTK(skdev, "drive not online\n");
1655                 rc = -ENXIO;
1656                 goto out;
1657         }
1658
1659         if ((rc = skd_sg_io_get_and_check_args(skdev, &sksgio)) ||
1660             (rc = skd_sg_io_obtain_skspcl(skdev, &sksgio)) ||
1661             (rc = skd_sg_io_prep_buffering(skdev, &sksgio)) ||
1662             (rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_TO_DEV)))
1663                 goto out;
1664
1665         if ((rc = skd_sg_io_send_fitmsg(skdev, &sksgio)) ||
1666             (rc = skd_sg_io_await(skdev, &sksgio)))
1667                 goto out;
1668
1669         if ((rc = skd_sg_io_copy_buffer(skdev, &sksgio, SG_DXFER_FROM_DEV)) ||
1670             (rc = skd_sg_io_put_status(skdev, &sksgio)))
1671                 goto out;
1672
1673         rc = 0;
1674
1675 out:
1676         skd_sg_io_release_skspcl(skdev, &sksgio);
1677
1678         if (sksgio.iov != NULL && sksgio.iov != &sksgio.no_iov_iov)
1679                 kfree(sksgio.iov);
1680         return rc;
1681 }
1682
1683 static int skd_sg_io_get_and_check_args(struct skd_device *skdev,
1684                                         struct skd_sg_io *sksgio)
1685 {
1686         struct sg_io_hdr *sgp = &sksgio->sg;
1687         int i, acc;
1688
1689         if (!access_ok(VERIFY_WRITE, sksgio->argp, sizeof(sg_io_hdr_t))) {
1690                 DPRINTK(skdev, "access sg failed %p\n", sksgio->argp);
1691                 return -EFAULT;
1692         }
1693
1694         if (__copy_from_user(sgp, sksgio->argp, sizeof(sg_io_hdr_t))) {
1695                 DPRINTK(skdev, "copy_from_user sg failed %p\n", sksgio->argp);
1696                 return -EFAULT;
1697         }
1698
1699         if (sgp->interface_id != SG_INTERFACE_ID_ORIG) {
1700                 DPRINTK(skdev, "interface_id invalid 0x%x\n",
1701                         sgp->interface_id);
1702                 return -EINVAL;
1703         }
1704
1705         if (sgp->cmd_len > sizeof(sksgio->cdb)) {
1706                 DPRINTK(skdev, "cmd_len invalid %d\n", sgp->cmd_len);
1707                 return -EINVAL;
1708         }
1709
1710         if (sgp->iovec_count > 256) {
1711                 DPRINTK(skdev, "iovec_count invalid %d\n", sgp->iovec_count);
1712                 return -EINVAL;
1713         }
1714
1715         if (sgp->dxfer_len > (PAGE_SIZE * SKD_N_SG_PER_SPECIAL)) {
1716                 DPRINTK(skdev, "dxfer_len invalid %d\n", sgp->dxfer_len);
1717                 return -EINVAL;
1718         }
1719
1720         switch (sgp->dxfer_direction) {
1721         case SG_DXFER_NONE:
1722                 acc = -1;
1723                 break;
1724
1725         case SG_DXFER_TO_DEV:
1726                 acc = VERIFY_READ;
1727                 break;
1728
1729         case SG_DXFER_FROM_DEV:
1730         case SG_DXFER_TO_FROM_DEV:
1731                 acc = VERIFY_WRITE;
1732                 break;
1733
1734         default:
1735                 DPRINTK(skdev, "dxfer_dir invalid %d\n", sgp->dxfer_direction);
1736                 return -EINVAL;
1737         }
1738
1739         if (copy_from_user(sksgio->cdb, sgp->cmdp, sgp->cmd_len)) {
1740                 DPRINTK(skdev, "copy_from_user cmdp failed %p\n", sgp->cmdp);
1741                 return -EFAULT;
1742         }
1743
1744         if (sgp->mx_sb_len != 0) {
1745                 if (!access_ok(VERIFY_WRITE, sgp->sbp, sgp->mx_sb_len)) {
1746                         DPRINTK(skdev, "access sbp failed %p\n", sgp->sbp);
1747                         return -EFAULT;
1748                 }
1749         }
1750
1751         if (sgp->iovec_count == 0) {
1752                 sksgio->iov[0].iov_base = sgp->dxferp;
1753                 sksgio->iov[0].iov_len = sgp->dxfer_len;
1754                 sksgio->iovcnt = 1;
1755                 sksgio->dxfer_len = sgp->dxfer_len;
1756         } else {
1757                 struct sg_iovec *iov;
1758                 uint nbytes = sizeof(*iov) * sgp->iovec_count;
1759                 size_t iov_data_len;
1760
1761                 iov = kmalloc(nbytes, GFP_KERNEL);
1762                 if (iov == NULL) {
1763                         DPRINTK(skdev, "alloc iovec failed %d\n",
1764                                 sgp->iovec_count);
1765                         return -ENOMEM;
1766                 }
1767                 sksgio->iov = iov;
1768                 sksgio->iovcnt = sgp->iovec_count;
1769
1770                 if (copy_from_user(iov, sgp->dxferp, nbytes)) {
1771                         DPRINTK(skdev, "copy_from_user iovec failed %p\n",
1772                                 sgp->dxferp);
1773                         return -EFAULT;
1774                 }
1775
1776                 /*
1777                  * Sum up the vecs, making sure they don't overflow
1778                  */
1779                 iov_data_len = 0;
1780                 for (i = 0; i < sgp->iovec_count; i++) {
1781                         if (iov_data_len + iov[i].iov_len < iov_data_len)
1782                                 return -EINVAL;
1783                         iov_data_len += iov[i].iov_len;
1784                 }
1785
1786                 /* SG_IO howto says that the shorter of the two wins */
1787                 if (sgp->dxfer_len < iov_data_len) {
1788                         sksgio->iovcnt = iov_shorten((struct iovec *)iov,
1789                                                      sgp->iovec_count,
1790                                                      sgp->dxfer_len);
1791                         sksgio->dxfer_len = sgp->dxfer_len;
1792                 } else
1793                         sksgio->dxfer_len = iov_data_len;
1794         }
1795
1796         if (sgp->dxfer_direction != SG_DXFER_NONE) {
1797                 struct sg_iovec *iov = sksgio->iov;
1798                 for (i = 0; i < sksgio->iovcnt; i++, iov++) {
1799                         if (!access_ok(acc, iov->iov_base, iov->iov_len)) {
1800                                 DPRINTK(skdev, "access data failed %p/%d\n",
1801                                         iov->iov_base, (int)iov->iov_len);
1802                                 return -EFAULT;
1803                         }
1804                 }
1805         }
1806
1807         return 0;
1808 }
1809
1810 static int skd_sg_io_obtain_skspcl(struct skd_device *skdev,
1811                                    struct skd_sg_io *sksgio)
1812 {
1813         struct skd_special_context *skspcl = NULL;
1814         int rc;
1815
1816         for (;; ) {
1817                 ulong flags;
1818
1819                 spin_lock_irqsave(&skdev->lock, flags);
1820                 skspcl = skdev->skspcl_free_list;
1821                 if (skspcl != NULL) {
1822                         skdev->skspcl_free_list =
1823                                 (struct skd_special_context *)skspcl->req.next;
1824                         skspcl->req.id += SKD_ID_INCR;
1825                         skspcl->req.state = SKD_REQ_STATE_SETUP;
1826                         skspcl->orphaned = 0;
1827                         skspcl->req.n_sg = 0;
1828                 }
1829                 spin_unlock_irqrestore(&skdev->lock, flags);
1830
1831                 if (skspcl != NULL) {
1832                         rc = 0;
1833                         break;
1834                 }
1835
1836                 DPRINTK(skdev, "blocking\n");
1837
1838                 rc = wait_event_interruptible_timeout(
1839                                 skdev->waitq,
1840                                 (skdev->skspcl_free_list != NULL),
1841                                 msecs_to_jiffies(sksgio->sg.timeout));
1842
1843                 DPRINTK(skdev, "unblocking, rc=%d\n", rc);
1844
1845                 if (rc <= 0) {
1846                         if (rc == 0)
1847                                 rc = -ETIMEDOUT;
1848                         else
1849                                 rc = -EINTR;
1850                         break;
1851                 }
1852                 /*
1853                  * If we get here rc > 0 meaning the timeout to
1854                  * wait_event_interruptible_timeout() had time left, hence the
1855                  * sought event -- non-empty free list -- happened.
1856                  * Retry the allocation.
1857                  */
1858         }
1859         sksgio->skspcl = skspcl;
1860
1861         return rc;
1862 }
1863
1864 static int skd_skreq_prep_buffering(struct skd_device *skdev,
1865                                     struct skd_request_context *skreq,
1866                                     u32 dxfer_len)
1867 {
1868         u32 resid = dxfer_len;
1869
1870         /*
1871          * The DMA engine must have aligned addresses and byte counts.
1872          */
1873         resid += (-resid) & 3;
1874         skreq->sg_byte_count = resid;
1875
1876         skreq->n_sg = 0;
1877
1878         while (resid > 0) {
1879                 u32 nbytes = PAGE_SIZE;
1880                 u32 ix = skreq->n_sg;
1881                 struct scatterlist *sg = &skreq->sg[ix];
1882                 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1883                 struct page *page;
1884
1885                 if (nbytes > resid)
1886                         nbytes = resid;
1887
1888                 page = alloc_page(GFP_KERNEL);
1889                 if (page == NULL)
1890                         return -ENOMEM;
1891
1892                 sg_set_page(sg, page, nbytes, 0);
1893
1894                 /* TODO: This should be going through a pci_???()
1895                  * routine to do proper mapping. */
1896                 sksg->control = FIT_SGD_CONTROL_NOT_LAST;
1897                 sksg->byte_count = nbytes;
1898
1899                 sksg->host_side_addr = sg_phys(sg);
1900
1901                 sksg->dev_side_addr = 0;
1902                 sksg->next_desc_ptr = skreq->sksg_dma_address +
1903                                       (ix + 1) * sizeof(*sksg);
1904
1905                 skreq->n_sg++;
1906                 resid -= nbytes;
1907         }
1908
1909         if (skreq->n_sg > 0) {
1910                 u32 ix = skreq->n_sg - 1;
1911                 struct fit_sg_descriptor *sksg = &skreq->sksg_list[ix];
1912
1913                 sksg->control = FIT_SGD_CONTROL_LAST;
1914                 sksg->next_desc_ptr = 0;
1915         }
1916
1917         if (unlikely(skdev->dbg_level > 1)) {
1918                 u32 i;
1919
1920                 VPRINTK(skdev, "skreq=%x sksg_list=%p sksg_dma=%llx\n",
1921                         skreq->id, skreq->sksg_list, skreq->sksg_dma_address);
1922                 for (i = 0; i < skreq->n_sg; i++) {
1923                         struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];
1924
1925                         VPRINTK(skdev, "  sg[%d] count=%u ctrl=0x%x "
1926                                 "addr=0x%llx next=0x%llx\n",
1927                                 i, sgd->byte_count, sgd->control,
1928                                 sgd->host_side_addr, sgd->next_desc_ptr);
1929                 }
1930         }
1931
1932         return 0;
1933 }
1934
1935 static int skd_sg_io_prep_buffering(struct skd_device *skdev,
1936                                     struct skd_sg_io *sksgio)
1937 {
1938         struct skd_special_context *skspcl = sksgio->skspcl;
1939         struct skd_request_context *skreq = &skspcl->req;
1940         u32 dxfer_len = sksgio->dxfer_len;
1941         int rc;
1942
1943         rc = skd_skreq_prep_buffering(skdev, skreq, dxfer_len);
1944         /*
1945          * Eventually, errors or not, skd_release_special() is called
1946          * to recover allocations including partial allocations.
1947          */
1948         return rc;
1949 }
1950
1951 static int skd_sg_io_copy_buffer(struct skd_device *skdev,
1952                                  struct skd_sg_io *sksgio, int dxfer_dir)
1953 {
1954         struct skd_special_context *skspcl = sksgio->skspcl;
1955         u32 iov_ix = 0;
1956         struct sg_iovec curiov;
1957         u32 sksg_ix = 0;
1958         u8 *bufp = NULL;
1959         u32 buf_len = 0;
1960         u32 resid = sksgio->dxfer_len;
1961         int rc;
1962
1963         curiov.iov_len = 0;
1964         curiov.iov_base = NULL;
1965
1966         if (dxfer_dir != sksgio->sg.dxfer_direction) {
1967                 if (dxfer_dir != SG_DXFER_TO_DEV ||
1968                     sksgio->sg.dxfer_direction != SG_DXFER_TO_FROM_DEV)
1969                         return 0;
1970         }
1971
1972         while (resid > 0) {
1973                 u32 nbytes = PAGE_SIZE;
1974
1975                 if (curiov.iov_len == 0) {
1976                         curiov = sksgio->iov[iov_ix++];
1977                         continue;
1978                 }
1979
1980                 if (buf_len == 0) {
1981                         struct page *page;
1982                         page = sg_page(&skspcl->req.sg[sksg_ix++]);
1983                         bufp = page_address(page);
1984                         buf_len = PAGE_SIZE;
1985                 }
1986
1987                 nbytes = min_t(u32, nbytes, resid);
1988                 nbytes = min_t(u32, nbytes, curiov.iov_len);
1989                 nbytes = min_t(u32, nbytes, buf_len);
1990
1991                 if (dxfer_dir == SG_DXFER_TO_DEV)
1992                         rc = __copy_from_user(bufp, curiov.iov_base, nbytes);
1993                 else
1994                         rc = __copy_to_user(curiov.iov_base, bufp, nbytes);
1995
1996                 if (rc)
1997                         return -EFAULT;
1998
1999                 resid -= nbytes;
2000                 curiov.iov_len -= nbytes;
2001                 curiov.iov_base += nbytes;
2002                 buf_len -= nbytes;
2003         }
2004
2005         return 0;
2006 }
2007
2008 static int skd_sg_io_send_fitmsg(struct skd_device *skdev,
2009                                  struct skd_sg_io *sksgio)
2010 {
2011         struct skd_special_context *skspcl = sksgio->skspcl;
2012         struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2013         struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
2014
2015         memset(skspcl->msg_buf, 0, SKD_N_SPECIAL_FITMSG_BYTES);
2016
2017         /* Initialize the FIT msg header */
2018         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
2019         fmh->num_protocol_cmds_coalesced = 1;
2020
2021         /* Initialize the SCSI request */
2022         if (sksgio->sg.dxfer_direction != SG_DXFER_NONE)
2023                 scsi_req->hdr.sg_list_dma_address =
2024                         cpu_to_be64(skspcl->req.sksg_dma_address);
2025         scsi_req->hdr.tag = skspcl->req.id;
2026         scsi_req->hdr.sg_list_len_bytes =
2027                 cpu_to_be32(skspcl->req.sg_byte_count);
2028         memcpy(scsi_req->cdb, sksgio->cdb, sizeof(scsi_req->cdb));
2029
2030         skspcl->req.state = SKD_REQ_STATE_BUSY;
2031         skd_send_special_fitmsg(skdev, skspcl);
2032
2033         return 0;
2034 }
2035
2036 static int skd_sg_io_await(struct skd_device *skdev, struct skd_sg_io *sksgio)
2037 {
2038         unsigned long flags;
2039         int rc;
2040
2041         rc = wait_event_interruptible_timeout(skdev->waitq,
2042                                               (sksgio->skspcl->req.state !=
2043                                                SKD_REQ_STATE_BUSY),
2044                                               msecs_to_jiffies(sksgio->sg.
2045                                                                timeout));
2046
2047         spin_lock_irqsave(&skdev->lock, flags);
2048
2049         if (sksgio->skspcl->req.state == SKD_REQ_STATE_ABORTED) {
2050                 DPRINTK(skdev, "skspcl %p aborted\n", sksgio->skspcl);
2051
2052                 /* Build check cond, sense and let command finish. */
2053                 /* For a timeout, we must fabricate completion and sense
2054                  * data to complete the command */
2055                 sksgio->skspcl->req.completion.status =
2056                         SAM_STAT_CHECK_CONDITION;
2057
2058                 memset(&sksgio->skspcl->req.err_info, 0,
2059                        sizeof(sksgio->skspcl->req.err_info));
2060                 sksgio->skspcl->req.err_info.type = 0x70;
2061                 sksgio->skspcl->req.err_info.key = ABORTED_COMMAND;
2062                 sksgio->skspcl->req.err_info.code = 0x44;
2063                 sksgio->skspcl->req.err_info.qual = 0;
2064                 rc = 0;
2065         } else if (sksgio->skspcl->req.state != SKD_REQ_STATE_BUSY)
2066                 /* No longer on the adapter. We finish. */
2067                 rc = 0;
2068         else {
2069                 /* Something's gone wrong. Still busy. Timeout or
2070                  * user interrupted (control-C). Mark as an orphan
2071                  * so it will be disposed when completed. */
2072                 sksgio->skspcl->orphaned = 1;
2073                 sksgio->skspcl = NULL;
2074                 if (rc == 0) {
2075                         DPRINTK(skdev, "timed out %p (%u ms)\n", sksgio,
2076                                 sksgio->sg.timeout);
2077                         rc = -ETIMEDOUT;
2078                 } else {
2079                         DPRINTK(skdev, "cntlc %p\n", sksgio);
2080                         rc = -EINTR;
2081                 }
2082         }
2083
2084         spin_unlock_irqrestore(&skdev->lock, flags);
2085
2086         return rc;
2087 }
2088
2089 static int skd_sg_io_put_status(struct skd_device *skdev,
2090                                 struct skd_sg_io *sksgio)
2091 {
2092         struct sg_io_hdr *sgp = &sksgio->sg;
2093         struct skd_special_context *skspcl = sksgio->skspcl;
2094         int resid = 0;
2095
2096         u32 nb = be32_to_cpu(skspcl->req.completion.num_returned_bytes);
2097
2098         sgp->status = skspcl->req.completion.status;
2099         resid = sksgio->dxfer_len - nb;
2100
2101         sgp->masked_status = sgp->status & STATUS_MASK;
2102         sgp->msg_status = 0;
2103         sgp->host_status = 0;
2104         sgp->driver_status = 0;
2105         sgp->resid = resid;
2106         if (sgp->masked_status || sgp->host_status || sgp->driver_status)
2107                 sgp->info |= SG_INFO_CHECK;
2108
2109         DPRINTK(skdev, "status %x masked %x resid 0x%x\n", sgp->status,
2110                 sgp->masked_status, sgp->resid);
2111
2112         if (sgp->masked_status == SAM_STAT_CHECK_CONDITION) {
2113                 if (sgp->mx_sb_len > 0) {
2114                         struct fit_comp_error_info *ei = &skspcl->req.err_info;
2115                         u32 nbytes = sizeof(*ei);
2116
2117                         nbytes = min_t(u32, nbytes, sgp->mx_sb_len);
2118
2119                         sgp->sb_len_wr = nbytes;
2120
2121                         if (__copy_to_user(sgp->sbp, ei, nbytes)) {
2122                                 DPRINTK(skdev, "copy_to_user sense failed %p\n",
2123                                         sgp->sbp);
2124                                 return -EFAULT;
2125                         }
2126                 }
2127         }
2128
2129         if (__copy_to_user(sksgio->argp, sgp, sizeof(sg_io_hdr_t))) {
2130                 DPRINTK(skdev, "copy_to_user sg failed %p\n", sksgio->argp);
2131                 return -EFAULT;
2132         }
2133
2134         return 0;
2135 }
2136
2137 static int skd_sg_io_release_skspcl(struct skd_device *skdev,
2138                                     struct skd_sg_io *sksgio)
2139 {
2140         struct skd_special_context *skspcl = sksgio->skspcl;
2141
2142         if (skspcl != NULL) {
2143                 ulong flags;
2144
2145                 sksgio->skspcl = NULL;
2146
2147                 spin_lock_irqsave(&skdev->lock, flags);
2148                 skd_release_special(skdev, skspcl);
2149                 spin_unlock_irqrestore(&skdev->lock, flags);
2150         }
2151
2152         return 0;
2153 }
2154
2155 /*
2156  *****************************************************************************
2157  * INTERNAL REQUESTS -- generated by driver itself
2158  *****************************************************************************
2159  */
2160
2161 static int skd_format_internal_skspcl(struct skd_device *skdev)
2162 {
2163         struct skd_special_context *skspcl = &skdev->internal_skspcl;
2164         struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
2165         struct fit_msg_hdr *fmh;
2166         uint64_t dma_address;
2167         struct skd_scsi_request *scsi;
2168
2169         fmh = (struct fit_msg_hdr *)&skspcl->msg_buf[0];
2170         fmh->protocol_id = FIT_PROTOCOL_ID_SOFIT;
2171         fmh->num_protocol_cmds_coalesced = 1;
2172
2173         scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
2174         memset(scsi, 0, sizeof(*scsi));
2175         dma_address = skspcl->req.sksg_dma_address;
2176         scsi->hdr.sg_list_dma_address = cpu_to_be64(dma_address);
2177         sgd->control = FIT_SGD_CONTROL_LAST;
2178         sgd->byte_count = 0;
2179         sgd->host_side_addr = skspcl->db_dma_address;
2180         sgd->dev_side_addr = 0;
2181         sgd->next_desc_ptr = 0LL;
2182
2183         return 1;
2184 }
2185
2186 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
2187
2188 static void skd_send_internal_skspcl(struct skd_device *skdev,
2189                                      struct skd_special_context *skspcl,
2190                                      u8 opcode)
2191 {
2192         struct fit_sg_descriptor *sgd = &skspcl->req.sksg_list[0];
2193         struct skd_scsi_request *scsi;
2194         unsigned char *buf = skspcl->data_buf;
2195         int i;
2196
2197         if (skspcl->req.state != SKD_REQ_STATE_IDLE)
2198                 /*
2199                  * A refresh is already in progress.
2200                  * Just wait for it to finish.
2201                  */
2202                 return;
2203
2204         SKD_ASSERT((skspcl->req.id & SKD_ID_INCR) == 0);
2205         skspcl->req.state = SKD_REQ_STATE_BUSY;
2206         skspcl->req.id += SKD_ID_INCR;
2207
2208         scsi = (struct skd_scsi_request *)&skspcl->msg_buf[64];
2209         scsi->hdr.tag = skspcl->req.id;
2210
2211         memset(scsi->cdb, 0, sizeof(scsi->cdb));
2212
2213         switch (opcode) {
2214         case TEST_UNIT_READY:
2215                 scsi->cdb[0] = TEST_UNIT_READY;
2216                 sgd->byte_count = 0;
2217                 scsi->hdr.sg_list_len_bytes = 0;
2218                 break;
2219
2220         case READ_CAPACITY:
2221                 scsi->cdb[0] = READ_CAPACITY;
2222                 sgd->byte_count = SKD_N_READ_CAP_BYTES;
2223                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
2224                 break;
2225
2226         case INQUIRY:
2227                 scsi->cdb[0] = INQUIRY;
2228                 scsi->cdb[1] = 0x01;    /* evpd */
2229                 scsi->cdb[2] = 0x80;    /* serial number page */
2230                 scsi->cdb[4] = 0x10;
2231                 sgd->byte_count = 16;
2232                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
2233                 break;
2234
2235         case SYNCHRONIZE_CACHE:
2236                 scsi->cdb[0] = SYNCHRONIZE_CACHE;
2237                 sgd->byte_count = 0;
2238                 scsi->hdr.sg_list_len_bytes = 0;
2239                 break;
2240
2241         case WRITE_BUFFER:
2242                 scsi->cdb[0] = WRITE_BUFFER;
2243                 scsi->cdb[1] = 0x02;
2244                 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
2245                 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
2246                 sgd->byte_count = WR_BUF_SIZE;
2247                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
2248                 /* fill incrementing byte pattern */
2249                 for (i = 0; i < sgd->byte_count; i++)
2250                         buf[i] = i & 0xFF;
2251                 break;
2252
2253         case READ_BUFFER:
2254                 scsi->cdb[0] = READ_BUFFER;
2255                 scsi->cdb[1] = 0x02;
2256                 scsi->cdb[7] = (WR_BUF_SIZE & 0xFF00) >> 8;
2257                 scsi->cdb[8] = WR_BUF_SIZE & 0xFF;
2258                 sgd->byte_count = WR_BUF_SIZE;
2259                 scsi->hdr.sg_list_len_bytes = cpu_to_be32(sgd->byte_count);
2260                 memset(skspcl->data_buf, 0, sgd->byte_count);
2261                 break;
2262
2263         default:
2264                 SKD_ASSERT("Don't know what to send");
2265                 return;
2266
2267         }
2268         skd_send_special_fitmsg(skdev, skspcl);
2269 }
2270
2271 static void skd_refresh_device_data(struct skd_device *skdev)
2272 {
2273         struct skd_special_context *skspcl = &skdev->internal_skspcl;
2274
2275         skd_send_internal_skspcl(skdev, skspcl, TEST_UNIT_READY);
2276 }
2277
2278 static int skd_chk_read_buf(struct skd_device *skdev,
2279                             struct skd_special_context *skspcl)
2280 {
2281         unsigned char *buf = skspcl->data_buf;
2282         int i;
2283
2284         /* check for incrementing byte pattern */
2285         for (i = 0; i < WR_BUF_SIZE; i++)
2286                 if (buf[i] != (i & 0xFF))
2287                         return 1;
2288
2289         return 0;
2290 }
2291
2292 static void skd_log_check_status(struct skd_device *skdev, u8 status, u8 key,
2293                                  u8 code, u8 qual, u8 fruc)
2294 {
2295         /* If the check condition is of special interest, log a message */
2296         if ((status == SAM_STAT_CHECK_CONDITION) && (key == 0x02)
2297             && (code == 0x04) && (qual == 0x06)) {
2298                 pr_err("(%s): *** LOST_WRITE_DATA ERROR *** key/asc/"
2299                        "ascq/fruc %02x/%02x/%02x/%02x\n",
2300                        skd_name(skdev), key, code, qual, fruc);
2301         }
2302 }
2303
2304 static void skd_complete_internal(struct skd_device *skdev,
2305                                   volatile struct fit_completion_entry_v1
2306                                   *skcomp,
2307                                   volatile struct fit_comp_error_info *skerr,
2308                                   struct skd_special_context *skspcl)
2309 {
2310         u8 *buf = skspcl->data_buf;
2311         u8 status;
2312         int i;
2313         struct skd_scsi_request *scsi =
2314                 (struct skd_scsi_request *)&skspcl->msg_buf[64];
2315
2316         SKD_ASSERT(skspcl == &skdev->internal_skspcl);
2317
2318         DPRINTK(skdev, "complete internal %x\n", scsi->cdb[0]);
2319
2320         skspcl->req.completion = *skcomp;
2321         skspcl->req.state = SKD_REQ_STATE_IDLE;
2322         skspcl->req.id += SKD_ID_INCR;
2323
2324         status = skspcl->req.completion.status;
2325
2326         skd_log_check_status(skdev, status, skerr->key, skerr->code,
2327                              skerr->qual, skerr->fruc);
2328
2329         switch (scsi->cdb[0]) {
2330         case TEST_UNIT_READY:
2331                 if (status == SAM_STAT_GOOD)
2332                         skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2333                 else if ((status == SAM_STAT_CHECK_CONDITION) &&
2334                          (skerr->key == MEDIUM_ERROR))
2335                         skd_send_internal_skspcl(skdev, skspcl, WRITE_BUFFER);
2336                 else {
2337                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2338                                 VPRINTK(skdev, "TUR failed, don't send anymore"
2339                                         "state 0x%x\n", skdev->state);
2340                                 return;
2341                         }
2342                         DPRINTK(skdev, "**** TUR failed, retry skerr\n");
2343                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
2344                 }
2345                 break;
2346
2347         case WRITE_BUFFER:
2348                 if (status == SAM_STAT_GOOD)
2349                         skd_send_internal_skspcl(skdev, skspcl, READ_BUFFER);
2350                 else {
2351                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2352                                 VPRINTK(skdev, "write buffer failed, don't send"
2353                                         " anymore state 0x%x\n", skdev->state);
2354                                 return;
2355                         }
2356                         DPRINTK(skdev,
2357                                 "**** write buffer failed, retry skerr\n");
2358                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
2359                 }
2360                 break;
2361
2362         case READ_BUFFER:
2363                 if (status == SAM_STAT_GOOD) {
2364                         if (skd_chk_read_buf(skdev, skspcl) == 0)
2365                                 skd_send_internal_skspcl(skdev, skspcl,
2366                                                          READ_CAPACITY);
2367                         else {
2368                                 pr_err(
2369                                        "(%s):*** W/R Buffer mismatch %d ***\n",
2370                                        skd_name(skdev), skdev->connect_retries);
2371                                 if (skdev->connect_retries <
2372                                     SKD_MAX_CONNECT_RETRIES) {
2373                                         skdev->connect_retries++;
2374                                         skd_soft_reset(skdev);
2375                                 } else {
2376                                         pr_err(
2377                                                "(%s): W/R Buffer Connect Error\n",
2378                                                skd_name(skdev));
2379                                         return;
2380                                 }
2381                         }
2382
2383                 } else {
2384                         if (skdev->state == SKD_DRVR_STATE_STOPPING) {
2385                                 VPRINTK(skdev,
2386                                         "read buffer failed, don't send anymore"
2387                                         "state 0x%x\n", skdev->state);
2388                                 return;
2389                         }
2390                         DPRINTK(skdev,
2391                                 "**** read buffer failed, retry skerr\n");
2392                         skd_send_internal_skspcl(skdev, skspcl, 0x00);
2393                 }
2394                 break;
2395
2396         case READ_CAPACITY:
2397                 skdev->read_cap_is_valid = 0;
2398                 if (status == SAM_STAT_GOOD) {
2399                         skdev->read_cap_last_lba =
2400                                 (buf[0] << 24) | (buf[1] << 16) |
2401                                 (buf[2] << 8) | buf[3];
2402                         skdev->read_cap_blocksize =
2403                                 (buf[4] << 24) | (buf[5] << 16) |
2404                                 (buf[6] << 8) | buf[7];
2405
2406                         DPRINTK(skdev, "last lba %d, bs %d\n",
2407                                 skdev->read_cap_last_lba,
2408                                 skdev->read_cap_blocksize);
2409
2410                         set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2411
2412                         skdev->read_cap_is_valid = 1;
2413
2414                         skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2415                 } else if ((status == SAM_STAT_CHECK_CONDITION) &&
2416                            (skerr->key == MEDIUM_ERROR)) {
2417                         skdev->read_cap_last_lba = ~0;
2418                         set_capacity(skdev->disk, skdev->read_cap_last_lba + 1);
2419                         DPRINTK(skdev,
2420                                 "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
2421                         skd_send_internal_skspcl(skdev, skspcl, INQUIRY);
2422                 } else {
2423                         DPRINTK(skdev, "**** READCAP failed, retry TUR\n");
2424                         skd_send_internal_skspcl(skdev, skspcl,
2425                                                  TEST_UNIT_READY);
2426                 }
2427                 break;
2428
2429         case INQUIRY:
2430                 skdev->inquiry_is_valid = 0;
2431                 if (status == SAM_STAT_GOOD) {
2432                         skdev->inquiry_is_valid = 1;
2433
2434                         for (i = 0; i < 12; i++)
2435                                 skdev->inq_serial_num[i] = buf[i + 4];
2436                         skdev->inq_serial_num[12] = 0;
2437                 }
2438
2439                 if (skd_unquiesce_dev(skdev) < 0)
2440                         DPRINTK(skdev, "**** failed, to ONLINE device\n");
2441                  /* connection is complete */
2442                 skdev->connect_retries = 0;
2443                 break;
2444
2445         case SYNCHRONIZE_CACHE:
2446                 if (status == SAM_STAT_GOOD)
2447                         skdev->sync_done = 1;
2448                 else
2449                         skdev->sync_done = -1;
2450                 wake_up_interruptible(&skdev->waitq);
2451                 break;
2452
2453         default:
2454                 SKD_ASSERT("we didn't send this");
2455         }
2456 }
2457
2458 /*
2459  *****************************************************************************
2460  * FIT MESSAGES
2461  *****************************************************************************
2462  */
2463
2464 static void skd_send_fitmsg(struct skd_device *skdev,
2465                             struct skd_fitmsg_context *skmsg)
2466 {
2467         u64 qcmd;
2468         struct fit_msg_hdr *fmh;
2469
2470         VPRINTK(skdev, "dma address 0x%llx, busy=%d\n",
2471                 skmsg->mb_dma_address, skdev->in_flight);
2472         VPRINTK(skdev, "msg_buf 0x%p, offset %x\n",
2473                 skmsg->msg_buf, skmsg->offset);
2474
2475         qcmd = skmsg->mb_dma_address;
2476         qcmd |= FIT_QCMD_QID_NORMAL;
2477
2478         fmh = (struct fit_msg_hdr *)skmsg->msg_buf;
2479         skmsg->outstanding = fmh->num_protocol_cmds_coalesced;
2480
2481         if (unlikely(skdev->dbg_level > 1)) {
2482                 u8 *bp = (u8 *)skmsg->msg_buf;
2483                 int i;
2484                 for (i = 0; i < skmsg->length; i += 8) {
2485                         VPRINTK(skdev, "  msg[%2d] %02x %02x %02x %02x "
2486                                 "%02x %02x %02x %02x\n",
2487                                 i, bp[i + 0], bp[i + 1], bp[i + 2],
2488                                 bp[i + 3], bp[i + 4], bp[i + 5],
2489                                 bp[i + 6], bp[i + 7]);
2490                         if (i == 0)
2491                                 i = 64 - 8;
2492                 }
2493         }
2494
2495         if (skmsg->length > 256)
2496                 qcmd |= FIT_QCMD_MSGSIZE_512;
2497         else if (skmsg->length > 128)
2498                 qcmd |= FIT_QCMD_MSGSIZE_256;
2499         else if (skmsg->length > 64)
2500                 qcmd |= FIT_QCMD_MSGSIZE_128;
2501         else
2502                 /*
2503                  * This makes no sense because the FIT msg header is
2504                  * 64 bytes. If the msg is only 64 bytes long it has
2505                  * no payload.
2506                  */
2507                 qcmd |= FIT_QCMD_MSGSIZE_64;
2508
2509         SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2510
2511 }
2512
2513 static void skd_send_special_fitmsg(struct skd_device *skdev,
2514                                     struct skd_special_context *skspcl)
2515 {
2516         u64 qcmd;
2517
2518         if (unlikely(skdev->dbg_level > 1)) {
2519                 u8 *bp = (u8 *)skspcl->msg_buf;
2520                 int i;
2521
2522                 for (i = 0; i < SKD_N_SPECIAL_FITMSG_BYTES; i += 8) {
2523                         VPRINTK(skdev,
2524                                 "  spcl[%2d] %02x %02x %02x %02x  "
2525                                 "%02x %02x %02x %02x\n", i,
2526                                 bp[i + 0], bp[i + 1], bp[i + 2], bp[i + 3],
2527                                 bp[i + 4], bp[i + 5], bp[i + 6], bp[i + 7]);
2528                         if (i == 0)
2529                                 i = 64 - 8;
2530                 }
2531
2532                 VPRINTK(skdev, "skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
2533                         skspcl, skspcl->req.id, skspcl->req.sksg_list,
2534                         skspcl->req.sksg_dma_address);
2535                 for (i = 0; i < skspcl->req.n_sg; i++) {
2536                         struct fit_sg_descriptor *sgd =
2537                                 &skspcl->req.sksg_list[i];
2538
2539                         VPRINTK(skdev, "  sg[%d] count=%u ctrl=0x%x "
2540                                 "addr=0x%llx next=0x%llx\n",
2541                                 i, sgd->byte_count, sgd->control,
2542                                 sgd->host_side_addr, sgd->next_desc_ptr);
2543                 }
2544         }
2545
2546         /*
2547          * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
2548          * and one 64-byte SSDI command.
2549          */
2550         qcmd = skspcl->mb_dma_address;
2551         qcmd |= FIT_QCMD_QID_NORMAL + FIT_QCMD_MSGSIZE_128;
2552
2553         SKD_WRITEQ(skdev, qcmd, FIT_Q_COMMAND);
2554 }
2555
2556 /*
2557  *****************************************************************************
2558  * COMPLETION QUEUE
2559  *****************************************************************************
2560  */
2561
2562 static void skd_complete_other(struct skd_device *skdev,
2563                                volatile struct fit_completion_entry_v1 *skcomp,
2564                                volatile struct fit_comp_error_info *skerr);
2565
2566
2567 static void skd_requeue_request(struct skd_device *skdev,
2568                                 struct skd_request_context *skreq);
2569
2570 struct sns_info {
2571         u8 type;
2572         u8 stat;
2573         u8 key;
2574         u8 asc;
2575         u8 ascq;
2576         u8 mask;
2577         enum skd_check_status_action action;
2578 };
2579
2580 static struct sns_info skd_chkstat_table[] = {
2581         /* Good */
2582         { 0x70, 0x02, RECOVERED_ERROR, 0,    0,    0x1c,
2583           SKD_CHECK_STATUS_REPORT_GOOD },
2584
2585         /* Smart alerts */
2586         { 0x70, 0x02, NO_SENSE,        0x0B, 0x00, 0x1E,        /* warnings */
2587           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2588         { 0x70, 0x02, NO_SENSE,        0x5D, 0x00, 0x1E,        /* thresholds */
2589           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2590         { 0x70, 0x02, RECOVERED_ERROR, 0x0B, 0x01, 0x1F,        /* temperature over trigger */
2591           SKD_CHECK_STATUS_REPORT_SMART_ALERT },
2592
2593         /* Retry (with limits) */
2594         { 0x70, 0x02, 0x0B,            0,    0,    0x1C,        /* This one is for DMA ERROR */
2595           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2596         { 0x70, 0x02, 0x06,            0x0B, 0x00, 0x1E,        /* warnings */
2597           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2598         { 0x70, 0x02, 0x06,            0x5D, 0x00, 0x1E,        /* thresholds */
2599           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2600         { 0x70, 0x02, 0x06,            0x80, 0x30, 0x1F,        /* backup power */
2601           SKD_CHECK_STATUS_REQUEUE_REQUEST },
2602
2603         /* Busy (or about to be) */
2604         { 0x70, 0x02, 0x06,            0x3f, 0x01, 0x1F, /* fw changed */
2605           SKD_CHECK_STATUS_BUSY_IMMINENT },
2606 };
2607
2608 /*
2609  * Look up status and sense data to decide how to handle the error
2610  * from the device.
2611  * mask says which fields must match e.g., mask=0x18 means check
2612  * type and stat, ignore key, asc, ascq.
2613  */
2614
2615 static enum skd_check_status_action skd_check_status(struct skd_device *skdev,
2616                                 u8 cmp_status,
2617                                 volatile struct fit_comp_error_info *skerr)
2618 {
2619         int i, n;
2620
2621         pr_err("(%s): key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
2622                skd_name(skdev), skerr->key, skerr->code, skerr->qual,
2623                skerr->fruc);
2624
2625         VPRINTK(skdev, "stat: t=%02x stat=%02x k=%02x c=%02x q=%02x "
2626                 "fruc=%02x\n", skerr->type, cmp_status, skerr->key,
2627                 skerr->code, skerr->qual, skerr->fruc);
2628
2629         /* Does the info match an entry in the good category? */
2630         n = sizeof(skd_chkstat_table) / sizeof(skd_chkstat_table[0]);
2631         for (i = 0; i < n; i++) {
2632                 struct sns_info *sns = &skd_chkstat_table[i];
2633
2634                 if (sns->mask & 0x10)
2635                         if (skerr->type != sns->type)
2636                                 continue;
2637
2638                 if (sns->mask & 0x08)
2639                         if (cmp_status != sns->stat)
2640                                 continue;
2641
2642                 if (sns->mask & 0x04)
2643                         if (skerr->key != sns->key)
2644                                 continue;
2645
2646                 if (sns->mask & 0x02)
2647                         if (skerr->code != sns->asc)
2648                                 continue;
2649
2650                 if (sns->mask & 0x01)
2651                         if (skerr->qual != sns->ascq)
2652                                 continue;
2653
2654                 if (sns->action == SKD_CHECK_STATUS_REPORT_SMART_ALERT) {
2655                         pr_err("(%s): SMART Alert: sense key/asc/ascq "
2656                                "%02x/%02x/%02x\n",
2657                                skd_name(skdev), skerr->key,
2658                                skerr->code, skerr->qual);
2659                 }
2660                 return sns->action;
2661         }
2662
2663         /* No other match, so nonzero status means error,
2664          * zero status means good
2665          */
2666         if (cmp_status) {
2667                 DPRINTK(skdev, "status check: error\n");
2668                 return SKD_CHECK_STATUS_REPORT_ERROR;
2669         }
2670
2671         DPRINTK(skdev, "status check good default\n");
2672         return SKD_CHECK_STATUS_REPORT_GOOD;
2673 }
2674
2675 static void skd_resolve_req_exception(struct skd_device *skdev,
2676                                       struct skd_request_context *skreq)
2677 {
2678         u8 cmp_status = skreq->completion.status;
2679
2680         switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {
2681         case SKD_CHECK_STATUS_REPORT_GOOD:
2682         case SKD_CHECK_STATUS_REPORT_SMART_ALERT:
2683                 skd_end_request(skdev, skreq, 0);
2684                 break;
2685
2686         case SKD_CHECK_STATUS_BUSY_IMMINENT:
2687                 skd_log_skreq(skdev, skreq, "retry(busy)");
2688                 skd_requeue_request(skdev, skreq);
2689                 pr_info("(%s) drive BUSY imminent\n", skd_name(skdev));
2690                 skdev->state = SKD_DRVR_STATE_BUSY_IMMINENT;
2691                 skdev->timer_countdown = SKD_TIMER_MINUTES(20);
2692                 skd_quiesce_dev(skdev);
2693                 break;
2694
2695         case SKD_CHECK_STATUS_REQUEUE_REQUEST:
2696                 if (!skd_bio) {
2697                         if ((unsigned long) ++skreq->req->special <
2698                             SKD_MAX_RETRIES) {
2699                                 skd_log_skreq(skdev, skreq, "retry");
2700                                 skd_requeue_request(skdev, skreq);
2701                                 break;
2702                         }
2703                 }
2704         /* fall through to report error */
2705
2706         case SKD_CHECK_STATUS_REPORT_ERROR:
2707         default:
2708                 skd_end_request(skdev, skreq, -EIO);
2709                 break;
2710         }
2711 }
2712
2713 static void skd_requeue_request(struct skd_device *skdev,
2714                                 struct skd_request_context *skreq)
2715 {
2716         if (!skd_bio) {
2717                 blk_requeue_request(skdev->queue, skreq->req);
2718         } else {
2719                 bio_list_add_head(&skdev->bio_queue, skreq->bio);
2720                 skreq->bio = NULL;
2721         }
2722 }
2723
2724
2725
2726 /* assume spinlock is already held */
2727 static void skd_release_skreq(struct skd_device *skdev,
2728                               struct skd_request_context *skreq)
2729 {
2730         u32 msg_slot;
2731         struct skd_fitmsg_context *skmsg;
2732
2733         u32 timo_slot;
2734
2735         /*
2736          * Reclaim the FIT msg buffer if this is
2737          * the first of the requests it carried to
2738          * be completed. The FIT msg buffer used to
2739          * send this request cannot be reused until
2740          * we are sure the s1120 card has copied
2741          * it to its memory. The FIT msg might have
2742          * contained several requests. As soon as
2743          * any of them are completed we know that
2744          * the entire FIT msg was transferred.
2745          * Only the first completed request will
2746          * match the FIT msg buffer id. The FIT
2747          * msg buffer id is immediately updated.
2748          * When subsequent requests complete the FIT
2749          * msg buffer id won't match, so we know
2750          * quite cheaply that it is already done.
2751          */
2752         msg_slot = skreq->fitmsg_id & SKD_ID_SLOT_MASK;
2753         SKD_ASSERT(msg_slot < skdev->num_fitmsg_context);
2754
2755         skmsg = &skdev->skmsg_table[msg_slot];
2756         if (skmsg->id == skreq->fitmsg_id) {
2757                 SKD_ASSERT(skmsg->state == SKD_MSG_STATE_BUSY);
2758                 SKD_ASSERT(skmsg->outstanding > 0);
2759                 skmsg->outstanding--;
2760                 if (skmsg->outstanding == 0) {
2761                         skmsg->state = SKD_MSG_STATE_IDLE;
2762                         skmsg->id += SKD_ID_INCR;
2763                         skmsg->next = skdev->skmsg_free_list;
2764                         skdev->skmsg_free_list = skmsg;
2765                 }
2766         }
2767
2768         /*
2769          * Decrease the number of active requests.
2770          * Also decrements the count in the timeout slot.
2771          */
2772         SKD_ASSERT(skdev->in_flight > 0);
2773         skdev->in_flight -= 1;
2774
2775         timo_slot = skreq->timeout_stamp & SKD_TIMEOUT_SLOT_MASK;
2776         SKD_ASSERT(skdev->timeout_slot[timo_slot] > 0);
2777         skdev->timeout_slot[timo_slot] -= 1;
2778
2779         /*
2780          * Reset backpointer
2781          */
2782         if (likely(!skd_bio))
2783                 skreq->req = NULL;
2784         else
2785                 skreq->bio = NULL;
2786
2787
2788         /*
2789          * Reclaim the skd_request_context
2790          */
2791         skreq->state = SKD_REQ_STATE_IDLE;
2792         skreq->id += SKD_ID_INCR;
2793         skreq->next = skdev->skreq_free_list;
2794         skdev->skreq_free_list = skreq;
2795 }
2796
2797 #define DRIVER_INQ_EVPD_PAGE_CODE   0xDA
2798
2799 static void skd_do_inq_page_00(struct skd_device *skdev,
2800                                volatile struct fit_completion_entry_v1 *skcomp,
2801                                volatile struct fit_comp_error_info *skerr,
2802                                uint8_t *cdb, uint8_t *buf)
2803 {
2804         uint16_t insert_pt, max_bytes, drive_pages, drive_bytes, new_size;
2805
2806         /* Caller requested "supported pages".  The driver needs to insert
2807          * its page.
2808          */
2809         VPRINTK(skdev, "skd_do_driver_inquiry: modify supported pages.\n");
2810
2811         /* If the device rejected the request because the CDB was
2812          * improperly formed, then just leave.
2813          */
2814         if (skcomp->status == SAM_STAT_CHECK_CONDITION &&
2815             skerr->key == ILLEGAL_REQUEST && skerr->code == 0x24)
2816                 return;
2817
2818         /* Get the amount of space the caller allocated */
2819         max_bytes = (cdb[3] << 8) | cdb[4];
2820
2821         /* Get the number of pages actually returned by the device */
2822         drive_pages = (buf[2] << 8) | buf[3];
2823         drive_bytes = drive_pages + 4;
2824         new_size = drive_pages + 1;
2825
2826         /* Supported pages must be in numerical order, so find where
2827          * the driver page needs to be inserted into the list of
2828          * pages returned by the device.
2829          */
2830         for (insert_pt = 4; insert_pt < drive_bytes; insert_pt++) {
2831                 if (buf[insert_pt] == DRIVER_INQ_EVPD_PAGE_CODE)
2832                         return; /* Device using this page code. abort */
2833                 else if (buf[insert_pt] > DRIVER_INQ_EVPD_PAGE_CODE)
2834                         break;
2835         }
2836
2837         if (insert_pt < max_bytes) {
2838                 uint16_t u;
2839
2840                 /* Shift everything up one byte to make room. */
2841                 for (u = new_size + 3; u > insert_pt; u--)
2842                         buf[u] = buf[u - 1];
2843                 buf[insert_pt] = DRIVER_INQ_EVPD_PAGE_CODE;
2844
2845                 /* SCSI byte order increment of num_returned_bytes by 1 */
2846                 skcomp->num_returned_bytes =
2847                         be32_to_cpu(skcomp->num_returned_bytes) + 1;
2848                 skcomp->num_returned_bytes =
2849                         be32_to_cpu(skcomp->num_returned_bytes);
2850         }
2851
2852         /* update page length field to reflect the driver's page too */
2853         buf[2] = (uint8_t)((new_size >> 8) & 0xFF);
2854         buf[3] = (uint8_t)((new_size >> 0) & 0xFF);
2855 }
2856
2857 static void skd_get_link_info(struct pci_dev *pdev, u8 *speed, u8 *width)
2858 {
2859         int pcie_reg;
2860         u16 pci_bus_speed;
2861         u8 pci_lanes;
2862
2863         pcie_reg = pci_find_capability(pdev, PCI_CAP_ID_EXP);
2864         if (pcie_reg) {
2865                 u16 linksta;
2866                 pci_read_config_word(pdev, pcie_reg + PCI_EXP_LNKSTA, &linksta);
2867
2868                 pci_bus_speed = linksta & 0xF;
2869                 pci_lanes = (linksta & 0x3F0) >> 4;
2870         } else {
2871                 *speed = STEC_LINK_UNKNOWN;
2872                 *width = 0xFF;
2873                 return;
2874         }
2875
2876         switch (pci_bus_speed) {
2877         case 1:
2878                 *speed = STEC_LINK_2_5GTS;
2879                 break;
2880         case 2:
2881                 *speed = STEC_LINK_5GTS;
2882                 break;
2883         case 3:
2884                 *speed = STEC_LINK_8GTS;
2885                 break;
2886         default:
2887                 *speed = STEC_LINK_UNKNOWN;
2888                 break;
2889         }
2890
2891         if (pci_lanes <= 0x20)
2892                 *width = pci_lanes;
2893         else
2894                 *width = 0xFF;
2895 }
2896
2897 static void skd_do_inq_page_da(struct skd_device *skdev,
2898                                volatile struct fit_completion_entry_v1 *skcomp,
2899                                volatile struct fit_comp_error_info *skerr,
2900                                uint8_t *cdb, uint8_t *buf)
2901 {
2902         unsigned max_bytes;
2903         struct driver_inquiry_data inq;
2904         u16 val;
2905
2906         VPRINTK(skdev, "skd_do_driver_inquiry: return driver page\n");
2907
2908         memset(&inq, 0, sizeof(inq));
2909
2910         inq.page_code = DRIVER_INQ_EVPD_PAGE_CODE;
2911
2912         if (skdev->pdev && skdev->pdev->bus) {
2913                 skd_get_link_info(skdev->pdev,
2914                                   &inq.pcie_link_speed, &inq.pcie_link_lanes);
2915                 inq.pcie_bus_number = cpu_to_be16(skdev->pdev->bus->number);
2916                 inq.pcie_device_number = PCI_SLOT(skdev->pdev->devfn);
2917                 inq.pcie_function_number = PCI_FUNC(skdev->pdev->devfn);
2918
2919                 pci_read_config_word(skdev->pdev, PCI_VENDOR_ID, &val);
2920                 inq.pcie_vendor_id = cpu_to_be16(val);
2921
2922                 pci_read_config_word(skdev->pdev, PCI_DEVICE_ID, &val);
2923                 inq.pcie_device_id = cpu_to_be16(val);
2924
2925                 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_VENDOR_ID,
2926                                      &val);
2927                 inq.pcie_subsystem_vendor_id = cpu_to_be16(val);
2928
2929                 pci_read_config_word(skdev->pdev, PCI_SUBSYSTEM_ID, &val);
2930                 inq.pcie_subsystem_device_id = cpu_to_be16(val);
2931         } else {
2932                 inq.pcie_bus_number = 0xFFFF;
2933                 inq.pcie_device_number = 0xFF;
2934                 inq.pcie_function_number = 0xFF;
2935                 inq.pcie_link_speed = 0xFF;
2936                 inq.pcie_link_lanes = 0xFF;
2937                 inq.pcie_vendor_id = 0xFFFF;
2938                 inq.pcie_device_id = 0xFFFF;
2939                 inq.pcie_subsystem_vendor_id = 0xFFFF;
2940                 inq.pcie_subsystem_device_id = 0xFFFF;
2941         }
2942
2943         /* Driver version, fixed lenth, padded with spaces on the right */
2944         inq.driver_version_length = sizeof(inq.driver_version);
2945         memset(&inq.driver_version, ' ', sizeof(inq.driver_version));
2946         memcpy(inq.driver_version, DRV_VER_COMPL,
2947                min(sizeof(inq.driver_version), strlen(DRV_VER_COMPL)));
2948
2949         inq.page_length = cpu_to_be16((sizeof(inq) - 4));
2950
2951         /* Clear the error set by the device */
2952         skcomp->status = SAM_STAT_GOOD;
2953         memset((void *)skerr, 0, sizeof(*skerr));
2954
2955         /* copy response into output buffer */
2956         max_bytes = (cdb[3] << 8) | cdb[4];
2957         memcpy(buf, &inq, min_t(unsigned, max_bytes, sizeof(inq)));
2958
2959         skcomp->num_returned_bytes =
2960                 be32_to_cpu(min_t(uint16_t, max_bytes, sizeof(inq)));
2961 }
2962
2963 static void skd_do_driver_inq(struct skd_device *skdev,
2964                               volatile struct fit_completion_entry_v1 *skcomp,
2965                               volatile struct fit_comp_error_info *skerr,
2966                               uint8_t *cdb, uint8_t *buf)
2967 {
2968         if (!buf)
2969                 return;
2970         else if (cdb[0] != INQUIRY)
2971                 return;         /* Not an INQUIRY */
2972         else if ((cdb[1] & 1) == 0)
2973                 return;         /* EVPD not set */
2974         else if (cdb[2] == 0)
2975                 /* Need to add driver's page to supported pages list */
2976                 skd_do_inq_page_00(skdev, skcomp, skerr, cdb, buf);
2977         else if (cdb[2] == DRIVER_INQ_EVPD_PAGE_CODE)
2978                 /* Caller requested driver's page */
2979                 skd_do_inq_page_da(skdev, skcomp, skerr, cdb, buf);
2980 }
2981
2982 static unsigned char *skd_sg_1st_page_ptr(struct scatterlist *sg)
2983 {
2984         if (!sg)
2985                 return NULL;
2986         if (!sg_page(sg))
2987                 return NULL;
2988         return sg_virt(sg);
2989 }
2990
2991 static void skd_process_scsi_inq(struct skd_device *skdev,
2992                                  volatile struct fit_completion_entry_v1
2993                                  *skcomp,
2994                                  volatile struct fit_comp_error_info *skerr,
2995                                  struct skd_special_context *skspcl)
2996 {
2997         uint8_t *buf;
2998         struct fit_msg_hdr *fmh = (struct fit_msg_hdr *)skspcl->msg_buf;
2999         struct skd_scsi_request *scsi_req = (struct skd_scsi_request *)&fmh[1];
3000
3001         dma_sync_sg_for_cpu(skdev->class_dev, skspcl->req.sg, skspcl->req.n_sg,
3002                             skspcl->req.sg_data_dir);
3003         buf = skd_sg_1st_page_ptr(skspcl->req.sg);
3004
3005         if (buf)
3006                 skd_do_driver_inq(skdev, skcomp, skerr, scsi_req->cdb, buf);
3007 }
3008
3009
3010 static int skd_isr_completion_posted(struct skd_device *skdev,
3011                                         int limit, int *enqueued)
3012 {
3013         volatile struct fit_completion_entry_v1 *skcmp = NULL;
3014         volatile struct fit_comp_error_info *skerr;
3015         u16 req_id;
3016         u32 req_slot;
3017         struct skd_request_context *skreq;
3018         u16 cmp_cntxt = 0;
3019         u8 cmp_status = 0;
3020         u8 cmp_cycle = 0;
3021         u32 cmp_bytes = 0;
3022         int rc = 0;
3023         int processed = 0;
3024         int ret;
3025
3026
3027         for (;; ) {
3028                 SKD_ASSERT(skdev->skcomp_ix < SKD_N_COMPLETION_ENTRY);
3029
3030                 skcmp = &skdev->skcomp_table[skdev->skcomp_ix];
3031                 cmp_cycle = skcmp->cycle;
3032                 cmp_cntxt = skcmp->tag;
3033                 cmp_status = skcmp->status;
3034                 cmp_bytes = be32_to_cpu(skcmp->num_returned_bytes);
3035
3036                 skerr = &skdev->skerr_table[skdev->skcomp_ix];
3037
3038                 VPRINTK(skdev,
3039                         "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d "
3040                         "busy=%d rbytes=0x%x proto=%d\n", skdev->skcomp_cycle,
3041                         skdev->skcomp_ix, cmp_cycle, cmp_cntxt, cmp_status,
3042                         skdev->in_flight, cmp_bytes, skdev->proto_ver);
3043
3044                 if (cmp_cycle != skdev->skcomp_cycle) {
3045                         VPRINTK(skdev, "end of completions\n");
3046                         break;
3047                 }
3048                 /*
3049                  * Update the completion queue head index and possibly
3050                  * the completion cycle count. 8-bit wrap-around.
3051                  */
3052                 skdev->skcomp_ix++;
3053                 if (skdev->skcomp_ix >= SKD_N_COMPLETION_ENTRY) {
3054                         skdev->skcomp_ix = 0;
3055                         skdev->skcomp_cycle++;
3056                 }
3057
3058                 /*
3059                  * The command context is a unique 32-bit ID. The low order
3060                  * bits help locate the request. The request is usually a
3061                  * r/w request (see skd_start() above) or a special request.
3062                  */
3063                 req_id = cmp_cntxt;
3064                 req_slot = req_id & SKD_ID_SLOT_AND_TABLE_MASK;
3065
3066                 /* Is this other than a r/w request? */
3067                 if (req_slot >= skdev->num_req_context) {
3068                         /*
3069                          * This is not a completion for a r/w request.
3070                          */
3071                         skd_complete_other(skdev, skcmp, skerr);
3072                         continue;
3073                 }
3074
3075                 skreq = &skdev->skreq_table[req_slot];
3076
3077                 /*
3078                  * Make sure the request ID for the slot matches.
3079                  */
3080                 if (skreq->id != req_id) {
3081                         DPRINTK(skdev, "mismatch comp_id=0x%x req_id=0x%x\n",
3082                                 req_id, skreq->id);
3083                         {
3084                                 u16 new_id = cmp_cntxt;
3085                                 pr_err("(%s): Completion mismatch "
3086                                        "comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
3087                                        skd_name(skdev), req_id,
3088                                        skreq->id, new_id);
3089
3090                                 continue;
3091                         }
3092                 }
3093
3094                 SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);
3095
3096                 if (skreq->state == SKD_REQ_STATE_ABORTED) {
3097                         DPRINTK(skdev, "reclaim req %p id=%04x\n",
3098                                 skreq, skreq->id);
3099                         /* a previously timed out command can
3100                          * now be cleaned up */
3101                         skd_release_skreq(skdev, skreq);
3102                         continue;
3103                 }
3104
3105                 skreq->completion = *skcmp;
3106                 if (unlikely(cmp_status == SAM_STAT_CHECK_CONDITION)) {
3107                         skreq->err_info = *skerr;
3108                         skd_log_check_status(skdev, cmp_status, skerr->key,
3109                                              skerr->code, skerr->qual,
3110                                              skerr->fruc);
3111                 }
3112                 /* Release DMA resources for the request. */
3113                 if (skreq->n_sg > 0)
3114                         skd_postop_sg_list(skdev, skreq);
3115
3116                 if (((!skd_bio) && !skreq->req) ||
3117                     ((skd_bio) && !skreq->bio)) {
3118                         DPRINTK(skdev, "NULL backptr skdreq %p, "
3119                                 "req=0x%x req_id=0x%x\n",
3120                                 skreq, skreq->id, req_id);
3121                 } else {
3122                         /*
3123                          * Capture the outcome and post it back to the
3124                          * native request.
3125                          */
3126                         if (likely(cmp_status == SAM_STAT_GOOD)) {
3127                                 if (unlikely(skreq->flush_cmd)) {
3128                                         if (skd_bio) {
3129                                                 /* if empty size bio, we are all done */
3130                                                 if (bio_sectors(skreq->bio) == 0) {
3131                                                         skd_end_request(skdev, skreq, 0);
3132                                                 } else {
3133                                                         ret = skd_flush_cmd_enqueue(skdev, (void *)skreq->bio);
3134                                                         if (ret != 0) {
3135                                                                 pr_err("Failed to enqueue flush bio with Data. Err=%d.\n", ret);
3136                                                                 skd_end_request(skdev, skreq, ret);
3137                                                         } else {
3138                                                                 ((*enqueued)++);
3139                                                         }
3140                                                 }
3141                                         } else {
3142                                                 skd_end_request(skdev, skreq, 0);
3143                                         }
3144                                 } else {
3145                                         skd_end_request(skdev, skreq, 0);
3146                                 }
3147                         } else {
3148                                 skd_resolve_req_exception(skdev, skreq);
3149                         }
3150                 }
3151
3152                 /*
3153                  * Release the skreq, its FIT msg (if one), timeout slot,
3154                  * and queue depth.
3155                  */
3156                 skd_release_skreq(skdev, skreq);
3157
3158                 /* skd_isr_comp_limit equal zero means no limit */
3159                 if (limit) {
3160                         if (++processed >= limit) {
3161                                 rc = 1;
3162                                 break;
3163                         }
3164                 }
3165         }
3166
3167         if ((skdev->state == SKD_DRVR_STATE_PAUSING)
3168                 && (skdev->in_flight) == 0) {
3169                 skdev->state = SKD_DRVR_STATE_PAUSED;
3170                 wake_up_interruptible(&skdev->waitq);
3171         }
3172
3173         return rc;
3174 }
3175
3176 static void skd_complete_other(struct skd_device *skdev,
3177                                volatile struct fit_completion_entry_v1 *skcomp,
3178                                volatile struct fit_comp_error_info *skerr)
3179 {
3180         u32 req_id = 0;
3181         u32 req_table;
3182         u32 req_slot;
3183         struct skd_special_context *skspcl;
3184
3185         req_id = skcomp->tag;
3186         req_table = req_id & SKD_ID_TABLE_MASK;
3187         req_slot = req_id & SKD_ID_SLOT_MASK;
3188
3189         DPRINTK(skdev, "table=0x%x id=0x%x slot=%d\n", req_table, req_id,
3190                 req_slot);
3191
3192         /*
3193          * Based on the request id, determine how to dispatch this completion.
3194          * This swich/case is finding the good cases and forwarding the
3195          * completion entry. Errors are reported below the switch.
3196          */
3197         switch (req_table) {
3198         case SKD_ID_RW_REQUEST:
3199                 /*
3200                  * The caller, skd_completion_posted_isr() above,
3201                  * handles r/w requests. The only way we get here
3202                  * is if the req_slot is out of bounds.
3203                  */
3204                 break;
3205
3206         case SKD_ID_SPECIAL_REQUEST:
3207                 /*
3208                  * Make sure the req_slot is in bounds and that the id
3209                  * matches.
3210                  */
3211                 if (req_slot < skdev->n_special) {
3212                         skspcl = &skdev->skspcl_table[req_slot];
3213                         if (skspcl->req.id == req_id &&
3214                             skspcl->req.state == SKD_REQ_STATE_BUSY) {
3215                                 skd_complete_special(skdev,
3216                                                      skcomp, skerr, skspcl);
3217                                 return;
3218                         }
3219                 }
3220                 break;
3221
3222         case SKD_ID_INTERNAL:
3223                 if (req_slot == 0) {
3224                         skspcl = &skdev->internal_skspcl;
3225                         if (skspcl->req.id == req_id &&
3226                             skspcl->req.state == SKD_REQ_STATE_BUSY) {
3227                                 skd_complete_internal(skdev,
3228                                                       skcomp, skerr, skspcl);
3229                                 return;
3230                         }
3231                 }
3232                 break;
3233
3234         case SKD_ID_FIT_MSG:
3235                 /*
3236                  * These id's should never appear in a completion record.
3237                  */
3238                 break;
3239
3240         default:
3241                 /*
3242                  * These id's should never appear anywhere;
3243                  */
3244                 break;
3245         }
3246
3247         /*
3248          * If we get here it is a bad or stale id.
3249          */
3250 }
3251
3252 static void skd_complete_special(struct skd_device *skdev,
3253                                  volatile struct fit_completion_entry_v1
3254                                  *skcomp,
3255                                  volatile struct fit_comp_error_info *skerr,
3256                                  struct skd_special_context *skspcl)
3257 {
3258         DPRINTK(skdev, " completing special request %p\n", skspcl);
3259         if (skspcl->orphaned) {
3260                 /* Discard orphaned request */
3261                 /* ?: Can this release directly or does it need
3262                  * to use a worker? */
3263                 DPRINTK(skdev, "release orphaned %p\n", skspcl);
3264                 skd_release_special(skdev, skspcl);
3265                 return;
3266         }
3267
3268         skd_process_scsi_inq(skdev, skcomp, skerr, skspcl);
3269
3270         skspcl->req.state = SKD_REQ_STATE_COMPLETED;
3271         skspcl->req.completion = *skcomp;
3272         skspcl->req.err_info = *skerr;
3273
3274         skd_log_check_status(skdev, skspcl->req.completion.status, skerr->key,
3275                              skerr->code, skerr->qual, skerr->fruc);
3276
3277         wake_up_interruptible(&skdev->waitq);
3278 }
3279
3280 /* assume spinlock is already held */
3281 static void skd_release_special(struct skd_device *skdev,
3282                                 struct skd_special_context *skspcl)
3283 {
3284         int i, was_depleted;
3285
3286         for (i = 0; i < skspcl->req.n_sg; i++) {
3287
3288                 struct page *page = sg_page(&skspcl->req.sg[i]);
3289                 __free_page(page);
3290         }
3291
3292         was_depleted = (skdev->skspcl_free_list == NULL);
3293
3294         skspcl->req.state = SKD_REQ_STATE_IDLE;
3295         skspcl->req.id += SKD_ID_INCR;
3296         skspcl->req.next =
3297                 (struct skd_request_context *)skdev->skspcl_free_list;
3298         skdev->skspcl_free_list = (struct skd_special_context *)skspcl;
3299
3300         if (was_depleted) {
3301                 DPRINTK(skdev, "skspcl was depleted\n");
3302                 /* Free list was depleted. Their might be waiters. */
3303                 wake_up_interruptible(&skdev->waitq);
3304         }
3305 }
3306
3307 static void skd_reset_skcomp(struct skd_device *skdev)
3308 {
3309         u32 nbytes;
3310         struct fit_completion_entry_v1 *skcomp;
3311
3312         nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
3313         nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
3314
3315         memset(skdev->skcomp_table, 0, nbytes);
3316
3317         skdev->skcomp_ix = 0;
3318         skdev->skcomp_cycle = 1;
3319 }
3320
3321 /*
3322  *****************************************************************************
3323  * INTERRUPTS
3324  *****************************************************************************
3325  */
3326 static void skd_completion_worker(struct work_struct *work)
3327 {
3328         struct skd_device *skdev =
3329                 container_of(work, struct skd_device, completion_worker);
3330         unsigned long flags;
3331         int flush_enqueued = 0;
3332
3333         spin_lock_irqsave(&skdev->lock, flags);
3334
3335         /*
3336          * pass in limit=0, which means no limit..
3337          * process everything in compq
3338          */
3339         skd_isr_completion_posted(skdev, 0, &flush_enqueued);
3340         skd_request_fn(skdev->queue);
3341
3342         spin_unlock_irqrestore(&skdev->lock, flags);
3343 }
3344
3345 static void skd_isr_msg_from_dev(struct skd_device *skdev);
3346
3347 irqreturn_t
3348 static skd_isr(int irq, void *ptr)
3349 {
3350         struct skd_device *skdev;
3351         u32 intstat;
3352         u32 ack;
3353         int rc = 0;
3354         int deferred = 0;
3355         int flush_enqueued = 0;
3356
3357         skdev = (struct skd_device *)ptr;
3358         spin_lock(&skdev->lock);
3359
3360         for (;; ) {
3361                 intstat = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3362
3363                 ack = FIT_INT_DEF_MASK;
3364                 ack &= intstat;
3365
3366                 VPRINTK(skdev, "intstat=0x%x ack=0x%x\n", intstat, ack);
3367
3368                 /* As long as there is an int pending on device, keep
3369                  * running loop.  When none, get out, but if we've never
3370                  * done any processing, call completion handler?
3371                  */
3372                 if (ack == 0) {
3373                         /* No interrupts on device, but run the completion
3374                          * processor anyway?
3375                          */
3376                         if (rc == 0)
3377                                 if (likely (skdev->state
3378                                         == SKD_DRVR_STATE_ONLINE))
3379                                         deferred = 1;
3380                         break;
3381                 }
3382
3383                 rc = IRQ_HANDLED;
3384
3385                 SKD_WRITEL(skdev, ack, FIT_INT_STATUS_HOST);
3386
3387                 if (likely((skdev->state != SKD_DRVR_STATE_LOAD) &&
3388                            (skdev->state != SKD_DRVR_STATE_STOPPING))) {
3389                         if (intstat & FIT_ISH_COMPLETION_POSTED) {
3390                                 /*
3391                                  * If we have already deferred completion
3392                                  * processing, don't bother running it again
3393                                  */
3394                                 if (deferred == 0)
3395                                         deferred =
3396                                                 skd_isr_completion_posted(skdev,
3397                                                 skd_isr_comp_limit, &flush_enqueued);
3398                         }
3399
3400                         if (intstat & FIT_ISH_FW_STATE_CHANGE) {
3401                                 skd_isr_fwstate(skdev);
3402                                 if (skdev->state == SKD_DRVR_STATE_FAULT ||
3403                                     skdev->state ==
3404                                     SKD_DRVR_STATE_DISAPPEARED) {
3405                                         spin_unlock(&skdev->lock);
3406                                         return rc;
3407                                 }
3408                         }
3409
3410                         if (intstat & FIT_ISH_MSG_FROM_DEV)
3411                                 skd_isr_msg_from_dev(skdev);
3412                 }
3413         }
3414
3415         if (unlikely(flush_enqueued))
3416                 skd_request_fn(skdev->queue);
3417
3418         if (deferred)
3419                 schedule_work(&skdev->completion_worker);
3420         else if (!flush_enqueued)
3421                 skd_request_fn(skdev->queue);
3422
3423         spin_unlock(&skdev->lock);
3424
3425         return rc;
3426 }
3427
3428
3429 static void skd_drive_fault(struct skd_device *skdev)
3430 {
3431         skdev->state = SKD_DRVR_STATE_FAULT;
3432         pr_err("(%s): Drive FAULT\n", skd_name(skdev));
3433 }
3434
3435 static void skd_drive_disappeared(struct skd_device *skdev)
3436 {
3437         skdev->state = SKD_DRVR_STATE_DISAPPEARED;
3438         pr_err("(%s): Drive DISAPPEARED\n", skd_name(skdev));
3439 }
3440
3441 static void skd_isr_fwstate(struct skd_device *skdev)
3442 {
3443         u32 sense;
3444         u32 state;
3445         u32 mtd;
3446         int prev_driver_state = skdev->state;
3447
3448         sense = SKD_READL(skdev, FIT_STATUS);
3449         state = sense & FIT_SR_DRIVE_STATE_MASK;
3450
3451         pr_err("(%s): s1120 state %s(%d)=>%s(%d)\n",
3452                skd_name(skdev),
3453                skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
3454                skd_drive_state_to_str(state), state);
3455
3456         skdev->drive_state = state;
3457
3458         switch (skdev->drive_state) {
3459         case FIT_SR_DRIVE_INIT:
3460                 if (skdev->state == SKD_DRVR_STATE_PROTOCOL_MISMATCH) {
3461                         skd_disable_interrupts(skdev);
3462                         break;
3463                 }
3464                 if (skdev->state == SKD_DRVR_STATE_RESTARTING)
3465                         skd_recover_requests(skdev, 0);
3466                 if (skdev->state == SKD_DRVR_STATE_WAIT_BOOT) {
3467                         skdev->timer_countdown = SKD_STARTING_TIMO;
3468                         skdev->state = SKD_DRVR_STATE_STARTING;
3469                         skd_soft_reset(skdev);
3470                         break;
3471                 }
3472                 mtd = FIT_MXD_CONS(FIT_MTD_FITFW_INIT, 0, 0);
3473                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3474                 skdev->last_mtd = mtd;
3475                 break;
3476
3477         case FIT_SR_DRIVE_ONLINE:
3478                 skdev->cur_max_queue_depth = skd_max_queue_depth;
3479                 if (skdev->cur_max_queue_depth > skdev->dev_max_queue_depth)
3480                         skdev->cur_max_queue_depth = skdev->dev_max_queue_depth;
3481
3482                 skdev->queue_low_water_mark =
3483                         skdev->cur_max_queue_depth * 2 / 3 + 1;
3484                 if (skdev->queue_low_water_mark < 1)
3485                         skdev->queue_low_water_mark = 1;
3486                 pr_info(
3487                        "(%s): Queue depth limit=%d dev=%d lowat=%d\n",
3488                        skd_name(skdev),
3489                        skdev->cur_max_queue_depth,
3490                        skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
3491
3492                 skd_refresh_device_data(skdev);
3493                 break;
3494
3495         case FIT_SR_DRIVE_BUSY:
3496                 skdev->state = SKD_DRVR_STATE_BUSY;
3497                 skdev->timer_countdown = SKD_BUSY_TIMO;
3498                 skd_quiesce_dev(skdev);
3499                 break;
3500         case FIT_SR_DRIVE_BUSY_SANITIZE:
3501                 /* set timer for 3 seconds, we'll abort any unfinished
3502                  * commands after that expires
3503                  */
3504                 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3505                 skdev->timer_countdown = SKD_TIMER_SECONDS(3);
3506                 skd_start_queue(skdev);
3507                 break;
3508         case FIT_SR_DRIVE_BUSY_ERASE:
3509                 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3510                 skdev->timer_countdown = SKD_BUSY_TIMO;
3511                 break;
3512         case FIT_SR_DRIVE_OFFLINE:
3513                 skdev->state = SKD_DRVR_STATE_IDLE;
3514                 break;
3515         case FIT_SR_DRIVE_SOFT_RESET:
3516                 switch (skdev->state) {
3517                 case SKD_DRVR_STATE_STARTING:
3518                 case SKD_DRVR_STATE_RESTARTING:
3519                         /* Expected by a caller of skd_soft_reset() */
3520                         break;
3521                 default:
3522                         skdev->state = SKD_DRVR_STATE_RESTARTING;
3523                         break;
3524                 }
3525                 break;
3526         case FIT_SR_DRIVE_FW_BOOTING:
3527                 VPRINTK(skdev, "ISR FIT_SR_DRIVE_FW_BOOTING %s\n", skdev->name);
3528                 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3529                 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3530                 break;
3531
3532         case FIT_SR_DRIVE_DEGRADED:
3533         case FIT_SR_PCIE_LINK_DOWN:
3534         case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
3535                 break;
3536
3537         case FIT_SR_DRIVE_FAULT:
3538                 skd_drive_fault(skdev);
3539                 skd_recover_requests(skdev, 0);
3540                 skd_start_queue(skdev);
3541                 break;
3542
3543         /* PCIe bus returned all Fs? */
3544         case 0xFF:
3545                 pr_info("(%s): state=0x%x sense=0x%x\n",
3546                        skd_name(skdev), state, sense);
3547                 skd_drive_disappeared(skdev);
3548                 skd_recover_requests(skdev, 0);
3549                 skd_start_queue(skdev);
3550                 break;
3551         default:
3552                 /*
3553                  * Uknown FW State. Wait for a state we recognize.
3554                  */
3555                 break;
3556         }
3557         pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
3558                skd_name(skdev),
3559                skd_skdev_state_to_str(prev_driver_state), prev_driver_state,
3560                skd_skdev_state_to_str(skdev->state), skdev->state);
3561 }
3562
3563 static void skd_recover_requests(struct skd_device *skdev, int requeue)
3564 {
3565         int i;
3566
3567         for (i = 0; i < skdev->num_req_context; i++) {
3568                 struct skd_request_context *skreq = &skdev->skreq_table[i];
3569
3570                 if (skreq->state == SKD_REQ_STATE_BUSY) {
3571                         skd_log_skreq(skdev, skreq, "recover");
3572
3573                         SKD_ASSERT((skreq->id & SKD_ID_INCR) != 0);
3574                         if (!skd_bio)
3575                                 SKD_ASSERT(skreq->req != NULL);
3576                         else
3577                                 SKD_ASSERT(skreq->bio != NULL);
3578
3579                         /* Release DMA resources for the request. */
3580                         if (skreq->n_sg > 0)
3581                                 skd_postop_sg_list(skdev, skreq);
3582
3583                         if (!skd_bio) {
3584                                 if (requeue &&
3585                                     (unsigned long) ++skreq->req->special <
3586                                     SKD_MAX_RETRIES)
3587                                         skd_requeue_request(skdev, skreq);
3588                                 else
3589                                 skd_end_request(skdev, skreq, -EIO);
3590                         } else
3591                                 skd_end_request(skdev, skreq, -EIO);
3592
3593                         if (!skd_bio)
3594                                 skreq->req = NULL;
3595                         else
3596                                 skreq->bio = NULL;
3597
3598                         skreq->state = SKD_REQ_STATE_IDLE;
3599                         skreq->id += SKD_ID_INCR;
3600
3601
3602                 }
3603                 if (i > 0)
3604                         skreq[-1].next = skreq;
3605                 skreq->next = NULL;
3606         }
3607         skdev->skreq_free_list = skdev->skreq_table;
3608
3609         for (i = 0; i < skdev->num_fitmsg_context; i++) {
3610                 struct skd_fitmsg_context *skmsg = &skdev->skmsg_table[i];
3611
3612                 if (skmsg->state == SKD_MSG_STATE_BUSY) {
3613                         skd_log_skmsg(skdev, skmsg, "salvaged");
3614                         SKD_ASSERT((skmsg->id & SKD_ID_INCR) != 0);
3615                         skmsg->state = SKD_MSG_STATE_IDLE;
3616                         skmsg->id += SKD_ID_INCR;
3617                 }
3618                 if (i > 0)
3619                         skmsg[-1].next = skmsg;
3620                 skmsg->next = NULL;
3621         }
3622         skdev->skmsg_free_list = skdev->skmsg_table;
3623
3624         for (i = 0; i < skdev->n_special; i++) {
3625                 struct skd_special_context *skspcl = &skdev->skspcl_table[i];
3626
3627                 /* If orphaned, reclaim it because it has already been reported
3628                  * to the process as an error (it was just waiting for
3629                  * a completion that didn't come, and now it will never come)
3630                  * If busy, change to a state that will cause it to error
3631                  * out in the wait routine and let it do the normal
3632                  * reporting and reclaiming
3633                  */
3634                 if (skspcl->req.state == SKD_REQ_STATE_BUSY) {
3635                         if (skspcl->orphaned) {
3636                                 DPRINTK(skdev, "orphaned %p\n", skspcl);
3637                                 skd_release_special(skdev, skspcl);
3638                         } else {
3639                                 DPRINTK(skdev, "not orphaned %p\n", skspcl);
3640                                 skspcl->req.state = SKD_REQ_STATE_ABORTED;
3641                         }
3642                 }
3643         }
3644         skdev->skspcl_free_list = skdev->skspcl_table;
3645
3646         for (i = 0; i < SKD_N_TIMEOUT_SLOT; i++)
3647                 skdev->timeout_slot[i] = 0;
3648
3649         skdev->in_flight = 0;
3650 }
3651
3652 static void skd_isr_msg_from_dev(struct skd_device *skdev)
3653 {
3654         u32 mfd;
3655         u32 mtd;
3656         u32 data;
3657
3658         mfd = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3659
3660         DPRINTK(skdev, "mfd=0x%x last_mtd=0x%x\n", mfd, skdev->last_mtd);
3661
3662         /* ignore any mtd that is an ack for something we didn't send */
3663         if (FIT_MXD_TYPE(mfd) != FIT_MXD_TYPE(skdev->last_mtd))
3664                 return;
3665
3666         switch (FIT_MXD_TYPE(mfd)) {
3667         case FIT_MTD_FITFW_INIT:
3668                 skdev->proto_ver = FIT_PROTOCOL_MAJOR_VER(mfd);
3669
3670                 if (skdev->proto_ver != FIT_PROTOCOL_VERSION_1) {
3671                         pr_err("(%s): protocol mismatch\n",
3672                                skdev->name);
3673                         pr_err("(%s):   got=%d support=%d\n",
3674                                skdev->name, skdev->proto_ver,
3675                                FIT_PROTOCOL_VERSION_1);
3676                         pr_err("(%s):   please upgrade driver\n",
3677                                skdev->name);
3678                         skdev->state = SKD_DRVR_STATE_PROTOCOL_MISMATCH;
3679                         skd_soft_reset(skdev);
3680                         break;
3681                 }
3682                 mtd = FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH, 0, 0);
3683                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3684                 skdev->last_mtd = mtd;
3685                 break;
3686
3687         case FIT_MTD_GET_CMDQ_DEPTH:
3688                 skdev->dev_max_queue_depth = FIT_MXD_DATA(mfd);
3689                 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH, 0,
3690                                    SKD_N_COMPLETION_ENTRY);
3691                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3692                 skdev->last_mtd = mtd;
3693                 break;
3694
3695         case FIT_MTD_SET_COMPQ_DEPTH:
3696                 SKD_WRITEQ(skdev, skdev->cq_dma_address, FIT_MSG_TO_DEVICE_ARG);
3697                 mtd = FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR, 0, 0);
3698                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3699                 skdev->last_mtd = mtd;
3700                 break;
3701
3702         case FIT_MTD_SET_COMPQ_ADDR:
3703                 skd_reset_skcomp(skdev);
3704                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID, 0, skdev->devno);
3705                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3706                 skdev->last_mtd = mtd;
3707                 break;
3708
3709         case FIT_MTD_CMD_LOG_HOST_ID:
3710                 skdev->connect_time_stamp = get_seconds();
3711                 data = skdev->connect_time_stamp & 0xFFFF;
3712                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO, 0, data);
3713                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3714                 skdev->last_mtd = mtd;
3715                 break;
3716
3717         case FIT_MTD_CMD_LOG_TIME_STAMP_LO:
3718                 skdev->drive_jiffies = FIT_MXD_DATA(mfd);
3719                 data = (skdev->connect_time_stamp >> 16) & 0xFFFF;
3720                 mtd = FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI, 0, data);
3721                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3722                 skdev->last_mtd = mtd;
3723                 break;
3724
3725         case FIT_MTD_CMD_LOG_TIME_STAMP_HI:
3726                 skdev->drive_jiffies |= (FIT_MXD_DATA(mfd) << 16);
3727                 mtd = FIT_MXD_CONS(FIT_MTD_ARM_QUEUE, 0, 0);
3728                 SKD_WRITEL(skdev, mtd, FIT_MSG_TO_DEVICE);
3729                 skdev->last_mtd = mtd;
3730
3731                 pr_err("(%s): Time sync driver=0x%x device=0x%x\n",
3732                        skd_name(skdev),
3733                        skdev->connect_time_stamp, skdev->drive_jiffies);
3734                 break;
3735
3736         case FIT_MTD_ARM_QUEUE:
3737                 skdev->last_mtd = 0;
3738                 /*
3739                  * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
3740                  */
3741                 break;
3742
3743         default:
3744                 break;
3745         }
3746 }
3747
3748 static void skd_disable_interrupts(struct skd_device *skdev)
3749 {
3750         u32 sense;
3751
3752         sense = SKD_READL(skdev, FIT_CONTROL);
3753         sense &= ~FIT_CR_ENABLE_INTERRUPTS;
3754         SKD_WRITEL(skdev, sense, FIT_CONTROL);
3755         DPRINTK(skdev, "sense 0x%x\n", sense);
3756
3757         /* Note that the 1s is written. A 1-bit means
3758          * disable, a 0 means enable.
3759          */
3760         SKD_WRITEL(skdev, ~0, FIT_INT_MASK_HOST);
3761 }
3762
3763 static void skd_enable_interrupts(struct skd_device *skdev)
3764 {
3765         u32 val;
3766
3767         /* unmask interrupts first */
3768         val = FIT_ISH_FW_STATE_CHANGE +
3769               FIT_ISH_COMPLETION_POSTED + FIT_ISH_MSG_FROM_DEV;
3770
3771         /* Note that the compliment of mask is written. A 1-bit means
3772          * disable, a 0 means enable. */
3773         SKD_WRITEL(skdev, ~val, FIT_INT_MASK_HOST);
3774         DPRINTK(skdev, "interrupt mask=0x%x\n", ~val);
3775
3776         val = SKD_READL(skdev, FIT_CONTROL);
3777         val |= FIT_CR_ENABLE_INTERRUPTS;
3778         DPRINTK(skdev, "control=0x%x\n", val);
3779         SKD_WRITEL(skdev, val, FIT_CONTROL);
3780 }
3781
3782 /*
3783  *****************************************************************************
3784  * START, STOP, RESTART, QUIESCE, UNQUIESCE
3785  *****************************************************************************
3786  */
3787
3788 static void skd_soft_reset(struct skd_device *skdev)
3789 {
3790         u32 val;
3791
3792         val = SKD_READL(skdev, FIT_CONTROL);
3793         val |= (FIT_CR_SOFT_RESET);
3794         DPRINTK(skdev, "control=0x%x\n", val);
3795         SKD_WRITEL(skdev, val, FIT_CONTROL);
3796 }
3797
3798 static void skd_start_device(struct skd_device *skdev)
3799 {
3800         unsigned long flags;
3801         u32 sense;
3802         u32 state;
3803
3804         spin_lock_irqsave(&skdev->lock, flags);
3805
3806         /* ack all ghost interrupts */
3807         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3808
3809         sense = SKD_READL(skdev, FIT_STATUS);
3810
3811         DPRINTK(skdev, "initial status=0x%x\n", sense);
3812
3813         state = sense & FIT_SR_DRIVE_STATE_MASK;
3814         skdev->drive_state = state;
3815         skdev->last_mtd = 0;
3816
3817         skdev->state = SKD_DRVR_STATE_STARTING;
3818         skdev->timer_countdown = SKD_STARTING_TIMO;
3819
3820         skd_enable_interrupts(skdev);
3821
3822         switch (skdev->drive_state) {
3823         case FIT_SR_DRIVE_OFFLINE:
3824                 pr_err("(%s): Drive offline...\n", skd_name(skdev));
3825                 break;
3826
3827         case FIT_SR_DRIVE_FW_BOOTING:
3828                 VPRINTK(skdev, "FIT_SR_DRIVE_FW_BOOTING %s\n", skdev->name);
3829                 skdev->state = SKD_DRVR_STATE_WAIT_BOOT;
3830                 skdev->timer_countdown = SKD_WAIT_BOOT_TIMO;
3831                 break;
3832
3833         case FIT_SR_DRIVE_BUSY_SANITIZE:
3834                 pr_info("(%s): Start: BUSY_SANITIZE\n",
3835                        skd_name(skdev));
3836                 skdev->state = SKD_DRVR_STATE_BUSY_SANITIZE;
3837                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3838                 break;
3839
3840         case FIT_SR_DRIVE_BUSY_ERASE:
3841                 pr_info("(%s): Start: BUSY_ERASE\n", skd_name(skdev));
3842                 skdev->state = SKD_DRVR_STATE_BUSY_ERASE;
3843                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3844                 break;
3845
3846         case FIT_SR_DRIVE_INIT:
3847         case FIT_SR_DRIVE_ONLINE:
3848                 skd_soft_reset(skdev);
3849                 break;
3850
3851         case FIT_SR_DRIVE_BUSY:
3852                 pr_err("(%s): Drive Busy...\n", skd_name(skdev));
3853                 skdev->state = SKD_DRVR_STATE_BUSY;
3854                 skdev->timer_countdown = SKD_STARTED_BUSY_TIMO;
3855                 break;
3856
3857         case FIT_SR_DRIVE_SOFT_RESET:
3858                 pr_err("(%s) drive soft reset in prog\n",
3859                        skd_name(skdev));
3860                 break;
3861
3862         case FIT_SR_DRIVE_FAULT:
3863                 /* Fault state is bad...soft reset won't do it...
3864                  * Hard reset, maybe, but does it work on device?
3865                  * For now, just fault so the system doesn't hang.
3866                  */
3867                 skd_drive_fault(skdev);
3868                 /*start the queue so we can respond with error to requests */
3869                 VPRINTK(skdev, "starting %s queue\n", skdev->name);
3870                 skd_start_queue(skdev);
3871                 skdev->gendisk_on = -1;
3872                 wake_up_interruptible(&skdev->waitq);
3873                 break;
3874
3875         case 0xFF:
3876                 /* Most likely the device isn't there or isn't responding
3877                  * to the BAR1 addresses. */
3878                 skd_drive_disappeared(skdev);
3879                 /*start the queue so we can respond with error to requests */
3880                 VPRINTK(skdev, "starting %s queue to error-out reqs\n",
3881                         skdev->name);
3882                 skd_start_queue(skdev);
3883                 skdev->gendisk_on = -1;
3884                 wake_up_interruptible(&skdev->waitq);
3885                 break;
3886
3887         default:
3888                 pr_err("(%s) Start: unknown state %x\n",
3889                        skd_name(skdev), skdev->drive_state);
3890                 break;
3891         }
3892
3893         state = SKD_READL(skdev, FIT_CONTROL);
3894         DPRINTK(skdev, "FIT Control Status=0x%x\n", state);
3895
3896         state = SKD_READL(skdev, FIT_INT_STATUS_HOST);
3897         DPRINTK(skdev, "Intr Status=0x%x\n", state);
3898
3899         state = SKD_READL(skdev, FIT_INT_MASK_HOST);
3900         DPRINTK(skdev, "Intr Mask=0x%x\n", state);
3901
3902         state = SKD_READL(skdev, FIT_MSG_FROM_DEVICE);
3903         DPRINTK(skdev, "Msg from Dev=0x%x\n", state);
3904
3905         state = SKD_READL(skdev, FIT_HW_VERSION);
3906         DPRINTK(skdev, "HW version=0x%x\n", state);
3907
3908         spin_unlock_irqrestore(&skdev->lock, flags);
3909 }
3910
3911 static void skd_stop_device(struct skd_device *skdev)
3912 {
3913         unsigned long flags;
3914         struct skd_special_context *skspcl = &skdev->internal_skspcl;
3915         u32 dev_state;
3916         int i;
3917
3918         spin_lock_irqsave(&skdev->lock, flags);
3919
3920         if (skdev->state != SKD_DRVR_STATE_ONLINE) {
3921                 pr_err("(%s): skd_stop_device not online no sync\n",
3922                        skd_name(skdev));
3923                 goto stop_out;
3924         }
3925
3926         if (skspcl->req.state != SKD_REQ_STATE_IDLE) {
3927                 pr_err("(%s): skd_stop_device no special\n",
3928                        skd_name(skdev));
3929                 goto stop_out;
3930         }
3931
3932         skdev->state = SKD_DRVR_STATE_SYNCING;
3933         skdev->sync_done = 0;
3934
3935         skd_send_internal_skspcl(skdev, skspcl, SYNCHRONIZE_CACHE);
3936
3937         spin_unlock_irqrestore(&skdev->lock, flags);
3938
3939         wait_event_interruptible_timeout(skdev->waitq,
3940                                          (skdev->sync_done), (10 * HZ));
3941
3942         spin_lock_irqsave(&skdev->lock, flags);
3943
3944         switch (skdev->sync_done) {
3945         case 0:
3946                 pr_err("(%s): skd_stop_device no sync\n",
3947                        skd_name(skdev));
3948                 break;
3949         case 1:
3950                 pr_err("(%s): skd_stop_device sync done\n",
3951                        skd_name(skdev));
3952                 break;
3953         default:
3954                 pr_err("(%s): skd_stop_device sync error\n",
3955                        skd_name(skdev));
3956         }
3957
3958 stop_out:
3959         skdev->state = SKD_DRVR_STATE_STOPPING;
3960         spin_unlock_irqrestore(&skdev->lock, flags);
3961
3962         skd_kill_timer(skdev);
3963
3964         spin_lock_irqsave(&skdev->lock, flags);
3965         skd_disable_interrupts(skdev);
3966
3967         /* ensure all ints on device are cleared */
3968         /* soft reset the device to unload with a clean slate */
3969         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3970         SKD_WRITEL(skdev, FIT_CR_SOFT_RESET, FIT_CONTROL);
3971
3972         spin_unlock_irqrestore(&skdev->lock, flags);
3973
3974         /* poll every 100ms, 1 second timeout */
3975         for (i = 0; i < 10; i++) {
3976                 dev_state =
3977                         SKD_READL(skdev, FIT_STATUS) & FIT_SR_DRIVE_STATE_MASK;
3978                 if (dev_state == FIT_SR_DRIVE_INIT)
3979                         break;
3980                 set_current_state(TASK_INTERRUPTIBLE);
3981                 schedule_timeout(msecs_to_jiffies(100));
3982         }
3983
3984         if (dev_state != FIT_SR_DRIVE_INIT)
3985                 pr_err("(%s): skd_stop_device state error 0x%02x\n",
3986                        skd_name(skdev), dev_state);
3987 }
3988
3989 /* assume spinlock is held */
3990 static void skd_restart_device(struct skd_device *skdev)
3991 {
3992         u32 state;
3993
3994         /* ack all ghost interrupts */
3995         SKD_WRITEL(skdev, FIT_INT_DEF_MASK, FIT_INT_STATUS_HOST);
3996
3997         state = SKD_READL(skdev, FIT_STATUS);
3998
3999         DPRINTK(skdev, "drive status=0x%x\n", state);
4000
4001         state &= FIT_SR_DRIVE_STATE_MASK;
4002         skdev->drive_state = state;
4003         skdev->last_mtd = 0;
4004
4005         skdev->state = SKD_DRVR_STATE_RESTARTING;
4006         skdev->timer_countdown = SKD_RESTARTING_TIMO;
4007
4008         skd_soft_reset(skdev);
4009 }
4010
4011 /* assume spinlock is held */
4012 static int skd_quiesce_dev(struct skd_device *skdev)
4013 {
4014         int rc = 0;
4015
4016         switch (skdev->state) {
4017         case SKD_DRVR_STATE_BUSY:
4018         case SKD_DRVR_STATE_BUSY_IMMINENT:
4019                 VPRINTK(skdev, "stopping %s queue\n", skdev->name);
4020                 skd_stop_queue(skdev);
4021                 break;
4022         case SKD_DRVR_STATE_ONLINE:
4023         case SKD_DRVR_STATE_STOPPING:
4024         case SKD_DRVR_STATE_SYNCING:
4025         case SKD_DRVR_STATE_PAUSING:
4026         case SKD_DRVR_STATE_PAUSED:
4027         case SKD_DRVR_STATE_STARTING:
4028         case SKD_DRVR_STATE_RESTARTING:
4029         case SKD_DRVR_STATE_RESUMING:
4030         default:
4031                 rc = -EINVAL;
4032                 VPRINTK(skdev, "state [%d] not implemented\n", skdev->state);
4033         }
4034         return rc;
4035 }
4036
4037 /* assume spinlock is held */
4038 static int skd_unquiesce_dev(struct skd_device *skdev)
4039 {
4040         int prev_driver_state = skdev->state;
4041
4042         skd_log_skdev(skdev, "unquiesce");
4043         if (skdev->state == SKD_DRVR_STATE_ONLINE) {
4044                 DPRINTK(skdev, "**** device already ONLINE\n");
4045                 return 0;
4046         }
4047         if (skdev->drive_state != FIT_SR_DRIVE_ONLINE) {
4048                 /*
4049                  * If there has been an state change to other than
4050                  * ONLINE, we will rely on controller state change
4051                  * to come back online and restart the queue.
4052                  * The BUSY state means that driver is ready to
4053                  * continue normal processing but waiting for controller
4054                  * to become available.
4055                  */
4056                 skdev->state = SKD_DRVR_STATE_BUSY;
4057                 DPRINTK(skdev, "drive BUSY state\n");
4058                 return 0;
4059         }
4060
4061         /*
4062          * Drive has just come online, driver is either in startup,
4063          * paused performing a task, or bust waiting for hardware.
4064          */
4065         switch (skdev->state) {
4066         case SKD_DRVR_STATE_PAUSED:
4067         case SKD_DRVR_STATE_BUSY:
4068         case SKD_DRVR_STATE_BUSY_IMMINENT:
4069         case SKD_DRVR_STATE_BUSY_ERASE:
4070         case SKD_DRVR_STATE_STARTING:
4071         case SKD_DRVR_STATE_RESTARTING:
4072         case SKD_DRVR_STATE_FAULT:
4073         case SKD_DRVR_STATE_IDLE:
4074         case SKD_DRVR_STATE_LOAD:
4075                 skdev->state = SKD_DRVR_STATE_ONLINE;
4076                 pr_err("(%s): Driver state %s(%d)=>%s(%d)\n",
4077                        skd_name(skdev),
4078                        skd_skdev_state_to_str(prev_driver_state),
4079                        prev_driver_state, skd_skdev_state_to_str(skdev->state),
4080                        skdev->state);
4081                 DPRINTK(skdev, "**** device ONLINE...starting block queue\n");
4082                 VPRINTK(skdev, "starting %s queue\n", skdev->name);
4083                 pr_info("(%s): STEC s1120 ONLINE\n", skd_name(skdev));
4084                 skd_start_queue(skdev);
4085                 skdev->gendisk_on = 1;
4086                 wake_up_interruptible(&skdev->waitq);
4087                 break;
4088
4089         case SKD_DRVR_STATE_DISAPPEARED:
4090         default:
4091                 DPRINTK(skdev, "**** driver state %d, not implemented \n",
4092                         skdev->state);
4093                 return -EBUSY;
4094         }
4095         return 0;
4096 }
4097
4098 /*
4099  *****************************************************************************
4100  * PCIe MSI/MSI-X INTERRUPT HANDLERS
4101  *****************************************************************************
4102  */
4103
4104 static irqreturn_t skd_reserved_isr(int irq, void *skd_host_data)
4105 {
4106         struct skd_device *skdev = skd_host_data;
4107         unsigned long flags;
4108
4109         spin_lock_irqsave(&skdev->lock, flags);
4110         VPRINTK(skdev, "MSIX = 0x%x\n", SKD_READL(skdev, FIT_INT_STATUS_HOST));
4111         pr_err("(%s): MSIX reserved irq %d = 0x%x\n", skd_name(skdev),
4112                irq, SKD_READL(skdev, FIT_INT_STATUS_HOST));
4113         SKD_WRITEL(skdev, FIT_INT_RESERVED_MASK, FIT_INT_STATUS_HOST);
4114         spin_unlock_irqrestore(&skdev->lock, flags);
4115         return IRQ_HANDLED;
4116 }
4117
4118 static irqreturn_t skd_statec_isr(int irq, void *skd_host_data)
4119 {
4120         struct skd_device *skdev = skd_host_data;
4121         unsigned long flags;
4122
4123         spin_lock_irqsave(&skdev->lock, flags);
4124         VPRINTK(skdev, "MSIX = 0x%x\n", SKD_READL(skdev, FIT_INT_STATUS_HOST));
4125         SKD_WRITEL(skdev, FIT_ISH_FW_STATE_CHANGE, FIT_INT_STATUS_HOST);
4126         skd_isr_fwstate(skdev);
4127         spin_unlock_irqrestore(&skdev->lock, flags);
4128         return IRQ_HANDLED;
4129 }
4130
4131 static irqreturn_t skd_comp_q(int irq, void *skd_host_data)
4132 {
4133         struct skd_device *skdev = skd_host_data;
4134         unsigned long flags;
4135         int flush_enqueued = 0;
4136         int deferred;
4137
4138         spin_lock_irqsave(&skdev->lock, flags);
4139         VPRINTK(skdev, "MSIX = 0x%x\n", SKD_READL(skdev, FIT_INT_STATUS_HOST));
4140         SKD_WRITEL(skdev, FIT_ISH_COMPLETION_POSTED, FIT_INT_STATUS_HOST);
4141         deferred = skd_isr_completion_posted(skdev, skd_isr_comp_limit,
4142                                                 &flush_enqueued);
4143
4144         if (flush_enqueued)
4145                 skd_request_fn(skdev->queue);
4146
4147         if (deferred)
4148                 schedule_work(&skdev->completion_worker);
4149         else if (!flush_enqueued)
4150                 skd_request_fn(skdev->queue);
4151
4152         spin_unlock_irqrestore(&skdev->lock, flags);
4153
4154         return IRQ_HANDLED;
4155 }
4156
4157 static irqreturn_t skd_msg_isr(int irq, void *skd_host_data)
4158 {
4159         struct skd_device *skdev = skd_host_data;
4160         unsigned long flags;
4161
4162         spin_lock_irqsave(&skdev->lock, flags);
4163         VPRINTK(skdev, "MSIX = 0x%x\n", SKD_READL(skdev, FIT_INT_STATUS_HOST));
4164         SKD_WRITEL(skdev, FIT_ISH_MSG_FROM_DEV, FIT_INT_STATUS_HOST);
4165         skd_isr_msg_from_dev(skdev);
4166         spin_unlock_irqrestore(&skdev->lock, flags);
4167         return IRQ_HANDLED;
4168 }
4169
4170 static irqreturn_t skd_qfull_isr(int irq, void *skd_host_data)
4171 {
4172         struct skd_device *skdev = skd_host_data;
4173         unsigned long flags;
4174
4175         spin_lock_irqsave(&skdev->lock, flags);
4176         VPRINTK(skdev, "MSIX = 0x%x\n", SKD_READL(skdev, FIT_INT_STATUS_HOST));
4177         SKD_WRITEL(skdev, FIT_INT_QUEUE_FULL, FIT_INT_STATUS_HOST);
4178         spin_unlock_irqrestore(&skdev->lock, flags);
4179         return IRQ_HANDLED;
4180 }
4181
4182 /*
4183  *****************************************************************************
4184  * PCIe MSI/MSI-X SETUP
4185  *****************************************************************************
4186  */
4187
4188 struct skd_msix_entry {
4189         int have_irq;
4190         u32 vector;
4191         u32 entry;
4192         struct skd_device *rsp;
4193         char isr_name[30];
4194 };
4195
4196 struct skd_init_msix_entry {
4197         const char *name;
4198         irq_handler_t handler;
4199 };
4200
4201 #define SKD_MAX_MSIX_COUNT              13
4202 #define SKD_MIN_MSIX_COUNT              7
4203 #define SKD_BASE_MSIX_IRQ               4
4204
4205 static struct skd_init_msix_entry msix_entries[SKD_MAX_MSIX_COUNT] = {
4206         { "(DMA 0)",        skd_reserved_isr },
4207         { "(DMA 1)",        skd_reserved_isr },
4208         { "(DMA 2)",        skd_reserved_isr },
4209         { "(DMA 3)",        skd_reserved_isr },
4210         { "(State Change)", skd_statec_isr   },
4211         { "(COMPL_Q)",      skd_comp_q       },
4212         { "(MSG)",          skd_msg_isr      },
4213         { "(Reserved)",     skd_reserved_isr },
4214         { "(Reserved)",     skd_reserved_isr },
4215         { "(Queue Full 0)", skd_qfull_isr    },
4216         { "(Queue Full 1)", skd_qfull_isr    },
4217         { "(Queue Full 2)", skd_qfull_isr    },
4218         { "(Queue Full 3)", skd_qfull_isr    },
4219 };
4220
4221 static void skd_release_msix(struct skd_device *skdev)
4222 {
4223         struct skd_msix_entry *qentry;
4224         int i;
4225
4226         if (skdev->msix_entries == NULL)
4227                 return;
4228         for (i = 0; i < skdev->msix_count; i++) {
4229                 qentry = &skdev->msix_entries[i];
4230                 skdev = qentry->rsp;
4231
4232                 if (qentry->have_irq)
4233                         devm_free_irq(&skdev->pdev->dev,
4234                                       qentry->vector, qentry->rsp);
4235         }
4236         pci_disable_msix(skdev->pdev);
4237         kfree(skdev->msix_entries);
4238         skdev->msix_count = 0;
4239         skdev->msix_entries = NULL;
4240 }
4241
4242 static int skd_acquire_msix(struct skd_device *skdev)
4243 {
4244         int i, rc;
4245         struct pci_dev *pdev;
4246         struct msix_entry *entries = NULL;
4247         struct skd_msix_entry *qentry;
4248
4249         pdev = skdev->pdev;
4250         skdev->msix_count = SKD_MAX_MSIX_COUNT;
4251         entries = kzalloc(sizeof(struct msix_entry) * SKD_MAX_MSIX_COUNT,
4252                           GFP_KERNEL);
4253         if (!entries)
4254                 return -ENOMEM;
4255
4256         for (i = 0; i < SKD_MAX_MSIX_COUNT; i++)
4257                 entries[i].entry = i;
4258
4259         rc = pci_enable_msix(pdev, entries, SKD_MAX_MSIX_COUNT);
4260         if (rc < 0)
4261                 goto msix_out;
4262         if (rc) {
4263                 if (rc < SKD_MIN_MSIX_COUNT) {
4264                         pr_err("(%s): failed to enable MSI-X %d\n",
4265                                skd_name(skdev), rc);
4266                         goto msix_out;
4267                 }
4268                 DPRINTK(skdev, "%s: <%s> allocated %d MSI-X vectors\n",
4269                         pci_name(pdev), skdev->name, rc);
4270
4271                 skdev->msix_count = rc;
4272                 rc = pci_enable_msix(pdev, entries, skdev->msix_count);
4273                 if (rc) {
4274                         pr_err("(%s): failed to enable MSI-X "
4275                                "support (%d) %d\n",
4276                                skd_name(skdev), skdev->msix_count, rc);
4277                         goto msix_out;
4278                 }
4279         }
4280         skdev->msix_entries = kzalloc(sizeof(struct skd_msix_entry) *
4281                                       skdev->msix_count, GFP_KERNEL);
4282         if (!skdev->msix_entries) {
4283                 rc = -ENOMEM;
4284                 skdev->msix_count = 0;
4285                 pr_err("(%s): msix table allocation error\n",
4286                        skd_name(skdev));
4287                 goto msix_out;
4288         }
4289
4290         qentry = skdev->msix_entries;
4291         for (i = 0; i < skdev->msix_count; i++) {
4292                 qentry->vector = entries[i].vector;
4293                 qentry->entry = entries[i].entry;
4294                 qentry->rsp = NULL;
4295                 qentry->have_irq = 0;
4296                 DPRINTK(skdev, "%s: <%s> msix (%d) vec %d, entry %x\n",
4297                         pci_name(pdev), skdev->name,
4298                         i, qentry->vector, qentry->entry);
4299                 qentry++;
4300         }
4301
4302         /* Enable MSI-X vectors for the base queue */
4303         for (i = 0; i < SKD_MAX_MSIX_COUNT; i++) {
4304                 qentry = &skdev->msix_entries[i];
4305                 snprintf(qentry->isr_name, sizeof(qentry->isr_name),
4306                          "%s%d-msix %s", DRV_NAME, skdev->devno,
4307                          msix_entries[i].name);
4308                 rc = devm_request_irq(&skdev->pdev->dev, qentry->vector,
4309                                       msix_entries[i].handler, 0,
4310                                       qentry->isr_name, skdev);
4311                 if (rc) {
4312                         pr_err("(%s): Unable to register(%d) MSI-X "
4313                                "handler %d: %s\n",
4314                                skd_name(skdev), rc, i, qentry->isr_name);
4315                         goto msix_out;
4316                 } else {
4317                         qentry->have_irq = 1;
4318                         qentry->rsp = skdev;
4319                 }
4320         }
4321         DPRINTK(skdev, "%s: <%s> msix %d irq(s) enabled\n",
4322                 pci_name(pdev), skdev->name, skdev->msix_count);
4323         return 0;
4324
4325 msix_out:
4326         if (entries)
4327                 kfree(entries);
4328         skd_release_msix(skdev);
4329         return rc;
4330 }
4331
4332 static int skd_acquire_irq(struct skd_device *skdev)
4333 {
4334         int rc;
4335         struct pci_dev *pdev;
4336
4337         pdev = skdev->pdev;
4338         skdev->msix_count = 0;
4339
4340 RETRY_IRQ_TYPE:
4341         switch (skdev->irq_type) {
4342         case SKD_IRQ_MSIX:
4343                 rc = skd_acquire_msix(skdev);
4344                 if (!rc)
4345                         pr_info("(%s): MSI-X %d irqs enabled\n",
4346                                skd_name(skdev), skdev->msix_count);
4347                 else {
4348                         pr_err(
4349                                "(%s): failed to enable MSI-X, re-trying with MSI %d\n",
4350                                skd_name(skdev), rc);
4351                         skdev->irq_type = SKD_IRQ_MSI;
4352                         goto RETRY_IRQ_TYPE;
4353                 }
4354                 break;
4355         case SKD_IRQ_MSI:
4356                 snprintf(skdev->isr_name, sizeof(skdev->isr_name), "%s%d-msi",
4357                          DRV_NAME, skdev->devno);
4358                 rc = pci_enable_msi(pdev);
4359                 if (!rc) {
4360                         rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr, 0,
4361                                               skdev->isr_name, skdev);
4362                         if (rc) {
4363                                 pci_disable_msi(pdev);
4364                                 pr_err(
4365                                        "(%s): failed to allocate the MSI interrupt %d\n",
4366                                        skd_name(skdev), rc);
4367                                 goto RETRY_IRQ_LEGACY;
4368                         }
4369                         pr_info("(%s): MSI irq %d enabled\n",
4370                                skd_name(skdev), pdev->irq);
4371                 } else {
4372 RETRY_IRQ_LEGACY:
4373                         pr_err(
4374                                "(%s): failed to enable MSI, re-trying with LEGACY %d\n",
4375                                skd_name(skdev), rc);
4376                         skdev->irq_type = SKD_IRQ_LEGACY;
4377                         goto RETRY_IRQ_TYPE;
4378                 }
4379                 break;
4380         case SKD_IRQ_LEGACY:
4381                 snprintf(skdev->isr_name, sizeof(skdev->isr_name),
4382                          "%s%d-legacy", DRV_NAME, skdev->devno);
4383                 rc = devm_request_irq(&pdev->dev, pdev->irq, skd_isr,
4384                                       IRQF_SHARED, skdev->isr_name, skdev);
4385                 if (!rc)
4386                         pr_info("(%s): LEGACY irq %d enabled\n",
4387                                skd_name(skdev), pdev->irq);
4388                 else
4389                         pr_err("(%s): request LEGACY irq error %d\n",
4390                                skd_name(skdev), rc);
4391                 break;
4392         default:
4393                 pr_info("(%s): irq_type %d invalid, re-set to %d\n",
4394                        skd_name(skdev), skdev->irq_type, SKD_IRQ_DEFAULT);
4395                 skdev->irq_type = SKD_IRQ_LEGACY;
4396                 goto RETRY_IRQ_TYPE;
4397         }
4398         return rc;
4399 }
4400
4401 static void skd_release_irq(struct skd_device *skdev)
4402 {
4403         switch (skdev->irq_type) {
4404         case SKD_IRQ_MSIX:
4405                 skd_release_msix(skdev);
4406                 break;
4407         case SKD_IRQ_MSI:
4408                 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4409                 pci_disable_msi(skdev->pdev);
4410                 break;
4411         case SKD_IRQ_LEGACY:
4412                 devm_free_irq(&skdev->pdev->dev, skdev->pdev->irq, skdev);
4413                 break;
4414         default:
4415                 pr_err("(%s): wrong irq type %d!",
4416                        skd_name(skdev), skdev->irq_type);
4417                 break;
4418         }
4419 }
4420
4421 /*
4422  *****************************************************************************
4423  * CONSTRUCT
4424  *****************************************************************************
4425  */
4426
4427 static int skd_cons_skcomp(struct skd_device *skdev);
4428 static int skd_cons_skmsg(struct skd_device *skdev);
4429 static int skd_cons_skreq(struct skd_device *skdev);
4430 static int skd_cons_skspcl(struct skd_device *skdev);
4431 static int skd_cons_sksb(struct skd_device *skdev);
4432 static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4433                                                   u32 n_sg,
4434                                                   dma_addr_t *ret_dma_addr);
4435 static int skd_cons_disk(struct skd_device *skdev);
4436
4437 #define SKD_N_DEV_TABLE         16u
4438 static u32 skd_next_devno;
4439
4440 static struct skd_device *skd_construct(struct pci_dev *pdev)
4441 {
4442         struct skd_device *skdev;
4443         int blk_major = skd_major;
4444         int rc;
4445
4446         skdev = kzalloc(sizeof(*skdev), GFP_KERNEL);
4447
4448         if (!skdev) {
4449                 pr_err(PFX "(%s): memory alloc failure\n",
4450                        pci_name(pdev));
4451                 return NULL;
4452         }
4453
4454         skdev->state = SKD_DRVR_STATE_LOAD;
4455         skdev->pdev = pdev;
4456         skdev->devno = skd_next_devno++;
4457         skdev->major = blk_major;
4458         skdev->irq_type = skd_isr_type;
4459         sprintf(skdev->name, DRV_NAME "%d", skdev->devno);
4460         skdev->dev_max_queue_depth = 0;
4461
4462         skdev->num_req_context = skd_max_queue_depth;
4463         skdev->num_fitmsg_context = skd_max_queue_depth;
4464         skdev->n_special = skd_max_pass_thru;
4465         skdev->cur_max_queue_depth = 1;
4466         skdev->queue_low_water_mark = 1;
4467         skdev->proto_ver = 99;
4468         skdev->sgs_per_request = skd_sgs_per_request;
4469         skdev->dbg_level = skd_dbg_level;
4470
4471         if (skd_bio)
4472                 bio_list_init(&skdev->bio_queue);
4473
4474
4475         atomic_set(&skdev->device_count, 0);
4476
4477         spin_lock_init(&skdev->lock);
4478
4479         INIT_WORK(&skdev->completion_worker, skd_completion_worker);
4480         INIT_LIST_HEAD(&skdev->flush_list);
4481
4482         VPRINTK(skdev, "skcomp\n");
4483         rc = skd_cons_skcomp(skdev);
4484         if (rc < 0)
4485                 goto err_out;
4486
4487         VPRINTK(skdev, "skmsg\n");
4488         rc = skd_cons_skmsg(skdev);
4489         if (rc < 0)
4490                 goto err_out;
4491
4492         VPRINTK(skdev, "skreq\n");
4493         rc = skd_cons_skreq(skdev);
4494         if (rc < 0)
4495                 goto err_out;
4496
4497         VPRINTK(skdev, "skspcl\n");
4498         rc = skd_cons_skspcl(skdev);
4499         if (rc < 0)
4500                 goto err_out;
4501
4502         VPRINTK(skdev, "sksb\n");
4503         rc = skd_cons_sksb(skdev);
4504         if (rc < 0)
4505                 goto err_out;
4506
4507         VPRINTK(skdev, "disk\n");
4508         rc = skd_cons_disk(skdev);
4509         if (rc < 0)
4510                 goto err_out;
4511
4512
4513
4514         DPRINTK(skdev, "VICTORY\n");
4515         return skdev;
4516
4517 err_out:
4518         DPRINTK(skdev, "construct failed\n");
4519         skd_destruct(skdev);
4520         return NULL;
4521 }
4522
4523 static int skd_cons_skcomp(struct skd_device *skdev)
4524 {
4525         int rc = 0;
4526         struct fit_completion_entry_v1 *skcomp;
4527         u32 nbytes;
4528
4529         nbytes = sizeof(*skcomp) * SKD_N_COMPLETION_ENTRY;
4530         nbytes += sizeof(struct fit_comp_error_info) * SKD_N_COMPLETION_ENTRY;
4531
4532         VPRINTK(skdev, "comp pci_alloc, total bytes %d entries %d\n", nbytes,
4533                 SKD_N_COMPLETION_ENTRY);
4534
4535         skcomp = pci_alloc_consistent(skdev->pdev, nbytes,
4536                                       &skdev->cq_dma_address);
4537
4538         if (skcomp == NULL) {
4539                 rc = -ENOMEM;
4540                 goto err_out;
4541         }
4542
4543         memset(skcomp, 0, nbytes);
4544
4545         skdev->skcomp_table = skcomp;
4546         skdev->skerr_table = (struct fit_comp_error_info *)((char *)skcomp +
4547                                                            sizeof(*skcomp) *
4548                                                            SKD_N_COMPLETION_ENTRY);
4549
4550 err_out:
4551         return rc;
4552 }
4553
4554 static int skd_cons_skmsg(struct skd_device *skdev)
4555 {
4556         int rc = 0;
4557         u32 i;
4558
4559         VPRINTK(skdev, "skmsg_table kzalloc, struct %u, count %u total %lu\n",
4560                 sizeof(struct skd_fitmsg_context),
4561                 skdev->num_fitmsg_context,
4562                 (unsigned long) sizeof(struct skd_fitmsg_context) *
4563                                         skdev->num_fitmsg_context);
4564
4565         skdev->skmsg_table = kzalloc(sizeof(struct skd_fitmsg_context)
4566                                      *skdev->num_fitmsg_context, GFP_KERNEL);
4567         if (skdev->skmsg_table == NULL) {
4568                 rc = -ENOMEM;
4569                 goto err_out;
4570         }
4571
4572         for (i = 0; i < skdev->num_fitmsg_context; i++) {
4573                 struct skd_fitmsg_context *skmsg;
4574
4575                 skmsg = &skdev->skmsg_table[i];
4576
4577                 skmsg->id = i + SKD_ID_FIT_MSG;
4578
4579                 skmsg->state = SKD_MSG_STATE_IDLE;
4580                 skmsg->msg_buf = pci_alloc_consistent(skdev->pdev,
4581                                                       SKD_N_FITMSG_BYTES + 64,
4582                                                       &skmsg->mb_dma_address);
4583
4584                 if (skmsg->msg_buf == NULL) {
4585                         rc = -ENOMEM;
4586                         goto err_out;
4587                 }
4588
4589                 skmsg->offset = (u32)((u64)skmsg->msg_buf &
4590                                       (~FIT_QCMD_BASE_ADDRESS_MASK));
4591                 skmsg->msg_buf += ~FIT_QCMD_BASE_ADDRESS_MASK;
4592                 skmsg->msg_buf = (u8 *)((u64)skmsg->msg_buf &
4593                                        FIT_QCMD_BASE_ADDRESS_MASK);
4594                 skmsg->mb_dma_address += ~FIT_QCMD_BASE_ADDRESS_MASK;
4595                 skmsg->mb_dma_address &= FIT_QCMD_BASE_ADDRESS_MASK;
4596                 memset(skmsg->msg_buf, 0, SKD_N_FITMSG_BYTES);
4597
4598                 skmsg->next = &skmsg[1];
4599         }
4600
4601         /* Free list is in order starting with the 0th entry. */
4602         skdev->skmsg_table[i - 1].next = NULL;
4603         skdev->skmsg_free_list = skdev->skmsg_table;
4604
4605 err_out:
4606         return rc;
4607 }
4608
4609 static int skd_cons_skreq(struct skd_device *skdev)
4610 {
4611         int rc = 0;
4612         u32 i;
4613
4614         VPRINTK(skdev, "skreq_table kzalloc, struct %u, count %u total %u\n",
4615                 sizeof(struct skd_request_context),
4616                 skdev->num_req_context,
4617                 sizeof(struct skd_request_context) * skdev->num_req_context);
4618
4619         skdev->skreq_table = kzalloc(sizeof(struct skd_request_context)
4620                                      * skdev->num_req_context, GFP_KERNEL);
4621         if (skdev->skreq_table == NULL) {
4622                 rc = -ENOMEM;
4623                 goto err_out;
4624         }
4625
4626         VPRINTK(skdev, "alloc sg_table sg_per_req %u scatlist %u total %u\n",
4627                 skdev->sgs_per_request, sizeof(struct scatterlist),
4628                 skdev->sgs_per_request * sizeof(struct scatterlist));
4629
4630         for (i = 0; i < skdev->num_req_context; i++) {
4631                 struct skd_request_context *skreq;
4632
4633                 skreq = &skdev->skreq_table[i];
4634
4635                 skreq->id = i + SKD_ID_RW_REQUEST;
4636                 skreq->state = SKD_REQ_STATE_IDLE;
4637
4638                 skreq->sg = kzalloc(sizeof(struct scatterlist) *
4639                                     skdev->sgs_per_request, GFP_KERNEL);
4640                 if (skreq->sg == NULL) {
4641                         rc = -ENOMEM;
4642                         goto err_out;
4643                 }
4644                 sg_init_table(skreq->sg, skdev->sgs_per_request);
4645
4646                 skreq->sksg_list = skd_cons_sg_list(skdev,
4647                                                     skdev->sgs_per_request,
4648                                                     &skreq->sksg_dma_address);
4649
4650                 if (skreq->sksg_list == NULL) {
4651                         rc = -ENOMEM;
4652                         goto err_out;
4653                 }
4654
4655                 skreq->next = &skreq[1];
4656         }
4657
4658         /* Free list is in order starting with the 0th entry. */
4659         skdev->skreq_table[i - 1].next = NULL;
4660         skdev->skreq_free_list = skdev->skreq_table;
4661
4662 err_out:
4663         return rc;
4664 }
4665
4666 static int skd_cons_skspcl(struct skd_device *skdev)
4667 {
4668         int rc = 0;
4669         u32 i, nbytes;
4670
4671         VPRINTK(skdev, "skspcl_table kzalloc, struct %u, count %u total %u\n",
4672                 sizeof(struct skd_special_context),
4673                 skdev->n_special,
4674                 sizeof(struct skd_special_context) * skdev->n_special);
4675
4676         skdev->skspcl_table = kzalloc(sizeof(struct skd_special_context)
4677                                       * skdev->n_special, GFP_KERNEL);
4678         if (skdev->skspcl_table == NULL) {
4679                 rc = -ENOMEM;
4680                 goto err_out;
4681         }
4682
4683         for (i = 0; i < skdev->n_special; i++) {
4684                 struct skd_special_context *skspcl;
4685
4686                 skspcl = &skdev->skspcl_table[i];
4687
4688                 skspcl->req.id = i + SKD_ID_SPECIAL_REQUEST;
4689                 skspcl->req.state = SKD_REQ_STATE_IDLE;
4690
4691                 skspcl->req.next = &skspcl[1].req;
4692
4693                 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4694
4695                 skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4696                                                        &skspcl->mb_dma_address);
4697                 if (skspcl->msg_buf == NULL) {
4698                         rc = -ENOMEM;
4699                         goto err_out;
4700                 }
4701
4702                 memset(skspcl->msg_buf, 0, nbytes);
4703
4704                 skspcl->req.sg = kzalloc(sizeof(struct scatterlist) *
4705                                          SKD_N_SG_PER_SPECIAL, GFP_KERNEL);
4706                 if (skspcl->req.sg == NULL) {
4707                         rc = -ENOMEM;
4708                         goto err_out;
4709                 }
4710
4711                 skspcl->req.sksg_list = skd_cons_sg_list(skdev,
4712                                                          SKD_N_SG_PER_SPECIAL,
4713                                                          &skspcl->req.
4714                                                          sksg_dma_address);
4715                 if (skspcl->req.sksg_list == NULL) {
4716                         rc = -ENOMEM;
4717                         goto err_out;
4718                 }
4719         }
4720
4721         /* Free list is in order starting with the 0th entry. */
4722         skdev->skspcl_table[i - 1].req.next = NULL;
4723         skdev->skspcl_free_list = skdev->skspcl_table;
4724
4725         return rc;
4726
4727 err_out:
4728         return rc;
4729 }
4730
4731 static int skd_cons_sksb(struct skd_device *skdev)
4732 {
4733         int rc = 0;
4734         struct skd_special_context *skspcl;
4735         u32 nbytes;
4736
4737         skspcl = &skdev->internal_skspcl;
4738
4739         skspcl->req.id = 0 + SKD_ID_INTERNAL;
4740         skspcl->req.state = SKD_REQ_STATE_IDLE;
4741
4742         nbytes = SKD_N_INTERNAL_BYTES;
4743
4744         skspcl->data_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4745                                                 &skspcl->db_dma_address);
4746         if (skspcl->data_buf == NULL) {
4747                 rc = -ENOMEM;
4748                 goto err_out;
4749         }
4750
4751         memset(skspcl->data_buf, 0, nbytes);
4752
4753         nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
4754         skspcl->msg_buf = pci_alloc_consistent(skdev->pdev, nbytes,
4755                                                &skspcl->mb_dma_address);
4756         if (skspcl->msg_buf == NULL) {
4757                 rc = -ENOMEM;
4758                 goto err_out;
4759         }
4760
4761         memset(skspcl->msg_buf, 0, nbytes);
4762
4763         skspcl->req.sksg_list = skd_cons_sg_list(skdev, 1,
4764                                                  &skspcl->req.sksg_dma_address);
4765         if (skspcl->req.sksg_list == NULL) {
4766                 rc = -ENOMEM;
4767                 goto err_out;
4768         }
4769
4770         if (!skd_format_internal_skspcl(skdev)) {
4771                 rc = -EINVAL;
4772                 goto err_out;
4773         }
4774
4775 err_out:
4776         return rc;
4777 }
4778
4779 static struct fit_sg_descriptor *skd_cons_sg_list(struct skd_device *skdev,
4780                                                   u32 n_sg,
4781                                                   dma_addr_t *ret_dma_addr)
4782 {
4783         struct fit_sg_descriptor *sg_list;
4784         u32 nbytes;
4785
4786         nbytes = sizeof(*sg_list) * n_sg;
4787
4788         sg_list = pci_alloc_consistent(skdev->pdev, nbytes, ret_dma_addr);
4789
4790         if (sg_list != NULL) {
4791                 uint64_t dma_address = *ret_dma_addr;
4792                 u32 i;
4793
4794                 memset(sg_list, 0, nbytes);
4795
4796                 for (i = 0; i < n_sg - 1; i++) {
4797                         uint64_t ndp_off;
4798                         ndp_off = (i + 1) * sizeof(struct fit_sg_descriptor);
4799
4800                         sg_list[i].next_desc_ptr = dma_address + ndp_off;
4801                 }
4802                 sg_list[i].next_desc_ptr = 0LL;
4803         }
4804
4805         return sg_list;
4806 }
4807
4808 static int skd_cons_disk(struct skd_device *skdev)
4809 {
4810         int rc = 0;
4811         struct gendisk *disk;
4812         struct request_queue *q;
4813         unsigned long flags;
4814
4815         disk = alloc_disk(SKD_MINORS_PER_DEVICE);
4816         if (!disk) {
4817                 rc = -ENOMEM;
4818                 goto err_out;
4819         }
4820
4821         skdev->disk = disk;
4822         sprintf(disk->disk_name, DRV_NAME "%u", skdev->devno);
4823
4824         disk->major = skdev->major;
4825         disk->first_minor = skdev->devno * SKD_MINORS_PER_DEVICE;
4826         disk->fops = &skd_blockdev_ops;
4827         disk->private_data = skdev;
4828
4829         if (!skd_bio) {
4830                 q = blk_init_queue(skd_request_fn, &skdev->lock);
4831         } else {
4832                 q = blk_alloc_queue(GFP_KERNEL);
4833                 q->queue_flags = QUEUE_FLAG_IO_STAT | QUEUE_FLAG_STACKABLE;
4834         }
4835
4836         if (!q) {
4837                 rc = -ENOMEM;
4838                 goto err_out;
4839         }
4840
4841         skdev->queue = q;
4842         disk->queue = q;
4843         q->queuedata = skdev;
4844
4845         if (skd_bio) {
4846                 q->queue_lock = &skdev->lock;
4847                 blk_queue_make_request(q, skd_make_request);
4848         }
4849
4850         blk_queue_flush(q, REQ_FLUSH | REQ_FUA);
4851         blk_queue_max_segments(q, skdev->sgs_per_request);
4852         blk_queue_max_hw_sectors(q, SKD_N_MAX_SECTORS);
4853
4854         /* set sysfs ptimal_io_size to 8K */
4855         blk_queue_io_opt(q, 8192);
4856
4857         /* DISCARD Flag initialization. */
4858         q->limits.discard_granularity = 8192;
4859         q->limits.discard_alignment = 0;
4860         q->limits.max_discard_sectors = UINT_MAX >> 9;
4861         q->limits.discard_zeroes_data = 1;
4862         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4863         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
4864
4865         spin_lock_irqsave(&skdev->lock, flags);
4866         VPRINTK(skdev, "stopping %s queue\n", skdev->name);
4867         skd_stop_queue(skdev);
4868         spin_unlock_irqrestore(&skdev->lock, flags);
4869
4870 err_out:
4871         return rc;
4872 }
4873
4874 /*
4875  *****************************************************************************
4876  * DESTRUCT (FREE)
4877  *****************************************************************************
4878  */
4879
4880 static void skd_free_skcomp(struct skd_device *skdev);
4881 static void skd_free_skmsg(struct skd_device *skdev);
4882 static void skd_free_skreq(struct skd_device *skdev);
4883 static void skd_free_skspcl(struct skd_device *skdev);
4884 static void skd_free_sksb(struct skd_device *skdev);
4885 static void skd_free_sg_list(struct skd_device *skdev,
4886                              struct fit_sg_descriptor *sg_list,
4887                              u32 n_sg, dma_addr_t dma_addr);
4888 static void skd_free_disk(struct skd_device *skdev);
4889
4890 static void skd_destruct(struct skd_device *skdev)
4891 {
4892         if (skdev == NULL)
4893                 return;
4894
4895
4896         VPRINTK(skdev, "disk\n");
4897         skd_free_disk(skdev);
4898
4899         VPRINTK(skdev, "sksb\n");
4900         skd_free_sksb(skdev);
4901
4902         VPRINTK(skdev, "skspcl\n");
4903         skd_free_skspcl(skdev);
4904
4905         VPRINTK(skdev, "skreq\n");
4906         skd_free_skreq(skdev);
4907
4908         VPRINTK(skdev, "skmsg\n");
4909         skd_free_skmsg(skdev);
4910
4911         VPRINTK(skdev, "skcomp\n");
4912         skd_free_skcomp(skdev);
4913
4914         VPRINTK(skdev, "skdev\n");
4915         kfree(skdev);
4916 }
4917
4918 static void skd_free_skcomp(struct skd_device *skdev)
4919 {
4920         if (skdev->skcomp_table != NULL) {
4921                 u32 nbytes;
4922
4923                 nbytes = sizeof(skdev->skcomp_table[0]) *
4924                          SKD_N_COMPLETION_ENTRY;
4925                 pci_free_consistent(skdev->pdev, nbytes,
4926                                     skdev->skcomp_table, skdev->cq_dma_address);
4927         }
4928
4929         skdev->skcomp_table = NULL;
4930         skdev->cq_dma_address = 0;
4931 }
4932
4933 static void skd_free_skmsg(struct skd_device *skdev)
4934 {
4935         u32 i;
4936
4937         if (skdev->skmsg_table == NULL)
4938                 return;
4939
4940         for (i = 0; i < skdev->num_fitmsg_context; i++) {
4941                 struct skd_fitmsg_context *skmsg;
4942
4943                 skmsg = &skdev->skmsg_table[i];
4944
4945                 if (skmsg->msg_buf != NULL) {
4946                         skmsg->msg_buf += skmsg->offset;
4947                         skmsg->mb_dma_address += skmsg->offset;
4948                         pci_free_consistent(skdev->pdev, SKD_N_FITMSG_BYTES,
4949                                             skmsg->msg_buf,
4950                                             skmsg->mb_dma_address);
4951                 }
4952                 skmsg->msg_buf = NULL;
4953                 skmsg->mb_dma_address = 0;
4954         }
4955
4956         kfree(skdev->skmsg_table);
4957         skdev->skmsg_table = NULL;
4958 }
4959
4960 static void skd_free_skreq(struct skd_device *skdev)
4961 {
4962         u32 i;
4963
4964         if (skdev->skreq_table == NULL)
4965                 return;
4966
4967         for (i = 0; i < skdev->num_req_context; i++) {
4968                 struct skd_request_context *skreq;
4969
4970                 skreq = &skdev->skreq_table[i];
4971
4972                 skd_free_sg_list(skdev, skreq->sksg_list,
4973                                  skdev->sgs_per_request,
4974                                  skreq->sksg_dma_address);
4975
4976                 skreq->sksg_list = NULL;
4977                 skreq->sksg_dma_address = 0;
4978
4979                 kfree(skreq->sg);
4980         }
4981
4982         kfree(skdev->skreq_table);
4983         skdev->skreq_table = NULL;
4984 }
4985
4986 static void skd_free_skspcl(struct skd_device *skdev)
4987 {
4988         u32 i;
4989         u32 nbytes;
4990
4991         if (skdev->skspcl_table == NULL)
4992                 return;
4993
4994         for (i = 0; i < skdev->n_special; i++) {
4995                 struct skd_special_context *skspcl;
4996
4997                 skspcl = &skdev->skspcl_table[i];
4998
4999                 if (skspcl->msg_buf != NULL) {
5000                         nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
5001                         pci_free_consistent(skdev->pdev, nbytes,
5002                                             skspcl->msg_buf,
5003                                             skspcl->mb_dma_address);
5004                 }
5005
5006                 skspcl->msg_buf = NULL;
5007                 skspcl->mb_dma_address = 0;
5008
5009                 skd_free_sg_list(skdev, skspcl->req.sksg_list,
5010                                  SKD_N_SG_PER_SPECIAL,
5011                                  skspcl->req.sksg_dma_address);
5012
5013                 skspcl->req.sksg_list = NULL;
5014                 skspcl->req.sksg_dma_address = 0;
5015
5016                 kfree(skspcl->req.sg);
5017         }
5018
5019         kfree(skdev->skspcl_table);
5020         skdev->skspcl_table = NULL;
5021 }
5022
5023 static void skd_free_sksb(struct skd_device *skdev)
5024 {
5025         struct skd_special_context *skspcl;
5026         u32 nbytes;
5027
5028         skspcl = &skdev->internal_skspcl;
5029
5030         if (skspcl->data_buf != NULL) {
5031                 nbytes = SKD_N_INTERNAL_BYTES;
5032
5033                 pci_free_consistent(skdev->pdev, nbytes,
5034                                     skspcl->data_buf, skspcl->db_dma_address);
5035         }
5036
5037         skspcl->data_buf = NULL;
5038         skspcl->db_dma_address = 0;
5039
5040         if (skspcl->msg_buf != NULL) {
5041                 nbytes = SKD_N_SPECIAL_FITMSG_BYTES;
5042                 pci_free_consistent(skdev->pdev, nbytes,
5043                                     skspcl->msg_buf, skspcl->mb_dma_address);
5044         }
5045
5046         skspcl->msg_buf = NULL;
5047         skspcl->mb_dma_address = 0;
5048
5049         skd_free_sg_list(skdev, skspcl->req.sksg_list, 1,
5050                          skspcl->req.sksg_dma_address);
5051
5052         skspcl->req.sksg_list = NULL;
5053         skspcl->req.sksg_dma_address = 0;
5054 }
5055
5056 static void skd_free_sg_list(struct skd_device *skdev,
5057                              struct fit_sg_descriptor *sg_list,
5058                              u32 n_sg, dma_addr_t dma_addr)
5059 {
5060         if (sg_list != NULL) {
5061                 u32 nbytes;
5062
5063                 nbytes = sizeof(*sg_list) * n_sg;
5064
5065                 pci_free_consistent(skdev->pdev, nbytes, sg_list, dma_addr);
5066         }
5067 }
5068
5069 static void skd_free_disk(struct skd_device *skdev)
5070 {
5071         struct gendisk *disk = skdev->disk;
5072
5073         if (disk != NULL) {
5074                 struct request_queue *q = disk->queue;
5075
5076                 if (disk->flags & GENHD_FL_UP)
5077                         del_gendisk(disk);
5078                 if (q)
5079                         blk_cleanup_queue(q);
5080                 put_disk(disk);
5081         }
5082         skdev->disk = NULL;
5083 }
5084
5085
5086
5087 /*
5088  *****************************************************************************
5089  * BLOCK DEVICE (BDEV) GLUE
5090  *****************************************************************************
5091  */
5092
5093 static int skd_bdev_getgeo(struct block_device *bdev, struct hd_geometry *geo)
5094 {
5095         struct skd_device *skdev;
5096         u64 capacity;
5097
5098         skdev = bdev->bd_disk->private_data;
5099
5100         DPRINTK(skdev, "%s: CMD[%s] getgeo device\n",
5101                 bdev->bd_disk->disk_name, current->comm);
5102
5103         if (skdev->read_cap_is_valid) {
5104                 capacity = get_capacity(skdev->disk);
5105                 geo->heads = 64;
5106                 geo->sectors = 255;
5107                 geo->cylinders = (capacity) / (255 * 64);
5108
5109                 return 0;
5110         }
5111         return -EIO;
5112 }
5113
5114 static int skd_bdev_attach(struct skd_device *skdev)
5115 {
5116         DPRINTK(skdev, "add_disk\n");
5117         add_disk(skdev->disk);
5118         return 0;
5119 }
5120
5121 static const struct block_device_operations skd_blockdev_ops = {
5122         .owner          = THIS_MODULE,
5123         .ioctl          = skd_bdev_ioctl,
5124         .getgeo         = skd_bdev_getgeo,
5125 };
5126
5127
5128 /*
5129  *****************************************************************************
5130  * PCIe DRIVER GLUE
5131  *****************************************************************************
5132  */
5133
5134 static DEFINE_PCI_DEVICE_TABLE(skd_pci_tbl) = {
5135         { PCI_VENDOR_ID_STEC, PCI_DEVICE_ID_S1120,
5136           PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
5137         { 0 }                     /* terminate list */
5138 };
5139
5140 MODULE_DEVICE_TABLE(pci, skd_pci_tbl);
5141
5142 static char *skd_pci_info(struct skd_device *skdev, char *str)
5143 {
5144         int pcie_reg;
5145
5146         strcpy(str, "PCIe (");
5147         pcie_reg = pci_find_capability(skdev->pdev, PCI_CAP_ID_EXP);
5148
5149         if (pcie_reg) {
5150
5151                 char lwstr[6];
5152                 uint16_t pcie_lstat, lspeed, lwidth;
5153
5154                 pcie_reg += 0x12;
5155                 pci_read_config_word(skdev->pdev, pcie_reg, &pcie_lstat);
5156                 lspeed = pcie_lstat & (0xF);
5157                 lwidth = (pcie_lstat & 0x3F0) >> 4;
5158
5159                 if (lspeed == 1)
5160                         strcat(str, "2.5GT/s ");
5161                 else if (lspeed == 2)
5162                         strcat(str, "5.0GT/s ");
5163                 else
5164                         strcat(str, "<unknown> ");
5165                 snprintf(lwstr, sizeof(lwstr), "%dX)", lwidth);
5166                 strcat(str, lwstr);
5167         }
5168         return str;
5169 }
5170
5171 static int skd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
5172 {
5173         int i;
5174         int rc = 0;
5175         char pci_str[32];
5176         struct skd_device *skdev;
5177
5178         pr_info("STEC s1120 Driver(%s) version %s-b%s\n",
5179                DRV_NAME, DRV_VERSION, DRV_BUILD_ID);
5180         pr_info("(skd?:??:[%s]): vendor=%04X device=%04x\n",
5181                pci_name(pdev), pdev->vendor, pdev->device);
5182
5183         rc = pci_enable_device(pdev);
5184         if (rc)
5185                 return rc;
5186         rc = pci_request_regions(pdev, DRV_NAME);
5187         if (rc)
5188                 goto err_out;
5189         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5190         if (!rc) {
5191                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5192
5193                         pr_err("(%s): consistent DMA mask error %d\n",
5194                                pci_name(pdev), rc);
5195                 }
5196         } else {
5197                 (rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)));
5198                 if (rc) {
5199
5200                         pr_err("(%s): DMA mask error %d\n",
5201                                pci_name(pdev), rc);
5202                         goto err_out_regions;
5203                 }
5204         }
5205
5206         skdev = skd_construct(pdev);
5207         if (skdev == NULL)
5208                 goto err_out_regions;
5209
5210         skd_pci_info(skdev, pci_str);
5211         pr_info("(%s): %s 64bit\n", skd_name(skdev), pci_str);
5212
5213         pci_set_master(pdev);
5214         rc = pci_enable_pcie_error_reporting(pdev);
5215         if (rc) {
5216                 pr_err(
5217                        "(%s): bad enable of PCIe error reporting rc=%d\n",
5218                        skd_name(skdev), rc);
5219                 skdev->pcie_error_reporting_is_enabled = 0;
5220         } else
5221                 skdev->pcie_error_reporting_is_enabled = 1;
5222
5223
5224         pci_set_drvdata(pdev, skdev);
5225         skdev->pdev = pdev;
5226         skdev->disk->driverfs_dev = &pdev->dev;
5227
5228         for (i = 0; i < SKD_MAX_BARS; i++) {
5229                 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5230                 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5231                 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5232                                             skdev->mem_size[i]);
5233                 if (!skdev->mem_map[i]) {
5234                         pr_err("(%s): Unable to map adapter memory!\n",
5235                                skd_name(skdev));
5236                         rc = -ENODEV;
5237                         goto err_out_iounmap;
5238                 }
5239                 DPRINTK(skdev, "mem_map=%p, phyd=%016llx, size=%d\n",
5240                         skdev->mem_map[i],
5241                         (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
5242         }
5243
5244         rc = skd_acquire_irq(skdev);
5245         if (rc) {
5246                 pr_err("(%s): interrupt resource error %d\n",
5247                        skd_name(skdev), rc);
5248                 goto err_out_iounmap;
5249         }
5250
5251         rc = skd_start_timer(skdev);
5252         if (rc)
5253                 goto err_out_timer;
5254
5255         init_waitqueue_head(&skdev->waitq);
5256
5257         skd_start_device(skdev);
5258
5259         rc = wait_event_interruptible_timeout(skdev->waitq,
5260                                               (skdev->gendisk_on),
5261                                               (SKD_START_WAIT_SECONDS * HZ));
5262         if (skdev->gendisk_on > 0) {
5263                 /* device came on-line after reset */
5264                 skd_bdev_attach(skdev);
5265                 rc = 0;
5266         } else {
5267                 /* we timed out, something is wrong with the device,
5268                    don't add the disk structure */
5269                 pr_err(
5270                        "(%s): error: waiting for s1120 timed out %d!\n",
5271                        skd_name(skdev), rc);
5272                 /* in case of no error; we timeout with ENXIO */
5273                 if (!rc)
5274                         rc = -ENXIO;
5275                 goto err_out_timer;
5276         }
5277
5278
5279 #ifdef SKD_VMK_POLL_HANDLER
5280         if (skdev->irq_type == SKD_IRQ_MSIX) {
5281                 /* MSIX completion handler is being used for coredump */
5282                 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5283                                                   skdev->msix_entries[5].vector,
5284                                                   skd_comp_q, skdev);
5285         } else {
5286                 vmklnx_scsi_register_poll_handler(skdev->scsi_host,
5287                                                   skdev->pdev->irq, skd_isr,
5288                                                   skdev);
5289         }
5290 #endif  /* SKD_VMK_POLL_HANDLER */
5291
5292         return rc;
5293
5294 err_out_timer:
5295         skd_stop_device(skdev);
5296         skd_release_irq(skdev);
5297
5298 err_out_iounmap:
5299         for (i = 0; i < SKD_MAX_BARS; i++)
5300                 if (skdev->mem_map[i])
5301                         iounmap(skdev->mem_map[i]);
5302
5303         if (skdev->pcie_error_reporting_is_enabled)
5304                 pci_disable_pcie_error_reporting(pdev);
5305
5306         skd_destruct(skdev);
5307
5308 err_out_regions:
5309         pci_release_regions(pdev);
5310
5311 err_out:
5312         pci_disable_device(pdev);
5313         pci_set_drvdata(pdev, NULL);
5314         return rc;
5315 }
5316
5317 static void skd_pci_remove(struct pci_dev *pdev)
5318 {
5319         int i;
5320         struct skd_device *skdev;
5321
5322         skdev = pci_get_drvdata(pdev);
5323         if (!skdev) {
5324                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5325                 return;
5326         }
5327         skd_stop_device(skdev);
5328         skd_release_irq(skdev);
5329
5330         for (i = 0; i < SKD_MAX_BARS; i++)
5331                 if (skdev->mem_map[i])
5332                         iounmap((u32 *)skdev->mem_map[i]);
5333
5334         if (skdev->pcie_error_reporting_is_enabled)
5335                 pci_disable_pcie_error_reporting(pdev);
5336
5337         skd_destruct(skdev);
5338
5339         pci_release_regions(pdev);
5340         pci_disable_device(pdev);
5341         pci_set_drvdata(pdev, NULL);
5342
5343         return;
5344 }
5345
5346 static int skd_pci_suspend(struct pci_dev *pdev, pm_message_t state)
5347 {
5348         int i;
5349         struct skd_device *skdev;
5350
5351         skdev = pci_get_drvdata(pdev);
5352         if (!skdev) {
5353                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5354                 return -EIO;
5355         }
5356
5357         skd_stop_device(skdev);
5358
5359         skd_release_irq(skdev);
5360
5361         for (i = 0; i < SKD_MAX_BARS; i++)
5362                 if (skdev->mem_map[i])
5363                         iounmap((u32 *)skdev->mem_map[i]);
5364
5365         if (skdev->pcie_error_reporting_is_enabled)
5366                 pci_disable_pcie_error_reporting(pdev);
5367
5368         pci_release_regions(pdev);
5369         pci_save_state(pdev);
5370         pci_disable_device(pdev);
5371         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5372         return 0;
5373 }
5374
5375 static int skd_pci_resume(struct pci_dev *pdev)
5376 {
5377         int i;
5378         int rc = 0;
5379         struct skd_device *skdev;
5380
5381         skdev = pci_get_drvdata(pdev);
5382         if (!skdev) {
5383                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5384                 return -1;
5385         }
5386
5387         pci_set_power_state(pdev, PCI_D0);
5388         pci_enable_wake(pdev, PCI_D0, 0);
5389         pci_restore_state(pdev);
5390
5391         rc = pci_enable_device(pdev);
5392         if (rc)
5393                 return rc;
5394         rc = pci_request_regions(pdev, DRV_NAME);
5395         if (rc)
5396                 goto err_out;
5397         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5398         if (!rc) {
5399                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) {
5400
5401                         pr_err("(%s): consistent DMA mask error %d\n",
5402                                pci_name(pdev), rc);
5403                 }
5404         } else {
5405                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5406                 if (rc) {
5407
5408                         pr_err("(%s): DMA mask error %d\n",
5409                                pci_name(pdev), rc);
5410                         goto err_out_regions;
5411                 }
5412         }
5413
5414         pci_set_master(pdev);
5415         rc = pci_enable_pcie_error_reporting(pdev);
5416         if (rc) {
5417                 pr_err("(%s): bad enable of PCIe error reporting rc=%d\n",
5418                        skdev->name, rc);
5419                 skdev->pcie_error_reporting_is_enabled = 0;
5420         } else
5421                 skdev->pcie_error_reporting_is_enabled = 1;
5422
5423         for (i = 0; i < SKD_MAX_BARS; i++) {
5424
5425                 skdev->mem_phys[i] = pci_resource_start(pdev, i);
5426                 skdev->mem_size[i] = (u32)pci_resource_len(pdev, i);
5427                 skdev->mem_map[i] = ioremap(skdev->mem_phys[i],
5428                                             skdev->mem_size[i]);
5429                 if (!skdev->mem_map[i]) {
5430                         pr_err("(%s): Unable to map adapter memory!\n",
5431                                skd_name(skdev));
5432                         rc = -ENODEV;
5433                         goto err_out_iounmap;
5434                 }
5435                 DPRINTK(skdev, "mem_map=%p, phyd=%016llx, size=%d\n",
5436                         skdev->mem_map[i],
5437                         (uint64_t)skdev->mem_phys[i], skdev->mem_size[i]);
5438         }
5439         rc = skd_acquire_irq(skdev);
5440         if (rc) {
5441
5442                 pr_err("(%s): interrupt resource error %d\n",
5443                        pci_name(pdev), rc);
5444                 goto err_out_iounmap;
5445         }
5446
5447         rc = skd_start_timer(skdev);
5448         if (rc)
5449                 goto err_out_timer;
5450
5451         init_waitqueue_head(&skdev->waitq);
5452
5453         skd_start_device(skdev);
5454
5455         return rc;
5456
5457 err_out_timer:
5458         skd_stop_device(skdev);
5459         skd_release_irq(skdev);
5460
5461 err_out_iounmap:
5462         for (i = 0; i < SKD_MAX_BARS; i++)
5463                 if (skdev->mem_map[i])
5464                         iounmap(skdev->mem_map[i]);
5465
5466         if (skdev->pcie_error_reporting_is_enabled)
5467                 pci_disable_pcie_error_reporting(pdev);
5468
5469 err_out_regions:
5470         pci_release_regions(pdev);
5471
5472 err_out:
5473         pci_disable_device(pdev);
5474         return rc;
5475 }
5476
5477 static void skd_pci_shutdown(struct pci_dev *pdev)
5478 {
5479         struct skd_device *skdev;
5480
5481         pr_err("skd_pci_shutdown called\n");
5482
5483         skdev = pci_get_drvdata(pdev);
5484         if (!skdev) {
5485                 pr_err("%s: no device data for PCI\n", pci_name(pdev));
5486                 return;
5487         }
5488
5489         pr_err("%s: calling stop\n", skd_name(skdev));
5490         skd_stop_device(skdev);
5491 }
5492
5493 static struct pci_driver skd_driver = {
5494         .name           = DRV_NAME,
5495         .id_table       = skd_pci_tbl,
5496         .probe          = skd_pci_probe,
5497         .remove         = skd_pci_remove,
5498         .suspend        = skd_pci_suspend,
5499         .resume         = skd_pci_resume,
5500         .shutdown       = skd_pci_shutdown,
5501 };
5502
5503 /*
5504  *****************************************************************************
5505  * LOGGING SUPPORT
5506  *****************************************************************************
5507  */
5508
5509 static const char *skd_name(struct skd_device *skdev)
5510 {
5511         memset(skdev->id_str, 0, sizeof(skdev->id_str));
5512
5513         if (skdev->inquiry_is_valid)
5514                 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:%s:[%s]",
5515                          skdev->name, skdev->inq_serial_num,
5516                          pci_name(skdev->pdev));
5517         else
5518                 snprintf(skdev->id_str, sizeof(skdev->id_str), "%s:??:[%s]",
5519                          skdev->name, pci_name(skdev->pdev));
5520
5521         return skdev->id_str;
5522 }
5523
5524 const char *skd_drive_state_to_str(int state)
5525 {
5526         switch (state) {
5527         case FIT_SR_DRIVE_OFFLINE:
5528                 return "OFFLINE";
5529         case FIT_SR_DRIVE_INIT:
5530                 return "INIT";
5531         case FIT_SR_DRIVE_ONLINE:
5532                 return "ONLINE";
5533         case FIT_SR_DRIVE_BUSY:
5534                 return "BUSY";
5535         case FIT_SR_DRIVE_FAULT:
5536                 return "FAULT";
5537         case FIT_SR_DRIVE_DEGRADED:
5538                 return "DEGRADED";
5539         case FIT_SR_PCIE_LINK_DOWN:
5540                 return "INK_DOWN";
5541         case FIT_SR_DRIVE_SOFT_RESET:
5542                 return "SOFT_RESET";
5543         case FIT_SR_DRIVE_NEED_FW_DOWNLOAD:
5544                 return "NEED_FW";
5545         case FIT_SR_DRIVE_INIT_FAULT:
5546                 return "INIT_FAULT";
5547         case FIT_SR_DRIVE_BUSY_SANITIZE:
5548                 return "BUSY_SANITIZE";
5549         case FIT_SR_DRIVE_BUSY_ERASE:
5550                 return "BUSY_ERASE";
5551         case FIT_SR_DRIVE_FW_BOOTING:
5552                 return "FW_BOOTING";
5553         default:
5554                 return "???";
5555         }
5556 }
5557
5558 const char *skd_skdev_state_to_str(enum skd_drvr_state state)
5559 {
5560         switch (state) {
5561         case SKD_DRVR_STATE_LOAD:
5562                 return "LOAD";
5563         case SKD_DRVR_STATE_IDLE:
5564                 return "IDLE";
5565         case SKD_DRVR_STATE_BUSY:
5566                 return "BUSY";
5567         case SKD_DRVR_STATE_STARTING:
5568                 return "STARTING";
5569         case SKD_DRVR_STATE_ONLINE:
5570                 return "ONLINE";
5571         case SKD_DRVR_STATE_PAUSING:
5572                 return "PAUSING";
5573         case SKD_DRVR_STATE_PAUSED:
5574                 return "PAUSED";
5575         case SKD_DRVR_STATE_DRAINING_TIMEOUT:
5576                 return "DRAINING_TIMEOUT";
5577         case SKD_DRVR_STATE_RESTARTING:
5578                 return "RESTARTING";
5579         case SKD_DRVR_STATE_RESUMING:
5580                 return "RESUMING";
5581         case SKD_DRVR_STATE_STOPPING:
5582                 return "STOPPING";
5583         case SKD_DRVR_STATE_SYNCING:
5584                 return "SYNCING";
5585         case SKD_DRVR_STATE_FAULT:
5586                 return "FAULT";
5587         case SKD_DRVR_STATE_DISAPPEARED:
5588                 return "DISAPPEARED";
5589         case SKD_DRVR_STATE_BUSY_ERASE:
5590                 return "BUSY_ERASE";
5591         case SKD_DRVR_STATE_BUSY_SANITIZE:
5592                 return "BUSY_SANITIZE";
5593         case SKD_DRVR_STATE_BUSY_IMMINENT:
5594                 return "BUSY_IMMINENT";
5595         case SKD_DRVR_STATE_WAIT_BOOT:
5596                 return "WAIT_BOOT";
5597
5598         default:
5599                 return "???";
5600         }
5601 }
5602
5603 const char *skd_skmsg_state_to_str(enum skd_fit_msg_state state)
5604 {
5605         switch (state) {
5606         case SKD_MSG_STATE_IDLE:
5607                 return "IDLE";
5608         case SKD_MSG_STATE_BUSY:
5609                 return "BUSY";
5610         default:
5611                 return "???";
5612         }
5613 }
5614
5615 const char *skd_skreq_state_to_str(enum skd_req_state state)
5616 {
5617         switch (state) {
5618         case SKD_REQ_STATE_IDLE:
5619                 return "IDLE";
5620         case SKD_REQ_STATE_SETUP:
5621                 return "SETUP";
5622         case SKD_REQ_STATE_BUSY:
5623                 return "BUSY";
5624         case SKD_REQ_STATE_COMPLETED:
5625                 return "COMPLETED";
5626         case SKD_REQ_STATE_TIMEOUT:
5627                 return "TIMEOUT";
5628         case SKD_REQ_STATE_ABORTED:
5629                 return "ABORTED";
5630         default:
5631                 return "???";
5632         }
5633 }
5634
5635 static void skd_log_skdev(struct skd_device *skdev, const char *event)
5636 {
5637         DPRINTK(skdev, "(%s) skdev=%p event='%s'\n", skdev->name, skdev, event);
5638         DPRINTK(skdev, "  drive_state=%s(%d) driver_state=%s(%d)\n",
5639                 skd_drive_state_to_str(skdev->drive_state), skdev->drive_state,
5640                 skd_skdev_state_to_str(skdev->state), skdev->state);
5641         DPRINTK(skdev, "  busy=%d limit=%d dev=%d lowat=%d\n",
5642                 skdev->in_flight, skdev->cur_max_queue_depth,
5643                 skdev->dev_max_queue_depth, skdev->queue_low_water_mark);
5644         DPRINTK(skdev, "  timestamp=0x%x cycle=%d cycle_ix=%d\n",
5645                 skdev->timeout_stamp, skdev->skcomp_cycle, skdev->skcomp_ix);
5646 }
5647
5648 static void skd_log_skmsg(struct skd_device *skdev,
5649                           struct skd_fitmsg_context *skmsg, const char *event)
5650 {
5651         DPRINTK(skdev, "(%s) skmsg=%p event='%s'\n", skdev->name, skmsg, event);
5652         DPRINTK(skdev, "  state=%s(%d) id=0x%04x length=%d\n",
5653                 skd_skmsg_state_to_str(skmsg->state), skmsg->state,
5654                 skmsg->id, skmsg->length);
5655 }
5656
5657 static void skd_log_skreq(struct skd_device *skdev,
5658                           struct skd_request_context *skreq, const char *event)
5659 {
5660         DPRINTK(skdev, "(%s) skreq=%p event='%s'\n", skdev->name, skreq, event);
5661         DPRINTK(skdev, "  state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
5662                 skd_skreq_state_to_str(skreq->state), skreq->state,
5663                 skreq->id, skreq->fitmsg_id);
5664         DPRINTK(skdev, "  timo=0x%x sg_dir=%d n_sg=%d\n",
5665                 skreq->timeout_stamp, skreq->sg_data_dir, skreq->n_sg);
5666
5667         if (!skd_bio) {
5668                 if (skreq->req != NULL) {
5669                         struct request *req = skreq->req;
5670                         u32 lba = (u32)blk_rq_pos(req);
5671                         u32 count = blk_rq_sectors(req);
5672
5673                         DPRINTK(skdev,
5674                                 "  req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5675                                 req, lba, lba, count, count,
5676                                 (int)rq_data_dir(req));
5677                 } else
5678                         DPRINTK(skdev, "  req=NULL\n");
5679         } else {
5680                 if (skreq->bio != NULL) {
5681                         struct bio *bio = skreq->bio;
5682                         u32 lba = (u32)bio->bi_sector;
5683                         u32 count = bio_sectors(bio);
5684
5685                         DPRINTK(skdev,
5686                                 "  bio=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n",
5687                                 bio, lba, lba, count, count,
5688                                 (int)bio_data_dir(bio));
5689                 } else
5690                         DPRINTK(skdev, "  req=NULL\n");
5691         }
5692 }
5693
5694 /*
5695  *****************************************************************************
5696  * MODULE GLUE
5697  *****************************************************************************
5698  */
5699
5700 static int __init skd_init(void)
5701 {
5702         int rc = 0;
5703
5704         pr_info(PFX " v%s-b%s loaded\n", DRV_VERSION, DRV_BUILD_ID);
5705
5706         switch (skd_isr_type) {
5707         case SKD_IRQ_LEGACY:
5708         case SKD_IRQ_MSI:
5709         case SKD_IRQ_MSIX:
5710                 break;
5711         default:
5712                 pr_info("skd_isr_type %d invalid, re-set to %d\n",
5713                        skd_isr_type, SKD_IRQ_DEFAULT);
5714                 skd_isr_type = SKD_IRQ_DEFAULT;
5715         }
5716
5717         skd_flush_slab = kmem_cache_create(SKD_FLUSH_JOB,
5718                                                 sizeof(struct skd_flush_cmd),
5719                                                 0, 0, NULL);
5720
5721         if (!skd_flush_slab) {
5722                 pr_err("failed to allocated flush slab.\n");
5723                 return -ENOMEM;
5724         }
5725
5726         if (skd_max_queue_depth < 1
5727             || skd_max_queue_depth > SKD_MAX_QUEUE_DEPTH) {
5728                 pr_info(
5729                        "skd_max_queue_depth %d invalid, re-set to %d\n",
5730                        skd_max_queue_depth, SKD_MAX_QUEUE_DEPTH_DEFAULT);
5731                 skd_max_queue_depth = SKD_MAX_QUEUE_DEPTH_DEFAULT;
5732         }
5733
5734         if (skd_max_req_per_msg < 1 || skd_max_req_per_msg > 14) {
5735                 pr_info(
5736                        "skd_max_req_per_msg %d invalid, re-set to %d\n",
5737                        skd_max_req_per_msg, SKD_MAX_REQ_PER_MSG_DEFAULT);
5738                 skd_max_req_per_msg = SKD_MAX_REQ_PER_MSG_DEFAULT;
5739         }
5740
5741         if (skd_sgs_per_request < 1 || skd_sgs_per_request > 4096) {
5742                 pr_info(
5743                        "skd_sg_per_request %d invalid, re-set to %d\n",
5744                        skd_sgs_per_request, SKD_N_SG_PER_REQ_DEFAULT);
5745                 skd_sgs_per_request = SKD_N_SG_PER_REQ_DEFAULT;
5746         }
5747
5748         if (skd_dbg_level < 0 || skd_dbg_level > 2) {
5749                 pr_info("skd_dbg_level %d invalid, re-set to %d\n",
5750                        skd_dbg_level, 0);
5751                 skd_dbg_level = 0;
5752         }
5753
5754         if (skd_isr_comp_limit < 0) {
5755                 pr_info("skd_isr_comp_limit %d invalid, set to %d\n",
5756                        skd_isr_comp_limit, 0);
5757                 skd_isr_comp_limit = 0;
5758         }
5759
5760         if (skd_max_pass_thru < 1 || skd_max_pass_thru > 50) {
5761                 pr_info("skd_max_pass_thru %d invalid, re-set to %d\n",
5762                        skd_max_pass_thru, SKD_N_SPECIAL_CONTEXT);
5763                 skd_max_pass_thru = SKD_N_SPECIAL_CONTEXT;
5764         }
5765
5766         /* Obtain major device number. */
5767         rc = register_blkdev(0, DRV_NAME);
5768         if (rc < 0)
5769                 return rc;
5770
5771         skd_major = rc;
5772
5773         return pci_register_driver(&skd_driver);
5774
5775 }
5776
5777 static void __exit skd_exit(void)
5778 {
5779         pr_info(PFX " v%s-b%s unloading\n", DRV_VERSION, DRV_BUILD_ID);
5780
5781         unregister_blkdev(skd_major, DRV_NAME);
5782         pci_unregister_driver(&skd_driver);
5783
5784         kmem_cache_destroy(skd_flush_slab);
5785 }
5786
5787 static int
5788 skd_flush_cmd_enqueue(struct skd_device *skdev, void *cmd)
5789 {
5790         struct skd_flush_cmd *item;
5791
5792         item = kmem_cache_zalloc(skd_flush_slab, GFP_ATOMIC);
5793         if (!item) {
5794                 pr_err("skd_flush_cmd_enqueue: Failed to allocated item.\n");
5795                 return -ENOMEM;
5796         }
5797
5798         item->cmd = cmd;
5799         list_add_tail(&item->flist, &skdev->flush_list);
5800         return 0;
5801 }
5802
5803 static void *
5804 skd_flush_cmd_dequeue(struct skd_device *skdev)
5805 {
5806         void *cmd;
5807         struct skd_flush_cmd *item;
5808
5809         item = list_entry(skdev->flush_list.next, struct skd_flush_cmd, flist);
5810         list_del_init(&item->flist);
5811         cmd = item->cmd;
5812         kmem_cache_free(skd_flush_slab, item);
5813         return cmd;
5814 }
5815
5816 module_init(skd_init);
5817 module_exit(skd_exit);