drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2012 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45 #include "be_cmds.h"
  46
  47 static unsigned int be_iopoll_budget = 10;
  48 static unsigned int be_max_phys_size = 64;
  49 static unsigned int enable_msix = 1;
  50
  51 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  53 MODULE_VERSION(BUILD_STR);
  54 MODULE_AUTHOR("Emulex Corporation");
  55 MODULE_LICENSE("GPL");
  56 module_param(be_iopoll_budget, int, 0);
  57 module_param(enable_msix, int, 0);
  58 module_param(be_max_phys_size, uint, S_IRUGO);
  59 MODULE_PARM_DESC(be_max_phys_size,
  60                 "Maximum Size (In Kilobytes) of physically contiguous "
  61                 "memory that can be allocated. Range is 16 - 128");
  62
  63 #define beiscsi_disp_param(_name)\
  64 ssize_t \
  65 beiscsi_##_name##_disp(struct device *dev,\
  66                         struct device_attribute *attrib, char *buf)     \
  67 {       \
  68         struct Scsi_Host *shost = class_to_shost(dev);\
  69         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  70         uint32_t param_val = 0; \
  71         param_val = phba->attr_##_name;\
  72         return snprintf(buf, PAGE_SIZE, "%d\n",\
  73                         phba->attr_##_name);\
  74 }
  75
  76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  77 int \
  78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  79 {\
  80         if (val >= _minval && val <= _maxval) {\
  81                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  82                             "BA_%d : beiscsi_"#_name" updated "\
  83                             "from 0x%x ==> 0x%x\n",\
  84                             phba->attr_##_name, val); \
  85                 phba->attr_##_name = val;\
  86                 return 0;\
  87         } \
  88         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  89                     "BA_%d beiscsi_"#_name" attribute "\
  90                     "cannot be updated to 0x%x, "\
  91                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  92                 return -EINVAL;\
  93 }
  94
  95 #define beiscsi_store_param(_name)  \
  96 ssize_t \
  97 beiscsi_##_name##_store(struct device *dev,\
  98                          struct device_attribute *attr, const char *buf,\
  99                          size_t count) \
 100 { \
 101         struct Scsi_Host  *shost = class_to_shost(dev);\
 102         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 103         uint32_t param_val = 0;\
 104         if (!isdigit(buf[0]))\
 105                 return -EINVAL;\
 106         if (sscanf(buf, "%i", &param_val) != 1)\
 107                 return -EINVAL;\
 108         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 109                 return strlen(buf);\
 110         else \
 111                 return -EINVAL;\
 112 }
 113
 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 115 int \
 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 117 { \
 118         if (val >= _minval && val <= _maxval) {\
 119                 phba->attr_##_name = val;\
 120                 return 0;\
 121         } \
 122         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 123                     "BA_%d beiscsi_"#_name" attribute " \
 124                     "cannot be updated to 0x%x, "\
 125                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 126         phba->attr_##_name = _defval;\
 127         return -EINVAL;\
 128 }
 129
 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 131 static uint beiscsi_##_name = _defval;\
 132 module_param(beiscsi_##_name, uint, S_IRUGO);\
 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 134 beiscsi_disp_param(_name)\
 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 136 beiscsi_store_param(_name)\
 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 139               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 140
 141 /*
 142  * When new log level added update the
 143  * the MAX allowed value for log_enable
 144  */
 145 BEISCSI_RW_ATTR(log_enable, 0x00,
 146                 0xFF, 0x00, "Enable logging Bit Mask\n"
 147                 "\t\t\t\tInitialization Events  : 0x01\n"
 148                 "\t\t\t\tMailbox Events         : 0x02\n"
 149                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 150                 "\t\t\t\tError Handling         : 0x08\n"
 151                 "\t\t\t\tIO Path Events         : 0x10\n"
 152                 "\t\t\t\tConfiguration Path     : 0x20\n");
 153
 154 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 155 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 156 struct device_attribute *beiscsi_attrs[] = {
 157         &dev_attr_beiscsi_log_enable,
 158         &dev_attr_beiscsi_drvr_ver,
 159         &dev_attr_beiscsi_adapter_family,
 160         NULL,
 161 };
 162
 163 static char const *cqe_desc[] = {
 164         "RESERVED_DESC",
 165         "SOL_CMD_COMPLETE",
 166         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 167         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 168         "CXN_KILLED_BURST_LEN_MISMATCH",
 169         "CXN_KILLED_AHS_RCVD",
 170         "CXN_KILLED_HDR_DIGEST_ERR",
 171         "CXN_KILLED_UNKNOWN_HDR",
 172         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 173         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 174         "CXN_KILLED_RST_RCVD",
 175         "CXN_KILLED_TIMED_OUT",
 176         "CXN_KILLED_RST_SENT",
 177         "CXN_KILLED_FIN_RCVD",
 178         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 179         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 180         "CXN_KILLED_OVER_RUN_RESIDUAL",
 181         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 182         "CMD_KILLED_INVALID_STATSN_RCVD",
 183         "CMD_KILLED_INVALID_R2T_RCVD",
 184         "CMD_CXN_KILLED_LUN_INVALID",
 185         "CMD_CXN_KILLED_ICD_INVALID",
 186         "CMD_CXN_KILLED_ITT_INVALID",
 187         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 188         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 189         "CXN_INVALIDATE_NOTIFY",
 190         "CXN_INVALIDATE_INDEX_NOTIFY",
 191         "CMD_INVALIDATED_NOTIFY",
 192         "UNSOL_HDR_NOTIFY",
 193         "UNSOL_DATA_NOTIFY",
 194         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 195         "DRIVERMSG_NOTIFY",
 196         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 197         "SOL_CMD_KILLED_DIF_ERR",
 198         "CXN_KILLED_SYN_RCVD",
 199         "CXN_KILLED_IMM_DATA_RCVD"
 200 };
 201
 202 static int beiscsi_slave_configure(struct scsi_device *sdev)
 203 {
 204         blk_queue_max_segment_size(sdev->request_queue, 65536);
 205         return 0;
 206 }
 207
 208 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 209 {
 210         struct iscsi_cls_session *cls_session;
 211         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 212         struct beiscsi_io_task *aborted_io_task;
 213         struct iscsi_conn *conn;
 214         struct beiscsi_conn *beiscsi_conn;
 215         struct beiscsi_hba *phba;
 216         struct iscsi_session *session;
 217         struct invalidate_command_table *inv_tbl;
 218         struct be_dma_mem nonemb_cmd;
 219         unsigned int cid, tag, num_invalidate;
 220
 221         cls_session = starget_to_session(scsi_target(sc->device));
 222         session = cls_session->dd_data;
 223
 224         spin_lock_bh(&session->lock);
 225         if (!aborted_task || !aborted_task->sc) {
 226                 /* we raced */
 227                 spin_unlock_bh(&session->lock);
 228                 return SUCCESS;
 229         }
 230
 231         aborted_io_task = aborted_task->dd_data;
 232         if (!aborted_io_task->scsi_cmnd) {
 233                 /* raced or invalid command */
 234                 spin_unlock_bh(&session->lock);
 235                 return SUCCESS;
 236         }
 237         spin_unlock_bh(&session->lock);
 238         conn = aborted_task->conn;
 239         beiscsi_conn = conn->dd_data;
 240         phba = beiscsi_conn->phba;
 241
 242         /* invalidate iocb */
 243         cid = beiscsi_conn->beiscsi_conn_cid;
 244         inv_tbl = phba->inv_tbl;
 245         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 246         inv_tbl->cid = cid;
 247         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 248         num_invalidate = 1;
 249         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 250                                 sizeof(struct invalidate_commands_params_in),
 251                                 &nonemb_cmd.dma);
 252         if (nonemb_cmd.va == NULL) {
 253                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 254                             "BM_%d : Failed to allocate memory for"
 255                             "mgmt_invalidate_icds\n");
 256                 return FAILED;
 257         }
 258         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 259
 260         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 261                                    cid, &nonemb_cmd);
 262         if (!tag) {
 263                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 264                             "BM_%d : mgmt_invalidate_icds could not be"
 265                             "submitted\n");
 266                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 267                                     nonemb_cmd.va, nonemb_cmd.dma);
 268
 269                 return FAILED;
 270         }
 271
 272         beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
 273         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 274                             nonemb_cmd.va, nonemb_cmd.dma);
 275         return iscsi_eh_abort(sc);
 276 }
 277
 278 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 279 {
 280         struct iscsi_task *abrt_task;
 281         struct beiscsi_io_task *abrt_io_task;
 282         struct iscsi_conn *conn;
 283         struct beiscsi_conn *beiscsi_conn;
 284         struct beiscsi_hba *phba;
 285         struct iscsi_session *session;
 286         struct iscsi_cls_session *cls_session;
 287         struct invalidate_command_table *inv_tbl;
 288         struct be_dma_mem nonemb_cmd;
 289         unsigned int cid, tag, i, num_invalidate;
 290
 291         /* invalidate iocbs */
 292         cls_session = starget_to_session(scsi_target(sc->device));
 293         session = cls_session->dd_data;
 294         spin_lock_bh(&session->lock);
 295         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 296                 spin_unlock_bh(&session->lock);
 297                 return FAILED;
 298         }
 299         conn = session->leadconn;
 300         beiscsi_conn = conn->dd_data;
 301         phba = beiscsi_conn->phba;
 302         cid = beiscsi_conn->beiscsi_conn_cid;
 303         inv_tbl = phba->inv_tbl;
 304         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 305         num_invalidate = 0;
 306         for (i = 0; i < conn->session->cmds_max; i++) {
 307                 abrt_task = conn->session->cmds[i];
 308                 abrt_io_task = abrt_task->dd_data;
 309                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 310                         continue;
 311
 312                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 313                         continue;
 314
 315                 inv_tbl->cid = cid;
 316                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 317                 num_invalidate++;
 318                 inv_tbl++;
 319         }
 320         spin_unlock_bh(&session->lock);
 321         inv_tbl = phba->inv_tbl;
 322
 323         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 324                                 sizeof(struct invalidate_commands_params_in),
 325                                 &nonemb_cmd.dma);
 326         if (nonemb_cmd.va == NULL) {
 327                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 328                             "BM_%d : Failed to allocate memory for"
 329                             "mgmt_invalidate_icds\n");
 330                 return FAILED;
 331         }
 332         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 333         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 334         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 335                                    cid, &nonemb_cmd);
 336         if (!tag) {
 337                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 338                             "BM_%d : mgmt_invalidate_icds could not be"
 339                             " submitted\n");
 340                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 341                                     nonemb_cmd.va, nonemb_cmd.dma);
 342                 return FAILED;
 343         }
 344
 345         beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
 346         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 347                             nonemb_cmd.va, nonemb_cmd.dma);
 348         return iscsi_eh_device_reset(sc);
 349 }
 350
 351 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 352 {
 353         struct beiscsi_hba *phba = data;
 354         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 355         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 356         char *str = buf;
 357         int rc;
 358
 359         switch (type) {
 360         case ISCSI_BOOT_TGT_NAME:
 361                 rc = sprintf(buf, "%.*s\n",
 362                             (int)strlen(boot_sess->target_name),
 363                             (char *)&boot_sess->target_name);
 364                 break;
 365         case ISCSI_BOOT_TGT_IP_ADDR:
 366                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 367                         rc = sprintf(buf, "%pI4\n",
 368                                 (char *)&boot_conn->dest_ipaddr.addr);
 369                 else
 370                         rc = sprintf(str, "%pI6\n",
 371                                 (char *)&boot_conn->dest_ipaddr.addr);
 372                 break;
 373         case ISCSI_BOOT_TGT_PORT:
 374                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 375                 break;
 376
 377         case ISCSI_BOOT_TGT_CHAP_NAME:
 378                 rc = sprintf(str,  "%.*s\n",
 379                              boot_conn->negotiated_login_options.auth_data.chap.
 380                              target_chap_name_length,
 381                              (char *)&boot_conn->negotiated_login_options.
 382                              auth_data.chap.target_chap_name);
 383                 break;
 384         case ISCSI_BOOT_TGT_CHAP_SECRET:
 385                 rc = sprintf(str,  "%.*s\n",
 386                              boot_conn->negotiated_login_options.auth_data.chap.
 387                              target_secret_length,
 388                              (char *)&boot_conn->negotiated_login_options.
 389                              auth_data.chap.target_secret);
 390                 break;
 391         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 392                 rc = sprintf(str,  "%.*s\n",
 393                              boot_conn->negotiated_login_options.auth_data.chap.
 394                              intr_chap_name_length,
 395                              (char *)&boot_conn->negotiated_login_options.
 396                              auth_data.chap.intr_chap_name);
 397                 break;
 398         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 399                 rc = sprintf(str,  "%.*s\n",
 400                              boot_conn->negotiated_login_options.auth_data.chap.
 401                              intr_secret_length,
 402                              (char *)&boot_conn->negotiated_login_options.
 403                              auth_data.chap.intr_secret);
 404                 break;
 405         case ISCSI_BOOT_TGT_FLAGS:
 406                 rc = sprintf(str, "2\n");
 407                 break;
 408         case ISCSI_BOOT_TGT_NIC_ASSOC:
 409                 rc = sprintf(str, "0\n");
 410                 break;
 411         default:
 412                 rc = -ENOSYS;
 413                 break;
 414         }
 415         return rc;
 416 }
 417
 418 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 419 {
 420         struct beiscsi_hba *phba = data;
 421         char *str = buf;
 422         int rc;
 423
 424         switch (type) {
 425         case ISCSI_BOOT_INI_INITIATOR_NAME:
 426                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 427                 break;
 428         default:
 429                 rc = -ENOSYS;
 430                 break;
 431         }
 432         return rc;
 433 }
 434
 435 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 436 {
 437         struct beiscsi_hba *phba = data;
 438         char *str = buf;
 439         int rc;
 440
 441         switch (type) {
 442         case ISCSI_BOOT_ETH_FLAGS:
 443                 rc = sprintf(str, "2\n");
 444                 break;
 445         case ISCSI_BOOT_ETH_INDEX:
 446                 rc = sprintf(str, "0\n");
 447                 break;
 448         case ISCSI_BOOT_ETH_MAC:
 449                 rc  = beiscsi_get_macaddr(str, phba);
 450                 break;
 451         default:
 452                 rc = -ENOSYS;
 453                 break;
 454         }
 455         return rc;
 456 }
 457
 458
 459 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 460 {
 461         umode_t rc;
 462
 463         switch (type) {
 464         case ISCSI_BOOT_TGT_NAME:
 465         case ISCSI_BOOT_TGT_IP_ADDR:
 466         case ISCSI_BOOT_TGT_PORT:
 467         case ISCSI_BOOT_TGT_CHAP_NAME:
 468         case ISCSI_BOOT_TGT_CHAP_SECRET:
 469         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 470         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 471         case ISCSI_BOOT_TGT_NIC_ASSOC:
 472         case ISCSI_BOOT_TGT_FLAGS:
 473                 rc = S_IRUGO;
 474                 break;
 475         default:
 476                 rc = 0;
 477                 break;
 478         }
 479         return rc;
 480 }
 481
 482 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 483 {
 484         umode_t rc;
 485
 486         switch (type) {
 487         case ISCSI_BOOT_INI_INITIATOR_NAME:
 488                 rc = S_IRUGO;
 489                 break;
 490         default:
 491                 rc = 0;
 492                 break;
 493         }
 494         return rc;
 495 }
 496
 497
 498 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 499 {
 500         umode_t rc;
 501
 502         switch (type) {
 503         case ISCSI_BOOT_ETH_FLAGS:
 504         case ISCSI_BOOT_ETH_MAC:
 505         case ISCSI_BOOT_ETH_INDEX:
 506                 rc = S_IRUGO;
 507                 break;
 508         default:
 509                 rc = 0;
 510                 break;
 511         }
 512         return rc;
 513 }
 514
 515 /*------------------- PCI Driver operations and data ----------------- */
 516 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 517         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 518         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 519         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 520         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 521         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 522         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 523         { 0 }
 524 };
 525 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 526
 527
 528 static struct scsi_host_template beiscsi_sht = {
 529         .module = THIS_MODULE,
 530         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 531         .proc_name = DRV_NAME,
 532         .queuecommand = iscsi_queuecommand,
 533         .change_queue_depth = iscsi_change_queue_depth,
 534         .slave_configure = beiscsi_slave_configure,
 535         .target_alloc = iscsi_target_alloc,
 536         .eh_abort_handler = beiscsi_eh_abort,
 537         .eh_device_reset_handler = beiscsi_eh_device_reset,
 538         .eh_target_reset_handler = iscsi_eh_session_reset,
 539         .shost_attrs = beiscsi_attrs,
 540         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 541         .can_queue = BE2_IO_DEPTH,
 542         .this_id = -1,
 543         .max_sectors = BEISCSI_MAX_SECTORS,
 544         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 545         .use_clustering = ENABLE_CLUSTERING,
 546         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 547
 548 };
 549
 550 static struct scsi_transport_template *beiscsi_scsi_transport;
 551
 552 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 553 {
 554         struct beiscsi_hba *phba;
 555         struct Scsi_Host *shost;
 556
 557         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 558         if (!shost) {
 559                 dev_err(&pcidev->dev,
 560                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 561                 return NULL;
 562         }
 563         shost->dma_boundary = pcidev->dma_mask;
 564         shost->max_id = BE2_MAX_SESSIONS;
 565         shost->max_channel = 0;
 566         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 567         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 568         shost->transportt = beiscsi_scsi_transport;
 569         phba = iscsi_host_priv(shost);
 570         memset(phba, 0, sizeof(*phba));
 571         phba->shost = shost;
 572         phba->pcidev = pci_dev_get(pcidev);
 573         pci_set_drvdata(pcidev, phba);
 574         phba->interface_handle = 0xFFFFFFFF;
 575
 576         if (iscsi_host_add(shost, &phba->pcidev->dev))
 577                 goto free_devices;
 578
 579         return phba;
 580
 581 free_devices:
 582         pci_dev_put(phba->pcidev);
 583         iscsi_host_free(phba->shost);
 584         return NULL;
 585 }
 586
 587 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 588 {
 589         if (phba->csr_va) {
 590                 iounmap(phba->csr_va);
 591                 phba->csr_va = NULL;
 592         }
 593         if (phba->db_va) {
 594                 iounmap(phba->db_va);
 595                 phba->db_va = NULL;
 596         }
 597         if (phba->pci_va) {
 598                 iounmap(phba->pci_va);
 599                 phba->pci_va = NULL;
 600         }
 601 }
 602
 603 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 604                                 struct pci_dev *pcidev)
 605 {
 606         u8 __iomem *addr;
 607         int pcicfg_reg;
 608
 609         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 610                                pci_resource_len(pcidev, 2));
 611         if (addr == NULL)
 612                 return -ENOMEM;
 613         phba->ctrl.csr = addr;
 614         phba->csr_va = addr;
 615         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 616
 617         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 618         if (addr == NULL)
 619                 goto pci_map_err;
 620         phba->ctrl.db = addr;
 621         phba->db_va = addr;
 622         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 623
 624         if (phba->generation == BE_GEN2)
 625                 pcicfg_reg = 1;
 626         else
 627                 pcicfg_reg = 0;
 628
 629         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 630                                pci_resource_len(pcidev, pcicfg_reg));
 631
 632         if (addr == NULL)
 633                 goto pci_map_err;
 634         phba->ctrl.pcicfg = addr;
 635         phba->pci_va = addr;
 636         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 637         return 0;
 638
 639 pci_map_err:
 640         beiscsi_unmap_pci_function(phba);
 641         return -ENOMEM;
 642 }
 643
 644 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 645 {
 646         int ret;
 647
 648         ret = pci_enable_device(pcidev);
 649         if (ret) {
 650                 dev_err(&pcidev->dev,
 651                         "beiscsi_enable_pci - enable device failed\n");
 652                 return ret;
 653         }
 654
 655         pci_set_master(pcidev);
 656         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 657                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 658                 if (ret) {
 659                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 660                         pci_disable_device(pcidev);
 661                         return ret;
 662                 }
 663         }
 664         return 0;
 665 }
 666
 667 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 668 {
 669         struct be_ctrl_info *ctrl = &phba->ctrl;
 670         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 671         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 672         int status = 0;
 673
 674         ctrl->pdev = pdev;
 675         status = beiscsi_map_pci_bars(phba, pdev);
 676         if (status)
 677                 return status;
 678         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 679         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 680                                                   mbox_mem_alloc->size,
 681                                                   &mbox_mem_alloc->dma);
 682         if (!mbox_mem_alloc->va) {
 683                 beiscsi_unmap_pci_function(phba);
 684                 return -ENOMEM;
 685         }
 686
 687         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 688         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 689         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 690         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 691         spin_lock_init(&ctrl->mbox_lock);
 692         spin_lock_init(&phba->ctrl.mcc_lock);
 693         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 694
 695         return status;
 696 }
 697
 698 static void beiscsi_get_params(struct beiscsi_hba *phba)
 699 {
 700         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 701                                     - (phba->fw_config.iscsi_cid_count
 702                                     + BE2_TMFS
 703                                     + BE2_NOPOUT_REQ));
 704         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 705         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 706         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 707         phba->params.num_sge_per_io = BE2_SGE;
 708         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 709         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 710         phba->params.eq_timer = 64;
 711         phba->params.num_eq_entries =
 712             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 713                                     + BE2_TMFS) / 512) + 1) * 512;
 714         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 715                                 ? 1024 : phba->params.num_eq_entries;
 716         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 717                     "BM_%d : phba->params.num_eq_entries=%d\n",
 718                     phba->params.num_eq_entries);
 719         phba->params.num_cq_entries =
 720             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 721                                     + BE2_TMFS) / 512) + 1) * 512;
 722         phba->params.wrbs_per_cxn = 256;
 723 }
 724
 725 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 726                            unsigned int id, unsigned int clr_interrupt,
 727                            unsigned int num_processed,
 728                            unsigned char rearm, unsigned char event)
 729 {
 730         u32 val = 0;
 731         val |= id & DB_EQ_RING_ID_MASK;
 732         if (rearm)
 733                 val |= 1 << DB_EQ_REARM_SHIFT;
 734         if (clr_interrupt)
 735                 val |= 1 << DB_EQ_CLR_SHIFT;
 736         if (event)
 737                 val |= 1 << DB_EQ_EVNT_SHIFT;
 738         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 739         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 740 }
 741
 742 /**
 743  * be_isr_mcc - The isr routine of the driver.
 744  * @irq: Not used
 745  * @dev_id: Pointer to host adapter structure
 746  */
 747 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 748 {
 749         struct beiscsi_hba *phba;
 750         struct be_eq_entry *eqe = NULL;
 751         struct be_queue_info *eq;
 752         struct be_queue_info *mcc;
 753         unsigned int num_eq_processed;
 754         struct be_eq_obj *pbe_eq;
 755         unsigned long flags;
 756
 757         pbe_eq = dev_id;
 758         eq = &pbe_eq->q;
 759         phba =  pbe_eq->phba;
 760         mcc = &phba->ctrl.mcc_obj.cq;
 761         eqe = queue_tail_node(eq);
 762
 763         num_eq_processed = 0;
 764
 765         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 766                                 & EQE_VALID_MASK) {
 767                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 768                      resource_id) / 32] &
 769                      EQE_RESID_MASK) >> 16) == mcc->id) {
 770                         spin_lock_irqsave(&phba->isr_lock, flags);
 771                         pbe_eq->todo_mcc_cq = true;
 772                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 773                 }
 774                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 775                 queue_tail_inc(eq);
 776                 eqe = queue_tail_node(eq);
 777                 num_eq_processed++;
 778         }
 779         if (pbe_eq->todo_mcc_cq)
 780                 queue_work(phba->wq, &pbe_eq->work_cqs);
 781         if (num_eq_processed)
 782                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 783
 784         return IRQ_HANDLED;
 785 }
 786
 787 /**
 788  * be_isr_msix - The isr routine of the driver.
 789  * @irq: Not used
 790  * @dev_id: Pointer to host adapter structure
 791  */
 792 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 793 {
 794         struct beiscsi_hba *phba;
 795         struct be_eq_entry *eqe = NULL;
 796         struct be_queue_info *eq;
 797         struct be_queue_info *cq;
 798         unsigned int num_eq_processed;
 799         struct be_eq_obj *pbe_eq;
 800         unsigned long flags;
 801
 802         pbe_eq = dev_id;
 803         eq = &pbe_eq->q;
 804         cq = pbe_eq->cq;
 805         eqe = queue_tail_node(eq);
 806
 807         phba = pbe_eq->phba;
 808         num_eq_processed = 0;
 809         if (blk_iopoll_enabled) {
 810                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 811                                         & EQE_VALID_MASK) {
 812                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 813                                 blk_iopoll_sched(&pbe_eq->iopoll);
 814
 815                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 816                         queue_tail_inc(eq);
 817                         eqe = queue_tail_node(eq);
 818                         num_eq_processed++;
 819                 }
 820         } else {
 821                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 822                                                 & EQE_VALID_MASK) {
 823                         spin_lock_irqsave(&phba->isr_lock, flags);
 824                         pbe_eq->todo_cq = true;
 825                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 826                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 827                         queue_tail_inc(eq);
 828                         eqe = queue_tail_node(eq);
 829                         num_eq_processed++;
 830                 }
 831
 832                 if (pbe_eq->todo_cq)
 833                         queue_work(phba->wq, &pbe_eq->work_cqs);
 834         }
 835
 836         if (num_eq_processed)
 837                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 838
 839         return IRQ_HANDLED;
 840 }
 841
 842 /**
 843  * be_isr - The isr routine of the driver.
 844  * @irq: Not used
 845  * @dev_id: Pointer to host adapter structure
 846  */
 847 static irqreturn_t be_isr(int irq, void *dev_id)
 848 {
 849         struct beiscsi_hba *phba;
 850         struct hwi_controller *phwi_ctrlr;
 851         struct hwi_context_memory *phwi_context;
 852         struct be_eq_entry *eqe = NULL;
 853         struct be_queue_info *eq;
 854         struct be_queue_info *cq;
 855         struct be_queue_info *mcc;
 856         unsigned long flags, index;
 857         unsigned int num_mcceq_processed, num_ioeq_processed;
 858         struct be_ctrl_info *ctrl;
 859         struct be_eq_obj *pbe_eq;
 860         int isr;
 861
 862         phba = dev_id;
 863         ctrl = &phba->ctrl;
 864         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 865                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 866         if (!isr)
 867                 return IRQ_NONE;
 868
 869         phwi_ctrlr = phba->phwi_ctrlr;
 870         phwi_context = phwi_ctrlr->phwi_ctxt;
 871         pbe_eq = &phwi_context->be_eq[0];
 872
 873         eq = &phwi_context->be_eq[0].q;
 874         mcc = &phba->ctrl.mcc_obj.cq;
 875         index = 0;
 876         eqe = queue_tail_node(eq);
 877
 878         num_ioeq_processed = 0;
 879         num_mcceq_processed = 0;
 880         if (blk_iopoll_enabled) {
 881                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 882                                         & EQE_VALID_MASK) {
 883                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 884                              resource_id) / 32] &
 885                              EQE_RESID_MASK) >> 16) == mcc->id) {
 886                                 spin_lock_irqsave(&phba->isr_lock, flags);
 887                                 pbe_eq->todo_mcc_cq = true;
 888                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 889                                 num_mcceq_processed++;
 890                         } else {
 891                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 892                                         blk_iopoll_sched(&pbe_eq->iopoll);
 893                                 num_ioeq_processed++;
 894                         }
 895                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 896                         queue_tail_inc(eq);
 897                         eqe = queue_tail_node(eq);
 898                 }
 899                 if (num_ioeq_processed || num_mcceq_processed) {
 900                         if (pbe_eq->todo_mcc_cq)
 901                                 queue_work(phba->wq, &pbe_eq->work_cqs);
 902
 903                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 904                                 hwi_ring_eq_db(phba, eq->id, 0,
 905                                               (num_ioeq_processed +
 906                                                num_mcceq_processed) , 1, 1);
 907                         else
 908                                 hwi_ring_eq_db(phba, eq->id, 0,
 909                                                (num_ioeq_processed +
 910                                                 num_mcceq_processed), 0, 1);
 911
 912                         return IRQ_HANDLED;
 913                 } else
 914                         return IRQ_NONE;
 915         } else {
 916                 cq = &phwi_context->be_cq[0];
 917                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 918                                                 & EQE_VALID_MASK) {
 919
 920                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 921                              resource_id) / 32] &
 922                              EQE_RESID_MASK) >> 16) != cq->id) {
 923                                 spin_lock_irqsave(&phba->isr_lock, flags);
 924                                 pbe_eq->todo_mcc_cq = true;
 925                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 926                         } else {
 927                                 spin_lock_irqsave(&phba->isr_lock, flags);
 928                                 pbe_eq->todo_cq = true;
 929                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 930                         }
 931                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 932                         queue_tail_inc(eq);
 933                         eqe = queue_tail_node(eq);
 934                         num_ioeq_processed++;
 935                 }
 936                 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
 937                         queue_work(phba->wq, &pbe_eq->work_cqs);
 938
 939                 if (num_ioeq_processed) {
 940                         hwi_ring_eq_db(phba, eq->id, 0,
 941                                        num_ioeq_processed, 1, 1);
 942                         return IRQ_HANDLED;
 943                 } else
 944                         return IRQ_NONE;
 945         }
 946 }
 947
 948 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 949 {
 950         struct pci_dev *pcidev = phba->pcidev;
 951         struct hwi_controller *phwi_ctrlr;
 952         struct hwi_context_memory *phwi_context;
 953         int ret, msix_vec, i, j;
 954
 955         phwi_ctrlr = phba->phwi_ctrlr;
 956         phwi_context = phwi_ctrlr->phwi_ctxt;
 957
 958         if (phba->msix_enabled) {
 959                 for (i = 0; i < phba->num_cpus; i++) {
 960                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 961                                                     GFP_KERNEL);
 962                         if (!phba->msi_name[i]) {
 963                                 ret = -ENOMEM;
 964                                 goto free_msix_irqs;
 965                         }
 966
 967                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 968                                 phba->shost->host_no, i);
 969                         msix_vec = phba->msix_entries[i].vector;
 970                         ret = request_irq(msix_vec, be_isr_msix, 0,
 971                                           phba->msi_name[i],
 972                                           &phwi_context->be_eq[i]);
 973                         if (ret) {
 974                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 975                                             "BM_%d : beiscsi_init_irqs-Failed to"
 976                                             "register msix for i = %d\n",
 977                                             i);
 978                                 kfree(phba->msi_name[i]);
 979                                 goto free_msix_irqs;
 980                         }
 981                 }
 982                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 983                 if (!phba->msi_name[i]) {
 984                         ret = -ENOMEM;
 985                         goto free_msix_irqs;
 986                 }
 987                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 988                         phba->shost->host_no);
 989                 msix_vec = phba->msix_entries[i].vector;
 990                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 991                                   &phwi_context->be_eq[i]);
 992                 if (ret) {
 993                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 994                                     "BM_%d : beiscsi_init_irqs-"
 995                                     "Failed to register beiscsi_msix_mcc\n");
 996                         kfree(phba->msi_name[i]);
 997                         goto free_msix_irqs;
 998                 }
 999
1000         } else {
1001                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1002                                   "beiscsi", phba);
1003                 if (ret) {
1004                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1005                                     "BM_%d : beiscsi_init_irqs-"
1006                                     "Failed to register irq\\n");
1007                         return ret;
1008                 }
1009         }
1010         return 0;
1011 free_msix_irqs:
1012         for (j = i - 1; j >= 0; j--) {
1013                 kfree(phba->msi_name[j]);
1014                 msix_vec = phba->msix_entries[j].vector;
1015                 free_irq(msix_vec, &phwi_context->be_eq[j]);
1016         }
1017         return ret;
1018 }
1019
1020 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1021                            unsigned int id, unsigned int num_processed,
1022                            unsigned char rearm, unsigned char event)
1023 {
1024         u32 val = 0;
1025         val |= id & DB_CQ_RING_ID_MASK;
1026         if (rearm)
1027                 val |= 1 << DB_CQ_REARM_SHIFT;
1028         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1029         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1030 }
1031
1032 static unsigned int
1033 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1034                           struct beiscsi_hba *phba,
1035                           unsigned short cid,
1036                           struct pdu_base *ppdu,
1037                           unsigned long pdu_len,
1038                           void *pbuffer, unsigned long buf_len)
1039 {
1040         struct iscsi_conn *conn = beiscsi_conn->conn;
1041         struct iscsi_session *session = conn->session;
1042         struct iscsi_task *task;
1043         struct beiscsi_io_task *io_task;
1044         struct iscsi_hdr *login_hdr;
1045
1046         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1047                                                 PDUBASE_OPCODE_MASK) {
1048         case ISCSI_OP_NOOP_IN:
1049                 pbuffer = NULL;
1050                 buf_len = 0;
1051                 break;
1052         case ISCSI_OP_ASYNC_EVENT:
1053                 break;
1054         case ISCSI_OP_REJECT:
1055                 WARN_ON(!pbuffer);
1056                 WARN_ON(!(buf_len == 48));
1057                 beiscsi_log(phba, KERN_ERR,
1058                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1059                             "BM_%d : In ISCSI_OP_REJECT\n");
1060                 break;
1061         case ISCSI_OP_LOGIN_RSP:
1062         case ISCSI_OP_TEXT_RSP:
1063                 task = conn->login_task;
1064                 io_task = task->dd_data;
1065                 login_hdr = (struct iscsi_hdr *)ppdu;
1066                 login_hdr->itt = io_task->libiscsi_itt;
1067                 break;
1068         default:
1069                 beiscsi_log(phba, KERN_WARNING,
1070                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1071                             "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1072                             (ppdu->
1073                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
1074                              & PDUBASE_OPCODE_MASK));
1075                 return 1;
1076         }
1077
1078         spin_lock_bh(&session->lock);
1079         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1080         spin_unlock_bh(&session->lock);
1081         return 0;
1082 }
1083
1084 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1085 {
1086         struct sgl_handle *psgl_handle;
1087
1088         if (phba->io_sgl_hndl_avbl) {
1089                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1090                             "BM_%d : In alloc_io_sgl_handle,"
1091                             " io_sgl_alloc_index=%d\n",
1092                             phba->io_sgl_alloc_index);
1093
1094                 psgl_handle = phba->io_sgl_hndl_base[phba->
1095                                                 io_sgl_alloc_index];
1096                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1097                 phba->io_sgl_hndl_avbl--;
1098                 if (phba->io_sgl_alloc_index == (phba->params.
1099                                                  ios_per_ctrl - 1))
1100                         phba->io_sgl_alloc_index = 0;
1101                 else
1102                         phba->io_sgl_alloc_index++;
1103         } else
1104                 psgl_handle = NULL;
1105         return psgl_handle;
1106 }
1107
1108 static void
1109 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1110 {
1111         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1112                     "BM_%d : In free_,io_sgl_free_index=%d\n",
1113                     phba->io_sgl_free_index);
1114
1115         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1116                 /*
1117                  * this can happen if clean_task is called on a task that
1118                  * failed in xmit_task or alloc_pdu.
1119                  */
1120                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1121                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1122                              "value there=%p\n", phba->io_sgl_free_index,
1123                              phba->io_sgl_hndl_base
1124                              [phba->io_sgl_free_index]);
1125                 return;
1126         }
1127         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1128         phba->io_sgl_hndl_avbl++;
1129         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1130                 phba->io_sgl_free_index = 0;
1131         else
1132                 phba->io_sgl_free_index++;
1133 }
1134
1135 /**
1136  * alloc_wrb_handle - To allocate a wrb handle
1137  * @phba: The hba pointer
1138  * @cid: The cid to use for allocation
1139  *
1140  * This happens under session_lock until submission to chip
1141  */
1142 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1143 {
1144         struct hwi_wrb_context *pwrb_context;
1145         struct hwi_controller *phwi_ctrlr;
1146         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1147
1148         phwi_ctrlr = phba->phwi_ctrlr;
1149         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1150         if (pwrb_context->wrb_handles_available >= 2) {
1151                 pwrb_handle = pwrb_context->pwrb_handle_base[
1152                                             pwrb_context->alloc_index];
1153                 pwrb_context->wrb_handles_available--;
1154                 if (pwrb_context->alloc_index ==
1155                                                 (phba->params.wrbs_per_cxn - 1))
1156                         pwrb_context->alloc_index = 0;
1157                 else
1158                         pwrb_context->alloc_index++;
1159                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1160                                                 pwrb_context->alloc_index];
1161                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1162         } else
1163                 pwrb_handle = NULL;
1164         return pwrb_handle;
1165 }
1166
1167 /**
1168  * free_wrb_handle - To free the wrb handle back to pool
1169  * @phba: The hba pointer
1170  * @pwrb_context: The context to free from
1171  * @pwrb_handle: The wrb_handle to free
1172  *
1173  * This happens under session_lock until submission to chip
1174  */
1175 static void
1176 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1177                 struct wrb_handle *pwrb_handle)
1178 {
1179         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1180         pwrb_context->wrb_handles_available++;
1181         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1182                 pwrb_context->free_index = 0;
1183         else
1184                 pwrb_context->free_index++;
1185
1186         beiscsi_log(phba, KERN_INFO,
1187                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1188                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1189                     "wrb_handles_available=%d\n",
1190                     pwrb_handle, pwrb_context->free_index,
1191                     pwrb_context->wrb_handles_available);
1192 }
1193
1194 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1195 {
1196         struct sgl_handle *psgl_handle;
1197
1198         if (phba->eh_sgl_hndl_avbl) {
1199                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1200                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1201                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1202                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1203                             phba->eh_sgl_alloc_index,
1204                             phba->eh_sgl_alloc_index);
1205
1206                 phba->eh_sgl_hndl_avbl--;
1207                 if (phba->eh_sgl_alloc_index ==
1208                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1209                      1))
1210                         phba->eh_sgl_alloc_index = 0;
1211                 else
1212                         phba->eh_sgl_alloc_index++;
1213         } else
1214                 psgl_handle = NULL;
1215         return psgl_handle;
1216 }
1217
1218 void
1219 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1220 {
1221
1222         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1223                     "BM_%d : In  free_mgmt_sgl_handle,"
1224                     "eh_sgl_free_index=%d\n",
1225                     phba->eh_sgl_free_index);
1226
1227         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1228                 /*
1229                  * this can happen if clean_task is called on a task that
1230                  * failed in xmit_task or alloc_pdu.
1231                  */
1232                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1233                             "BM_%d : Double Free in eh SGL ,"
1234                             "eh_sgl_free_index=%d\n",
1235                             phba->eh_sgl_free_index);
1236                 return;
1237         }
1238         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1239         phba->eh_sgl_hndl_avbl++;
1240         if (phba->eh_sgl_free_index ==
1241             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1242                 phba->eh_sgl_free_index = 0;
1243         else
1244                 phba->eh_sgl_free_index++;
1245 }
1246
1247 static void
1248 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1249                 struct iscsi_task *task,
1250                 struct common_sol_cqe *csol_cqe)
1251 {
1252         struct beiscsi_io_task *io_task = task->dd_data;
1253         struct be_status_bhs *sts_bhs =
1254                                 (struct be_status_bhs *)io_task->cmd_bhs;
1255         struct iscsi_conn *conn = beiscsi_conn->conn;
1256         unsigned char *sense;
1257         u32 resid = 0, exp_cmdsn, max_cmdsn;
1258         u8 rsp, status, flags;
1259
1260         exp_cmdsn = csol_cqe->exp_cmdsn;
1261         max_cmdsn = (csol_cqe->exp_cmdsn +
1262                      csol_cqe->cmd_wnd - 1);
1263         rsp = csol_cqe->i_resp;
1264         status = csol_cqe->i_sts;
1265         flags = csol_cqe->i_flags;
1266         resid = csol_cqe->res_cnt;
1267
1268         if (!task->sc) {
1269                 if (io_task->scsi_cmnd)
1270                         scsi_dma_unmap(io_task->scsi_cmnd);
1271
1272                 return;
1273         }
1274         task->sc->result = (DID_OK << 16) | status;
1275         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1276                 task->sc->result = DID_ERROR << 16;
1277                 goto unmap;
1278         }
1279
1280         /* bidi not initially supported */
1281         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1282                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1283                         task->sc->result = DID_ERROR << 16;
1284
1285                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1286                         scsi_set_resid(task->sc, resid);
1287                         if (!status && (scsi_bufflen(task->sc) - resid <
1288                             task->sc->underflow))
1289                                 task->sc->result = DID_ERROR << 16;
1290                 }
1291         }
1292
1293         if (status == SAM_STAT_CHECK_CONDITION) {
1294                 u16 sense_len;
1295                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1296
1297                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1298                 sense_len = be16_to_cpu(*slen);
1299                 memcpy(task->sc->sense_buffer, sense,
1300                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1301         }
1302
1303         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1304                 conn->rxdata_octets += resid;
1305 unmap:
1306         scsi_dma_unmap(io_task->scsi_cmnd);
1307         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1308 }
1309
1310 static void
1311 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1312                     struct iscsi_task *task,
1313                     struct common_sol_cqe *csol_cqe)
1314 {
1315         struct iscsi_logout_rsp *hdr;
1316         struct beiscsi_io_task *io_task = task->dd_data;
1317         struct iscsi_conn *conn = beiscsi_conn->conn;
1318
1319         hdr = (struct iscsi_logout_rsp *)task->hdr;
1320         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1321         hdr->t2wait = 5;
1322         hdr->t2retain = 0;
1323         hdr->flags = csol_cqe->i_flags;
1324         hdr->response = csol_cqe->i_resp;
1325         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1326         hdr->max_cmdsn = (csol_cqe->exp_cmdsn + csol_cqe->cmd_wnd - 1);
1327
1328         hdr->dlength[0] = 0;
1329         hdr->dlength[1] = 0;
1330         hdr->dlength[2] = 0;
1331         hdr->hlength = 0;
1332         hdr->itt = io_task->libiscsi_itt;
1333         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1334 }
1335
1336 static void
1337 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1338                  struct iscsi_task *task,
1339                  struct common_sol_cqe *csol_cqe)
1340 {
1341         struct iscsi_tm_rsp *hdr;
1342         struct iscsi_conn *conn = beiscsi_conn->conn;
1343         struct beiscsi_io_task *io_task = task->dd_data;
1344
1345         hdr = (struct iscsi_tm_rsp *)task->hdr;
1346         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1347         hdr->flags = csol_cqe->i_flags;
1348         hdr->response = csol_cqe->i_resp;
1349         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1350         hdr->max_cmdsn = (csol_cqe->exp_cmdsn +
1351                           csol_cqe->cmd_wnd - 1);
1352
1353         hdr->itt = io_task->libiscsi_itt;
1354         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1355 }
1356
1357 static void
1358 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1359                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1360 {
1361         struct hwi_wrb_context *pwrb_context;
1362         struct wrb_handle *pwrb_handle = NULL;
1363         struct hwi_controller *phwi_ctrlr;
1364         struct iscsi_task *task;
1365         struct beiscsi_io_task *io_task;
1366         struct iscsi_conn *conn = beiscsi_conn->conn;
1367         struct iscsi_session *session = conn->session;
1368         uint16_t wrb_index, cid;
1369
1370         phwi_ctrlr = phba->phwi_ctrlr;
1371         if (chip_skh_r(phba->pcidev)) {
1372                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1373                                           wrb_idx, psol);
1374                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1375                                     cid, psol);
1376         } else {
1377                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1378                                           wrb_idx, psol);
1379                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1380                                     cid, psol);
1381         }
1382
1383         pwrb_context = &phwi_ctrlr->wrb_context[
1384                         cid - phba->fw_config.iscsi_cid_start];
1385         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1386         task = pwrb_handle->pio_handle;
1387
1388         io_task = task->dd_data;
1389         spin_lock_bh(&phba->mgmt_sgl_lock);
1390         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1391         spin_unlock_bh(&phba->mgmt_sgl_lock);
1392         spin_lock_bh(&session->lock);
1393         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1394         spin_unlock_bh(&session->lock);
1395 }
1396
1397 static void
1398 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1399                         struct iscsi_task *task,
1400                         struct common_sol_cqe *csol_cqe)
1401 {
1402         struct iscsi_nopin *hdr;
1403         struct iscsi_conn *conn = beiscsi_conn->conn;
1404         struct beiscsi_io_task *io_task = task->dd_data;
1405
1406         hdr = (struct iscsi_nopin *)task->hdr;
1407         hdr->flags = csol_cqe->i_flags;
1408         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1409         hdr->max_cmdsn = be32_to_cpu(hdr->exp_cmdsn +
1410                          csol_cqe->cmd_wnd - 1);
1411
1412         hdr->opcode = ISCSI_OP_NOOP_IN;
1413         hdr->itt = io_task->libiscsi_itt;
1414         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1415 }
1416
1417 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1418                 struct sol_cqe *psol,
1419                 struct common_sol_cqe *csol_cqe)
1420 {
1421         if (chip_skh_r(phba->pcidev)) {
1422                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1423                                                     i_exp_cmd_sn, psol);
1424                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1425                                                   i_res_cnt, psol);
1426                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1427                                                     wrb_index, psol);
1428                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1429                                               cid, psol);
1430                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1431                                                  hw_sts, psol);
1432                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1433                                                   i_cmd_wnd, psol);
1434                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1435                                   cmd_cmpl, psol))
1436                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1437                                                         i_sts, psol);
1438                 else
1439                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1440                                                          i_sts, psol);
1441                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1442                                   u, psol))
1443                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1444
1445                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1446                                   o, psol))
1447                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1448         } else {
1449                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1450                                                     i_exp_cmd_sn, psol);
1451                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1452                                                   i_res_cnt, psol);
1453                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1454                                                   i_cmd_wnd, psol);
1455                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1456                                                     wrb_index, psol);
1457                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1458                                               cid, psol);
1459                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1460                                                  hw_sts, psol);
1461                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1462                                                  i_resp, psol);
1463                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1464                                                 i_sts, psol);
1465                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1466                                                   i_flags, psol);
1467         }
1468 }
1469
1470
1471 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1472                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1473 {
1474         struct hwi_wrb_context *pwrb_context;
1475         struct wrb_handle *pwrb_handle;
1476         struct iscsi_wrb *pwrb = NULL;
1477         struct hwi_controller *phwi_ctrlr;
1478         struct iscsi_task *task;
1479         unsigned int type;
1480         struct iscsi_conn *conn = beiscsi_conn->conn;
1481         struct iscsi_session *session = conn->session;
1482         struct common_sol_cqe csol_cqe = {0};
1483
1484         phwi_ctrlr = phba->phwi_ctrlr;
1485
1486         /* Copy the elements to a common structure */
1487         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1488
1489         pwrb_context = &phwi_ctrlr->wrb_context[
1490                         csol_cqe.cid - phba->fw_config.iscsi_cid_start];
1491
1492         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1493                       csol_cqe.wrb_index];
1494
1495         task = pwrb_handle->pio_handle;
1496         pwrb = pwrb_handle->pwrb;
1497         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1498
1499         spin_lock_bh(&session->lock);
1500         switch (type) {
1501         case HWH_TYPE_IO:
1502         case HWH_TYPE_IO_RD:
1503                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1504                      ISCSI_OP_NOOP_OUT)
1505                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1506                 else
1507                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1508                 break;
1509
1510         case HWH_TYPE_LOGOUT:
1511                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1512                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1513                 else
1514                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1515                 break;
1516
1517         case HWH_TYPE_LOGIN:
1518                 beiscsi_log(phba, KERN_ERR,
1519                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1520                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1521                             " hwi_complete_cmd- Solicited path\n");
1522                 break;
1523
1524         case HWH_TYPE_NOP:
1525                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1526                 break;
1527
1528         default:
1529                 beiscsi_log(phba, KERN_WARNING,
1530                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1531                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1532                             "wrb_index 0x%x CID 0x%x\n", type,
1533                             csol_cqe.wrb_index,
1534                             csol_cqe.cid);
1535                 break;
1536         }
1537
1538         spin_unlock_bh(&session->lock);
1539 }
1540
1541 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1542                                           *pasync_ctx, unsigned int is_header,
1543                                           unsigned int host_write_ptr)
1544 {
1545         if (is_header)
1546                 return &pasync_ctx->async_entry[host_write_ptr].
1547                     header_busy_list;
1548         else
1549                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1550 }
1551
1552 static struct async_pdu_handle *
1553 hwi_get_async_handle(struct beiscsi_hba *phba,
1554                      struct beiscsi_conn *beiscsi_conn,
1555                      struct hwi_async_pdu_context *pasync_ctx,
1556                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1557 {
1558         struct be_bus_address phys_addr;
1559         struct list_head *pbusy_list;
1560         struct async_pdu_handle *pasync_handle = NULL;
1561         unsigned char is_header = 0;
1562         unsigned int index, dpl;
1563
1564         if (chip_skh_r(phba->pcidev)) {
1565                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1566                                     dpl, pdpdu_cqe);
1567                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1568                                       index, pdpdu_cqe);
1569         } else {
1570                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1571                                     dpl, pdpdu_cqe);
1572                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1573                                       index, pdpdu_cqe);
1574         }
1575
1576         phys_addr.u.a32.address_lo =
1577                 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1578                                         db_addr_lo) / 32] - dpl);
1579         phys_addr.u.a32.address_hi =
1580                 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1581                                        db_addr_hi) / 32];
1582
1583         phys_addr.u.a64.address =
1584                         *((unsigned long long *)(&phys_addr.u.a64.address));
1585
1586         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1587                         & PDUCQE_CODE_MASK) {
1588         case UNSOL_HDR_NOTIFY:
1589                 is_header = 1;
1590
1591                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1592                                                       is_header, index);
1593                 break;
1594         case UNSOL_DATA_NOTIFY:
1595                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1596                                                       is_header, index);
1597                 break;
1598         default:
1599                 pbusy_list = NULL;
1600                 beiscsi_log(phba, KERN_WARNING,
1601                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1602                             "BM_%d : Unexpected code=%d\n",
1603                             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1604                             code) / 32] & PDUCQE_CODE_MASK);
1605                 return NULL;
1606         }
1607
1608         WARN_ON(list_empty(pbusy_list));
1609         list_for_each_entry(pasync_handle, pbusy_list, link) {
1610                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1611                         break;
1612         }
1613
1614         WARN_ON(!pasync_handle);
1615
1616         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1617                                              phba->fw_config.iscsi_cid_start;
1618         pasync_handle->is_header = is_header;
1619         pasync_handle->buffer_len = dpl;
1620         *pcq_index = index;
1621
1622         return pasync_handle;
1623 }
1624
1625 static unsigned int
1626 hwi_update_async_writables(struct beiscsi_hba *phba,
1627                             struct hwi_async_pdu_context *pasync_ctx,
1628                             unsigned int is_header, unsigned int cq_index)
1629 {
1630         struct list_head *pbusy_list;
1631         struct async_pdu_handle *pasync_handle;
1632         unsigned int num_entries, writables = 0;
1633         unsigned int *pep_read_ptr, *pwritables;
1634
1635         num_entries = pasync_ctx->num_entries;
1636         if (is_header) {
1637                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1638                 pwritables = &pasync_ctx->async_header.writables;
1639         } else {
1640                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1641                 pwritables = &pasync_ctx->async_data.writables;
1642         }
1643
1644         while ((*pep_read_ptr) != cq_index) {
1645                 (*pep_read_ptr)++;
1646                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1647
1648                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1649                                                      *pep_read_ptr);
1650                 if (writables == 0)
1651                         WARN_ON(list_empty(pbusy_list));
1652
1653                 if (!list_empty(pbusy_list)) {
1654                         pasync_handle = list_entry(pbusy_list->next,
1655                                                    struct async_pdu_handle,
1656                                                    link);
1657                         WARN_ON(!pasync_handle);
1658                         pasync_handle->consumed = 1;
1659                 }
1660
1661                 writables++;
1662         }
1663
1664         if (!writables) {
1665                 beiscsi_log(phba, KERN_ERR,
1666                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1667                             "BM_%d : Duplicate notification received - index 0x%x!!\n",
1668                             cq_index);
1669                 WARN_ON(1);
1670         }
1671
1672         *pwritables = *pwritables + writables;
1673         return 0;
1674 }
1675
1676 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1677                                        unsigned int cri)
1678 {
1679         struct hwi_controller *phwi_ctrlr;
1680         struct hwi_async_pdu_context *pasync_ctx;
1681         struct async_pdu_handle *pasync_handle, *tmp_handle;
1682         struct list_head *plist;
1683
1684         phwi_ctrlr = phba->phwi_ctrlr;
1685         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1686
1687         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1688
1689         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1690                 list_del(&pasync_handle->link);
1691
1692                 if (pasync_handle->is_header) {
1693                         list_add_tail(&pasync_handle->link,
1694                                       &pasync_ctx->async_header.free_list);
1695                         pasync_ctx->async_header.free_entries++;
1696                 } else {
1697                         list_add_tail(&pasync_handle->link,
1698                                       &pasync_ctx->async_data.free_list);
1699                         pasync_ctx->async_data.free_entries++;
1700                 }
1701         }
1702
1703         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1704         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1705         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1706 }
1707
1708 static struct phys_addr *
1709 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1710                      unsigned int is_header, unsigned int host_write_ptr)
1711 {
1712         struct phys_addr *pasync_sge = NULL;
1713
1714         if (is_header)
1715                 pasync_sge = pasync_ctx->async_header.ring_base;
1716         else
1717                 pasync_sge = pasync_ctx->async_data.ring_base;
1718
1719         return pasync_sge + host_write_ptr;
1720 }
1721
1722 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1723                                    unsigned int is_header)
1724 {
1725         struct hwi_controller *phwi_ctrlr;
1726         struct hwi_async_pdu_context *pasync_ctx;
1727         struct async_pdu_handle *pasync_handle;
1728         struct list_head *pfree_link, *pbusy_list;
1729         struct phys_addr *pasync_sge;
1730         unsigned int ring_id, num_entries;
1731         unsigned int host_write_num;
1732         unsigned int writables;
1733         unsigned int i = 0;
1734         u32 doorbell = 0;
1735
1736         phwi_ctrlr = phba->phwi_ctrlr;
1737         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1738         num_entries = pasync_ctx->num_entries;
1739
1740         if (is_header) {
1741                 writables = min(pasync_ctx->async_header.writables,
1742                                 pasync_ctx->async_header.free_entries);
1743                 pfree_link = pasync_ctx->async_header.free_list.next;
1744                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1745                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1746         } else {
1747                 writables = min(pasync_ctx->async_data.writables,
1748                                 pasync_ctx->async_data.free_entries);
1749                 pfree_link = pasync_ctx->async_data.free_list.next;
1750                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1751                 ring_id = phwi_ctrlr->default_pdu_data.id;
1752         }
1753
1754         writables = (writables / 8) * 8;
1755         if (writables) {
1756                 for (i = 0; i < writables; i++) {
1757                         pbusy_list =
1758                             hwi_get_async_busy_list(pasync_ctx, is_header,
1759                                                     host_write_num);
1760                         pasync_handle =
1761                             list_entry(pfree_link, struct async_pdu_handle,
1762                                                                 link);
1763                         WARN_ON(!pasync_handle);
1764                         pasync_handle->consumed = 0;
1765
1766                         pfree_link = pfree_link->next;
1767
1768                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1769                                                 is_header, host_write_num);
1770
1771                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1772                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1773
1774                         list_move(&pasync_handle->link, pbusy_list);
1775
1776                         host_write_num++;
1777                         host_write_num = host_write_num % num_entries;
1778                 }
1779
1780                 if (is_header) {
1781                         pasync_ctx->async_header.host_write_ptr =
1782                                                         host_write_num;
1783                         pasync_ctx->async_header.free_entries -= writables;
1784                         pasync_ctx->async_header.writables -= writables;
1785                         pasync_ctx->async_header.busy_entries += writables;
1786                 } else {
1787                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1788                         pasync_ctx->async_data.free_entries -= writables;
1789                         pasync_ctx->async_data.writables -= writables;
1790                         pasync_ctx->async_data.busy_entries += writables;
1791                 }
1792
1793                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1794                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1795                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1796                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1797                                         << DB_DEF_PDU_CQPROC_SHIFT;
1798
1799                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1800         }
1801 }
1802
1803 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1804                                          struct beiscsi_conn *beiscsi_conn,
1805                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1806 {
1807         struct hwi_controller *phwi_ctrlr;
1808         struct hwi_async_pdu_context *pasync_ctx;
1809         struct async_pdu_handle *pasync_handle = NULL;
1810         unsigned int cq_index = -1;
1811
1812         phwi_ctrlr = phba->phwi_ctrlr;
1813         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1814
1815         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1816                                              pdpdu_cqe, &cq_index);
1817         BUG_ON(pasync_handle->is_header != 0);
1818         if (pasync_handle->consumed == 0)
1819                 hwi_update_async_writables(phba, pasync_ctx,
1820                                            pasync_handle->is_header, cq_index);
1821
1822         hwi_free_async_msg(phba, pasync_handle->cri);
1823         hwi_post_async_buffers(phba, pasync_handle->is_header);
1824 }
1825
1826 static unsigned int
1827 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1828                   struct beiscsi_hba *phba,
1829                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1830 {
1831         struct list_head *plist;
1832         struct async_pdu_handle *pasync_handle;
1833         void *phdr = NULL;
1834         unsigned int hdr_len = 0, buf_len = 0;
1835         unsigned int status, index = 0, offset = 0;
1836         void *pfirst_buffer = NULL;
1837         unsigned int num_buf = 0;
1838
1839         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1840
1841         list_for_each_entry(pasync_handle, plist, link) {
1842                 if (index == 0) {
1843                         phdr = pasync_handle->pbuffer;
1844                         hdr_len = pasync_handle->buffer_len;
1845                 } else {
1846                         buf_len = pasync_handle->buffer_len;
1847                         if (!num_buf) {
1848                                 pfirst_buffer = pasync_handle->pbuffer;
1849                                 num_buf++;
1850                         }
1851                         memcpy(pfirst_buffer + offset,
1852                                pasync_handle->pbuffer, buf_len);
1853                         offset += buf_len;
1854                 }
1855                 index++;
1856         }
1857
1858         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1859                                            (beiscsi_conn->beiscsi_conn_cid -
1860                                             phba->fw_config.iscsi_cid_start),
1861                                             phdr, hdr_len, pfirst_buffer,
1862                                             offset);
1863
1864         hwi_free_async_msg(phba, cri);
1865         return 0;
1866 }
1867
1868 static unsigned int
1869 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1870                      struct beiscsi_hba *phba,
1871                      struct async_pdu_handle *pasync_handle)
1872 {
1873         struct hwi_async_pdu_context *pasync_ctx;
1874         struct hwi_controller *phwi_ctrlr;
1875         unsigned int bytes_needed = 0, status = 0;
1876         unsigned short cri = pasync_handle->cri;
1877         struct pdu_base *ppdu;
1878
1879         phwi_ctrlr = phba->phwi_ctrlr;
1880         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1881
1882         list_del(&pasync_handle->link);
1883         if (pasync_handle->is_header) {
1884                 pasync_ctx->async_header.busy_entries--;
1885                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1886                         hwi_free_async_msg(phba, cri);
1887                         BUG();
1888                 }
1889
1890                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1891                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1892                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1893                                 (unsigned short)pasync_handle->buffer_len;
1894                 list_add_tail(&pasync_handle->link,
1895                               &pasync_ctx->async_entry[cri].wait_queue.list);
1896
1897                 ppdu = pasync_handle->pbuffer;
1898                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1899                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1900                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1901                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1902                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1903
1904                 if (status == 0) {
1905                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1906                             bytes_needed;
1907
1908                         if (bytes_needed == 0)
1909                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1910                                                            pasync_ctx, cri);
1911                 }
1912         } else {
1913                 pasync_ctx->async_data.busy_entries--;
1914                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1915                         list_add_tail(&pasync_handle->link,
1916                                       &pasync_ctx->async_entry[cri].wait_queue.
1917                                       list);
1918                         pasync_ctx->async_entry[cri].wait_queue.
1919                                 bytes_received +=
1920                                 (unsigned short)pasync_handle->buffer_len;
1921
1922                         if (pasync_ctx->async_entry[cri].wait_queue.
1923                             bytes_received >=
1924                             pasync_ctx->async_entry[cri].wait_queue.
1925                             bytes_needed)
1926                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1927                                                            pasync_ctx, cri);
1928                 }
1929         }
1930         return status;
1931 }
1932
1933 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1934                                          struct beiscsi_hba *phba,
1935                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1936 {
1937         struct hwi_controller *phwi_ctrlr;
1938         struct hwi_async_pdu_context *pasync_ctx;
1939         struct async_pdu_handle *pasync_handle = NULL;
1940         unsigned int cq_index = -1;
1941
1942         phwi_ctrlr = phba->phwi_ctrlr;
1943         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1944         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1945                                              pdpdu_cqe, &cq_index);
1946
1947         if (pasync_handle->consumed == 0)
1948                 hwi_update_async_writables(phba, pasync_ctx,
1949                                            pasync_handle->is_header, cq_index);
1950
1951         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1952         hwi_post_async_buffers(phba, pasync_handle->is_header);
1953 }
1954
1955 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1956 {
1957         struct be_queue_info *mcc_cq;
1958         struct  be_mcc_compl *mcc_compl;
1959         unsigned int num_processed = 0;
1960
1961         mcc_cq = &phba->ctrl.mcc_obj.cq;
1962         mcc_compl = queue_tail_node(mcc_cq);
1963         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1964         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1965
1966                 if (num_processed >= 32) {
1967                         hwi_ring_cq_db(phba, mcc_cq->id,
1968                                         num_processed, 0, 0);
1969                         num_processed = 0;
1970                 }
1971                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1972                         /* Interpret flags as an async trailer */
1973                         if (is_link_state_evt(mcc_compl->flags))
1974                                 /* Interpret compl as a async link evt */
1975                                 beiscsi_async_link_state_process(phba,
1976                                 (struct be_async_event_link_state *) mcc_compl);
1977                         else
1978                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
1979                                             "BM_%d :  Unsupported Async Event, flags"
1980                                             " = 0x%08x\n",
1981                                             mcc_compl->flags);
1982                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1983                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1984                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1985                 }
1986
1987                 mcc_compl->flags = 0;
1988                 queue_tail_inc(mcc_cq);
1989                 mcc_compl = queue_tail_node(mcc_cq);
1990                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1991                 num_processed++;
1992         }
1993
1994         if (num_processed > 0)
1995                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1996
1997 }
1998
1999 /**
2000  * beiscsi_process_cq()- Process the Completion Queue
2001  * @pbe_eq: Event Q on which the Completion has come
2002  *
2003  * return
2004  *     Number of Completion Entries processed.
2005  **/
2006 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
2007 {
2008         struct be_queue_info *cq;
2009         struct sol_cqe *sol;
2010         struct dmsg_cqe *dmsg;
2011         unsigned int num_processed = 0;
2012         unsigned int tot_nump = 0;
2013         unsigned short code = 0, cid = 0;
2014         struct beiscsi_conn *beiscsi_conn;
2015         struct beiscsi_endpoint *beiscsi_ep;
2016         struct iscsi_endpoint *ep;
2017         struct beiscsi_hba *phba;
2018
2019         cq = pbe_eq->cq;
2020         sol = queue_tail_node(cq);
2021         phba = pbe_eq->phba;
2022
2023         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2024                CQE_VALID_MASK) {
2025                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2026
2027                  code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2028                          32] & CQE_CODE_MASK);
2029
2030                  /* Get the CID */
2031                 if (chip_skh_r(phba->pcidev)) {
2032                         if ((code == DRIVERMSG_NOTIFY) ||
2033                             (code == UNSOL_HDR_NOTIFY) ||
2034                             (code == UNSOL_DATA_NOTIFY))
2035                                 cid = AMAP_GET_BITS(
2036                                                     struct amap_i_t_dpdu_cqe_v2,
2037                                                     cid, sol);
2038                          else
2039                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2040                                                      cid, sol);
2041                    } else
2042                          cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2043
2044                 ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start];
2045                 beiscsi_ep = ep->dd_data;
2046                 beiscsi_conn = beiscsi_ep->conn;
2047
2048                 if (num_processed >= 32) {
2049                         hwi_ring_cq_db(phba, cq->id,
2050                                         num_processed, 0, 0);
2051                         tot_nump += num_processed;
2052                         num_processed = 0;
2053                 }
2054
2055                 switch (code) {
2056                 case SOL_CMD_COMPLETE:
2057                         hwi_complete_cmd(beiscsi_conn, phba, sol);
2058                         break;
2059                 case DRIVERMSG_NOTIFY:
2060                         beiscsi_log(phba, KERN_INFO,
2061                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2062                                     "BM_%d : Received %s[%d] on CID : %d\n",
2063                                     cqe_desc[code], code, cid);
2064
2065                         dmsg = (struct dmsg_cqe *)sol;
2066                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2067                         break;
2068                 case UNSOL_HDR_NOTIFY:
2069                         beiscsi_log(phba, KERN_INFO,
2070                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2071                                     "BM_%d : Received %s[%d] on CID : %d\n",
2072                                     cqe_desc[code], code, cid);
2073
2074                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2075                                              (struct i_t_dpdu_cqe *)sol);
2076                         break;
2077                 case UNSOL_DATA_NOTIFY:
2078                         beiscsi_log(phba, KERN_INFO,
2079                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2080                                     "BM_%d : Received %s[%d] on CID : %d\n",
2081                                     cqe_desc[code], code, cid);
2082
2083                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2084                                              (struct i_t_dpdu_cqe *)sol);
2085                         break;
2086                 case CXN_INVALIDATE_INDEX_NOTIFY:
2087                 case CMD_INVALIDATED_NOTIFY:
2088                 case CXN_INVALIDATE_NOTIFY:
2089                         beiscsi_log(phba, KERN_ERR,
2090                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2091                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
2092                                     cqe_desc[code], code, cid);
2093                         break;
2094                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2095                 case CMD_KILLED_INVALID_STATSN_RCVD:
2096                 case CMD_KILLED_INVALID_R2T_RCVD:
2097                 case CMD_CXN_KILLED_LUN_INVALID:
2098                 case CMD_CXN_KILLED_ICD_INVALID:
2099                 case CMD_CXN_KILLED_ITT_INVALID:
2100                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2101                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
2102                         beiscsi_log(phba, KERN_ERR,
2103                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2104                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2105                                     cqe_desc[code], code,  cid);
2106                         break;
2107                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2108                         beiscsi_log(phba, KERN_ERR,
2109                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2110                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2111                                     cqe_desc[code], code, cid);
2112                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2113                                              (struct i_t_dpdu_cqe *) sol);
2114                         break;
2115                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2116                 case CXN_KILLED_BURST_LEN_MISMATCH:
2117                 case CXN_KILLED_AHS_RCVD:
2118                 case CXN_KILLED_HDR_DIGEST_ERR:
2119                 case CXN_KILLED_UNKNOWN_HDR:
2120                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2121                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2122                 case CXN_KILLED_TIMED_OUT:
2123                 case CXN_KILLED_FIN_RCVD:
2124                 case CXN_KILLED_RST_SENT:
2125                 case CXN_KILLED_RST_RCVD:
2126                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2127                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2128                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2129                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2130                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2131                         beiscsi_log(phba, KERN_ERR,
2132                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2133                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2134                                     cqe_desc[code], code, cid);
2135                         if (beiscsi_conn)
2136                                 iscsi_conn_failure(beiscsi_conn->conn,
2137                                                    ISCSI_ERR_CONN_FAILED);
2138                         break;
2139                 default:
2140                         beiscsi_log(phba, KERN_ERR,
2141                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2142                                     "BM_%d : Invalid CQE Event Received Code : %d"
2143                                     "CID 0x%x...\n",
2144                                     code, cid);
2145                         break;
2146                 }
2147
2148                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2149                 queue_tail_inc(cq);
2150                 sol = queue_tail_node(cq);
2151                 num_processed++;
2152         }
2153
2154         if (num_processed > 0) {
2155                 tot_nump += num_processed;
2156                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
2157         }
2158         return tot_nump;
2159 }
2160
2161 void beiscsi_process_all_cqs(struct work_struct *work)
2162 {
2163         unsigned long flags;
2164         struct hwi_controller *phwi_ctrlr;
2165         struct hwi_context_memory *phwi_context;
2166         struct beiscsi_hba *phba;
2167         struct be_eq_obj *pbe_eq =
2168             container_of(work, struct be_eq_obj, work_cqs);
2169
2170         phba = pbe_eq->phba;
2171         phwi_ctrlr = phba->phwi_ctrlr;
2172         phwi_context = phwi_ctrlr->phwi_ctxt;
2173
2174         if (pbe_eq->todo_mcc_cq) {
2175                 spin_lock_irqsave(&phba->isr_lock, flags);
2176                 pbe_eq->todo_mcc_cq = false;
2177                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2178                 beiscsi_process_mcc_isr(phba);
2179         }
2180
2181         if (pbe_eq->todo_cq) {
2182                 spin_lock_irqsave(&phba->isr_lock, flags);
2183                 pbe_eq->todo_cq = false;
2184                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2185                 beiscsi_process_cq(pbe_eq);
2186         }
2187
2188         /* rearm EQ for further interrupts */
2189         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2190 }
2191
2192 static int be_iopoll(struct blk_iopoll *iop, int budget)
2193 {
2194         static unsigned int ret;
2195         struct beiscsi_hba *phba;
2196         struct be_eq_obj *pbe_eq;
2197
2198         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2199         ret = beiscsi_process_cq(pbe_eq);
2200         if (ret < budget) {
2201                 phba = pbe_eq->phba;
2202                 blk_iopoll_complete(iop);
2203                 beiscsi_log(phba, KERN_INFO,
2204                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2205                             "BM_%d : rearm pbe_eq->q.id =%d\n",
2206                             pbe_eq->q.id);
2207                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2208         }
2209         return ret;
2210 }
2211
2212 static void
2213 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2214                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2215 {
2216         struct iscsi_sge *psgl;
2217         unsigned int sg_len, index;
2218         unsigned int sge_len = 0;
2219         unsigned long long addr;
2220         struct scatterlist *l_sg;
2221         unsigned int offset;
2222
2223         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2224                       io_task->bhs_pa.u.a32.address_lo);
2225         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2226                       io_task->bhs_pa.u.a32.address_hi);
2227
2228         l_sg = sg;
2229         for (index = 0; (index < num_sg) && (index < 2); index++,
2230                         sg = sg_next(sg)) {
2231                 if (index == 0) {
2232                         sg_len = sg_dma_len(sg);
2233                         addr = (u64) sg_dma_address(sg);
2234                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2235                                       sge0_addr_lo, pwrb,
2236                                       lower_32_bits(addr));
2237                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2238                                       sge0_addr_hi, pwrb,
2239                                       upper_32_bits(addr));
2240                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2241                                       sge0_len, pwrb,
2242                                       sg_len);
2243                         sge_len = sg_len;
2244                 } else {
2245                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2246                                       pwrb, sge_len);
2247                         sg_len = sg_dma_len(sg);
2248                         addr = (u64) sg_dma_address(sg);
2249                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2250                                       sge1_addr_lo, pwrb,
2251                                       lower_32_bits(addr));
2252                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2253                                       sge1_addr_hi, pwrb,
2254                                       upper_32_bits(addr));
2255                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2256                                       sge1_len, pwrb,
2257                                       sg_len);
2258                 }
2259         }
2260         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2261         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2262
2263         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2264
2265         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2266                       io_task->bhs_pa.u.a32.address_hi);
2267         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2268                       io_task->bhs_pa.u.a32.address_lo);
2269
2270         if (num_sg == 1) {
2271                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2272                               1);
2273                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2274                               0);
2275         } else if (num_sg == 2) {
2276                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2277                               0);
2278                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2279                               1);
2280         } else {
2281                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2282                               0);
2283                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2284                               0);
2285         }
2286
2287         sg = l_sg;
2288         psgl++;
2289         psgl++;
2290         offset = 0;
2291         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2292                 sg_len = sg_dma_len(sg);
2293                 addr = (u64) sg_dma_address(sg);
2294                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2295                               lower_32_bits(addr));
2296                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2297                               upper_32_bits(addr));
2298                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2299                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2300                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2301                 offset += sg_len;
2302         }
2303         psgl--;
2304         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2305 }
2306
2307 static void
2308 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2309               unsigned int num_sg, struct beiscsi_io_task *io_task)
2310 {
2311         struct iscsi_sge *psgl;
2312         unsigned int sg_len, index;
2313         unsigned int sge_len = 0;
2314         unsigned long long addr;
2315         struct scatterlist *l_sg;
2316         unsigned int offset;
2317
2318         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2319                                       io_task->bhs_pa.u.a32.address_lo);
2320         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2321                                       io_task->bhs_pa.u.a32.address_hi);
2322
2323         l_sg = sg;
2324         for (index = 0; (index < num_sg) && (index < 2); index++,
2325                                                          sg = sg_next(sg)) {
2326                 if (index == 0) {
2327                         sg_len = sg_dma_len(sg);
2328                         addr = (u64) sg_dma_address(sg);
2329                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2330                                                 ((u32)(addr & 0xFFFFFFFF)));
2331                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2332                                                         ((u32)(addr >> 32)));
2333                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2334                                                         sg_len);
2335                         sge_len = sg_len;
2336                 } else {
2337                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2338                                                         pwrb, sge_len);
2339                         sg_len = sg_dma_len(sg);
2340                         addr = (u64) sg_dma_address(sg);
2341                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2342                                                 ((u32)(addr & 0xFFFFFFFF)));
2343                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2344                                                         ((u32)(addr >> 32)));
2345                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2346                                                         sg_len);
2347                 }
2348         }
2349         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2350         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2351
2352         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2353
2354         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2355                         io_task->bhs_pa.u.a32.address_hi);
2356         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2357                         io_task->bhs_pa.u.a32.address_lo);
2358
2359         if (num_sg == 1) {
2360                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2361                                                                 1);
2362                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2363                                                                 0);
2364         } else if (num_sg == 2) {
2365                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2366                                                                 0);
2367                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2368                                                                 1);
2369         } else {
2370                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2371                                                                 0);
2372                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2373                                                                 0);
2374         }
2375         sg = l_sg;
2376         psgl++;
2377         psgl++;
2378         offset = 0;
2379         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2380                 sg_len = sg_dma_len(sg);
2381                 addr = (u64) sg_dma_address(sg);
2382                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2383                                                 (addr & 0xFFFFFFFF));
2384                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2385                                                 (addr >> 32));
2386                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2387                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2388                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2389                 offset += sg_len;
2390         }
2391         psgl--;
2392         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2393 }
2394
2395 /**
2396  * hwi_write_buffer()- Populate the WRB with task info
2397  * @pwrb: ptr to the WRB entry
2398  * @task: iscsi task which is to be executed
2399  **/
2400 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2401 {
2402         struct iscsi_sge *psgl;
2403         struct beiscsi_io_task *io_task = task->dd_data;
2404         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2405         struct beiscsi_hba *phba = beiscsi_conn->phba;
2406         uint8_t dsp_value = 0;
2407
2408         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2409         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2410                                 io_task->bhs_pa.u.a32.address_lo);
2411         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2412                                 io_task->bhs_pa.u.a32.address_hi);
2413
2414         if (task->data) {
2415
2416                 /* Check for the data_count */
2417                 dsp_value = (task->data_count) ? 1 : 0;
2418
2419                 if (chip_skh_r(phba->pcidev))
2420                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2421                                       pwrb, dsp_value);
2422                 else
2423                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2424                                       pwrb, dsp_value);
2425
2426                 /* Map addr only if there is data_count */
2427                 if (dsp_value) {
2428                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2429                                                              task->data,
2430                                                              task->data_count,
2431                                                              PCI_DMA_TODEVICE);
2432                         io_task->mtask_data_count = task->data_count;
2433                 } else
2434                         io_task->mtask_addr = 0;
2435
2436                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2437                               lower_32_bits(io_task->mtask_addr));
2438                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2439                               upper_32_bits(io_task->mtask_addr));
2440                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2441                                                 task->data_count);
2442
2443                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2444         } else {
2445                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2446                 io_task->mtask_addr = 0;
2447         }
2448
2449         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2450
2451         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2452
2453         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2454                       io_task->bhs_pa.u.a32.address_hi);
2455         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2456                       io_task->bhs_pa.u.a32.address_lo);
2457         if (task->data) {
2458                 psgl++;
2459                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2460                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2461                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2462                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2463                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2464                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2465
2466                 psgl++;
2467                 if (task->data) {
2468                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2469                                       lower_32_bits(io_task->mtask_addr));
2470                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2471                                       upper_32_bits(io_task->mtask_addr));
2472                 }
2473                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2474         }
2475         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2476 }
2477
2478 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2479 {
2480         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2481         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2482         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2483
2484         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2485                                       sizeof(struct sol_cqe));
2486         num_async_pdu_buf_pages =
2487                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2488                                        phba->params.defpdu_hdr_sz);
2489         num_async_pdu_buf_sgl_pages =
2490                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2491                                        sizeof(struct phys_addr));
2492         num_async_pdu_data_pages =
2493                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2494                                        phba->params.defpdu_data_sz);
2495         num_async_pdu_data_sgl_pages =
2496                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2497                                        sizeof(struct phys_addr));
2498
2499         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2500
2501         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2502                                                  BE_ISCSI_PDU_HEADER_SIZE;
2503         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2504                                             sizeof(struct hwi_context_memory);
2505
2506
2507         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2508             * (phba->params.wrbs_per_cxn)
2509             * phba->params.cxns_per_ctrl;
2510         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2511                                  (phba->params.wrbs_per_cxn);
2512         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2513                                 phba->params.cxns_per_ctrl);
2514
2515         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2516                 phba->params.icds_per_ctrl;
2517         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2518                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2519
2520         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2521                 num_async_pdu_buf_pages * PAGE_SIZE;
2522         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2523                 num_async_pdu_data_pages * PAGE_SIZE;
2524         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2525                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2526         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2527                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2528         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2529                 phba->params.asyncpdus_per_ctrl *
2530                 sizeof(struct async_pdu_handle);
2531         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2532                 phba->params.asyncpdus_per_ctrl *
2533                 sizeof(struct async_pdu_handle);
2534         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2535                 sizeof(struct hwi_async_pdu_context) +
2536                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2537 }
2538
2539 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2540 {
2541         struct be_mem_descriptor *mem_descr;
2542         dma_addr_t bus_add;
2543         struct mem_array *mem_arr, *mem_arr_orig;
2544         unsigned int i, j, alloc_size, curr_alloc_size;
2545
2546         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2547         if (!phba->phwi_ctrlr)
2548                 return -ENOMEM;
2549
2550         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2551                                  GFP_KERNEL);
2552         if (!phba->init_mem) {
2553                 kfree(phba->phwi_ctrlr);
2554                 return -ENOMEM;
2555         }
2556
2557         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2558                                GFP_KERNEL);
2559         if (!mem_arr_orig) {
2560                 kfree(phba->init_mem);
2561                 kfree(phba->phwi_ctrlr);
2562                 return -ENOMEM;
2563         }
2564
2565         mem_descr = phba->init_mem;
2566         for (i = 0; i < SE_MEM_MAX; i++) {
2567                 j = 0;
2568                 mem_arr = mem_arr_orig;
2569                 alloc_size = phba->mem_req[i];
2570                 memset(mem_arr, 0, sizeof(struct mem_array) *
2571                        BEISCSI_MAX_FRAGS_INIT);
2572                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2573                 do {
2574                         mem_arr->virtual_address = pci_alloc_consistent(
2575                                                         phba->pcidev,
2576                                                         curr_alloc_size,
2577                                                         &bus_add);
2578                         if (!mem_arr->virtual_address) {
2579                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2580                                         goto free_mem;
2581                                 if (curr_alloc_size -
2582                                         rounddown_pow_of_two(curr_alloc_size))
2583                                         curr_alloc_size = rounddown_pow_of_two
2584                                                              (curr_alloc_size);
2585                                 else
2586                                         curr_alloc_size = curr_alloc_size / 2;
2587                         } else {
2588                                 mem_arr->bus_address.u.
2589                                     a64.address = (__u64) bus_add;
2590                                 mem_arr->size = curr_alloc_size;
2591                                 alloc_size -= curr_alloc_size;
2592                                 curr_alloc_size = min(be_max_phys_size *
2593                                                       1024, alloc_size);
2594                                 j++;
2595                                 mem_arr++;
2596                         }
2597                 } while (alloc_size);
2598                 mem_descr->num_elements = j;
2599                 mem_descr->size_in_bytes = phba->mem_req[i];
2600                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2601                                                GFP_KERNEL);
2602                 if (!mem_descr->mem_array)
2603                         goto free_mem;
2604
2605                 memcpy(mem_descr->mem_array, mem_arr_orig,
2606                        sizeof(struct mem_array) * j);
2607                 mem_descr++;
2608         }
2609         kfree(mem_arr_orig);
2610         return 0;
2611 free_mem:
2612         mem_descr->num_elements = j;
2613         while ((i) || (j)) {
2614                 for (j = mem_descr->num_elements; j > 0; j--) {
2615                         pci_free_consistent(phba->pcidev,
2616                                             mem_descr->mem_array[j - 1].size,
2617                                             mem_descr->mem_array[j - 1].
2618                                             virtual_address,
2619                                             (unsigned long)mem_descr->
2620                                             mem_array[j - 1].
2621                                             bus_address.u.a64.address);
2622                 }
2623                 if (i) {
2624                         i--;
2625                         kfree(mem_descr->mem_array);
2626                         mem_descr--;
2627                 }
2628         }
2629         kfree(mem_arr_orig);
2630         kfree(phba->init_mem);
2631         kfree(phba->phwi_ctrlr);
2632         return -ENOMEM;
2633 }
2634
2635 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2636 {
2637         beiscsi_find_mem_req(phba);
2638         return beiscsi_alloc_mem(phba);
2639 }
2640
2641 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2642 {
2643         struct pdu_data_out *pdata_out;
2644         struct pdu_nop_out *pnop_out;
2645         struct be_mem_descriptor *mem_descr;
2646
2647         mem_descr = phba->init_mem;
2648         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2649         pdata_out =
2650             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2651         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2652
2653         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2654                       IIOC_SCSI_DATA);
2655
2656         pnop_out =
2657             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2658                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2659
2660         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2661         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2662         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2663         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2664 }
2665
2666 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2667 {
2668         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2669         struct wrb_handle *pwrb_handle = NULL;
2670         struct hwi_controller *phwi_ctrlr;
2671         struct hwi_wrb_context *pwrb_context;
2672         struct iscsi_wrb *pwrb = NULL;
2673         unsigned int num_cxn_wrbh = 0;
2674         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2675
2676         mem_descr_wrbh = phba->init_mem;
2677         mem_descr_wrbh += HWI_MEM_WRBH;
2678
2679         mem_descr_wrb = phba->init_mem;
2680         mem_descr_wrb += HWI_MEM_WRB;
2681         phwi_ctrlr = phba->phwi_ctrlr;
2682
2683         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2684                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2685                 pwrb_context->pwrb_handle_base =
2686                                 kzalloc(sizeof(struct wrb_handle *) *
2687                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2688                 if (!pwrb_context->pwrb_handle_base) {
2689                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2690                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2691                         goto init_wrb_hndl_failed;
2692                 }
2693                 pwrb_context->pwrb_handle_basestd =
2694                                 kzalloc(sizeof(struct wrb_handle *) *
2695                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2696                 if (!pwrb_context->pwrb_handle_basestd) {
2697                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2698                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2699                         goto init_wrb_hndl_failed;
2700                 }
2701                 if (!num_cxn_wrbh) {
2702                         pwrb_handle =
2703                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2704                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2705                                         ((sizeof(struct wrb_handle)) *
2706                                          phba->params.wrbs_per_cxn));
2707                         idx++;
2708                 }
2709                 pwrb_context->alloc_index = 0;
2710                 pwrb_context->wrb_handles_available = 0;
2711                 pwrb_context->free_index = 0;
2712
2713                 if (num_cxn_wrbh) {
2714                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2715                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2716                                 pwrb_context->pwrb_handle_basestd[j] =
2717                                                                 pwrb_handle;
2718                                 pwrb_context->wrb_handles_available++;
2719                                 pwrb_handle->wrb_index = j;
2720                                 pwrb_handle++;
2721                         }
2722                         num_cxn_wrbh--;
2723                 }
2724         }
2725         idx = 0;
2726         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2727                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2728                 if (!num_cxn_wrb) {
2729                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2730                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2731                                 ((sizeof(struct iscsi_wrb) *
2732                                   phba->params.wrbs_per_cxn));
2733                         idx++;
2734                 }
2735
2736                 if (num_cxn_wrb) {
2737                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2738                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2739                                 pwrb_handle->pwrb = pwrb;
2740                                 pwrb++;
2741                         }
2742                         num_cxn_wrb--;
2743                 }
2744         }
2745         return 0;
2746 init_wrb_hndl_failed:
2747         for (j = index; j > 0; j--) {
2748                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2749                 kfree(pwrb_context->pwrb_handle_base);
2750                 kfree(pwrb_context->pwrb_handle_basestd);
2751         }
2752         return -ENOMEM;
2753 }
2754
2755 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2756 {
2757         struct hwi_controller *phwi_ctrlr;
2758         struct hba_parameters *p = &phba->params;
2759         struct hwi_async_pdu_context *pasync_ctx;
2760         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2761         unsigned int index, idx, num_per_mem, num_async_data;
2762         struct be_mem_descriptor *mem_descr;
2763
2764         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2765         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2766
2767         phwi_ctrlr = phba->phwi_ctrlr;
2768         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2769                                 mem_descr->mem_array[0].virtual_address;
2770         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2771         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2772
2773         pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2774         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2775
2776         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2777         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2778         if (mem_descr->mem_array[0].virtual_address) {
2779                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2780                             "BM_%d : hwi_init_async_pdu_ctx"
2781                             " HWI_MEM_ASYNC_HEADER_BUF va=%p\n",
2782                             mem_descr->mem_array[0].virtual_address);
2783         } else
2784                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2785                             "BM_%d : No Virtual address\n");
2786
2787         pasync_ctx->async_header.va_base =
2788                         mem_descr->mem_array[0].virtual_address;
2789
2790         pasync_ctx->async_header.pa_base.u.a64.address =
2791                         mem_descr->mem_array[0].bus_address.u.a64.address;
2792
2793         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2794         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2795         if (mem_descr->mem_array[0].virtual_address) {
2796                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2797                             "BM_%d : hwi_init_async_pdu_ctx"
2798                             " HWI_MEM_ASYNC_HEADER_RING va=%p\n",
2799                             mem_descr->mem_array[0].virtual_address);
2800         } else
2801                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2802                             "BM_%d : No Virtual address\n");
2803
2804         pasync_ctx->async_header.ring_base =
2805                         mem_descr->mem_array[0].virtual_address;
2806
2807         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2808         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2809         if (mem_descr->mem_array[0].virtual_address) {
2810                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2811                             "BM_%d : hwi_init_async_pdu_ctx"
2812                             " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n",
2813                             mem_descr->mem_array[0].virtual_address);
2814         } else
2815                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2816                             "BM_%d : No Virtual address\n");
2817
2818         pasync_ctx->async_header.handle_base =
2819                         mem_descr->mem_array[0].virtual_address;
2820         pasync_ctx->async_header.writables = 0;
2821         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2822
2823
2824         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2825         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2826         if (mem_descr->mem_array[0].virtual_address) {
2827                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2828                             "BM_%d : hwi_init_async_pdu_ctx"
2829                             " HWI_MEM_ASYNC_DATA_RING va=%p\n",
2830                             mem_descr->mem_array[0].virtual_address);
2831         } else
2832                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2833                             "BM_%d : No Virtual address\n");
2834
2835         pasync_ctx->async_data.ring_base =
2836                         mem_descr->mem_array[0].virtual_address;
2837
2838         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2839         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2840         if (!mem_descr->mem_array[0].virtual_address)
2841                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2842                             "BM_%d : No Virtual address\n");
2843
2844         pasync_ctx->async_data.handle_base =
2845                         mem_descr->mem_array[0].virtual_address;
2846         pasync_ctx->async_data.writables = 0;
2847         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2848
2849         pasync_header_h =
2850                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2851         pasync_data_h =
2852                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2853
2854         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2855         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2856         if (mem_descr->mem_array[0].virtual_address) {
2857                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2858                             "BM_%d : hwi_init_async_pdu_ctx"
2859                             " HWI_MEM_ASYNC_DATA_BUF va=%p\n",
2860                             mem_descr->mem_array[0].virtual_address);
2861         } else
2862                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2863                             "BM_%d : No Virtual address\n");
2864
2865         idx = 0;
2866         pasync_ctx->async_data.va_base =
2867                         mem_descr->mem_array[idx].virtual_address;
2868         pasync_ctx->async_data.pa_base.u.a64.address =
2869                         mem_descr->mem_array[idx].bus_address.u.a64.address;
2870
2871         num_async_data = ((mem_descr->mem_array[idx].size) /
2872                                 phba->params.defpdu_data_sz);
2873         num_per_mem = 0;
2874
2875         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2876                 pasync_header_h->cri = -1;
2877                 pasync_header_h->index = (char)index;
2878                 INIT_LIST_HEAD(&pasync_header_h->link);
2879                 pasync_header_h->pbuffer =
2880                         (void *)((unsigned long)
2881                         (pasync_ctx->async_header.va_base) +
2882                         (p->defpdu_hdr_sz * index));
2883
2884                 pasync_header_h->pa.u.a64.address =
2885                         pasync_ctx->async_header.pa_base.u.a64.address +
2886                         (p->defpdu_hdr_sz * index);
2887
2888                 list_add_tail(&pasync_header_h->link,
2889                                 &pasync_ctx->async_header.free_list);
2890                 pasync_header_h++;
2891                 pasync_ctx->async_header.free_entries++;
2892                 pasync_ctx->async_header.writables++;
2893
2894                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2895                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2896                                header_busy_list);
2897                 pasync_data_h->cri = -1;
2898                 pasync_data_h->index = (char)index;
2899                 INIT_LIST_HEAD(&pasync_data_h->link);
2900
2901                 if (!num_async_data) {
2902                         num_per_mem = 0;
2903                         idx++;
2904                         pasync_ctx->async_data.va_base =
2905                                 mem_descr->mem_array[idx].virtual_address;
2906                         pasync_ctx->async_data.pa_base.u.a64.address =
2907                                 mem_descr->mem_array[idx].
2908                                 bus_address.u.a64.address;
2909
2910                         num_async_data = ((mem_descr->mem_array[idx].size) /
2911                                         phba->params.defpdu_data_sz);
2912                 }
2913                 pasync_data_h->pbuffer =
2914                         (void *)((unsigned long)
2915                         (pasync_ctx->async_data.va_base) +
2916                         (p->defpdu_data_sz * num_per_mem));
2917
2918                 pasync_data_h->pa.u.a64.address =
2919                     pasync_ctx->async_data.pa_base.u.a64.address +
2920                     (p->defpdu_data_sz * num_per_mem);
2921                 num_per_mem++;
2922                 num_async_data--;
2923
2924                 list_add_tail(&pasync_data_h->link,
2925                               &pasync_ctx->async_data.free_list);
2926                 pasync_data_h++;
2927                 pasync_ctx->async_data.free_entries++;
2928                 pasync_ctx->async_data.writables++;
2929
2930                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2931         }
2932
2933         pasync_ctx->async_header.host_write_ptr = 0;
2934         pasync_ctx->async_header.ep_read_ptr = -1;
2935         pasync_ctx->async_data.host_write_ptr = 0;
2936         pasync_ctx->async_data.ep_read_ptr = -1;
2937 }
2938
2939 static int
2940 be_sgl_create_contiguous(void *virtual_address,
2941                          u64 physical_address, u32 length,
2942                          struct be_dma_mem *sgl)
2943 {
2944         WARN_ON(!virtual_address);
2945         WARN_ON(!physical_address);
2946         WARN_ON(!length > 0);
2947         WARN_ON(!sgl);
2948
2949         sgl->va = virtual_address;
2950         sgl->dma = (unsigned long)physical_address;
2951         sgl->size = length;
2952
2953         return 0;
2954 }
2955
2956 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2957 {
2958         memset(sgl, 0, sizeof(*sgl));
2959 }
2960
2961 static void
2962 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2963                      struct mem_array *pmem, struct be_dma_mem *sgl)
2964 {
2965         if (sgl->va)
2966                 be_sgl_destroy_contiguous(sgl);
2967
2968         be_sgl_create_contiguous(pmem->virtual_address,
2969                                  pmem->bus_address.u.a64.address,
2970                                  pmem->size, sgl);
2971 }
2972
2973 static void
2974 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2975                            struct mem_array *pmem, struct be_dma_mem *sgl)
2976 {
2977         if (sgl->va)
2978                 be_sgl_destroy_contiguous(sgl);
2979
2980         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2981                                  pmem->bus_address.u.a64.address,
2982                                  pmem->size, sgl);
2983 }
2984
2985 static int be_fill_queue(struct be_queue_info *q,
2986                 u16 len, u16 entry_size, void *vaddress)
2987 {
2988         struct be_dma_mem *mem = &q->dma_mem;
2989
2990         memset(q, 0, sizeof(*q));
2991         q->len = len;
2992         q->entry_size = entry_size;
2993         mem->size = len * entry_size;
2994         mem->va = vaddress;
2995         if (!mem->va)
2996                 return -ENOMEM;
2997         memset(mem->va, 0, mem->size);
2998         return 0;
2999 }
3000
3001 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3002                              struct hwi_context_memory *phwi_context)
3003 {
3004         unsigned int i, num_eq_pages;
3005         int ret = 0, eq_for_mcc;
3006         struct be_queue_info *eq;
3007         struct be_dma_mem *mem;
3008         void *eq_vaddress;
3009         dma_addr_t paddr;
3010
3011         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3012                                       sizeof(struct be_eq_entry));
3013
3014         if (phba->msix_enabled)
3015                 eq_for_mcc = 1;
3016         else
3017                 eq_for_mcc = 0;
3018         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3019                 eq = &phwi_context->be_eq[i].q;
3020                 mem = &eq->dma_mem;
3021                 phwi_context->be_eq[i].phba = phba;
3022                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3023                                                      num_eq_pages * PAGE_SIZE,
3024                                                      &paddr);
3025                 if (!eq_vaddress)
3026                         goto create_eq_error;
3027
3028                 mem->va = eq_vaddress;
3029                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3030                                     sizeof(struct be_eq_entry), eq_vaddress);
3031                 if (ret) {
3032                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3033                                     "BM_%d : be_fill_queue Failed for EQ\n");
3034                         goto create_eq_error;
3035                 }
3036
3037                 mem->dma = paddr;
3038                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3039                                             phwi_context->cur_eqd);
3040                 if (ret) {
3041                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3042                                     "BM_%d : beiscsi_cmd_eq_create"
3043                                     "Failed for EQ\n");
3044                         goto create_eq_error;
3045                 }
3046
3047                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3048                             "BM_%d : eqid = %d\n",
3049                             phwi_context->be_eq[i].q.id);
3050         }
3051         return 0;
3052 create_eq_error:
3053         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3054                 eq = &phwi_context->be_eq[i].q;
3055                 mem = &eq->dma_mem;
3056                 if (mem->va)
3057                         pci_free_consistent(phba->pcidev, num_eq_pages
3058                                             * PAGE_SIZE,
3059                                             mem->va, mem->dma);
3060         }
3061         return ret;
3062 }
3063
3064 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3065                              struct hwi_context_memory *phwi_context)
3066 {
3067         unsigned int i, num_cq_pages;
3068         int ret = 0;
3069         struct be_queue_info *cq, *eq;
3070         struct be_dma_mem *mem;
3071         struct be_eq_obj *pbe_eq;
3072         void *cq_vaddress;
3073         dma_addr_t paddr;
3074
3075         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3076                                       sizeof(struct sol_cqe));
3077
3078         for (i = 0; i < phba->num_cpus; i++) {
3079                 cq = &phwi_context->be_cq[i];
3080                 eq = &phwi_context->be_eq[i].q;
3081                 pbe_eq = &phwi_context->be_eq[i];
3082                 pbe_eq->cq = cq;
3083                 pbe_eq->phba = phba;
3084                 mem = &cq->dma_mem;
3085                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3086                                                      num_cq_pages * PAGE_SIZE,
3087                                                      &paddr);
3088                 if (!cq_vaddress)
3089                         goto create_cq_error;
3090                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3091                                     sizeof(struct sol_cqe), cq_vaddress);
3092                 if (ret) {
3093                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3094                                     "BM_%d : be_fill_queue Failed "
3095                                     "for ISCSI CQ\n");
3096                         goto create_cq_error;
3097                 }
3098
3099                 mem->dma = paddr;
3100                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3101                                             false, 0);
3102                 if (ret) {
3103                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3104                                     "BM_%d : beiscsi_cmd_eq_create"
3105                                     "Failed for ISCSI CQ\n");
3106                         goto create_cq_error;
3107                 }
3108                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3109                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3110                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3111         }
3112         return 0;
3113
3114 create_cq_error:
3115         for (i = 0; i < phba->num_cpus; i++) {
3116                 cq = &phwi_context->be_cq[i];
3117                 mem = &cq->dma_mem;
3118                 if (mem->va)
3119                         pci_free_consistent(phba->pcidev, num_cq_pages
3120                                             * PAGE_SIZE,
3121                                             mem->va, mem->dma);
3122         }
3123         return ret;
3124
3125 }
3126
3127 static int
3128 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3129                        struct hwi_context_memory *phwi_context,
3130                        struct hwi_controller *phwi_ctrlr,
3131                        unsigned int def_pdu_ring_sz)
3132 {
3133         unsigned int idx;
3134         int ret;
3135         struct be_queue_info *dq, *cq;
3136         struct be_dma_mem *mem;
3137         struct be_mem_descriptor *mem_descr;
3138         void *dq_vaddress;
3139
3140         idx = 0;
3141         dq = &phwi_context->be_def_hdrq;
3142         cq = &phwi_context->be_cq[0];
3143         mem = &dq->dma_mem;
3144         mem_descr = phba->init_mem;
3145         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
3146         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3147         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3148                             sizeof(struct phys_addr),
3149                             sizeof(struct phys_addr), dq_vaddress);
3150         if (ret) {
3151                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3152                             "BM_%d : be_fill_queue Failed for DEF PDU HDR\n");
3153                 return ret;
3154         }
3155         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3156                                   bus_address.u.a64.address;
3157         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3158                                               def_pdu_ring_sz,
3159                                               phba->params.defpdu_hdr_sz);
3160         if (ret) {
3161                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3162                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n");
3163                 return ret;
3164         }
3165         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
3166         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3167                     "BM_%d : iscsi def pdu id is %d\n",
3168                     phwi_context->be_def_hdrq.id);
3169
3170         hwi_post_async_buffers(phba, 1);
3171         return 0;
3172 }
3173
3174 static int
3175 beiscsi_create_def_data(struct beiscsi_hba *phba,
3176                         struct hwi_context_memory *phwi_context,
3177                         struct hwi_controller *phwi_ctrlr,
3178                         unsigned int def_pdu_ring_sz)
3179 {
3180         unsigned int idx;
3181         int ret;
3182         struct be_queue_info *dataq, *cq;
3183         struct be_dma_mem *mem;
3184         struct be_mem_descriptor *mem_descr;
3185         void *dq_vaddress;
3186
3187         idx = 0;
3188         dataq = &phwi_context->be_def_dataq;
3189         cq = &phwi_context->be_cq[0];
3190         mem = &dataq->dma_mem;
3191         mem_descr = phba->init_mem;
3192         mem_descr += HWI_MEM_ASYNC_DATA_RING;
3193         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3194         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3195                             sizeof(struct phys_addr),
3196                             sizeof(struct phys_addr), dq_vaddress);
3197         if (ret) {
3198                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3199                             "BM_%d : be_fill_queue Failed for DEF PDU DATA\n");
3200                 return ret;
3201         }
3202         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3203                                   bus_address.u.a64.address;
3204         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3205                                               def_pdu_ring_sz,
3206                                               phba->params.defpdu_data_sz);
3207         if (ret) {
3208                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3209                             "BM_%d be_cmd_create_default_pdu_queue"
3210                             " Failed for DEF PDU DATA\n");
3211                 return ret;
3212         }
3213         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
3214         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3215                     "BM_%d : iscsi def data id is %d\n",
3216                     phwi_context->be_def_dataq.id);
3217
3218         hwi_post_async_buffers(phba, 0);
3219         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3220                     "BM_%d : DEFAULT PDU DATA RING CREATED\n");
3221
3222         return 0;
3223 }
3224
3225 static int
3226 beiscsi_post_pages(struct beiscsi_hba *phba)
3227 {
3228         struct be_mem_descriptor *mem_descr;
3229         struct mem_array *pm_arr;
3230         unsigned int page_offset, i;
3231         struct be_dma_mem sgl;
3232         int status;
3233
3234         mem_descr = phba->init_mem;
3235         mem_descr += HWI_MEM_SGE;
3236         pm_arr = mem_descr->mem_array;
3237
3238         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3239                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
3240         for (i = 0; i < mem_descr->num_elements; i++) {
3241                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3242                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3243                                                 page_offset,
3244                                                 (pm_arr->size / PAGE_SIZE));
3245                 page_offset += pm_arr->size / PAGE_SIZE;
3246                 if (status != 0) {
3247                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3248                                     "BM_%d : post sgl failed.\n");
3249                         return status;
3250                 }
3251                 pm_arr++;
3252         }
3253         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3254                     "BM_%d : POSTED PAGES\n");
3255         return 0;
3256 }
3257
3258 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3259 {
3260         struct be_dma_mem *mem = &q->dma_mem;
3261         if (mem->va) {
3262                 pci_free_consistent(phba->pcidev, mem->size,
3263                         mem->va, mem->dma);
3264                 mem->va = NULL;
3265         }
3266 }
3267
3268 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3269                 u16 len, u16 entry_size)
3270 {
3271         struct be_dma_mem *mem = &q->dma_mem;
3272
3273         memset(q, 0, sizeof(*q));
3274         q->len = len;
3275         q->entry_size = entry_size;
3276         mem->size = len * entry_size;
3277         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3278         if (!mem->va)
3279                 return -ENOMEM;
3280         memset(mem->va, 0, mem->size);
3281         return 0;
3282 }
3283
3284 static int
3285 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3286                          struct hwi_context_memory *phwi_context,
3287                          struct hwi_controller *phwi_ctrlr)
3288 {
3289         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3290         u64 pa_addr_lo;
3291         unsigned int idx, num, i;
3292         struct mem_array *pwrb_arr;
3293         void *wrb_vaddr;
3294         struct be_dma_mem sgl;
3295         struct be_mem_descriptor *mem_descr;
3296         int status;
3297
3298         idx = 0;
3299         mem_descr = phba->init_mem;
3300         mem_descr += HWI_MEM_WRB;
3301         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3302                            GFP_KERNEL);
3303         if (!pwrb_arr) {
3304                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3305                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3306                 return -ENOMEM;
3307         }
3308         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3309         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3310         num_wrb_rings = mem_descr->mem_array[idx].size /
3311                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3312
3313         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3314                 if (num_wrb_rings) {
3315                         pwrb_arr[num].virtual_address = wrb_vaddr;
3316                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3317                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3318                                             sizeof(struct iscsi_wrb);
3319                         wrb_vaddr += pwrb_arr[num].size;
3320                         pa_addr_lo += pwrb_arr[num].size;
3321                         num_wrb_rings--;
3322                 } else {
3323                         idx++;
3324                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3325                         pa_addr_lo = mem_descr->mem_array[idx].\
3326                                         bus_address.u.a64.address;
3327                         num_wrb_rings = mem_descr->mem_array[idx].size /
3328                                         (phba->params.wrbs_per_cxn *
3329                                         sizeof(struct iscsi_wrb));
3330                         pwrb_arr[num].virtual_address = wrb_vaddr;
3331                         pwrb_arr[num].bus_address.u.a64.address\
3332                                                 = pa_addr_lo;
3333                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3334                                                  sizeof(struct iscsi_wrb);
3335                         wrb_vaddr += pwrb_arr[num].size;
3336                         pa_addr_lo   += pwrb_arr[num].size;
3337                         num_wrb_rings--;
3338                 }
3339         }
3340         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3341                 wrb_mem_index = 0;
3342                 offset = 0;
3343                 size = 0;
3344
3345                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3346                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3347                                             &phwi_context->be_wrbq[i]);
3348                 if (status != 0) {
3349                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3350                                     "BM_%d : wrbq create failed.");
3351                         kfree(pwrb_arr);
3352                         return status;
3353                 }
3354                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3355                                                                    id;
3356         }
3357         kfree(pwrb_arr);
3358         return 0;
3359 }
3360
3361 static void free_wrb_handles(struct beiscsi_hba *phba)
3362 {
3363         unsigned int index;
3364         struct hwi_controller *phwi_ctrlr;
3365         struct hwi_wrb_context *pwrb_context;
3366
3367         phwi_ctrlr = phba->phwi_ctrlr;
3368         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3369                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3370                 kfree(pwrb_context->pwrb_handle_base);
3371                 kfree(pwrb_context->pwrb_handle_basestd);
3372         }
3373 }
3374
3375 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3376 {
3377         struct be_queue_info *q;
3378         struct be_ctrl_info *ctrl = &phba->ctrl;
3379
3380         q = &phba->ctrl.mcc_obj.q;
3381         if (q->created)
3382                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3383         be_queue_free(phba, q);
3384
3385         q = &phba->ctrl.mcc_obj.cq;
3386         if (q->created)
3387                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3388         be_queue_free(phba, q);
3389 }
3390
3391 static void hwi_cleanup(struct beiscsi_hba *phba)
3392 {
3393         struct be_queue_info *q;
3394         struct be_ctrl_info *ctrl = &phba->ctrl;
3395         struct hwi_controller *phwi_ctrlr;
3396         struct hwi_context_memory *phwi_context;
3397         int i, eq_num;
3398
3399         phwi_ctrlr = phba->phwi_ctrlr;
3400         phwi_context = phwi_ctrlr->phwi_ctxt;
3401         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3402                 q = &phwi_context->be_wrbq[i];
3403                 if (q->created)
3404                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3405         }
3406         free_wrb_handles(phba);
3407
3408         q = &phwi_context->be_def_hdrq;
3409         if (q->created)
3410                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3411
3412         q = &phwi_context->be_def_dataq;
3413         if (q->created)
3414                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3415
3416         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3417
3418         for (i = 0; i < (phba->num_cpus); i++) {
3419                 q = &phwi_context->be_cq[i];
3420                 if (q->created)
3421                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3422         }
3423         if (phba->msix_enabled)
3424                 eq_num = 1;
3425         else
3426                 eq_num = 0;
3427         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3428                 q = &phwi_context->be_eq[i].q;
3429                 if (q->created)
3430                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3431         }
3432         be_mcc_queues_destroy(phba);
3433 }
3434
3435 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3436                                 struct hwi_context_memory *phwi_context)
3437 {
3438         struct be_queue_info *q, *cq;
3439         struct be_ctrl_info *ctrl = &phba->ctrl;
3440
3441         /* Alloc MCC compl queue */
3442         cq = &phba->ctrl.mcc_obj.cq;
3443         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3444                         sizeof(struct be_mcc_compl)))
3445                 goto err;
3446         /* Ask BE to create MCC compl queue; */
3447         if (phba->msix_enabled) {
3448                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3449                                          [phba->num_cpus].q, false, true, 0))
3450                 goto mcc_cq_free;
3451         } else {
3452                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3453                                           false, true, 0))
3454                 goto mcc_cq_free;
3455         }
3456
3457         /* Alloc MCC queue */
3458         q = &phba->ctrl.mcc_obj.q;
3459         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3460                 goto mcc_cq_destroy;
3461
3462         /* Ask BE to create MCC queue */
3463         if (beiscsi_cmd_mccq_create(phba, q, cq))
3464                 goto mcc_q_free;
3465
3466         return 0;
3467
3468 mcc_q_free:
3469         be_queue_free(phba, q);
3470 mcc_cq_destroy:
3471         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3472 mcc_cq_free:
3473         be_queue_free(phba, cq);
3474 err:
3475         return -ENOMEM;
3476 }
3477
3478 /**
3479  * find_num_cpus()- Get the CPU online count
3480  * @phba: ptr to priv structure
3481  *
3482  * CPU count is used for creating EQ.
3483  **/
3484 static void find_num_cpus(struct beiscsi_hba *phba)
3485 {
3486         int  num_cpus = 0;
3487
3488         num_cpus = num_online_cpus();
3489
3490         switch (phba->generation) {
3491         case BE_GEN2:
3492         case BE_GEN3:
3493                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3494                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3495                 break;
3496         case BE_GEN4:
3497                 phba->num_cpus = (num_cpus > OC_SKH_MAX_NUM_CPUS) ?
3498                                   OC_SKH_MAX_NUM_CPUS : num_cpus;
3499                 break;
3500         default:
3501                 phba->num_cpus = 1;
3502         }
3503 }
3504
3505 static int hwi_init_port(struct beiscsi_hba *phba)
3506 {
3507         struct hwi_controller *phwi_ctrlr;
3508         struct hwi_context_memory *phwi_context;
3509         unsigned int def_pdu_ring_sz;
3510         struct be_ctrl_info *ctrl = &phba->ctrl;
3511         int status;
3512
3513         def_pdu_ring_sz =
3514                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3515         phwi_ctrlr = phba->phwi_ctrlr;
3516         phwi_context = phwi_ctrlr->phwi_ctxt;
3517         phwi_context->max_eqd = 0;
3518         phwi_context->min_eqd = 0;
3519         phwi_context->cur_eqd = 64;
3520         be_cmd_fw_initialize(&phba->ctrl);
3521
3522         status = beiscsi_create_eqs(phba, phwi_context);
3523         if (status != 0) {
3524                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3525                             "BM_%d : EQ not created\n");
3526                 goto error;
3527         }
3528
3529         status = be_mcc_queues_create(phba, phwi_context);
3530         if (status != 0)
3531                 goto error;
3532
3533         status = mgmt_check_supported_fw(ctrl, phba);
3534         if (status != 0) {
3535                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3536                             "BM_%d : Unsupported fw version\n");
3537                 goto error;
3538         }
3539
3540         status = beiscsi_create_cqs(phba, phwi_context);
3541         if (status != 0) {
3542                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3543                             "BM_%d : CQ not created\n");
3544                 goto error;
3545         }
3546
3547         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3548                                         def_pdu_ring_sz);
3549         if (status != 0) {
3550                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3551                             "BM_%d : Default Header not created\n");
3552                 goto error;
3553         }
3554
3555         status = beiscsi_create_def_data(phba, phwi_context,
3556                                          phwi_ctrlr, def_pdu_ring_sz);
3557         if (status != 0) {
3558                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3559                             "BM_%d : Default Data not created\n");
3560                 goto error;
3561         }
3562
3563         status = beiscsi_post_pages(phba);
3564         if (status != 0) {
3565                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3566                             "BM_%d : Post SGL Pages Failed\n");
3567                 goto error;
3568         }
3569
3570         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3571         if (status != 0) {
3572                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3573                             "BM_%d : WRB Rings not created\n");
3574                 goto error;
3575         }
3576
3577         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3578                     "BM_%d : hwi_init_port success\n");
3579         return 0;
3580
3581 error:
3582         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3583                     "BM_%d : hwi_init_port failed");
3584         hwi_cleanup(phba);
3585         return status;
3586 }
3587
3588 static int hwi_init_controller(struct beiscsi_hba *phba)
3589 {
3590         struct hwi_controller *phwi_ctrlr;
3591
3592         phwi_ctrlr = phba->phwi_ctrlr;
3593         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3594                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3595                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3596                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3597                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3598                             phwi_ctrlr->phwi_ctxt);
3599         } else {
3600                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3601                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3602                             "than one element.Failing to load\n");
3603                 return -ENOMEM;
3604         }
3605
3606         iscsi_init_global_templates(phba);
3607         if (beiscsi_init_wrb_handle(phba))
3608                 return -ENOMEM;
3609
3610         hwi_init_async_pdu_ctx(phba);
3611         if (hwi_init_port(phba) != 0) {
3612                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3613                             "BM_%d : hwi_init_controller failed\n");
3614
3615                 return -ENOMEM;
3616         }
3617         return 0;
3618 }
3619
3620 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3621 {
3622         struct be_mem_descriptor *mem_descr;
3623         int i, j;
3624
3625         mem_descr = phba->init_mem;
3626         i = 0;
3627         j = 0;
3628         for (i = 0; i < SE_MEM_MAX; i++) {
3629                 for (j = mem_descr->num_elements; j > 0; j--) {
3630                         pci_free_consistent(phba->pcidev,
3631                           mem_descr->mem_array[j - 1].size,
3632                           mem_descr->mem_array[j - 1].virtual_address,
3633                           (unsigned long)mem_descr->mem_array[j - 1].
3634                           bus_address.u.a64.address);
3635                 }
3636                 kfree(mem_descr->mem_array);
3637                 mem_descr++;
3638         }
3639         kfree(phba->init_mem);
3640         kfree(phba->phwi_ctrlr);
3641 }
3642
3643 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3644 {
3645         int ret = -ENOMEM;
3646
3647         ret = beiscsi_get_memory(phba);
3648         if (ret < 0) {
3649                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3650                             "BM_%d : beiscsi_dev_probe -"
3651                             "Failed in beiscsi_alloc_memory\n");
3652                 return ret;
3653         }
3654
3655         ret = hwi_init_controller(phba);
3656         if (ret)
3657                 goto free_init;
3658         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3659                     "BM_%d : Return success from beiscsi_init_controller");
3660
3661         return 0;
3662
3663 free_init:
3664         beiscsi_free_mem(phba);
3665         return ret;
3666 }
3667
3668 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3669 {
3670         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3671         struct sgl_handle *psgl_handle;
3672         struct iscsi_sge *pfrag;
3673         unsigned int arr_index, i, idx;
3674
3675         phba->io_sgl_hndl_avbl = 0;
3676         phba->eh_sgl_hndl_avbl = 0;
3677
3678         mem_descr_sglh = phba->init_mem;
3679         mem_descr_sglh += HWI_MEM_SGLH;
3680         if (1 == mem_descr_sglh->num_elements) {
3681                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3682                                                  phba->params.ios_per_ctrl,
3683                                                  GFP_KERNEL);
3684                 if (!phba->io_sgl_hndl_base) {
3685                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3686                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3687                         return -ENOMEM;
3688                 }
3689                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3690                                                  (phba->params.icds_per_ctrl -
3691                                                  phba->params.ios_per_ctrl),
3692                                                  GFP_KERNEL);
3693                 if (!phba->eh_sgl_hndl_base) {
3694                         kfree(phba->io_sgl_hndl_base);
3695                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3696                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3697                         return -ENOMEM;
3698                 }
3699         } else {
3700                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3701                             "BM_%d : HWI_MEM_SGLH is more than one element."
3702                             "Failing to load\n");
3703                 return -ENOMEM;
3704         }
3705
3706         arr_index = 0;
3707         idx = 0;
3708         while (idx < mem_descr_sglh->num_elements) {
3709                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3710
3711                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3712                       sizeof(struct sgl_handle)); i++) {
3713                         if (arr_index < phba->params.ios_per_ctrl) {
3714                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3715                                 phba->io_sgl_hndl_avbl++;
3716                                 arr_index++;
3717                         } else {
3718                                 phba->eh_sgl_hndl_base[arr_index -
3719                                         phba->params.ios_per_ctrl] =
3720                                                                 psgl_handle;
3721                                 arr_index++;
3722                                 phba->eh_sgl_hndl_avbl++;
3723                         }
3724                         psgl_handle++;
3725                 }
3726                 idx++;
3727         }
3728         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3729                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
3730                     "phba->eh_sgl_hndl_avbl=%d\n",
3731                     phba->io_sgl_hndl_avbl,
3732                     phba->eh_sgl_hndl_avbl);
3733
3734         mem_descr_sg = phba->init_mem;
3735         mem_descr_sg += HWI_MEM_SGE;
3736         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3737                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3738                     mem_descr_sg->num_elements);
3739
3740         arr_index = 0;
3741         idx = 0;
3742         while (idx < mem_descr_sg->num_elements) {
3743                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3744
3745                 for (i = 0;
3746                      i < (mem_descr_sg->mem_array[idx].size) /
3747                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3748                      i++) {
3749                         if (arr_index < phba->params.ios_per_ctrl)
3750                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3751                         else
3752                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3753                                                 phba->params.ios_per_ctrl];
3754                         psgl_handle->pfrag = pfrag;
3755                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3756                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3757                         pfrag += phba->params.num_sge_per_io;
3758                         psgl_handle->sgl_index =
3759                                 phba->fw_config.iscsi_icd_start + arr_index++;
3760                 }
3761                 idx++;
3762         }
3763         phba->io_sgl_free_index = 0;
3764         phba->io_sgl_alloc_index = 0;
3765         phba->eh_sgl_free_index = 0;
3766         phba->eh_sgl_alloc_index = 0;
3767         return 0;
3768 }
3769
3770 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3771 {
3772         int i, new_cid;
3773
3774         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3775                                   GFP_KERNEL);
3776         if (!phba->cid_array) {
3777                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3778                             "BM_%d : Failed to allocate memory in "
3779                             "hba_setup_cid_tbls\n");
3780                 return -ENOMEM;
3781         }
3782         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3783                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3784         if (!phba->ep_array) {
3785                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3786                             "BM_%d : Failed to allocate memory in "
3787                             "hba_setup_cid_tbls\n");
3788                 kfree(phba->cid_array);
3789                 return -ENOMEM;
3790         }
3791         new_cid = phba->fw_config.iscsi_cid_start;
3792         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3793                 phba->cid_array[i] = new_cid;
3794                 new_cid += 2;
3795         }
3796         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3797         return 0;
3798 }
3799
3800 static void hwi_enable_intr(struct beiscsi_hba *phba)
3801 {
3802         struct be_ctrl_info *ctrl = &phba->ctrl;
3803         struct hwi_controller *phwi_ctrlr;
3804         struct hwi_context_memory *phwi_context;
3805         struct be_queue_info *eq;
3806         u8 __iomem *addr;
3807         u32 reg, i;
3808         u32 enabled;
3809
3810         phwi_ctrlr = phba->phwi_ctrlr;
3811         phwi_context = phwi_ctrlr->phwi_ctxt;
3812
3813         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3814                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3815         reg = ioread32(addr);
3816
3817         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3818         if (!enabled) {
3819                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3820                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3821                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
3822                 iowrite32(reg, addr);
3823         }
3824
3825         if (!phba->msix_enabled) {
3826                 eq = &phwi_context->be_eq[0].q;
3827                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3828                             "BM_%d : eq->id=%d\n", eq->id);
3829
3830                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3831         } else {
3832                 for (i = 0; i <= phba->num_cpus; i++) {
3833                         eq = &phwi_context->be_eq[i].q;
3834                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3835                                     "BM_%d : eq->id=%d\n", eq->id);
3836                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3837                 }
3838         }
3839 }
3840
3841 static void hwi_disable_intr(struct beiscsi_hba *phba)
3842 {
3843         struct be_ctrl_info *ctrl = &phba->ctrl;
3844
3845         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3846         u32 reg = ioread32(addr);
3847
3848         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3849         if (enabled) {
3850                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3851                 iowrite32(reg, addr);
3852         } else
3853                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
3854                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
3855 }
3856
3857 /**
3858  * beiscsi_get_boot_info()- Get the boot session info
3859  * @phba: The device priv structure instance
3860  *
3861  * Get the boot target info and store in driver priv structure
3862  *
3863  * return values
3864  *      Success: 0
3865  *      Failure: Non-Zero Value
3866  **/
3867 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3868 {
3869         struct be_cmd_get_session_resp *session_resp;
3870         struct be_dma_mem nonemb_cmd;
3871         unsigned int tag;
3872         unsigned int s_handle;
3873         int ret = -ENOMEM;
3874
3875         /* Get the session handle of the boot target */
3876         ret = be_mgmt_get_boot_shandle(phba, &s_handle);
3877         if (ret) {
3878                 beiscsi_log(phba, KERN_ERR,
3879                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3880                             "BM_%d : No boot session\n");
3881                 return ret;
3882         }
3883         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3884                                 sizeof(*session_resp),
3885                                 &nonemb_cmd.dma);
3886         if (nonemb_cmd.va == NULL) {
3887                 beiscsi_log(phba, KERN_ERR,
3888                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3889                             "BM_%d : Failed to allocate memory for"
3890                             "beiscsi_get_session_info\n");
3891
3892                 return -ENOMEM;
3893         }
3894
3895         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3896         tag = mgmt_get_session_info(phba, s_handle,
3897                                     &nonemb_cmd);
3898         if (!tag) {
3899                 beiscsi_log(phba, KERN_ERR,
3900                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3901                             "BM_%d : beiscsi_get_session_info"
3902                             " Failed\n");
3903
3904                 goto boot_freemem;
3905         }
3906
3907         ret = beiscsi_mccq_compl(phba, tag, NULL, nonemb_cmd.va);
3908         if (ret) {
3909                 beiscsi_log(phba, KERN_ERR,
3910                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3911                             "BM_%d : beiscsi_get_session_info Failed");
3912                 goto boot_freemem;
3913         }
3914
3915         session_resp = nonemb_cmd.va ;
3916
3917         memcpy(&phba->boot_sess, &session_resp->session_info,
3918                sizeof(struct mgmt_session_info));
3919         ret = 0;
3920
3921 boot_freemem:
3922         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3923                     nonemb_cmd.va, nonemb_cmd.dma);
3924         return ret;
3925 }
3926
3927 static void beiscsi_boot_release(void *data)
3928 {
3929         struct beiscsi_hba *phba = data;
3930
3931         scsi_host_put(phba->shost);
3932 }
3933
3934 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3935 {
3936         struct iscsi_boot_kobj *boot_kobj;
3937
3938         /* get boot info using mgmt cmd */
3939         if (beiscsi_get_boot_info(phba))
3940                 /* Try to see if we can carry on without this */
3941                 return 0;
3942
3943         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3944         if (!phba->boot_kset)
3945                 return -ENOMEM;
3946
3947         /* get a ref because the show function will ref the phba */
3948         if (!scsi_host_get(phba->shost))
3949                 goto free_kset;
3950         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3951                                              beiscsi_show_boot_tgt_info,
3952                                              beiscsi_tgt_get_attr_visibility,
3953                                              beiscsi_boot_release);
3954         if (!boot_kobj)
3955                 goto put_shost;
3956
3957         if (!scsi_host_get(phba->shost))
3958                 goto free_kset;
3959         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3960                                                 beiscsi_show_boot_ini_info,
3961                                                 beiscsi_ini_get_attr_visibility,
3962                                                 beiscsi_boot_release);
3963         if (!boot_kobj)
3964                 goto put_shost;
3965
3966         if (!scsi_host_get(phba->shost))
3967                 goto free_kset;
3968         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3969                                                beiscsi_show_boot_eth_info,
3970                                                beiscsi_eth_get_attr_visibility,
3971                                                beiscsi_boot_release);
3972         if (!boot_kobj)
3973                 goto put_shost;
3974         return 0;
3975
3976 put_shost:
3977         scsi_host_put(phba->shost);
3978 free_kset:
3979         iscsi_boot_destroy_kset(phba->boot_kset);
3980         return -ENOMEM;
3981 }
3982
3983 static int beiscsi_init_port(struct beiscsi_hba *phba)
3984 {
3985         int ret;
3986
3987         ret = beiscsi_init_controller(phba);
3988         if (ret < 0) {
3989                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3990                             "BM_%d : beiscsi_dev_probe - Failed in"
3991                             "beiscsi_init_controller\n");
3992                 return ret;
3993         }
3994         ret = beiscsi_init_sgl_handle(phba);
3995         if (ret < 0) {
3996                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3997                             "BM_%d : beiscsi_dev_probe - Failed in"
3998                             "beiscsi_init_sgl_handle\n");
3999                 goto do_cleanup_ctrlr;
4000         }
4001
4002         if (hba_setup_cid_tbls(phba)) {
4003                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4004                             "BM_%d : Failed in hba_setup_cid_tbls\n");
4005                 kfree(phba->io_sgl_hndl_base);
4006                 kfree(phba->eh_sgl_hndl_base);
4007                 goto do_cleanup_ctrlr;
4008         }
4009
4010         return ret;
4011
4012 do_cleanup_ctrlr:
4013         hwi_cleanup(phba);
4014         return ret;
4015 }
4016
4017 static void hwi_purge_eq(struct beiscsi_hba *phba)
4018 {
4019         struct hwi_controller *phwi_ctrlr;
4020         struct hwi_context_memory *phwi_context;
4021         struct be_queue_info *eq;
4022         struct be_eq_entry *eqe = NULL;
4023         int i, eq_msix;
4024         unsigned int num_processed;
4025
4026         phwi_ctrlr = phba->phwi_ctrlr;
4027         phwi_context = phwi_ctrlr->phwi_ctxt;
4028         if (phba->msix_enabled)
4029                 eq_msix = 1;
4030         else
4031                 eq_msix = 0;
4032
4033         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4034                 eq = &phwi_context->be_eq[i].q;
4035                 eqe = queue_tail_node(eq);
4036                 num_processed = 0;
4037                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4038                                         & EQE_VALID_MASK) {
4039                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4040                         queue_tail_inc(eq);
4041                         eqe = queue_tail_node(eq);
4042                         num_processed++;
4043                 }
4044
4045                 if (num_processed)
4046                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
4047         }
4048 }
4049
4050 static void beiscsi_clean_port(struct beiscsi_hba *phba)
4051 {
4052         int mgmt_status;
4053
4054         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
4055         if (mgmt_status)
4056                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4057                             "BM_%d : mgmt_epfw_cleanup FAILED\n");
4058
4059         hwi_purge_eq(phba);
4060         hwi_cleanup(phba);
4061         kfree(phba->io_sgl_hndl_base);
4062         kfree(phba->eh_sgl_hndl_base);
4063         kfree(phba->cid_array);
4064         kfree(phba->ep_array);
4065 }
4066
4067 /**
4068  * beiscsi_cleanup_task()- Free driver resources of the task
4069  * @task: ptr to the iscsi task
4070  *
4071  **/
4072 static void beiscsi_cleanup_task(struct iscsi_task *task)
4073 {
4074         struct beiscsi_io_task *io_task = task->dd_data;
4075         struct iscsi_conn *conn = task->conn;
4076         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4077         struct beiscsi_hba *phba = beiscsi_conn->phba;
4078         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4079         struct hwi_wrb_context *pwrb_context;
4080         struct hwi_controller *phwi_ctrlr;
4081
4082         phwi_ctrlr = phba->phwi_ctrlr;
4083         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
4084                         - phba->fw_config.iscsi_cid_start];
4085
4086         if (io_task->cmd_bhs) {
4087                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4088                               io_task->bhs_pa.u.a64.address);
4089                 io_task->cmd_bhs = NULL;
4090         }
4091
4092         if (task->sc) {
4093                 if (io_task->pwrb_handle) {
4094                         free_wrb_handle(phba, pwrb_context,
4095                                         io_task->pwrb_handle);
4096                         io_task->pwrb_handle = NULL;
4097                 }
4098
4099                 if (io_task->psgl_handle) {
4100                         spin_lock(&phba->io_sgl_lock);
4101                         free_io_sgl_handle(phba, io_task->psgl_handle);
4102                         spin_unlock(&phba->io_sgl_lock);
4103                         io_task->psgl_handle = NULL;
4104                 }
4105         } else {
4106                 if (!beiscsi_conn->login_in_progress) {
4107                         if (io_task->pwrb_handle) {
4108                                 free_wrb_handle(phba, pwrb_context,
4109                                                 io_task->pwrb_handle);
4110                                 io_task->pwrb_handle = NULL;
4111                         }
4112                         if (io_task->psgl_handle) {
4113                                 spin_lock(&phba->mgmt_sgl_lock);
4114                                 free_mgmt_sgl_handle(phba,
4115                                                      io_task->psgl_handle);
4116                                 spin_unlock(&phba->mgmt_sgl_lock);
4117                                 io_task->psgl_handle = NULL;
4118                         }
4119                         if (io_task->mtask_addr) {
4120                                 pci_unmap_single(phba->pcidev,
4121                                                  io_task->mtask_addr,
4122                                                  io_task->mtask_data_count,
4123                                                  PCI_DMA_TODEVICE);
4124                                 io_task->mtask_addr = 0;
4125                         }
4126                 }
4127         }
4128 }
4129
4130 void
4131 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4132                            struct beiscsi_offload_params *params)
4133 {
4134         struct wrb_handle *pwrb_handle;
4135         struct beiscsi_hba *phba = beiscsi_conn->phba;
4136         struct iscsi_task *task = beiscsi_conn->task;
4137         struct iscsi_session *session = task->conn->session;
4138         u32 doorbell = 0;
4139
4140         /*
4141          * We can always use 0 here because it is reserved by libiscsi for
4142          * login/startup related tasks.
4143          */
4144         beiscsi_conn->login_in_progress = 0;
4145         spin_lock_bh(&session->lock);
4146         beiscsi_cleanup_task(task);
4147         spin_unlock_bh(&session->lock);
4148
4149         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
4150                                        phba->fw_config.iscsi_cid_start));
4151
4152         /* Check for the adapter family */
4153         if (chip_skh_r(phba->pcidev))
4154                 beiscsi_offload_cxn_v2(params, pwrb_handle);
4155         else
4156                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4157                                        phba->init_mem);
4158
4159         be_dws_le_to_cpu(pwrb_handle->pwrb,
4160                          sizeof(struct iscsi_target_context_update_wrb));
4161
4162         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4163         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4164                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4165         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4166
4167         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4168 }
4169
4170 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4171                               int *index, int *age)
4172 {
4173         *index = (int)itt;
4174         if (age)
4175                 *age = conn->session->age;
4176 }
4177
4178 /**
4179  * beiscsi_alloc_pdu - allocates pdu and related resources
4180  * @task: libiscsi task
4181  * @opcode: opcode of pdu for task
4182  *
4183  * This is called with the session lock held. It will allocate
4184  * the wrb and sgl if needed for the command. And it will prep
4185  * the pdu's itt. beiscsi_parse_pdu will later translate
4186  * the pdu itt to the libiscsi task itt.
4187  */
4188 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4189 {
4190         struct beiscsi_io_task *io_task = task->dd_data;
4191         struct iscsi_conn *conn = task->conn;
4192         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4193         struct beiscsi_hba *phba = beiscsi_conn->phba;
4194         struct hwi_wrb_context *pwrb_context;
4195         struct hwi_controller *phwi_ctrlr;
4196         itt_t itt;
4197         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4198         dma_addr_t paddr;
4199
4200         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4201                                           GFP_ATOMIC, &paddr);
4202         if (!io_task->cmd_bhs)
4203                 return -ENOMEM;
4204         io_task->bhs_pa.u.a64.address = paddr;
4205         io_task->libiscsi_itt = (itt_t)task->itt;
4206         io_task->conn = beiscsi_conn;
4207
4208         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4209         task->hdr_max = sizeof(struct be_cmd_bhs);
4210         io_task->psgl_handle = NULL;
4211         io_task->pwrb_handle = NULL;
4212
4213         if (task->sc) {
4214                 spin_lock(&phba->io_sgl_lock);
4215                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4216                 spin_unlock(&phba->io_sgl_lock);
4217                 if (!io_task->psgl_handle) {
4218                         beiscsi_log(phba, KERN_ERR,
4219                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4220                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4221                                     "for the CID : %d\n",
4222                                     beiscsi_conn->beiscsi_conn_cid);
4223                         goto free_hndls;
4224                 }
4225                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4226                                         beiscsi_conn->beiscsi_conn_cid -
4227                                         phba->fw_config.iscsi_cid_start);
4228                 if (!io_task->pwrb_handle) {
4229                         beiscsi_log(phba, KERN_ERR,
4230                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4231                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4232                                     "for the CID : %d\n",
4233                                     beiscsi_conn->beiscsi_conn_cid);
4234                         goto free_io_hndls;
4235                 }
4236         } else {
4237                 io_task->scsi_cmnd = NULL;
4238                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4239                         if (!beiscsi_conn->login_in_progress) {
4240                                 spin_lock(&phba->mgmt_sgl_lock);
4241                                 io_task->psgl_handle = (struct sgl_handle *)
4242                                                 alloc_mgmt_sgl_handle(phba);
4243                                 spin_unlock(&phba->mgmt_sgl_lock);
4244                                 if (!io_task->psgl_handle) {
4245                                         beiscsi_log(phba, KERN_ERR,
4246                                                     BEISCSI_LOG_IO |
4247                                                     BEISCSI_LOG_CONFIG,
4248                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4249                                                     "for the CID : %d\n",
4250                                                     beiscsi_conn->
4251                                                     beiscsi_conn_cid);
4252                                         goto free_hndls;
4253                                 }
4254
4255                                 beiscsi_conn->login_in_progress = 1;
4256                                 beiscsi_conn->plogin_sgl_handle =
4257                                                         io_task->psgl_handle;
4258                                 io_task->pwrb_handle =
4259                                         alloc_wrb_handle(phba,
4260                                         beiscsi_conn->beiscsi_conn_cid -
4261                                         phba->fw_config.iscsi_cid_start);
4262                                 if (!io_task->pwrb_handle) {
4263                                         beiscsi_log(phba, KERN_ERR,
4264                                                     BEISCSI_LOG_IO |
4265                                                     BEISCSI_LOG_CONFIG,
4266                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4267                                                     "for the CID : %d\n",
4268                                                     beiscsi_conn->
4269                                                     beiscsi_conn_cid);
4270                                         goto free_mgmt_hndls;
4271                                 }
4272                                 beiscsi_conn->plogin_wrb_handle =
4273                                                         io_task->pwrb_handle;
4274
4275                         } else {
4276                                 io_task->psgl_handle =
4277                                                 beiscsi_conn->plogin_sgl_handle;
4278                                 io_task->pwrb_handle =
4279                                                 beiscsi_conn->plogin_wrb_handle;
4280                         }
4281                         beiscsi_conn->task = task;
4282                 } else {
4283                         spin_lock(&phba->mgmt_sgl_lock);
4284                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4285                         spin_unlock(&phba->mgmt_sgl_lock);
4286                         if (!io_task->psgl_handle) {
4287                                 beiscsi_log(phba, KERN_ERR,
4288                                             BEISCSI_LOG_IO |
4289                                             BEISCSI_LOG_CONFIG,
4290                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4291                                             "for the CID : %d\n",
4292                                             beiscsi_conn->
4293                                             beiscsi_conn_cid);
4294                                 goto free_hndls;
4295                         }
4296                         io_task->pwrb_handle =
4297                                         alloc_wrb_handle(phba,
4298                                         beiscsi_conn->beiscsi_conn_cid -
4299                                         phba->fw_config.iscsi_cid_start);
4300                         if (!io_task->pwrb_handle) {
4301                                 beiscsi_log(phba, KERN_ERR,
4302                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4303                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4304                                             "for the CID : %d\n",
4305                                             beiscsi_conn->beiscsi_conn_cid);
4306                                 goto free_mgmt_hndls;
4307                         }
4308
4309                 }
4310         }
4311         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4312                                  wrb_index << 16) | (unsigned int)
4313                                 (io_task->psgl_handle->sgl_index));
4314         io_task->pwrb_handle->pio_handle = task;
4315
4316         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4317         return 0;
4318
4319 free_io_hndls:
4320         spin_lock(&phba->io_sgl_lock);
4321         free_io_sgl_handle(phba, io_task->psgl_handle);
4322         spin_unlock(&phba->io_sgl_lock);
4323         goto free_hndls;
4324 free_mgmt_hndls:
4325         spin_lock(&phba->mgmt_sgl_lock);
4326         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4327         spin_unlock(&phba->mgmt_sgl_lock);
4328 free_hndls:
4329         phwi_ctrlr = phba->phwi_ctrlr;
4330         pwrb_context = &phwi_ctrlr->wrb_context[
4331                         beiscsi_conn->beiscsi_conn_cid -
4332                         phba->fw_config.iscsi_cid_start];
4333         if (io_task->pwrb_handle)
4334                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4335         io_task->pwrb_handle = NULL;
4336         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4337                       io_task->bhs_pa.u.a64.address);
4338         io_task->cmd_bhs = NULL;
4339         return -ENOMEM;
4340 }
4341 int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4342                        unsigned int num_sg, unsigned int xferlen,
4343                        unsigned int writedir)
4344 {
4345
4346         struct beiscsi_io_task *io_task = task->dd_data;
4347         struct iscsi_conn *conn = task->conn;
4348         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4349         struct beiscsi_hba *phba = beiscsi_conn->phba;
4350         struct iscsi_wrb *pwrb = NULL;
4351         unsigned int doorbell = 0;
4352
4353         pwrb = io_task->pwrb_handle->pwrb;
4354         memset(pwrb, 0, sizeof(*pwrb));
4355
4356         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4357         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4358
4359         if (writedir) {
4360                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4361                               INI_WR_CMD);
4362                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4363         } else {
4364                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4365                               INI_RD_CMD);
4366                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4367         }
4368
4369         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4370                                           type, pwrb);
4371
4372         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4373                       cpu_to_be16(*(unsigned short *)
4374                       &io_task->cmd_bhs->iscsi_hdr.lun));
4375         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4376         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4377                       io_task->pwrb_handle->wrb_index);
4378         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4379                       be32_to_cpu(task->cmdsn));
4380         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4381                       io_task->psgl_handle->sgl_index);
4382
4383         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4384         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4385                       io_task->pwrb_handle->nxt_wrb_index);
4386
4387         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4388
4389         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4390         doorbell |= (io_task->pwrb_handle->wrb_index &
4391                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4392                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4393         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4394         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4395         return 0;
4396 }
4397
4398 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4399                           unsigned int num_sg, unsigned int xferlen,
4400                           unsigned int writedir)
4401 {
4402
4403         struct beiscsi_io_task *io_task = task->dd_data;
4404         struct iscsi_conn *conn = task->conn;
4405         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4406         struct beiscsi_hba *phba = beiscsi_conn->phba;
4407         struct iscsi_wrb *pwrb = NULL;
4408         unsigned int doorbell = 0;
4409
4410         pwrb = io_task->pwrb_handle->pwrb;
4411         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4412         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4413
4414         if (writedir) {
4415                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4416                               INI_WR_CMD);
4417                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4418         } else {
4419                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4420                               INI_RD_CMD);
4421                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4422         }
4423
4424         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4425                                           type, pwrb);
4426
4427         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4428                       cpu_to_be16(*(unsigned short *)
4429                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4430         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4431         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4432                       io_task->pwrb_handle->wrb_index);
4433         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4434                       be32_to_cpu(task->cmdsn));
4435         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4436                       io_task->psgl_handle->sgl_index);
4437
4438         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4439
4440         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4441                       io_task->pwrb_handle->nxt_wrb_index);
4442         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4443
4444         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4445         doorbell |= (io_task->pwrb_handle->wrb_index &
4446                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4447         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4448
4449         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4450         return 0;
4451 }
4452
4453 static int beiscsi_mtask(struct iscsi_task *task)
4454 {
4455         struct beiscsi_io_task *io_task = task->dd_data;
4456         struct iscsi_conn *conn = task->conn;
4457         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4458         struct beiscsi_hba *phba = beiscsi_conn->phba;
4459         struct iscsi_wrb *pwrb = NULL;
4460         unsigned int doorbell = 0;
4461         unsigned int cid;
4462         unsigned int pwrb_typeoffset = 0;
4463
4464         cid = beiscsi_conn->beiscsi_conn_cid;
4465         pwrb = io_task->pwrb_handle->pwrb;
4466         memset(pwrb, 0, sizeof(*pwrb));
4467
4468         if (chip_skh_r(phba->pcidev)) {
4469                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4470                               be32_to_cpu(task->cmdsn));
4471                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4472                               io_task->pwrb_handle->wrb_index);
4473                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4474                               io_task->psgl_handle->sgl_index);
4475                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4476                               task->data_count);
4477                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4478                               io_task->pwrb_handle->nxt_wrb_index);
4479                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
4480         } else {
4481                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4482                               be32_to_cpu(task->cmdsn));
4483                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4484                               io_task->pwrb_handle->wrb_index);
4485                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4486                               io_task->psgl_handle->sgl_index);
4487                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4488                               task->data_count);
4489                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4490                               io_task->pwrb_handle->nxt_wrb_index);
4491                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4492         }
4493
4494
4495         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4496         case ISCSI_OP_LOGIN:
4497                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4498                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4499                 hwi_write_buffer(pwrb, task);
4500                 break;
4501         case ISCSI_OP_NOOP_OUT:
4502                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4503                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4504                         if (chip_skh_r(phba->pcidev))
4505                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4506                                               dmsg, pwrb, 1);
4507                         else
4508                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4509                                               dmsg, pwrb, 1);
4510                 } else {
4511                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
4512                         if (chip_skh_r(phba->pcidev))
4513                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4514                                               dmsg, pwrb, 0);
4515                         else
4516                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4517                                               dmsg, pwrb, 0);
4518                 }
4519                 hwi_write_buffer(pwrb, task);
4520                 break;
4521         case ISCSI_OP_TEXT:
4522                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4523                 hwi_write_buffer(pwrb, task);
4524                 break;
4525         case ISCSI_OP_SCSI_TMFUNC:
4526                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
4527                 hwi_write_buffer(pwrb, task);
4528                 break;
4529         case ISCSI_OP_LOGOUT:
4530                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
4531                 hwi_write_buffer(pwrb, task);
4532                 break;
4533
4534         default:
4535                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4536                             "BM_%d : opcode =%d Not supported\n",
4537                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4538
4539                 return -EINVAL;
4540         }
4541
4542         /* Set the task type */
4543         io_task->wrb_type = (chip_skh_r(phba->pcidev)) ?
4544                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb) :
4545                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb);
4546
4547         doorbell |= cid & DB_WRB_POST_CID_MASK;
4548         doorbell |= (io_task->pwrb_handle->wrb_index &
4549                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4550         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4551         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4552         return 0;
4553 }
4554
4555 static int beiscsi_task_xmit(struct iscsi_task *task)
4556 {
4557         struct beiscsi_io_task *io_task = task->dd_data;
4558         struct scsi_cmnd *sc = task->sc;
4559         struct beiscsi_hba *phba = NULL;
4560         struct scatterlist *sg;
4561         int num_sg;
4562         unsigned int  writedir = 0, xferlen = 0;
4563
4564         phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
4565
4566         if (!sc)
4567                 return beiscsi_mtask(task);
4568
4569         io_task->scsi_cmnd = sc;
4570         num_sg = scsi_dma_map(sc);
4571         if (num_sg < 0) {
4572                 struct iscsi_conn *conn = task->conn;
4573                 struct beiscsi_hba *phba = NULL;
4574
4575                 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
4576                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO,
4577                             "BM_%d : scsi_dma_map Failed\n");
4578
4579                 return num_sg;
4580         }
4581         xferlen = scsi_bufflen(sc);
4582         sg = scsi_sglist(sc);
4583         if (sc->sc_data_direction == DMA_TO_DEVICE)
4584                 writedir = 1;
4585          else
4586                 writedir = 0;
4587
4588          return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
4589 }
4590
4591 /**
4592  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4593  * @job: job to handle
4594  */
4595 static int beiscsi_bsg_request(struct bsg_job *job)
4596 {
4597         struct Scsi_Host *shost;
4598         struct beiscsi_hba *phba;
4599         struct iscsi_bsg_request *bsg_req = job->request;
4600         int rc = -EINVAL;
4601         unsigned int tag;
4602         struct be_dma_mem nonemb_cmd;
4603         struct be_cmd_resp_hdr *resp;
4604         struct iscsi_bsg_reply *bsg_reply = job->reply;
4605         unsigned short status, extd_status;
4606
4607         shost = iscsi_job_to_shost(job);
4608         phba = iscsi_host_priv(shost);
4609
4610         switch (bsg_req->msgcode) {
4611         case ISCSI_BSG_HST_VENDOR:
4612                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4613                                         job->request_payload.payload_len,
4614                                         &nonemb_cmd.dma);
4615                 if (nonemb_cmd.va == NULL) {
4616                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4617                                     "BM_%d : Failed to allocate memory for "
4618                                     "beiscsi_bsg_request\n");
4619                         return -ENOMEM;
4620                 }
4621                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4622                                                   &nonemb_cmd);
4623                 if (!tag) {
4624                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4625                                     "BM_%d : MBX Tag Allocation Failed\n");
4626
4627                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4628                                             nonemb_cmd.va, nonemb_cmd.dma);
4629                         return -EAGAIN;
4630                 }
4631
4632                 rc = wait_event_interruptible_timeout(
4633                                         phba->ctrl.mcc_wait[tag],
4634                                         phba->ctrl.mcc_numtag[tag],
4635                                         msecs_to_jiffies(
4636                                         BEISCSI_HOST_MBX_TIMEOUT));
4637                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4638                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4639                 free_mcc_tag(&phba->ctrl, tag);
4640                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4641                 sg_copy_from_buffer(job->reply_payload.sg_list,
4642                                     job->reply_payload.sg_cnt,
4643                                     nonemb_cmd.va, (resp->response_length
4644                                     + sizeof(*resp)));
4645                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4646                 bsg_reply->result = status;
4647                 bsg_job_done(job, bsg_reply->result,
4648                              bsg_reply->reply_payload_rcv_len);
4649                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4650                                     nonemb_cmd.va, nonemb_cmd.dma);
4651                 if (status || extd_status) {
4652                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4653                                     "BM_%d : MBX Cmd Failed"
4654                                     " status = %d extd_status = %d\n",
4655                                     status, extd_status);
4656
4657                         return -EIO;
4658                 } else {
4659                         rc = 0;
4660                 }
4661                 break;
4662
4663         default:
4664                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4665                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4666                                 bsg_req->msgcode);
4667                 break;
4668         }
4669
4670         return rc;
4671 }
4672
4673 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4674 {
4675         /* Set the logging parameter */
4676         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4677 }
4678
4679 /*
4680  * beiscsi_quiesce()- Cleanup Driver resources
4681  * @phba: Instance Priv structure
4682  *
4683  * Free the OS and HW resources held by the driver
4684  **/
4685 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4686 {
4687         struct hwi_controller *phwi_ctrlr;
4688         struct hwi_context_memory *phwi_context;
4689         struct be_eq_obj *pbe_eq;
4690         unsigned int i, msix_vec;
4691
4692         phwi_ctrlr = phba->phwi_ctrlr;
4693         phwi_context = phwi_ctrlr->phwi_ctxt;
4694         hwi_disable_intr(phba);
4695         if (phba->msix_enabled) {
4696                 for (i = 0; i <= phba->num_cpus; i++) {
4697                         msix_vec = phba->msix_entries[i].vector;
4698                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4699                         kfree(phba->msi_name[i]);
4700                 }
4701         } else
4702                 if (phba->pcidev->irq)
4703                         free_irq(phba->pcidev->irq, phba);
4704         pci_disable_msix(phba->pcidev);
4705         destroy_workqueue(phba->wq);
4706         if (blk_iopoll_enabled)
4707                 for (i = 0; i < phba->num_cpus; i++) {
4708                         pbe_eq = &phwi_context->be_eq[i];
4709                         blk_iopoll_disable(&pbe_eq->iopoll);
4710                 }
4711
4712         beiscsi_clean_port(phba);
4713         beiscsi_free_mem(phba);
4714
4715         beiscsi_unmap_pci_function(phba);
4716         pci_free_consistent(phba->pcidev,
4717                             phba->ctrl.mbox_mem_alloced.size,
4718                             phba->ctrl.mbox_mem_alloced.va,
4719                             phba->ctrl.mbox_mem_alloced.dma);
4720
4721         cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
4722 }
4723
4724 static void beiscsi_remove(struct pci_dev *pcidev)
4725 {
4726
4727         struct beiscsi_hba *phba = NULL;
4728
4729         phba = pci_get_drvdata(pcidev);
4730         if (!phba) {
4731                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4732                 return;
4733         }
4734
4735         beiscsi_destroy_def_ifaces(phba);
4736         beiscsi_quiesce(phba);
4737         iscsi_boot_destroy_kset(phba->boot_kset);
4738         iscsi_host_remove(phba->shost);
4739         pci_dev_put(phba->pcidev);
4740         iscsi_host_free(phba->shost);
4741         pci_disable_device(pcidev);
4742 }
4743
4744 static void beiscsi_shutdown(struct pci_dev *pcidev)
4745 {
4746
4747         struct beiscsi_hba *phba = NULL;
4748
4749         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4750         if (!phba) {
4751                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4752                 return;
4753         }
4754
4755         beiscsi_quiesce(phba);
4756         pci_disable_device(pcidev);
4757 }
4758
4759 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4760 {
4761         int i, status;
4762
4763         for (i = 0; i <= phba->num_cpus; i++)
4764                 phba->msix_entries[i].entry = i;
4765
4766         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4767                                  (phba->num_cpus + 1));
4768         if (!status)
4769                 phba->msix_enabled = true;
4770
4771         return;
4772 }
4773
4774 /*
4775  * beiscsi_hw_health_check()- Check adapter health
4776  * @work: work item to check HW health
4777  *
4778  * Check if adapter in an unrecoverable state or not.
4779  **/
4780 static void
4781 beiscsi_hw_health_check(struct work_struct *work)
4782 {
4783         struct beiscsi_hba *phba =
4784                 container_of(work, struct beiscsi_hba,
4785                              beiscsi_hw_check_task.work);
4786
4787         beiscsi_ue_detect(phba);
4788
4789         schedule_delayed_work(&phba->beiscsi_hw_check_task,
4790                               msecs_to_jiffies(1000));
4791 }
4792
4793 static int beiscsi_dev_probe(struct pci_dev *pcidev,
4794                              const struct pci_device_id *id)
4795 {
4796         struct beiscsi_hba *phba = NULL;
4797         struct hwi_controller *phwi_ctrlr;
4798         struct hwi_context_memory *phwi_context;
4799         struct be_eq_obj *pbe_eq;
4800         int ret, i;
4801
4802         ret = beiscsi_enable_pci(pcidev);
4803         if (ret < 0) {
4804                 dev_err(&pcidev->dev,
4805                         "beiscsi_dev_probe - Failed to enable pci device\n");
4806                 return ret;
4807         }
4808
4809         phba = beiscsi_hba_alloc(pcidev);
4810         if (!phba) {
4811                 dev_err(&pcidev->dev,
4812                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
4813                 goto disable_pci;
4814         }
4815
4816         /* Initialize Driver configuration Paramters */
4817         beiscsi_hba_attrs_init(phba);
4818
4819         phba->fw_timeout = false;
4820
4821
4822         switch (pcidev->device) {
4823         case BE_DEVICE_ID1:
4824         case OC_DEVICE_ID1:
4825         case OC_DEVICE_ID2:
4826                 phba->generation = BE_GEN2;
4827                 phba->iotask_fn = beiscsi_iotask;
4828                 break;
4829         case BE_DEVICE_ID2:
4830         case OC_DEVICE_ID3:
4831                 phba->generation = BE_GEN3;
4832                 phba->iotask_fn = beiscsi_iotask;
4833                 break;
4834         case OC_SKH_ID1:
4835                 phba->generation = BE_GEN4;
4836                 phba->iotask_fn = beiscsi_iotask_v2;
4837         default:
4838                 phba->generation = 0;
4839         }
4840
4841         if (enable_msix)
4842                 find_num_cpus(phba);
4843         else
4844                 phba->num_cpus = 1;
4845
4846         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4847                     "BM_%d : num_cpus = %d\n",
4848                     phba->num_cpus);
4849
4850         if (enable_msix) {
4851                 beiscsi_msix_enable(phba);
4852                 if (!phba->msix_enabled)
4853                         phba->num_cpus = 1;
4854         }
4855         ret = be_ctrl_init(phba, pcidev);
4856         if (ret) {
4857                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4858                             "BM_%d : beiscsi_dev_probe-"
4859                             "Failed in be_ctrl_init\n");
4860                 goto hba_free;
4861         }
4862
4863         ret = beiscsi_cmd_reset_function(phba);
4864         if (ret) {
4865                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4866                             "BM_%d : Reset Failed. Aborting Crashdump\n");
4867                 goto hba_free;
4868         }
4869         ret = be_chk_reset_complete(phba);
4870         if (ret) {
4871                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4872                             "BM_%d : Failed to get out of reset."
4873                             "Aborting Crashdump\n");
4874                 goto hba_free;
4875         }
4876
4877         spin_lock_init(&phba->io_sgl_lock);
4878         spin_lock_init(&phba->mgmt_sgl_lock);
4879         spin_lock_init(&phba->isr_lock);
4880         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4881         if (ret != 0) {
4882                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4883                             "BM_%d : Error getting fw config\n");
4884                 goto free_port;
4885         }
4886         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4887         beiscsi_get_params(phba);
4888         phba->shost->can_queue = phba->params.ios_per_ctrl;
4889         ret = beiscsi_init_port(phba);
4890         if (ret < 0) {
4891                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4892                             "BM_%d : beiscsi_dev_probe-"
4893                             "Failed in beiscsi_init_port\n");
4894                 goto free_port;
4895         }
4896
4897         for (i = 0; i < MAX_MCC_CMD ; i++) {
4898                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4899                 phba->ctrl.mcc_tag[i] = i + 1;
4900                 phba->ctrl.mcc_numtag[i + 1] = 0;
4901                 phba->ctrl.mcc_tag_available++;
4902         }
4903
4904         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4905
4906         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
4907                  phba->shost->host_no);
4908         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4909         if (!phba->wq) {
4910                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4911                             "BM_%d : beiscsi_dev_probe-"
4912                             "Failed to allocate work queue\n");
4913                 goto free_twq;
4914         }
4915
4916         INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
4917                           beiscsi_hw_health_check);
4918
4919         phwi_ctrlr = phba->phwi_ctrlr;
4920         phwi_context = phwi_ctrlr->phwi_ctxt;
4921
4922         if (blk_iopoll_enabled) {
4923                 for (i = 0; i < phba->num_cpus; i++) {
4924                         pbe_eq = &phwi_context->be_eq[i];
4925                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4926                                         be_iopoll);
4927                         blk_iopoll_enable(&pbe_eq->iopoll);
4928                 }
4929
4930                 i = (phba->msix_enabled) ? i : 0;
4931                 /* Work item for MCC handling */
4932                 pbe_eq = &phwi_context->be_eq[i];
4933                 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
4934         } else {
4935                 if (phba->msix_enabled) {
4936                         for (i = 0; i <= phba->num_cpus; i++) {
4937                                 pbe_eq = &phwi_context->be_eq[i];
4938                                 INIT_WORK(&pbe_eq->work_cqs,
4939                                           beiscsi_process_all_cqs);
4940                         }
4941                 } else {
4942                                 pbe_eq = &phwi_context->be_eq[0];
4943                                 INIT_WORK(&pbe_eq->work_cqs,
4944                                           beiscsi_process_all_cqs);
4945                         }
4946         }
4947
4948         ret = beiscsi_init_irqs(phba);
4949         if (ret < 0) {
4950                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4951                             "BM_%d : beiscsi_dev_probe-"
4952                             "Failed to beiscsi_init_irqs\n");
4953                 goto free_blkenbld;
4954         }
4955         hwi_enable_intr(phba);
4956
4957         if (beiscsi_setup_boot_info(phba))
4958                 /*
4959                  * log error but continue, because we may not be using
4960                  * iscsi boot.
4961                  */
4962                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4963                             "BM_%d : Could not set up "
4964                             "iSCSI boot info.\n");
4965
4966         beiscsi_create_def_ifaces(phba);
4967         schedule_delayed_work(&phba->beiscsi_hw_check_task,
4968                               msecs_to_jiffies(1000));
4969
4970         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4971                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
4972         return 0;
4973
4974 free_blkenbld:
4975         destroy_workqueue(phba->wq);
4976         if (blk_iopoll_enabled)
4977                 for (i = 0; i < phba->num_cpus; i++) {
4978                         pbe_eq = &phwi_context->be_eq[i];
4979                         blk_iopoll_disable(&pbe_eq->iopoll);
4980                 }
4981 free_twq:
4982         beiscsi_clean_port(phba);
4983         beiscsi_free_mem(phba);
4984 free_port:
4985         pci_free_consistent(phba->pcidev,
4986                             phba->ctrl.mbox_mem_alloced.size,
4987                             phba->ctrl.mbox_mem_alloced.va,
4988                            phba->ctrl.mbox_mem_alloced.dma);
4989         beiscsi_unmap_pci_function(phba);
4990 hba_free:
4991         if (phba->msix_enabled)
4992                 pci_disable_msix(phba->pcidev);
4993         iscsi_host_remove(phba->shost);
4994         pci_dev_put(phba->pcidev);
4995         iscsi_host_free(phba->shost);
4996 disable_pci:
4997         pci_disable_device(pcidev);
4998         return ret;
4999 }
5000
5001 struct iscsi_transport beiscsi_iscsi_transport = {
5002         .owner = THIS_MODULE,
5003         .name = DRV_NAME,
5004         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
5005                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
5006         .create_session = beiscsi_session_create,
5007         .destroy_session = beiscsi_session_destroy,
5008         .create_conn = beiscsi_conn_create,
5009         .bind_conn = beiscsi_conn_bind,
5010         .destroy_conn = iscsi_conn_teardown,
5011         .attr_is_visible = be2iscsi_attr_is_visible,
5012         .set_iface_param = be2iscsi_iface_set_param,
5013         .get_iface_param = be2iscsi_iface_get_param,
5014         .set_param = beiscsi_set_param,
5015         .get_conn_param = iscsi_conn_get_param,
5016         .get_session_param = iscsi_session_get_param,
5017         .get_host_param = beiscsi_get_host_param,
5018         .start_conn = beiscsi_conn_start,
5019         .stop_conn = iscsi_conn_stop,
5020         .send_pdu = iscsi_conn_send_pdu,
5021         .xmit_task = beiscsi_task_xmit,
5022         .cleanup_task = beiscsi_cleanup_task,
5023         .alloc_pdu = beiscsi_alloc_pdu,
5024         .parse_pdu_itt = beiscsi_parse_pdu,
5025         .get_stats = beiscsi_conn_get_stats,
5026         .get_ep_param = beiscsi_ep_get_param,
5027         .ep_connect = beiscsi_ep_connect,
5028         .ep_poll = beiscsi_ep_poll,
5029         .ep_disconnect = beiscsi_ep_disconnect,
5030         .session_recovery_timedout = iscsi_session_recovery_timedout,
5031         .bsg_request = beiscsi_bsg_request,
5032 };
5033
5034 static struct pci_driver beiscsi_pci_driver = {
5035         .name = DRV_NAME,
5036         .probe = beiscsi_dev_probe,
5037         .remove = beiscsi_remove,
5038         .shutdown = beiscsi_shutdown,
5039         .id_table = beiscsi_pci_id_table
5040 };
5041
5042
5043 static int __init beiscsi_module_init(void)
5044 {
5045         int ret;
5046
5047         beiscsi_scsi_transport =
5048                         iscsi_register_transport(&beiscsi_iscsi_transport);
5049         if (!beiscsi_scsi_transport) {
5050                 printk(KERN_ERR
5051                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5052                 return -ENOMEM;
5053         }
5054         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5055                &beiscsi_iscsi_transport);
5056
5057         ret = pci_register_driver(&beiscsi_pci_driver);
5058         if (ret) {
5059                 printk(KERN_ERR
5060                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5061                 goto unregister_iscsi_transport;
5062         }
5063         return 0;
5064
5065 unregister_iscsi_transport:
5066         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5067         return ret;
5068 }
5069
5070 static void __exit beiscsi_module_exit(void)
5071 {
5072         pci_unregister_driver(&beiscsi_pci_driver);
5073         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5074 }
5075
5076 module_init(beiscsi_module_init);
5077 module_exit(beiscsi_module_exit);