drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43 #include <rdma/ib_cache.h>
  44
  45 #include <linux/atomic.h>
  46
  47 #include <scsi/scsi.h>
  48 #include <scsi/scsi_device.h>
  49 #include <scsi/scsi_dbg.h>
  50 #include <scsi/scsi_tcq.h>
  51 #include <scsi/srp.h>
  52 #include <scsi/scsi_transport_srp.h>
  53
  54 #include "ib_srp.h"
  55
  56 #define DRV_NAME        "ib_srp"
  57 #define PFX             DRV_NAME ": "
  58 #define DRV_VERSION     "2.0"
  59 #define DRV_RELDATE     "July 26, 2015"
  60
  61 MODULE_AUTHOR("Roland Dreier");
  62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64 MODULE_VERSION(DRV_VERSION);
  65 MODULE_INFO(release_date, DRV_RELDATE);
  66
  67 static unsigned int srp_sg_tablesize;
  68 static unsigned int cmd_sg_entries;
  69 static unsigned int indirect_sg_entries;
  70 static bool allow_ext_sg;
  71 static bool prefer_fr = true;
  72 static bool register_always = true;
  73 static bool never_register;
  74 static int topspin_workarounds = 1;
  75
  76 module_param(srp_sg_tablesize, uint, 0444);
  77 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  78
  79 module_param(cmd_sg_entries, uint, 0444);
  80 MODULE_PARM_DESC(cmd_sg_entries,
  81                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  82
  83 module_param(indirect_sg_entries, uint, 0444);
  84 MODULE_PARM_DESC(indirect_sg_entries,
  85                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
  86
  87 module_param(allow_ext_sg, bool, 0444);
  88 MODULE_PARM_DESC(allow_ext_sg,
  89                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  90
  91 module_param(topspin_workarounds, int, 0444);
  92 MODULE_PARM_DESC(topspin_workarounds,
  93                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  94
  95 module_param(prefer_fr, bool, 0444);
  96 MODULE_PARM_DESC(prefer_fr,
  97 "Whether to use fast registration if both FMR and fast registration are supported");
  98
  99 module_param(register_always, bool, 0444);
 100 MODULE_PARM_DESC(register_always,
 101                  "Use memory registration even for contiguous memory regions");
 102
 103 module_param(never_register, bool, 0444);
 104 MODULE_PARM_DESC(never_register, "Never register memory");
 105
 106 static const struct kernel_param_ops srp_tmo_ops;
 107
 108 static int srp_reconnect_delay = 10;
 109 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 110                 S_IRUGO | S_IWUSR);
 111 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 112
 113 static int srp_fast_io_fail_tmo = 15;
 114 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 115                 S_IRUGO | S_IWUSR);
 116 MODULE_PARM_DESC(fast_io_fail_tmo,
 117                  "Number of seconds between the observation of a transport"
 118                  " layer error and failing all I/O. \"off\" means that this"
 119                  " functionality is disabled.");
 120
 121 static int srp_dev_loss_tmo = 600;
 122 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 123                 S_IRUGO | S_IWUSR);
 124 MODULE_PARM_DESC(dev_loss_tmo,
 125                  "Maximum number of seconds that the SRP transport should"
 126                  " insulate transport layer errors. After this time has been"
 127                  " exceeded the SCSI host is removed. Should be"
 128                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 129                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 130                  " this functionality is disabled.");
 131
 132 static unsigned ch_count;
 133 module_param(ch_count, uint, 0444);
 134 MODULE_PARM_DESC(ch_count,
 135                  "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
 136
 137 static void srp_add_one(struct ib_device *device);
 138 static void srp_remove_one(struct ib_device *device, void *client_data);
 139 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 140 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 141                 const char *opname);
 142 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 143
 144 static struct scsi_transport_template *ib_srp_transport_template;
 145 static struct workqueue_struct *srp_remove_wq;
 146
 147 static struct ib_client srp_client = {
 148         .name   = "srp",
 149         .add    = srp_add_one,
 150         .remove = srp_remove_one
 151 };
 152
 153 static struct ib_sa_client srp_sa_client;
 154
 155 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 156 {
 157         int tmo = *(int *)kp->arg;
 158
 159         if (tmo >= 0)
 160                 return sprintf(buffer, "%d", tmo);
 161         else
 162                 return sprintf(buffer, "off");
 163 }
 164
 165 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 166 {
 167         int tmo, res;
 168
 169         res = srp_parse_tmo(&tmo, val);
 170         if (res)
 171                 goto out;
 172
 173         if (kp->arg == &srp_reconnect_delay)
 174                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 175                                     srp_dev_loss_tmo);
 176         else if (kp->arg == &srp_fast_io_fail_tmo)
 177                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 178         else
 179                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 180                                     tmo);
 181         if (res)
 182                 goto out;
 183         *(int *)kp->arg = tmo;
 184
 185 out:
 186         return res;
 187 }
 188
 189 static const struct kernel_param_ops srp_tmo_ops = {
 190         .get = srp_tmo_get,
 191         .set = srp_tmo_set,
 192 };
 193
 194 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 195 {
 196         return (struct srp_target_port *) host->hostdata;
 197 }
 198
 199 static const char *srp_target_info(struct Scsi_Host *host)
 200 {
 201         return host_to_target(host)->target_name;
 202 }
 203
 204 static int srp_target_is_topspin(struct srp_target_port *target)
 205 {
 206         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 207         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 208
 209         return topspin_workarounds &&
 210                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 211                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 212 }
 213
 214 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 215                                    gfp_t gfp_mask,
 216                                    enum dma_data_direction direction)
 217 {
 218         struct srp_iu *iu;
 219
 220         iu = kmalloc(sizeof *iu, gfp_mask);
 221         if (!iu)
 222                 goto out;
 223
 224         iu->buf = kzalloc(size, gfp_mask);
 225         if (!iu->buf)
 226                 goto out_free_iu;
 227
 228         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 229                                     direction);
 230         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 231                 goto out_free_buf;
 232
 233         iu->size      = size;
 234         iu->direction = direction;
 235
 236         return iu;
 237
 238 out_free_buf:
 239         kfree(iu->buf);
 240 out_free_iu:
 241         kfree(iu);
 242 out:
 243         return NULL;
 244 }
 245
 246 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 247 {
 248         if (!iu)
 249                 return;
 250
 251         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 252                             iu->direction);
 253         kfree(iu->buf);
 254         kfree(iu);
 255 }
 256
 257 static void srp_qp_event(struct ib_event *event, void *context)
 258 {
 259         pr_debug("QP event %s (%d)\n",
 260                  ib_event_msg(event->event), event->event);
 261 }
 262
 263 static int srp_init_qp(struct srp_target_port *target,
 264                        struct ib_qp *qp)
 265 {
 266         struct ib_qp_attr *attr;
 267         int ret;
 268
 269         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 270         if (!attr)
 271                 return -ENOMEM;
 272
 273         ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 274                                   target->srp_host->port,
 275                                   be16_to_cpu(target->pkey),
 276                                   &attr->pkey_index);
 277         if (ret)
 278                 goto out;
 279
 280         attr->qp_state        = IB_QPS_INIT;
 281         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 282                                     IB_ACCESS_REMOTE_WRITE);
 283         attr->port_num        = target->srp_host->port;
 284
 285         ret = ib_modify_qp(qp, attr,
 286                            IB_QP_STATE          |
 287                            IB_QP_PKEY_INDEX     |
 288                            IB_QP_ACCESS_FLAGS   |
 289                            IB_QP_PORT);
 290
 291 out:
 292         kfree(attr);
 293         return ret;
 294 }
 295
 296 static int srp_new_cm_id(struct srp_rdma_ch *ch)
 297 {
 298         struct srp_target_port *target = ch->target;
 299         struct ib_cm_id *new_cm_id;
 300
 301         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 302                                     srp_cm_handler, ch);
 303         if (IS_ERR(new_cm_id))
 304                 return PTR_ERR(new_cm_id);
 305
 306         if (ch->cm_id)
 307                 ib_destroy_cm_id(ch->cm_id);
 308         ch->cm_id = new_cm_id;
 309         ch->path.sgid = target->sgid;
 310         ch->path.dgid = target->orig_dgid;
 311         ch->path.pkey = target->pkey;
 312         ch->path.service_id = target->service_id;
 313
 314         return 0;
 315 }
 316
 317 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 318 {
 319         struct srp_device *dev = target->srp_host->srp_dev;
 320         struct ib_fmr_pool_param fmr_param;
 321
 322         memset(&fmr_param, 0, sizeof(fmr_param));
 323         fmr_param.pool_size         = target->mr_pool_size;
 324         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 325         fmr_param.cache             = 1;
 326         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 327         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 328         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 329                                        IB_ACCESS_REMOTE_WRITE |
 330                                        IB_ACCESS_REMOTE_READ);
 331
 332         return ib_create_fmr_pool(dev->pd, &fmr_param);
 333 }
 334
 335 /**
 336  * srp_destroy_fr_pool() - free the resources owned by a pool
 337  * @pool: Fast registration pool to be destroyed.
 338  */
 339 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 340 {
 341         int i;
 342         struct srp_fr_desc *d;
 343
 344         if (!pool)
 345                 return;
 346
 347         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 348                 if (d->mr)
 349                         ib_dereg_mr(d->mr);
 350         }
 351         kfree(pool);
 352 }
 353
 354 /**
 355  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 356  * @device:            IB device to allocate fast registration descriptors for.
 357  * @pd:                Protection domain associated with the FR descriptors.
 358  * @pool_size:         Number of descriptors to allocate.
 359  * @max_page_list_len: Maximum fast registration work request page list length.
 360  */
 361 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 362                                               struct ib_pd *pd, int pool_size,
 363                                               int max_page_list_len)
 364 {
 365         struct srp_fr_pool *pool;
 366         struct srp_fr_desc *d;
 367         struct ib_mr *mr;
 368         int i, ret = -EINVAL;
 369
 370         if (pool_size <= 0)
 371                 goto err;
 372         ret = -ENOMEM;
 373         pool = kzalloc(sizeof(struct srp_fr_pool) +
 374                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 375         if (!pool)
 376                 goto err;
 377         pool->size = pool_size;
 378         pool->max_page_list_len = max_page_list_len;
 379         spin_lock_init(&pool->lock);
 380         INIT_LIST_HEAD(&pool->free_list);
 381
 382         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 383                 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
 384                                  max_page_list_len);
 385                 if (IS_ERR(mr)) {
 386                         ret = PTR_ERR(mr);
 387                         goto destroy_pool;
 388                 }
 389                 d->mr = mr;
 390                 list_add_tail(&d->entry, &pool->free_list);
 391         }
 392
 393 out:
 394         return pool;
 395
 396 destroy_pool:
 397         srp_destroy_fr_pool(pool);
 398
 399 err:
 400         pool = ERR_PTR(ret);
 401         goto out;
 402 }
 403
 404 /**
 405  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 406  * @pool: Pool to obtain descriptor from.
 407  */
 408 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 409 {
 410         struct srp_fr_desc *d = NULL;
 411         unsigned long flags;
 412
 413         spin_lock_irqsave(&pool->lock, flags);
 414         if (!list_empty(&pool->free_list)) {
 415                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 416                 list_del(&d->entry);
 417         }
 418         spin_unlock_irqrestore(&pool->lock, flags);
 419
 420         return d;
 421 }
 422
 423 /**
 424  * srp_fr_pool_put() - put an FR descriptor back in the free list
 425  * @pool: Pool the descriptor was allocated from.
 426  * @desc: Pointer to an array of fast registration descriptor pointers.
 427  * @n:    Number of descriptors to put back.
 428  *
 429  * Note: The caller must already have queued an invalidation request for
 430  * desc->mr->rkey before calling this function.
 431  */
 432 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 433                             int n)
 434 {
 435         unsigned long flags;
 436         int i;
 437
 438         spin_lock_irqsave(&pool->lock, flags);
 439         for (i = 0; i < n; i++)
 440                 list_add(&desc[i]->entry, &pool->free_list);
 441         spin_unlock_irqrestore(&pool->lock, flags);
 442 }
 443
 444 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 445 {
 446         struct srp_device *dev = target->srp_host->srp_dev;
 447
 448         return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
 449                                   dev->max_pages_per_mr);
 450 }
 451
 452 /**
 453  * srp_destroy_qp() - destroy an RDMA queue pair
 454  * @qp: RDMA queue pair.
 455  *
 456  * Drain the qp before destroying it.  This avoids that the receive
 457  * completion handler can access the queue pair while it is
 458  * being destroyed.
 459  */
 460 static void srp_destroy_qp(struct ib_qp *qp)
 461 {
 462         ib_drain_rq(qp);
 463         ib_destroy_qp(qp);
 464 }
 465
 466 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 467 {
 468         struct srp_target_port *target = ch->target;
 469         struct srp_device *dev = target->srp_host->srp_dev;
 470         struct ib_qp_init_attr *init_attr;
 471         struct ib_cq *recv_cq, *send_cq;
 472         struct ib_qp *qp;
 473         struct ib_fmr_pool *fmr_pool = NULL;
 474         struct srp_fr_pool *fr_pool = NULL;
 475         const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
 476         int ret;
 477
 478         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 479         if (!init_attr)
 480                 return -ENOMEM;
 481
 482         /* queue_size + 1 for ib_drain_rq() */
 483         recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
 484                                 ch->comp_vector, IB_POLL_SOFTIRQ);
 485         if (IS_ERR(recv_cq)) {
 486                 ret = PTR_ERR(recv_cq);
 487                 goto err;
 488         }
 489
 490         send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
 491                                 ch->comp_vector, IB_POLL_DIRECT);
 492         if (IS_ERR(send_cq)) {
 493                 ret = PTR_ERR(send_cq);
 494                 goto err_recv_cq;
 495         }
 496
 497         init_attr->event_handler       = srp_qp_event;
 498         init_attr->cap.max_send_wr     = m * target->queue_size;
 499         init_attr->cap.max_recv_wr     = target->queue_size + 1;
 500         init_attr->cap.max_recv_sge    = 1;
 501         init_attr->cap.max_send_sge    = 1;
 502         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 503         init_attr->qp_type             = IB_QPT_RC;
 504         init_attr->send_cq             = send_cq;
 505         init_attr->recv_cq             = recv_cq;
 506
 507         qp = ib_create_qp(dev->pd, init_attr);
 508         if (IS_ERR(qp)) {
 509                 ret = PTR_ERR(qp);
 510                 goto err_send_cq;
 511         }
 512
 513         ret = srp_init_qp(target, qp);
 514         if (ret)
 515                 goto err_qp;
 516
 517         if (dev->use_fast_reg) {
 518                 fr_pool = srp_alloc_fr_pool(target);
 519                 if (IS_ERR(fr_pool)) {
 520                         ret = PTR_ERR(fr_pool);
 521                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 522                                      "FR pool allocation failed (%d)\n", ret);
 523                         goto err_qp;
 524                 }
 525         } else if (dev->use_fmr) {
 526                 fmr_pool = srp_alloc_fmr_pool(target);
 527                 if (IS_ERR(fmr_pool)) {
 528                         ret = PTR_ERR(fmr_pool);
 529                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 530                                      "FMR pool allocation failed (%d)\n", ret);
 531                         goto err_qp;
 532                 }
 533         }
 534
 535         if (ch->qp)
 536                 srp_destroy_qp(ch->qp);
 537         if (ch->recv_cq)
 538                 ib_free_cq(ch->recv_cq);
 539         if (ch->send_cq)
 540                 ib_free_cq(ch->send_cq);
 541
 542         ch->qp = qp;
 543         ch->recv_cq = recv_cq;
 544         ch->send_cq = send_cq;
 545
 546         if (dev->use_fast_reg) {
 547                 if (ch->fr_pool)
 548                         srp_destroy_fr_pool(ch->fr_pool);
 549                 ch->fr_pool = fr_pool;
 550         } else if (dev->use_fmr) {
 551                 if (ch->fmr_pool)
 552                         ib_destroy_fmr_pool(ch->fmr_pool);
 553                 ch->fmr_pool = fmr_pool;
 554         }
 555
 556         kfree(init_attr);
 557         return 0;
 558
 559 err_qp:
 560         srp_destroy_qp(qp);
 561
 562 err_send_cq:
 563         ib_free_cq(send_cq);
 564
 565 err_recv_cq:
 566         ib_free_cq(recv_cq);
 567
 568 err:
 569         kfree(init_attr);
 570         return ret;
 571 }
 572
 573 /*
 574  * Note: this function may be called without srp_alloc_iu_bufs() having been
 575  * invoked. Hence the ch->[rt]x_ring checks.
 576  */
 577 static void srp_free_ch_ib(struct srp_target_port *target,
 578                            struct srp_rdma_ch *ch)
 579 {
 580         struct srp_device *dev = target->srp_host->srp_dev;
 581         int i;
 582
 583         if (!ch->target)
 584                 return;
 585
 586         if (ch->cm_id) {
 587                 ib_destroy_cm_id(ch->cm_id);
 588                 ch->cm_id = NULL;
 589         }
 590
 591         /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
 592         if (!ch->qp)
 593                 return;
 594
 595         if (dev->use_fast_reg) {
 596                 if (ch->fr_pool)
 597                         srp_destroy_fr_pool(ch->fr_pool);
 598         } else if (dev->use_fmr) {
 599                 if (ch->fmr_pool)
 600                         ib_destroy_fmr_pool(ch->fmr_pool);
 601         }
 602
 603         srp_destroy_qp(ch->qp);
 604         ib_free_cq(ch->send_cq);
 605         ib_free_cq(ch->recv_cq);
 606
 607         /*
 608          * Avoid that the SCSI error handler tries to use this channel after
 609          * it has been freed. The SCSI error handler can namely continue
 610          * trying to perform recovery actions after scsi_remove_host()
 611          * returned.
 612          */
 613         ch->target = NULL;
 614
 615         ch->qp = NULL;
 616         ch->send_cq = ch->recv_cq = NULL;
 617
 618         if (ch->rx_ring) {
 619                 for (i = 0; i < target->queue_size; ++i)
 620                         srp_free_iu(target->srp_host, ch->rx_ring[i]);
 621                 kfree(ch->rx_ring);
 622                 ch->rx_ring = NULL;
 623         }
 624         if (ch->tx_ring) {
 625                 for (i = 0; i < target->queue_size; ++i)
 626                         srp_free_iu(target->srp_host, ch->tx_ring[i]);
 627                 kfree(ch->tx_ring);
 628                 ch->tx_ring = NULL;
 629         }
 630 }
 631
 632 static void srp_path_rec_completion(int status,
 633                                     struct ib_sa_path_rec *pathrec,
 634                                     void *ch_ptr)
 635 {
 636         struct srp_rdma_ch *ch = ch_ptr;
 637         struct srp_target_port *target = ch->target;
 638
 639         ch->status = status;
 640         if (status)
 641                 shost_printk(KERN_ERR, target->scsi_host,
 642                              PFX "Got failed path rec status %d\n", status);
 643         else
 644                 ch->path = *pathrec;
 645         complete(&ch->done);
 646 }
 647
 648 static int srp_lookup_path(struct srp_rdma_ch *ch)
 649 {
 650         struct srp_target_port *target = ch->target;
 651         int ret;
 652
 653         ch->path.numb_path = 1;
 654
 655         init_completion(&ch->done);
 656
 657         ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 658                                                target->srp_host->srp_dev->dev,
 659                                                target->srp_host->port,
 660                                                &ch->path,
 661                                                IB_SA_PATH_REC_SERVICE_ID |
 662                                                IB_SA_PATH_REC_DGID       |
 663                                                IB_SA_PATH_REC_SGID       |
 664                                                IB_SA_PATH_REC_NUMB_PATH  |
 665                                                IB_SA_PATH_REC_PKEY,
 666                                                SRP_PATH_REC_TIMEOUT_MS,
 667                                                GFP_KERNEL,
 668                                                srp_path_rec_completion,
 669                                                ch, &ch->path_query);
 670         if (ch->path_query_id < 0)
 671                 return ch->path_query_id;
 672
 673         ret = wait_for_completion_interruptible(&ch->done);
 674         if (ret < 0)
 675                 return ret;
 676
 677         if (ch->status < 0)
 678                 shost_printk(KERN_WARNING, target->scsi_host,
 679                              PFX "Path record query failed\n");
 680
 681         return ch->status;
 682 }
 683
 684 static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 685 {
 686         struct srp_target_port *target = ch->target;
 687         struct {
 688                 struct ib_cm_req_param param;
 689                 struct srp_login_req   priv;
 690         } *req = NULL;
 691         int status;
 692
 693         req = kzalloc(sizeof *req, GFP_KERNEL);
 694         if (!req)
 695                 return -ENOMEM;
 696
 697         req->param.primary_path               = &ch->path;
 698         req->param.alternate_path             = NULL;
 699         req->param.service_id                 = target->service_id;
 700         req->param.qp_num                     = ch->qp->qp_num;
 701         req->param.qp_type                    = ch->qp->qp_type;
 702         req->param.private_data               = &req->priv;
 703         req->param.private_data_len           = sizeof req->priv;
 704         req->param.flow_control               = 1;
 705
 706         get_random_bytes(&req->param.starting_psn, 4);
 707         req->param.starting_psn              &= 0xffffff;
 708
 709         /*
 710          * Pick some arbitrary defaults here; we could make these
 711          * module parameters if anyone cared about setting them.
 712          */
 713         req->param.responder_resources        = 4;
 714         req->param.remote_cm_response_timeout = 20;
 715         req->param.local_cm_response_timeout  = 20;
 716         req->param.retry_count                = target->tl_retry_count;
 717         req->param.rnr_retry_count            = 7;
 718         req->param.max_cm_retries             = 15;
 719
 720         req->priv.opcode        = SRP_LOGIN_REQ;
 721         req->priv.tag           = 0;
 722         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 723         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 724                                               SRP_BUF_FORMAT_INDIRECT);
 725         req->priv.req_flags     = (multich ? SRP_MULTICHAN_MULTI :
 726                                    SRP_MULTICHAN_SINGLE);
 727         /*
 728          * In the published SRP specification (draft rev. 16a), the
 729          * port identifier format is 8 bytes of ID extension followed
 730          * by 8 bytes of GUID.  Older drafts put the two halves in the
 731          * opposite order, so that the GUID comes first.
 732          *
 733          * Targets conforming to these obsolete drafts can be
 734          * recognized by the I/O Class they report.
 735          */
 736         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 737                 memcpy(req->priv.initiator_port_id,
 738                        &target->sgid.global.interface_id, 8);
 739                 memcpy(req->priv.initiator_port_id + 8,
 740                        &target->initiator_ext, 8);
 741                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 742                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 743         } else {
 744                 memcpy(req->priv.initiator_port_id,
 745                        &target->initiator_ext, 8);
 746                 memcpy(req->priv.initiator_port_id + 8,
 747                        &target->sgid.global.interface_id, 8);
 748                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 749                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 750         }
 751
 752         /*
 753          * Topspin/Cisco SRP targets will reject our login unless we
 754          * zero out the first 8 bytes of our initiator port ID and set
 755          * the second 8 bytes to the local node GUID.
 756          */
 757         if (srp_target_is_topspin(target)) {
 758                 shost_printk(KERN_DEBUG, target->scsi_host,
 759                              PFX "Topspin/Cisco initiator port ID workaround "
 760                              "activated for target GUID %016llx\n",
 761                              be64_to_cpu(target->ioc_guid));
 762                 memset(req->priv.initiator_port_id, 0, 8);
 763                 memcpy(req->priv.initiator_port_id + 8,
 764                        &target->srp_host->srp_dev->dev->node_guid, 8);
 765         }
 766
 767         status = ib_send_cm_req(ch->cm_id, &req->param);
 768
 769         kfree(req);
 770
 771         return status;
 772 }
 773
 774 static bool srp_queue_remove_work(struct srp_target_port *target)
 775 {
 776         bool changed = false;
 777
 778         spin_lock_irq(&target->lock);
 779         if (target->state != SRP_TARGET_REMOVED) {
 780                 target->state = SRP_TARGET_REMOVED;
 781                 changed = true;
 782         }
 783         spin_unlock_irq(&target->lock);
 784
 785         if (changed)
 786                 queue_work(srp_remove_wq, &target->remove_work);
 787
 788         return changed;
 789 }
 790
 791 static void srp_disconnect_target(struct srp_target_port *target)
 792 {
 793         struct srp_rdma_ch *ch;
 794         int i;
 795
 796         /* XXX should send SRP_I_LOGOUT request */
 797
 798         for (i = 0; i < target->ch_count; i++) {
 799                 ch = &target->ch[i];
 800                 ch->connected = false;
 801                 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
 802                         shost_printk(KERN_DEBUG, target->scsi_host,
 803                                      PFX "Sending CM DREQ failed\n");
 804                 }
 805         }
 806 }
 807
 808 static void srp_free_req_data(struct srp_target_port *target,
 809                               struct srp_rdma_ch *ch)
 810 {
 811         struct srp_device *dev = target->srp_host->srp_dev;
 812         struct ib_device *ibdev = dev->dev;
 813         struct srp_request *req;
 814         int i;
 815
 816         if (!ch->req_ring)
 817                 return;
 818
 819         for (i = 0; i < target->req_ring_size; ++i) {
 820                 req = &ch->req_ring[i];
 821                 if (dev->use_fast_reg) {
 822                         kfree(req->fr_list);
 823                 } else {
 824                         kfree(req->fmr_list);
 825                         kfree(req->map_page);
 826                 }
 827                 if (req->indirect_dma_addr) {
 828                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 829                                             target->indirect_size,
 830                                             DMA_TO_DEVICE);
 831                 }
 832                 kfree(req->indirect_desc);
 833         }
 834
 835         kfree(ch->req_ring);
 836         ch->req_ring = NULL;
 837 }
 838
 839 static int srp_alloc_req_data(struct srp_rdma_ch *ch)
 840 {
 841         struct srp_target_port *target = ch->target;
 842         struct srp_device *srp_dev = target->srp_host->srp_dev;
 843         struct ib_device *ibdev = srp_dev->dev;
 844         struct srp_request *req;
 845         void *mr_list;
 846         dma_addr_t dma_addr;
 847         int i, ret = -ENOMEM;
 848
 849         ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring),
 850                                GFP_KERNEL);
 851         if (!ch->req_ring)
 852                 goto out;
 853
 854         for (i = 0; i < target->req_ring_size; ++i) {
 855                 req = &ch->req_ring[i];
 856                 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *),
 857                                   GFP_KERNEL);
 858                 if (!mr_list)
 859                         goto out;
 860                 if (srp_dev->use_fast_reg) {
 861                         req->fr_list = mr_list;
 862                 } else {
 863                         req->fmr_list = mr_list;
 864                         req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 865                                                 sizeof(void *), GFP_KERNEL);
 866                         if (!req->map_page)
 867                                 goto out;
 868                 }
 869                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 870                 if (!req->indirect_desc)
 871                         goto out;
 872
 873                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 874                                              target->indirect_size,
 875                                              DMA_TO_DEVICE);
 876                 if (ib_dma_mapping_error(ibdev, dma_addr))
 877                         goto out;
 878
 879                 req->indirect_dma_addr = dma_addr;
 880         }
 881         ret = 0;
 882
 883 out:
 884         return ret;
 885 }
 886
 887 /**
 888  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 889  * @shost: SCSI host whose attributes to remove from sysfs.
 890  *
 891  * Note: Any attributes defined in the host template and that did not exist
 892  * before invocation of this function will be ignored.
 893  */
 894 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 895 {
 896         struct device_attribute **attr;
 897
 898         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 899                 device_remove_file(&shost->shost_dev, *attr);
 900 }
 901
 902 static void srp_remove_target(struct srp_target_port *target)
 903 {
 904         struct srp_rdma_ch *ch;
 905         int i;
 906
 907         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 908
 909         srp_del_scsi_host_attr(target->scsi_host);
 910         srp_rport_get(target->rport);
 911         srp_remove_host(target->scsi_host);
 912         scsi_remove_host(target->scsi_host);
 913         srp_stop_rport_timers(target->rport);
 914         srp_disconnect_target(target);
 915         for (i = 0; i < target->ch_count; i++) {
 916                 ch = &target->ch[i];
 917                 srp_free_ch_ib(target, ch);
 918         }
 919         cancel_work_sync(&target->tl_err_work);
 920         srp_rport_put(target->rport);
 921         for (i = 0; i < target->ch_count; i++) {
 922                 ch = &target->ch[i];
 923                 srp_free_req_data(target, ch);
 924         }
 925         kfree(target->ch);
 926         target->ch = NULL;
 927
 928         spin_lock(&target->srp_host->target_lock);
 929         list_del(&target->list);
 930         spin_unlock(&target->srp_host->target_lock);
 931
 932         scsi_host_put(target->scsi_host);
 933 }
 934
 935 static void srp_remove_work(struct work_struct *work)
 936 {
 937         struct srp_target_port *target =
 938                 container_of(work, struct srp_target_port, remove_work);
 939
 940         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 941
 942         srp_remove_target(target);
 943 }
 944
 945 static void srp_rport_delete(struct srp_rport *rport)
 946 {
 947         struct srp_target_port *target = rport->lld_data;
 948
 949         srp_queue_remove_work(target);
 950 }
 951
 952 /**
 953  * srp_connected_ch() - number of connected channels
 954  * @target: SRP target port.
 955  */
 956 static int srp_connected_ch(struct srp_target_port *target)
 957 {
 958         int i, c = 0;
 959
 960         for (i = 0; i < target->ch_count; i++)
 961                 c += target->ch[i].connected;
 962
 963         return c;
 964 }
 965
 966 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
 967 {
 968         struct srp_target_port *target = ch->target;
 969         int ret;
 970
 971         WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
 972
 973         ret = srp_lookup_path(ch);
 974         if (ret)
 975                 goto out;
 976
 977         while (1) {
 978                 init_completion(&ch->done);
 979                 ret = srp_send_req(ch, multich);
 980                 if (ret)
 981                         goto out;
 982                 ret = wait_for_completion_interruptible(&ch->done);
 983                 if (ret < 0)
 984                         goto out;
 985
 986                 /*
 987                  * The CM event handling code will set status to
 988                  * SRP_PORT_REDIRECT if we get a port redirect REJ
 989                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
 990                  * redirect REJ back.
 991                  */
 992                 ret = ch->status;
 993                 switch (ret) {
 994                 case 0:
 995                         ch->connected = true;
 996                         goto out;
 997
 998                 case SRP_PORT_REDIRECT:
 999                         ret = srp_lookup_path(ch);
1000                         if (ret)
1001                                 goto out;
1002                         break;
1003
1004                 case SRP_DLID_REDIRECT:
1005                         break;
1006
1007                 case SRP_STALE_CONN:
1008                         shost_printk(KERN_ERR, target->scsi_host, PFX
1009                                      "giving up on stale connection\n");
1010                         ret = -ECONNRESET;
1011                         goto out;
1012
1013                 default:
1014                         goto out;
1015                 }
1016         }
1017
1018 out:
1019         return ret <= 0 ? ret : -ENODEV;
1020 }
1021
1022 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1023 {
1024         srp_handle_qp_err(cq, wc, "INV RKEY");
1025 }
1026
1027 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1028                 u32 rkey)
1029 {
1030         struct ib_send_wr *bad_wr;
1031         struct ib_send_wr wr = {
1032                 .opcode             = IB_WR_LOCAL_INV,
1033                 .next               = NULL,
1034                 .num_sge            = 0,
1035                 .send_flags         = 0,
1036                 .ex.invalidate_rkey = rkey,
1037         };
1038
1039         wr.wr_cqe = &req->reg_cqe;
1040         req->reg_cqe.done = srp_inv_rkey_err_done;
1041         return ib_post_send(ch->qp, &wr, &bad_wr);
1042 }
1043
1044 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1045                            struct srp_rdma_ch *ch,
1046                            struct srp_request *req)
1047 {
1048         struct srp_target_port *target = ch->target;
1049         struct srp_device *dev = target->srp_host->srp_dev;
1050         struct ib_device *ibdev = dev->dev;
1051         int i, res;
1052
1053         if (!scsi_sglist(scmnd) ||
1054             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1055              scmnd->sc_data_direction != DMA_FROM_DEVICE))
1056                 return;
1057
1058         if (dev->use_fast_reg) {
1059                 struct srp_fr_desc **pfr;
1060
1061                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1062                         res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1063                         if (res < 0) {
1064                                 shost_printk(KERN_ERR, target->scsi_host, PFX
1065                                   "Queueing INV WR for rkey %#x failed (%d)\n",
1066                                   (*pfr)->mr->rkey, res);
1067                                 queue_work(system_long_wq,
1068                                            &target->tl_err_work);
1069                         }
1070                 }
1071                 if (req->nmdesc)
1072                         srp_fr_pool_put(ch->fr_pool, req->fr_list,
1073                                         req->nmdesc);
1074         } else if (dev->use_fmr) {
1075                 struct ib_pool_fmr **pfmr;
1076
1077                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1078                         ib_fmr_pool_unmap(*pfmr);
1079         }
1080
1081         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1082                         scmnd->sc_data_direction);
1083 }
1084
1085 /**
1086  * srp_claim_req - Take ownership of the scmnd associated with a request.
1087  * @ch: SRP RDMA channel.
1088  * @req: SRP request.
1089  * @sdev: If not NULL, only take ownership for this SCSI device.
1090  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1091  *         ownership of @req->scmnd if it equals @scmnd.
1092  *
1093  * Return value:
1094  * Either NULL or a pointer to the SCSI command the caller became owner of.
1095  */
1096 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1097                                        struct srp_request *req,
1098                                        struct scsi_device *sdev,
1099                                        struct scsi_cmnd *scmnd)
1100 {
1101         unsigned long flags;
1102
1103         spin_lock_irqsave(&ch->lock, flags);
1104         if (req->scmnd &&
1105             (!sdev || req->scmnd->device == sdev) &&
1106             (!scmnd || req->scmnd == scmnd)) {
1107                 scmnd = req->scmnd;
1108                 req->scmnd = NULL;
1109         } else {
1110                 scmnd = NULL;
1111         }
1112         spin_unlock_irqrestore(&ch->lock, flags);
1113
1114         return scmnd;
1115 }
1116
1117 /**
1118  * srp_free_req() - Unmap data and adjust ch->req_lim.
1119  * @ch:     SRP RDMA channel.
1120  * @req:    Request to be freed.
1121  * @scmnd:  SCSI command associated with @req.
1122  * @req_lim_delta: Amount to be added to @target->req_lim.
1123  */
1124 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1125                          struct scsi_cmnd *scmnd, s32 req_lim_delta)
1126 {
1127         unsigned long flags;
1128
1129         srp_unmap_data(scmnd, ch, req);
1130
1131         spin_lock_irqsave(&ch->lock, flags);
1132         ch->req_lim += req_lim_delta;
1133         spin_unlock_irqrestore(&ch->lock, flags);
1134 }
1135
1136 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1137                            struct scsi_device *sdev, int result)
1138 {
1139         struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1140
1141         if (scmnd) {
1142                 srp_free_req(ch, req, scmnd, 0);
1143                 scmnd->result = result;
1144                 scmnd->scsi_done(scmnd);
1145         }
1146 }
1147
1148 static void srp_terminate_io(struct srp_rport *rport)
1149 {
1150         struct srp_target_port *target = rport->lld_data;
1151         struct srp_rdma_ch *ch;
1152         struct Scsi_Host *shost = target->scsi_host;
1153         struct scsi_device *sdev;
1154         int i, j;
1155
1156         /*
1157          * Invoking srp_terminate_io() while srp_queuecommand() is running
1158          * is not safe. Hence the warning statement below.
1159          */
1160         shost_for_each_device(sdev, shost)
1161                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1162
1163         for (i = 0; i < target->ch_count; i++) {
1164                 ch = &target->ch[i];
1165
1166                 for (j = 0; j < target->req_ring_size; ++j) {
1167                         struct srp_request *req = &ch->req_ring[j];
1168
1169                         srp_finish_req(ch, req, NULL,
1170                                        DID_TRANSPORT_FAILFAST << 16);
1171                 }
1172         }
1173 }
1174
1175 /*
1176  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1177  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1178  * srp_reset_device() or srp_reset_host() calls will occur while this function
1179  * is in progress. One way to realize that is not to call this function
1180  * directly but to call srp_reconnect_rport() instead since that last function
1181  * serializes calls of this function via rport->mutex and also blocks
1182  * srp_queuecommand() calls before invoking this function.
1183  */
1184 static int srp_rport_reconnect(struct srp_rport *rport)
1185 {
1186         struct srp_target_port *target = rport->lld_data;
1187         struct srp_rdma_ch *ch;
1188         int i, j, ret = 0;
1189         bool multich = false;
1190
1191         srp_disconnect_target(target);
1192
1193         if (target->state == SRP_TARGET_SCANNING)
1194                 return -ENODEV;
1195
1196         /*
1197          * Now get a new local CM ID so that we avoid confusing the target in
1198          * case things are really fouled up. Doing so also ensures that all CM
1199          * callbacks will have finished before a new QP is allocated.
1200          */
1201         for (i = 0; i < target->ch_count; i++) {
1202                 ch = &target->ch[i];
1203                 ret += srp_new_cm_id(ch);
1204         }
1205         for (i = 0; i < target->ch_count; i++) {
1206                 ch = &target->ch[i];
1207                 for (j = 0; j < target->req_ring_size; ++j) {
1208                         struct srp_request *req = &ch->req_ring[j];
1209
1210                         srp_finish_req(ch, req, NULL, DID_RESET << 16);
1211                 }
1212         }
1213         for (i = 0; i < target->ch_count; i++) {
1214                 ch = &target->ch[i];
1215                 /*
1216                  * Whether or not creating a new CM ID succeeded, create a new
1217                  * QP. This guarantees that all completion callback function
1218                  * invocations have finished before request resetting starts.
1219                  */
1220                 ret += srp_create_ch_ib(ch);
1221
1222                 INIT_LIST_HEAD(&ch->free_tx);
1223                 for (j = 0; j < target->queue_size; ++j)
1224                         list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1225         }
1226
1227         target->qp_in_error = false;
1228
1229         for (i = 0; i < target->ch_count; i++) {
1230                 ch = &target->ch[i];
1231                 if (ret)
1232                         break;
1233                 ret = srp_connect_ch(ch, multich);
1234                 multich = true;
1235         }
1236
1237         if (ret == 0)
1238                 shost_printk(KERN_INFO, target->scsi_host,
1239                              PFX "reconnect succeeded\n");
1240
1241         return ret;
1242 }
1243
1244 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1245                          unsigned int dma_len, u32 rkey)
1246 {
1247         struct srp_direct_buf *desc = state->desc;
1248
1249         WARN_ON_ONCE(!dma_len);
1250
1251         desc->va = cpu_to_be64(dma_addr);
1252         desc->key = cpu_to_be32(rkey);
1253         desc->len = cpu_to_be32(dma_len);
1254
1255         state->total_len += dma_len;
1256         state->desc++;
1257         state->ndesc++;
1258 }
1259
1260 static int srp_map_finish_fmr(struct srp_map_state *state,
1261                               struct srp_rdma_ch *ch)
1262 {
1263         struct srp_target_port *target = ch->target;
1264         struct srp_device *dev = target->srp_host->srp_dev;
1265         struct ib_pool_fmr *fmr;
1266         u64 io_addr = 0;
1267
1268         if (state->fmr.next >= state->fmr.end)
1269                 return -ENOMEM;
1270
1271         WARN_ON_ONCE(!dev->use_fmr);
1272
1273         if (state->npages == 0)
1274                 return 0;
1275
1276         if (state->npages == 1 && target->global_mr) {
1277                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1278                              target->global_mr->rkey);
1279                 goto reset_state;
1280         }
1281
1282         fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages,
1283                                    state->npages, io_addr);
1284         if (IS_ERR(fmr))
1285                 return PTR_ERR(fmr);
1286
1287         *state->fmr.next++ = fmr;
1288         state->nmdesc++;
1289
1290         srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask,
1291                      state->dma_len, fmr->fmr->rkey);
1292
1293 reset_state:
1294         state->npages = 0;
1295         state->dma_len = 0;
1296
1297         return 0;
1298 }
1299
1300 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1301 {
1302         srp_handle_qp_err(cq, wc, "FAST REG");
1303 }
1304
1305 /*
1306  * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1307  * where to start in the first element. If sg_offset_p != NULL then
1308  * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1309  * byte that has not yet been mapped.
1310  */
1311 static int srp_map_finish_fr(struct srp_map_state *state,
1312                              struct srp_request *req,
1313                              struct srp_rdma_ch *ch, int sg_nents,
1314                              unsigned int *sg_offset_p)
1315 {
1316         struct srp_target_port *target = ch->target;
1317         struct srp_device *dev = target->srp_host->srp_dev;
1318         struct ib_send_wr *bad_wr;
1319         struct ib_reg_wr wr;
1320         struct srp_fr_desc *desc;
1321         u32 rkey;
1322         int n, err;
1323
1324         if (state->fr.next >= state->fr.end)
1325                 return -ENOMEM;
1326
1327         WARN_ON_ONCE(!dev->use_fast_reg);
1328
1329         if (sg_nents == 1 && target->global_mr) {
1330                 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1331
1332                 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1333                              sg_dma_len(state->sg) - sg_offset,
1334                              target->global_mr->rkey);
1335                 if (sg_offset_p)
1336                         *sg_offset_p = 0;
1337                 return 1;
1338         }
1339
1340         desc = srp_fr_pool_get(ch->fr_pool);
1341         if (!desc)
1342                 return -ENOMEM;
1343
1344         rkey = ib_inc_rkey(desc->mr->rkey);
1345         ib_update_fast_reg_key(desc->mr, rkey);
1346
1347         n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1348                          dev->mr_page_size);
1349         if (unlikely(n < 0)) {
1350                 srp_fr_pool_put(ch->fr_pool, &desc, 1);
1351                 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1352                          dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1353                          sg_offset_p ? *sg_offset_p : -1, n);
1354                 return n;
1355         }
1356
1357         WARN_ON_ONCE(desc->mr->length == 0);
1358
1359         req->reg_cqe.done = srp_reg_mr_err_done;
1360
1361         wr.wr.next = NULL;
1362         wr.wr.opcode = IB_WR_REG_MR;
1363         wr.wr.wr_cqe = &req->reg_cqe;
1364         wr.wr.num_sge = 0;
1365         wr.wr.send_flags = 0;
1366         wr.mr = desc->mr;
1367         wr.key = desc->mr->rkey;
1368         wr.access = (IB_ACCESS_LOCAL_WRITE |
1369                      IB_ACCESS_REMOTE_READ |
1370                      IB_ACCESS_REMOTE_WRITE);
1371
1372         *state->fr.next++ = desc;
1373         state->nmdesc++;
1374
1375         srp_map_desc(state, desc->mr->iova,
1376                      desc->mr->length, desc->mr->rkey);
1377
1378         err = ib_post_send(ch->qp, &wr.wr, &bad_wr);
1379         if (unlikely(err)) {
1380                 WARN_ON_ONCE(err == -ENOMEM);
1381                 return err;
1382         }
1383
1384         return n;
1385 }
1386
1387 static int srp_map_sg_entry(struct srp_map_state *state,
1388                             struct srp_rdma_ch *ch,
1389                             struct scatterlist *sg, int sg_index)
1390 {
1391         struct srp_target_port *target = ch->target;
1392         struct srp_device *dev = target->srp_host->srp_dev;
1393         struct ib_device *ibdev = dev->dev;
1394         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1395         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1396         unsigned int len = 0;
1397         int ret;
1398
1399         WARN_ON_ONCE(!dma_len);
1400
1401         while (dma_len) {
1402                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1403                 if (state->npages == dev->max_pages_per_mr || offset != 0) {
1404                         ret = srp_map_finish_fmr(state, ch);
1405                         if (ret)
1406                                 return ret;
1407                 }
1408
1409                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1410
1411                 if (!state->npages)
1412                         state->base_dma_addr = dma_addr;
1413                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1414                 state->dma_len += len;
1415                 dma_addr += len;
1416                 dma_len -= len;
1417         }
1418
1419         /*
1420          * If the last entry of the MR wasn't a full page, then we need to
1421          * close it out and start a new one -- we can only merge at page
1422          * boundaries.
1423          */
1424         ret = 0;
1425         if (len != dev->mr_page_size)
1426                 ret = srp_map_finish_fmr(state, ch);
1427         return ret;
1428 }
1429
1430 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1431                           struct srp_request *req, struct scatterlist *scat,
1432                           int count)
1433 {
1434         struct scatterlist *sg;
1435         int i, ret;
1436
1437         state->pages = req->map_page;
1438         state->fmr.next = req->fmr_list;
1439         state->fmr.end = req->fmr_list + ch->target->mr_per_cmd;
1440
1441         for_each_sg(scat, sg, count, i) {
1442                 ret = srp_map_sg_entry(state, ch, sg, i);
1443                 if (ret)
1444                         return ret;
1445         }
1446
1447         ret = srp_map_finish_fmr(state, ch);
1448         if (ret)
1449                 return ret;
1450
1451         return 0;
1452 }
1453
1454 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1455                          struct srp_request *req, struct scatterlist *scat,
1456                          int count)
1457 {
1458         unsigned int sg_offset = 0;
1459
1460         state->desc = req->indirect_desc;
1461         state->fr.next = req->fr_list;
1462         state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1463         state->sg = scat;
1464
1465         if (count == 0)
1466                 return 0;
1467
1468         while (count) {
1469                 int i, n;
1470
1471                 n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1472                 if (unlikely(n < 0))
1473                         return n;
1474
1475                 count -= n;
1476                 for (i = 0; i < n; i++)
1477                         state->sg = sg_next(state->sg);
1478         }
1479
1480         return 0;
1481 }
1482
1483 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1484                           struct srp_request *req, struct scatterlist *scat,
1485                           int count)
1486 {
1487         struct srp_target_port *target = ch->target;
1488         struct srp_device *dev = target->srp_host->srp_dev;
1489         struct scatterlist *sg;
1490         int i;
1491
1492         state->desc = req->indirect_desc;
1493         for_each_sg(scat, sg, count, i) {
1494                 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg),
1495                              ib_sg_dma_len(dev->dev, sg),
1496                              target->global_mr->rkey);
1497         }
1498
1499         return 0;
1500 }
1501
1502 /*
1503  * Register the indirect data buffer descriptor with the HCA.
1504  *
1505  * Note: since the indirect data buffer descriptor has been allocated with
1506  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1507  * memory buffer.
1508  */
1509 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1510                        void **next_mr, void **end_mr, u32 idb_len,
1511                        __be32 *idb_rkey)
1512 {
1513         struct srp_target_port *target = ch->target;
1514         struct srp_device *dev = target->srp_host->srp_dev;
1515         struct srp_map_state state;
1516         struct srp_direct_buf idb_desc;
1517         u64 idb_pages[1];
1518         struct scatterlist idb_sg[1];
1519         int ret;
1520
1521         memset(&state, 0, sizeof(state));
1522         memset(&idb_desc, 0, sizeof(idb_desc));
1523         state.gen.next = next_mr;
1524         state.gen.end = end_mr;
1525         state.desc = &idb_desc;
1526         state.base_dma_addr = req->indirect_dma_addr;
1527         state.dma_len = idb_len;
1528
1529         if (dev->use_fast_reg) {
1530                 state.sg = idb_sg;
1531                 sg_init_one(idb_sg, req->indirect_desc, idb_len);
1532                 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1533 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1534                 idb_sg->dma_length = idb_sg->length;          /* hack^2 */
1535 #endif
1536                 ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1537                 if (ret < 0)
1538                         return ret;
1539                 WARN_ON_ONCE(ret < 1);
1540         } else if (dev->use_fmr) {
1541                 state.pages = idb_pages;
1542                 state.pages[0] = (req->indirect_dma_addr &
1543                                   dev->mr_page_mask);
1544                 state.npages = 1;
1545                 ret = srp_map_finish_fmr(&state, ch);
1546                 if (ret < 0)
1547                         return ret;
1548         } else {
1549                 return -EINVAL;
1550         }
1551
1552         *idb_rkey = idb_desc.key;
1553
1554         return 0;
1555 }
1556
1557 #if defined(DYNAMIC_DATA_DEBUG)
1558 static void srp_check_mapping(struct srp_map_state *state,
1559                               struct srp_rdma_ch *ch, struct srp_request *req,
1560                               struct scatterlist *scat, int count)
1561 {
1562         struct srp_device *dev = ch->target->srp_host->srp_dev;
1563         struct srp_fr_desc **pfr;
1564         u64 desc_len = 0, mr_len = 0;
1565         int i;
1566
1567         for (i = 0; i < state->ndesc; i++)
1568                 desc_len += be32_to_cpu(req->indirect_desc[i].len);
1569         if (dev->use_fast_reg)
1570                 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1571                         mr_len += (*pfr)->mr->length;
1572         else if (dev->use_fmr)
1573                 for (i = 0; i < state->nmdesc; i++)
1574                         mr_len += be32_to_cpu(req->indirect_desc[i].len);
1575         if (desc_len != scsi_bufflen(req->scmnd) ||
1576             mr_len > scsi_bufflen(req->scmnd))
1577                 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1578                        scsi_bufflen(req->scmnd), desc_len, mr_len,
1579                        state->ndesc, state->nmdesc);
1580 }
1581 #endif
1582
1583 /**
1584  * srp_map_data() - map SCSI data buffer onto an SRP request
1585  * @scmnd: SCSI command to map
1586  * @ch: SRP RDMA channel
1587  * @req: SRP request
1588  *
1589  * Returns the length in bytes of the SRP_CMD IU or a negative value if
1590  * mapping failed.
1591  */
1592 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1593                         struct srp_request *req)
1594 {
1595         struct srp_target_port *target = ch->target;
1596         struct scatterlist *scat;
1597         struct srp_cmd *cmd = req->cmd->buf;
1598         int len, nents, count, ret;
1599         struct srp_device *dev;
1600         struct ib_device *ibdev;
1601         struct srp_map_state state;
1602         struct srp_indirect_buf *indirect_hdr;
1603         u32 idb_len, table_len;
1604         __be32 idb_rkey;
1605         u8 fmt;
1606
1607         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1608                 return sizeof (struct srp_cmd);
1609
1610         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1611             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1612                 shost_printk(KERN_WARNING, target->scsi_host,
1613                              PFX "Unhandled data direction %d\n",
1614                              scmnd->sc_data_direction);
1615                 return -EINVAL;
1616         }
1617
1618         nents = scsi_sg_count(scmnd);
1619         scat  = scsi_sglist(scmnd);
1620
1621         dev = target->srp_host->srp_dev;
1622         ibdev = dev->dev;
1623
1624         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1625         if (unlikely(count == 0))
1626                 return -EIO;
1627
1628         fmt = SRP_DATA_DESC_DIRECT;
1629         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1630
1631         if (count == 1 && target->global_mr) {
1632                 /*
1633                  * The midlayer only generated a single gather/scatter
1634                  * entry, or DMA mapping coalesced everything to a
1635                  * single entry.  So a direct descriptor along with
1636                  * the DMA MR suffices.
1637                  */
1638                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1639
1640                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1641                 buf->key = cpu_to_be32(target->global_mr->rkey);
1642                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1643
1644                 req->nmdesc = 0;
1645                 goto map_complete;
1646         }
1647
1648         /*
1649          * We have more than one scatter/gather entry, so build our indirect
1650          * descriptor table, trying to merge as many entries as we can.
1651          */
1652         indirect_hdr = (void *) cmd->add_data;
1653
1654         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1655                                    target->indirect_size, DMA_TO_DEVICE);
1656
1657         memset(&state, 0, sizeof(state));
1658         if (dev->use_fast_reg)
1659                 ret = srp_map_sg_fr(&state, ch, req, scat, count);
1660         else if (dev->use_fmr)
1661                 ret = srp_map_sg_fmr(&state, ch, req, scat, count);
1662         else
1663                 ret = srp_map_sg_dma(&state, ch, req, scat, count);
1664         req->nmdesc = state.nmdesc;
1665         if (ret < 0)
1666                 goto unmap;
1667
1668 #if defined(DYNAMIC_DEBUG)
1669         {
1670                 DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1671                         "Memory mapping consistency check");
1672                 if (unlikely(ddm.flags & _DPRINTK_FLAGS_PRINT))
1673                         srp_check_mapping(&state, ch, req, scat, count);
1674         }
1675 #endif
1676
1677         /* We've mapped the request, now pull as much of the indirect
1678          * descriptor table as we can into the command buffer. If this
1679          * target is not using an external indirect table, we are
1680          * guaranteed to fit into the command, as the SCSI layer won't
1681          * give us more S/G entries than we allow.
1682          */
1683         if (state.ndesc == 1) {
1684                 /*
1685                  * Memory registration collapsed the sg-list into one entry,
1686                  * so use a direct descriptor.
1687                  */
1688                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1689
1690                 *buf = req->indirect_desc[0];
1691                 goto map_complete;
1692         }
1693
1694         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1695                                                 !target->allow_ext_sg)) {
1696                 shost_printk(KERN_ERR, target->scsi_host,
1697                              "Could not fit S/G list into SRP_CMD\n");
1698                 ret = -EIO;
1699                 goto unmap;
1700         }
1701
1702         count = min(state.ndesc, target->cmd_sg_cnt);
1703         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1704         idb_len = sizeof(struct srp_indirect_buf) + table_len;
1705
1706         fmt = SRP_DATA_DESC_INDIRECT;
1707         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1708         len += count * sizeof (struct srp_direct_buf);
1709
1710         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1711                count * sizeof (struct srp_direct_buf));
1712
1713         if (!target->global_mr) {
1714                 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1715                                   idb_len, &idb_rkey);
1716                 if (ret < 0)
1717                         goto unmap;
1718                 req->nmdesc++;
1719         } else {
1720                 idb_rkey = cpu_to_be32(target->global_mr->rkey);
1721         }
1722
1723         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1724         indirect_hdr->table_desc.key = idb_rkey;
1725         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1726         indirect_hdr->len = cpu_to_be32(state.total_len);
1727
1728         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1729                 cmd->data_out_desc_cnt = count;
1730         else
1731                 cmd->data_in_desc_cnt = count;
1732
1733         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1734                                       DMA_TO_DEVICE);
1735
1736 map_complete:
1737         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1738                 cmd->buf_fmt = fmt << 4;
1739         else
1740                 cmd->buf_fmt = fmt;
1741
1742         return len;
1743
1744 unmap:
1745         srp_unmap_data(scmnd, ch, req);
1746         if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1747                 ret = -E2BIG;
1748         return ret;
1749 }
1750
1751 /*
1752  * Return an IU and possible credit to the free pool
1753  */
1754 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1755                           enum srp_iu_type iu_type)
1756 {
1757         unsigned long flags;
1758
1759         spin_lock_irqsave(&ch->lock, flags);
1760         list_add(&iu->list, &ch->free_tx);
1761         if (iu_type != SRP_IU_RSP)
1762                 ++ch->req_lim;
1763         spin_unlock_irqrestore(&ch->lock, flags);
1764 }
1765
1766 /*
1767  * Must be called with ch->lock held to protect req_lim and free_tx.
1768  * If IU is not sent, it must be returned using srp_put_tx_iu().
1769  *
1770  * Note:
1771  * An upper limit for the number of allocated information units for each
1772  * request type is:
1773  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1774  *   more than Scsi_Host.can_queue requests.
1775  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1776  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1777  *   one unanswered SRP request to an initiator.
1778  */
1779 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1780                                       enum srp_iu_type iu_type)
1781 {
1782         struct srp_target_port *target = ch->target;
1783         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1784         struct srp_iu *iu;
1785
1786         ib_process_cq_direct(ch->send_cq, -1);
1787
1788         if (list_empty(&ch->free_tx))
1789                 return NULL;
1790
1791         /* Initiator responses to target requests do not consume credits */
1792         if (iu_type != SRP_IU_RSP) {
1793                 if (ch->req_lim <= rsv) {
1794                         ++target->zero_req_lim;
1795                         return NULL;
1796                 }
1797
1798                 --ch->req_lim;
1799         }
1800
1801         iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1802         list_del(&iu->list);
1803         return iu;
1804 }
1805
1806 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1807 {
1808         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1809         struct srp_rdma_ch *ch = cq->cq_context;
1810
1811         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1812                 srp_handle_qp_err(cq, wc, "SEND");
1813                 return;
1814         }
1815
1816         list_add(&iu->list, &ch->free_tx);
1817 }
1818
1819 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1820 {
1821         struct srp_target_port *target = ch->target;
1822         struct ib_sge list;
1823         struct ib_send_wr wr, *bad_wr;
1824
1825         list.addr   = iu->dma;
1826         list.length = len;
1827         list.lkey   = target->lkey;
1828
1829         iu->cqe.done = srp_send_done;
1830
1831         wr.next       = NULL;
1832         wr.wr_cqe     = &iu->cqe;
1833         wr.sg_list    = &list;
1834         wr.num_sge    = 1;
1835         wr.opcode     = IB_WR_SEND;
1836         wr.send_flags = IB_SEND_SIGNALED;
1837
1838         return ib_post_send(ch->qp, &wr, &bad_wr);
1839 }
1840
1841 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1842 {
1843         struct srp_target_port *target = ch->target;
1844         struct ib_recv_wr wr, *bad_wr;
1845         struct ib_sge list;
1846
1847         list.addr   = iu->dma;
1848         list.length = iu->size;
1849         list.lkey   = target->lkey;
1850
1851         iu->cqe.done = srp_recv_done;
1852
1853         wr.next     = NULL;
1854         wr.wr_cqe   = &iu->cqe;
1855         wr.sg_list  = &list;
1856         wr.num_sge  = 1;
1857
1858         return ib_post_recv(ch->qp, &wr, &bad_wr);
1859 }
1860
1861 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1862 {
1863         struct srp_target_port *target = ch->target;
1864         struct srp_request *req;
1865         struct scsi_cmnd *scmnd;
1866         unsigned long flags;
1867
1868         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1869                 spin_lock_irqsave(&ch->lock, flags);
1870                 ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1871                 spin_unlock_irqrestore(&ch->lock, flags);
1872
1873                 ch->tsk_mgmt_status = -1;
1874                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1875                         ch->tsk_mgmt_status = rsp->data[3];
1876                 complete(&ch->tsk_mgmt_done);
1877         } else {
1878                 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1879                 if (scmnd) {
1880                         req = (void *)scmnd->host_scribble;
1881                         scmnd = srp_claim_req(ch, req, NULL, scmnd);
1882                 }
1883                 if (!scmnd) {
1884                         shost_printk(KERN_ERR, target->scsi_host,
1885                                      "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1886                                      rsp->tag, ch - target->ch, ch->qp->qp_num);
1887
1888                         spin_lock_irqsave(&ch->lock, flags);
1889                         ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1890                         spin_unlock_irqrestore(&ch->lock, flags);
1891
1892                         return;
1893                 }
1894                 scmnd->result = rsp->status;
1895
1896                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1897                         memcpy(scmnd->sense_buffer, rsp->data +
1898                                be32_to_cpu(rsp->resp_data_len),
1899                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1900                                      SCSI_SENSE_BUFFERSIZE));
1901                 }
1902
1903                 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1904                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1905                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER))
1906                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt));
1907                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1908                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1909                 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER))
1910                         scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt));
1911
1912                 srp_free_req(ch, req, scmnd,
1913                              be32_to_cpu(rsp->req_lim_delta));
1914
1915                 scmnd->host_scribble = NULL;
1916                 scmnd->scsi_done(scmnd);
1917         }
1918 }
1919
1920 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1921                                void *rsp, int len)
1922 {
1923         struct srp_target_port *target = ch->target;
1924         struct ib_device *dev = target->srp_host->srp_dev->dev;
1925         unsigned long flags;
1926         struct srp_iu *iu;
1927         int err;
1928
1929         spin_lock_irqsave(&ch->lock, flags);
1930         ch->req_lim += req_delta;
1931         iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
1932         spin_unlock_irqrestore(&ch->lock, flags);
1933
1934         if (!iu) {
1935                 shost_printk(KERN_ERR, target->scsi_host, PFX
1936                              "no IU available to send response\n");
1937                 return 1;
1938         }
1939
1940         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1941         memcpy(iu->buf, rsp, len);
1942         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1943
1944         err = srp_post_send(ch, iu, len);
1945         if (err) {
1946                 shost_printk(KERN_ERR, target->scsi_host, PFX
1947                              "unable to post response: %d\n", err);
1948                 srp_put_tx_iu(ch, iu, SRP_IU_RSP);
1949         }
1950
1951         return err;
1952 }
1953
1954 static void srp_process_cred_req(struct srp_rdma_ch *ch,
1955                                  struct srp_cred_req *req)
1956 {
1957         struct srp_cred_rsp rsp = {
1958                 .opcode = SRP_CRED_RSP,
1959                 .tag = req->tag,
1960         };
1961         s32 delta = be32_to_cpu(req->req_lim_delta);
1962
1963         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1964                 shost_printk(KERN_ERR, ch->target->scsi_host, PFX
1965                              "problems processing SRP_CRED_REQ\n");
1966 }
1967
1968 static void srp_process_aer_req(struct srp_rdma_ch *ch,
1969                                 struct srp_aer_req *req)
1970 {
1971         struct srp_target_port *target = ch->target;
1972         struct srp_aer_rsp rsp = {
1973                 .opcode = SRP_AER_RSP,
1974                 .tag = req->tag,
1975         };
1976         s32 delta = be32_to_cpu(req->req_lim_delta);
1977
1978         shost_printk(KERN_ERR, target->scsi_host, PFX
1979                      "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
1980
1981         if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
1982                 shost_printk(KERN_ERR, target->scsi_host, PFX
1983                              "problems processing SRP_AER_REQ\n");
1984 }
1985
1986 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1987 {
1988         struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1989         struct srp_rdma_ch *ch = cq->cq_context;
1990         struct srp_target_port *target = ch->target;
1991         struct ib_device *dev = target->srp_host->srp_dev->dev;
1992         int res;
1993         u8 opcode;
1994
1995         if (unlikely(wc->status != IB_WC_SUCCESS)) {
1996                 srp_handle_qp_err(cq, wc, "RECV");
1997                 return;
1998         }
1999
2000         ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2001                                    DMA_FROM_DEVICE);
2002
2003         opcode = *(u8 *) iu->buf;
2004
2005         if (0) {
2006                 shost_printk(KERN_ERR, target->scsi_host,
2007                              PFX "recv completion, opcode 0x%02x\n", opcode);
2008                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2009                                iu->buf, wc->byte_len, true);
2010         }
2011
2012         switch (opcode) {
2013         case SRP_RSP:
2014                 srp_process_rsp(ch, iu->buf);
2015                 break;
2016
2017         case SRP_CRED_REQ:
2018                 srp_process_cred_req(ch, iu->buf);
2019                 break;
2020
2021         case SRP_AER_REQ:
2022                 srp_process_aer_req(ch, iu->buf);
2023                 break;
2024
2025         case SRP_T_LOGOUT:
2026                 /* XXX Handle target logout */
2027                 shost_printk(KERN_WARNING, target->scsi_host,
2028                              PFX "Got target logout request\n");
2029                 break;
2030
2031         default:
2032                 shost_printk(KERN_WARNING, target->scsi_host,
2033                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2034                 break;
2035         }
2036
2037         ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2038                                       DMA_FROM_DEVICE);
2039
2040         res = srp_post_recv(ch, iu);
2041         if (res != 0)
2042                 shost_printk(KERN_ERR, target->scsi_host,
2043                              PFX "Recv failed with error code %d\n", res);
2044 }
2045
2046 /**
2047  * srp_tl_err_work() - handle a transport layer error
2048  * @work: Work structure embedded in an SRP target port.
2049  *
2050  * Note: This function may get invoked before the rport has been created,
2051  * hence the target->rport test.
2052  */
2053 static void srp_tl_err_work(struct work_struct *work)
2054 {
2055         struct srp_target_port *target;
2056
2057         target = container_of(work, struct srp_target_port, tl_err_work);
2058         if (target->rport)
2059                 srp_start_tl_fail_timers(target->rport);
2060 }
2061
2062 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2063                 const char *opname)
2064 {
2065         struct srp_rdma_ch *ch = cq->cq_context;
2066         struct srp_target_port *target = ch->target;
2067
2068         if (ch->connected && !target->qp_in_error) {
2069                 shost_printk(KERN_ERR, target->scsi_host,
2070                              PFX "failed %s status %s (%d) for CQE %p\n",
2071                              opname, ib_wc_status_msg(wc->status), wc->status,
2072                              wc->wr_cqe);
2073                 queue_work(system_long_wq, &target->tl_err_work);
2074         }
2075         target->qp_in_error = true;
2076 }
2077
2078 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2079 {
2080         struct srp_target_port *target = host_to_target(shost);
2081         struct srp_rport *rport = target->rport;
2082         struct srp_rdma_ch *ch;
2083         struct srp_request *req;
2084         struct srp_iu *iu;
2085         struct srp_cmd *cmd;
2086         struct ib_device *dev;
2087         unsigned long flags;
2088         u32 tag;
2089         u16 idx;
2090         int len, ret;
2091         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
2092
2093         /*
2094          * The SCSI EH thread is the only context from which srp_queuecommand()
2095          * can get invoked for blocked devices (SDEV_BLOCK /
2096          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
2097          * locking the rport mutex if invoked from inside the SCSI EH.
2098          */
2099         if (in_scsi_eh)
2100                 mutex_lock(&rport->mutex);
2101
2102         scmnd->result = srp_chkready(target->rport);
2103         if (unlikely(scmnd->result))
2104                 goto err;
2105
2106         WARN_ON_ONCE(scmnd->request->tag < 0);
2107         tag = blk_mq_unique_tag(scmnd->request);
2108         ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2109         idx = blk_mq_unique_tag_to_tag(tag);
2110         WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n",
2111                   dev_name(&shost->shost_gendev), tag, idx,
2112                   target->req_ring_size);
2113
2114         spin_lock_irqsave(&ch->lock, flags);
2115         iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2116         spin_unlock_irqrestore(&ch->lock, flags);
2117
2118         if (!iu)
2119                 goto err;
2120
2121         req = &ch->req_ring[idx];
2122         dev = target->srp_host->srp_dev->dev;
2123         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
2124                                    DMA_TO_DEVICE);
2125
2126         scmnd->host_scribble = (void *) req;
2127
2128         cmd = iu->buf;
2129         memset(cmd, 0, sizeof *cmd);
2130
2131         cmd->opcode = SRP_CMD;
2132         int_to_scsilun(scmnd->device->lun, &cmd->lun);
2133         cmd->tag    = tag;
2134         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2135
2136         req->scmnd    = scmnd;
2137         req->cmd      = iu;
2138
2139         len = srp_map_data(scmnd, ch, req);
2140         if (len < 0) {
2141                 shost_printk(KERN_ERR, target->scsi_host,
2142                              PFX "Failed to map data (%d)\n", len);
2143                 /*
2144                  * If we ran out of memory descriptors (-ENOMEM) because an
2145                  * application is queuing many requests with more than
2146                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2147                  * to reduce queue depth temporarily.
2148                  */
2149                 scmnd->result = len == -ENOMEM ?
2150                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
2151                 goto err_iu;
2152         }
2153
2154         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
2155                                       DMA_TO_DEVICE);
2156
2157         if (srp_post_send(ch, iu, len)) {
2158                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2159                 goto err_unmap;
2160         }
2161
2162         ret = 0;
2163
2164 unlock_rport:
2165         if (in_scsi_eh)
2166                 mutex_unlock(&rport->mutex);
2167
2168         return ret;
2169
2170 err_unmap:
2171         srp_unmap_data(scmnd, ch, req);
2172
2173 err_iu:
2174         srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2175
2176         /*
2177          * Avoid that the loops that iterate over the request ring can
2178          * encounter a dangling SCSI command pointer.
2179          */
2180         req->scmnd = NULL;
2181
2182 err:
2183         if (scmnd->result) {
2184                 scmnd->scsi_done(scmnd);
2185                 ret = 0;
2186         } else {
2187                 ret = SCSI_MLQUEUE_HOST_BUSY;
2188         }
2189
2190         goto unlock_rport;
2191 }
2192
2193 /*
2194  * Note: the resources allocated in this function are freed in
2195  * srp_free_ch_ib().
2196  */
2197 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2198 {
2199         struct srp_target_port *target = ch->target;
2200         int i;
2201
2202         ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2203                               GFP_KERNEL);
2204         if (!ch->rx_ring)
2205                 goto err_no_ring;
2206         ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2207                               GFP_KERNEL);
2208         if (!ch->tx_ring)
2209                 goto err_no_ring;
2210
2211         for (i = 0; i < target->queue_size; ++i) {
2212                 ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2213                                               ch->max_ti_iu_len,
2214                                               GFP_KERNEL, DMA_FROM_DEVICE);
2215                 if (!ch->rx_ring[i])
2216                         goto err;
2217         }
2218
2219         for (i = 0; i < target->queue_size; ++i) {
2220                 ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2221                                               target->max_iu_len,
2222                                               GFP_KERNEL, DMA_TO_DEVICE);
2223                 if (!ch->tx_ring[i])
2224                         goto err;
2225
2226                 list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2227         }
2228
2229         return 0;
2230
2231 err:
2232         for (i = 0; i < target->queue_size; ++i) {
2233                 srp_free_iu(target->srp_host, ch->rx_ring[i]);
2234                 srp_free_iu(target->srp_host, ch->tx_ring[i]);
2235         }
2236
2237
2238 err_no_ring:
2239         kfree(ch->tx_ring);
2240         ch->tx_ring = NULL;
2241         kfree(ch->rx_ring);
2242         ch->rx_ring = NULL;
2243
2244         return -ENOMEM;
2245 }
2246
2247 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2248 {
2249         uint64_t T_tr_ns, max_compl_time_ms;
2250         uint32_t rq_tmo_jiffies;
2251
2252         /*
2253          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2254          * table 91), both the QP timeout and the retry count have to be set
2255          * for RC QP's during the RTR to RTS transition.
2256          */
2257         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2258                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2259
2260         /*
2261          * Set target->rq_tmo_jiffies to one second more than the largest time
2262          * it can take before an error completion is generated. See also
2263          * C9-140..142 in the IBTA spec for more information about how to
2264          * convert the QP Local ACK Timeout value to nanoseconds.
2265          */
2266         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2267         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2268         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2269         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2270
2271         return rq_tmo_jiffies;
2272 }
2273
2274 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2275                                const struct srp_login_rsp *lrsp,
2276                                struct srp_rdma_ch *ch)
2277 {
2278         struct srp_target_port *target = ch->target;
2279         struct ib_qp_attr *qp_attr = NULL;
2280         int attr_mask = 0;
2281         int ret;
2282         int i;
2283
2284         if (lrsp->opcode == SRP_LOGIN_RSP) {
2285                 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2286                 ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2287
2288                 /*
2289                  * Reserve credits for task management so we don't
2290                  * bounce requests back to the SCSI mid-layer.
2291                  */
2292                 target->scsi_host->can_queue
2293                         = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2294                               target->scsi_host->can_queue);
2295                 target->scsi_host->cmd_per_lun
2296                         = min_t(int, target->scsi_host->can_queue,
2297                                 target->scsi_host->cmd_per_lun);
2298         } else {
2299                 shost_printk(KERN_WARNING, target->scsi_host,
2300                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2301                 ret = -ECONNRESET;
2302                 goto error;
2303         }
2304
2305         if (!ch->rx_ring) {
2306                 ret = srp_alloc_iu_bufs(ch);
2307                 if (ret)
2308                         goto error;
2309         }
2310
2311         ret = -ENOMEM;
2312         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2313         if (!qp_attr)
2314                 goto error;
2315
2316         qp_attr->qp_state = IB_QPS_RTR;
2317         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2318         if (ret)
2319                 goto error_free;
2320
2321         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2322         if (ret)
2323                 goto error_free;
2324
2325         for (i = 0; i < target->queue_size; i++) {
2326                 struct srp_iu *iu = ch->rx_ring[i];
2327
2328                 ret = srp_post_recv(ch, iu);
2329                 if (ret)
2330                         goto error_free;
2331         }
2332
2333         qp_attr->qp_state = IB_QPS_RTS;
2334         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2335         if (ret)
2336                 goto error_free;
2337
2338         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2339
2340         ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2341         if (ret)
2342                 goto error_free;
2343
2344         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2345
2346 error_free:
2347         kfree(qp_attr);
2348
2349 error:
2350         ch->status = ret;
2351 }
2352
2353 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2354                                struct ib_cm_event *event,
2355                                struct srp_rdma_ch *ch)
2356 {
2357         struct srp_target_port *target = ch->target;
2358         struct Scsi_Host *shost = target->scsi_host;
2359         struct ib_class_port_info *cpi;
2360         int opcode;
2361
2362         switch (event->param.rej_rcvd.reason) {
2363         case IB_CM_REJ_PORT_CM_REDIRECT:
2364                 cpi = event->param.rej_rcvd.ari;
2365                 ch->path.dlid = cpi->redirect_lid;
2366                 ch->path.pkey = cpi->redirect_pkey;
2367                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2368                 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
2369
2370                 ch->status = ch->path.dlid ?
2371                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2372                 break;
2373
2374         case IB_CM_REJ_PORT_REDIRECT:
2375                 if (srp_target_is_topspin(target)) {
2376                         /*
2377                          * Topspin/Cisco SRP gateways incorrectly send
2378                          * reject reason code 25 when they mean 24
2379                          * (port redirect).
2380                          */
2381                         memcpy(ch->path.dgid.raw,
2382                                event->param.rej_rcvd.ari, 16);
2383
2384                         shost_printk(KERN_DEBUG, shost,
2385                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2386                                      be64_to_cpu(ch->path.dgid.global.subnet_prefix),
2387                                      be64_to_cpu(ch->path.dgid.global.interface_id));
2388
2389                         ch->status = SRP_PORT_REDIRECT;
2390                 } else {
2391                         shost_printk(KERN_WARNING, shost,
2392                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2393                         ch->status = -ECONNRESET;
2394                 }
2395                 break;
2396
2397         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2398                 shost_printk(KERN_WARNING, shost,
2399                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2400                 ch->status = -ECONNRESET;
2401                 break;
2402
2403         case IB_CM_REJ_CONSUMER_DEFINED:
2404                 opcode = *(u8 *) event->private_data;
2405                 if (opcode == SRP_LOGIN_REJ) {
2406                         struct srp_login_rej *rej = event->private_data;
2407                         u32 reason = be32_to_cpu(rej->reason);
2408
2409                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2410                                 shost_printk(KERN_WARNING, shost,
2411                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2412                         else
2413                                 shost_printk(KERN_WARNING, shost, PFX
2414                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2415                                              target->sgid.raw,
2416                                              target->orig_dgid.raw, reason);
2417                 } else
2418                         shost_printk(KERN_WARNING, shost,
2419                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2420                                      " opcode 0x%02x\n", opcode);
2421                 ch->status = -ECONNRESET;
2422                 break;
2423
2424         case IB_CM_REJ_STALE_CONN:
2425                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2426                 ch->status = SRP_STALE_CONN;
2427                 break;
2428
2429         default:
2430                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2431                              event->param.rej_rcvd.reason);
2432                 ch->status = -ECONNRESET;
2433         }
2434 }
2435
2436 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2437 {
2438         struct srp_rdma_ch *ch = cm_id->context;
2439         struct srp_target_port *target = ch->target;
2440         int comp = 0;
2441
2442         switch (event->event) {
2443         case IB_CM_REQ_ERROR:
2444                 shost_printk(KERN_DEBUG, target->scsi_host,
2445                              PFX "Sending CM REQ failed\n");
2446                 comp = 1;
2447                 ch->status = -ECONNRESET;
2448                 break;
2449
2450         case IB_CM_REP_RECEIVED:
2451                 comp = 1;
2452                 srp_cm_rep_handler(cm_id, event->private_data, ch);
2453                 break;
2454
2455         case IB_CM_REJ_RECEIVED:
2456                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2457                 comp = 1;
2458
2459                 srp_cm_rej_handler(cm_id, event, ch);
2460                 break;
2461
2462         case IB_CM_DREQ_RECEIVED:
2463                 shost_printk(KERN_WARNING, target->scsi_host,
2464                              PFX "DREQ received - connection closed\n");
2465                 ch->connected = false;
2466                 if (ib_send_cm_drep(cm_id, NULL, 0))
2467                         shost_printk(KERN_ERR, target->scsi_host,
2468                                      PFX "Sending CM DREP failed\n");
2469                 queue_work(system_long_wq, &target->tl_err_work);
2470                 break;
2471
2472         case IB_CM_TIMEWAIT_EXIT:
2473                 shost_printk(KERN_ERR, target->scsi_host,
2474                              PFX "connection closed\n");
2475                 comp = 1;
2476
2477                 ch->status = 0;
2478                 break;
2479
2480         case IB_CM_MRA_RECEIVED:
2481         case IB_CM_DREQ_ERROR:
2482         case IB_CM_DREP_RECEIVED:
2483                 break;
2484
2485         default:
2486                 shost_printk(KERN_WARNING, target->scsi_host,
2487                              PFX "Unhandled CM event %d\n", event->event);
2488                 break;
2489         }
2490
2491         if (comp)
2492                 complete(&ch->done);
2493
2494         return 0;
2495 }
2496
2497 /**
2498  * srp_change_queue_depth - setting device queue depth
2499  * @sdev: scsi device struct
2500  * @qdepth: requested queue depth
2501  *
2502  * Returns queue depth.
2503  */
2504 static int
2505 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2506 {
2507         if (!sdev->tagged_supported)
2508                 qdepth = 1;
2509         return scsi_change_queue_depth(sdev, qdepth);
2510 }
2511
2512 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2513                              u8 func)
2514 {
2515         struct srp_target_port *target = ch->target;
2516         struct srp_rport *rport = target->rport;
2517         struct ib_device *dev = target->srp_host->srp_dev->dev;
2518         struct srp_iu *iu;
2519         struct srp_tsk_mgmt *tsk_mgmt;
2520
2521         if (!ch->connected || target->qp_in_error)
2522                 return -1;
2523
2524         init_completion(&ch->tsk_mgmt_done);
2525
2526         /*
2527          * Lock the rport mutex to avoid that srp_create_ch_ib() is
2528          * invoked while a task management function is being sent.
2529          */
2530         mutex_lock(&rport->mutex);
2531         spin_lock_irq(&ch->lock);
2532         iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2533         spin_unlock_irq(&ch->lock);
2534
2535         if (!iu) {
2536                 mutex_unlock(&rport->mutex);
2537
2538                 return -1;
2539         }
2540
2541         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2542                                    DMA_TO_DEVICE);
2543         tsk_mgmt = iu->buf;
2544         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2545
2546         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2547         int_to_scsilun(lun, &tsk_mgmt->lun);
2548         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
2549         tsk_mgmt->tsk_mgmt_func = func;
2550         tsk_mgmt->task_tag      = req_tag;
2551
2552         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2553                                       DMA_TO_DEVICE);
2554         if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2555                 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2556                 mutex_unlock(&rport->mutex);
2557
2558                 return -1;
2559         }
2560         mutex_unlock(&rport->mutex);
2561
2562         if (!wait_for_completion_timeout(&ch->tsk_mgmt_done,
2563                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2564                 return -1;
2565
2566         return 0;
2567 }
2568
2569 static int srp_abort(struct scsi_cmnd *scmnd)
2570 {
2571         struct srp_target_port *target = host_to_target(scmnd->device->host);
2572         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2573         u32 tag;
2574         u16 ch_idx;
2575         struct srp_rdma_ch *ch;
2576         int ret;
2577
2578         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2579
2580         if (!req)
2581                 return SUCCESS;
2582         tag = blk_mq_unique_tag(scmnd->request);
2583         ch_idx = blk_mq_unique_tag_to_hwq(tag);
2584         if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2585                 return SUCCESS;
2586         ch = &target->ch[ch_idx];
2587         if (!srp_claim_req(ch, req, NULL, scmnd))
2588                 return SUCCESS;
2589         shost_printk(KERN_ERR, target->scsi_host,
2590                      "Sending SRP abort for tag %#x\n", tag);
2591         if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2592                               SRP_TSK_ABORT_TASK) == 0)
2593                 ret = SUCCESS;
2594         else if (target->rport->state == SRP_RPORT_LOST)
2595                 ret = FAST_IO_FAIL;
2596         else
2597                 ret = FAILED;
2598         srp_free_req(ch, req, scmnd, 0);
2599         scmnd->result = DID_ABORT << 16;
2600         scmnd->scsi_done(scmnd);
2601
2602         return ret;
2603 }
2604
2605 static int srp_reset_device(struct scsi_cmnd *scmnd)
2606 {
2607         struct srp_target_port *target = host_to_target(scmnd->device->host);
2608         struct srp_rdma_ch *ch;
2609         int i;
2610
2611         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2612
2613         ch = &target->ch[0];
2614         if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2615                               SRP_TSK_LUN_RESET))
2616                 return FAILED;
2617         if (ch->tsk_mgmt_status)
2618                 return FAILED;
2619
2620         for (i = 0; i < target->ch_count; i++) {
2621                 ch = &target->ch[i];
2622                 for (i = 0; i < target->req_ring_size; ++i) {
2623                         struct srp_request *req = &ch->req_ring[i];
2624
2625                         srp_finish_req(ch, req, scmnd->device, DID_RESET << 16);
2626                 }
2627         }
2628
2629         return SUCCESS;
2630 }
2631
2632 static int srp_reset_host(struct scsi_cmnd *scmnd)
2633 {
2634         struct srp_target_port *target = host_to_target(scmnd->device->host);
2635
2636         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2637
2638         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2639 }
2640
2641 static int srp_slave_alloc(struct scsi_device *sdev)
2642 {
2643         struct Scsi_Host *shost = sdev->host;
2644         struct srp_target_port *target = host_to_target(shost);
2645         struct srp_device *srp_dev = target->srp_host->srp_dev;
2646         struct ib_device *ibdev = srp_dev->dev;
2647
2648         if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG))
2649                 blk_queue_virt_boundary(sdev->request_queue,
2650                                         ~srp_dev->mr_page_mask);
2651
2652         return 0;
2653 }
2654
2655 static int srp_slave_configure(struct scsi_device *sdev)
2656 {
2657         struct Scsi_Host *shost = sdev->host;
2658         struct srp_target_port *target = host_to_target(shost);
2659         struct request_queue *q = sdev->request_queue;
2660         unsigned long timeout;
2661
2662         if (sdev->type == TYPE_DISK) {
2663                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2664                 blk_queue_rq_timeout(q, timeout);
2665         }
2666
2667         return 0;
2668 }
2669
2670 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2671                            char *buf)
2672 {
2673         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2674
2675         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2676 }
2677
2678 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2679                              char *buf)
2680 {
2681         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2682
2683         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2684 }
2685
2686 static ssize_t show_service_id(struct device *dev,
2687                                struct device_attribute *attr, char *buf)
2688 {
2689         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2690
2691         return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
2692 }
2693
2694 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2695                          char *buf)
2696 {
2697         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2698
2699         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
2700 }
2701
2702 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2703                          char *buf)
2704 {
2705         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2706
2707         return sprintf(buf, "%pI6\n", target->sgid.raw);
2708 }
2709
2710 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2711                          char *buf)
2712 {
2713         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2714         struct srp_rdma_ch *ch = &target->ch[0];
2715
2716         return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
2717 }
2718
2719 static ssize_t show_orig_dgid(struct device *dev,
2720                               struct device_attribute *attr, char *buf)
2721 {
2722         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2723
2724         return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
2725 }
2726
2727 static ssize_t show_req_lim(struct device *dev,
2728                             struct device_attribute *attr, char *buf)
2729 {
2730         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2731         struct srp_rdma_ch *ch;
2732         int i, req_lim = INT_MAX;
2733
2734         for (i = 0; i < target->ch_count; i++) {
2735                 ch = &target->ch[i];
2736                 req_lim = min(req_lim, ch->req_lim);
2737         }
2738         return sprintf(buf, "%d\n", req_lim);
2739 }
2740
2741 static ssize_t show_zero_req_lim(struct device *dev,
2742                                  struct device_attribute *attr, char *buf)
2743 {
2744         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2745
2746         return sprintf(buf, "%d\n", target->zero_req_lim);
2747 }
2748
2749 static ssize_t show_local_ib_port(struct device *dev,
2750                                   struct device_attribute *attr, char *buf)
2751 {
2752         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2753
2754         return sprintf(buf, "%d\n", target->srp_host->port);
2755 }
2756
2757 static ssize_t show_local_ib_device(struct device *dev,
2758                                     struct device_attribute *attr, char *buf)
2759 {
2760         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2761
2762         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2763 }
2764
2765 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr,
2766                              char *buf)
2767 {
2768         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2769
2770         return sprintf(buf, "%d\n", target->ch_count);
2771 }
2772
2773 static ssize_t show_comp_vector(struct device *dev,
2774                                 struct device_attribute *attr, char *buf)
2775 {
2776         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2777
2778         return sprintf(buf, "%d\n", target->comp_vector);
2779 }
2780
2781 static ssize_t show_tl_retry_count(struct device *dev,
2782                                    struct device_attribute *attr, char *buf)
2783 {
2784         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2785
2786         return sprintf(buf, "%d\n", target->tl_retry_count);
2787 }
2788
2789 static ssize_t show_cmd_sg_entries(struct device *dev,
2790                                    struct device_attribute *attr, char *buf)
2791 {
2792         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2793
2794         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2795 }
2796
2797 static ssize_t show_allow_ext_sg(struct device *dev,
2798                                  struct device_attribute *attr, char *buf)
2799 {
2800         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2801
2802         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2803 }
2804
2805 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2806 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2807 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2808 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2809 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2810 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2811 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2812 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2813 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2814 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2815 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2816 static DEVICE_ATTR(ch_count,        S_IRUGO, show_ch_count,        NULL);
2817 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2818 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2819 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2820 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2821
2822 static struct device_attribute *srp_host_attrs[] = {
2823         &dev_attr_id_ext,
2824         &dev_attr_ioc_guid,
2825         &dev_attr_service_id,
2826         &dev_attr_pkey,
2827         &dev_attr_sgid,
2828         &dev_attr_dgid,
2829         &dev_attr_orig_dgid,
2830         &dev_attr_req_lim,
2831         &dev_attr_zero_req_lim,
2832         &dev_attr_local_ib_port,
2833         &dev_attr_local_ib_device,
2834         &dev_attr_ch_count,
2835         &dev_attr_comp_vector,
2836         &dev_attr_tl_retry_count,
2837         &dev_attr_cmd_sg_entries,
2838         &dev_attr_allow_ext_sg,
2839         NULL
2840 };
2841
2842 static struct scsi_host_template srp_template = {
2843         .module                         = THIS_MODULE,
2844         .name                           = "InfiniBand SRP initiator",
2845         .proc_name                      = DRV_NAME,
2846         .slave_alloc                    = srp_slave_alloc,
2847         .slave_configure                = srp_slave_configure,
2848         .info                           = srp_target_info,
2849         .queuecommand                   = srp_queuecommand,
2850         .change_queue_depth             = srp_change_queue_depth,
2851         .eh_abort_handler               = srp_abort,
2852         .eh_device_reset_handler        = srp_reset_device,
2853         .eh_host_reset_handler          = srp_reset_host,
2854         .skip_settle_delay              = true,
2855         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2856         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2857         .this_id                        = -1,
2858         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2859         .use_clustering                 = ENABLE_CLUSTERING,
2860         .shost_attrs                    = srp_host_attrs,
2861         .track_queue_depth              = 1,
2862 };
2863
2864 static int srp_sdev_count(struct Scsi_Host *host)
2865 {
2866         struct scsi_device *sdev;
2867         int c = 0;
2868
2869         shost_for_each_device(sdev, host)
2870                 c++;
2871
2872         return c;
2873 }
2874
2875 /*
2876  * Return values:
2877  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
2878  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
2879  *    removal has been scheduled.
2880  * 0 and target->state != SRP_TARGET_REMOVED upon success.
2881  */
2882 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2883 {
2884         struct srp_rport_identifiers ids;
2885         struct srp_rport *rport;
2886
2887         target->state = SRP_TARGET_SCANNING;
2888         sprintf(target->target_name, "SRP.T10:%016llX",
2889                 be64_to_cpu(target->id_ext));
2890
2891         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2892                 return -ENODEV;
2893
2894         memcpy(ids.port_id, &target->id_ext, 8);
2895         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2896         ids.roles = SRP_RPORT_ROLE_TARGET;
2897         rport = srp_rport_add(target->scsi_host, &ids);
2898         if (IS_ERR(rport)) {
2899                 scsi_remove_host(target->scsi_host);
2900                 return PTR_ERR(rport);
2901         }
2902
2903         rport->lld_data = target;
2904         target->rport = rport;
2905
2906         spin_lock(&host->target_lock);
2907         list_add_tail(&target->list, &host->target_list);
2908         spin_unlock(&host->target_lock);
2909
2910         scsi_scan_target(&target->scsi_host->shost_gendev,
2911                          0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
2912
2913         if (srp_connected_ch(target) < target->ch_count ||
2914             target->qp_in_error) {
2915                 shost_printk(KERN_INFO, target->scsi_host,
2916                              PFX "SCSI scan failed - removing SCSI host\n");
2917                 srp_queue_remove_work(target);
2918                 goto out;
2919         }
2920
2921         pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
2922                  dev_name(&target->scsi_host->shost_gendev),
2923                  srp_sdev_count(target->scsi_host));
2924
2925         spin_lock_irq(&target->lock);
2926         if (target->state == SRP_TARGET_SCANNING)
2927                 target->state = SRP_TARGET_LIVE;
2928         spin_unlock_irq(&target->lock);
2929
2930 out:
2931         return 0;
2932 }
2933
2934 static void srp_release_dev(struct device *dev)
2935 {
2936         struct srp_host *host =
2937                 container_of(dev, struct srp_host, dev);
2938
2939         complete(&host->released);
2940 }
2941
2942 static struct class srp_class = {
2943         .name    = "infiniband_srp",
2944         .dev_release = srp_release_dev
2945 };
2946
2947 /**
2948  * srp_conn_unique() - check whether the connection to a target is unique
2949  * @host:   SRP host.
2950  * @target: SRP target port.
2951  */
2952 static bool srp_conn_unique(struct srp_host *host,
2953                             struct srp_target_port *target)
2954 {
2955         struct srp_target_port *t;
2956         bool ret = false;
2957
2958         if (target->state == SRP_TARGET_REMOVED)
2959                 goto out;
2960
2961         ret = true;
2962
2963         spin_lock(&host->target_lock);
2964         list_for_each_entry(t, &host->target_list, list) {
2965                 if (t != target &&
2966                     target->id_ext == t->id_ext &&
2967                     target->ioc_guid == t->ioc_guid &&
2968                     target->initiator_ext == t->initiator_ext) {
2969                         ret = false;
2970                         break;
2971                 }
2972         }
2973         spin_unlock(&host->target_lock);
2974
2975 out:
2976         return ret;
2977 }
2978
2979 /*
2980  * Target ports are added by writing
2981  *
2982  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2983  *     pkey=<P_Key>,service_id=<service ID>
2984  *
2985  * to the add_target sysfs attribute.
2986  */
2987 enum {
2988         SRP_OPT_ERR             = 0,
2989         SRP_OPT_ID_EXT          = 1 << 0,
2990         SRP_OPT_IOC_GUID        = 1 << 1,
2991         SRP_OPT_DGID            = 1 << 2,
2992         SRP_OPT_PKEY            = 1 << 3,
2993         SRP_OPT_SERVICE_ID      = 1 << 4,
2994         SRP_OPT_MAX_SECT        = 1 << 5,
2995         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2996         SRP_OPT_IO_CLASS        = 1 << 7,
2997         SRP_OPT_INITIATOR_EXT   = 1 << 8,
2998         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
2999         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
3000         SRP_OPT_SG_TABLESIZE    = 1 << 11,
3001         SRP_OPT_COMP_VECTOR     = 1 << 12,
3002         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
3003         SRP_OPT_QUEUE_SIZE      = 1 << 14,
3004         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
3005                                    SRP_OPT_IOC_GUID     |
3006                                    SRP_OPT_DGID         |
3007                                    SRP_OPT_PKEY         |
3008                                    SRP_OPT_SERVICE_ID),
3009 };
3010
3011 static const match_table_t srp_opt_tokens = {
3012         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
3013         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
3014         { SRP_OPT_DGID,                 "dgid=%s"               },
3015         { SRP_OPT_PKEY,                 "pkey=%x"               },
3016         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
3017         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
3018         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
3019         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
3020         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
3021         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
3022         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
3023         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
3024         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
3025         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
3026         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
3027         { SRP_OPT_ERR,                  NULL                    }
3028 };
3029
3030 static int srp_parse_options(const char *buf, struct srp_target_port *target)
3031 {
3032         char *options, *sep_opt;
3033         char *p;
3034         char dgid[3];
3035         substring_t args[MAX_OPT_ARGS];
3036         int opt_mask = 0;
3037         int token;
3038         int ret = -EINVAL;
3039         int i;
3040
3041         options = kstrdup(buf, GFP_KERNEL);
3042         if (!options)
3043                 return -ENOMEM;
3044
3045         sep_opt = options;
3046         while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3047                 if (!*p)
3048                         continue;
3049
3050                 token = match_token(p, srp_opt_tokens, args);
3051                 opt_mask |= token;
3052
3053                 switch (token) {
3054                 case SRP_OPT_ID_EXT:
3055                         p = match_strdup(args);
3056                         if (!p) {
3057                                 ret = -ENOMEM;
3058                                 goto out;
3059                         }
3060                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3061                         kfree(p);
3062                         break;
3063
3064                 case SRP_OPT_IOC_GUID:
3065                         p = match_strdup(args);
3066                         if (!p) {
3067                                 ret = -ENOMEM;
3068                                 goto out;
3069                         }
3070                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
3071                         kfree(p);
3072                         break;
3073
3074                 case SRP_OPT_DGID:
3075                         p = match_strdup(args);
3076                         if (!p) {
3077                                 ret = -ENOMEM;
3078                                 goto out;
3079                         }
3080                         if (strlen(p) != 32) {
3081                                 pr_warn("bad dest GID parameter '%s'\n", p);
3082                                 kfree(p);
3083                                 goto out;
3084                         }
3085
3086                         for (i = 0; i < 16; ++i) {
3087                                 strlcpy(dgid, p + i * 2, sizeof(dgid));
3088                                 if (sscanf(dgid, "%hhx",
3089                                            &target->orig_dgid.raw[i]) < 1) {
3090                                         ret = -EINVAL;
3091                                         kfree(p);
3092                                         goto out;
3093                                 }
3094                         }
3095                         kfree(p);
3096                         break;
3097
3098                 case SRP_OPT_PKEY:
3099                         if (match_hex(args, &token)) {
3100                                 pr_warn("bad P_Key parameter '%s'\n", p);
3101                                 goto out;
3102                         }
3103                         target->pkey = cpu_to_be16(token);
3104                         break;
3105
3106                 case SRP_OPT_SERVICE_ID:
3107                         p = match_strdup(args);
3108                         if (!p) {
3109                                 ret = -ENOMEM;
3110                                 goto out;
3111                         }
3112                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
3113                         kfree(p);
3114                         break;
3115
3116                 case SRP_OPT_MAX_SECT:
3117                         if (match_int(args, &token)) {
3118                                 pr_warn("bad max sect parameter '%s'\n", p);
3119                                 goto out;
3120                         }
3121                         target->scsi_host->max_sectors = token;
3122                         break;
3123
3124                 case SRP_OPT_QUEUE_SIZE:
3125                         if (match_int(args, &token) || token < 1) {
3126                                 pr_warn("bad queue_size parameter '%s'\n", p);
3127                                 goto out;
3128                         }
3129                         target->scsi_host->can_queue = token;
3130                         target->queue_size = token + SRP_RSP_SQ_SIZE +
3131                                              SRP_TSK_MGMT_SQ_SIZE;
3132                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3133                                 target->scsi_host->cmd_per_lun = token;
3134                         break;
3135
3136                 case SRP_OPT_MAX_CMD_PER_LUN:
3137                         if (match_int(args, &token) || token < 1) {
3138                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
3139                                         p);
3140                                 goto out;
3141                         }
3142                         target->scsi_host->cmd_per_lun = token;
3143                         break;
3144
3145                 case SRP_OPT_IO_CLASS:
3146                         if (match_hex(args, &token)) {
3147                                 pr_warn("bad IO class parameter '%s'\n", p);
3148                                 goto out;
3149                         }
3150                         if (token != SRP_REV10_IB_IO_CLASS &&
3151                             token != SRP_REV16A_IB_IO_CLASS) {
3152                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3153                                         token, SRP_REV10_IB_IO_CLASS,
3154                                         SRP_REV16A_IB_IO_CLASS);
3155                                 goto out;
3156                         }
3157                         target->io_class = token;
3158                         break;
3159
3160                 case SRP_OPT_INITIATOR_EXT:
3161                         p = match_strdup(args);
3162                         if (!p) {
3163                                 ret = -ENOMEM;
3164                                 goto out;
3165                         }
3166                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
3167                         kfree(p);
3168                         break;
3169
3170                 case SRP_OPT_CMD_SG_ENTRIES:
3171                         if (match_int(args, &token) || token < 1 || token > 255) {
3172                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3173                                         p);
3174                                 goto out;
3175                         }
3176                         target->cmd_sg_cnt = token;
3177                         break;
3178
3179                 case SRP_OPT_ALLOW_EXT_SG:
3180                         if (match_int(args, &token)) {
3181                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3182                                 goto out;
3183                         }
3184                         target->allow_ext_sg = !!token;
3185                         break;
3186
3187                 case SRP_OPT_SG_TABLESIZE:
3188                         if (match_int(args, &token) || token < 1 ||
3189                                         token > SG_MAX_SEGMENTS) {
3190                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
3191                                         p);
3192                                 goto out;
3193                         }
3194                         target->sg_tablesize = token;
3195                         break;
3196
3197                 case SRP_OPT_COMP_VECTOR:
3198                         if (match_int(args, &token) || token < 0) {
3199                                 pr_warn("bad comp_vector parameter '%s'\n", p);
3200                                 goto out;
3201                         }
3202                         target->comp_vector = token;
3203                         break;
3204
3205                 case SRP_OPT_TL_RETRY_COUNT:
3206                         if (match_int(args, &token) || token < 2 || token > 7) {
3207                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3208                                         p);
3209                                 goto out;
3210                         }
3211                         target->tl_retry_count = token;
3212                         break;
3213
3214                 default:
3215                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3216                                 p);
3217                         goto out;
3218                 }
3219         }
3220
3221         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
3222                 ret = 0;
3223         else
3224                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
3225                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
3226                             !(srp_opt_tokens[i].token & opt_mask))
3227                                 pr_warn("target creation request is missing parameter '%s'\n",
3228                                         srp_opt_tokens[i].pattern);
3229
3230         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3231             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3232                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3233                         target->scsi_host->cmd_per_lun,
3234                         target->scsi_host->can_queue);
3235
3236 out:
3237         kfree(options);
3238         return ret;
3239 }
3240
3241 static ssize_t srp_create_target(struct device *dev,
3242                                  struct device_attribute *attr,
3243                                  const char *buf, size_t count)
3244 {
3245         struct srp_host *host =
3246                 container_of(dev, struct srp_host, dev);
3247         struct Scsi_Host *target_host;
3248         struct srp_target_port *target;
3249         struct srp_rdma_ch *ch;
3250         struct srp_device *srp_dev = host->srp_dev;
3251         struct ib_device *ibdev = srp_dev->dev;
3252         int ret, node_idx, node, cpu, i;
3253         unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3254         bool multich = false;
3255
3256         target_host = scsi_host_alloc(&srp_template,
3257                                       sizeof (struct srp_target_port));
3258         if (!target_host)
3259                 return -ENOMEM;
3260
3261         target_host->transportt  = ib_srp_transport_template;
3262         target_host->max_channel = 0;
3263         target_host->max_id      = 1;
3264         target_host->max_lun     = -1LL;
3265         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3266
3267         target = host_to_target(target_host);
3268
3269         target->io_class        = SRP_REV16A_IB_IO_CLASS;
3270         target->scsi_host       = target_host;
3271         target->srp_host        = host;
3272         target->lkey            = host->srp_dev->pd->local_dma_lkey;
3273         target->global_mr       = host->srp_dev->global_mr;
3274         target->cmd_sg_cnt      = cmd_sg_entries;
3275         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
3276         target->allow_ext_sg    = allow_ext_sg;
3277         target->tl_retry_count  = 7;
3278         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
3279
3280         /*
3281          * Avoid that the SCSI host can be removed by srp_remove_target()
3282          * before this function returns.
3283          */
3284         scsi_host_get(target->scsi_host);
3285
3286         mutex_lock(&host->add_target_mutex);
3287
3288         ret = srp_parse_options(buf, target);
3289         if (ret)
3290                 goto out;
3291
3292         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
3293
3294         if (!srp_conn_unique(target->srp_host, target)) {
3295                 shost_printk(KERN_INFO, target->scsi_host,
3296                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3297                              be64_to_cpu(target->id_ext),
3298                              be64_to_cpu(target->ioc_guid),
3299                              be64_to_cpu(target->initiator_ext));
3300                 ret = -EEXIST;
3301                 goto out;
3302         }
3303
3304         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3305             target->cmd_sg_cnt < target->sg_tablesize) {
3306                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3307                 target->sg_tablesize = target->cmd_sg_cnt;
3308         }
3309
3310         if (srp_dev->use_fast_reg || srp_dev->use_fmr) {
3311                 /*
3312                  * FR and FMR can only map one HCA page per entry. If the
3313                  * start address is not aligned on a HCA page boundary two
3314                  * entries will be used for the head and the tail although
3315                  * these two entries combined contain at most one HCA page of
3316                  * data. Hence the "+ 1" in the calculation below.
3317                  *
3318                  * The indirect data buffer descriptor is contiguous so the
3319                  * memory for that buffer will only be registered if
3320                  * register_always is true. Hence add one to mr_per_cmd if
3321                  * register_always has been set.
3322                  */
3323                 max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3324                                   (ilog2(srp_dev->mr_page_size) - 9);
3325                 mr_per_cmd = register_always +
3326                         (target->scsi_host->max_sectors + 1 +
3327                          max_sectors_per_mr - 1) / max_sectors_per_mr;
3328                 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3329                          target->scsi_host->max_sectors,
3330                          srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3331                          max_sectors_per_mr, mr_per_cmd);
3332         }
3333
3334         target_host->sg_tablesize = target->sg_tablesize;
3335         target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3336         target->mr_per_cmd = mr_per_cmd;
3337         target->indirect_size = target->sg_tablesize *
3338                                 sizeof (struct srp_direct_buf);
3339         target->max_iu_len = sizeof (struct srp_cmd) +
3340                              sizeof (struct srp_indirect_buf) +
3341                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3342
3343         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3344         INIT_WORK(&target->remove_work, srp_remove_work);
3345         spin_lock_init(&target->lock);
3346         ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL);
3347         if (ret)
3348                 goto out;
3349
3350         ret = -ENOMEM;
3351         target->ch_count = max_t(unsigned, num_online_nodes(),
3352                                  min(ch_count ? :
3353                                      min(4 * num_online_nodes(),
3354                                          ibdev->num_comp_vectors),
3355                                      num_online_cpus()));
3356         target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3357                              GFP_KERNEL);
3358         if (!target->ch)
3359                 goto out;
3360
3361         node_idx = 0;
3362         for_each_online_node(node) {
3363                 const int ch_start = (node_idx * target->ch_count /
3364                                       num_online_nodes());
3365                 const int ch_end = ((node_idx + 1) * target->ch_count /
3366                                     num_online_nodes());
3367                 const int cv_start = (node_idx * ibdev->num_comp_vectors /
3368                                       num_online_nodes() + target->comp_vector)
3369                                      % ibdev->num_comp_vectors;
3370                 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors /
3371                                     num_online_nodes() + target->comp_vector)
3372                                    % ibdev->num_comp_vectors;
3373                 int cpu_idx = 0;
3374
3375                 for_each_online_cpu(cpu) {
3376                         if (cpu_to_node(cpu) != node)
3377                                 continue;
3378                         if (ch_start + cpu_idx >= ch_end)
3379                                 continue;
3380                         ch = &target->ch[ch_start + cpu_idx];
3381                         ch->target = target;
3382                         ch->comp_vector = cv_start == cv_end ? cv_start :
3383                                 cv_start + cpu_idx % (cv_end - cv_start);
3384                         spin_lock_init(&ch->lock);
3385                         INIT_LIST_HEAD(&ch->free_tx);
3386                         ret = srp_new_cm_id(ch);
3387                         if (ret)
3388                                 goto err_disconnect;
3389
3390                         ret = srp_create_ch_ib(ch);
3391                         if (ret)
3392                                 goto err_disconnect;
3393
3394                         ret = srp_alloc_req_data(ch);
3395                         if (ret)
3396                                 goto err_disconnect;
3397
3398                         ret = srp_connect_ch(ch, multich);
3399                         if (ret) {
3400                                 shost_printk(KERN_ERR, target->scsi_host,
3401                                              PFX "Connection %d/%d failed\n",
3402                                              ch_start + cpu_idx,
3403                                              target->ch_count);
3404                                 if (node_idx == 0 && cpu_idx == 0) {
3405                                         goto err_disconnect;
3406                                 } else {
3407                                         srp_free_ch_ib(target, ch);
3408                                         srp_free_req_data(target, ch);
3409                                         target->ch_count = ch - target->ch;
3410                                         goto connected;
3411                                 }
3412                         }
3413
3414                         multich = true;
3415                         cpu_idx++;
3416                 }
3417                 node_idx++;
3418         }
3419
3420 connected:
3421         target->scsi_host->nr_hw_queues = target->ch_count;
3422
3423         ret = srp_add_target(host, target);
3424         if (ret)
3425                 goto err_disconnect;
3426
3427         if (target->state != SRP_TARGET_REMOVED) {
3428                 shost_printk(KERN_DEBUG, target->scsi_host, PFX
3429                              "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3430                              be64_to_cpu(target->id_ext),
3431                              be64_to_cpu(target->ioc_guid),
3432                              be16_to_cpu(target->pkey),
3433                              be64_to_cpu(target->service_id),
3434                              target->sgid.raw, target->orig_dgid.raw);
3435         }
3436
3437         ret = count;
3438
3439 out:
3440         mutex_unlock(&host->add_target_mutex);
3441
3442         scsi_host_put(target->scsi_host);
3443         if (ret < 0)
3444                 scsi_host_put(target->scsi_host);
3445
3446         return ret;
3447
3448 err_disconnect:
3449         srp_disconnect_target(target);
3450
3451         for (i = 0; i < target->ch_count; i++) {
3452                 ch = &target->ch[i];
3453                 srp_free_ch_ib(target, ch);
3454                 srp_free_req_data(target, ch);
3455         }
3456
3457         kfree(target->ch);
3458         goto out;
3459 }
3460
3461 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3462
3463 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3464                           char *buf)
3465 {
3466         struct srp_host *host = container_of(dev, struct srp_host, dev);
3467
3468         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3469 }
3470
3471 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3472
3473 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3474                          char *buf)
3475 {
3476         struct srp_host *host = container_of(dev, struct srp_host, dev);
3477
3478         return sprintf(buf, "%d\n", host->port);
3479 }
3480
3481 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3482
3483 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3484 {
3485         struct srp_host *host;
3486
3487         host = kzalloc(sizeof *host, GFP_KERNEL);
3488         if (!host)
3489                 return NULL;
3490
3491         INIT_LIST_HEAD(&host->target_list);
3492         spin_lock_init(&host->target_lock);
3493         init_completion(&host->released);
3494         mutex_init(&host->add_target_mutex);
3495         host->srp_dev = device;
3496         host->port = port;
3497
3498         host->dev.class = &srp_class;
3499         host->dev.parent = device->dev->dma_device;
3500         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3501
3502         if (device_register(&host->dev))
3503                 goto free_host;
3504         if (device_create_file(&host->dev, &dev_attr_add_target))
3505                 goto err_class;
3506         if (device_create_file(&host->dev, &dev_attr_ibdev))
3507                 goto err_class;
3508         if (device_create_file(&host->dev, &dev_attr_port))
3509                 goto err_class;
3510
3511         return host;
3512
3513 err_class:
3514         device_unregister(&host->dev);
3515
3516 free_host:
3517         kfree(host);
3518
3519         return NULL;
3520 }
3521
3522 static void srp_add_one(struct ib_device *device)
3523 {
3524         struct srp_device *srp_dev;
3525         struct srp_host *host;
3526         int mr_page_shift, p;
3527         u64 max_pages_per_mr;
3528
3529         srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3530         if (!srp_dev)
3531                 return;
3532
3533         /*
3534          * Use the smallest page size supported by the HCA, down to a
3535          * minimum of 4096 bytes. We're unlikely to build large sglists
3536          * out of smaller entries.
3537          */
3538         mr_page_shift           = max(12, ffs(device->attrs.page_size_cap) - 1);
3539         srp_dev->mr_page_size   = 1 << mr_page_shift;
3540         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3541         max_pages_per_mr        = device->attrs.max_mr_size;
3542         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3543         pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
3544                  device->attrs.max_mr_size, srp_dev->mr_page_size,
3545                  max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
3546         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3547                                           max_pages_per_mr);
3548
3549         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3550                             device->map_phys_fmr && device->unmap_fmr);
3551         srp_dev->has_fr = (device->attrs.device_cap_flags &
3552                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3553         if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) {
3554                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3555         } else if (!never_register &&
3556                    device->attrs.max_mr_size >= 2 * srp_dev->mr_page_size) {
3557                 srp_dev->use_fast_reg = (srp_dev->has_fr &&
3558                                          (!srp_dev->has_fmr || prefer_fr));
3559                 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr;
3560         }
3561
3562         if (srp_dev->use_fast_reg) {
3563                 srp_dev->max_pages_per_mr =
3564                         min_t(u32, srp_dev->max_pages_per_mr,
3565                               device->attrs.max_fast_reg_page_list_len);
3566         }
3567         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3568                                    srp_dev->max_pages_per_mr;
3569         pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3570                  device->name, mr_page_shift, device->attrs.max_mr_size,
3571                  device->attrs.max_fast_reg_page_list_len,
3572                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3573
3574         INIT_LIST_HEAD(&srp_dev->dev_list);
3575
3576         srp_dev->dev = device;
3577         srp_dev->pd  = ib_alloc_pd(device);
3578         if (IS_ERR(srp_dev->pd))
3579                 goto free_dev;
3580
3581         if (never_register || !register_always ||
3582             (!srp_dev->has_fmr && !srp_dev->has_fr)) {
3583                 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd,
3584                                                    IB_ACCESS_LOCAL_WRITE |
3585                                                    IB_ACCESS_REMOTE_READ |
3586                                                    IB_ACCESS_REMOTE_WRITE);
3587                 if (IS_ERR(srp_dev->global_mr))
3588                         goto err_pd;
3589         } else {
3590                 srp_dev->global_mr = NULL;
3591         }
3592
3593         for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) {
3594                 host = srp_add_port(srp_dev, p);
3595                 if (host)
3596                         list_add_tail(&host->list, &srp_dev->dev_list);
3597         }
3598
3599         ib_set_client_data(device, &srp_client, srp_dev);
3600         return;
3601
3602 err_pd:
3603         ib_dealloc_pd(srp_dev->pd);
3604
3605 free_dev:
3606         kfree(srp_dev);
3607 }
3608
3609 static void srp_remove_one(struct ib_device *device, void *client_data)
3610 {
3611         struct srp_device *srp_dev;
3612         struct srp_host *host, *tmp_host;
3613         struct srp_target_port *target;
3614
3615         srp_dev = client_data;
3616         if (!srp_dev)
3617                 return;
3618
3619         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3620                 device_unregister(&host->dev);
3621                 /*
3622                  * Wait for the sysfs entry to go away, so that no new
3623                  * target ports can be created.
3624                  */
3625                 wait_for_completion(&host->released);
3626
3627                 /*
3628                  * Remove all target ports.
3629                  */
3630                 spin_lock(&host->target_lock);
3631                 list_for_each_entry(target, &host->target_list, list)
3632                         srp_queue_remove_work(target);
3633                 spin_unlock(&host->target_lock);
3634
3635                 /*
3636                  * Wait for tl_err and target port removal tasks.
3637                  */
3638                 flush_workqueue(system_long_wq);
3639                 flush_workqueue(srp_remove_wq);
3640
3641                 kfree(host);
3642         }
3643
3644         if (srp_dev->global_mr)
3645                 ib_dereg_mr(srp_dev->global_mr);
3646         ib_dealloc_pd(srp_dev->pd);
3647
3648         kfree(srp_dev);
3649 }
3650
3651 static struct srp_function_template ib_srp_transport_functions = {
3652         .has_rport_state         = true,
3653         .reset_timer_if_blocked  = true,
3654         .reconnect_delay         = &srp_reconnect_delay,
3655         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3656         .dev_loss_tmo            = &srp_dev_loss_tmo,
3657         .reconnect               = srp_rport_reconnect,
3658         .rport_delete            = srp_rport_delete,
3659         .terminate_rport_io      = srp_terminate_io,
3660 };
3661
3662 static int __init srp_init_module(void)
3663 {
3664         int ret;
3665
3666         if (srp_sg_tablesize) {
3667                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3668                 if (!cmd_sg_entries)
3669                         cmd_sg_entries = srp_sg_tablesize;
3670         }
3671
3672         if (!cmd_sg_entries)
3673                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3674
3675         if (cmd_sg_entries > 255) {
3676                 pr_warn("Clamping cmd_sg_entries to 255\n");
3677                 cmd_sg_entries = 255;
3678         }
3679
3680         if (!indirect_sg_entries)
3681                 indirect_sg_entries = cmd_sg_entries;
3682         else if (indirect_sg_entries < cmd_sg_entries) {
3683                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3684                         cmd_sg_entries);
3685                 indirect_sg_entries = cmd_sg_entries;
3686         }
3687
3688         srp_remove_wq = create_workqueue("srp_remove");
3689         if (!srp_remove_wq) {
3690                 ret = -ENOMEM;
3691                 goto out;
3692         }
3693
3694         ret = -ENOMEM;
3695         ib_srp_transport_template =
3696                 srp_attach_transport(&ib_srp_transport_functions);
3697         if (!ib_srp_transport_template)
3698                 goto destroy_wq;
3699
3700         ret = class_register(&srp_class);
3701         if (ret) {
3702                 pr_err("couldn't register class infiniband_srp\n");
3703                 goto release_tr;
3704         }
3705
3706         ib_sa_register_client(&srp_sa_client);
3707
3708         ret = ib_register_client(&srp_client);
3709         if (ret) {
3710                 pr_err("couldn't register IB client\n");
3711                 goto unreg_sa;
3712         }
3713
3714 out:
3715         return ret;
3716
3717 unreg_sa:
3718         ib_sa_unregister_client(&srp_sa_client);
3719         class_unregister(&srp_class);
3720
3721 release_tr:
3722         srp_release_transport(ib_srp_transport_template);
3723
3724 destroy_wq:
3725         destroy_workqueue(srp_remove_wq);
3726         goto out;
3727 }
3728
3729 static void __exit srp_cleanup_module(void)
3730 {
3731         ib_unregister_client(&srp_client);
3732         ib_sa_unregister_client(&srp_sa_client);
3733         class_unregister(&srp_class);
3734         srp_release_transport(ib_srp_transport_template);
3735         destroy_workqueue(srp_remove_wq);
3736 }
3737
3738 module_init(srp_init_module);
3739 module_exit(srp_cleanup_module);