drivers/target/iscsi/iscsi_target_util.c

   1 /*******************************************************************************
   2  * This file contains the iSCSI Target specific utility functions.
   3  *
   4  * (c) Copyright 2007-2013 Datera, Inc.
   5  *
   6  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  ******************************************************************************/
  18
  19 #include <linux/list.h>
  20 #include <linux/percpu_ida.h>
  21 #include <scsi/scsi_tcq.h>
  22 #include <scsi/iscsi_proto.h>
  23 #include <target/target_core_base.h>
  24 #include <target/target_core_fabric.h>
  25 #include <target/iscsi/iscsi_transport.h>
  26
  27 #include <target/iscsi/iscsi_target_core.h>
  28 #include "iscsi_target_parameters.h"
  29 #include "iscsi_target_seq_pdu_list.h"
  30 #include "iscsi_target_datain_values.h"
  31 #include "iscsi_target_erl0.h"
  32 #include "iscsi_target_erl1.h"
  33 #include "iscsi_target_erl2.h"
  34 #include "iscsi_target_tpg.h"
  35 #include "iscsi_target_util.h"
  36 #include "iscsi_target.h"
  37
  38 #define PRINT_BUFF(buff, len)                                   \
  39 {                                                               \
  40         int zzz;                                                \
  41                                                                 \
  42         pr_debug("%d:\n", __LINE__);                            \
  43         for (zzz = 0; zzz < len; zzz++) {                       \
  44                 if (zzz % 16 == 0) {                            \
  45                         if (zzz)                                \
  46                                 pr_debug("\n");                 \
  47                         pr_debug("%4i: ", zzz);                 \
  48                 }                                               \
  49                 pr_debug("%02x ", (unsigned char) (buff)[zzz]); \
  50         }                                                       \
  51         if ((len + 1) % 16)                                     \
  52                 pr_debug("\n");                                 \
  53 }
  54
  55 extern struct list_head g_tiqn_list;
  56 extern spinlock_t tiqn_lock;
  57
  58 /*
  59  *      Called with cmd->r2t_lock held.
  60  */
  61 int iscsit_add_r2t_to_list(
  62         struct iscsi_cmd *cmd,
  63         u32 offset,
  64         u32 xfer_len,
  65         int recovery,
  66         u32 r2t_sn)
  67 {
  68         struct iscsi_r2t *r2t;
  69
  70         r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
  71         if (!r2t) {
  72                 pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
  73                 return -1;
  74         }
  75         INIT_LIST_HEAD(&r2t->r2t_list);
  76
  77         r2t->recovery_r2t = recovery;
  78         r2t->r2t_sn = (!r2t_sn) ? cmd->r2t_sn++ : r2t_sn;
  79         r2t->offset = offset;
  80         r2t->xfer_len = xfer_len;
  81         list_add_tail(&r2t->r2t_list, &cmd->cmd_r2t_list);
  82         spin_unlock_bh(&cmd->r2t_lock);
  83
  84         iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
  85
  86         spin_lock_bh(&cmd->r2t_lock);
  87         return 0;
  88 }
  89
  90 struct iscsi_r2t *iscsit_get_r2t_for_eos(
  91         struct iscsi_cmd *cmd,
  92         u32 offset,
  93         u32 length)
  94 {
  95         struct iscsi_r2t *r2t;
  96
  97         spin_lock_bh(&cmd->r2t_lock);
  98         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
  99                 if ((r2t->offset <= offset) &&
 100                     (r2t->offset + r2t->xfer_len) >= (offset + length)) {
 101                         spin_unlock_bh(&cmd->r2t_lock);
 102                         return r2t;
 103                 }
 104         }
 105         spin_unlock_bh(&cmd->r2t_lock);
 106
 107         pr_err("Unable to locate R2T for Offset: %u, Length:"
 108                         " %u\n", offset, length);
 109         return NULL;
 110 }
 111
 112 struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *cmd)
 113 {
 114         struct iscsi_r2t *r2t;
 115
 116         spin_lock_bh(&cmd->r2t_lock);
 117         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 118                 if (!r2t->sent_r2t) {
 119                         spin_unlock_bh(&cmd->r2t_lock);
 120                         return r2t;
 121                 }
 122         }
 123         spin_unlock_bh(&cmd->r2t_lock);
 124
 125         pr_err("Unable to locate next R2T to send for ITT:"
 126                         " 0x%08x.\n", cmd->init_task_tag);
 127         return NULL;
 128 }
 129
 130 /*
 131  *      Called with cmd->r2t_lock held.
 132  */
 133 void iscsit_free_r2t(struct iscsi_r2t *r2t, struct iscsi_cmd *cmd)
 134 {
 135         list_del(&r2t->r2t_list);
 136         kmem_cache_free(lio_r2t_cache, r2t);
 137 }
 138
 139 void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd)
 140 {
 141         struct iscsi_r2t *r2t, *r2t_tmp;
 142
 143         spin_lock_bh(&cmd->r2t_lock);
 144         list_for_each_entry_safe(r2t, r2t_tmp, &cmd->cmd_r2t_list, r2t_list)
 145                 iscsit_free_r2t(r2t, cmd);
 146         spin_unlock_bh(&cmd->r2t_lock);
 147 }
 148
 149 /*
 150  * May be called from software interrupt (timer) context for allocating
 151  * iSCSI NopINs.
 152  */
 153 struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, int state)
 154 {
 155         struct iscsi_cmd *cmd;
 156         struct se_session *se_sess = conn->sess->se_sess;
 157         int size, tag;
 158
 159         tag = percpu_ida_alloc(&se_sess->sess_tag_pool, state);
 160         if (tag < 0)
 161                 return NULL;
 162
 163         size = sizeof(struct iscsi_cmd) + conn->conn_transport->priv_size;
 164         cmd = (struct iscsi_cmd *)(se_sess->sess_cmd_map + (tag * size));
 165         memset(cmd, 0, size);
 166
 167         cmd->se_cmd.map_tag = tag;
 168         cmd->conn = conn;
 169         INIT_LIST_HEAD(&cmd->i_conn_node);
 170         INIT_LIST_HEAD(&cmd->datain_list);
 171         INIT_LIST_HEAD(&cmd->cmd_r2t_list);
 172         spin_lock_init(&cmd->datain_lock);
 173         spin_lock_init(&cmd->dataout_timeout_lock);
 174         spin_lock_init(&cmd->istate_lock);
 175         spin_lock_init(&cmd->error_lock);
 176         spin_lock_init(&cmd->r2t_lock);
 177
 178         return cmd;
 179 }
 180 EXPORT_SYMBOL(iscsit_allocate_cmd);
 181
 182 struct iscsi_seq *iscsit_get_seq_holder_for_datain(
 183         struct iscsi_cmd *cmd,
 184         u32 seq_send_order)
 185 {
 186         u32 i;
 187
 188         for (i = 0; i < cmd->seq_count; i++)
 189                 if (cmd->seq_list[i].seq_send_order == seq_send_order)
 190                         return &cmd->seq_list[i];
 191
 192         return NULL;
 193 }
 194
 195 struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *cmd)
 196 {
 197         u32 i;
 198
 199         if (!cmd->seq_list) {
 200                 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
 201                 return NULL;
 202         }
 203
 204         for (i = 0; i < cmd->seq_count; i++) {
 205                 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
 206                         continue;
 207                 if (cmd->seq_list[i].seq_send_order == cmd->seq_send_order) {
 208                         cmd->seq_send_order++;
 209                         return &cmd->seq_list[i];
 210                 }
 211         }
 212
 213         return NULL;
 214 }
 215
 216 struct iscsi_r2t *iscsit_get_holder_for_r2tsn(
 217         struct iscsi_cmd *cmd,
 218         u32 r2t_sn)
 219 {
 220         struct iscsi_r2t *r2t;
 221
 222         spin_lock_bh(&cmd->r2t_lock);
 223         list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
 224                 if (r2t->r2t_sn == r2t_sn) {
 225                         spin_unlock_bh(&cmd->r2t_lock);
 226                         return r2t;
 227                 }
 228         }
 229         spin_unlock_bh(&cmd->r2t_lock);
 230
 231         return NULL;
 232 }
 233
 234 static inline int iscsit_check_received_cmdsn(struct iscsi_session *sess, u32 cmdsn)
 235 {
 236         int ret;
 237
 238         /*
 239          * This is the proper method of checking received CmdSN against
 240          * ExpCmdSN and MaxCmdSN values, as well as accounting for out
 241          * or order CmdSNs due to multiple connection sessions and/or
 242          * CRC failures.
 243          */
 244         if (iscsi_sna_gt(cmdsn, sess->max_cmd_sn)) {
 245                 pr_err("Received CmdSN: 0x%08x is greater than"
 246                        " MaxCmdSN: 0x%08x, ignoring.\n", cmdsn,
 247                        sess->max_cmd_sn);
 248                 ret = CMDSN_MAXCMDSN_OVERRUN;
 249
 250         } else if (cmdsn == sess->exp_cmd_sn) {
 251                 sess->exp_cmd_sn++;
 252                 pr_debug("Received CmdSN matches ExpCmdSN,"
 253                       " incremented ExpCmdSN to: 0x%08x\n",
 254                       sess->exp_cmd_sn);
 255                 ret = CMDSN_NORMAL_OPERATION;
 256
 257         } else if (iscsi_sna_gt(cmdsn, sess->exp_cmd_sn)) {
 258                 pr_debug("Received CmdSN: 0x%08x is greater"
 259                       " than ExpCmdSN: 0x%08x, not acknowledging.\n",
 260                       cmdsn, sess->exp_cmd_sn);
 261                 ret = CMDSN_HIGHER_THAN_EXP;
 262
 263         } else {
 264                 pr_err("Received CmdSN: 0x%08x is less than"
 265                        " ExpCmdSN: 0x%08x, ignoring.\n", cmdsn,
 266                        sess->exp_cmd_sn);
 267                 ret = CMDSN_LOWER_THAN_EXP;
 268         }
 269
 270         return ret;
 271 }
 272
 273 /*
 274  * Commands may be received out of order if MC/S is in use.
 275  * Ensure they are executed in CmdSN order.
 276  */
 277 int iscsit_sequence_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 278                         unsigned char *buf, __be32 cmdsn)
 279 {
 280         int ret, cmdsn_ret;
 281         bool reject = false;
 282         u8 reason = ISCSI_REASON_BOOKMARK_NO_RESOURCES;
 283
 284         mutex_lock(&conn->sess->cmdsn_mutex);
 285
 286         cmdsn_ret = iscsit_check_received_cmdsn(conn->sess, be32_to_cpu(cmdsn));
 287         switch (cmdsn_ret) {
 288         case CMDSN_NORMAL_OPERATION:
 289                 ret = iscsit_execute_cmd(cmd, 0);
 290                 if ((ret >= 0) && !list_empty(&conn->sess->sess_ooo_cmdsn_list))
 291                         iscsit_execute_ooo_cmdsns(conn->sess);
 292                 else if (ret < 0) {
 293                         reject = true;
 294                         ret = CMDSN_ERROR_CANNOT_RECOVER;
 295                 }
 296                 break;
 297         case CMDSN_HIGHER_THAN_EXP:
 298                 ret = iscsit_handle_ooo_cmdsn(conn->sess, cmd, be32_to_cpu(cmdsn));
 299                 if (ret < 0) {
 300                         reject = true;
 301                         ret = CMDSN_ERROR_CANNOT_RECOVER;
 302                         break;
 303                 }
 304                 ret = CMDSN_HIGHER_THAN_EXP;
 305                 break;
 306         case CMDSN_LOWER_THAN_EXP:
 307         case CMDSN_MAXCMDSN_OVERRUN:
 308         default:
 309                 cmd->i_state = ISTATE_REMOVE;
 310                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
 311                 /*
 312                  * Existing callers for iscsit_sequence_cmd() will silently
 313                  * ignore commands with CMDSN_LOWER_THAN_EXP, so force this
 314                  * return for CMDSN_MAXCMDSN_OVERRUN as well..
 315                  */
 316                 ret = CMDSN_LOWER_THAN_EXP;
 317                 break;
 318         }
 319         mutex_unlock(&conn->sess->cmdsn_mutex);
 320
 321         if (reject)
 322                 iscsit_reject_cmd(cmd, reason, buf);
 323
 324         return ret;
 325 }
 326 EXPORT_SYMBOL(iscsit_sequence_cmd);
 327
 328 int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf)
 329 {
 330         struct iscsi_conn *conn = cmd->conn;
 331         struct se_cmd *se_cmd = &cmd->se_cmd;
 332         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 333         u32 payload_length = ntoh24(hdr->dlength);
 334
 335         if (conn->sess->sess_ops->InitialR2T) {
 336                 pr_err("Received unexpected unsolicited data"
 337                         " while InitialR2T=Yes, protocol error.\n");
 338                 transport_send_check_condition_and_sense(se_cmd,
 339                                 TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
 340                 return -1;
 341         }
 342
 343         if ((cmd->first_burst_len + payload_length) >
 344              conn->sess->sess_ops->FirstBurstLength) {
 345                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 346                         " for this Unsolicited DataOut Burst.\n",
 347                         (cmd->first_burst_len + payload_length),
 348                                 conn->sess->sess_ops->FirstBurstLength);
 349                 transport_send_check_condition_and_sense(se_cmd,
 350                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 351                 return -1;
 352         }
 353
 354         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))
 355                 return 0;
 356
 357         if (((cmd->first_burst_len + payload_length) != cmd->se_cmd.data_length) &&
 358             ((cmd->first_burst_len + payload_length) !=
 359               conn->sess->sess_ops->FirstBurstLength)) {
 360                 pr_err("Unsolicited non-immediate data received %u"
 361                         " does not equal FirstBurstLength: %u, and does"
 362                         " not equal ExpXferLen %u.\n",
 363                         (cmd->first_burst_len + payload_length),
 364                         conn->sess->sess_ops->FirstBurstLength, cmd->se_cmd.data_length);
 365                 transport_send_check_condition_and_sense(se_cmd,
 366                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 367                 return -1;
 368         }
 369         return 0;
 370 }
 371
 372 struct iscsi_cmd *iscsit_find_cmd_from_itt(
 373         struct iscsi_conn *conn,
 374         itt_t init_task_tag)
 375 {
 376         struct iscsi_cmd *cmd;
 377
 378         spin_lock_bh(&conn->cmd_lock);
 379         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 380                 if (cmd->init_task_tag == init_task_tag) {
 381                         spin_unlock_bh(&conn->cmd_lock);
 382                         return cmd;
 383                 }
 384         }
 385         spin_unlock_bh(&conn->cmd_lock);
 386
 387         pr_err("Unable to locate ITT: 0x%08x on CID: %hu",
 388                         init_task_tag, conn->cid);
 389         return NULL;
 390 }
 391 EXPORT_SYMBOL(iscsit_find_cmd_from_itt);
 392
 393 struct iscsi_cmd *iscsit_find_cmd_from_itt_or_dump(
 394         struct iscsi_conn *conn,
 395         itt_t init_task_tag,
 396         u32 length)
 397 {
 398         struct iscsi_cmd *cmd;
 399
 400         spin_lock_bh(&conn->cmd_lock);
 401         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 402                 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT)
 403                         continue;
 404                 if (cmd->init_task_tag == init_task_tag) {
 405                         spin_unlock_bh(&conn->cmd_lock);
 406                         return cmd;
 407                 }
 408         }
 409         spin_unlock_bh(&conn->cmd_lock);
 410
 411         pr_err("Unable to locate ITT: 0x%08x on CID: %hu,"
 412                         " dumping payload\n", init_task_tag, conn->cid);
 413         if (length)
 414                 iscsit_dump_data_payload(conn, length, 1);
 415
 416         return NULL;
 417 }
 418
 419 struct iscsi_cmd *iscsit_find_cmd_from_ttt(
 420         struct iscsi_conn *conn,
 421         u32 targ_xfer_tag)
 422 {
 423         struct iscsi_cmd *cmd = NULL;
 424
 425         spin_lock_bh(&conn->cmd_lock);
 426         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
 427                 if (cmd->targ_xfer_tag == targ_xfer_tag) {
 428                         spin_unlock_bh(&conn->cmd_lock);
 429                         return cmd;
 430                 }
 431         }
 432         spin_unlock_bh(&conn->cmd_lock);
 433
 434         pr_err("Unable to locate TTT: 0x%08x on CID: %hu\n",
 435                         targ_xfer_tag, conn->cid);
 436         return NULL;
 437 }
 438
 439 int iscsit_find_cmd_for_recovery(
 440         struct iscsi_session *sess,
 441         struct iscsi_cmd **cmd_ptr,
 442         struct iscsi_conn_recovery **cr_ptr,
 443         itt_t init_task_tag)
 444 {
 445         struct iscsi_cmd *cmd = NULL;
 446         struct iscsi_conn_recovery *cr;
 447         /*
 448          * Scan through the inactive connection recovery list's command list.
 449          * If init_task_tag matches the command is still alligent.
 450          */
 451         spin_lock(&sess->cr_i_lock);
 452         list_for_each_entry(cr, &sess->cr_inactive_list, cr_list) {
 453                 spin_lock(&cr->conn_recovery_cmd_lock);
 454                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 455                         if (cmd->init_task_tag == init_task_tag) {
 456                                 spin_unlock(&cr->conn_recovery_cmd_lock);
 457                                 spin_unlock(&sess->cr_i_lock);
 458
 459                                 *cr_ptr = cr;
 460                                 *cmd_ptr = cmd;
 461                                 return -2;
 462                         }
 463                 }
 464                 spin_unlock(&cr->conn_recovery_cmd_lock);
 465         }
 466         spin_unlock(&sess->cr_i_lock);
 467         /*
 468          * Scan through the active connection recovery list's command list.
 469          * If init_task_tag matches the command is ready to be reassigned.
 470          */
 471         spin_lock(&sess->cr_a_lock);
 472         list_for_each_entry(cr, &sess->cr_active_list, cr_list) {
 473                 spin_lock(&cr->conn_recovery_cmd_lock);
 474                 list_for_each_entry(cmd, &cr->conn_recovery_cmd_list, i_conn_node) {
 475                         if (cmd->init_task_tag == init_task_tag) {
 476                                 spin_unlock(&cr->conn_recovery_cmd_lock);
 477                                 spin_unlock(&sess->cr_a_lock);
 478
 479                                 *cr_ptr = cr;
 480                                 *cmd_ptr = cmd;
 481                                 return 0;
 482                         }
 483                 }
 484                 spin_unlock(&cr->conn_recovery_cmd_lock);
 485         }
 486         spin_unlock(&sess->cr_a_lock);
 487
 488         return -1;
 489 }
 490
 491 void iscsit_add_cmd_to_immediate_queue(
 492         struct iscsi_cmd *cmd,
 493         struct iscsi_conn *conn,
 494         u8 state)
 495 {
 496         struct iscsi_queue_req *qr;
 497
 498         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 499         if (!qr) {
 500                 pr_err("Unable to allocate memory for"
 501                                 " struct iscsi_queue_req\n");
 502                 return;
 503         }
 504         INIT_LIST_HEAD(&qr->qr_list);
 505         qr->cmd = cmd;
 506         qr->state = state;
 507
 508         spin_lock_bh(&conn->immed_queue_lock);
 509         list_add_tail(&qr->qr_list, &conn->immed_queue_list);
 510         atomic_inc(&cmd->immed_queue_count);
 511         atomic_set(&conn->check_immediate_queue, 1);
 512         spin_unlock_bh(&conn->immed_queue_lock);
 513
 514         wake_up(&conn->queues_wq);
 515 }
 516
 517 struct iscsi_queue_req *iscsit_get_cmd_from_immediate_queue(struct iscsi_conn *conn)
 518 {
 519         struct iscsi_queue_req *qr;
 520
 521         spin_lock_bh(&conn->immed_queue_lock);
 522         if (list_empty(&conn->immed_queue_list)) {
 523                 spin_unlock_bh(&conn->immed_queue_lock);
 524                 return NULL;
 525         }
 526         qr = list_first_entry(&conn->immed_queue_list,
 527                               struct iscsi_queue_req, qr_list);
 528
 529         list_del(&qr->qr_list);
 530         if (qr->cmd)
 531                 atomic_dec(&qr->cmd->immed_queue_count);
 532         spin_unlock_bh(&conn->immed_queue_lock);
 533
 534         return qr;
 535 }
 536
 537 static void iscsit_remove_cmd_from_immediate_queue(
 538         struct iscsi_cmd *cmd,
 539         struct iscsi_conn *conn)
 540 {
 541         struct iscsi_queue_req *qr, *qr_tmp;
 542
 543         spin_lock_bh(&conn->immed_queue_lock);
 544         if (!atomic_read(&cmd->immed_queue_count)) {
 545                 spin_unlock_bh(&conn->immed_queue_lock);
 546                 return;
 547         }
 548
 549         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 550                 if (qr->cmd != cmd)
 551                         continue;
 552
 553                 atomic_dec(&qr->cmd->immed_queue_count);
 554                 list_del(&qr->qr_list);
 555                 kmem_cache_free(lio_qr_cache, qr);
 556         }
 557         spin_unlock_bh(&conn->immed_queue_lock);
 558
 559         if (atomic_read(&cmd->immed_queue_count)) {
 560                 pr_err("ITT: 0x%08x immed_queue_count: %d\n",
 561                         cmd->init_task_tag,
 562                         atomic_read(&cmd->immed_queue_count));
 563         }
 564 }
 565
 566 void iscsit_add_cmd_to_response_queue(
 567         struct iscsi_cmd *cmd,
 568         struct iscsi_conn *conn,
 569         u8 state)
 570 {
 571         struct iscsi_queue_req *qr;
 572
 573         qr = kmem_cache_zalloc(lio_qr_cache, GFP_ATOMIC);
 574         if (!qr) {
 575                 pr_err("Unable to allocate memory for"
 576                         " struct iscsi_queue_req\n");
 577                 return;
 578         }
 579         INIT_LIST_HEAD(&qr->qr_list);
 580         qr->cmd = cmd;
 581         qr->state = state;
 582
 583         spin_lock_bh(&conn->response_queue_lock);
 584         list_add_tail(&qr->qr_list, &conn->response_queue_list);
 585         atomic_inc(&cmd->response_queue_count);
 586         spin_unlock_bh(&conn->response_queue_lock);
 587
 588         wake_up(&conn->queues_wq);
 589 }
 590
 591 struct iscsi_queue_req *iscsit_get_cmd_from_response_queue(struct iscsi_conn *conn)
 592 {
 593         struct iscsi_queue_req *qr;
 594
 595         spin_lock_bh(&conn->response_queue_lock);
 596         if (list_empty(&conn->response_queue_list)) {
 597                 spin_unlock_bh(&conn->response_queue_lock);
 598                 return NULL;
 599         }
 600
 601         qr = list_first_entry(&conn->response_queue_list,
 602                               struct iscsi_queue_req, qr_list);
 603
 604         list_del(&qr->qr_list);
 605         if (qr->cmd)
 606                 atomic_dec(&qr->cmd->response_queue_count);
 607         spin_unlock_bh(&conn->response_queue_lock);
 608
 609         return qr;
 610 }
 611
 612 static void iscsit_remove_cmd_from_response_queue(
 613         struct iscsi_cmd *cmd,
 614         struct iscsi_conn *conn)
 615 {
 616         struct iscsi_queue_req *qr, *qr_tmp;
 617
 618         spin_lock_bh(&conn->response_queue_lock);
 619         if (!atomic_read(&cmd->response_queue_count)) {
 620                 spin_unlock_bh(&conn->response_queue_lock);
 621                 return;
 622         }
 623
 624         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 625                                 qr_list) {
 626                 if (qr->cmd != cmd)
 627                         continue;
 628
 629                 atomic_dec(&qr->cmd->response_queue_count);
 630                 list_del(&qr->qr_list);
 631                 kmem_cache_free(lio_qr_cache, qr);
 632         }
 633         spin_unlock_bh(&conn->response_queue_lock);
 634
 635         if (atomic_read(&cmd->response_queue_count)) {
 636                 pr_err("ITT: 0x%08x response_queue_count: %d\n",
 637                         cmd->init_task_tag,
 638                         atomic_read(&cmd->response_queue_count));
 639         }
 640 }
 641
 642 bool iscsit_conn_all_queues_empty(struct iscsi_conn *conn)
 643 {
 644         bool empty;
 645
 646         spin_lock_bh(&conn->immed_queue_lock);
 647         empty = list_empty(&conn->immed_queue_list);
 648         spin_unlock_bh(&conn->immed_queue_lock);
 649
 650         if (!empty)
 651                 return empty;
 652
 653         spin_lock_bh(&conn->response_queue_lock);
 654         empty = list_empty(&conn->response_queue_list);
 655         spin_unlock_bh(&conn->response_queue_lock);
 656
 657         return empty;
 658 }
 659
 660 void iscsit_free_queue_reqs_for_conn(struct iscsi_conn *conn)
 661 {
 662         struct iscsi_queue_req *qr, *qr_tmp;
 663
 664         spin_lock_bh(&conn->immed_queue_lock);
 665         list_for_each_entry_safe(qr, qr_tmp, &conn->immed_queue_list, qr_list) {
 666                 list_del(&qr->qr_list);
 667                 if (qr->cmd)
 668                         atomic_dec(&qr->cmd->immed_queue_count);
 669
 670                 kmem_cache_free(lio_qr_cache, qr);
 671         }
 672         spin_unlock_bh(&conn->immed_queue_lock);
 673
 674         spin_lock_bh(&conn->response_queue_lock);
 675         list_for_each_entry_safe(qr, qr_tmp, &conn->response_queue_list,
 676                         qr_list) {
 677                 list_del(&qr->qr_list);
 678                 if (qr->cmd)
 679                         atomic_dec(&qr->cmd->response_queue_count);
 680
 681                 kmem_cache_free(lio_qr_cache, qr);
 682         }
 683         spin_unlock_bh(&conn->response_queue_lock);
 684 }
 685
 686 void iscsit_release_cmd(struct iscsi_cmd *cmd)
 687 {
 688         struct iscsi_session *sess;
 689         struct se_cmd *se_cmd = &cmd->se_cmd;
 690
 691         if (cmd->conn)
 692                 sess = cmd->conn->sess;
 693         else
 694                 sess = cmd->sess;
 695
 696         BUG_ON(!sess || !sess->se_sess);
 697
 698         kfree(cmd->buf_ptr);
 699         kfree(cmd->pdu_list);
 700         kfree(cmd->seq_list);
 701         kfree(cmd->tmr_req);
 702         kfree(cmd->iov_data);
 703         kfree(cmd->text_in_ptr);
 704
 705         percpu_ida_free(&sess->se_sess->sess_tag_pool, se_cmd->map_tag);
 706 }
 707 EXPORT_SYMBOL(iscsit_release_cmd);
 708
 709 void __iscsit_free_cmd(struct iscsi_cmd *cmd, bool scsi_cmd,
 710                        bool check_queues)
 711 {
 712         struct iscsi_conn *conn = cmd->conn;
 713
 714         if (scsi_cmd) {
 715                 if (cmd->data_direction == DMA_TO_DEVICE) {
 716                         iscsit_stop_dataout_timer(cmd);
 717                         iscsit_free_r2ts_from_list(cmd);
 718                 }
 719                 if (cmd->data_direction == DMA_FROM_DEVICE)
 720                         iscsit_free_all_datain_reqs(cmd);
 721         }
 722
 723         if (conn && check_queues) {
 724                 iscsit_remove_cmd_from_immediate_queue(cmd, conn);
 725                 iscsit_remove_cmd_from_response_queue(cmd, conn);
 726         }
 727 }
 728
 729 void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
 730 {
 731         struct se_cmd *se_cmd = NULL;
 732         int rc;
 733         /*
 734          * Determine if a struct se_cmd is associated with
 735          * this struct iscsi_cmd.
 736          */
 737         switch (cmd->iscsi_opcode) {
 738         case ISCSI_OP_SCSI_CMD:
 739                 se_cmd = &cmd->se_cmd;
 740                 __iscsit_free_cmd(cmd, true, shutdown);
 741                 /*
 742                  * Fallthrough
 743                  */
 744         case ISCSI_OP_SCSI_TMFUNC:
 745                 rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
 746                 if (!rc && shutdown && se_cmd && se_cmd->se_sess) {
 747                         __iscsit_free_cmd(cmd, true, shutdown);
 748                         target_put_sess_cmd(se_cmd);
 749                 }
 750                 break;
 751         case ISCSI_OP_REJECT:
 752                 /*
 753                  * Handle special case for REJECT when iscsi_add_reject*() has
 754                  * overwritten the original iscsi_opcode assignment, and the
 755                  * associated cmd->se_cmd needs to be released.
 756                  */
 757                 if (cmd->se_cmd.se_tfo != NULL) {
 758                         se_cmd = &cmd->se_cmd;
 759                         __iscsit_free_cmd(cmd, true, shutdown);
 760
 761                         rc = transport_generic_free_cmd(&cmd->se_cmd, shutdown);
 762                         if (!rc && shutdown && se_cmd->se_sess) {
 763                                 __iscsit_free_cmd(cmd, true, shutdown);
 764                                 target_put_sess_cmd(se_cmd);
 765                         }
 766                         break;
 767                 }
 768                 /* Fall-through */
 769         default:
 770                 __iscsit_free_cmd(cmd, false, shutdown);
 771                 iscsit_release_cmd(cmd);
 772                 break;
 773         }
 774 }
 775
 776 int iscsit_check_session_usage_count(struct iscsi_session *sess)
 777 {
 778         spin_lock_bh(&sess->session_usage_lock);
 779         if (sess->session_usage_count != 0) {
 780                 sess->session_waiting_on_uc = 1;
 781                 spin_unlock_bh(&sess->session_usage_lock);
 782                 if (in_interrupt())
 783                         return 2;
 784
 785                 wait_for_completion(&sess->session_waiting_on_uc_comp);
 786                 return 1;
 787         }
 788         spin_unlock_bh(&sess->session_usage_lock);
 789
 790         return 0;
 791 }
 792
 793 void iscsit_dec_session_usage_count(struct iscsi_session *sess)
 794 {
 795         spin_lock_bh(&sess->session_usage_lock);
 796         sess->session_usage_count--;
 797
 798         if (!sess->session_usage_count && sess->session_waiting_on_uc)
 799                 complete(&sess->session_waiting_on_uc_comp);
 800
 801         spin_unlock_bh(&sess->session_usage_lock);
 802 }
 803
 804 void iscsit_inc_session_usage_count(struct iscsi_session *sess)
 805 {
 806         spin_lock_bh(&sess->session_usage_lock);
 807         sess->session_usage_count++;
 808         spin_unlock_bh(&sess->session_usage_lock);
 809 }
 810
 811 struct iscsi_conn *iscsit_get_conn_from_cid(struct iscsi_session *sess, u16 cid)
 812 {
 813         struct iscsi_conn *conn;
 814
 815         spin_lock_bh(&sess->conn_lock);
 816         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 817                 if ((conn->cid == cid) &&
 818                     (conn->conn_state == TARG_CONN_STATE_LOGGED_IN)) {
 819                         iscsit_inc_conn_usage_count(conn);
 820                         spin_unlock_bh(&sess->conn_lock);
 821                         return conn;
 822                 }
 823         }
 824         spin_unlock_bh(&sess->conn_lock);
 825
 826         return NULL;
 827 }
 828
 829 struct iscsi_conn *iscsit_get_conn_from_cid_rcfr(struct iscsi_session *sess, u16 cid)
 830 {
 831         struct iscsi_conn *conn;
 832
 833         spin_lock_bh(&sess->conn_lock);
 834         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
 835                 if (conn->cid == cid) {
 836                         iscsit_inc_conn_usage_count(conn);
 837                         spin_lock(&conn->state_lock);
 838                         atomic_set(&conn->connection_wait_rcfr, 1);
 839                         spin_unlock(&conn->state_lock);
 840                         spin_unlock_bh(&sess->conn_lock);
 841                         return conn;
 842                 }
 843         }
 844         spin_unlock_bh(&sess->conn_lock);
 845
 846         return NULL;
 847 }
 848
 849 void iscsit_check_conn_usage_count(struct iscsi_conn *conn)
 850 {
 851         spin_lock_bh(&conn->conn_usage_lock);
 852         if (conn->conn_usage_count != 0) {
 853                 conn->conn_waiting_on_uc = 1;
 854                 spin_unlock_bh(&conn->conn_usage_lock);
 855
 856                 wait_for_completion(&conn->conn_waiting_on_uc_comp);
 857                 return;
 858         }
 859         spin_unlock_bh(&conn->conn_usage_lock);
 860 }
 861
 862 void iscsit_dec_conn_usage_count(struct iscsi_conn *conn)
 863 {
 864         spin_lock_bh(&conn->conn_usage_lock);
 865         conn->conn_usage_count--;
 866
 867         if (!conn->conn_usage_count && conn->conn_waiting_on_uc)
 868                 complete(&conn->conn_waiting_on_uc_comp);
 869
 870         spin_unlock_bh(&conn->conn_usage_lock);
 871 }
 872
 873 void iscsit_inc_conn_usage_count(struct iscsi_conn *conn)
 874 {
 875         spin_lock_bh(&conn->conn_usage_lock);
 876         conn->conn_usage_count++;
 877         spin_unlock_bh(&conn->conn_usage_lock);
 878 }
 879
 880 static int iscsit_add_nopin(struct iscsi_conn *conn, int want_response)
 881 {
 882         u8 state;
 883         struct iscsi_cmd *cmd;
 884
 885         cmd = iscsit_allocate_cmd(conn, TASK_RUNNING);
 886         if (!cmd)
 887                 return -1;
 888
 889         cmd->iscsi_opcode = ISCSI_OP_NOOP_IN;
 890         state = (want_response) ? ISTATE_SEND_NOPIN_WANT_RESPONSE :
 891                                 ISTATE_SEND_NOPIN_NO_RESPONSE;
 892         cmd->init_task_tag = RESERVED_ITT;
 893         cmd->targ_xfer_tag = (want_response) ?
 894                              session_get_next_ttt(conn->sess) : 0xFFFFFFFF;
 895         spin_lock_bh(&conn->cmd_lock);
 896         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 897         spin_unlock_bh(&conn->cmd_lock);
 898
 899         if (want_response)
 900                 iscsit_start_nopin_response_timer(conn);
 901         iscsit_add_cmd_to_immediate_queue(cmd, conn, state);
 902
 903         return 0;
 904 }
 905
 906 static void iscsit_handle_nopin_response_timeout(unsigned long data)
 907 {
 908         struct iscsi_conn *conn = (struct iscsi_conn *) data;
 909
 910         iscsit_inc_conn_usage_count(conn);
 911
 912         spin_lock_bh(&conn->nopin_timer_lock);
 913         if (conn->nopin_response_timer_flags & ISCSI_TF_STOP) {
 914                 spin_unlock_bh(&conn->nopin_timer_lock);
 915                 iscsit_dec_conn_usage_count(conn);
 916                 return;
 917         }
 918
 919         pr_debug("Did not receive response to NOPIN on CID: %hu on"
 920                 " SID: %u, failing connection.\n", conn->cid,
 921                         conn->sess->sid);
 922         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
 923         spin_unlock_bh(&conn->nopin_timer_lock);
 924
 925         {
 926         struct iscsi_portal_group *tpg = conn->sess->tpg;
 927         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 928
 929         if (tiqn) {
 930                 spin_lock_bh(&tiqn->sess_err_stats.lock);
 931                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 932                                 conn->sess->sess_ops->InitiatorName);
 933                 tiqn->sess_err_stats.last_sess_failure_type =
 934                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 935                 tiqn->sess_err_stats.cxn_timeout_errors++;
 936                 atomic_long_inc(&conn->sess->conn_timeout_errors);
 937                 spin_unlock_bh(&tiqn->sess_err_stats.lock);
 938         }
 939         }
 940
 941         iscsit_cause_connection_reinstatement(conn, 0);
 942         iscsit_dec_conn_usage_count(conn);
 943 }
 944
 945 void iscsit_mod_nopin_response_timer(struct iscsi_conn *conn)
 946 {
 947         struct iscsi_session *sess = conn->sess;
 948         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
 949
 950         spin_lock_bh(&conn->nopin_timer_lock);
 951         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
 952                 spin_unlock_bh(&conn->nopin_timer_lock);
 953                 return;
 954         }
 955
 956         mod_timer(&conn->nopin_response_timer,
 957                 (get_jiffies_64() + na->nopin_response_timeout * HZ));
 958         spin_unlock_bh(&conn->nopin_timer_lock);
 959 }
 960
 961 /*
 962  *      Called with conn->nopin_timer_lock held.
 963  */
 964 void iscsit_start_nopin_response_timer(struct iscsi_conn *conn)
 965 {
 966         struct iscsi_session *sess = conn->sess;
 967         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
 968
 969         spin_lock_bh(&conn->nopin_timer_lock);
 970         if (conn->nopin_response_timer_flags & ISCSI_TF_RUNNING) {
 971                 spin_unlock_bh(&conn->nopin_timer_lock);
 972                 return;
 973         }
 974
 975         init_timer(&conn->nopin_response_timer);
 976         conn->nopin_response_timer.expires =
 977                 (get_jiffies_64() + na->nopin_response_timeout * HZ);
 978         conn->nopin_response_timer.data = (unsigned long)conn;
 979         conn->nopin_response_timer.function = iscsit_handle_nopin_response_timeout;
 980         conn->nopin_response_timer_flags &= ~ISCSI_TF_STOP;
 981         conn->nopin_response_timer_flags |= ISCSI_TF_RUNNING;
 982         add_timer(&conn->nopin_response_timer);
 983
 984         pr_debug("Started NOPIN Response Timer on CID: %d to %u"
 985                 " seconds\n", conn->cid, na->nopin_response_timeout);
 986         spin_unlock_bh(&conn->nopin_timer_lock);
 987 }
 988
 989 void iscsit_stop_nopin_response_timer(struct iscsi_conn *conn)
 990 {
 991         spin_lock_bh(&conn->nopin_timer_lock);
 992         if (!(conn->nopin_response_timer_flags & ISCSI_TF_RUNNING)) {
 993                 spin_unlock_bh(&conn->nopin_timer_lock);
 994                 return;
 995         }
 996         conn->nopin_response_timer_flags |= ISCSI_TF_STOP;
 997         spin_unlock_bh(&conn->nopin_timer_lock);
 998
 999         del_timer_sync(&conn->nopin_response_timer);
1000
1001         spin_lock_bh(&conn->nopin_timer_lock);
1002         conn->nopin_response_timer_flags &= ~ISCSI_TF_RUNNING;
1003         spin_unlock_bh(&conn->nopin_timer_lock);
1004 }
1005
1006 static void iscsit_handle_nopin_timeout(unsigned long data)
1007 {
1008         struct iscsi_conn *conn = (struct iscsi_conn *) data;
1009
1010         iscsit_inc_conn_usage_count(conn);
1011
1012         spin_lock_bh(&conn->nopin_timer_lock);
1013         if (conn->nopin_timer_flags & ISCSI_TF_STOP) {
1014                 spin_unlock_bh(&conn->nopin_timer_lock);
1015                 iscsit_dec_conn_usage_count(conn);
1016                 return;
1017         }
1018         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1019         spin_unlock_bh(&conn->nopin_timer_lock);
1020
1021         iscsit_add_nopin(conn, 1);
1022         iscsit_dec_conn_usage_count(conn);
1023 }
1024
1025 /*
1026  * Called with conn->nopin_timer_lock held.
1027  */
1028 void __iscsit_start_nopin_timer(struct iscsi_conn *conn)
1029 {
1030         struct iscsi_session *sess = conn->sess;
1031         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1032         /*
1033         * NOPIN timeout is disabled.
1034          */
1035         if (!na->nopin_timeout)
1036                 return;
1037
1038         if (conn->nopin_timer_flags & ISCSI_TF_RUNNING)
1039                 return;
1040
1041         init_timer(&conn->nopin_timer);
1042         conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1043         conn->nopin_timer.data = (unsigned long)conn;
1044         conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1045         conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1046         conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1047         add_timer(&conn->nopin_timer);
1048
1049         pr_debug("Started NOPIN Timer on CID: %d at %u second"
1050                 " interval\n", conn->cid, na->nopin_timeout);
1051 }
1052
1053 void iscsit_start_nopin_timer(struct iscsi_conn *conn)
1054 {
1055         struct iscsi_session *sess = conn->sess;
1056         struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1057         /*
1058          * NOPIN timeout is disabled..
1059          */
1060         if (!na->nopin_timeout)
1061                 return;
1062
1063         spin_lock_bh(&conn->nopin_timer_lock);
1064         if (conn->nopin_timer_flags & ISCSI_TF_RUNNING) {
1065                 spin_unlock_bh(&conn->nopin_timer_lock);
1066                 return;
1067         }
1068
1069         init_timer(&conn->nopin_timer);
1070         conn->nopin_timer.expires = (get_jiffies_64() + na->nopin_timeout * HZ);
1071         conn->nopin_timer.data = (unsigned long)conn;
1072         conn->nopin_timer.function = iscsit_handle_nopin_timeout;
1073         conn->nopin_timer_flags &= ~ISCSI_TF_STOP;
1074         conn->nopin_timer_flags |= ISCSI_TF_RUNNING;
1075         add_timer(&conn->nopin_timer);
1076
1077         pr_debug("Started NOPIN Timer on CID: %d at %u second"
1078                         " interval\n", conn->cid, na->nopin_timeout);
1079         spin_unlock_bh(&conn->nopin_timer_lock);
1080 }
1081
1082 void iscsit_stop_nopin_timer(struct iscsi_conn *conn)
1083 {
1084         spin_lock_bh(&conn->nopin_timer_lock);
1085         if (!(conn->nopin_timer_flags & ISCSI_TF_RUNNING)) {
1086                 spin_unlock_bh(&conn->nopin_timer_lock);
1087                 return;
1088         }
1089         conn->nopin_timer_flags |= ISCSI_TF_STOP;
1090         spin_unlock_bh(&conn->nopin_timer_lock);
1091
1092         del_timer_sync(&conn->nopin_timer);
1093
1094         spin_lock_bh(&conn->nopin_timer_lock);
1095         conn->nopin_timer_flags &= ~ISCSI_TF_RUNNING;
1096         spin_unlock_bh(&conn->nopin_timer_lock);
1097 }
1098
1099 int iscsit_send_tx_data(
1100         struct iscsi_cmd *cmd,
1101         struct iscsi_conn *conn,
1102         int use_misc)
1103 {
1104         int tx_sent, tx_size;
1105         u32 iov_count;
1106         struct kvec *iov;
1107
1108 send_data:
1109         tx_size = cmd->tx_size;
1110
1111         if (!use_misc) {
1112                 iov = &cmd->iov_data[0];
1113                 iov_count = cmd->iov_data_count;
1114         } else {
1115                 iov = &cmd->iov_misc[0];
1116                 iov_count = cmd->iov_misc_count;
1117         }
1118
1119         tx_sent = tx_data(conn, &iov[0], iov_count, tx_size);
1120         if (tx_size != tx_sent) {
1121                 if (tx_sent == -EAGAIN) {
1122                         pr_err("tx_data() returned -EAGAIN\n");
1123                         goto send_data;
1124                 } else
1125                         return -1;
1126         }
1127         cmd->tx_size = 0;
1128
1129         return 0;
1130 }
1131
1132 int iscsit_fe_sendpage_sg(
1133         struct iscsi_cmd *cmd,
1134         struct iscsi_conn *conn)
1135 {
1136         struct scatterlist *sg = cmd->first_data_sg;
1137         struct kvec iov;
1138         u32 tx_hdr_size, data_len;
1139         u32 offset = cmd->first_data_sg_off;
1140         int tx_sent, iov_off;
1141
1142 send_hdr:
1143         tx_hdr_size = ISCSI_HDR_LEN;
1144         if (conn->conn_ops->HeaderDigest)
1145                 tx_hdr_size += ISCSI_CRC_LEN;
1146
1147         iov.iov_base = cmd->pdu;
1148         iov.iov_len = tx_hdr_size;
1149
1150         tx_sent = tx_data(conn, &iov, 1, tx_hdr_size);
1151         if (tx_hdr_size != tx_sent) {
1152                 if (tx_sent == -EAGAIN) {
1153                         pr_err("tx_data() returned -EAGAIN\n");
1154                         goto send_hdr;
1155                 }
1156                 return -1;
1157         }
1158
1159         data_len = cmd->tx_size - tx_hdr_size - cmd->padding;
1160         /*
1161          * Set iov_off used by padding and data digest tx_data() calls below
1162          * in order to determine proper offset into cmd->iov_data[]
1163          */
1164         if (conn->conn_ops->DataDigest) {
1165                 data_len -= ISCSI_CRC_LEN;
1166                 if (cmd->padding)
1167                         iov_off = (cmd->iov_data_count - 2);
1168                 else
1169                         iov_off = (cmd->iov_data_count - 1);
1170         } else {
1171                 iov_off = (cmd->iov_data_count - 1);
1172         }
1173         /*
1174          * Perform sendpage() for each page in the scatterlist
1175          */
1176         while (data_len) {
1177                 u32 space = (sg->length - offset);
1178                 u32 sub_len = min_t(u32, data_len, space);
1179 send_pg:
1180                 tx_sent = conn->sock->ops->sendpage(conn->sock,
1181                                         sg_page(sg), sg->offset + offset, sub_len, 0);
1182                 if (tx_sent != sub_len) {
1183                         if (tx_sent == -EAGAIN) {
1184                                 pr_err("tcp_sendpage() returned"
1185                                                 " -EAGAIN\n");
1186                                 goto send_pg;
1187                         }
1188
1189                         pr_err("tcp_sendpage() failure: %d\n",
1190                                         tx_sent);
1191                         return -1;
1192                 }
1193
1194                 data_len -= sub_len;
1195                 offset = 0;
1196                 sg = sg_next(sg);
1197         }
1198
1199 send_padding:
1200         if (cmd->padding) {
1201                 struct kvec *iov_p = &cmd->iov_data[iov_off++];
1202
1203                 tx_sent = tx_data(conn, iov_p, 1, cmd->padding);
1204                 if (cmd->padding != tx_sent) {
1205                         if (tx_sent == -EAGAIN) {
1206                                 pr_err("tx_data() returned -EAGAIN\n");
1207                                 goto send_padding;
1208                         }
1209                         return -1;
1210                 }
1211         }
1212
1213 send_datacrc:
1214         if (conn->conn_ops->DataDigest) {
1215                 struct kvec *iov_d = &cmd->iov_data[iov_off];
1216
1217                 tx_sent = tx_data(conn, iov_d, 1, ISCSI_CRC_LEN);
1218                 if (ISCSI_CRC_LEN != tx_sent) {
1219                         if (tx_sent == -EAGAIN) {
1220                                 pr_err("tx_data() returned -EAGAIN\n");
1221                                 goto send_datacrc;
1222                         }
1223                         return -1;
1224                 }
1225         }
1226
1227         return 0;
1228 }
1229
1230 /*
1231  *      This function is used for mainly sending a ISCSI_TARG_LOGIN_RSP PDU
1232  *      back to the Initiator when an expection condition occurs with the
1233  *      errors set in status_class and status_detail.
1234  *
1235  *      Parameters:     iSCSI Connection, Status Class, Status Detail.
1236  *      Returns:        0 on success, -1 on error.
1237  */
1238 int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail)
1239 {
1240         struct iscsi_login_rsp *hdr;
1241         struct iscsi_login *login = conn->conn_login;
1242
1243         login->login_failed = 1;
1244         iscsit_collect_login_stats(conn, status_class, status_detail);
1245
1246         memset(&login->rsp[0], 0, ISCSI_HDR_LEN);
1247
1248         hdr     = (struct iscsi_login_rsp *)&login->rsp[0];
1249         hdr->opcode             = ISCSI_OP_LOGIN_RSP;
1250         hdr->status_class       = status_class;
1251         hdr->status_detail      = status_detail;
1252         hdr->itt                = conn->login_itt;
1253
1254         return conn->conn_transport->iscsit_put_login_tx(conn, login, 0);
1255 }
1256
1257 void iscsit_print_session_params(struct iscsi_session *sess)
1258 {
1259         struct iscsi_conn *conn;
1260
1261         pr_debug("-----------------------------[Session Params for"
1262                 " SID: %u]-----------------------------\n", sess->sid);
1263         spin_lock_bh(&sess->conn_lock);
1264         list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
1265                 iscsi_dump_conn_ops(conn->conn_ops);
1266         spin_unlock_bh(&sess->conn_lock);
1267
1268         iscsi_dump_sess_ops(sess->sess_ops);
1269 }
1270
1271 static int iscsit_do_rx_data(
1272         struct iscsi_conn *conn,
1273         struct iscsi_data_count *count)
1274 {
1275         int data = count->data_length, rx_loop = 0, total_rx = 0;
1276         struct msghdr msg;
1277
1278         if (!conn || !conn->sock || !conn->conn_ops)
1279                 return -1;
1280
1281         memset(&msg, 0, sizeof(struct msghdr));
1282         iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC,
1283                       count->iov, count->iov_count, data);
1284
1285         while (total_rx < data) {
1286                 rx_loop = sock_recvmsg(conn->sock, &msg,
1287                                       (data - total_rx), MSG_WAITALL);
1288                 if (rx_loop <= 0) {
1289                         pr_debug("rx_loop: %d total_rx: %d\n",
1290                                 rx_loop, total_rx);
1291                         return rx_loop;
1292                 }
1293                 total_rx += rx_loop;
1294                 pr_debug("rx_loop: %d, total_rx: %d, data: %d\n",
1295                                 rx_loop, total_rx, data);
1296         }
1297
1298         return total_rx;
1299 }
1300
1301 static int iscsit_do_tx_data(
1302         struct iscsi_conn *conn,
1303         struct iscsi_data_count *count)
1304 {
1305         int ret, iov_len;
1306         struct kvec *iov_p;
1307         struct msghdr msg;
1308
1309         if (!conn || !conn->sock || !conn->conn_ops)
1310                 return -1;
1311
1312         if (count->data_length <= 0) {
1313                 pr_err("Data length is: %d\n", count->data_length);
1314                 return -1;
1315         }
1316
1317         memset(&msg, 0, sizeof(struct msghdr));
1318
1319         iov_p = count->iov;
1320         iov_len = count->iov_count;
1321
1322         ret = kernel_sendmsg(conn->sock, &msg, iov_p, iov_len,
1323                              count->data_length);
1324         if (ret != count->data_length) {
1325                 pr_err("Unexpected ret: %d send data %d\n",
1326                        ret, count->data_length);
1327                 return -EPIPE;
1328         }
1329         pr_debug("ret: %d, sent data: %d\n", ret, count->data_length);
1330
1331         return ret;
1332 }
1333
1334 int rx_data(
1335         struct iscsi_conn *conn,
1336         struct kvec *iov,
1337         int iov_count,
1338         int data)
1339 {
1340         struct iscsi_data_count c;
1341
1342         if (!conn || !conn->sock || !conn->conn_ops)
1343                 return -1;
1344
1345         memset(&c, 0, sizeof(struct iscsi_data_count));
1346         c.iov = iov;
1347         c.iov_count = iov_count;
1348         c.data_length = data;
1349         c.type = ISCSI_RX_DATA;
1350
1351         return iscsit_do_rx_data(conn, &c);
1352 }
1353
1354 int tx_data(
1355         struct iscsi_conn *conn,
1356         struct kvec *iov,
1357         int iov_count,
1358         int data)
1359 {
1360         struct iscsi_data_count c;
1361
1362         if (!conn || !conn->sock || !conn->conn_ops)
1363                 return -1;
1364
1365         memset(&c, 0, sizeof(struct iscsi_data_count));
1366         c.iov = iov;
1367         c.iov_count = iov_count;
1368         c.data_length = data;
1369         c.type = ISCSI_TX_DATA;
1370
1371         return iscsit_do_tx_data(conn, &c);
1372 }
1373
1374 void iscsit_collect_login_stats(
1375         struct iscsi_conn *conn,
1376         u8 status_class,
1377         u8 status_detail)
1378 {
1379         struct iscsi_param *intrname = NULL;
1380         struct iscsi_tiqn *tiqn;
1381         struct iscsi_login_stats *ls;
1382
1383         tiqn = iscsit_snmp_get_tiqn(conn);
1384         if (!tiqn)
1385                 return;
1386
1387         ls = &tiqn->login_stats;
1388
1389         spin_lock(&ls->lock);
1390         if (!strcmp(conn->login_ip, ls->last_intr_fail_ip_addr) &&
1391             ((get_jiffies_64() - ls->last_fail_time) < 10)) {
1392                 /* We already have the failure info for this login */
1393                 spin_unlock(&ls->lock);
1394                 return;
1395         }
1396
1397         if (status_class == ISCSI_STATUS_CLS_SUCCESS)
1398                 ls->accepts++;
1399         else if (status_class == ISCSI_STATUS_CLS_REDIRECT) {
1400                 ls->redirects++;
1401                 ls->last_fail_type = ISCSI_LOGIN_FAIL_REDIRECT;
1402         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1403                  (status_detail == ISCSI_LOGIN_STATUS_AUTH_FAILED)) {
1404                 ls->authenticate_fails++;
1405                 ls->last_fail_type =  ISCSI_LOGIN_FAIL_AUTHENTICATE;
1406         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR)  &&
1407                  (status_detail == ISCSI_LOGIN_STATUS_TGT_FORBIDDEN)) {
1408                 ls->authorize_fails++;
1409                 ls->last_fail_type = ISCSI_LOGIN_FAIL_AUTHORIZE;
1410         } else if ((status_class == ISCSI_STATUS_CLS_INITIATOR_ERR) &&
1411                  (status_detail == ISCSI_LOGIN_STATUS_INIT_ERR)) {
1412                 ls->negotiate_fails++;
1413                 ls->last_fail_type = ISCSI_LOGIN_FAIL_NEGOTIATE;
1414         } else {
1415                 ls->other_fails++;
1416                 ls->last_fail_type = ISCSI_LOGIN_FAIL_OTHER;
1417         }
1418
1419         /* Save initiator name, ip address and time, if it is a failed login */
1420         if (status_class != ISCSI_STATUS_CLS_SUCCESS) {
1421                 if (conn->param_list)
1422                         intrname = iscsi_find_param_from_key(INITIATORNAME,
1423                                                              conn->param_list);
1424                 strlcpy(ls->last_intr_fail_name,
1425                        (intrname ? intrname->value : "Unknown"),
1426                        sizeof(ls->last_intr_fail_name));
1427
1428                 ls->last_intr_fail_ip_family = conn->login_family;
1429
1430                 snprintf(ls->last_intr_fail_ip_addr, IPV6_ADDRESS_SPACE,
1431                                 "%s", conn->login_ip);
1432                 ls->last_fail_time = get_jiffies_64();
1433         }
1434
1435         spin_unlock(&ls->lock);
1436 }
1437
1438 struct iscsi_tiqn *iscsit_snmp_get_tiqn(struct iscsi_conn *conn)
1439 {
1440         struct iscsi_portal_group *tpg;
1441
1442         if (!conn || !conn->sess)
1443                 return NULL;
1444
1445         tpg = conn->sess->tpg;
1446         if (!tpg)
1447                 return NULL;
1448
1449         if (!tpg->tpg_tiqn)
1450                 return NULL;
1451
1452         return tpg->tpg_tiqn;
1453 }