2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2013 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
36 #include <linux/scatterlist.h>
37 #include <linux/kfifo.h>
38 #include <scsi/scsi_cmnd.h>
39 #include <scsi/scsi_host.h>
41 #include "iscsi_iser.h"
43 /* Register user buffer memory and initialize passive rdma
44 * dto descriptor. Total data size is stored in
45 * iser_task->data[ISER_DIR_IN].data_len
47 static int iser_prepare_read_cmd(struct iscsi_task *task,
51 struct iscsi_iser_task *iser_task = task->dd_data;
52 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
53 struct iser_regd_buf *regd_buf;
55 struct iser_hdr *hdr = &iser_task->desc.iser_header;
56 struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN];
58 err = iser_dma_map_task_data(iser_task,
65 if (edtl > iser_task->data[ISER_DIR_IN].data_len) {
66 iser_err("Total data length: %ld, less than EDTL: "
67 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
68 iser_task->data[ISER_DIR_IN].data_len, edtl,
69 task->itt, iser_task->iser_conn);
73 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
75 iser_err("Failed to set up Data-IN RDMA\n");
78 regd_buf = &iser_task->rdma_regd[ISER_DIR_IN];
80 hdr->flags |= ISER_RSV;
81 hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey);
82 hdr->read_va = cpu_to_be64(regd_buf->reg.va);
84 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
85 task->itt, regd_buf->reg.rkey,
86 (unsigned long long)regd_buf->reg.va);
91 /* Register user buffer memory and initialize passive rdma
92 * dto descriptor. Total data size is stored in
93 * task->data[ISER_DIR_OUT].data_len
96 iser_prepare_write_cmd(struct iscsi_task *task,
98 unsigned int unsol_sz,
101 struct iscsi_iser_task *iser_task = task->dd_data;
102 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
103 struct iser_regd_buf *regd_buf;
105 struct iser_hdr *hdr = &iser_task->desc.iser_header;
106 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT];
107 struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1];
109 err = iser_dma_map_task_data(iser_task,
116 if (edtl > iser_task->data[ISER_DIR_OUT].data_len) {
117 iser_err("Total data length: %ld, less than EDTL: %d, "
118 "in WRITE cmd BHS itt: %d, conn: 0x%p\n",
119 iser_task->data[ISER_DIR_OUT].data_len,
120 edtl, task->itt, task->conn);
124 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
126 iser_err("Failed to register write cmd RDMA mem\n");
130 regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT];
132 if (unsol_sz < edtl) {
133 hdr->flags |= ISER_WSV;
134 hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey);
135 hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz);
137 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
138 "VA:%#llX + unsol:%d\n",
139 task->itt, regd_buf->reg.rkey,
140 (unsigned long long)regd_buf->reg.va, unsol_sz);
144 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
146 tx_dsg->addr = regd_buf->reg.va;
147 tx_dsg->length = imm_sz;
148 tx_dsg->lkey = regd_buf->reg.lkey;
149 iser_task->desc.num_sge = 2;
155 /* creates a new tx descriptor and adds header regd buffer */
156 static void iser_create_send_desc(struct iser_conn *ib_conn,
157 struct iser_tx_desc *tx_desc)
159 struct iser_device *device = ib_conn->device;
161 ib_dma_sync_single_for_cpu(device->ib_device,
162 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
164 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
165 tx_desc->iser_header.flags = ISER_VER;
167 tx_desc->num_sge = 1;
169 if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
170 tx_desc->tx_sg[0].lkey = device->mr->lkey;
171 iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc);
175 static void iser_free_login_buf(struct iser_conn *ib_conn)
177 if (!ib_conn->login_buf)
180 if (ib_conn->login_req_dma)
181 ib_dma_unmap_single(ib_conn->device->ib_device,
182 ib_conn->login_req_dma,
183 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
185 if (ib_conn->login_resp_dma)
186 ib_dma_unmap_single(ib_conn->device->ib_device,
187 ib_conn->login_resp_dma,
188 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
190 kfree(ib_conn->login_buf);
192 /* make sure we never redo any unmapping */
193 ib_conn->login_req_dma = 0;
194 ib_conn->login_resp_dma = 0;
195 ib_conn->login_buf = NULL;
198 static int iser_alloc_login_buf(struct iser_conn *ib_conn)
200 struct iser_device *device;
201 int req_err, resp_err;
203 BUG_ON(ib_conn->device == NULL);
205 device = ib_conn->device;
207 ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
208 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
209 if (!ib_conn->login_buf)
212 ib_conn->login_req_buf = ib_conn->login_buf;
213 ib_conn->login_resp_buf = ib_conn->login_buf +
214 ISCSI_DEF_MAX_RECV_SEG_LEN;
216 ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device,
217 (void *)ib_conn->login_req_buf,
218 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
220 ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device,
221 (void *)ib_conn->login_resp_buf,
222 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
224 req_err = ib_dma_mapping_error(device->ib_device,
225 ib_conn->login_req_dma);
226 resp_err = ib_dma_mapping_error(device->ib_device,
227 ib_conn->login_resp_dma);
229 if (req_err || resp_err) {
231 ib_conn->login_req_dma = 0;
233 ib_conn->login_resp_dma = 0;
239 iser_free_login_buf(ib_conn);
242 iser_err("unable to alloc or map login buf\n");
246 int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session)
250 struct iser_rx_desc *rx_desc;
251 struct ib_sge *rx_sg;
252 struct iser_device *device = ib_conn->device;
254 ib_conn->qp_max_recv_dtos = session->cmds_max;
255 ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
256 ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2;
258 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max))
259 goto create_rdma_reg_res_failed;
261 if (iser_alloc_login_buf(ib_conn))
262 goto alloc_login_buf_fail;
264 ib_conn->rx_descs = kmalloc(session->cmds_max *
265 sizeof(struct iser_rx_desc), GFP_KERNEL);
266 if (!ib_conn->rx_descs)
267 goto rx_desc_alloc_fail;
269 rx_desc = ib_conn->rx_descs;
271 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) {
272 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
273 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
274 if (ib_dma_mapping_error(device->ib_device, dma_addr))
275 goto rx_desc_dma_map_failed;
277 rx_desc->dma_addr = dma_addr;
279 rx_sg = &rx_desc->rx_sg;
280 rx_sg->addr = rx_desc->dma_addr;
281 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
282 rx_sg->lkey = device->mr->lkey;
285 ib_conn->rx_desc_head = 0;
288 rx_desc_dma_map_failed:
289 rx_desc = ib_conn->rx_descs;
290 for (j = 0; j < i; j++, rx_desc++)
291 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
292 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
293 kfree(ib_conn->rx_descs);
294 ib_conn->rx_descs = NULL;
296 iser_free_login_buf(ib_conn);
297 alloc_login_buf_fail:
298 device->iser_free_rdma_reg_res(ib_conn);
299 create_rdma_reg_res_failed:
300 iser_err("failed allocating rx descriptors / data buffers\n");
304 void iser_free_rx_descriptors(struct iser_conn *ib_conn)
307 struct iser_rx_desc *rx_desc;
308 struct iser_device *device = ib_conn->device;
310 if (!ib_conn->rx_descs)
313 if (device->iser_free_rdma_reg_res)
314 device->iser_free_rdma_reg_res(ib_conn);
316 rx_desc = ib_conn->rx_descs;
317 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++)
318 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
319 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
320 kfree(ib_conn->rx_descs);
321 /* make sure we never redo any unmapping */
322 ib_conn->rx_descs = NULL;
325 iser_free_login_buf(ib_conn);
328 static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
330 struct iscsi_iser_conn *iser_conn = conn->dd_data;
332 iser_dbg("req op %x flags %x\n", req->opcode, req->flags);
333 /* check if this is the last login - going to full feature phase */
334 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE)
338 * Check that there is one posted recv buffer (for the last login
339 * response) and no posted send buffers left - they must have been
340 * consumed during previous login phases.
342 WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1);
343 WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0);
345 iser_dbg("Initially post: %d\n", iser_conn->ib_conn->min_posted_rx);
346 /* Initial post receive buffers */
347 if (iser_post_recvm(iser_conn->ib_conn,
348 iser_conn->ib_conn->min_posted_rx))
355 * iser_send_command - send command PDU
357 int iser_send_command(struct iscsi_conn *conn,
358 struct iscsi_task *task)
360 struct iscsi_iser_conn *iser_conn = conn->dd_data;
361 struct iscsi_iser_task *iser_task = task->dd_data;
364 struct iser_data_buf *data_buf;
365 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
366 struct scsi_cmnd *sc = task->sc;
367 struct iser_tx_desc *tx_desc = &iser_task->desc;
369 edtl = ntohl(hdr->data_length);
371 /* build the tx desc regd header and add it to the tx desc dto */
372 tx_desc->type = ISCSI_TX_SCSI_COMMAND;
373 iser_create_send_desc(iser_conn->ib_conn, tx_desc);
375 if (hdr->flags & ISCSI_FLAG_CMD_READ)
376 data_buf = &iser_task->data[ISER_DIR_IN];
378 data_buf = &iser_task->data[ISER_DIR_OUT];
380 if (scsi_sg_count(sc)) { /* using a scatter list */
381 data_buf->buf = scsi_sglist(sc);
382 data_buf->size = scsi_sg_count(sc);
385 data_buf->data_len = scsi_bufflen(sc);
387 if (hdr->flags & ISCSI_FLAG_CMD_READ) {
388 err = iser_prepare_read_cmd(task, edtl);
390 goto send_command_error;
392 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) {
393 err = iser_prepare_write_cmd(task,
396 task->unsol_r2t.data_length,
399 goto send_command_error;
402 iser_task->status = ISER_TASK_STATUS_STARTED;
404 err = iser_post_send(iser_conn->ib_conn, tx_desc);
409 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err);
414 * iser_send_data_out - send data out PDU
416 int iser_send_data_out(struct iscsi_conn *conn,
417 struct iscsi_task *task,
418 struct iscsi_data *hdr)
420 struct iscsi_iser_conn *iser_conn = conn->dd_data;
421 struct iscsi_iser_task *iser_task = task->dd_data;
422 struct iser_tx_desc *tx_desc = NULL;
423 struct iser_regd_buf *regd_buf;
424 unsigned long buf_offset;
425 unsigned long data_seg_len;
428 struct ib_sge *tx_dsg;
430 itt = (__force uint32_t)hdr->itt;
431 data_seg_len = ntoh24(hdr->dlength);
432 buf_offset = ntohl(hdr->offset);
434 iser_dbg("%s itt %d dseg_len %d offset %d\n",
435 __func__,(int)itt,(int)data_seg_len,(int)buf_offset);
437 tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC);
438 if (tx_desc == NULL) {
439 iser_err("Failed to alloc desc for post dataout\n");
443 tx_desc->type = ISCSI_TX_DATAOUT;
444 tx_desc->iser_header.flags = ISER_VER;
445 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr));
447 /* build the tx desc */
448 iser_initialize_task_headers(task, tx_desc);
450 regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT];
451 tx_dsg = &tx_desc->tx_sg[1];
452 tx_dsg->addr = regd_buf->reg.va + buf_offset;
453 tx_dsg->length = data_seg_len;
454 tx_dsg->lkey = regd_buf->reg.lkey;
455 tx_desc->num_sge = 2;
457 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) {
458 iser_err("Offset:%ld & DSL:%ld in Data-Out "
459 "inconsistent with total len:%ld, itt:%d\n",
460 buf_offset, data_seg_len,
461 iser_task->data[ISER_DIR_OUT].data_len, itt);
463 goto send_data_out_error;
465 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
466 itt, buf_offset, data_seg_len);
469 err = iser_post_send(iser_conn->ib_conn, tx_desc);
474 kmem_cache_free(ig.desc_cache, tx_desc);
475 iser_err("conn %p failed err %d\n",conn, err);
479 int iser_send_control(struct iscsi_conn *conn,
480 struct iscsi_task *task)
482 struct iscsi_iser_conn *iser_conn = conn->dd_data;
483 struct iscsi_iser_task *iser_task = task->dd_data;
484 struct iser_tx_desc *mdesc = &iser_task->desc;
485 unsigned long data_seg_len;
487 struct iser_device *device;
488 struct iser_conn *ib_conn = iser_conn->ib_conn;
490 /* build the tx desc regd header and add it to the tx desc dto */
491 mdesc->type = ISCSI_TX_CONTROL;
492 iser_create_send_desc(iser_conn->ib_conn, mdesc);
494 device = iser_conn->ib_conn->device;
496 data_seg_len = ntoh24(task->hdr->dlength);
498 if (data_seg_len > 0) {
499 struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
500 if (task != conn->login_task) {
501 iser_err("data present on non login task!!!\n");
502 goto send_control_error;
505 ib_dma_sync_single_for_cpu(device->ib_device,
506 ib_conn->login_req_dma, task->data_count,
509 memcpy(iser_conn->ib_conn->login_req_buf, task->data,
512 ib_dma_sync_single_for_device(device->ib_device,
513 ib_conn->login_req_dma, task->data_count,
516 tx_dsg->addr = iser_conn->ib_conn->login_req_dma;
517 tx_dsg->length = task->data_count;
518 tx_dsg->lkey = device->mr->lkey;
522 if (task == conn->login_task) {
523 err = iser_post_recvl(iser_conn->ib_conn);
525 goto send_control_error;
526 err = iser_post_rx_bufs(conn, task->hdr);
528 goto send_control_error;
531 err = iser_post_send(iser_conn->ib_conn, mdesc);
536 iser_err("conn %p failed err %d\n",conn, err);
541 * iser_rcv_dto_completion - recv DTO completion
543 void iser_rcv_completion(struct iser_rx_desc *rx_desc,
544 unsigned long rx_xfer_len,
545 struct iser_conn *ib_conn)
547 struct iscsi_iser_conn *conn = ib_conn->iser_conn;
548 struct iscsi_hdr *hdr;
550 int rx_buflen, outstanding, count, err;
552 /* differentiate between login to all other PDUs */
553 if ((char *)rx_desc == ib_conn->login_resp_buf) {
554 rx_dma = ib_conn->login_resp_dma;
555 rx_buflen = ISER_RX_LOGIN_SIZE;
557 rx_dma = rx_desc->dma_addr;
558 rx_buflen = ISER_RX_PAYLOAD_SIZE;
561 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
562 rx_buflen, DMA_FROM_DEVICE);
564 hdr = &rx_desc->iscsi_header;
566 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode,
567 hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN));
569 iscsi_iser_recv(conn->iscsi_conn, hdr,
570 rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN);
572 ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
573 rx_buflen, DMA_FROM_DEVICE);
575 /* decrementing conn->post_recv_buf_count only --after-- freeing the *
576 * task eliminates the need to worry on tasks which are completed in *
577 * parallel to the execution of iser_conn_term. So the code that waits *
578 * for the posted rx bufs refcount to become zero handles everything */
579 conn->ib_conn->post_recv_buf_count--;
581 if (rx_dma == ib_conn->login_resp_dma)
584 outstanding = ib_conn->post_recv_buf_count;
585 if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) {
586 count = min(ib_conn->qp_max_recv_dtos - outstanding,
587 ib_conn->min_posted_rx);
588 err = iser_post_recvm(ib_conn, count);
590 iser_err("posting %d rx bufs err %d\n", count, err);
594 void iser_snd_completion(struct iser_tx_desc *tx_desc,
595 struct iser_conn *ib_conn)
597 struct iscsi_task *task;
598 struct iser_device *device = ib_conn->device;
600 if (tx_desc->type == ISCSI_TX_DATAOUT) {
601 ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
602 ISER_HEADERS_LEN, DMA_TO_DEVICE);
603 kmem_cache_free(ig.desc_cache, tx_desc);
606 atomic_dec(&ib_conn->post_send_buf_count);
608 if (tx_desc->type == ISCSI_TX_CONTROL) {
609 /* this arithmetic is legal by libiscsi dd_data allocation */
610 task = (void *) ((long)(void *)tx_desc -
611 sizeof(struct iscsi_task));
612 if (task->hdr->itt == RESERVED_ITT)
613 iscsi_put_task(task);
617 void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
620 iser_task->status = ISER_TASK_STATUS_INIT;
622 iser_task->dir[ISER_DIR_IN] = 0;
623 iser_task->dir[ISER_DIR_OUT] = 0;
625 iser_task->data[ISER_DIR_IN].data_len = 0;
626 iser_task->data[ISER_DIR_OUT].data_len = 0;
628 memset(&iser_task->rdma_regd[ISER_DIR_IN], 0,
629 sizeof(struct iser_regd_buf));
630 memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0,
631 sizeof(struct iser_regd_buf));
634 void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
636 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
637 int is_rdma_aligned = 1;
639 /* if we were reading, copy back to unaligned sglist,
640 * anyway dma_unmap and free the copy
642 if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) {
644 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN);
646 if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) {
648 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT);
651 if (iser_task->dir[ISER_DIR_IN])
652 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
654 if (iser_task->dir[ISER_DIR_OUT])
655 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
657 /* if the data was unaligned, it was already unmapped and then copied */
659 iser_dma_unmap_task_data(iser_task);