libceph: don't time out osd requests that haven't been received

Keep track of when an outgoing message is ACKed (i.e., the server fully
received it and, presumably, queued it for processing).  Time out OSD
requests only if it's been too long since they've been received.

This prevents timeouts and connection thrashing when the OSDs are simply
busy and are throttling the requests they read off the network.

Reviewed-by: Yehuda Sadeh <yehuda@hq.newdream.net>
Signed-off-by: Sage Weil <sage@newdream.net>

diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index 31d91a64838be0326ee3d8d76032f67181008dbc..d7adf151d3351c4fb315952231c7768a6e6c9cf7 100644 (file)
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -94,6 +94,7 @@ struct ceph_msg {
        bool more_to_follow;
        bool needs_out_seq;
        int front_max;
+       unsigned long ack_stamp;        /* tx: when we were acked */
 
        struct ceph_msgpool *pool;
 };

diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 78b55f49de7cba5bbdf0fb872b4b3c0464190f04..c340e2e0765b4bdf72efc9f2bd484bc7f7f36de5 100644 (file)
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -486,13 +486,10 @@ static void prepare_write_message(struct ceph_connection *con)
        m = list_first_entry(&con->out_queue,
                       struct ceph_msg, list_head);
        con->out_msg = m;
-       if (test_bit(LOSSYTX, &con->state)) {
-               list_del_init(&m->list_head);
-       } else {
-               /* put message on sent list */
-               ceph_msg_get(m);
-               list_move_tail(&m->list_head, &con->out_sent);
-       }
+
+       /* put message on sent list */
+       ceph_msg_get(m);
+       list_move_tail(&m->list_head, &con->out_sent);
 
        /*
         * only assign outgoing seq # if we haven't sent this message
@@ -1399,6 +1396,7 @@ static void process_ack(struct ceph_connection *con)
                        break;
                dout("got ack for seq %llu type %d at %p\n", seq,
                     le16_to_cpu(m->hdr.type), m);
+               m->ack_stamp = jiffies;
                ceph_msg_remove(m);
        }
        prepare_read_tag(con);

diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 7330c2757c0c23c6cf0f11b1331d59de82983690..ce310eee708d9f76c0a631b32edb6eb046cff153 100644 (file)
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1085,9 +1085,15 @@ static void handle_timeout(struct work_struct *work)
                req = list_entry(osdc->req_lru.next, struct ceph_osd_request,
                                 r_req_lru_item);
 
+               /* hasn't been long enough since we sent it? */
                if (time_before(jiffies, req->r_stamp + timeout))
                        break;
 
+               /* hasn't been long enough since it was acked? */
+               if (req->r_request->ack_stamp == 0 ||
+                   time_before(jiffies, req->r_request->ack_stamp + timeout))
+                       break;
+
                BUG_ON(req == last_req && req->r_stamp == last_stamp);
                last_req = req;
                last_stamp = req->r_stamp;