From e5e372da9a469dfe3ece40277090a7056c566838 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 11:41:43 -0500 Subject: libceph: eliminate connection state "DEAD" The ceph connection state "DEAD" is never set and is therefore not needed. Eliminate it. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/messenger.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 2521a95fa6d..aa506cadea6 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -119,7 +119,6 @@ struct ceph_msg_pos { #define CLOSED 10 /* we've closed the connection */ #define SOCK_CLOSED 11 /* socket state changed to closed */ #define OPENING 13 /* open connection w/ (possibly new) peer */ -#define DEAD 14 /* dead, about to kfree */ #define BACKOFF 15 /* -- cgit v1.2.3-70-g09d2 From 6384bb8b8e88a9c6bf2ae0d9517c2c0199177c34 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 29 May 2012 21:47:38 -0500 Subject: libceph: kill bad_proto ceph connection op No code sets a bad_proto method in its ceph connection operations vector, so just get rid of it. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/messenger.h | 3 --- net/ceph/messenger.c | 5 ----- 2 files changed, 8 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index aa506cadea6..74f6c9bd807 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -31,9 +31,6 @@ struct ceph_connection_operations { int (*verify_authorizer_reply) (struct ceph_connection *con, int len); int (*invalidate_authorizer)(struct ceph_connection *con); - /* protocol version mismatch */ - void (*bad_proto) (struct ceph_connection *con); - /* there was some error on the socket (disconnect, whatever) */ void (*fault) (struct ceph_connection *con); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 42ca8aab6dc..07af9948e3f 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1356,11 +1356,6 @@ static void fail_protocol(struct ceph_connection *con) { reset_connection(con); set_bit(CLOSED, &con->state); /* in case there's queued work */ - - mutex_unlock(&con->mutex); - if (con->ops->bad_proto) - con->ops->bad_proto(con); - mutex_lock(&con->mutex); } static int process_connect(struct ceph_connection *con) -- cgit v1.2.3-70-g09d2 From 15d9882c336db2db73ccf9871ae2398e452f694c Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: embed ceph messenger structure in ceph_client A ceph client has a pointer to a ceph messenger structure in it. There is always exactly one ceph messenger for a ceph client, so there is no need to allocate it separate from the ceph client structure. Switch the ceph_client structure to embed its ceph_messenger structure. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh Reviewed-by: Sage Weil --- fs/ceph/mds_client.c | 2 +- include/linux/ceph/libceph.h | 2 +- include/linux/ceph/messenger.h | 9 +++++---- net/ceph/ceph_common.c | 18 +++++------------- net/ceph/messenger.c | 30 +++++++++--------------------- net/ceph/mon_client.c | 6 +++--- net/ceph/osd_client.c | 4 ++-- 7 files changed, 26 insertions(+), 45 deletions(-) (limited to 'include') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 200bc87eceb..ad30261cd4c 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -394,7 +394,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, s->s_seq = 0; mutex_init(&s->s_mutex); - ceph_con_init(mdsc->fsc->client->msgr, &s->s_con); + ceph_con_init(&mdsc->fsc->client->msgr, &s->s_con); s->s_con.private = s; s->s_con.ops = &mds_con_ops; s->s_con.peer_name.type = CEPH_ENTITY_TYPE_MDS; diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 92eef7c3d3c..927361c4b0a 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -131,7 +131,7 @@ struct ceph_client { u32 supported_features; u32 required_features; - struct ceph_messenger *msgr; /* messenger instance */ + struct ceph_messenger msgr; /* messenger instance */ struct ceph_mon_client monc; struct ceph_osd_client osdc; diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 74f6c9bd807..3fbd4be804e 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -211,10 +211,11 @@ extern int ceph_msgr_init(void); extern void ceph_msgr_exit(void); extern void ceph_msgr_flush(void); -extern struct ceph_messenger *ceph_messenger_create( - struct ceph_entity_addr *myaddr, - u32 features, u32 required); -extern void ceph_messenger_destroy(struct ceph_messenger *); +extern void ceph_messenger_init(struct ceph_messenger *msgr, + struct ceph_entity_addr *myaddr, + u32 supported_features, + u32 required_features, + bool nocrc); extern void ceph_con_init(struct ceph_messenger *msgr, struct ceph_connection *con); diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index cc913193d99..2de3ea1bbd6 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -468,19 +468,15 @@ struct ceph_client *ceph_create_client(struct ceph_options *opt, void *private, /* msgr */ if (ceph_test_opt(client, MYIP)) myaddr = &client->options->my_addr; - client->msgr = ceph_messenger_create(myaddr, - client->supported_features, - client->required_features); - if (IS_ERR(client->msgr)) { - err = PTR_ERR(client->msgr); - goto fail; - } - client->msgr->nocrc = ceph_test_opt(client, NOCRC); + ceph_messenger_init(&client->msgr, myaddr, + client->supported_features, + client->required_features, + ceph_test_opt(client, NOCRC)); /* subsystems */ err = ceph_monc_init(&client->monc, client); if (err < 0) - goto fail_msgr; + goto fail; err = ceph_osdc_init(&client->osdc, client); if (err < 0) goto fail_monc; @@ -489,8 +485,6 @@ struct ceph_client *ceph_create_client(struct ceph_options *opt, void *private, fail_monc: ceph_monc_stop(&client->monc); -fail_msgr: - ceph_messenger_destroy(client->msgr); fail: kfree(client); return ERR_PTR(err); @@ -515,8 +509,6 @@ void ceph_destroy_client(struct ceph_client *client) ceph_debugfs_client_cleanup(client); - ceph_messenger_destroy(client->msgr); - ceph_destroy_options(client->options); kfree(client); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 2ca491fc50e..d8423a3f669 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -2245,18 +2245,14 @@ out: /* - * create a new messenger instance + * initialize a new messenger instance */ -struct ceph_messenger *ceph_messenger_create(struct ceph_entity_addr *myaddr, - u32 supported_features, - u32 required_features) +void ceph_messenger_init(struct ceph_messenger *msgr, + struct ceph_entity_addr *myaddr, + u32 supported_features, + u32 required_features, + bool nocrc) { - struct ceph_messenger *msgr; - - msgr = kzalloc(sizeof(*msgr), GFP_KERNEL); - if (msgr == NULL) - return ERR_PTR(-ENOMEM); - msgr->supported_features = supported_features; msgr->required_features = required_features; @@ -2269,19 +2265,11 @@ struct ceph_messenger *ceph_messenger_create(struct ceph_entity_addr *myaddr, msgr->inst.addr.type = 0; get_random_bytes(&msgr->inst.addr.nonce, sizeof(msgr->inst.addr.nonce)); encode_my_addr(msgr); + msgr->nocrc = nocrc; - dout("messenger_create %p\n", msgr); - return msgr; -} -EXPORT_SYMBOL(ceph_messenger_create); - -void ceph_messenger_destroy(struct ceph_messenger *msgr) -{ - dout("destroy %p\n", msgr); - kfree(msgr); - dout("destroyed messenger %p\n", msgr); + dout("%s %p\n", __func__, msgr); } -EXPORT_SYMBOL(ceph_messenger_destroy); +EXPORT_SYMBOL(ceph_messenger_init); static void clear_standby(struct ceph_connection *con) { diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 1845cde2622..704dc95dc62 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -763,7 +763,7 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl) monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL); if (!monc->con) goto out_monmap; - ceph_con_init(monc->client->msgr, monc->con); + ceph_con_init(&monc->client->msgr, monc->con); monc->con->private = monc; monc->con->ops = &mon_con_ops; @@ -880,8 +880,8 @@ static void handle_auth_reply(struct ceph_mon_client *monc, } else if (!was_auth && monc->auth->ops->is_authenticated(monc->auth)) { dout("authenticated, starting session\n"); - monc->client->msgr->inst.name.type = CEPH_ENTITY_TYPE_CLIENT; - monc->client->msgr->inst.name.num = + monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT; + monc->client->msgr.inst.name.num = cpu_to_le64(monc->auth->global_id); __send_subscribe(monc); diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index b098e7b591f..cca4c7f1c78 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -639,7 +639,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc) INIT_LIST_HEAD(&osd->o_osd_lru); osd->o_incarnation = 1; - ceph_con_init(osdc->client->msgr, &osd->o_con); + ceph_con_init(&osdc->client->msgr, &osd->o_con); osd->o_con.private = osd; osd->o_con.ops = &osd_con_ops; osd->o_con.peer_name.type = CEPH_ENTITY_TYPE_OSD; @@ -1391,7 +1391,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg) epoch, maplen); newmap = osdmap_apply_incremental(&p, next, osdc->osdmap, - osdc->client->msgr); + &osdc->client->msgr); if (IS_ERR(newmap)) { err = PTR_ERR(newmap); goto bad; -- cgit v1.2.3-70-g09d2 From 928443cd9644e7cfd46f687dbeffda2d1a357ff9 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 11:41:43 -0500 Subject: libceph: start separating connection flags from state A ceph_connection holds a mixture of connection state (as in "state machine" state) and connection flags in a single "state" field. To make the distinction more clear, define a new "flags" field and use it rather than the "state" field to hold Boolean flag values. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 18 ++++++++++----- net/ceph/messenger.c | 50 +++++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 31 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 3fbd4be804e..920235e114a 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -103,20 +103,25 @@ struct ceph_msg_pos { #define MAX_DELAY_INTERVAL (5 * 60 * HZ) /* - * ceph_connection state bit flags + * ceph_connection flag bits */ + #define LOSSYTX 0 /* we can close channel or drop messages on errors */ -#define CONNECTING 1 -#define NEGOTIATING 2 #define KEEPALIVE_PENDING 3 #define WRITE_PENDING 4 /* we have data ready to send */ +#define SOCK_CLOSED 11 /* socket state changed to closed */ +#define BACKOFF 15 + +/* + * ceph_connection states + */ +#define CONNECTING 1 +#define NEGOTIATING 2 #define STANDBY 8 /* no outgoing messages, socket closed. we keep * the ceph_connection around to maintain shared * state with the peer. */ #define CLOSED 10 /* we've closed the connection */ -#define SOCK_CLOSED 11 /* socket state changed to closed */ #define OPENING 13 /* open connection w/ (possibly new) peer */ -#define BACKOFF 15 /* * A single connection with another host. @@ -133,7 +138,8 @@ struct ceph_connection { struct ceph_messenger *msgr; struct socket *sock; - unsigned long state; /* connection state (see flags above) */ + unsigned long flags; + unsigned long state; const char *error_msg; /* error message, if any */ struct ceph_entity_addr peer_addr; /* peer address */ diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index d8423a3f669..e84e4fd86bb 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -176,7 +176,7 @@ static void ceph_sock_write_space(struct sock *sk) * buffer. See net/ipv4/tcp_input.c:tcp_check_space() * and net/core/stream.c:sk_stream_write_space(). */ - if (test_bit(WRITE_PENDING, &con->state)) { + if (test_bit(WRITE_PENDING, &con->flags)) { if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) { dout("%s %p queueing write work\n", __func__, con); clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); @@ -203,7 +203,7 @@ static void ceph_sock_state_change(struct sock *sk) dout("%s TCP_CLOSE\n", __func__); case TCP_CLOSE_WAIT: dout("%s TCP_CLOSE_WAIT\n", __func__); - if (test_and_set_bit(SOCK_CLOSED, &con->state) == 0) { + if (test_and_set_bit(SOCK_CLOSED, &con->flags) == 0) { if (test_bit(CONNECTING, &con->state)) con->error_msg = "connection failed"; else @@ -395,9 +395,9 @@ void ceph_con_close(struct ceph_connection *con) ceph_pr_addr(&con->peer_addr.in_addr)); set_bit(CLOSED, &con->state); /* in case there's queued work */ clear_bit(STANDBY, &con->state); /* avoid connect_seq bump */ - clear_bit(LOSSYTX, &con->state); /* so we retry next connect */ - clear_bit(KEEPALIVE_PENDING, &con->state); - clear_bit(WRITE_PENDING, &con->state); + clear_bit(LOSSYTX, &con->flags); /* so we retry next connect */ + clear_bit(KEEPALIVE_PENDING, &con->flags); + clear_bit(WRITE_PENDING, &con->flags); mutex_lock(&con->mutex); reset_connection(con); con->peer_global_seq = 0; @@ -614,7 +614,7 @@ static void prepare_write_message(struct ceph_connection *con) prepare_write_message_footer(con); } - set_bit(WRITE_PENDING, &con->state); + set_bit(WRITE_PENDING, &con->flags); } /* @@ -635,7 +635,7 @@ static void prepare_write_ack(struct ceph_connection *con) &con->out_temp_ack); con->out_more = 1; /* more will follow.. eventually.. */ - set_bit(WRITE_PENDING, &con->state); + set_bit(WRITE_PENDING, &con->flags); } /* @@ -646,7 +646,7 @@ static void prepare_write_keepalive(struct ceph_connection *con) dout("prepare_write_keepalive %p\n", con); con_out_kvec_reset(con); con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive); - set_bit(WRITE_PENDING, &con->state); + set_bit(WRITE_PENDING, &con->flags); } /* @@ -675,7 +675,7 @@ static struct ceph_auth_handshake *get_connect_authorizer(struct ceph_connection if (IS_ERR(auth)) return auth; - if (test_bit(CLOSED, &con->state) || test_bit(OPENING, &con->state)) + if (test_bit(CLOSED, &con->state) || test_bit(OPENING, &con->flags)) return ERR_PTR(-EAGAIN); con->auth_reply_buf = auth->authorizer_reply_buf; @@ -695,7 +695,7 @@ static void prepare_write_banner(struct ceph_connection *con) &con->msgr->my_enc_addr); con->out_more = 0; - set_bit(WRITE_PENDING, &con->state); + set_bit(WRITE_PENDING, &con->flags); } static int prepare_write_connect(struct ceph_connection *con) @@ -745,7 +745,7 @@ static int prepare_write_connect(struct ceph_connection *con) auth->authorizer_buf); con->out_more = 0; - set_bit(WRITE_PENDING, &con->state); + set_bit(WRITE_PENDING, &con->flags); return 0; } @@ -1492,7 +1492,7 @@ static int process_connect(struct ceph_connection *con) le32_to_cpu(con->in_reply.connect_seq)); if (con->in_reply.flags & CEPH_MSG_CONNECT_LOSSY) - set_bit(LOSSYTX, &con->state); + set_bit(LOSSYTX, &con->flags); prepare_read_tag(con); break; @@ -1933,14 +1933,14 @@ do_next: prepare_write_ack(con); goto more; } - if (test_and_clear_bit(KEEPALIVE_PENDING, &con->state)) { + if (test_and_clear_bit(KEEPALIVE_PENDING, &con->flags)) { prepare_write_keepalive(con); goto more; } } /* Nothing to do! */ - clear_bit(WRITE_PENDING, &con->state); + clear_bit(WRITE_PENDING, &con->flags); dout("try_write nothing else to write.\n"); ret = 0; out: @@ -2106,7 +2106,7 @@ static void con_work(struct work_struct *work) mutex_lock(&con->mutex); restart: - if (test_and_clear_bit(BACKOFF, &con->state)) { + if (test_and_clear_bit(BACKOFF, &con->flags)) { dout("con_work %p backing off\n", con); if (queue_delayed_work(ceph_msgr_wq, &con->work, round_jiffies_relative(con->delay))) { @@ -2135,7 +2135,7 @@ restart: con_close_socket(con); } - if (test_and_clear_bit(SOCK_CLOSED, &con->state)) + if (test_and_clear_bit(SOCK_CLOSED, &con->flags)) goto fault; ret = try_read(con); @@ -2174,7 +2174,7 @@ static void ceph_fault(struct ceph_connection *con) dout("fault %p state %lu to peer %s\n", con, con->state, ceph_pr_addr(&con->peer_addr.in_addr)); - if (test_bit(LOSSYTX, &con->state)) { + if (test_bit(LOSSYTX, &con->flags)) { dout("fault on LOSSYTX channel\n"); goto out; } @@ -2196,9 +2196,9 @@ static void ceph_fault(struct ceph_connection *con) /* If there are no messages queued or keepalive pending, place * the connection in a STANDBY state */ if (list_empty(&con->out_queue) && - !test_bit(KEEPALIVE_PENDING, &con->state)) { + !test_bit(KEEPALIVE_PENDING, &con->flags)) { dout("fault %p setting STANDBY clearing WRITE_PENDING\n", con); - clear_bit(WRITE_PENDING, &con->state); + clear_bit(WRITE_PENDING, &con->flags); set_bit(STANDBY, &con->state); } else { /* retry after a delay. */ @@ -2222,7 +2222,7 @@ static void ceph_fault(struct ceph_connection *con) * that when con_work restarts we schedule the * delay then. */ - set_bit(BACKOFF, &con->state); + set_bit(BACKOFF, &con->flags); } } @@ -2278,8 +2278,8 @@ static void clear_standby(struct ceph_connection *con) mutex_lock(&con->mutex); dout("clear_standby %p and ++connect_seq\n", con); con->connect_seq++; - WARN_ON(test_bit(WRITE_PENDING, &con->state)); - WARN_ON(test_bit(KEEPALIVE_PENDING, &con->state)); + WARN_ON(test_bit(WRITE_PENDING, &con->flags)); + WARN_ON(test_bit(KEEPALIVE_PENDING, &con->flags)); mutex_unlock(&con->mutex); } } @@ -2317,7 +2317,7 @@ void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg) /* if there wasn't anything waiting to send before, queue * new work */ clear_standby(con); - if (test_and_set_bit(WRITE_PENDING, &con->state) == 0) + if (test_and_set_bit(WRITE_PENDING, &con->flags) == 0) queue_con(con); } EXPORT_SYMBOL(ceph_con_send); @@ -2384,8 +2384,8 @@ void ceph_con_keepalive(struct ceph_connection *con) { dout("con_keepalive %p\n", con); clear_standby(con); - if (test_and_set_bit(KEEPALIVE_PENDING, &con->state) == 0 && - test_and_set_bit(WRITE_PENDING, &con->state) == 0) + if (test_and_set_bit(KEEPALIVE_PENDING, &con->flags) == 0 && + test_and_set_bit(WRITE_PENDING, &con->flags) == 0) queue_con(con); } EXPORT_SYMBOL(ceph_con_keepalive); -- cgit v1.2.3-70-g09d2 From ce2c8903e76e690846a00a0284e4bd9ee954d680 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 22 May 2012 22:15:49 -0500 Subject: libceph: start tracking connection socket state Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself. Create placeholder functions to encapsulate the state transitions. -------- | NEW* | transient initial state -------- | con_sock_state_init() v ---------- | CLOSED | initialized, but no socket (and no ---------- TCP connection) ^ \ | \ con_sock_state_connecting() | ---------------------- | \ + con_sock_state_closed() \ |\ \ | \ \ | ----------- \ | | CLOSING | socket event; \ | ----------- await close \ | ^ | | | | | + con_sock_state_closing() | | / \ | | / --------------- | | / \ v | / -------------- | / -----------------| CONNECTING | socket created, TCP | | / -------------- connect initiated | | | con_sock_state_connected() | | v ------------- | CONNECTED | TCP connection established ------------- Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held). We can back out the atomicity later. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 8 ++++-- net/ceph/messenger.c | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 920235e114a..5e852f444f6 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -137,14 +137,18 @@ struct ceph_connection { const struct ceph_connection_operations *ops; struct ceph_messenger *msgr; + + atomic_t sock_state; struct socket *sock; + struct ceph_entity_addr peer_addr; /* peer address */ + struct ceph_entity_addr peer_addr_for_me; + unsigned long flags; unsigned long state; const char *error_msg; /* error message, if any */ - struct ceph_entity_addr peer_addr; /* peer address */ struct ceph_entity_name peer_name; /* peer name */ - struct ceph_entity_addr peer_addr_for_me; + unsigned peer_features; u32 connect_seq; /* identify the most recent connection attempt for this connection, client */ diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index e84e4fd86bb..a4ac3deec16 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -29,6 +29,14 @@ * the sender. */ +/* State values for ceph_connection->sock_state; NEW is assumed to be 0 */ + +#define CON_SOCK_STATE_NEW 0 /* -> CLOSED */ +#define CON_SOCK_STATE_CLOSED 1 /* -> CONNECTING */ +#define CON_SOCK_STATE_CONNECTING 2 /* -> CONNECTED or -> CLOSING */ +#define CON_SOCK_STATE_CONNECTED 3 /* -> CLOSING or -> CLOSED */ +#define CON_SOCK_STATE_CLOSING 4 /* -> CLOSED */ + /* static tag bytes (protocol control messages) */ static char tag_msg = CEPH_MSGR_TAG_MSG; static char tag_ack = CEPH_MSGR_TAG_ACK; @@ -147,6 +155,55 @@ void ceph_msgr_flush(void) } EXPORT_SYMBOL(ceph_msgr_flush); +/* Connection socket state transition functions */ + +static void con_sock_state_init(struct ceph_connection *con) +{ + int old_state; + + old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED); + if (WARN_ON(old_state != CON_SOCK_STATE_NEW)) + printk("%s: unexpected old state %d\n", __func__, old_state); +} + +static void con_sock_state_connecting(struct ceph_connection *con) +{ + int old_state; + + old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTING); + if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED)) + printk("%s: unexpected old state %d\n", __func__, old_state); +} + +static void con_sock_state_connected(struct ceph_connection *con) +{ + int old_state; + + old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED); + if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING)) + printk("%s: unexpected old state %d\n", __func__, old_state); +} + +static void con_sock_state_closing(struct ceph_connection *con) +{ + int old_state; + + old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING); + if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING && + old_state != CON_SOCK_STATE_CONNECTED && + old_state != CON_SOCK_STATE_CLOSING)) + printk("%s: unexpected old state %d\n", __func__, old_state); +} + +static void con_sock_state_closed(struct ceph_connection *con) +{ + int old_state; + + old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED); + if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED && + old_state != CON_SOCK_STATE_CLOSING)) + printk("%s: unexpected old state %d\n", __func__, old_state); +} /* * socket callback functions @@ -203,6 +260,7 @@ static void ceph_sock_state_change(struct sock *sk) dout("%s TCP_CLOSE\n", __func__); case TCP_CLOSE_WAIT: dout("%s TCP_CLOSE_WAIT\n", __func__); + con_sock_state_closing(con); if (test_and_set_bit(SOCK_CLOSED, &con->flags) == 0) { if (test_bit(CONNECTING, &con->state)) con->error_msg = "connection failed"; @@ -213,6 +271,7 @@ static void ceph_sock_state_change(struct sock *sk) break; case TCP_ESTABLISHED: dout("%s TCP_ESTABLISHED\n", __func__); + con_sock_state_connected(con); queue_con(con); break; default: /* Everything else is uninteresting */ @@ -277,6 +336,7 @@ static int ceph_tcp_connect(struct ceph_connection *con) return ret; } con->sock = sock; + con_sock_state_connecting(con); return 0; } @@ -343,6 +403,7 @@ static int con_close_socket(struct ceph_connection *con) sock_release(con->sock); con->sock = NULL; clear_bit(SOCK_CLOSED, &con->state); + con_sock_state_closed(con); return rc; } @@ -462,6 +523,9 @@ void ceph_con_init(struct ceph_messenger *msgr, struct ceph_connection *con) memset(con, 0, sizeof(*con)); atomic_set(&con->nref, 1); con->msgr = msgr; + + con_sock_state_init(con); + mutex_init(&con->mutex); INIT_LIST_HEAD(&con->out_queue); INIT_LIST_HEAD(&con->out_sent); -- cgit v1.2.3-70-g09d2 From 67130934fb579fdf0f2f6d745960264378b57dc8 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: embed ceph connection structure in mon_client A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure. So switch the ceph_mon_client structure to embed its ceph_connection structure. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/mon_client.h | 2 +- net/ceph/mon_client.c | 47 ++++++++++++++++++----------------------- 2 files changed, 21 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/mon_client.h b/include/linux/ceph/mon_client.h index 545f8591778..2113e3850a4 100644 --- a/include/linux/ceph/mon_client.h +++ b/include/linux/ceph/mon_client.h @@ -70,7 +70,7 @@ struct ceph_mon_client { bool hunting; int cur_mon; /* last monitor i contacted */ unsigned long sub_sent, sub_renew_after; - struct ceph_connection *con; + struct ceph_connection con; bool have_fsid; /* pending generic requests */ diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 704dc95dc62..ac4d6b10073 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -106,9 +106,9 @@ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) monc->pending_auth = 1; monc->m_auth->front.iov_len = len; monc->m_auth->hdr.front_len = cpu_to_le32(len); - ceph_con_revoke(monc->con, monc->m_auth); + ceph_con_revoke(&monc->con, monc->m_auth); ceph_msg_get(monc->m_auth); /* keep our ref */ - ceph_con_send(monc->con, monc->m_auth); + ceph_con_send(&monc->con, monc->m_auth); } /* @@ -117,8 +117,8 @@ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) static void __close_session(struct ceph_mon_client *monc) { dout("__close_session closing mon%d\n", monc->cur_mon); - ceph_con_revoke(monc->con, monc->m_auth); - ceph_con_close(monc->con); + ceph_con_revoke(&monc->con, monc->m_auth); + ceph_con_close(&monc->con); monc->cur_mon = -1; monc->pending_auth = 0; ceph_auth_reset(monc->auth); @@ -142,9 +142,9 @@ static int __open_session(struct ceph_mon_client *monc) monc->want_next_osdmap = !!monc->want_next_osdmap; dout("open_session mon%d opening\n", monc->cur_mon); - monc->con->peer_name.type = CEPH_ENTITY_TYPE_MON; - monc->con->peer_name.num = cpu_to_le64(monc->cur_mon); - ceph_con_open(monc->con, + monc->con.peer_name.type = CEPH_ENTITY_TYPE_MON; + monc->con.peer_name.num = cpu_to_le64(monc->cur_mon); + ceph_con_open(&monc->con, &monc->monmap->mon_inst[monc->cur_mon].addr); /* initiatiate authentication handshake */ @@ -226,8 +226,8 @@ static void __send_subscribe(struct ceph_mon_client *monc) msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); - ceph_con_revoke(monc->con, msg); - ceph_con_send(monc->con, ceph_msg_get(msg)); + ceph_con_revoke(&monc->con, msg); + ceph_con_send(&monc->con, ceph_msg_get(msg)); monc->sub_sent = jiffies | 1; /* never 0 */ } @@ -247,7 +247,7 @@ static void handle_subscribe_ack(struct ceph_mon_client *monc, if (monc->hunting) { pr_info("mon%d %s session established\n", monc->cur_mon, - ceph_pr_addr(&monc->con->peer_addr.in_addr)); + ceph_pr_addr(&monc->con.peer_addr.in_addr)); monc->hunting = false; } dout("handle_subscribe_ack after %d seconds\n", seconds); @@ -461,7 +461,7 @@ static int do_generic_request(struct ceph_mon_client *monc, req->request->hdr.tid = cpu_to_le64(req->tid); __insert_generic_request(monc, req); monc->num_generic_requests++; - ceph_con_send(monc->con, ceph_msg_get(req->request)); + ceph_con_send(&monc->con, ceph_msg_get(req->request)); mutex_unlock(&monc->mutex); err = wait_for_completion_interruptible(&req->completion); @@ -684,8 +684,8 @@ static void __resend_generic_request(struct ceph_mon_client *monc) for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) { req = rb_entry(p, struct ceph_mon_generic_request, node); - ceph_con_revoke(monc->con, req->request); - ceph_con_send(monc->con, ceph_msg_get(req->request)); + ceph_con_revoke(&monc->con, req->request); + ceph_con_send(&monc->con, ceph_msg_get(req->request)); } } @@ -705,7 +705,7 @@ static void delayed_work(struct work_struct *work) __close_session(monc); __open_session(monc); /* continue hunting */ } else { - ceph_con_keepalive(monc->con); + ceph_con_keepalive(&monc->con); __validate_auth(monc); @@ -760,19 +760,16 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl) goto out; /* connection */ - monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL); - if (!monc->con) - goto out_monmap; - ceph_con_init(&monc->client->msgr, monc->con); - monc->con->private = monc; - monc->con->ops = &mon_con_ops; + ceph_con_init(&monc->client->msgr, &monc->con); + monc->con.private = monc; + monc->con.ops = &mon_con_ops; /* authentication */ monc->auth = ceph_auth_init(cl->options->name, cl->options->key); if (IS_ERR(monc->auth)) { err = PTR_ERR(monc->auth); - goto out_con; + goto out_monmap; } monc->auth->want_keys = CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON | @@ -824,8 +821,6 @@ out_subscribe_ack: ceph_msg_put(monc->m_subscribe_ack); out_auth: ceph_auth_destroy(monc->auth); -out_con: - monc->con->ops->put(monc->con); out_monmap: kfree(monc->monmap); out: @@ -841,9 +836,7 @@ void ceph_monc_stop(struct ceph_mon_client *monc) mutex_lock(&monc->mutex); __close_session(monc); - monc->con->private = NULL; - monc->con->ops->put(monc->con); - monc->con = NULL; + monc->con.private = NULL; mutex_unlock(&monc->mutex); @@ -1021,7 +1014,7 @@ static void mon_fault(struct ceph_connection *con) if (!monc->hunting) pr_info("mon%d %s session lost, " "hunting for new mon\n", monc->cur_mon, - ceph_pr_addr(&monc->con->peer_addr.in_addr)); + ceph_pr_addr(&monc->con.peer_addr.in_addr)); __close_session(monc); if (!monc->hunting) { -- cgit v1.2.3-70-g09d2 From 1bfd89f4e6e1adc6a782d94aa5d4c53be1e404d7 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Sat, 26 May 2012 23:26:43 -0500 Subject: libceph: fully initialize connection in con_init() Move the initialization of a ceph connection's private pointer, operations vector pointer, and peer name information into ceph_con_init(). Rearrange the arguments so the connection pointer is first. Hide the byte-swapping of the peer entity number inside ceph_con_init() Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- fs/ceph/mds_client.c | 7 ++----- include/linux/ceph/messenger.h | 6 ++++-- net/ceph/messenger.c | 9 ++++++++- net/ceph/mon_client.c | 8 +++----- net/ceph/osd_client.c | 7 ++----- 5 files changed, 19 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ad30261cd4c..ecd7f15741c 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -394,11 +394,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, s->s_seq = 0; mutex_init(&s->s_mutex); - ceph_con_init(&mdsc->fsc->client->msgr, &s->s_con); - s->s_con.private = s; - s->s_con.ops = &mds_con_ops; - s->s_con.peer_name.type = CEPH_ENTITY_TYPE_MDS; - s->s_con.peer_name.num = cpu_to_le64(mds); + ceph_con_init(&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr, + CEPH_ENTITY_TYPE_MDS, mds); spin_lock_init(&s->s_gen_ttl_lock); s->s_cap_gen = 0; diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 5e852f444f6..dd27837f79a 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -227,8 +227,10 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr, u32 required_features, bool nocrc); -extern void ceph_con_init(struct ceph_messenger *msgr, - struct ceph_connection *con); +extern void ceph_con_init(struct ceph_connection *con, void *private, + const struct ceph_connection_operations *ops, + struct ceph_messenger *msgr, __u8 entity_type, + __u64 entity_num); extern void ceph_con_open(struct ceph_connection *con, struct ceph_entity_addr *addr); extern bool ceph_con_opened(struct ceph_connection *con); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 36b440a00cc..3b65f6e6911 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -521,15 +521,22 @@ void ceph_con_put(struct ceph_connection *con) /* * initialize a new connection. */ -void ceph_con_init(struct ceph_messenger *msgr, struct ceph_connection *con) +void ceph_con_init(struct ceph_connection *con, void *private, + const struct ceph_connection_operations *ops, + struct ceph_messenger *msgr, __u8 entity_type, __u64 entity_num) { dout("con_init %p\n", con); memset(con, 0, sizeof(*con)); + con->private = private; + con->ops = ops; atomic_set(&con->nref, 1); con->msgr = msgr; con_sock_state_init(con); + con->peer_name.type = (__u8) entity_type; + con->peer_name.num = cpu_to_le64(entity_num); + mutex_init(&con->mutex); INIT_LIST_HEAD(&con->out_queue); INIT_LIST_HEAD(&con->out_sent); diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 6adbea78b16..ab6b24a5169 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -142,11 +142,9 @@ static int __open_session(struct ceph_mon_client *monc) monc->sub_renew_after = jiffies; /* i.e., expired */ monc->want_next_osdmap = !!monc->want_next_osdmap; - ceph_con_init(&monc->client->msgr, &monc->con); - monc->con.private = monc; - monc->con.ops = &mon_con_ops; - monc->con.peer_name.type = CEPH_ENTITY_TYPE_MON; - monc->con.peer_name.num = cpu_to_le64(monc->cur_mon); + ceph_con_init(&monc->con, monc, &mon_con_ops, + &monc->client->msgr, + CEPH_ENTITY_TYPE_MON, monc->cur_mon); dout("open_session mon%d opening\n", monc->cur_mon); ceph_con_open(&monc->con, diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 5b41a6929cd..448c9da8bef 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -640,11 +640,8 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc, int onum) INIT_LIST_HEAD(&osd->o_osd_lru); osd->o_incarnation = 1; - ceph_con_init(&osdc->client->msgr, &osd->o_con); - osd->o_con.private = osd; - osd->o_con.ops = &osd_con_ops; - osd->o_con.peer_name.type = CEPH_ENTITY_TYPE_OSD; - osd->o_con.peer_name.num = cpu_to_le64(onum); + ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr, + CEPH_ENTITY_TYPE_OSD, onum); INIT_LIST_HEAD(&osd->o_keepalive_item); return osd; -- cgit v1.2.3-70-g09d2 From 38941f8031bf042dba3ced6394ba3a3b16c244ea Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: have messages point to their connection When a ceph message is queued for sending it is placed on a list of pending messages (ceph_connection->out_queue). When they are actually sent over the wire, they are moved from that list to another (ceph_connection->out_sent). When acknowledgement for the message is received, it is removed from the sent messages list. During that entire time the message is "in the possession" of a single ceph connection. Keep track of that connection in the message. This will be used in the next patch (and is a helpful bit of information for debugging anyway). Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 3 +++ net/ceph/messenger.c | 27 +++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index dd27837f79a..6df837f7276 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -77,7 +77,10 @@ struct ceph_msg { unsigned nr_pages; /* size of page array */ unsigned page_alignment; /* io offset in first page */ struct ceph_pagelist *pagelist; /* instead of pages */ + + struct ceph_connection *con; struct list_head list_head; + struct kref kref; struct bio *bio; /* instead of pages/pagelist */ struct bio *bio_iter; /* bio iterator */ diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 98ca23726ea..68b49b5b8e8 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -414,6 +414,9 @@ static int con_close_socket(struct ceph_connection *con) static void ceph_msg_remove(struct ceph_msg *msg) { list_del_init(&msg->list_head); + BUG_ON(msg->con == NULL); + msg->con = NULL; + ceph_msg_put(msg); } static void ceph_msg_remove_list(struct list_head *head) @@ -433,6 +436,8 @@ static void reset_connection(struct ceph_connection *con) ceph_msg_remove_list(&con->out_sent); if (con->in_msg) { + BUG_ON(con->in_msg->con != con); + con->in_msg->con = NULL; ceph_msg_put(con->in_msg); con->in_msg = NULL; } @@ -625,8 +630,10 @@ static void prepare_write_message(struct ceph_connection *con) &con->out_temp_ack); } + BUG_ON(list_empty(&con->out_queue)); m = list_first_entry(&con->out_queue, struct ceph_msg, list_head); con->out_msg = m; + BUG_ON(m->con != con); /* put message on sent list */ ceph_msg_get(m); @@ -1806,6 +1813,8 @@ static int read_partial_message(struct ceph_connection *con) "error allocating memory for incoming message"; return -ENOMEM; } + + BUG_ON(con->in_msg->con != con); m = con->in_msg; m->front.iov_len = 0; /* haven't read it yet */ if (m->middle) @@ -1901,6 +1910,8 @@ static void process_message(struct ceph_connection *con) { struct ceph_msg *msg; + BUG_ON(con->in_msg->con != con); + con->in_msg->con = NULL; msg = con->in_msg; con->in_msg = NULL; @@ -2260,6 +2271,8 @@ static void ceph_fault(struct ceph_connection *con) con_close_socket(con); if (con->in_msg) { + BUG_ON(con->in_msg->con != con); + con->in_msg->con = NULL; ceph_msg_put(con->in_msg); con->in_msg = NULL; } @@ -2378,6 +2391,8 @@ void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg) /* queue */ mutex_lock(&con->mutex); + BUG_ON(msg->con != NULL); + msg->con = con; BUG_ON(!list_empty(&msg->list_head)); list_add_tail(&msg->list_head, &con->out_queue); dout("----- %p to %s%lld %d=%s len %d+%d+%d -----\n", msg, @@ -2403,13 +2418,16 @@ void ceph_con_revoke(struct ceph_connection *con, struct ceph_msg *msg) { mutex_lock(&con->mutex); if (!list_empty(&msg->list_head)) { - dout("con_revoke %p msg %p - was on queue\n", con, msg); + dout("%s %p msg %p - was on queue\n", __func__, con, msg); list_del_init(&msg->list_head); + BUG_ON(msg->con == NULL); + msg->con = NULL; + ceph_msg_put(msg); msg->hdr.seq = 0; } if (con->out_msg == msg) { - dout("con_revoke %p msg %p - was sending\n", con, msg); + dout("%s %p msg %p - was sending\n", __func__, con, msg); con->out_msg = NULL; if (con->out_kvec_is_msg) { con->out_skip = con->out_kvec_bytes; @@ -2478,6 +2496,8 @@ struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags, if (m == NULL) goto out; kref_init(&m->kref); + + m->con = NULL; INIT_LIST_HEAD(&m->list_head); m->hdr.tid = 0; @@ -2598,6 +2618,8 @@ static bool ceph_con_in_msg_alloc(struct ceph_connection *con, mutex_unlock(&con->mutex); con->in_msg = con->ops->alloc_msg(con, hdr, &skip); mutex_lock(&con->mutex); + if (con->in_msg) + con->in_msg->con = con; if (skip) con->in_msg = NULL; @@ -2611,6 +2633,7 @@ static bool ceph_con_in_msg_alloc(struct ceph_connection *con, type, front_len); return false; } + con->in_msg->con = con; con->in_msg->page_alignment = le16_to_cpu(hdr->data_off); } memcpy(&con->in_msg->hdr, &con->in_hdr, sizeof(con->in_hdr)); -- cgit v1.2.3-70-g09d2 From 6740a845b2543cc46e1902ba21bac743fbadd0dc Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: make ceph_con_revoke() a msg operation ceph_con_revoke() is passed both a message and a ceph connection. Now that any message associated with a connection holds a pointer to that connection, there's no need to provide the connection when revoking a message. This has the added benefit of precluding the possibility of the providing the wrong connection pointer. If the message's connection pointer is null, it is not being tracked by any connection, so revoking it is a no-op. This is supported as a convenience for upper layers, so they can revoke a message that is not actually "in flight." Rename the function ceph_msg_revoke() to reflect that it is really an operation on a message, not a connection. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 3 ++- net/ceph/messenger.c | 7 ++++++- net/ceph/mon_client.c | 8 ++++---- net/ceph/osd_client.c | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 6df837f7276..9008f81c20c 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -239,7 +239,8 @@ extern void ceph_con_open(struct ceph_connection *con, extern bool ceph_con_opened(struct ceph_connection *con); extern void ceph_con_close(struct ceph_connection *con); extern void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg); -extern void ceph_con_revoke(struct ceph_connection *con, struct ceph_msg *msg); + +extern void ceph_msg_revoke(struct ceph_msg *msg); extern void ceph_con_revoke_message(struct ceph_connection *con, struct ceph_msg *msg); extern void ceph_con_keepalive(struct ceph_connection *con); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 88ac083bb99..d636903ad4b 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -2421,8 +2421,13 @@ EXPORT_SYMBOL(ceph_con_send); /* * Revoke a message that was previously queued for send */ -void ceph_con_revoke(struct ceph_connection *con, struct ceph_msg *msg) +void ceph_msg_revoke(struct ceph_msg *msg) { + struct ceph_connection *con = msg->con; + + if (!con) + return; /* Message not in our possession */ + mutex_lock(&con->mutex); if (!list_empty(&msg->list_head)) { dout("%s %p msg %p - was on queue\n", __func__, con, msg); diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 8462ccec633..7a16750d62a 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -106,7 +106,7 @@ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) monc->pending_auth = 1; monc->m_auth->front.iov_len = len; monc->m_auth->hdr.front_len = cpu_to_le32(len); - ceph_con_revoke(&monc->con, monc->m_auth); + ceph_msg_revoke(monc->m_auth); ceph_msg_get(monc->m_auth); /* keep our ref */ ceph_con_send(&monc->con, monc->m_auth); } @@ -117,7 +117,7 @@ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) static void __close_session(struct ceph_mon_client *monc) { dout("__close_session closing mon%d\n", monc->cur_mon); - ceph_con_revoke(&monc->con, monc->m_auth); + ceph_msg_revoke(monc->m_auth); ceph_con_close(&monc->con); monc->con.private = NULL; monc->cur_mon = -1; @@ -229,7 +229,7 @@ static void __send_subscribe(struct ceph_mon_client *monc) msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); - ceph_con_revoke(&monc->con, msg); + ceph_msg_revoke(msg); ceph_con_send(&monc->con, ceph_msg_get(msg)); monc->sub_sent = jiffies | 1; /* never 0 */ @@ -688,7 +688,7 @@ static void __resend_generic_request(struct ceph_mon_client *monc) for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) { req = rb_entry(p, struct ceph_mon_generic_request, node); - ceph_con_revoke(&monc->con, req->request); + ceph_msg_revoke(req->request); ceph_con_send(&monc->con, ceph_msg_get(req->request)); } } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 24b427b1eca..ad78705a4af 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -852,7 +852,7 @@ static void __unregister_request(struct ceph_osd_client *osdc, if (req->r_osd) { /* make sure the original request isn't in flight. */ - ceph_con_revoke(&req->r_osd->o_con, req->r_request); + ceph_msg_revoke(req->r_request); list_del_init(&req->r_osd_item); if (list_empty(&req->r_osd->o_requests) && @@ -879,7 +879,7 @@ static void __unregister_request(struct ceph_osd_client *osdc, static void __cancel_request(struct ceph_osd_request *req) { if (req->r_sent && req->r_osd) { - ceph_con_revoke(&req->r_osd->o_con, req->r_request); + ceph_msg_revoke(req->r_request); req->r_sent = 0; } } -- cgit v1.2.3-70-g09d2 From 8921d114f5574c6da2cdd00749d185633ecf88f3 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 1 Jun 2012 14:56:43 -0500 Subject: libceph: make ceph_con_revoke_message() a msg op ceph_con_revoke_message() is passed both a message and a ceph connection. A ceph_msg allocated for incoming messages on a connection always has a pointer to that connection, so there's no need to provide the connection when revoking such a message. Note that the existing logic does not preclude the message supplied being a null/bogus message pointer. The only user of this interface is the OSD client, and the only value an osd client passes is a request's r_reply field. That is always non-null (except briefly in an error path in ceph_osdc_alloc_request(), and that drops the only reference so the request won't ever have a reply to revoke). So we can safely assume the passed-in message is non-null, but add a BUG_ON() to make it very obvious we are imposing this restriction. Rename the function ceph_msg_revoke_incoming() to reflect that it is really an operation on an incoming message. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 4 ++-- net/ceph/messenger.c | 22 ++++++++++++++++------ net/ceph/osd_client.c | 9 ++++----- 3 files changed, 22 insertions(+), 13 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 9008f81c20c..a334dbd1b32 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -241,8 +241,8 @@ extern void ceph_con_close(struct ceph_connection *con); extern void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg); extern void ceph_msg_revoke(struct ceph_msg *msg); -extern void ceph_con_revoke_message(struct ceph_connection *con, - struct ceph_msg *msg); +extern void ceph_msg_revoke_incoming(struct ceph_msg *msg); + extern void ceph_con_keepalive(struct ceph_connection *con); extern struct ceph_connection *ceph_con_get(struct ceph_connection *con); extern void ceph_con_put(struct ceph_connection *con); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index d636903ad4b..3857f815c03 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -2456,17 +2456,27 @@ void ceph_msg_revoke(struct ceph_msg *msg) /* * Revoke a message that we may be reading data into */ -void ceph_con_revoke_message(struct ceph_connection *con, struct ceph_msg *msg) +void ceph_msg_revoke_incoming(struct ceph_msg *msg) { + struct ceph_connection *con; + + BUG_ON(msg == NULL); + if (!msg->con) { + dout("%s msg %p null con\n", __func__, msg); + + return; /* Message not in our possession */ + } + + con = msg->con; mutex_lock(&con->mutex); - if (con->in_msg && con->in_msg == msg) { + if (con->in_msg == msg) { unsigned front_len = le32_to_cpu(con->in_hdr.front_len); unsigned middle_len = le32_to_cpu(con->in_hdr.middle_len); unsigned data_len = le32_to_cpu(con->in_hdr.data_len); /* skip rest of message */ - dout("con_revoke_pages %p msg %p revoked\n", con, msg); - con->in_base_pos = con->in_base_pos - + dout("%s %p msg %p revoked\n", __func__, con, msg); + con->in_base_pos = con->in_base_pos - sizeof(struct ceph_msg_header) - front_len - middle_len - @@ -2477,8 +2487,8 @@ void ceph_con_revoke_message(struct ceph_connection *con, struct ceph_msg *msg) con->in_tag = CEPH_MSGR_TAG_READY; con->in_seq++; } else { - dout("con_revoke_pages %p msg %p pages %p no-op\n", - con, con->in_msg, msg); + dout("%s %p in_msg %p msg %p no-op\n", + __func__, con, con->in_msg, msg); } mutex_unlock(&con->mutex); } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index ad78705a4af..c178c770acb 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -140,10 +140,9 @@ void ceph_osdc_release_request(struct kref *kref) if (req->r_request) ceph_msg_put(req->r_request); if (req->r_con_filling_msg) { - dout("release_request revoking pages %p from con %p\n", + dout("%s revoking pages %p from con %p\n", __func__, req->r_pages, req->r_con_filling_msg); - ceph_con_revoke_message(req->r_con_filling_msg, - req->r_reply); + ceph_msg_revoke_incoming(req->r_reply); req->r_con_filling_msg->ops->put(req->r_con_filling_msg); } if (req->r_reply) @@ -2022,9 +2021,9 @@ static struct ceph_msg *get_reply(struct ceph_connection *con, } if (req->r_con_filling_msg) { - dout("get_reply revoking msg %p from old con %p\n", + dout("%s revoking msg %p from old con %p\n", __func__, req->r_reply, req->r_con_filling_msg); - ceph_con_revoke_message(req->r_con_filling_msg, req->r_reply); + ceph_msg_revoke_incoming(req->r_reply); req->r_con_filling_msg->ops->put(req->r_con_filling_msg); req->r_con_filling_msg = NULL; } -- cgit v1.2.3-70-g09d2 From d59315ca8c0de00df9b363f94a2641a30961ca1c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 21 Jun 2012 12:49:23 -0700 Subject: libceph: drop ceph_con_get/put helpers and nref member These are no longer used. Every ceph_connection instance is embedded in another structure, and refcounts manipulated via the get/put ops. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 1 - net/ceph/messenger.c | 28 +--------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index a334dbd1b32..cc6f9bdcf46 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -135,7 +135,6 @@ struct ceph_msg_pos { */ struct ceph_connection { void *private; - atomic_t nref; const struct ceph_connection_operations *ops; diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index fc0cee7c9aa..ab690e2e120 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -500,30 +500,6 @@ bool ceph_con_opened(struct ceph_connection *con) return con->connect_seq > 0; } -/* - * generic get/put - */ -struct ceph_connection *ceph_con_get(struct ceph_connection *con) -{ - int nref = __atomic_add_unless(&con->nref, 1, 0); - - dout("con_get %p nref = %d -> %d\n", con, nref, nref + 1); - - return nref ? con : NULL; -} - -void ceph_con_put(struct ceph_connection *con) -{ - int nref = atomic_dec_return(&con->nref); - - BUG_ON(nref < 0); - if (nref == 0) { - BUG_ON(con->sock); - kfree(con); - } - dout("con_put %p nref = %d -> %d\n", con, nref + 1, nref); -} - /* * initialize a new connection. */ @@ -535,7 +511,6 @@ void ceph_con_init(struct ceph_connection *con, void *private, memset(con, 0, sizeof(*con)); con->private = private; con->ops = ops; - atomic_set(&con->nref, 1); con->msgr = msgr; con_sock_state_init(con); @@ -1951,8 +1926,7 @@ static int try_write(struct ceph_connection *con) { int ret = 1; - dout("try_write start %p state %lu nref %d\n", con, con->state, - atomic_read(&con->nref)); + dout("try_write start %p state %lu\n", con, con->state); more: dout("try_write out_kvec_bytes %d\n", con->out_kvec_bytes); -- cgit v1.2.3-70-g09d2 From e27947c767f5bed15048f4e4dad3e2eb69133697 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Wed, 23 May 2012 14:35:23 -0500 Subject: libceph: define and use an explicit CONNECTED state There is no state explicitly defined when a ceph connection is fully operational. So define one. It's set when the connection sequence completes successfully, and is cleared when the connection gets closed. Be a little more careful when examining the old state when a socket disconnect event is reported. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 1 + net/ceph/messenger.c | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index cc6f9bdcf46..002d504df3b 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -120,6 +120,7 @@ struct ceph_msg_pos { */ #define CONNECTING 1 #define NEGOTIATING 2 +#define CONNECTED 5 #define STANDBY 8 /* no outgoing messages, socket closed. we keep * the ceph_connection around to maintain shared * state with the peer. */ diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 500207bad5d..83bcf977e9b 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -463,6 +463,7 @@ void ceph_con_close(struct ceph_connection *con) ceph_pr_addr(&con->peer_addr.in_addr)); clear_bit(NEGOTIATING, &con->state); clear_bit(CONNECTING, &con->state); + clear_bit(CONNECTED, &con->state); clear_bit(STANDBY, &con->state); /* avoid connect_seq bump */ set_bit(CLOSED, &con->state); @@ -1564,6 +1565,7 @@ static int process_connect(struct ceph_connection *con) } clear_bit(NEGOTIATING, &con->state); clear_bit(CONNECTING, &con->state); + set_bit(CONNECTED, &con->state); con->peer_global_seq = le32_to_cpu(con->in_reply.global_seq); con->connect_seq++; con->peer_features = server_feat; @@ -2114,6 +2116,7 @@ more: prepare_read_ack(con); break; case CEPH_MSGR_TAG_CLOSE: + clear_bit(CONNECTED, &con->state); set_bit(CLOSED, &con->state); /* fixme */ goto out; default: @@ -2190,11 +2193,13 @@ static void con_work(struct work_struct *work) mutex_lock(&con->mutex); restart: if (test_and_clear_bit(SOCK_CLOSED, &con->flags)) { - if (test_and_clear_bit(CONNECTING, &con->state)) { + if (test_and_clear_bit(CONNECTED, &con->state)) + con->error_msg = "socket closed"; + else if (test_and_clear_bit(CONNECTING, &con->state)) { clear_bit(NEGOTIATING, &con->state); con->error_msg = "connection failed"; } else { - con->error_msg = "socket closed"; + con->error_msg = "unrecognized con state"; } goto fault; } -- cgit v1.2.3-70-g09d2 From 261030215d970c62f799e6e508e3c68fc7ec2aa9 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Thu, 21 Jun 2012 12:49:23 -0700 Subject: libceph: drop declaration of ceph_con_get() For some reason the declaration of ceph_con_get() and ceph_con_put() did not get deleted in this commit: d59315ca libceph: drop ceph_con_get/put helpers and nref member Clean that up. Signed-off-by: Alex Elder --- include/linux/ceph/messenger.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 002d504df3b..dd4ef1f8ec9 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -244,8 +244,6 @@ extern void ceph_msg_revoke(struct ceph_msg *msg); extern void ceph_msg_revoke_incoming(struct ceph_msg *msg); extern void ceph_con_keepalive(struct ceph_connection *con); -extern struct ceph_connection *ceph_con_get(struct ceph_connection *con); -extern void ceph_con_put(struct ceph_connection *con); extern struct ceph_msg *ceph_msg_new(int type, int front_len, gfp_t flags, bool can_fail); -- cgit v1.2.3-70-g09d2 From b7a9e5dd40f17a48a72f249b8bbc989b63bae5fd Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 27 Jun 2012 12:24:08 -0700 Subject: libceph: set peer name on con_open, not init The peer name may change on each open attempt, even when the connection is reused. Signed-off-by: Sage Weil --- fs/ceph/mds_client.c | 7 ++++--- include/linux/ceph/messenger.h | 4 ++-- net/ceph/messenger.c | 12 +++++++----- net/ceph/mon_client.c | 4 ++-- net/ceph/osd_client.c | 10 ++++++---- 5 files changed, 21 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ecd7f15741c..5ac6434185a 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -394,8 +394,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, s->s_seq = 0; mutex_init(&s->s_mutex); - ceph_con_init(&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr, - CEPH_ENTITY_TYPE_MDS, mds); + ceph_con_init(&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr); spin_lock_init(&s->s_gen_ttl_lock); s->s_cap_gen = 0; @@ -437,7 +436,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, mdsc->sessions[mds] = s; atomic_inc(&s->s_ref); /* one ref to sessions[], one to caller */ - ceph_con_open(&s->s_con, ceph_mdsmap_get_addr(mdsc->mdsmap, mds)); + ceph_con_open(&s->s_con, CEPH_ENTITY_TYPE_MDS, mds, + ceph_mdsmap_get_addr(mdsc->mdsmap, mds)); return s; @@ -2529,6 +2529,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, session->s_seq = 0; ceph_con_open(&session->s_con, + CEPH_ENTITY_TYPE_MDS, mds, ceph_mdsmap_get_addr(mdsc->mdsmap, mds)); /* replay unsafe requests */ diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index dd4ef1f8ec9..478f814f210 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -232,9 +232,9 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr, extern void ceph_con_init(struct ceph_connection *con, void *private, const struct ceph_connection_operations *ops, - struct ceph_messenger *msgr, __u8 entity_type, - __u64 entity_num); + struct ceph_messenger *msgr); extern void ceph_con_open(struct ceph_connection *con, + __u8 entity_type, __u64 entity_num, struct ceph_entity_addr *addr); extern bool ceph_con_opened(struct ceph_connection *con); extern void ceph_con_close(struct ceph_connection *con); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index dcc50e4cd5c..ae082d95fc7 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -523,12 +523,17 @@ EXPORT_SYMBOL(ceph_con_close); /* * Reopen a closed connection, with a new peer address. */ -void ceph_con_open(struct ceph_connection *con, struct ceph_entity_addr *addr) +void ceph_con_open(struct ceph_connection *con, + __u8 entity_type, __u64 entity_num, + struct ceph_entity_addr *addr) { dout("con_open %p %s\n", con, ceph_pr_addr(&addr->in_addr)); set_bit(OPENING, &con->state); WARN_ON(!test_and_clear_bit(CLOSED, &con->state)); + con->peer_name.type = (__u8) entity_type; + con->peer_name.num = cpu_to_le64(entity_num); + memcpy(&con->peer_addr, addr, sizeof(*addr)); con->delay = 0; /* reset backoff memory */ queue_con(con); @@ -548,7 +553,7 @@ bool ceph_con_opened(struct ceph_connection *con) */ void ceph_con_init(struct ceph_connection *con, void *private, const struct ceph_connection_operations *ops, - struct ceph_messenger *msgr, __u8 entity_type, __u64 entity_num) + struct ceph_messenger *msgr) { dout("con_init %p\n", con); memset(con, 0, sizeof(*con)); @@ -558,9 +563,6 @@ void ceph_con_init(struct ceph_connection *con, void *private, con_sock_state_init(con); - con->peer_name.type = (__u8) entity_type; - con->peer_name.num = cpu_to_le64(entity_num); - mutex_init(&con->mutex); INIT_LIST_HEAD(&con->out_queue); INIT_LIST_HEAD(&con->out_sent); diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index e9db3de20b2..bcc80a0e2a9 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -143,11 +143,11 @@ static int __open_session(struct ceph_mon_client *monc) monc->want_next_osdmap = !!monc->want_next_osdmap; ceph_con_init(&monc->con, monc, &mon_con_ops, - &monc->client->msgr, - CEPH_ENTITY_TYPE_MON, monc->cur_mon); + &monc->client->msgr); dout("open_session mon%d opening\n", monc->cur_mon); ceph_con_open(&monc->con, + CEPH_ENTITY_TYPE_MON, monc->cur_mon, &monc->monmap->mon_inst[monc->cur_mon].addr); /* initiatiate authentication handshake */ diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index db2da54f733..c2527113d2a 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -639,8 +639,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc, int onum) INIT_LIST_HEAD(&osd->o_osd_lru); osd->o_incarnation = 1; - ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr, - CEPH_ENTITY_TYPE_OSD, onum); + ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); INIT_LIST_HEAD(&osd->o_keepalive_item); return osd; @@ -750,7 +749,8 @@ static int __reset_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd) ret = -EAGAIN; } else { ceph_con_close(&osd->o_con); - ceph_con_open(&osd->o_con, &osdc->osdmap->osd_addr[osd->o_osd]); + ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, + &osdc->osdmap->osd_addr[osd->o_osd]); osd->o_incarnation++; } return ret; @@ -1005,7 +1005,9 @@ static int __map_request(struct ceph_osd_client *osdc, dout("map_request osd %p is osd%d\n", req->r_osd, o); __insert_osd(osdc, req->r_osd); - ceph_con_open(&req->r_osd->o_con, &osdc->osdmap->osd_addr[o]); + ceph_con_open(&req->r_osd->o_con, + CEPH_ENTITY_TYPE_OSD, o, + &osdc->osdmap->osd_addr[o]); } if (req->r_osd) { -- cgit v1.2.3-70-g09d2 From d50b409fb8698571d8209e5adfe122e287e31290 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 9 Jul 2012 14:22:34 -0700 Subject: libceph: initialize msgpool message types Initialize the type field for messages in a msgpool. The caller was doing this for osd ops, but not for the reply messages. Reported-by: Alex Elder Signed-off-by: Sage Weil --- include/linux/ceph/msgpool.h | 3 ++- net/ceph/msgpool.c | 7 ++++--- net/ceph/osd_client.c | 7 ++++--- 3 files changed, 10 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/msgpool.h b/include/linux/ceph/msgpool.h index a362605f936..09fa96b4343 100644 --- a/include/linux/ceph/msgpool.h +++ b/include/linux/ceph/msgpool.h @@ -11,10 +11,11 @@ struct ceph_msgpool { const char *name; mempool_t *pool; + int type; /* preallocated message type */ int front_len; /* preallocated payload size */ }; -extern int ceph_msgpool_init(struct ceph_msgpool *pool, +extern int ceph_msgpool_init(struct ceph_msgpool *pool, int type, int front_len, int size, bool blocking, const char *name); extern void ceph_msgpool_destroy(struct ceph_msgpool *pool); diff --git a/net/ceph/msgpool.c b/net/ceph/msgpool.c index 11d5f4196a7..ddec1c10ac8 100644 --- a/net/ceph/msgpool.c +++ b/net/ceph/msgpool.c @@ -12,7 +12,7 @@ static void *msgpool_alloc(gfp_t gfp_mask, void *arg) struct ceph_msgpool *pool = arg; struct ceph_msg *msg; - msg = ceph_msg_new(0, pool->front_len, gfp_mask, true); + msg = ceph_msg_new(pool->type, pool->front_len, gfp_mask, true); if (!msg) { dout("msgpool_alloc %s failed\n", pool->name); } else { @@ -32,10 +32,11 @@ static void msgpool_free(void *element, void *arg) ceph_msg_put(msg); } -int ceph_msgpool_init(struct ceph_msgpool *pool, +int ceph_msgpool_init(struct ceph_msgpool *pool, int type, int front_len, int size, bool blocking, const char *name) { dout("msgpool %s init\n", name); + pool->type = type; pool->front_len = front_len; pool->pool = mempool_create(size, msgpool_alloc, msgpool_free, pool); if (!pool->pool) @@ -61,7 +62,7 @@ struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool, WARN_ON(1); /* try to alloc a fresh message */ - return ceph_msg_new(0, front_len, GFP_NOFS, false); + return ceph_msg_new(pool->type, front_len, GFP_NOFS, false); } msg = mempool_alloc(pool->pool, GFP_NOFS); diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index c2527113d2a..4475d17863e 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -242,6 +242,7 @@ struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc, } ceph_pagelist_init(req->r_trail); } + /* create request message; allow space for oid */ msg_size += MAX_OBJ_NAME_SIZE; if (snapc) @@ -255,7 +256,6 @@ struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc, return NULL; } - msg->hdr.type = cpu_to_le16(CEPH_MSG_OSD_OP); memset(msg->front.iov_base, 0, msg->front.iov_len); req->r_request = msg; @@ -1837,11 +1837,12 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) if (!osdc->req_mempool) goto out; - err = ceph_msgpool_init(&osdc->msgpool_op, OSD_OP_FRONT_LEN, 10, true, + err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, + OSD_OP_FRONT_LEN, 10, true, "osd_op"); if (err < 0) goto out_mempool; - err = ceph_msgpool_init(&osdc->msgpool_op_reply, + err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, OSD_OPREPLY_FRONT_LEN, 10, true, "osd_op_reply"); if (err < 0) -- cgit v1.2.3-70-g09d2 From a16cb1f70799c851410d9dca0a24122e258df06c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 10 Jul 2012 11:53:34 -0700 Subject: libceph: fix messenger retry In ancient times, the messenger could both initiate and accept connections. An artifact if that was data structures to store/process an incoming ceph_msg_connect request and send an outgoing ceph_msg_connect_reply. Sadly, the negotiation code was referencing those structures and ignoring important information (like the peer's connect_seq) from the correct ones. Among other things, this fixes tight reconnect loops where the server sends RETRY_SESSION and we (the client) retries with the same connect_seq as last time. This bug pretty easily triggered by injecting socket failures on the MDS and running some fs workload like workunits/direct_io/test_sync_io. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 12 ++---------- net/ceph/messenger.c | 12 ++++++------ 2 files changed, 8 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index 478f814f210..cfb1bbdac62 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -172,16 +172,8 @@ struct ceph_connection { /* connection negotiation temps */ char in_banner[CEPH_BANNER_MAX_LEN]; - union { - struct { /* outgoing connection */ - struct ceph_msg_connect out_connect; - struct ceph_msg_connect_reply in_reply; - }; - struct { /* incoming */ - struct ceph_msg_connect in_connect; - struct ceph_msg_connect_reply out_reply; - }; - }; + struct ceph_msg_connect out_connect; + struct ceph_msg_connect_reply in_reply; struct ceph_entity_addr actual_peer_addr; /* message out temps */ diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 09ada792487..16814d1f477 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -1540,7 +1540,7 @@ static int process_connect(struct ceph_connection *con) * dropped messages. */ dout("process_connect got RESET peer seq %u\n", - le32_to_cpu(con->in_connect.connect_seq)); + le32_to_cpu(con->in_reply.connect_seq)); pr_err("%s%lld %s connection reset\n", ENTITY_NAME(con->peer_name), ceph_pr_addr(&con->peer_addr.in_addr)); @@ -1566,10 +1566,10 @@ static int process_connect(struct ceph_connection *con) * If we sent a smaller connect_seq than the peer has, try * again with a larger value. */ - dout("process_connect got RETRY my seq = %u, peer_seq = %u\n", + dout("process_connect got RETRY_SESSION my seq %u, peer %u\n", le32_to_cpu(con->out_connect.connect_seq), - le32_to_cpu(con->in_connect.connect_seq)); - con->connect_seq = le32_to_cpu(con->in_connect.connect_seq); + le32_to_cpu(con->in_reply.connect_seq)); + con->connect_seq = le32_to_cpu(con->in_reply.connect_seq); ret = prepare_write_connect(con); if (ret < 0) return ret; @@ -1583,9 +1583,9 @@ static int process_connect(struct ceph_connection *con) */ dout("process_connect got RETRY_GLOBAL my %u peer_gseq %u\n", con->peer_global_seq, - le32_to_cpu(con->in_connect.global_seq)); + le32_to_cpu(con->in_reply.global_seq)); get_global_seq(con->msgr, - le32_to_cpu(con->in_connect.global_seq)); + le32_to_cpu(con->in_reply.global_seq)); ret = prepare_write_connect(con); if (ret < 0) return ret; -- cgit v1.2.3-70-g09d2 From a2a3258417eb6a1799cf893350771428875a8287 Mon Sep 17 00:00:00 2001 From: Guanjun He Date: Sun, 8 Jul 2012 19:50:33 -0700 Subject: libceph: prevent the race of incoming work during teardown Add an atomic variable 'stopping' as flag in struct ceph_messenger, set this flag to 1 in function ceph_destroy_client(), and add the condition code in function ceph_data_ready() to test the flag value, if true(1), just return. Signed-off-by: Guanjun He Reviewed-by: Sage Weil --- include/linux/ceph/messenger.h | 1 + net/ceph/ceph_common.c | 2 ++ net/ceph/messenger.c | 5 +++++ 3 files changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index cfb1bbdac62..a310d7fe6e2 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -50,6 +50,7 @@ struct ceph_messenger { struct ceph_entity_inst inst; /* my name+address */ struct ceph_entity_addr my_enc_addr; + atomic_t stopping; bool nocrc; /* diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 58b09efb528..3b45e01fa8d 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -495,6 +495,8 @@ void ceph_destroy_client(struct ceph_client *client) { dout("destroy_client %p\n", client); + atomic_set(&client->msgr.stopping, 1); + /* unmount */ ceph_osdc_stop(&client->osdc); diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 16814d1f477..63e1252d3af 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -254,6 +254,9 @@ static void con_sock_state_closed(struct ceph_connection *con) static void ceph_sock_data_ready(struct sock *sk, int count_unused) { struct ceph_connection *con = sk->sk_user_data; + if (atomic_read(&con->msgr->stopping)) { + return; + } if (sk->sk_state != TCP_CLOSE_WAIT) { dout("%s on %p state = %lu, queueing work\n", __func__, @@ -2413,6 +2416,8 @@ void ceph_messenger_init(struct ceph_messenger *msgr, encode_my_addr(msgr); msgr->nocrc = nocrc; + atomic_set(&msgr->stopping, 0); + dout("%s %p\n", __func__, msgr); } EXPORT_SYMBOL(ceph_messenger_init); -- cgit v1.2.3-70-g09d2 From c61a1abd215c1ccd6fa73104c79e79987ed3aa98 Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Tue, 3 Jul 2012 16:01:18 -0500 Subject: libceph: fix off-by-one bug in ceph_encode_filepath() There is a BUG_ON() call that doesn't account for the single byte structure version at the start of an encoded filepath in ceph_encode_filepath(). Fix that. Signed-off-by: Alex Elder Reviewed-by: Yehuda Sadeh Reviewed-by: Josh Durgin --- include/linux/ceph/decode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h index d8615dee580..bcbd66c8489 100644 --- a/include/linux/ceph/decode.h +++ b/include/linux/ceph/decode.h @@ -151,7 +151,7 @@ static inline void ceph_encode_filepath(void **p, void *end, u64 ino, const char *path) { u32 len = path ? strlen(path) : 0; - BUG_ON(*p + sizeof(ino) + sizeof(len) + len > end); + BUG_ON(*p + 1 + sizeof(ino) + sizeof(len) + len > end); ceph_encode_8(p, 1); ceph_encode_64(p, ino); ceph_encode_32(p, len); -- cgit v1.2.3-70-g09d2 From f8c36c58accd5c53a472b5c289910565b3df9f9d Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Wed, 11 Jul 2012 08:24:45 -0500 Subject: libceph: define ceph_extract_encoded_string() This adds a new utility routine which will return a dynamically- allocated buffer containing a string that has been decoded from ceph over-the-wire format. It also returns the length of the string if the address of a size variable is supplied to receive it. Signed-off-by: Alex Elder Reviewed-by: Sage Weil --- include/linux/ceph/decode.h | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) (limited to 'include') diff --git a/include/linux/ceph/decode.h b/include/linux/ceph/decode.h index bcbd66c8489..4bbf2db45f4 100644 --- a/include/linux/ceph/decode.h +++ b/include/linux/ceph/decode.h @@ -1,6 +1,7 @@ #ifndef __CEPH_DECODE_H #define __CEPH_DECODE_H +#include #include #include #include @@ -84,6 +85,52 @@ static inline int ceph_has_room(void **p, void *end, size_t n) ceph_decode_copy(p, pv, n); \ } while (0) +/* + * Allocate a buffer big enough to hold the wire-encoded string, and + * decode the string into it. The resulting string will always be + * terminated with '\0'. If successful, *p will be advanced + * past the decoded data. Also, if lenp is not a null pointer, the + * length (not including the terminating '\0') will be recorded in + * *lenp. Note that a zero-length string is a valid return value. + * + * Returns a pointer to the newly-allocated string buffer, or a + * pointer-coded errno if an error occurs. Neither *p nor *lenp + * will have been updated if an error is returned. + * + * There are two possible failures: + * - converting the string would require accessing memory at or + * beyond the "end" pointer provided (-E + * - memory could not be allocated for the result + */ +static inline char *ceph_extract_encoded_string(void **p, void *end, + size_t *lenp, gfp_t gfp) +{ + u32 len; + void *sp = *p; + char *buf; + + ceph_decode_32_safe(&sp, end, len, bad); + if (!ceph_has_room(&sp, end, len)) + goto bad; + + buf = kmalloc(len + 1, gfp); + if (!buf) + return ERR_PTR(-ENOMEM); + + if (len) + memcpy(buf, sp, len); + buf[len] = '\0'; + + *p = (char *) *p + sizeof (u32) + len; + if (lenp) + *lenp = (size_t) len; + + return buf; + +bad: + return ERR_PTR(-ERANGE); +} + /* * struct ceph_timespec <-> struct timespec */ -- cgit v1.2.3-70-g09d2 From 1fe60e51a3744528f3939b1b1167ca909133d9ae Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 30 Jul 2012 16:23:22 -0700 Subject: libceph: move feature bits to separate header This is simply cleanup that will keep things more closely synced with the userland code. Signed-off-by: Sage Weil Reviewed-by: Alex Elder Reviewed-by: Yehuda Sadeh --- fs/ceph/mds_client.c | 1 + fs/ceph/super.c | 1 + include/linux/ceph/ceph_features.h | 24 ++++++++++++++++++++++++ include/linux/ceph/ceph_fs.h | 14 -------------- include/linux/ceph/libceph.h | 6 ------ net/ceph/ceph_common.c | 5 +++-- 6 files changed, 29 insertions(+), 22 deletions(-) create mode 100644 include/linux/ceph/ceph_features.h (limited to 'include') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 418f6a82c90..39b76d66bc5 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -10,6 +10,7 @@ #include "super.h" #include "mds_client.h" +#include #include #include #include diff --git a/fs/ceph/super.c b/fs/ceph/super.c index 1e67dd7305a..2c47ecfe437 100644 --- a/fs/ceph/super.c +++ b/fs/ceph/super.c @@ -18,6 +18,7 @@ #include "super.h" #include "mds_client.h" +#include #include #include #include diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h new file mode 100644 index 00000000000..342f93dbe16 --- /dev/null +++ b/include/linux/ceph/ceph_features.h @@ -0,0 +1,24 @@ +#ifndef __CEPH_FEATURES +#define __CEPH_FEATURES + +/* + * feature bits + */ +#define CEPH_FEATURE_UID (1<<0) +#define CEPH_FEATURE_NOSRCADDR (1<<1) +#define CEPH_FEATURE_MONCLOCKCHECK (1<<2) +#define CEPH_FEATURE_FLOCK (1<<3) +#define CEPH_FEATURE_SUBSCRIBE2 (1<<4) +#define CEPH_FEATURE_MONNAMES (1<<5) +#define CEPH_FEATURE_RECONNECT_SEQ (1<<6) +#define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) + +/* + * Features supported. + */ +#define CEPH_FEATURES_SUPPORTED_DEFAULT \ + (CEPH_FEATURE_NOSRCADDR) + +#define CEPH_FEATURES_REQUIRED_DEFAULT \ + (CEPH_FEATURE_NOSRCADDR) +#endif diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index e81ab30d489..d021610efd6 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -35,20 +35,6 @@ /* arbitrary limit on max # of monitors (cluster of 3 is typical) */ #define CEPH_MAX_MON 31 - -/* - * feature bits - */ -#define CEPH_FEATURE_UID (1<<0) -#define CEPH_FEATURE_NOSRCADDR (1<<1) -#define CEPH_FEATURE_MONCLOCKCHECK (1<<2) -#define CEPH_FEATURE_FLOCK (1<<3) -#define CEPH_FEATURE_SUBSCRIBE2 (1<<4) -#define CEPH_FEATURE_MONNAMES (1<<5) -#define CEPH_FEATURE_RECONNECT_SEQ (1<<6) -#define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) - - /* * ceph_file_layout - describe data layout for a file/inode */ diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 98ec36ae8a3..ea072e1f9db 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -22,12 +22,6 @@ #include "osd_client.h" #include "ceph_fs.h" -/* - * Supported features - */ -#define CEPH_FEATURE_SUPPORTED_DEFAULT CEPH_FEATURE_NOSRCADDR -#define CEPH_FEATURE_REQUIRED_DEFAULT CEPH_FEATURE_NOSRCADDR - /* * mount options */ diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 3b45e01fa8d..69e38db28e5 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -17,6 +17,7 @@ #include +#include #include #include #include @@ -460,9 +461,9 @@ struct ceph_client *ceph_create_client(struct ceph_options *opt, void *private, client->auth_err = 0; client->extra_mon_dispatch = NULL; - client->supported_features = CEPH_FEATURE_SUPPORTED_DEFAULT | + client->supported_features = CEPH_FEATURES_SUPPORTED_DEFAULT | supported_features; - client->required_features = CEPH_FEATURE_REQUIRED_DEFAULT | + client->required_features = CEPH_FEATURES_REQUIRED_DEFAULT | required_features; /* msgr */ -- cgit v1.2.3-70-g09d2 From 546f04ef716dd49521774653d8b032a7d64c05d9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 30 Jul 2012 18:15:23 -0700 Subject: libceph: support crush tunables The server side recently added support for tuning some magic crush variables. Decode these variables if they are present, or use the default values if they are not present. Corresponds to ceph.git commit 89af369c25f274fe62ef730e5e8aad0c54f1e5a5. Signed-off-by: caleb miles Reviewed-by: Sage Weil Reviewed-by: Alex Elder Reviewed-by: Yehuda Sadeh --- include/linux/ceph/ceph_features.h | 5 ++++- include/linux/crush/crush.h | 8 ++++++++ net/ceph/crush/mapper.c | 13 +++++++------ net/ceph/osdmap.c | 39 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h index 342f93dbe16..dad579b0c0e 100644 --- a/include/linux/ceph/ceph_features.h +++ b/include/linux/ceph/ceph_features.h @@ -12,12 +12,15 @@ #define CEPH_FEATURE_MONNAMES (1<<5) #define CEPH_FEATURE_RECONNECT_SEQ (1<<6) #define CEPH_FEATURE_DIRLAYOUTHASH (1<<7) +/* bits 8-17 defined by user-space; not supported yet here */ +#define CEPH_FEATURE_CRUSH_TUNABLES (1<<18) /* * Features supported. */ #define CEPH_FEATURES_SUPPORTED_DEFAULT \ - (CEPH_FEATURE_NOSRCADDR) + (CEPH_FEATURE_NOSRCADDR | \ + CEPH_FEATURE_CRUSH_TUNABLES) #define CEPH_FEATURES_REQUIRED_DEFAULT \ (CEPH_FEATURE_NOSRCADDR) diff --git a/include/linux/crush/crush.h b/include/linux/crush/crush.h index 7c4750811b9..25baa287cff 100644 --- a/include/linux/crush/crush.h +++ b/include/linux/crush/crush.h @@ -154,6 +154,14 @@ struct crush_map { __s32 max_buckets; __u32 max_rules; __s32 max_devices; + + /* choose local retries before re-descent */ + __u32 choose_local_tries; + /* choose local attempts using a fallback permutation before + * re-descent */ + __u32 choose_local_fallback_tries; + /* choose attempts before giving up */ + __u32 choose_total_tries; }; diff --git a/net/ceph/crush/mapper.c b/net/ceph/crush/mapper.c index d7edc24333b..35fce755ce1 100644 --- a/net/ceph/crush/mapper.c +++ b/net/ceph/crush/mapper.c @@ -306,7 +306,6 @@ static int crush_choose(const struct crush_map *map, int item = 0; int itemtype; int collide, reject; - const unsigned int orig_tries = 5; /* attempts before we fall back to search */ dprintk("CHOOSE%s bucket %d x %d outpos %d numrep %d\n", recurse_to_leaf ? "_LEAF" : "", bucket->id, x, outpos, numrep); @@ -351,8 +350,9 @@ static int crush_choose(const struct crush_map *map, reject = 1; goto reject; } - if (flocal >= (in->size>>1) && - flocal > orig_tries) + if (map->choose_local_fallback_tries > 0 && + flocal >= (in->size>>1) && + flocal > map->choose_local_fallback_tries) item = bucket_perm_choose(in, x, r); else item = crush_bucket_choose(in, x, r); @@ -422,13 +422,14 @@ reject: ftotal++; flocal++; - if (collide && flocal < 3) + if (collide && flocal <= map->choose_local_tries) /* retry locally a few times */ retry_bucket = 1; - else if (flocal <= in->size + orig_tries) + else if (map->choose_local_fallback_tries > 0 && + flocal <= in->size + map->choose_local_fallback_tries) /* exhaustive bucket search */ retry_bucket = 1; - else if (ftotal < 20) + else if (ftotal <= map->choose_total_tries) /* then retry descent */ retry_descent = 1; else diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index 9600674c2c3..3124b71a888 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -135,6 +135,21 @@ bad: return -EINVAL; } +static int skip_name_map(void **p, void *end) +{ + int len; + ceph_decode_32_safe(p, end, len ,bad); + while (len--) { + int strlen; + *p += sizeof(u32); + ceph_decode_32_safe(p, end, strlen, bad); + *p += strlen; +} + return 0; +bad: + return -EINVAL; +} + static struct crush_map *crush_decode(void *pbyval, void *end) { struct crush_map *c; @@ -143,6 +158,7 @@ static struct crush_map *crush_decode(void *pbyval, void *end) void **p = &pbyval; void *start = pbyval; u32 magic; + u32 num_name_maps; dout("crush_decode %p to %p len %d\n", *p, end, (int)(end - *p)); @@ -150,6 +166,11 @@ static struct crush_map *crush_decode(void *pbyval, void *end) if (c == NULL) return ERR_PTR(-ENOMEM); + /* set tunables to default values */ + c->choose_local_tries = 2; + c->choose_local_fallback_tries = 5; + c->choose_total_tries = 19; + ceph_decode_need(p, end, 4*sizeof(u32), bad); magic = ceph_decode_32(p); if (magic != CRUSH_MAGIC) { @@ -297,7 +318,25 @@ static struct crush_map *crush_decode(void *pbyval, void *end) } /* ignore trailing name maps. */ + for (num_name_maps = 0; num_name_maps < 3; num_name_maps++) { + err = skip_name_map(p, end); + if (err < 0) + goto done; + } + + /* tunables */ + ceph_decode_need(p, end, 3*sizeof(u32), done); + c->choose_local_tries = ceph_decode_32(p); + c->choose_local_fallback_tries = ceph_decode_32(p); + c->choose_total_tries = ceph_decode_32(p); + dout("crush decode tunable choose_local_tries = %d", + c->choose_local_tries); + dout("crush decode tunable choose_local_fallback_tries = %d", + c->choose_local_fallback_tries); + dout("crush decode tunable choose_total_tries = %d", + c->choose_total_tries); +done: dout("crush_decode success\n"); return c; -- cgit v1.2.3-70-g09d2 From aa711ee3402ad10ffd5b70ce0417fadc9a95cccf Mon Sep 17 00:00:00 2001 From: Alex Elder Date: Fri, 13 Jul 2012 20:35:11 -0500 Subject: ceph: define snap counts as u32 everywhere There are two structures in which a count of snapshots are maintained: struct ceph_snap_context { ... u32 num_snaps; ... } and struct ceph_snap_realm { ... u32 num_prior_parent_snaps; /* had prior to parent_since */ ... u32 num_snaps; ... } These fields never take on negative values (e.g., to hold special meaning), and so are really inherently unsigned. Furthermore they take their value from over-the-wire or on-disk formatted 32-bit values. So change their definition to have type u32, and change some spots elsewhere in the code to account for this change. Signed-off-by: Alex Elder Reviewed-by: Josh Durgin --- fs/ceph/snap.c | 18 ++++++++++-------- fs/ceph/super.h | 4 ++-- include/linux/ceph/libceph.h | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c index e5206fc7656..cbb2f54a301 100644 --- a/fs/ceph/snap.c +++ b/fs/ceph/snap.c @@ -296,8 +296,7 @@ static int build_snap_context(struct ceph_snap_realm *realm) struct ceph_snap_realm *parent = realm->parent; struct ceph_snap_context *snapc; int err = 0; - int i; - int num = realm->num_prior_parent_snaps + realm->num_snaps; + u32 num = realm->num_prior_parent_snaps + realm->num_snaps; /* * build parent context, if it hasn't been built. @@ -321,11 +320,11 @@ static int build_snap_context(struct ceph_snap_realm *realm) realm->cached_context->seq == realm->seq && (!parent || realm->cached_context->seq >= parent->cached_context->seq)) { - dout("build_snap_context %llx %p: %p seq %lld (%d snaps)" + dout("build_snap_context %llx %p: %p seq %lld (%u snaps)" " (unchanged)\n", realm->ino, realm, realm->cached_context, realm->cached_context->seq, - realm->cached_context->num_snaps); + (unsigned int) realm->cached_context->num_snaps); return 0; } @@ -342,6 +341,8 @@ static int build_snap_context(struct ceph_snap_realm *realm) num = 0; snapc->seq = realm->seq; if (parent) { + u32 i; + /* include any of parent's snaps occurring _after_ my parent became my parent */ for (i = 0; i < parent->cached_context->num_snaps; i++) @@ -361,8 +362,9 @@ static int build_snap_context(struct ceph_snap_realm *realm) sort(snapc->snaps, num, sizeof(u64), cmpu64_rev, NULL); snapc->num_snaps = num; - dout("build_snap_context %llx %p: %p seq %lld (%d snaps)\n", - realm->ino, realm, snapc, snapc->seq, snapc->num_snaps); + dout("build_snap_context %llx %p: %p seq %lld (%u snaps)\n", + realm->ino, realm, snapc, snapc->seq, + (unsigned int) snapc->num_snaps); if (realm->cached_context) ceph_put_snap_context(realm->cached_context); @@ -402,9 +404,9 @@ static void rebuild_snap_realms(struct ceph_snap_realm *realm) * helper to allocate and decode an array of snapids. free prior * instance, if any. */ -static int dup_array(u64 **dst, __le64 *src, int num) +static int dup_array(u64 **dst, __le64 *src, u32 num) { - int i; + u32 i; kfree(*dst); if (num) { diff --git a/fs/ceph/super.h b/fs/ceph/super.h index fc35036d258..3ea48b7b98b 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -612,9 +612,9 @@ struct ceph_snap_realm { u64 parent_since; /* snapid when our current parent became so */ u64 *prior_parent_snaps; /* snaps inherited from any parents we */ - int num_prior_parent_snaps; /* had prior to parent_since */ + u32 num_prior_parent_snaps; /* had prior to parent_since */ u64 *snaps; /* snaps specific to this realm */ - int num_snaps; + u32 num_snaps; struct ceph_snap_realm *parent; struct list_head children; /* list of child realms */ diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index ea072e1f9db..42624789b06 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -154,7 +154,7 @@ struct ceph_client { struct ceph_snap_context { atomic_t nref; u64 seq; - int num_snaps; + u32 num_snaps; u64 snaps[]; }; -- cgit v1.2.3-70-g09d2 From 8dacc7da69a491c515851e68de6036f21b5663ce Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 20 Jul 2012 17:24:40 -0700 Subject: libceph: replace connection state bits with states Use a simple set of 6 enumerated values for the socket states (CON_STATE_*) and use those instead of the state bits. All of the con->state checks are now under the protection of the con mutex, so this is safe. It also simplifies many of the state checks because we can check for anything other than the expected state instead of various bits for races we can think of. This appears to hold up well to stress testing both with and without socket failure injection on the server side. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 12 ---- net/ceph/messenger.c | 130 +++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 74 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index a310d7fe6e2..d9c2b8f5abd 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -116,18 +116,6 @@ struct ceph_msg_pos { #define SOCK_CLOSED 11 /* socket state changed to closed */ #define BACKOFF 15 -/* - * ceph_connection states - */ -#define CONNECTING 1 -#define NEGOTIATING 2 -#define CONNECTED 5 -#define STANDBY 8 /* no outgoing messages, socket closed. we keep - * the ceph_connection around to maintain shared - * state with the peer. */ -#define CLOSED 10 /* we've closed the connection */ -#define OPENING 13 /* open connection w/ (possibly new) peer */ - /* * A single connection with another host. * diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index e7320cd5fdb..563e46aa4d6 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -77,6 +77,17 @@ #define CON_SOCK_STATE_CONNECTED 3 /* -> CLOSING or -> CLOSED */ #define CON_SOCK_STATE_CLOSING 4 /* -> CLOSED */ +/* + * connection states + */ +#define CON_STATE_CLOSED 1 /* -> PREOPEN */ +#define CON_STATE_PREOPEN 2 /* -> CONNECTING, CLOSED */ +#define CON_STATE_CONNECTING 3 /* -> NEGOTIATING, CLOSED */ +#define CON_STATE_NEGOTIATING 4 /* -> OPEN, CLOSED */ +#define CON_STATE_OPEN 5 /* -> STANDBY, CLOSED */ +#define CON_STATE_STANDBY 6 /* -> PREOPEN, CLOSED */ + + /* static tag bytes (protocol control messages) */ static char tag_msg = CEPH_MSGR_TAG_MSG; static char tag_ack = CEPH_MSGR_TAG_ACK; @@ -503,11 +514,7 @@ void ceph_con_close(struct ceph_connection *con) mutex_lock(&con->mutex); dout("con_close %p peer %s\n", con, ceph_pr_addr(&con->peer_addr.in_addr)); - clear_bit(NEGOTIATING, &con->state); - clear_bit(CONNECTING, &con->state); - clear_bit(CONNECTED, &con->state); - clear_bit(STANDBY, &con->state); /* avoid connect_seq bump */ - set_bit(CLOSED, &con->state); + con->state = CON_STATE_CLOSED; clear_bit(LOSSYTX, &con->flags); /* so we retry next connect */ clear_bit(KEEPALIVE_PENDING, &con->flags); @@ -530,8 +537,9 @@ void ceph_con_open(struct ceph_connection *con, { mutex_lock(&con->mutex); dout("con_open %p %s\n", con, ceph_pr_addr(&addr->in_addr)); - set_bit(OPENING, &con->state); - WARN_ON(!test_and_clear_bit(CLOSED, &con->state)); + + BUG_ON(con->state != CON_STATE_CLOSED); + con->state = CON_STATE_PREOPEN; con->peer_name.type = (__u8) entity_type; con->peer_name.num = cpu_to_le64(entity_num); @@ -571,7 +579,7 @@ void ceph_con_init(struct ceph_connection *con, void *private, INIT_LIST_HEAD(&con->out_sent); INIT_DELAYED_WORK(&con->work, con_work); - set_bit(CLOSED, &con->state); + con->state = CON_STATE_CLOSED; } EXPORT_SYMBOL(ceph_con_init); @@ -809,27 +817,21 @@ static struct ceph_auth_handshake *get_connect_authorizer(struct ceph_connection if (!con->ops->get_authorizer) { con->out_connect.authorizer_protocol = CEPH_AUTH_UNKNOWN; con->out_connect.authorizer_len = 0; - return NULL; } /* Can't hold the mutex while getting authorizer */ - mutex_unlock(&con->mutex); - auth = con->ops->get_authorizer(con, auth_proto, con->auth_retry); - mutex_lock(&con->mutex); if (IS_ERR(auth)) return auth; - if (test_bit(CLOSED, &con->state) || test_bit(OPENING, &con->flags)) + if (con->state != CON_STATE_NEGOTIATING) return ERR_PTR(-EAGAIN); con->auth_reply_buf = auth->authorizer_reply_buf; con->auth_reply_buf_len = auth->authorizer_reply_buf_len; - - return auth; } @@ -1484,7 +1486,8 @@ static int process_banner(struct ceph_connection *con) static void fail_protocol(struct ceph_connection *con) { reset_connection(con); - set_bit(CLOSED, &con->state); /* in case there's queued work */ + BUG_ON(con->state != CON_STATE_NEGOTIATING); + con->state = CON_STATE_CLOSED; } static int process_connect(struct ceph_connection *con) @@ -1558,8 +1561,7 @@ static int process_connect(struct ceph_connection *con) if (con->ops->peer_reset) con->ops->peer_reset(con); mutex_lock(&con->mutex); - if (test_bit(CLOSED, &con->state) || - test_bit(OPENING, &con->state)) + if (con->state != CON_STATE_NEGOTIATING) return -EAGAIN; break; @@ -1605,8 +1607,10 @@ static int process_connect(struct ceph_connection *con) fail_protocol(con); return -1; } - clear_bit(NEGOTIATING, &con->state); - set_bit(CONNECTED, &con->state); + + BUG_ON(con->state != CON_STATE_NEGOTIATING); + con->state = CON_STATE_OPEN; + con->peer_global_seq = le32_to_cpu(con->in_reply.global_seq); con->connect_seq++; con->peer_features = server_feat; @@ -1994,8 +1998,9 @@ more: dout("try_write out_kvec_bytes %d\n", con->out_kvec_bytes); /* open the socket first? */ - if (con->sock == NULL) { - set_bit(CONNECTING, &con->state); + if (con->state == CON_STATE_PREOPEN) { + BUG_ON(con->sock); + con->state = CON_STATE_CONNECTING; con_out_kvec_reset(con); prepare_write_banner(con); @@ -2046,8 +2051,7 @@ more_kvec: } do_next: - if (!test_bit(CONNECTING, &con->state) && - !test_bit(NEGOTIATING, &con->state)) { + if (con->state == CON_STATE_OPEN) { /* is anything else pending? */ if (!list_empty(&con->out_queue)) { prepare_write_message(con); @@ -2081,29 +2085,19 @@ static int try_read(struct ceph_connection *con) { int ret = -1; - if (!con->sock) - return 0; - - if (test_bit(STANDBY, &con->state)) +more: + dout("try_read start on %p state %lu\n", con, con->state); + if (con->state != CON_STATE_CONNECTING && + con->state != CON_STATE_NEGOTIATING && + con->state != CON_STATE_OPEN) return 0; - dout("try_read start on %p\n", con); + BUG_ON(!con->sock); -more: dout("try_read tag %d in_base_pos %d\n", (int)con->in_tag, con->in_base_pos); - /* - * process_connect and process_message drop and re-take - * con->mutex. make sure we handle a racing close or reopen. - */ - if (test_bit(CLOSED, &con->state) || - test_bit(OPENING, &con->state)) { - ret = -EAGAIN; - goto out; - } - - if (test_bit(CONNECTING, &con->state)) { + if (con->state == CON_STATE_CONNECTING) { dout("try_read connecting\n"); ret = read_partial_banner(con); if (ret <= 0) @@ -2112,8 +2106,8 @@ more: if (ret < 0) goto out; - clear_bit(CONNECTING, &con->state); - set_bit(NEGOTIATING, &con->state); + BUG_ON(con->state != CON_STATE_CONNECTING); + con->state = CON_STATE_NEGOTIATING; /* Banner is good, exchange connection info */ ret = prepare_write_connect(con); @@ -2125,7 +2119,7 @@ more: goto out; } - if (test_bit(NEGOTIATING, &con->state)) { + if (con->state == CON_STATE_NEGOTIATING) { dout("try_read negotiating\n"); ret = read_partial_connect(con); if (ret <= 0) @@ -2136,6 +2130,8 @@ more: goto more; } + BUG_ON(con->state != CON_STATE_OPEN); + if (con->in_base_pos < 0) { /* * skipping + discarding content. @@ -2169,8 +2165,8 @@ more: prepare_read_ack(con); break; case CEPH_MSGR_TAG_CLOSE: - clear_bit(CONNECTED, &con->state); - set_bit(CLOSED, &con->state); /* fixme */ + con_close_socket(con); + con->state = CON_STATE_CLOSED; goto out; default: goto bad_tag; @@ -2246,14 +2242,21 @@ static void con_work(struct work_struct *work) mutex_lock(&con->mutex); restart: if (test_and_clear_bit(SOCK_CLOSED, &con->flags)) { - if (test_and_clear_bit(CONNECTED, &con->state)) - con->error_msg = "socket closed"; - else if (test_and_clear_bit(NEGOTIATING, &con->state)) - con->error_msg = "negotiation failed"; - else if (test_and_clear_bit(CONNECTING, &con->state)) + switch (con->state) { + case CON_STATE_CONNECTING: con->error_msg = "connection failed"; - else + break; + case CON_STATE_NEGOTIATING: + con->error_msg = "negotiation failed"; + break; + case CON_STATE_OPEN: + con->error_msg = "socket closed"; + break; + default: + dout("unrecognized con state %d\n", (int)con->state); con->error_msg = "unrecognized con state"; + BUG(); + } goto fault; } @@ -2271,17 +2274,16 @@ restart: } } - if (test_bit(STANDBY, &con->state)) { + if (con->state == CON_STATE_STANDBY) { dout("con_work %p STANDBY\n", con); goto done; } - if (test_bit(CLOSED, &con->state)) { + if (con->state == CON_STATE_CLOSED) { dout("con_work %p CLOSED\n", con); BUG_ON(con->sock); goto done; } - if (test_and_clear_bit(OPENING, &con->state)) { - /* reopen w/ new peer */ + if (con->state == CON_STATE_PREOPEN) { dout("con_work OPENING\n"); BUG_ON(con->sock); } @@ -2328,13 +2330,15 @@ static void ceph_fault(struct ceph_connection *con) dout("fault %p state %lu to peer %s\n", con, con->state, ceph_pr_addr(&con->peer_addr.in_addr)); - if (test_bit(CLOSED, &con->state)) - goto out_unlock; + BUG_ON(con->state != CON_STATE_CONNECTING && + con->state != CON_STATE_NEGOTIATING && + con->state != CON_STATE_OPEN); con_close_socket(con); if (test_bit(LOSSYTX, &con->flags)) { - dout("fault on LOSSYTX channel\n"); + dout("fault on LOSSYTX channel, marking CLOSED\n"); + con->state = CON_STATE_CLOSED; goto out_unlock; } @@ -2355,9 +2359,10 @@ static void ceph_fault(struct ceph_connection *con) !test_bit(KEEPALIVE_PENDING, &con->flags)) { dout("fault %p setting STANDBY clearing WRITE_PENDING\n", con); clear_bit(WRITE_PENDING, &con->flags); - set_bit(STANDBY, &con->state); + con->state = CON_STATE_STANDBY; } else { /* retry after a delay. */ + con->state = CON_STATE_PREOPEN; if (con->delay == 0) con->delay = BASE_DELAY_INTERVAL; else if (con->delay < MAX_DELAY_INTERVAL) @@ -2431,8 +2436,9 @@ EXPORT_SYMBOL(ceph_messenger_init); static void clear_standby(struct ceph_connection *con) { /* come back from STANDBY? */ - if (test_and_clear_bit(STANDBY, &con->state)) { + if (con->state == CON_STATE_STANDBY) { dout("clear_standby %p and ++connect_seq\n", con); + con->state = CON_STATE_PREOPEN; con->connect_seq++; WARN_ON(test_bit(WRITE_PENDING, &con->flags)); WARN_ON(test_bit(KEEPALIVE_PENDING, &con->flags)); @@ -2451,7 +2457,7 @@ void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg) mutex_lock(&con->mutex); - if (test_bit(CLOSED, &con->state)) { + if (con->state == CON_STATE_CLOSED) { dout("con_send %p closed, dropping %p\n", con, msg); ceph_msg_put(msg); mutex_unlock(&con->mutex); -- cgit v1.2.3-70-g09d2 From 4a8616920860920abaa51193146fe36b38ef09aa Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 20 Jul 2012 17:29:55 -0700 Subject: libceph: clean up con flags Rename flags with CON_FLAG prefix, move the definitions into the c file, and (better) document their meaning. Signed-off-by: Sage Weil --- include/linux/ceph/messenger.h | 10 ------- net/ceph/messenger.c | 62 ++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 36 deletions(-) (limited to 'include') diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index d9c2b8f5abd..189ae063763 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -106,16 +106,6 @@ struct ceph_msg_pos { #define BASE_DELAY_INTERVAL (HZ/2) #define MAX_DELAY_INTERVAL (5 * 60 * HZ) -/* - * ceph_connection flag bits - */ - -#define LOSSYTX 0 /* we can close channel or drop messages on errors */ -#define KEEPALIVE_PENDING 3 -#define WRITE_PENDING 4 /* we have data ready to send */ -#define SOCK_CLOSED 11 /* socket state changed to closed */ -#define BACKOFF 15 - /* * A single connection with another host. * diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 563e46aa4d6..b872db5c498 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -87,6 +87,15 @@ #define CON_STATE_OPEN 5 /* -> STANDBY, CLOSED */ #define CON_STATE_STANDBY 6 /* -> PREOPEN, CLOSED */ +/* + * ceph_connection flag bits + */ +#define CON_FLAG_LOSSYTX 0 /* we can close channel or drop + * messages on errors */ +#define CON_FLAG_KEEPALIVE_PENDING 1 /* we need to send a keepalive */ +#define CON_FLAG_WRITE_PENDING 2 /* we have data ready to send */ +#define CON_FLAG_SOCK_CLOSED 3 /* socket state changed to closed */ +#define CON_FLAG_BACKOFF 4 /* need to retry queuing delayed work */ /* static tag bytes (protocol control messages) */ static char tag_msg = CEPH_MSGR_TAG_MSG; @@ -288,7 +297,7 @@ static void ceph_sock_write_space(struct sock *sk) * buffer. See net/ipv4/tcp_input.c:tcp_check_space() * and net/core/stream.c:sk_stream_write_space(). */ - if (test_bit(WRITE_PENDING, &con->flags)) { + if (test_bit(CON_FLAG_WRITE_PENDING, &con->flags)) { if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) { dout("%s %p queueing write work\n", __func__, con); clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); @@ -313,7 +322,7 @@ static void ceph_sock_state_change(struct sock *sk) case TCP_CLOSE_WAIT: dout("%s TCP_CLOSE_WAIT\n", __func__); con_sock_state_closing(con); - set_bit(SOCK_CLOSED, &con->flags); + set_bit(CON_FLAG_SOCK_CLOSED, &con->flags); queue_con(con); break; case TCP_ESTABLISHED: @@ -449,12 +458,12 @@ static int con_close_socket(struct ceph_connection *con) con->sock = NULL; /* - * Forcibly clear the SOCK_CLOSE flag. It gets set + * Forcibly clear the SOCK_CLOSED flag. It gets set * independent of the connection mutex, and we could have * received a socket close event before we had the chance to * shut the socket down. */ - clear_bit(SOCK_CLOSED, &con->flags); + clear_bit(CON_FLAG_SOCK_CLOSED, &con->flags); con_sock_state_closed(con); return rc; } @@ -516,9 +525,9 @@ void ceph_con_close(struct ceph_connection *con) ceph_pr_addr(&con->peer_addr.in_addr)); con->state = CON_STATE_CLOSED; - clear_bit(LOSSYTX, &con->flags); /* so we retry next connect */ - clear_bit(KEEPALIVE_PENDING, &con->flags); - clear_bit(WRITE_PENDING, &con->flags); + clear_bit(CON_FLAG_LOSSYTX, &con->flags); /* so we retry next connect */ + clear_bit(CON_FLAG_KEEPALIVE_PENDING, &con->flags); + clear_bit(CON_FLAG_WRITE_PENDING, &con->flags); reset_connection(con); con->peer_global_seq = 0; @@ -770,7 +779,7 @@ static void prepare_write_message(struct ceph_connection *con) /* no, queue up footer too and be done */ prepare_write_message_footer(con); - set_bit(WRITE_PENDING, &con->flags); + set_bit(CON_FLAG_WRITE_PENDING, &con->flags); } /* @@ -791,7 +800,7 @@ static void prepare_write_ack(struct ceph_connection *con) &con->out_temp_ack); con->out_more = 1; /* more will follow.. eventually.. */ - set_bit(WRITE_PENDING, &con->flags); + set_bit(CON_FLAG_WRITE_PENDING, &con->flags); } /* @@ -802,7 +811,7 @@ static void prepare_write_keepalive(struct ceph_connection *con) dout("prepare_write_keepalive %p\n", con); con_out_kvec_reset(con); con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive); - set_bit(WRITE_PENDING, &con->flags); + set_bit(CON_FLAG_WRITE_PENDING, &con->flags); } /* @@ -845,7 +854,7 @@ static void prepare_write_banner(struct ceph_connection *con) &con->msgr->my_enc_addr); con->out_more = 0; - set_bit(WRITE_PENDING, &con->flags); + set_bit(CON_FLAG_WRITE_PENDING, &con->flags); } static int prepare_write_connect(struct ceph_connection *con) @@ -896,7 +905,7 @@ static int prepare_write_connect(struct ceph_connection *con) auth->authorizer_buf); con->out_more = 0; - set_bit(WRITE_PENDING, &con->flags); + set_bit(CON_FLAG_WRITE_PENDING, &con->flags); return 0; } @@ -1622,7 +1631,7 @@ static int process_connect(struct ceph_connection *con) le32_to_cpu(con->in_reply.connect_seq)); if (con->in_reply.flags & CEPH_MSG_CONNECT_LOSSY) - set_bit(LOSSYTX, &con->flags); + set_bit(CON_FLAG_LOSSYTX, &con->flags); con->delay = 0; /* reset backoff memory */ @@ -2061,14 +2070,15 @@ do_next: prepare_write_ack(con); goto more; } - if (test_and_clear_bit(KEEPALIVE_PENDING, &con->flags)) { + if (test_and_clear_bit(CON_FLAG_KEEPALIVE_PENDING, + &con->flags)) { prepare_write_keepalive(con); goto more; } } /* Nothing to do! */ - clear_bit(WRITE_PENDING, &con->flags); + clear_bit(CON_FLAG_WRITE_PENDING, &con->flags); dout("try_write nothing else to write.\n"); ret = 0; out: @@ -2241,7 +2251,7 @@ static void con_work(struct work_struct *work) mutex_lock(&con->mutex); restart: - if (test_and_clear_bit(SOCK_CLOSED, &con->flags)) { + if (test_and_clear_bit(CON_FLAG_SOCK_CLOSED, &con->flags)) { switch (con->state) { case CON_STATE_CONNECTING: con->error_msg = "connection failed"; @@ -2260,7 +2270,7 @@ restart: goto fault; } - if (test_and_clear_bit(BACKOFF, &con->flags)) { + if (test_and_clear_bit(CON_FLAG_BACKOFF, &con->flags)) { dout("con_work %p backing off\n", con); if (queue_delayed_work(ceph_msgr_wq, &con->work, round_jiffies_relative(con->delay))) { @@ -2336,7 +2346,7 @@ static void ceph_fault(struct ceph_connection *con) con_close_socket(con); - if (test_bit(LOSSYTX, &con->flags)) { + if (test_bit(CON_FLAG_LOSSYTX, &con->flags)) { dout("fault on LOSSYTX channel, marking CLOSED\n"); con->state = CON_STATE_CLOSED; goto out_unlock; @@ -2356,9 +2366,9 @@ static void ceph_fault(struct ceph_connection *con) /* If there are no messages queued or keepalive pending, place * the connection in a STANDBY state */ if (list_empty(&con->out_queue) && - !test_bit(KEEPALIVE_PENDING, &con->flags)) { + !test_bit(CON_FLAG_KEEPALIVE_PENDING, &con->flags)) { dout("fault %p setting STANDBY clearing WRITE_PENDING\n", con); - clear_bit(WRITE_PENDING, &con->flags); + clear_bit(CON_FLAG_WRITE_PENDING, &con->flags); con->state = CON_STATE_STANDBY; } else { /* retry after a delay. */ @@ -2383,7 +2393,7 @@ static void ceph_fault(struct ceph_connection *con) * that when con_work restarts we schedule the * delay then. */ - set_bit(BACKOFF, &con->flags); + set_bit(CON_FLAG_BACKOFF, &con->flags); } } @@ -2440,8 +2450,8 @@ static void clear_standby(struct ceph_connection *con) dout("clear_standby %p and ++connect_seq\n", con); con->state = CON_STATE_PREOPEN; con->connect_seq++; - WARN_ON(test_bit(WRITE_PENDING, &con->flags)); - WARN_ON(test_bit(KEEPALIVE_PENDING, &con->flags)); + WARN_ON(test_bit(CON_FLAG_WRITE_PENDING, &con->flags)); + WARN_ON(test_bit(CON_FLAG_KEEPALIVE_PENDING, &con->flags)); } } @@ -2482,7 +2492,7 @@ void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg) /* if there wasn't anything waiting to send before, queue * new work */ - if (test_and_set_bit(WRITE_PENDING, &con->flags) == 0) + if (test_and_set_bit(CON_FLAG_WRITE_PENDING, &con->flags) == 0) queue_con(con); } EXPORT_SYMBOL(ceph_con_send); @@ -2571,8 +2581,8 @@ void ceph_con_keepalive(struct ceph_connection *con) mutex_lock(&con->mutex); clear_standby(con); mutex_unlock(&con->mutex); - if (test_and_set_bit(KEEPALIVE_PENDING, &con->flags) == 0 && - test_and_set_bit(WRITE_PENDING, &con->flags) == 0) + if (test_and_set_bit(CON_FLAG_KEEPALIVE_PENDING, &con->flags) == 0 && + test_and_set_bit(CON_FLAG_WRITE_PENDING, &con->flags) == 0) queue_con(con); } EXPORT_SYMBOL(ceph_con_keepalive); -- cgit v1.2.3-70-g09d2