From 8fc91fd85950d106883852c6d215614ec28cc92d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 6 Oct 2009 11:31:13 -0700 Subject: ceph: message pools The msgpool is a basic mempool_t-like structure to preallocate messages we expect to receive over the wire. This ensures we have the necessary memory preallocated to process replies to requests, or to process unsolicited messages from various servers. Signed-off-by: Sage Weil --- fs/ceph/msgpool.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 fs/ceph/msgpool.h (limited to 'fs/ceph/msgpool.h') diff --git a/fs/ceph/msgpool.h b/fs/ceph/msgpool.h new file mode 100644 index 00000000000..07a2decaa6d --- /dev/null +++ b/fs/ceph/msgpool.h @@ -0,0 +1,26 @@ +#ifndef _FS_CEPH_MSGPOOL +#define _FS_CEPH_MSGPOOL + +#include "messenger.h" + +/* + * we use memory pools for preallocating messages we may receive, to + * avoid unexpected OOM conditions. + */ +struct ceph_msgpool { + spinlock_t lock; + int front_len; /* preallocated payload size */ + struct list_head msgs; /* msgs in the pool; each has 1 ref */ + int num, min; /* cur, min # msgs in the pool */ + bool blocking; + wait_queue_head_t wait; +}; + +extern int ceph_msgpool_init(struct ceph_msgpool *pool, + int front_len, int size, bool blocking); +extern void ceph_msgpool_destroy(struct ceph_msgpool *pool); +extern int ceph_msgpool_resv(struct ceph_msgpool *, int delta); +extern struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *); +extern void ceph_msgpool_put(struct ceph_msgpool *, struct ceph_msg *); + +#endif -- cgit v1.2.3-70-g09d2 From 8f3bc053c610826a657714649ea596f07875db2e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 14 Oct 2009 17:36:07 -0700 Subject: ceph: warn on allocation from msgpool with larger front_len Pass the front_len we need when pulling a message off a msgpool, and WARN if it is greater than the pool's size. Then try to allocate a new message (to continue without failing). Signed-off-by: Sage Weil --- fs/ceph/mon_client.c | 7 ++++--- fs/ceph/msgpool.c | 20 +++++++++++++++++--- fs/ceph/msgpool.h | 3 ++- fs/ceph/osd_client.c | 5 +++-- 4 files changed, 26 insertions(+), 9 deletions(-) (limited to 'fs/ceph/msgpool.h') diff --git a/fs/ceph/mon_client.c b/fs/ceph/mon_client.c index d52e52968d0..e6e954cac6b 100644 --- a/fs/ceph/mon_client.c +++ b/fs/ceph/mon_client.c @@ -639,14 +639,15 @@ static struct ceph_msg *mon_alloc_msg(struct ceph_connection *con, { struct ceph_mon_client *monc = con->private; int type = le16_to_cpu(hdr->type); + int front = le32_to_cpu(hdr->front_len); switch (type) { case CEPH_MSG_CLIENT_MOUNT_ACK: - return ceph_msgpool_get(&monc->msgpool_mount_ack); + return ceph_msgpool_get(&monc->msgpool_mount_ack, front); case CEPH_MSG_MON_SUBSCRIBE_ACK: - return ceph_msgpool_get(&monc->msgpool_subscribe_ack); + return ceph_msgpool_get(&monc->msgpool_subscribe_ack, front); case CEPH_MSG_STATFS_REPLY: - return ceph_msgpool_get(&monc->msgpool_statfs_reply); + return ceph_msgpool_get(&monc->msgpool_statfs_reply, front); } return ceph_alloc_msg(con, hdr); } diff --git a/fs/ceph/msgpool.c b/fs/ceph/msgpool.c index 39d4d7ed82c..7599b338207 100644 --- a/fs/ceph/msgpool.c +++ b/fs/ceph/msgpool.c @@ -101,14 +101,28 @@ int ceph_msgpool_resv(struct ceph_msgpool *pool, int delta) return ret; } -struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool) +struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool, int front_len) { wait_queue_t wait; struct ceph_msg *msg; + if (front_len && front_len > pool->front_len) { + pr_err("msgpool_get pool %p need front %d, pool size is %d\n", + pool, front_len, pool->front_len); + WARN_ON(1); + + /* try to alloc a fresh message */ + msg = ceph_msg_new(0, front_len, 0, 0, NULL); + if (!IS_ERR(msg)) + return msg; + } + + if (!front_len) + front_len = pool->front_len; + if (pool->blocking) { /* mempool_t behavior; first try to alloc */ - msg = ceph_msg_new(0, pool->front_len, 0, 0, NULL); + msg = ceph_msg_new(0, front_len, 0, 0, NULL); if (!IS_ERR(msg)) return msg; } @@ -133,7 +147,7 @@ struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *pool) WARN_ON(1); /* maybe we can allocate it now? */ - msg = ceph_msg_new(0, pool->front_len, 0, 0, NULL); + msg = ceph_msg_new(0, front_len, 0, 0, NULL); if (!IS_ERR(msg)) return msg; diff --git a/fs/ceph/msgpool.h b/fs/ceph/msgpool.h index 07a2decaa6d..bc834bfcd72 100644 --- a/fs/ceph/msgpool.h +++ b/fs/ceph/msgpool.h @@ -20,7 +20,8 @@ extern int ceph_msgpool_init(struct ceph_msgpool *pool, int front_len, int size, bool blocking); extern void ceph_msgpool_destroy(struct ceph_msgpool *pool); extern int ceph_msgpool_resv(struct ceph_msgpool *, int delta); -extern struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *); +extern struct ceph_msg *ceph_msgpool_get(struct ceph_msgpool *, + int front_len); extern void ceph_msgpool_put(struct ceph_msgpool *, struct ceph_msg *); #endif diff --git a/fs/ceph/osd_client.c b/fs/ceph/osd_client.c index bbd9a5d2371..0a254054a82 100644 --- a/fs/ceph/osd_client.c +++ b/fs/ceph/osd_client.c @@ -161,7 +161,7 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc, if (snapc) msg_size += sizeof(u64) * snapc->num_snaps; if (use_mempool) - msg = ceph_msgpool_get(&osdc->msgpool_op); + msg = ceph_msgpool_get(&osdc->msgpool_op, 0); else msg = ceph_msg_new(CEPH_MSG_OSD_OP, msg_size, 0, 0, NULL); if (IS_ERR(msg)) { @@ -1271,10 +1271,11 @@ static struct ceph_msg *alloc_msg(struct ceph_connection *con, struct ceph_osd *osd = con->private; struct ceph_osd_client *osdc = osd->o_osdc; int type = le16_to_cpu(hdr->type); + int front = le32_to_cpu(hdr->front_len); switch (type) { case CEPH_MSG_OSD_OPREPLY: - return ceph_msgpool_get(&osdc->msgpool_op_reply); + return ceph_msgpool_get(&osdc->msgpool_op_reply, front); } return ceph_alloc_msg(con, hdr); } -- cgit v1.2.3-70-g09d2