From 09a423a3d6c70905f1090f01aadb8e6abff527ce Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:20 +0000 Subject: xfs: split tail_lsn assignments from log space wakeups Currently xfs_log_move_tail has a tail_lsn argument that is horribly overloaded: it may contain either an actual lsn to assign to the log tail, 0 as a special case to use the last sync LSN, or 1 to indicate that no tail LSN assignment should be performed, and we should opportunisticly wake up at one task waiting for log space even if we did not move the LSN. Remove the tail lsn assigned from xfs_log_move_tail and make the two callers use xlog_assign_tail_lsn instead of the current variant of partially using the code in xfs_log_move_tail and partially opencoding it. Note that means we grow an addition lock roundtrip on the AIL lock for each bulk update or delete, which is still far less than what we had before introducing the bulk operations. If this proves to be a problem we can still add a variant of xlog_assign_tail_lsn that expects the lock to be held already. Also rename the remainder of xfs_log_move_tail to xfs_log_space_wake as that name describes its functionality much better. Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 74 ++++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 43 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index e2cc3568c29..372642d3987 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -760,37 +760,35 @@ xfs_log_item_init( INIT_LIST_HEAD(&item->li_cil); } +/* + * Wake up processes waiting for log space after we have moved the log tail. + * + * If opportunistic is set wake up one waiter even if we do not have enough + * free space by our strict accounting. + */ void -xfs_log_move_tail(xfs_mount_t *mp, - xfs_lsn_t tail_lsn) +xfs_log_space_wake( + struct xfs_mount *mp, + bool opportunistic) { - xlog_ticket_t *tic; - xlog_t *log = mp->m_log; - int need_bytes, free_bytes; + struct xlog_ticket *tic; + struct log *log = mp->m_log; + int need_bytes, free_bytes; if (XLOG_FORCED_SHUTDOWN(log)) return; - if (tail_lsn == 0) - tail_lsn = atomic64_read(&log->l_last_sync_lsn); - - /* tail_lsn == 1 implies that we weren't passed a valid value. */ - if (tail_lsn != 1) - atomic64_set(&log->l_tail_lsn, tail_lsn); - if (!list_empty_careful(&log->l_writeq)) { -#ifdef DEBUG - if (log->l_flags & XLOG_ACTIVE_RECOVERY) - panic("Recovery problem"); -#endif + ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); + spin_lock(&log->l_grant_write_lock); free_bytes = xlog_space_left(log, &log->l_grant_write_head); list_for_each_entry(tic, &log->l_writeq, t_queue) { ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); - if (free_bytes < tic->t_unit_res && tail_lsn != 1) + if (free_bytes < tic->t_unit_res && !opportunistic) break; - tail_lsn = 0; + opportunistic = false; free_bytes -= tic->t_unit_res; trace_xfs_log_regrant_write_wake_up(log, tic); wake_up(&tic->t_wait); @@ -799,10 +797,8 @@ xfs_log_move_tail(xfs_mount_t *mp, } if (!list_empty_careful(&log->l_reserveq)) { -#ifdef DEBUG - if (log->l_flags & XLOG_ACTIVE_RECOVERY) - panic("Recovery problem"); -#endif + ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); + spin_lock(&log->l_grant_reserve_lock); free_bytes = xlog_space_left(log, &log->l_grant_reserve_head); list_for_each_entry(tic, &log->l_reserveq, t_queue) { @@ -810,9 +806,9 @@ xfs_log_move_tail(xfs_mount_t *mp, need_bytes = tic->t_unit_res*tic->t_cnt; else need_bytes = tic->t_unit_res; - if (free_bytes < need_bytes && tail_lsn != 1) + if (free_bytes < need_bytes && !opportunistic) break; - tail_lsn = 0; + opportunistic = false; free_bytes -= need_bytes; trace_xfs_log_grant_wake_up(log, tic); wake_up(&tic->t_wait); @@ -867,21 +863,7 @@ xfs_log_need_covered(xfs_mount_t *mp) return needed; } -/****************************************************************************** - * - * local routines - * - ****************************************************************************** - */ - -/* xfs_trans_tail_ail returns 0 when there is nothing in the list. - * The log manager must keep track of the last LR which was committed - * to disk. The lsn of this LR will become the new tail_lsn whenever - * xfs_trans_tail_ail returns 0. If we don't do this, we run into - * the situation where stuff could be written into the log but nothing - * was ever in the AIL when asked. Eventually, we panic since the - * tail hits the head. - * +/* * We may be holding the log iclog lock upon entering this routine. */ xfs_lsn_t @@ -891,10 +873,17 @@ xlog_assign_tail_lsn( xfs_lsn_t tail_lsn; struct log *log = mp->m_log; + /* + * To make sure we always have a valid LSN for the log tail we keep + * track of the last LSN which was committed in log->l_last_sync_lsn, + * and use that when the AIL was empty and xfs_ail_min_lsn returns 0. + * + * If the AIL has been emptied we also need to wake any process + * waiting for this condition. + */ tail_lsn = xfs_ail_min_lsn(mp->m_ail); if (!tail_lsn) tail_lsn = atomic64_read(&log->l_last_sync_lsn); - atomic64_set(&log->l_tail_lsn, tail_lsn); return tail_lsn; } @@ -2759,9 +2748,8 @@ xlog_ungrant_log_space(xlog_t *log, trace_xfs_log_ungrant_exit(log, ticket); - xfs_log_move_tail(log->l_mp, 1); -} /* xlog_ungrant_log_space */ - + xfs_log_space_wake(log->l_mp, true); +} /* * Flush iclog to disk if this is the last reference to the given iclog and -- cgit v1.2.3-70-g09d2 From 3af1de753b3caf9fa3762b4b1b85d833c121847e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:21 +0000 Subject: xfs: do exact log space wakeups in xlog_ungrant_log_space The only reason that xfs_log_space_wake had to do opportunistic wakeups was that the old xfs_log_move_tail calling convention didn't allow for exact wakeups when not updating the log tail LSN. Since this issue has been fixed we can do exact wakeups now. Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 372642d3987..9161e8a76e7 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -2748,7 +2748,7 @@ xlog_ungrant_log_space(xlog_t *log, trace_xfs_log_ungrant_exit(log, ticket); - xfs_log_space_wake(log->l_mp, true); + xfs_log_space_wake(log->l_mp, false); } /* -- cgit v1.2.3-70-g09d2 From cfb7cdca0aca5ee2e2ef491284bf1edc3b581885 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:23 +0000 Subject: xfs: cleanup xfs_log_space_wake Remove the now unused opportunistic parameter, and use the the xlog_writeq_wake and xlog_reserveq_wake helpers now that we don't have to care about the opportunistic wakeups. Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 35 +++++------------------------------ fs/xfs/xfs_log.h | 3 +-- fs/xfs/xfs_trans_ail.c | 4 ++-- 3 files changed, 8 insertions(+), 34 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 9161e8a76e7..2db39df5a57 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -762,18 +762,13 @@ xfs_log_item_init( /* * Wake up processes waiting for log space after we have moved the log tail. - * - * If opportunistic is set wake up one waiter even if we do not have enough - * free space by our strict accounting. */ void xfs_log_space_wake( - struct xfs_mount *mp, - bool opportunistic) + struct xfs_mount *mp) { - struct xlog_ticket *tic; struct log *log = mp->m_log; - int need_bytes, free_bytes; + int free_bytes; if (XLOG_FORCED_SHUTDOWN(log)) return; @@ -783,16 +778,7 @@ xfs_log_space_wake( spin_lock(&log->l_grant_write_lock); free_bytes = xlog_space_left(log, &log->l_grant_write_head); - list_for_each_entry(tic, &log->l_writeq, t_queue) { - ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); - - if (free_bytes < tic->t_unit_res && !opportunistic) - break; - opportunistic = false; - free_bytes -= tic->t_unit_res; - trace_xfs_log_regrant_write_wake_up(log, tic); - wake_up(&tic->t_wait); - } + xlog_writeq_wake(log, &free_bytes); spin_unlock(&log->l_grant_write_lock); } @@ -801,18 +787,7 @@ xfs_log_space_wake( spin_lock(&log->l_grant_reserve_lock); free_bytes = xlog_space_left(log, &log->l_grant_reserve_head); - list_for_each_entry(tic, &log->l_reserveq, t_queue) { - if (tic->t_flags & XLOG_TIC_PERM_RESERV) - need_bytes = tic->t_unit_res*tic->t_cnt; - else - need_bytes = tic->t_unit_res; - if (free_bytes < need_bytes && !opportunistic) - break; - opportunistic = false; - free_bytes -= need_bytes; - trace_xfs_log_grant_wake_up(log, tic); - wake_up(&tic->t_wait); - } + xlog_reserveq_wake(log, &free_bytes); spin_unlock(&log->l_grant_reserve_lock); } } @@ -2748,7 +2723,7 @@ xlog_ungrant_log_space(xlog_t *log, trace_xfs_log_ungrant_exit(log, ticket); - xfs_log_space_wake(log->l_mp, false); + xfs_log_space_wake(log->l_mp); } /* diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index 58d858074e6..fe32c692787 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -161,8 +161,7 @@ int xfs_log_mount(struct xfs_mount *mp, int num_bblocks); int xfs_log_mount_finish(struct xfs_mount *mp); xfs_lsn_t xlog_assign_tail_lsn(struct xfs_mount *mp); -void xfs_log_space_wake(struct xfs_mount *mp, - bool opportunistic); +void xfs_log_space_wake(struct xfs_mount *mp); int xfs_log_notify(struct xfs_mount *mp, struct xlog_in_core *iclog, xfs_log_callback_t *callback_entry); diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 9d5fc089ea2..1dead07f092 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -671,7 +671,7 @@ xfs_trans_ail_update_bulk( if (mlip_changed && !XFS_FORCED_SHUTDOWN(ailp->xa_mount)) { xlog_assign_tail_lsn(ailp->xa_mount); - xfs_log_space_wake(ailp->xa_mount, false); + xfs_log_space_wake(ailp->xa_mount); } } @@ -733,7 +733,7 @@ xfs_trans_ail_delete_bulk( if (mlip_changed && !XFS_FORCED_SHUTDOWN(ailp->xa_mount)) { xlog_assign_tail_lsn(ailp->xa_mount); - xfs_log_space_wake(ailp->xa_mount, false); + xfs_log_space_wake(ailp->xa_mount); } } -- cgit v1.2.3-70-g09d2 From 14a7235fba4302a82d61150eda92ec90d3ae9cfb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:24 +0000 Subject: xfs: remove log space waitqueues The tic->t_wait waitqueues can never have more than a single waiter on them, so we can easily replace them with a task_struct pointer and wake_up_process. Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 24 +++++++++++++++--------- fs/xfs/xfs_log_priv.h | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 2db39df5a57..02a35fba5ea 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -169,7 +169,7 @@ xlog_reserveq_wake( *free_bytes -= need_bytes; trace_xfs_log_grant_wake_up(log, tic); - wake_up(&tic->t_wait); + wake_up_process(tic->t_task); } return true; @@ -193,7 +193,7 @@ xlog_writeq_wake( *free_bytes -= need_bytes; trace_xfs_log_regrant_write_wake_up(log, tic); - wake_up(&tic->t_wait); + wake_up_process(tic->t_task); } return true; @@ -212,10 +212,13 @@ xlog_reserveq_wait( goto shutdown; xlog_grant_push_ail(log, need_bytes); + __set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock(&log->l_grant_reserve_lock); + XFS_STATS_INC(xs_sleep_logspace); - trace_xfs_log_grant_sleep(log, tic); - xlog_wait(&tic->t_wait, &log->l_grant_reserve_lock); + trace_xfs_log_grant_sleep(log, tic); + schedule(); trace_xfs_log_grant_wake(log, tic); spin_lock(&log->l_grant_reserve_lock); @@ -243,10 +246,13 @@ xlog_writeq_wait( goto shutdown; xlog_grant_push_ail(log, need_bytes); + __set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock(&log->l_grant_write_lock); + XFS_STATS_INC(xs_sleep_logspace); - trace_xfs_log_regrant_write_sleep(log, tic); - xlog_wait(&tic->t_wait, &log->l_grant_write_lock); + trace_xfs_log_regrant_write_sleep(log, tic); + schedule(); trace_xfs_log_regrant_write_wake(log, tic); spin_lock(&log->l_grant_write_lock); @@ -3276,6 +3282,7 @@ xlog_ticket_alloc( } atomic_set(&tic->t_ref, 1); + tic->t_task = current; INIT_LIST_HEAD(&tic->t_queue); tic->t_unit_res = unit_bytes; tic->t_curr_res = unit_bytes; @@ -3287,7 +3294,6 @@ xlog_ticket_alloc( tic->t_trans_type = 0; if (xflags & XFS_LOG_PERM_RESERV) tic->t_flags |= XLOG_TIC_PERM_RESERV; - init_waitqueue_head(&tic->t_wait); xlog_tic_reset_res(tic); @@ -3615,12 +3621,12 @@ xfs_log_force_umount( */ spin_lock(&log->l_grant_reserve_lock); list_for_each_entry(tic, &log->l_reserveq, t_queue) - wake_up(&tic->t_wait); + wake_up_process(tic->t_task); spin_unlock(&log->l_grant_reserve_lock); spin_lock(&log->l_grant_write_lock); list_for_each_entry(tic, &log->l_writeq, t_queue) - wake_up(&tic->t_wait); + wake_up_process(tic->t_task); spin_unlock(&log->l_grant_write_lock); if (!(log->l_iclog->ic_state & XLOG_STATE_IOERROR)) { diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index 785905e3cf0..d8c5e47bbc2 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -239,8 +239,8 @@ typedef struct xlog_res { } xlog_res_t; typedef struct xlog_ticket { - wait_queue_head_t t_wait; /* ticket wait queue */ struct list_head t_queue; /* reserve/write queue */ + struct task_struct *t_task; /* task that owns this ticket */ xlog_tid_t t_tid; /* transaction identifier : 4 */ atomic_t t_ref; /* ticket reference count : 4 */ int t_curr_res; /* current reservation in bytes : 4 */ -- cgit v1.2.3-70-g09d2 From 28496968a6ac37c8b8c44b5156e633c581bb8378 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:25 +0000 Subject: xfs: add the xlog_grant_head structure Add a new data structure to allow sharing code between the log grant and regrant code. Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 112 +++++++++++++++++++++++------------------------ fs/xfs/xfs_log_priv.h | 23 +++++----- fs/xfs/xfs_log_recover.c | 4 +- fs/xfs/xfs_trace.h | 8 ++-- 4 files changed, 74 insertions(+), 73 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 02a35fba5ea..ad0cac378e9 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -158,7 +158,7 @@ xlog_reserveq_wake( struct xlog_ticket *tic; int need_bytes; - list_for_each_entry(tic, &log->l_reserveq, t_queue) { + list_for_each_entry(tic, &log->l_reserve_head.waiters, t_queue) { if (tic->t_flags & XLOG_TIC_PERM_RESERV) need_bytes = tic->t_unit_res * tic->t_cnt; else @@ -183,7 +183,7 @@ xlog_writeq_wake( struct xlog_ticket *tic; int need_bytes; - list_for_each_entry(tic, &log->l_writeq, t_queue) { + list_for_each_entry(tic, &log->l_write_head.waiters, t_queue) { ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); need_bytes = tic->t_unit_res; @@ -205,7 +205,7 @@ xlog_reserveq_wait( struct xlog_ticket *tic, int need_bytes) { - list_add_tail(&tic->t_queue, &log->l_reserveq); + list_add_tail(&tic->t_queue, &log->l_reserve_head.waiters); do { if (XLOG_FORCED_SHUTDOWN(log)) @@ -213,7 +213,7 @@ xlog_reserveq_wait( xlog_grant_push_ail(log, need_bytes); __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock(&log->l_grant_reserve_lock); + spin_unlock(&log->l_reserve_head.lock); XFS_STATS_INC(xs_sleep_logspace); @@ -221,10 +221,10 @@ xlog_reserveq_wait( schedule(); trace_xfs_log_grant_wake(log, tic); - spin_lock(&log->l_grant_reserve_lock); + spin_lock(&log->l_reserve_head.lock); if (XLOG_FORCED_SHUTDOWN(log)) goto shutdown; - } while (xlog_space_left(log, &log->l_grant_reserve_head) < need_bytes); + } while (xlog_space_left(log, &log->l_reserve_head.grant) < need_bytes); list_del_init(&tic->t_queue); return 0; @@ -239,7 +239,7 @@ xlog_writeq_wait( struct xlog_ticket *tic, int need_bytes) { - list_add_tail(&tic->t_queue, &log->l_writeq); + list_add_tail(&tic->t_queue, &log->l_write_head.waiters); do { if (XLOG_FORCED_SHUTDOWN(log)) @@ -247,7 +247,7 @@ xlog_writeq_wait( xlog_grant_push_ail(log, need_bytes); __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock(&log->l_grant_write_lock); + spin_unlock(&log->l_write_head.lock); XFS_STATS_INC(xs_sleep_logspace); @@ -255,10 +255,10 @@ xlog_writeq_wait( schedule(); trace_xfs_log_regrant_write_wake(log, tic); - spin_lock(&log->l_grant_write_lock); + spin_lock(&log->l_write_head.lock); if (XLOG_FORCED_SHUTDOWN(log)) goto shutdown; - } while (xlog_space_left(log, &log->l_grant_write_head) < need_bytes); + } while (xlog_space_left(log, &log->l_write_head.grant) < need_bytes); list_del_init(&tic->t_queue); return 0; @@ -779,22 +779,22 @@ xfs_log_space_wake( if (XLOG_FORCED_SHUTDOWN(log)) return; - if (!list_empty_careful(&log->l_writeq)) { + if (!list_empty_careful(&log->l_write_head.waiters)) { ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); - spin_lock(&log->l_grant_write_lock); - free_bytes = xlog_space_left(log, &log->l_grant_write_head); + spin_lock(&log->l_write_head.lock); + free_bytes = xlog_space_left(log, &log->l_write_head.grant); xlog_writeq_wake(log, &free_bytes); - spin_unlock(&log->l_grant_write_lock); + spin_unlock(&log->l_write_head.lock); } - if (!list_empty_careful(&log->l_reserveq)) { + if (!list_empty_careful(&log->l_reserve_head.waiters)) { ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); - spin_lock(&log->l_grant_reserve_lock); - free_bytes = xlog_space_left(log, &log->l_grant_reserve_head); + spin_lock(&log->l_reserve_head.lock); + free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); xlog_reserveq_wake(log, &free_bytes); - spin_unlock(&log->l_grant_reserve_lock); + spin_unlock(&log->l_reserve_head.lock); } } @@ -1070,12 +1070,12 @@ xlog_alloc_log(xfs_mount_t *mp, xlog_assign_atomic_lsn(&log->l_tail_lsn, 1, 0); xlog_assign_atomic_lsn(&log->l_last_sync_lsn, 1, 0); log->l_curr_cycle = 1; /* 0 is bad since this is initial value */ - xlog_assign_grant_head(&log->l_grant_reserve_head, 1, 0); - xlog_assign_grant_head(&log->l_grant_write_head, 1, 0); - INIT_LIST_HEAD(&log->l_reserveq); - INIT_LIST_HEAD(&log->l_writeq); - spin_lock_init(&log->l_grant_reserve_lock); - spin_lock_init(&log->l_grant_write_lock); + xlog_assign_grant_head(&log->l_reserve_head.grant, 1, 0); + xlog_assign_grant_head(&log->l_write_head.grant, 1, 0); + INIT_LIST_HEAD(&log->l_reserve_head.waiters); + INIT_LIST_HEAD(&log->l_write_head.waiters); + spin_lock_init(&log->l_reserve_head.lock); + spin_lock_init(&log->l_write_head.lock); error = EFSCORRUPTED; if (xfs_sb_version_hassector(&mp->m_sb)) { @@ -1250,7 +1250,7 @@ xlog_grant_push_ail( ASSERT(BTOBB(need_bytes) < log->l_logBBsize); - free_bytes = xlog_space_left(log, &log->l_grant_reserve_head); + free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); free_blocks = BTOBBT(free_bytes); /* @@ -1382,8 +1382,8 @@ xlog_sync(xlog_t *log, roundoff < BBTOB(1))); /* move grant heads by roundoff in sync */ - xlog_grant_add_space(log, &log->l_grant_reserve_head, roundoff); - xlog_grant_add_space(log, &log->l_grant_write_head, roundoff); + xlog_grant_add_space(log, &log->l_reserve_head.grant, roundoff); + xlog_grant_add_space(log, &log->l_write_head.grant, roundoff); /* put cycle number in every block */ xlog_pack_data(log, iclog, roundoff); @@ -2547,8 +2547,8 @@ restart: * path. Hence any lock will be globally hot if we take it unconditionally on * every pass. * - * As tickets are only ever moved on and off the reserveq under the - * l_grant_reserve_lock, we only need to take that lock if we are going to add + * As tickets are only ever moved on and off the l_reserve.waiters under the + * l_reserve.lock, we only need to take that lock if we are going to add * the ticket to the queue and sleep. We can avoid taking the lock if the ticket * was never added to the reserveq because the t_queue list head will be empty * and we hold the only reference to it so it can safely be checked unlocked. @@ -2574,23 +2574,23 @@ xlog_grant_log_space( need_bytes = tic->t_unit_res; if (tic->t_flags & XFS_LOG_PERM_RESERV) need_bytes *= tic->t_ocnt; - free_bytes = xlog_space_left(log, &log->l_grant_reserve_head); - if (!list_empty_careful(&log->l_reserveq)) { - spin_lock(&log->l_grant_reserve_lock); + free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); + if (!list_empty_careful(&log->l_reserve_head.waiters)) { + spin_lock(&log->l_reserve_head.lock); if (!xlog_reserveq_wake(log, &free_bytes) || free_bytes < need_bytes) error = xlog_reserveq_wait(log, tic, need_bytes); - spin_unlock(&log->l_grant_reserve_lock); + spin_unlock(&log->l_reserve_head.lock); } else if (free_bytes < need_bytes) { - spin_lock(&log->l_grant_reserve_lock); + spin_lock(&log->l_reserve_head.lock); error = xlog_reserveq_wait(log, tic, need_bytes); - spin_unlock(&log->l_grant_reserve_lock); + spin_unlock(&log->l_reserve_head.lock); } if (error) return error; - xlog_grant_add_space(log, &log->l_grant_reserve_head, need_bytes); - xlog_grant_add_space(log, &log->l_grant_write_head, need_bytes); + xlog_grant_add_space(log, &log->l_reserve_head.grant, need_bytes); + xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); trace_xfs_log_grant_exit(log, tic); xlog_verify_grant_tail(log); return 0; @@ -2627,23 +2627,23 @@ xlog_regrant_write_log_space( * otherwise try to get some space for this transaction. */ need_bytes = tic->t_unit_res; - free_bytes = xlog_space_left(log, &log->l_grant_write_head); - if (!list_empty_careful(&log->l_writeq)) { - spin_lock(&log->l_grant_write_lock); + free_bytes = xlog_space_left(log, &log->l_write_head.grant); + if (!list_empty_careful(&log->l_write_head.waiters)) { + spin_lock(&log->l_write_head.lock); if (!xlog_writeq_wake(log, &free_bytes) || free_bytes < need_bytes) error = xlog_writeq_wait(log, tic, need_bytes); - spin_unlock(&log->l_grant_write_lock); + spin_unlock(&log->l_write_head.lock); } else if (free_bytes < need_bytes) { - spin_lock(&log->l_grant_write_lock); + spin_lock(&log->l_write_head.lock); error = xlog_writeq_wait(log, tic, need_bytes); - spin_unlock(&log->l_grant_write_lock); + spin_unlock(&log->l_write_head.lock); } if (error) return error; - xlog_grant_add_space(log, &log->l_grant_write_head, need_bytes); + xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); trace_xfs_log_regrant_write_exit(log, tic); xlog_verify_grant_tail(log); return 0; @@ -2665,9 +2665,9 @@ xlog_regrant_reserve_log_space(xlog_t *log, if (ticket->t_cnt > 0) ticket->t_cnt--; - xlog_grant_sub_space(log, &log->l_grant_reserve_head, + xlog_grant_sub_space(log, &log->l_reserve_head.grant, ticket->t_curr_res); - xlog_grant_sub_space(log, &log->l_grant_write_head, + xlog_grant_sub_space(log, &log->l_write_head.grant, ticket->t_curr_res); ticket->t_curr_res = ticket->t_unit_res; xlog_tic_reset_res(ticket); @@ -2678,7 +2678,7 @@ xlog_regrant_reserve_log_space(xlog_t *log, if (ticket->t_cnt > 0) return; - xlog_grant_add_space(log, &log->l_grant_reserve_head, + xlog_grant_add_space(log, &log->l_reserve_head.grant, ticket->t_unit_res); trace_xfs_log_regrant_reserve_exit(log, ticket); @@ -2724,8 +2724,8 @@ xlog_ungrant_log_space(xlog_t *log, bytes += ticket->t_unit_res*ticket->t_cnt; } - xlog_grant_sub_space(log, &log->l_grant_reserve_head, bytes); - xlog_grant_sub_space(log, &log->l_grant_write_head, bytes); + xlog_grant_sub_space(log, &log->l_reserve_head.grant, bytes); + xlog_grant_sub_space(log, &log->l_write_head.grant, bytes); trace_xfs_log_ungrant_exit(log, ticket); @@ -3349,7 +3349,7 @@ xlog_verify_grant_tail( int tail_cycle, tail_blocks; int cycle, space; - xlog_crack_grant_head(&log->l_grant_write_head, &cycle, &space); + xlog_crack_grant_head(&log->l_write_head.grant, &cycle, &space); xlog_crack_atomic_lsn(&log->l_tail_lsn, &tail_cycle, &tail_blocks); if (tail_cycle != cycle) { if (cycle - 1 != tail_cycle && @@ -3619,15 +3619,15 @@ xfs_log_force_umount( * we don't enqueue anything once the SHUTDOWN flag is set, and this * action is protected by the grant locks. */ - spin_lock(&log->l_grant_reserve_lock); - list_for_each_entry(tic, &log->l_reserveq, t_queue) + spin_lock(&log->l_reserve_head.lock); + list_for_each_entry(tic, &log->l_reserve_head.waiters, t_queue) wake_up_process(tic->t_task); - spin_unlock(&log->l_grant_reserve_lock); + spin_unlock(&log->l_reserve_head.lock); - spin_lock(&log->l_grant_write_lock); - list_for_each_entry(tic, &log->l_writeq, t_queue) + spin_lock(&log->l_write_head.lock); + list_for_each_entry(tic, &log->l_write_head.waiters, t_queue) wake_up_process(tic->t_task); - spin_unlock(&log->l_grant_write_lock); + spin_unlock(&log->l_write_head.lock); if (!(log->l_iclog->ic_state & XLOG_STATE_IOERROR)) { ASSERT(!logerror); diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index d8c5e47bbc2..eba4ec925a4 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -469,6 +469,16 @@ struct xfs_cil { #define XLOG_CIL_SPACE_LIMIT(log) (log->l_logsize >> 3) #define XLOG_CIL_HARD_SPACE_LIMIT(log) (3 * (log->l_logsize >> 4)) +/* + * ticket grant locks, queues and accounting have their own cachlines + * as these are quite hot and can be operated on concurrently. + */ +struct xlog_grant_head { + spinlock_t lock ____cacheline_aligned_in_smp; + struct list_head waiters; + atomic64_t grant; +}; + /* * The reservation head lsn is not made up of a cycle number and block number. * Instead, it uses a cycle number and byte number. Logs don't expect to @@ -520,17 +530,8 @@ typedef struct log { /* lsn of 1st LR with unflushed * buffers */ atomic64_t l_tail_lsn ____cacheline_aligned_in_smp; - /* - * ticket grant locks, queues and accounting have their own cachlines - * as these are quite hot and can be operated on concurrently. - */ - spinlock_t l_grant_reserve_lock ____cacheline_aligned_in_smp; - struct list_head l_reserveq; - atomic64_t l_grant_reserve_head; - - spinlock_t l_grant_write_lock ____cacheline_aligned_in_smp; - struct list_head l_writeq; - atomic64_t l_grant_write_head; + struct xlog_grant_head l_reserve_head; + struct xlog_grant_head l_write_head; /* The following field are used for debugging; need to hold icloglock */ #ifdef DEBUG diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 8a3d8aedd1f..7c75c7374d5 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -965,9 +965,9 @@ xlog_find_tail( log->l_curr_cycle++; atomic64_set(&log->l_tail_lsn, be64_to_cpu(rhead->h_tail_lsn)); atomic64_set(&log->l_last_sync_lsn, be64_to_cpu(rhead->h_lsn)); - xlog_assign_grant_head(&log->l_grant_reserve_head, log->l_curr_cycle, + xlog_assign_grant_head(&log->l_reserve_head.grant, log->l_curr_cycle, BBTOB(log->l_curr_block)); - xlog_assign_grant_head(&log->l_grant_write_head, log->l_curr_cycle, + xlog_assign_grant_head(&log->l_write_head.grant, log->l_curr_cycle, BBTOB(log->l_curr_block)); /* diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index bb134a81993..a80e30854fc 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -782,12 +782,12 @@ DECLARE_EVENT_CLASS(xfs_loggrant_class, __entry->curr_res = tic->t_curr_res; __entry->unit_res = tic->t_unit_res; __entry->flags = tic->t_flags; - __entry->reserveq = list_empty(&log->l_reserveq); - __entry->writeq = list_empty(&log->l_writeq); - xlog_crack_grant_head(&log->l_grant_reserve_head, + __entry->reserveq = list_empty(&log->l_reserve_head.waiters); + __entry->writeq = list_empty(&log->l_write_head.waiters); + xlog_crack_grant_head(&log->l_reserve_head.grant, &__entry->grant_reserve_cycle, &__entry->grant_reserve_bytes); - xlog_crack_grant_head(&log->l_grant_write_head, + xlog_crack_grant_head(&log->l_write_head.grant, &__entry->grant_write_cycle, &__entry->grant_write_bytes); __entry->curr_cycle = log->l_curr_cycle; -- cgit v1.2.3-70-g09d2 From c303c5b8c3b8eace41c4fba26205b50c0f8e4ca0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:26 +0000 Subject: xfs: add xlog_grant_head_init Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index ad0cac378e9..30fec0a2a21 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -150,6 +150,15 @@ xlog_grant_add_space( } while (head_val != old); } +STATIC void +xlog_grant_head_init( + struct xlog_grant_head *head) +{ + xlog_assign_grant_head(&head->grant, 1, 0); + INIT_LIST_HEAD(&head->waiters); + spin_lock_init(&head->lock); +} + STATIC bool xlog_reserveq_wake( struct log *log, @@ -1070,12 +1079,9 @@ xlog_alloc_log(xfs_mount_t *mp, xlog_assign_atomic_lsn(&log->l_tail_lsn, 1, 0); xlog_assign_atomic_lsn(&log->l_last_sync_lsn, 1, 0); log->l_curr_cycle = 1; /* 0 is bad since this is initial value */ - xlog_assign_grant_head(&log->l_reserve_head.grant, 1, 0); - xlog_assign_grant_head(&log->l_write_head.grant, 1, 0); - INIT_LIST_HEAD(&log->l_reserve_head.waiters); - INIT_LIST_HEAD(&log->l_write_head.waiters); - spin_lock_init(&log->l_reserve_head.lock); - spin_lock_init(&log->l_write_head.lock); + + xlog_grant_head_init(&log->l_reserve_head); + xlog_grant_head_init(&log->l_write_head); error = EFSCORRUPTED; if (xfs_sb_version_hassector(&mp->m_sb)) { -- cgit v1.2.3-70-g09d2 From a79bf2d75b8f96bcdb6714138cd53cb3c358669c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:27 +0000 Subject: xfs: add xlog_grant_head_wake_all Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 30fec0a2a21..a0d1376b3d4 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -159,6 +159,18 @@ xlog_grant_head_init( spin_lock_init(&head->lock); } +STATIC void +xlog_grant_head_wake_all( + struct xlog_grant_head *head) +{ + struct xlog_ticket *tic; + + spin_lock(&head->lock); + list_for_each_entry(tic, &head->waiters, t_queue) + wake_up_process(tic->t_task); + spin_unlock(&head->lock); +} + STATIC bool xlog_reserveq_wake( struct log *log, @@ -3557,7 +3569,6 @@ xfs_log_force_umount( struct xfs_mount *mp, int logerror) { - xlog_ticket_t *tic; xlog_t *log; int retval; @@ -3625,15 +3636,8 @@ xfs_log_force_umount( * we don't enqueue anything once the SHUTDOWN flag is set, and this * action is protected by the grant locks. */ - spin_lock(&log->l_reserve_head.lock); - list_for_each_entry(tic, &log->l_reserve_head.waiters, t_queue) - wake_up_process(tic->t_task); - spin_unlock(&log->l_reserve_head.lock); - - spin_lock(&log->l_write_head.lock); - list_for_each_entry(tic, &log->l_write_head.waiters, t_queue) - wake_up_process(tic->t_task); - spin_unlock(&log->l_write_head.lock); + xlog_grant_head_wake_all(&log->l_reserve_head); + xlog_grant_head_wake_all(&log->l_write_head); if (!(log->l_iclog->ic_state & XLOG_STATE_IOERROR)) { ASSERT(!logerror); -- cgit v1.2.3-70-g09d2 From 23ee3df349b8b8fd153bd02fccf08b31aec5bce3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:28 +0000 Subject: xfs: share code for grant head waiting Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 63 ++++++++++++++++-------------------------------------- fs/xfs/xfs_trace.h | 2 -- 2 files changed, 18 insertions(+), 47 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index a0d1376b3d4..c2d13827582 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -221,12 +221,13 @@ xlog_writeq_wake( } STATIC int -xlog_reserveq_wait( +xlog_grant_head_wait( struct log *log, + struct xlog_grant_head *head, struct xlog_ticket *tic, int need_bytes) { - list_add_tail(&tic->t_queue, &log->l_reserve_head.waiters); + list_add_tail(&tic->t_queue, &head->waiters); do { if (XLOG_FORCED_SHUTDOWN(log)) @@ -234,7 +235,7 @@ xlog_reserveq_wait( xlog_grant_push_ail(log, need_bytes); __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock(&log->l_reserve_head.lock); + spin_unlock(&head->lock); XFS_STATS_INC(xs_sleep_logspace); @@ -242,44 +243,10 @@ xlog_reserveq_wait( schedule(); trace_xfs_log_grant_wake(log, tic); - spin_lock(&log->l_reserve_head.lock); + spin_lock(&head->lock); if (XLOG_FORCED_SHUTDOWN(log)) goto shutdown; - } while (xlog_space_left(log, &log->l_reserve_head.grant) < need_bytes); - - list_del_init(&tic->t_queue); - return 0; -shutdown: - list_del_init(&tic->t_queue); - return XFS_ERROR(EIO); -} - -STATIC int -xlog_writeq_wait( - struct log *log, - struct xlog_ticket *tic, - int need_bytes) -{ - list_add_tail(&tic->t_queue, &log->l_write_head.waiters); - - do { - if (XLOG_FORCED_SHUTDOWN(log)) - goto shutdown; - xlog_grant_push_ail(log, need_bytes); - - __set_current_state(TASK_UNINTERRUPTIBLE); - spin_unlock(&log->l_write_head.lock); - - XFS_STATS_INC(xs_sleep_logspace); - - trace_xfs_log_regrant_write_sleep(log, tic); - schedule(); - trace_xfs_log_regrant_write_wake(log, tic); - - spin_lock(&log->l_write_head.lock); - if (XLOG_FORCED_SHUTDOWN(log)) - goto shutdown; - } while (xlog_space_left(log, &log->l_write_head.grant) < need_bytes); + } while (xlog_space_left(log, &head->grant) < need_bytes); list_del_init(&tic->t_queue); return 0; @@ -2596,12 +2563,15 @@ xlog_grant_log_space( if (!list_empty_careful(&log->l_reserve_head.waiters)) { spin_lock(&log->l_reserve_head.lock); if (!xlog_reserveq_wake(log, &free_bytes) || - free_bytes < need_bytes) - error = xlog_reserveq_wait(log, tic, need_bytes); + free_bytes < need_bytes) { + error = xlog_grant_head_wait(log, &log->l_reserve_head, + tic, need_bytes); + } spin_unlock(&log->l_reserve_head.lock); } else if (free_bytes < need_bytes) { spin_lock(&log->l_reserve_head.lock); - error = xlog_reserveq_wait(log, tic, need_bytes); + error = xlog_grant_head_wait(log, &log->l_reserve_head, tic, + need_bytes); spin_unlock(&log->l_reserve_head.lock); } if (error) @@ -2649,12 +2619,15 @@ xlog_regrant_write_log_space( if (!list_empty_careful(&log->l_write_head.waiters)) { spin_lock(&log->l_write_head.lock); if (!xlog_writeq_wake(log, &free_bytes) || - free_bytes < need_bytes) - error = xlog_writeq_wait(log, tic, need_bytes); + free_bytes < need_bytes) { + error = xlog_grant_head_wait(log, &log->l_write_head, + tic, need_bytes); + } spin_unlock(&log->l_write_head.lock); } else if (free_bytes < need_bytes) { spin_lock(&log->l_write_head.lock); - error = xlog_writeq_wait(log, tic, need_bytes); + error = xlog_grant_head_wait(log, &log->l_write_head, tic, + need_bytes); spin_unlock(&log->l_write_head.lock); } diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index a80e30854fc..7ee18e91bf8 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -837,8 +837,6 @@ DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake_up); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_enter); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_exit); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_error); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_sleep); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_wake); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_wake_up); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_enter); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_exit); -- cgit v1.2.3-70-g09d2 From e179840d74606ab1935c83fe5ad9d93c95ddc956 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:29 +0000 Subject: xfs: share code for grant head wakeups Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 50 ++++++++++++++++++++------------------------------ fs/xfs/xfs_trace.h | 1 - 2 files changed, 20 insertions(+), 31 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index c2d13827582..685997548fb 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -171,49 +171,39 @@ xlog_grant_head_wake_all( spin_unlock(&head->lock); } -STATIC bool -xlog_reserveq_wake( +static inline int +xlog_ticket_reservation( struct log *log, - int *free_bytes) + struct xlog_grant_head *head, + struct xlog_ticket *tic) { - struct xlog_ticket *tic; - int need_bytes; - - list_for_each_entry(tic, &log->l_reserve_head.waiters, t_queue) { + if (head == &log->l_write_head) { + ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); + return tic->t_unit_res; + } else { if (tic->t_flags & XLOG_TIC_PERM_RESERV) - need_bytes = tic->t_unit_res * tic->t_cnt; + return tic->t_unit_res * tic->t_cnt; else - need_bytes = tic->t_unit_res; - - if (*free_bytes < need_bytes) - return false; - *free_bytes -= need_bytes; - - trace_xfs_log_grant_wake_up(log, tic); - wake_up_process(tic->t_task); + return tic->t_unit_res; } - - return true; } STATIC bool -xlog_writeq_wake( +xlog_grant_head_wake( struct log *log, + struct xlog_grant_head *head, int *free_bytes) { struct xlog_ticket *tic; int need_bytes; - list_for_each_entry(tic, &log->l_write_head.waiters, t_queue) { - ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); - - need_bytes = tic->t_unit_res; - + list_for_each_entry(tic, &head->waiters, t_queue) { + need_bytes = xlog_ticket_reservation(log, head, tic); if (*free_bytes < need_bytes) return false; - *free_bytes -= need_bytes; - trace_xfs_log_regrant_write_wake_up(log, tic); + *free_bytes -= need_bytes; + trace_xfs_log_grant_wake_up(log, tic); wake_up_process(tic->t_task); } @@ -772,7 +762,7 @@ xfs_log_space_wake( spin_lock(&log->l_write_head.lock); free_bytes = xlog_space_left(log, &log->l_write_head.grant); - xlog_writeq_wake(log, &free_bytes); + xlog_grant_head_wake(log, &log->l_write_head, &free_bytes); spin_unlock(&log->l_write_head.lock); } @@ -781,7 +771,7 @@ xfs_log_space_wake( spin_lock(&log->l_reserve_head.lock); free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); - xlog_reserveq_wake(log, &free_bytes); + xlog_grant_head_wake(log, &log->l_reserve_head, &free_bytes); spin_unlock(&log->l_reserve_head.lock); } } @@ -2562,7 +2552,7 @@ xlog_grant_log_space( free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); if (!list_empty_careful(&log->l_reserve_head.waiters)) { spin_lock(&log->l_reserve_head.lock); - if (!xlog_reserveq_wake(log, &free_bytes) || + if (!xlog_grant_head_wake(log, &log->l_reserve_head, &free_bytes) || free_bytes < need_bytes) { error = xlog_grant_head_wait(log, &log->l_reserve_head, tic, need_bytes); @@ -2618,7 +2608,7 @@ xlog_regrant_write_log_space( free_bytes = xlog_space_left(log, &log->l_write_head.grant); if (!list_empty_careful(&log->l_write_head.waiters)) { spin_lock(&log->l_write_head.lock); - if (!xlog_writeq_wake(log, &free_bytes) || + if (!xlog_grant_head_wake(log, &log->l_write_head, &free_bytes) || free_bytes < need_bytes) { error = xlog_grant_head_wait(log, &log->l_write_head, tic, need_bytes); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 7ee18e91bf8..482bebf8c7e 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -837,7 +837,6 @@ DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake_up); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_enter); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_exit); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_error); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_wake_up); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_enter); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_exit); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_sub); -- cgit v1.2.3-70-g09d2 From 42ceedb3caffe67c4ec0dfbb78ce410832d429b9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:30 +0000 Subject: xfs: share code for grant head availability checks Reviewed-by: Mark Tinguely Signed-off-by: Christoph Hellwig Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 133 +++++++++++++++++++++++++------------------------------ 1 file changed, 60 insertions(+), 73 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 685997548fb..c6a29a05c60 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -245,6 +245,60 @@ shutdown: return XFS_ERROR(EIO); } +/* + * Atomically get the log space required for a log ticket. + * + * Once a ticket gets put onto head->waiters, it will only return after the + * needed reservation is satisfied. + * + * This function is structured so that it has a lock free fast path. This is + * necessary because every new transaction reservation will come through this + * path. Hence any lock will be globally hot if we take it unconditionally on + * every pass. + * + * As tickets are only ever moved on and off head->waiters under head->lock, we + * only need to take that lock if we are going to add the ticket to the queue + * and sleep. We can avoid taking the lock if the ticket was never added to + * head->waiters because the t_queue list head will be empty and we hold the + * only reference to it so it can safely be checked unlocked. + */ +STATIC int +xlog_grant_head_check( + struct log *log, + struct xlog_grant_head *head, + struct xlog_ticket *tic, + int *need_bytes) +{ + int free_bytes; + int error = 0; + + ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); + + /* + * If there are other waiters on the queue then give them a chance at + * logspace before us. Wake up the first waiters, if we do not wake + * up all the waiters then go to sleep waiting for more free space, + * otherwise try to get some space for this transaction. + */ + *need_bytes = xlog_ticket_reservation(log, head, tic); + free_bytes = xlog_space_left(log, &head->grant); + if (!list_empty_careful(&head->waiters)) { + spin_lock(&head->lock); + if (!xlog_grant_head_wake(log, head, &free_bytes) || + free_bytes < *need_bytes) { + error = xlog_grant_head_wait(log, head, tic, + *need_bytes); + } + spin_unlock(&head->lock); + } else if (free_bytes < *need_bytes) { + spin_lock(&head->lock); + error = xlog_grant_head_wait(log, head, tic, *need_bytes); + spin_unlock(&head->lock); + } + + return error; +} + static void xlog_tic_reset_res(xlog_ticket_t *tic) { @@ -2511,59 +2565,18 @@ restart: return 0; } /* xlog_state_get_iclog_space */ -/* - * Atomically get the log space required for a log ticket. - * - * Once a ticket gets put onto the reserveq, it will only return after the - * needed reservation is satisfied. - * - * This function is structured so that it has a lock free fast path. This is - * necessary because every new transaction reservation will come through this - * path. Hence any lock will be globally hot if we take it unconditionally on - * every pass. - * - * As tickets are only ever moved on and off the l_reserve.waiters under the - * l_reserve.lock, we only need to take that lock if we are going to add - * the ticket to the queue and sleep. We can avoid taking the lock if the ticket - * was never added to the reserveq because the t_queue list head will be empty - * and we hold the only reference to it so it can safely be checked unlocked. - */ STATIC int xlog_grant_log_space( struct log *log, struct xlog_ticket *tic) { - int free_bytes, need_bytes; + int need_bytes; int error = 0; - ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); - trace_xfs_log_grant_enter(log, tic); - /* - * If there are other waiters on the queue then give them a chance at - * logspace before us. Wake up the first waiters, if we do not wake - * up all the waiters then go to sleep waiting for more free space, - * otherwise try to get some space for this transaction. - */ - need_bytes = tic->t_unit_res; - if (tic->t_flags & XFS_LOG_PERM_RESERV) - need_bytes *= tic->t_ocnt; - free_bytes = xlog_space_left(log, &log->l_reserve_head.grant); - if (!list_empty_careful(&log->l_reserve_head.waiters)) { - spin_lock(&log->l_reserve_head.lock); - if (!xlog_grant_head_wake(log, &log->l_reserve_head, &free_bytes) || - free_bytes < need_bytes) { - error = xlog_grant_head_wait(log, &log->l_reserve_head, - tic, need_bytes); - } - spin_unlock(&log->l_reserve_head.lock); - } else if (free_bytes < need_bytes) { - spin_lock(&log->l_reserve_head.lock); - error = xlog_grant_head_wait(log, &log->l_reserve_head, tic, - need_bytes); - spin_unlock(&log->l_reserve_head.lock); - } + error = xlog_grant_head_check(log, &log->l_reserve_head, tic, + &need_bytes); if (error) return error; @@ -2576,16 +2589,13 @@ xlog_grant_log_space( /* * Replenish the byte reservation required by moving the grant write head. - * - * Similar to xlog_grant_log_space, the function is structured to have a lock - * free fast path. */ STATIC int xlog_regrant_write_log_space( struct log *log, struct xlog_ticket *tic) { - int free_bytes, need_bytes; + int need_bytes; int error = 0; tic->t_curr_res = tic->t_unit_res; @@ -2594,33 +2604,10 @@ xlog_regrant_write_log_space( if (tic->t_cnt > 0) return 0; - ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY)); - trace_xfs_log_regrant_write_enter(log, tic); - /* - * If there are other waiters on the queue then give them a chance at - * logspace before us. Wake up the first waiters, if we do not wake - * up all the waiters then go to sleep waiting for more free space, - * otherwise try to get some space for this transaction. - */ - need_bytes = tic->t_unit_res; - free_bytes = xlog_space_left(log, &log->l_write_head.grant); - if (!list_empty_careful(&log->l_write_head.waiters)) { - spin_lock(&log->l_write_head.lock); - if (!xlog_grant_head_wake(log, &log->l_write_head, &free_bytes) || - free_bytes < need_bytes) { - error = xlog_grant_head_wait(log, &log->l_write_head, - tic, need_bytes); - } - spin_unlock(&log->l_write_head.lock); - } else if (free_bytes < need_bytes) { - spin_lock(&log->l_write_head.lock); - error = xlog_grant_head_wait(log, &log->l_write_head, tic, - need_bytes); - spin_unlock(&log->l_write_head.lock); - } - + error = xlog_grant_head_check(log, &log->l_write_head, tic, + &need_bytes); if (error) return error; -- cgit v1.2.3-70-g09d2 From 9006fb91cfdf22812923f0536c7531c429c1aeab Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 20 Feb 2012 02:31:31 +0000 Subject: xfs: split and cleanup xfs_log_reserve Split the log regrant case out of xfs_log_reserve into a separate function, and merge xlog_grant_log_space and xlog_regrant_write_log_space into their respective callers. Also replace the XFS_LOG_PERM_RESERV flag, which easily got misused before the previous cleanups with a simple boolean parameter. Signed-off-by: Christoph Hellwig Reviewed-by: Mark Tinguely Reviewed-by: Dave Chinner Signed-off-by: Ben Myers --- fs/xfs/xfs_log.c | 265 +++++++++++++++++++++++--------------------------- fs/xfs/xfs_log.h | 12 +-- fs/xfs/xfs_log_priv.h | 2 +- fs/xfs/xfs_trace.h | 11 +-- fs/xfs/xfs_trans.c | 31 +++--- 5 files changed, 151 insertions(+), 170 deletions(-) (limited to 'fs/xfs/xfs_log.c') diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index c6a29a05c60..98a9cb5ffd1 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -67,15 +67,10 @@ STATIC void xlog_state_switch_iclogs(xlog_t *log, int eventual_size); STATIC void xlog_state_want_sync(xlog_t *log, xlog_in_core_t *iclog); -/* local functions to manipulate grant head */ -STATIC int xlog_grant_log_space(xlog_t *log, - xlog_ticket_t *xtic); STATIC void xlog_grant_push_ail(struct log *log, int need_bytes); STATIC void xlog_regrant_reserve_log_space(xlog_t *log, xlog_ticket_t *ticket); -STATIC int xlog_regrant_write_log_space(xlog_t *log, - xlog_ticket_t *ticket); STATIC void xlog_ungrant_log_space(xlog_t *log, xlog_ticket_t *ticket); @@ -323,6 +318,128 @@ xlog_tic_add_region(xlog_ticket_t *tic, uint len, uint type) tic->t_res_num++; } +/* + * Replenish the byte reservation required by moving the grant write head. + */ +int +xfs_log_regrant( + struct xfs_mount *mp, + struct xlog_ticket *tic) +{ + struct log *log = mp->m_log; + int need_bytes; + int error = 0; + + if (XLOG_FORCED_SHUTDOWN(log)) + return XFS_ERROR(EIO); + + XFS_STATS_INC(xs_try_logspace); + + /* + * This is a new transaction on the ticket, so we need to change the + * transaction ID so that the next transaction has a different TID in + * the log. Just add one to the existing tid so that we can see chains + * of rolling transactions in the log easily. + */ + tic->t_tid++; + + xlog_grant_push_ail(log, tic->t_unit_res); + + tic->t_curr_res = tic->t_unit_res; + xlog_tic_reset_res(tic); + + if (tic->t_cnt > 0) + return 0; + + trace_xfs_log_regrant(log, tic); + + error = xlog_grant_head_check(log, &log->l_write_head, tic, + &need_bytes); + if (error) + goto out_error; + + xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); + trace_xfs_log_regrant_exit(log, tic); + xlog_verify_grant_tail(log); + return 0; + +out_error: + /* + * If we are failing, make sure the ticket doesn't have any current + * reservations. We don't want to add this back when the ticket/ + * transaction gets cancelled. + */ + tic->t_curr_res = 0; + tic->t_cnt = 0; /* ungrant will give back unit_res * t_cnt. */ + return error; +} + +/* + * Reserve log space and return a ticket corresponding the reservation. + * + * Each reservation is going to reserve extra space for a log record header. + * When writes happen to the on-disk log, we don't subtract the length of the + * log record header from any reservation. By wasting space in each + * reservation, we prevent over allocation problems. + */ +int +xfs_log_reserve( + struct xfs_mount *mp, + int unit_bytes, + int cnt, + struct xlog_ticket **ticp, + __uint8_t client, + bool permanent, + uint t_type) +{ + struct log *log = mp->m_log; + struct xlog_ticket *tic; + int need_bytes; + int error = 0; + + ASSERT(client == XFS_TRANSACTION || client == XFS_LOG); + + if (XLOG_FORCED_SHUTDOWN(log)) + return XFS_ERROR(EIO); + + XFS_STATS_INC(xs_try_logspace); + + ASSERT(*ticp == NULL); + tic = xlog_ticket_alloc(log, unit_bytes, cnt, client, permanent, + KM_SLEEP | KM_MAYFAIL); + if (!tic) + return XFS_ERROR(ENOMEM); + + tic->t_trans_type = t_type; + *ticp = tic; + + xlog_grant_push_ail(log, tic->t_unit_res * tic->t_cnt); + + trace_xfs_log_reserve(log, tic); + + error = xlog_grant_head_check(log, &log->l_reserve_head, tic, + &need_bytes); + if (error) + goto out_error; + + xlog_grant_add_space(log, &log->l_reserve_head.grant, need_bytes); + xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); + trace_xfs_log_reserve_exit(log, tic); + xlog_verify_grant_tail(log); + return 0; + +out_error: + /* + * If we are failing, make sure the ticket doesn't have any current + * reservations. We don't want to add this back when the ticket/ + * transaction gets cancelled. + */ + tic->t_curr_res = 0; + tic->t_cnt = 0; /* ungrant will give back unit_res * t_cnt. */ + return error; +} + + /* * NOTES: * @@ -432,88 +549,6 @@ xfs_log_release_iclog( return 0; } -/* - * 1. Reserve an amount of on-disk log space and return a ticket corresponding - * to the reservation. - * 2. Potentially, push buffers at tail of log to disk. - * - * Each reservation is going to reserve extra space for a log record header. - * When writes happen to the on-disk log, we don't subtract the length of the - * log record header from any reservation. By wasting space in each - * reservation, we prevent over allocation problems. - */ -int -xfs_log_reserve( - struct xfs_mount *mp, - int unit_bytes, - int cnt, - struct xlog_ticket **ticket, - __uint8_t client, - uint flags, - uint t_type) -{ - struct log *log = mp->m_log; - struct xlog_ticket *internal_ticket; - int retval = 0; - - ASSERT(client == XFS_TRANSACTION || client == XFS_LOG); - - if (XLOG_FORCED_SHUTDOWN(log)) - return XFS_ERROR(EIO); - - XFS_STATS_INC(xs_try_logspace); - - - if (*ticket != NULL) { - ASSERT(flags & XFS_LOG_PERM_RESERV); - internal_ticket = *ticket; - - /* - * this is a new transaction on the ticket, so we need to - * change the transaction ID so that the next transaction has a - * different TID in the log. Just add one to the existing tid - * so that we can see chains of rolling transactions in the log - * easily. - */ - internal_ticket->t_tid++; - - trace_xfs_log_reserve(log, internal_ticket); - - xlog_grant_push_ail(log, internal_ticket->t_unit_res); - retval = xlog_regrant_write_log_space(log, internal_ticket); - } else { - /* may sleep if need to allocate more tickets */ - internal_ticket = xlog_ticket_alloc(log, unit_bytes, cnt, - client, flags, - KM_SLEEP|KM_MAYFAIL); - if (!internal_ticket) - return XFS_ERROR(ENOMEM); - internal_ticket->t_trans_type = t_type; - *ticket = internal_ticket; - - trace_xfs_log_reserve(log, internal_ticket); - - xlog_grant_push_ail(log, - (internal_ticket->t_unit_res * - internal_ticket->t_cnt)); - retval = xlog_grant_log_space(log, internal_ticket); - } - - if (unlikely(retval)) { - /* - * If we are failing, make sure the ticket doesn't have any - * current reservations. We don't want to add this back - * when the ticket/ transaction gets cancelled. - */ - internal_ticket->t_curr_res = 0; - /* ungrant will give back unit_res * t_cnt. */ - internal_ticket->t_cnt = 0; - } - - return retval; -} - - /* * Mount a log filesystem * @@ -2565,58 +2600,6 @@ restart: return 0; } /* xlog_state_get_iclog_space */ -STATIC int -xlog_grant_log_space( - struct log *log, - struct xlog_ticket *tic) -{ - int need_bytes; - int error = 0; - - trace_xfs_log_grant_enter(log, tic); - - error = xlog_grant_head_check(log, &log->l_reserve_head, tic, - &need_bytes); - if (error) - return error; - - xlog_grant_add_space(log, &log->l_reserve_head.grant, need_bytes); - xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); - trace_xfs_log_grant_exit(log, tic); - xlog_verify_grant_tail(log); - return 0; -} - -/* - * Replenish the byte reservation required by moving the grant write head. - */ -STATIC int -xlog_regrant_write_log_space( - struct log *log, - struct xlog_ticket *tic) -{ - int need_bytes; - int error = 0; - - tic->t_curr_res = tic->t_unit_res; - xlog_tic_reset_res(tic); - - if (tic->t_cnt > 0) - return 0; - - trace_xfs_log_regrant_write_enter(log, tic); - - error = xlog_grant_head_check(log, &log->l_write_head, tic, - &need_bytes); - if (error) - return error; - - xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes); - trace_xfs_log_regrant_write_exit(log, tic); - xlog_verify_grant_tail(log); - return 0; -} - /* The first cnt-1 times through here we don't need to * move the grant write head because the permanent * reservation has reserved cnt times the unit amount. @@ -3156,7 +3139,7 @@ xlog_ticket_alloc( int unit_bytes, int cnt, char client, - uint xflags, + bool permanent, int alloc_flags) { struct xlog_ticket *tic; @@ -3260,7 +3243,7 @@ xlog_ticket_alloc( tic->t_clientid = client; tic->t_flags = XLOG_TIC_INITED; tic->t_trans_type = 0; - if (xflags & XFS_LOG_PERM_RESERV) + if (permanent) tic->t_flags |= XLOG_TIC_PERM_RESERV; xlog_tic_reset_res(tic); diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h index fe32c692787..2c622bedb30 100644 --- a/fs/xfs/xfs_log.h +++ b/fs/xfs/xfs_log.h @@ -52,15 +52,6 @@ static inline xfs_lsn_t _lsn_cmp(xfs_lsn_t lsn1, xfs_lsn_t lsn2) */ #define XFS_LOG_REL_PERM_RESERV 0x1 -/* - * Flags to xfs_log_reserve() - * - * XFS_LOG_PERM_RESERV: Permanent reservation. When writes are - * performed against this type of reservation, the reservation - * is not decreased. Long running transactions should use this. - */ -#define XFS_LOG_PERM_RESERV 0x2 - /* * Flags to xfs_log_force() * @@ -172,8 +163,9 @@ int xfs_log_reserve(struct xfs_mount *mp, int count, struct xlog_ticket **ticket, __uint8_t clientid, - uint flags, + bool permanent, uint t_type); +int xfs_log_regrant(struct xfs_mount *mp, struct xlog_ticket *tic); int xfs_log_unmount_write(struct xfs_mount *mp); void xfs_log_unmount(struct xfs_mount *mp); int xfs_log_force_umount(struct xfs_mount *mp, int logerror); diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h index eba4ec925a4..2152900b79d 100644 --- a/fs/xfs/xfs_log_priv.h +++ b/fs/xfs/xfs_log_priv.h @@ -552,7 +552,7 @@ extern void xlog_pack_data(xlog_t *log, xlog_in_core_t *iclog, int); extern kmem_zone_t *xfs_log_ticket_zone; struct xlog_ticket *xlog_ticket_alloc(struct log *log, int unit_bytes, - int count, char client, uint xflags, + int count, char client, bool permanent, int alloc_flags); diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index 482bebf8c7e..3b369c1277f 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -826,17 +826,14 @@ DEFINE_EVENT(xfs_loggrant_class, name, \ TP_ARGS(log, tic)) DEFINE_LOGGRANT_EVENT(xfs_log_done_nonperm); DEFINE_LOGGRANT_EVENT(xfs_log_done_perm); -DEFINE_LOGGRANT_EVENT(xfs_log_reserve); DEFINE_LOGGRANT_EVENT(xfs_log_umount_write); -DEFINE_LOGGRANT_EVENT(xfs_log_grant_enter); -DEFINE_LOGGRANT_EVENT(xfs_log_grant_exit); -DEFINE_LOGGRANT_EVENT(xfs_log_grant_error); DEFINE_LOGGRANT_EVENT(xfs_log_grant_sleep); DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake); DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake_up); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_enter); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_exit); -DEFINE_LOGGRANT_EVENT(xfs_log_regrant_write_error); +DEFINE_LOGGRANT_EVENT(xfs_log_reserve); +DEFINE_LOGGRANT_EVENT(xfs_log_reserve_exit); +DEFINE_LOGGRANT_EVENT(xfs_log_regrant); +DEFINE_LOGGRANT_EVENT(xfs_log_regrant_exit); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_enter); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_exit); DEFINE_LOGGRANT_EVENT(xfs_log_regrant_reserve_sub); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 7adcdf15ae0..103b00c9000 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -681,7 +681,6 @@ xfs_trans_reserve( uint flags, uint logcount) { - int log_flags; int error = 0; int rsvd = (tp->t_flags & XFS_TRANS_RESERVE) != 0; @@ -707,24 +706,32 @@ xfs_trans_reserve( * Reserve the log space needed for this transaction. */ if (logspace > 0) { - ASSERT((tp->t_log_res == 0) || (tp->t_log_res == logspace)); - ASSERT((tp->t_log_count == 0) || - (tp->t_log_count == logcount)); + bool permanent = false; + + ASSERT(tp->t_log_res == 0 || tp->t_log_res == logspace); + ASSERT(tp->t_log_count == 0 || tp->t_log_count == logcount); + if (flags & XFS_TRANS_PERM_LOG_RES) { - log_flags = XFS_LOG_PERM_RESERV; tp->t_flags |= XFS_TRANS_PERM_LOG_RES; + permanent = true; } else { ASSERT(tp->t_ticket == NULL); ASSERT(!(tp->t_flags & XFS_TRANS_PERM_LOG_RES)); - log_flags = 0; } - error = xfs_log_reserve(tp->t_mountp, logspace, logcount, - &tp->t_ticket, - XFS_TRANSACTION, log_flags, tp->t_type); - if (error) { - goto undo_blocks; + if (tp->t_ticket != NULL) { + ASSERT(flags & XFS_TRANS_PERM_LOG_RES); + error = xfs_log_regrant(tp->t_mountp, tp->t_ticket); + } else { + error = xfs_log_reserve(tp->t_mountp, logspace, + logcount, &tp->t_ticket, + XFS_TRANSACTION, permanent, + tp->t_type); } + + if (error) + goto undo_blocks; + tp->t_log_res = logspace; tp->t_log_count = logcount; } @@ -752,6 +759,8 @@ xfs_trans_reserve( */ undo_log: if (logspace > 0) { + int log_flags; + if (flags & XFS_TRANS_PERM_LOG_RES) { log_flags = XFS_LOG_REL_PERM_RESERV; } else { -- cgit v1.2.3-70-g09d2