From 91c0dc93a1a6bbdd79707ed311e48b4397df177f Mon Sep 17 00:00:00 2001 From: David Teigland Date: Tue, 31 Oct 2006 11:56:01 -0600 Subject: [DLM] fix aborted recovery during node removal Red Hat BZ 211914 With the new cluster infrastructure, dlm recovery for a node removal can be aborted and restarted for a node addition. When this happens, the restarted recovery isn't aware that it's doing recovery for the earlier removal as well as the addition. So, it then skips the recovery steps only required when nodes are removed. This can result in locks not being purged for failed/removed nodes. The fix is to check for removed nodes for which recovery has not been completed at the start of a new recovery sequence. Signed-off-by: David Teigland Signed-off-by: Steven Whitehouse --- fs/dlm/recoverd.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'fs/dlm/recoverd.c') diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c index 362e3eff4dc..4a1d6023fd9 100644 --- a/fs/dlm/recoverd.c +++ b/fs/dlm/recoverd.c @@ -164,6 +164,13 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) */ dlm_recover_rsbs(ls); + } else { + /* + * Other lockspace members may be going through the "neg" steps + * while also adding us to the lockspace, in which case they'll + * be looking for this status bit during dlm_recover_locks(). + */ + dlm_set_recover_status(ls, DLM_RS_LOCKS); } dlm_release_root_list(ls); -- cgit v1.2.3-70-g09d2 From 2cdc98aaf072d573df10c503d3b3b0b74e2a6d06 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Tue, 31 Oct 2006 11:56:08 -0600 Subject: [DLM] fix stopping unstarted recovery Red Hat BZ 211914 When many nodes are joining a lockspace simultaneously, the dlm gets a quick sequence of stop/start events, a pair for adding each node. dlm_controld in user space sends dlm_recoverd in the kernel each stop and start event. dlm_controld will sometimes send the stop before dlm_recoverd has had a chance to take up the previously queued start. The stop aborts the processing of the previous start by setting the RECOVERY_STOP flag. dlm_recoverd is erroneously clearing this flag and ignoring the stop/abort if it happens to take up the start after the stop meant to abort it. The fix is to check the sequence number that's incremented for each stop/start before clearing the flag. Signed-off-by: David Teigland Signed-off-by: Steven Whitehouse --- fs/dlm/recoverd.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'fs/dlm/recoverd.c') diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c index 4a1d6023fd9..6e4ee94ce7d 100644 --- a/fs/dlm/recoverd.c +++ b/fs/dlm/recoverd.c @@ -219,6 +219,10 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) return error; } +/* The dlm_ls_start() that created the rv we take here may already have been + stopped via dlm_ls_stop(); in that case we need to leave the RECOVERY_STOP + flag set. */ + static void do_ls_recovery(struct dlm_ls *ls) { struct dlm_recover *rv = NULL; @@ -226,7 +230,8 @@ static void do_ls_recovery(struct dlm_ls *ls) spin_lock(&ls->ls_recover_lock); rv = ls->ls_recover_args; ls->ls_recover_args = NULL; - clear_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); + if (rv && ls->ls_recover_seq == rv->seq) + clear_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); spin_unlock(&ls->ls_recover_lock); if (rv) { -- cgit v1.2.3-70-g09d2 From 4b77f2c93d052adca8cc8690b9b5e7f8798f4ddd Mon Sep 17 00:00:00 2001 From: David Teigland Date: Wed, 1 Nov 2006 09:31:48 -0600 Subject: [DLM] do full recover_locks barrier Red Hat BZ 211914 The previous patch "[DLM] fix aborted recovery during node removal" was incomplete as discovered with further testing. It set the bit for the RS_LOCKS barrier but did not then wait for the barrier. This is often ok, but sometimes it will cause yet another recovery hang. If it's a new node that also has the lowest nodeid that skips the barrier wait, then it misses the important step of collecting and reporting the barrier status from the other nodes (which is the job of the low nodeid in the barrier wait routine). Signed-off-by: David Teigland Signed-off-by: Steven Whitehouse --- fs/dlm/recoverd.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'fs/dlm/recoverd.c') diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c index 6e4ee94ce7d..8bb895ffd90 100644 --- a/fs/dlm/recoverd.c +++ b/fs/dlm/recoverd.c @@ -168,9 +168,15 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) /* * Other lockspace members may be going through the "neg" steps * while also adding us to the lockspace, in which case they'll - * be looking for this status bit during dlm_recover_locks(). + * be doing the recover_locks (RS_LOCKS) barrier. */ dlm_set_recover_status(ls, DLM_RS_LOCKS); + + error = dlm_recover_locks_wait(ls); + if (error) { + log_error(ls, "recover_locks_wait failed %d", error); + goto fail; + } } dlm_release_root_list(ls); -- cgit v1.2.3-70-g09d2 From 2896ee37ccc1f9acb244c9b02becb74a43661009 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Mon, 27 Nov 2006 11:31:22 -0600 Subject: [DLM] fix add_requestqueue checking nodes list Requests that arrive after recovery has started are saved in the requestqueue and processed after recovery is done. Some of these requests are purged during recovery if they are from nodes that have been removed. We move the purging of the requests (dlm_purge_requestqueue) to later in the recovery sequence which allows the routine saving requests (dlm_add_requestqueue) to avoid filtering out requests by nodeid since the same will be done by the purge. The current code has add_requestqueue filtering by nodeid but doesn't hold any locks when accessing the list of current nodes. This also means that we need to call the purge routine when the lockspace is being shut down since the add routine will not be rejecting requests itself any more. Signed-off-by: David Teigland Signed-off-by: Steven Whitehouse --- fs/dlm/lockspace.c | 2 ++ fs/dlm/recoverd.c | 16 ++++++++-------- fs/dlm/requestqueue.c | 7 ++++--- 3 files changed, 14 insertions(+), 11 deletions(-) (limited to 'fs/dlm/recoverd.c') diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index f8842ca443c..791388b25c3 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -22,6 +22,7 @@ #include "memory.h" #include "lock.h" #include "recover.h" +#include "requestqueue.h" #ifdef CONFIG_DLM_DEBUG int dlm_create_debug_file(struct dlm_ls *ls); @@ -684,6 +685,7 @@ static int release_lockspace(struct dlm_ls *ls, int force) * Free structures on any other lists */ + dlm_purge_requestqueue(ls); kfree(ls->ls_recover_args); dlm_clear_free_entries(ls); dlm_clear_members(ls); diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c index 8bb895ffd90..9dc2f9156f1 100644 --- a/fs/dlm/recoverd.c +++ b/fs/dlm/recoverd.c @@ -93,14 +93,6 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) goto fail; } - /* - * Purge directory-related requests that are saved in requestqueue. - * All dir requests from before recovery are invalid now due to the dir - * rebuild and will be resent by the requesting nodes. - */ - - dlm_purge_requestqueue(ls); - /* * Wait for all nodes to complete directory rebuild. */ @@ -181,6 +173,14 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) dlm_release_root_list(ls); + /* + * Purge directory-related requests that are saved in requestqueue. + * All dir requests from before recovery are invalid now due to the dir + * rebuild and will be resent by the requesting nodes. + */ + + dlm_purge_requestqueue(ls); + dlm_set_recover_status(ls, DLM_RS_DONE); error = dlm_recover_done_wait(ls); if (error) { diff --git a/fs/dlm/requestqueue.c b/fs/dlm/requestqueue.c index 0226d2a0a0f..65008d79c96 100644 --- a/fs/dlm/requestqueue.c +++ b/fs/dlm/requestqueue.c @@ -36,9 +36,6 @@ int dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, struct dlm_header *hd) int length = hd->h_length; int rv = 0; - if (dlm_is_removed(ls, nodeid)) - return 0; - e = kmalloc(sizeof(struct rq_entry) + length, GFP_KERNEL); if (!e) { log_print("dlm_add_requestqueue: out of memory\n"); @@ -133,6 +130,10 @@ static int purge_request(struct dlm_ls *ls, struct dlm_message *ms, int nodeid) { uint32_t type = ms->m_type; + /* the ls is being cleaned up and freed by release_lockspace */ + if (!ls->ls_count) + return 1; + if (dlm_is_removed(ls, nodeid)) return 1; -- cgit v1.2.3-70-g09d2 From 57adf7eede38d315e0e328c52484d6a596e9a238 Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Wed, 29 Nov 2006 09:33:48 -0500 Subject: [DLM] fix format warnings in rcom.c and recoverd.c This fixes the following gcc warnings generated on the architectures where uint64_t != unsigned long long (e.g. ppc64). fs/dlm/rcom.c:154: warning: format '%llx' expects type 'long long unsigned int', but argument 4 has type 'uint64_t' fs/dlm/rcom.c:154: warning: format '%llx' expects type 'long long unsigned int', but argument 5 has type 'uint64_t' fs/dlm/recoverd.c:48: warning: format '%llx' expects type 'long long unsigned int', but argument 3 has type 'uint64_t' fs/dlm/recoverd.c:202: warning: format '%llx' expects type 'long long unsigned int', but argument 3 has type 'uint64_t' fs/dlm/recoverd.c:210: warning: format '%llx' expects type 'long long unsigned int', but argument 3 has type 'uint64_t' Signed-off-by: Ryusuke Konishi Signed-off-by: Patrick Caulfield Signed-off-by: Steven Whitehouse --- fs/dlm/rcom.c | 3 ++- fs/dlm/recoverd.c | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'fs/dlm/recoverd.c') diff --git a/fs/dlm/rcom.c b/fs/dlm/rcom.c index c42f2db6c4b..4cc31be9cd9 100644 --- a/fs/dlm/rcom.c +++ b/fs/dlm/rcom.c @@ -172,7 +172,8 @@ static void receive_sync_reply(struct dlm_ls *ls, struct dlm_rcom *rc_in) rc_in->rc_id != ls->ls_rcom_seq) { log_debug(ls, "reject reply %d from %d seq %llx expect %llx", rc_in->rc_type, rc_in->rc_header.h_nodeid, - rc_in->rc_id, ls->ls_rcom_seq); + (unsigned long long)rc_in->rc_id, + (unsigned long long)ls->ls_rcom_seq); goto out; } memcpy(ls->ls_recover_buf, rc_in, rc_in->rc_header.h_length); diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c index 9dc2f9156f1..650536aa513 100644 --- a/fs/dlm/recoverd.c +++ b/fs/dlm/recoverd.c @@ -45,7 +45,7 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) unsigned long start; int error, neg = 0; - log_debug(ls, "recover %llx", rv->seq); + log_debug(ls, "recover %llx", (unsigned long long)rv->seq); mutex_lock(&ls->ls_recoverd_active); @@ -212,7 +212,8 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) dlm_astd_wake(); - log_debug(ls, "recover %llx done: %u ms", rv->seq, + log_debug(ls, "recover %llx done: %u ms", + (unsigned long long)rv->seq, jiffies_to_msecs(jiffies - start)); mutex_unlock(&ls->ls_recoverd_active); @@ -220,7 +221,8 @@ static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) fail: dlm_release_root_list(ls); - log_debug(ls, "recover %llx error %d", rv->seq, error); + log_debug(ls, "recover %llx error %d", + (unsigned long long)rv->seq, error); mutex_unlock(&ls->ls_recoverd_active); return error; } -- cgit v1.2.3-70-g09d2