From cb54fa78fdd5e94f890c3fa1c03481358e3c82ce Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Thu, 9 Jul 2015 09:59:46 -0400
Subject: [PATCH 13/20] Ticket 48217 - cleanAllRUV hangs shutdown if not all of
the replicas are online
Bug Description: There are race conditions where we might not notify the
clean task when a shutdown is occuring. This casues the
task refcount to be not decremented, which hangs the
destructor function.
Fix Description: Check that the server is not shutting down before going
to sleep, and notify the clean/abort tasks to stop in
the destructor functions(instead of in the mmr plugin stop
function).
https://fedorahosted.org/389/ticket/48217
Reviewed by: lkrispen(Thanks!)
(cherry picked from commit d6269f2e6898a187d43e3368860b13cdbd39ec55)
(cherry picked from commit 0bb881aea92d64e509cf7604e86559779e4f9b77)
---
.../plugins/replication/repl5_replica_config.c | 49 ++++++++++++++--------
1 file changed, 31 insertions(+), 18 deletions(-)
diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c
index faa86b8..446da3f 100644
--- a/ldap/servers/plugins/replication/repl5_replica_config.c
+++ b/ldap/servers/plugins/replication/repl5_replica_config.c
@@ -1738,7 +1738,9 @@ replica_cleanallruv_thread(void *arg)
}
if (data->task) {
slapi_task_inc_refcount(data->task);
- slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name, "replica_cleanallruv_thread --> refcount incremented.\n");
+ slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name,
+ "replica_cleanallruv_thread --> refcount incremented (%d).\n",
+ data->task->task_refcount);
}
/*
* Initialize our settings
@@ -1871,10 +1873,11 @@ replica_cleanallruv_thread(void *arg)
*/
cleanruv_log(data->task, data->rid, CLEANALLRUV_ID, "Not all replicas have received the "
"cleanallruv extended op, retrying in %d seconds",interval);
- PR_Lock( notify_lock );
- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
- PR_Unlock( notify_lock );
-
+ if(!slapi_is_shutting_down()){
+ PR_Lock( notify_lock );
+ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
+ PR_Unlock( notify_lock );
+ }
if(interval < 14400){ /* 4 hour max */
interval = interval * 2;
} else {
@@ -1974,6 +1977,7 @@ done:
if(data->repl_obj && free_obj){
object_release(data->repl_obj);
}
+
csn_free(&data->maxcsn);
slapi_sdn_free(&data->sdn);
slapi_ch_free_string(&data->repl_root);
@@ -1987,6 +1991,7 @@ replica_cleanall_ruv_destructor(Slapi_Task *task)
{
slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
"replica_cleanall_ruv_destructor -->\n" );
+ stop_ruv_cleaning();
if (task) {
while (slapi_task_get_refcount(task) > 0) {
/* Yield to wait for the fixup task finishes. */
@@ -2002,6 +2007,7 @@ replica_cleanall_ruv_abort_destructor(Slapi_Task *task)
{
slapi_log_error( SLAPI_LOG_PLUGIN, repl_plugin_name,
"replica_cleanall_ruv_abort_destructor -->\n" );
+ stop_ruv_cleaning();
if (task) {
while (slapi_task_get_refcount(task) > 0) {
/* Yield to wait for the fixup task finishes. */
@@ -2055,9 +2061,11 @@ check_replicas_are_done_cleaning(cleanruv_data *data )
break;
}
cleanruv_log(data->task, data->rid, CLEANALLRUV_ID, "Not all replicas finished cleaning, retrying in %d seconds",interval);
- PR_Lock( notify_lock );
- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
- PR_Unlock( notify_lock );
+ if(!slapi_is_shutting_down()){
+ PR_Lock( notify_lock );
+ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
+ PR_Unlock( notify_lock );
+ }
if(interval < 14400){ /* 4 hour max */
interval = interval * 2;
} else {
@@ -2158,9 +2166,11 @@ check_replicas_are_done_aborting(cleanruv_data *data )
break;
}
cleanruv_log(data->task, data->rid, ABORT_CLEANALLRUV_ID, "Not all replicas finished aborting, retrying in %d seconds",interval);
- PR_Lock( notify_lock );
- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
- PR_Unlock( notify_lock );
+ if(!slapi_is_shutting_down()){
+ PR_Lock( notify_lock );
+ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
+ PR_Unlock( notify_lock );
+ }
if(interval < 14400){ /* 4 hour max */
interval = interval * 2;
} else {
@@ -2212,10 +2222,11 @@ check_agmts_are_caught_up(cleanruv_data *data, char *maxcsn)
}
cleanruv_log(data->task, data->rid, CLEANALLRUV_ID,
"Not all replicas caught up, retrying in %d seconds",interval);
- PR_Lock( notify_lock );
- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
- PR_Unlock( notify_lock );
-
+ if(!slapi_is_shutting_down()){
+ PR_Lock( notify_lock );
+ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
+ PR_Unlock( notify_lock );
+ }
if(interval < 14400){ /* 4 hour max */
interval = interval * 2;
} else {
@@ -2271,10 +2282,12 @@ check_agmts_are_alive(Replica *replica, ReplicaId rid, Slapi_Task *task)
}
cleanruv_log(task, rid, CLEANALLRUV_ID, "Not all replicas online, retrying in %d seconds...",
interval);
- PR_Lock( notify_lock );
- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
- PR_Unlock( notify_lock );
+ if(!slapi_is_shutting_down()){
+ PR_Lock( notify_lock );
+ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) );
+ PR_Unlock( notify_lock );
+ }
if(interval < 14400){ /* 4 hour max */
interval = interval * 2;
} else {
--
1.9.3