From d966e476e0c2e9c6c59e6fc95a9090a473c97771 Mon Sep 17 00:00:00 2001 From: Thierry Bordaz Date: Jul 13 2022 09:53:27 +0000 Subject: Issue 45 - RFE - Allow to rebuild the compat tree Description: For different reasons, the compat tree maps might get corrupted and some entries would be missing. This RFE provides a way to rebuild online the whole compat tree on demand via a DS task. Fix: It registers tasks callbacks under cn=chema compatibility refresh task,cn=tasks,cn=config. The task meta data contains the filter (SCH_CONTAINER_CONFIGURATION_FILTER) and the plugin_state (not retrievable via pblock). The rebuild task (backend_shr_refresh_thread) is shared as it can be used by NIS or Schema Compat. The rebuild task disable access to the maps, free and reinit them. Then rebuild them using the already implemented backend_shr_data_initialize_thread. Review: Alexander Bokovoy (thanks !) --- diff --git a/src/back-nis.c b/src/back-nis.c index 2001222..4144d21 100644 --- a/src/back-nis.c +++ b/src/back-nis.c @@ -1046,12 +1046,103 @@ backend_entry_is_delete_related(const char *group, const char *set, bool_t flag, { return FALSE; } +/* This task is called when the following entry is added + * + * dn: cn=rebuild 0,cn=NIS Server refresh task,cn=tasks,cn=config + * objectClass: top + * objectClass: extensibleObject + * cn: rebuild 0 + */ +static int +backend_nis_refresh_task(Slapi_PBlock *pb, + Slapi_Entry *e, + Slapi_Entry *eAfter __attribute__((unused)), + int *returncode, + char *returntext __attribute__((unused)), + void *arg) +{ + PRThread *thread = NULL; + int rv = SLAPI_DSE_CALLBACK_OK; + Slapi_Task *task = NULL; + task_data *mytaskdata = NULL; + char *bind_dn; + const char *filter; + + *returncode = LDAP_SUCCESS; + + /* get the requestor dn for our thread data*/ + slapi_pblock_get(pb, SLAPI_REQUESTOR_DN, &bind_dn); + + /* default filter for mapped entries */ + if ((filter = slapi_fetch_attr(e, "filter", NIS_MAP_CONFIGURATION_FILTER)) == NULL) { + *returncode = LDAP_OBJECT_CLASS_VIOLATION; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + + /* allocate new task now */ + task = slapi_plugin_new_task(slapi_entry_get_ndn(e), arg); + if (task == NULL) { + slapi_log_error(SLAPI_LOG_INFO, "nis-server", "backend_nis_refresh_task - Unable to allocate new task!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata = (task_data *)slapi_ch_malloc(sizeof(task_data)); + if (mytaskdata == NULL) { + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->state = nis_get_plugin_state(); + mytaskdata->bind_dn = slapi_ch_strdup(bind_dn); + if (mytaskdata->bind_dn == NULL) { + slapi_log_error(SLAPI_LOG_INFO, "nis-server", "backend_nis_refresh_task - Unable initialize task data bind_dn!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->filter = slapi_ch_strdup(filter); + if (mytaskdata->filter == NULL) { + slapi_log_error(SLAPI_LOG_INFO, "nis-server", "backend_nis_refresh_task - Unable initialize task data filter!\n"); + slapi_ch_free_string(&mytaskdata->bind_dn); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->plugin_log_id = "nis-server"; + + /* set a destructor that will clean up schemadir for us when the task is complete */ + slapi_task_set_destructor_fn(task, backend_shr_refresh_destructor); + + /* Stash our task_data for use by the task thread */ + slapi_task_set_data(task, mytaskdata); + + /* start the schema reload task as a separate thread */ + thread = PR_CreateThread(PR_USER_THREAD, backend_shr_refresh_thread, + (void *)task, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE); + if (thread == NULL) { + slapi_log_error(SLAPI_LOG_ERR, "nis-server", + "backend_nis_refresh_task - Unable to create schema compat refresh task thread!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + } else { + /* thread successful */ + rv = SLAPI_DSE_CALLBACK_OK; + } + +out: + + return rv; +} /* Scan for the list of configured domains and maps. */ void backend_startup(Slapi_PBlock *pb, struct plugin_state *state) { - backend_shr_startup(state, pb, NIS_MAP_CONFIGURATION_FILTER); + backend_shr_startup(state, pb, NIS_MAP_CONFIGURATION_FILTER); + slapi_plugin_task_register_handler("NIS Server refresh task", backend_nis_refresh_task, pb); } void diff --git a/src/back-sch.c b/src/back-sch.c index 172d619..04ae8cc 100644 --- a/src/back-sch.c +++ b/src/back-sch.c @@ -1060,13 +1060,7 @@ backend_get_set_config(Slapi_PBlock *parent_pb, struct plugin_state *state, slapi_sdn_free(&groupdn); } -/* Given an entry, return the filter which will match a container entry beneath - * the plugin's configuration entry. */ -const char * -backend_entry_get_set_config_entry_filter(void) -{ - return SCH_CONTAINER_CONFIGURATION_FILTER; -} + /* Re-read plugin-wide settings that may have changed. Nothing to do. */ void @@ -2918,12 +2912,119 @@ backend_check_empty(struct plugin_state *state, set, strlen(set) ? ", " : "", group); } } +/* Given an entry, return the filter which will match a container entry beneath + * the plugin's configuration entry. */ +char * +backend_entry_get_set_config_entry_filter(void) +{ + char *filter; + filter = strdup(SCH_CONTAINER_CONFIGURATION_FILTER); + return filter; +} +/* This task is called when the following entry is added + * + * dn: cn=rebuild 0,cn=Schema compatibility refresh task,cn=tasks,cn=config + * objectClass: top + * objectClass: extensibleObject + * cn: rebuild 0 + */ +static int +backend_sch_refresh_task(Slapi_PBlock *pb, + Slapi_Entry *e, + Slapi_Entry *eAfter __attribute__((unused)), + int *returncode, + char *returntext __attribute__((unused)), + void *arg) +{ + PRThread *thread = NULL; + int rv = SLAPI_DSE_CALLBACK_OK; + Slapi_Task *task = NULL; + task_data *mytaskdata = NULL; + char *bind_dn; + const char *filter; + char *f; + + *returncode = LDAP_SUCCESS; + + /* get the requestor dn for our thread data*/ + slapi_pblock_get(pb, SLAPI_REQUESTOR_DN, &bind_dn); + + /* default filter for mapped entries */ + if ((f = backend_entry_get_set_config_entry_filter()) == NULL) { + slapi_log_error(SLAPI_LOG_ERR, "schema-compat-plugin", "backend_sch_refresh_task - Unable retrieve the container filter!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + if ((filter = slapi_fetch_attr(e, "filter", f)) == NULL) { + *returncode = LDAP_OBJECT_CLASS_VIOLATION; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + /* allocate new task now */ + task = slapi_plugin_new_task(slapi_entry_get_ndn(e), arg); + if (task == NULL) { + slapi_log_error(SLAPI_LOG_ERR, "schema-compat-plugin", "backend_sch_refresh_task - Unable to allocate new task!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + + mytaskdata = (task_data *)slapi_ch_malloc(sizeof(task_data)); + if (mytaskdata == NULL) { + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->state = schema_compat_get_plugin_state(); + mytaskdata->bind_dn = slapi_ch_strdup(bind_dn); + if (mytaskdata->bind_dn == NULL) { + slapi_log_error(SLAPI_LOG_INFO, "schema-compat-plugin", "backend_sch_refresh_task - Unable initialize task data bind_dn!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->filter = slapi_ch_strdup(filter); + if (mytaskdata->filter == NULL) { + slapi_log_error(SLAPI_LOG_INFO, "schema-compat-plugin", "backend_sch_refresh_task - Unable initialize task data filter!\n"); + slapi_ch_free_string(&mytaskdata->bind_dn); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + goto out; + } + mytaskdata->plugin_log_id = "schema-compat-plugin"; + + /* set a destructor that will clean up schemadir for us when the task is complete */ + slapi_task_set_destructor_fn(task, backend_shr_refresh_destructor); + + /* Stash our task_data for use by the task thread */ + slapi_task_set_data(task, mytaskdata); + + /* start the schema reload task as a separate thread */ + thread = PR_CreateThread(PR_USER_THREAD, backend_shr_refresh_thread, + (void *)task, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, SLAPD_DEFAULT_THREAD_STACKSIZE); + if (thread == NULL) { + slapi_log_error(SLAPI_LOG_ERR, "schema-compat-plugin", + "backend_sch_refresh_task - Unable to create schema compat refresh task thread!\n"); + *returncode = LDAP_OPERATIONS_ERROR; + rv = SLAPI_DSE_CALLBACK_ERROR; + } else { + /* thread successful */ + rv = SLAPI_DSE_CALLBACK_OK; + } + +out: + + return rv; +} /* Populate our data cache. */ void backend_startup(Slapi_PBlock *pb, struct plugin_state *state) { - backend_shr_startup(state, pb, SCH_CONTAINER_CONFIGURATION_FILTER); + backend_shr_startup(state, pb, SCH_CONTAINER_CONFIGURATION_FILTER); + slapi_plugin_task_register_handler("Schema compatibility refresh task", backend_sch_refresh_task, pb); } void diff --git a/src/back-shr.c b/src/back-shr.c index c6a181e..8d1c949 100644 --- a/src/back-shr.c +++ b/src/back-shr.c @@ -796,9 +796,11 @@ backend_shr_data_initialize_thread(time_t when, void *arg) if (!cbdata->state->start_priming_thread) { slapi_log_error(SLAPI_LOG_PLUGIN, cbdata->state->plugin_desc->spd_id, - "Likely a shutdown occurred before we started \n"); + "Shutdown occurred before we started or other priming thread is already running\n"); goto done; } + /* We are going to prim, time to prevent other priming to start */ + cbdata->state->start_priming_thread = 0; cbdata->state->priming_tid = wrap_start_thread(&backend_shr_data_initialize_thread_cb, arg); if (cbdata->state->priming_tid == NULL) { @@ -811,11 +813,131 @@ backend_shr_data_initialize_thread(time_t when, void *arg) "%s tree scan will start in about %d seconds!\n", cbdata->state->plugin_desc->spd_id, PLUGIN_SCAN_DELAY); } + /* Now it is primed let others priming to proceed */ + cbdata->state->start_priming_thread = 1; done: wrap_mutex_unlock(cbdata->state->priming_mutex); } +/* routine that free the structure attached as task data */ +void +backend_shr_refresh_destructor(Slapi_Task *task) +{ + if (task) { + task_data *mydata = (task_data *)slapi_task_get_data(task); + while (slapi_task_get_refcount(task) > 0) { + /* Yield to wait for the fixup task finishes. */ + DS_Sleep(PR_MillisecondsToInterval(100)); + } + if (mydata) { + /* mydata->state is global, do not free it + * mydata->plugin_log_id is a constant, do not free it + */ + slapi_ch_free_string(&mydata->bind_dn); + slapi_ch_free_string(&mydata->filter); + /* Need to cast to avoid a compiler warning */ + slapi_ch_free((void **)&mydata); + } + } +} + +/* + * Task thread + + */ +void +backend_shr_refresh_thread(void *arg) +{ + Slapi_Task *task = (Slapi_Task *)arg; + int rv = 0; + int total_work = 2; + task_data *td = NULL; + + if (!task) { + return; /* no task */ + } + slapi_task_inc_refcount(task); + + /* Fetch our task data from the task */ + td = (task_data *)slapi_task_get_data(task); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, "refresh_thread --> refcount incremented.\n"); + + /* Initialize and set the bind dn in the thread data. + * Used for setting creatorname of the task entry. + */ + slapi_td_set_dn(slapi_ch_strdup(td->bind_dn)); + + /* update task state to show it's running */ + slapi_task_begin(task, total_work); + + /* Check if refresh is not already running or will start soon */ + if (td->state->ready_to_serve == 0) { + slapi_task_log_notice(task, "Refresh task already running or initial priming not completed yet."); + slapi_task_log_status(task, "Refresh task already running or initial priming not completed yet."); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, + "backend_shr_refresh_thread - Refresh task already running or initial priming not completed yet\n"); + goto done; + } + + slapi_task_log_notice(task, "Refresh task starts\n"); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, "backend_shr_refresh_thread - Refresh task starts\n"); + slapi_task_inc_progress(task); + + if (slapi_is_shutting_down()) { + slapi_task_log_notice(task, "Server is shutting down; Refresh aborted."); + slapi_task_log_status(task, "Server is shutting down; Refresh aborted."); + slapi_log_error(SLAPI_LOG_ERR, td->plugin_log_id, "backend_shr_refresh_thread - Server is shutting down; Refresh aborted."); + } else { + struct backend_shr_data_init_cbdata *cbdata = NULL; + + slapi_task_log_notice(task, "Refresh maps starting soon."); + slapi_task_log_status(task, "Refresh maps starting soon."); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, "backend_shr_refresh_thread - Refresh maps starting soon.\n"); + + cbdata = (struct backend_shr_data_init_cbdata *) slapi_ch_malloc(sizeof(struct backend_shr_data_init_cbdata)); + if (cbdata == NULL) { + slapi_log_error(SLAPI_LOG_FATAL, + td->plugin_log_id, + "failed to create a task for populating the maps"); + rv = SLAPI_PLUGIN_FAILURE; + goto done; + } + cbdata->state = td->state; + cbdata->filter = td->filter; + + /* disable access to the maps */ + PR_AtomicSet(&td->state->ready_to_serve, 0); + + /* Free the old maps and prepare the new ones */ + map_done(td->state); + map_init(NULL, td->state); + + /* Schedule the initialization of the maps */ + slapi_eq_once(backend_shr_data_initialize_thread, cbdata, PR_SecondsToInterval(1)); + PR_Sleep(PR_SecondsToInterval(1)); + + /* Then wait for its completion */ + slapi_task_log_notice(task, "Refresh maps still going on."); + slapi_task_log_status(task, "Refresh maps still going on."); + while (td->state->ready_to_serve == 0) { + PR_Sleep(PR_SecondsToInterval(5)); + } + slapi_task_inc_progress(task); + + /* update task state to say we're finished */ + slapi_task_log_notice(task, "Refresh maps task finished."); + slapi_task_log_status(task, "Refresh maps task finished."); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, "backend_shr_refresh_thread - Refresh maps task finished.\n"); + + } +done: + + /* this will queue the destruction of the task */ + slapi_task_finish(task, rv); + slapi_task_dec_refcount(task); + slapi_log_error(SLAPI_LOG_INFO, td->plugin_log_id, "backend_shr_refresh_thread <-- refcount decremented.\n"); +} /* Scan for the list of configured groups and sets. */ void backend_shr_startup(struct plugin_state *state, @@ -1045,10 +1167,10 @@ static bool_t backend_shr_entry_is_a_set(struct plugin_state *state, Slapi_PBlock *pb, Slapi_Entry *e) { - return backend_shr_entry_matches(pb, e, - state->plugin_base, - LDAP_SCOPE_ONELEVEL, - backend_entry_get_set_config_entry_filter()); + return backend_shr_entry_matches(pb, e, + state->plugin_base, + LDAP_SCOPE_ONELEVEL, + backend_entry_get_set_config_entry_filter()); } /* Build a filter which includes the basic_filter, if given, and ANDs that diff --git a/src/back-shr.h b/src/back-shr.h index e0d154b..fad1632 100644 --- a/src/back-shr.h +++ b/src/back-shr.h @@ -43,9 +43,10 @@ PRBool backend_shr_write_ignore(Slapi_PBlock *pb); struct plugin_state; +void backend_shr_refresh_destructor(Slapi_Task *task); +void backend_shr_refresh_thread(void *arg); void backend_shr_free_server_name(struct plugin_state *state, char *master); -int backend_shr_read_server_name(Slapi_PBlock *pb, struct plugin_state *state, - char **master); +int backend_shr_read_server_name(Slapi_PBlock *pb, struct plugin_state *state, char **master); void backend_shr_free_strlist(char **strlist); char **backend_shr_dup_strlist_n(char **strlist, int n); diff --git a/src/backend.h b/src/backend.h index 4034704..aee49a2 100644 --- a/src/backend.h +++ b/src/backend.h @@ -87,11 +87,21 @@ void backend_get_set_config(Slapi_PBlock *pb, struct plugin_state *state, void backend_free_set_config(char **bases, char *entry_filter); /* Check if an entry is a set configuration, and add or remove one. */ -const char *backend_entry_get_set_config_entry_filter(void); struct backend_set_config_entry_add_cbdata { - struct plugin_state *state; - Slapi_PBlock *pb; + struct plugin_state *state; + Slapi_PBlock *pb; }; +/* This structure is used during the refresh tasks*/ +typedef struct _task_data +{ + struct plugin_state *state; /* used to provide the plugin state to the refresh thread */ + char *bind_dn; /* not used ATM */ + char *filter; /* it can be set in the task entry + * by default standard plugin filter are used + * NIS_MAP_CONFIGURATION_FILTER or SCH_CONTAINER_CONFIGURATION_FILTER + */ + char *plugin_log_id; /* used for error log */ +} task_data; int backend_set_config_entry_add_cb(Slapi_Entry *e, void *callback_data); int backend_set_config_entry_delete_cb(Slapi_Entry *e, void *callback_data); diff --git a/src/plug-nis.c b/src/plug-nis.c index 0ce4ae6..1e9b3c9 100644 --- a/src/plug-nis.c +++ b/src/plug-nis.c @@ -73,6 +73,15 @@ plugin_description = { }; static struct plugin_state *global_plugin_state; +/* This function is call by the refresh task + * it is required because with the task mechanism, plugin state + * is not present in pblock(SLAPI_PRIVATE) so it needs to be retrieved + * by the plugin task, before calling the shared task + */ +struct plugin_state *nis_get_plugin_state() +{ + return global_plugin_state; +} /* Populate the map cache, register with the local portmapper, and then start * the plugin's work thread to answer requests using the cache. */ static int @@ -598,7 +607,13 @@ nis_plugin_init(Slapi_PBlock *pb) #endif slapi_log_error(SLAPI_LOG_PLUGIN, state->plugin_desc->spd_id, "registered plugin hooks\n"); - global_plugin_state = NULL; + /* The registered plugin callbacks retrieve the global_plugin_state + * from the pblock(SLAPI_PLUGIN_PRIVATE). + * The task mechanism does not provide the reference to global_plugin_state. + * So for the map refresh task, global_plugin_state should not be reset + * and it is provided by nis_get_plugin_state(). + * global_plugin_state = NULL; + */ /* Note that the plugin was successfully loaded. */ slapi_log_error(SLAPI_LOG_PLUGIN, plugin_description.spd_id, "plugin initialized\n"); diff --git a/src/plug-sch.c b/src/plug-sch.c index 913abe2..a73cbd8 100644 --- a/src/plug-sch.c +++ b/src/plug-sch.c @@ -79,6 +79,15 @@ plugin_description = { }; static struct plugin_state *global_plugin_state; +/* This function is call by the refresh task + * it is required because with the task mechanism, plugin state + * is not present in pblock(SLAPI_PRIVATE) so it needs to be retrieved + * by the plugin task, before calling the shared task + */ +struct plugin_state *schema_compat_get_plugin_state() +{ + return global_plugin_state; +} /* Handle the part of startup that needs to be done before we drop privileges, * which for this plugin isn't much at all. */ static int @@ -441,7 +450,13 @@ schema_compat_plugin_init(Slapi_PBlock *pb) } slapi_log_error(SLAPI_LOG_PLUGIN, state->plugin_desc->spd_id, "registered plugin hooks\n"); - global_plugin_state = NULL; + /* The registered plugin callbacks retrieve the global_plugin_state + * from the pblock(SLAPI_PLUGIN_PRIVATE). + * The task mechanism does not provide the reference to global_plugin_state. + * So for the map refresh task, global_plugin_state should not be reset + * and it is provided by schema_compat_get_plugin_state(). + * global_plugin_state = NULL; + */ /* Note that the plugin was successfully loaded. */ slapi_log_error(SLAPI_LOG_PLUGIN, plugin_description.spd_id, "plugin initialized\n"); diff --git a/src/plugin.h b/src/plugin.h index 56d672f..4020e65 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -55,5 +55,6 @@ struct plugin_state { void *cached_entries; struct wrapped_rwlock *cached_entries_lock; }; - +struct plugin_state *nis_get_plugin_state(); +struct plugin_state *schema_compat_get_plugin_state(); #endif