From 298ada3b2f7b8aa770df9a5a7d8129f46b4417d7 Mon Sep 17 00:00:00 2001
From: Noriko Hosoi <nhosoi@redhat.com>
Date: Mon, 16 Dec 2013 13:03:19 -0800
Subject: [PATCH 73/78] Ticket #47606 - replica init/bulk import errors should
be more verbose
Description:
1. maxbersize: If the size of an entry is larger than the consumer's
maxbersize, the following error used to be logged:
Incoming BER Element was too long, max allowable is ### bytes.
Change the nsslapd-maxbersize attribute in cn=config to increase.
This message does not indicate how large the maxbersize needs to be.
This patch adds the code to retrieve the failed ber size.
Revised message:
Incoming BER Element was @@@ bytes, max allowable is ### bytes.
Change the nsslapd-maxbersize attribute in cn=config to increase.
Note: There is no lber API that returns the ber size if it fails to
handle the ber. This patch borrows the internal structure of ber
and get the size. This could be risky since the size or structure
of the ber could be updated in the openldap/mozldap lber.
2. cache size: The bulk import depends upon the nsslapd-cachememsize
value in the backend instance entry (e.g., cn=userRoot,cn=ldbm
database,cn=plugins,cn=config). If an entry size is larger than
the cachememsize, the bulk import used to fail with this message:
import userRoot: REASON: entry too large (@@@ bytes) for the
import buffer size (### bytes). Try increasing nsslapd-
cachememsize.
Also, the message follows the skipping entry message:
import userRoot: WARNING: skipping entry "<DN>"
but actually, it did NOT "skip" the entry and continue the bulk
import, but it failed there and completely wiped out the backend
database.
This patch modifies the message as follows:
import userRoot: REASON: entry too large (@@@ bytes) for the
effective import buffer size (### bytes). Try increasing nsslapd-
cachememsize for the backend instance "userRoot".
and as the message mentions, it just skips the failed entry and
continues the bulk import.
3. In repl5_tot_result_threadmain, when conn_read_result_ex returns
non zero (non SUCCESS), it sets abort, but does not set any error
code to rc (return code), which is not considered as "finished" in
repl5_tot_waitfor_async_results and it contines waiting until the
code reaches the max loop count (about 5 minutes). This patch sets
LDAP_CONNECT_ERROR to the return code along with setting abort, if
conn_read_result_ex returns CONN_NOT_CONNECTED. This makes the bulk
import finishes quickly when it fails.
https://fedorahosted.org/389/ticket/47606
Reviewed by rmeggins@redhat.com (Thank you, Rich!!)
(cherry picked from commit 1119083d3d99993421609783efcb8962d78724fc)
(cherry picked from commit fde9ed5bf74b4ea1fff875bcb421137c78af1227)
(cherry picked from commit c9d0b6ccad84dd56a536da883f5a8e5acb01bc4e)
---
.../plugins/replication/repl5_tot_protocol.c | 3 ++
ldap/servers/slapd/back-ldbm/import-threads.c | 8 ++---
ldap/servers/slapd/connection.c | 36 ++++++++++++++++++----
ldap/servers/slapd/openldapber.h | 25 +++++++++++++++
4 files changed, 62 insertions(+), 10 deletions(-)
create mode 100644 ldap/servers/slapd/openldapber.h
diff --git a/ldap/servers/plugins/replication/repl5_tot_protocol.c b/ldap/servers/plugins/replication/repl5_tot_protocol.c
index a241128..3895ace 100644
--- a/ldap/servers/plugins/replication/repl5_tot_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_tot_protocol.c
@@ -203,6 +203,9 @@ static void repl5_tot_result_threadmain(void *param)
/* If so then we need to take steps to abort the update process */
PR_Lock(cb->lock);
cb->abort = 1;
+ if (conres == CONN_NOT_CONNECTED) {
+ cb->rc = LDAP_CONNECT_ERROR;
+ }
PR_Unlock(cb->lock);
}
/* Should we stop ? */
diff --git a/ldap/servers/slapd/back-ldbm/import-threads.c b/ldap/servers/slapd/back-ldbm/import-threads.c
index c0475c6..95433aa 100644
--- a/ldap/servers/slapd/back-ldbm/import-threads.c
+++ b/ldap/servers/slapd/back-ldbm/import-threads.c
@@ -3330,11 +3330,11 @@ static int bulk_import_queue(ImportJob *job, Slapi_Entry *entry)
newesize = (slapi_entry_size(ep->ep_entry) + sizeof(struct backentry));
if (newesize > job->fifo.bsize) { /* entry too big */
- import_log_notice(job, "WARNING: skipping entry \"%s\"",
- slapi_entry_get_dn(ep->ep_entry));
import_log_notice(job, "REASON: entry too large (%lu bytes) for "
- "the import buffer size (%lu bytes). Try increasing nsslapd-cachememsize.",
- (long unsigned int)newesize, (long unsigned int)job->fifo.bsize);
+ "the effective import buffer size (%lu bytes). "
+ "Try increasing nsslapd-cachememsize for the backend instance \"%s\".",
+ (long unsigned int)newesize, (long unsigned int)job->fifo.bsize,
+ job->inst->inst_name);
backentry_clear_entry(ep); /* entry is released in the frontend on failure*/
backentry_free( &ep ); /* release the backend wrapper, here */
PR_Unlock(job->wire_lock);
diff --git a/ldap/servers/slapd/connection.c b/ldap/servers/slapd/connection.c
index fed3512..02c86c5 100644
--- a/ldap/servers/slapd/connection.c
+++ b/ldap/servers/slapd/connection.c
@@ -1749,6 +1749,32 @@ void connection_make_new_pb(Slapi_PBlock **ppb, Connection *conn)
}
+#ifdef USE_OPENLDAP
+#include "openldapber.h"
+#else
+#include "mozldap.h"
+#endif
+
+static ber_tag_t
+_ber_get_len(BerElement *ber, ber_len_t *lenp)
+{
+#ifdef USE_OPENLDAP
+ OLBerElement *lber = (OLBerElement *)ber;
+#else
+ MozElement *lber = (MozElement *)ber;
+#endif
+
+ if (NULL == lenp) {
+ return LBER_DEFAULT;
+ }
+ *lenp = 0;
+ if (NULL == lber) {
+ return LBER_DEFAULT;
+ }
+ *lenp = lber->ber_len;
+ return lber->ber_tag;
+}
+
/*
* Utility function called by connection_read_operation(). This is a
* small wrapper on top of libldap's ber_get_next_buffer_ext().
@@ -1787,18 +1813,16 @@ get_next_from_buffer( void *buffer, size_t buffer_size, ber_len_t *lenp,
if ((LBER_OVERFLOW == *tagp || LBER_DEFAULT == *tagp) && 0 == bytes_scanned &&
!SLAPD_SYSTEM_WOULD_BLOCK_ERROR(errno))
{
- if (LBER_OVERFLOW == *tagp)
- {
- err = SLAPD_DISCONNECT_BER_TOO_BIG;
- }
- else if (errno == ERANGE)
+ if ((LBER_OVERFLOW == *tagp) || (errno == ERANGE))
{
ber_len_t maxbersize = config_get_maxbersize();
+ ber_len_t tmplen = 0;
+ (void)_ber_get_len(ber, &tmplen);
/* openldap does not differentiate between length == 0
and length > max - all we know is that there was a
problem with the length - assume too big */
err = SLAPD_DISCONNECT_BER_TOO_BIG;
- log_ber_too_big_error(conn, 0, maxbersize);
+ log_ber_too_big_error(conn, tmplen, maxbersize);
}
else
{
diff --git a/ldap/servers/slapd/openldapber.h b/ldap/servers/slapd/openldapber.h
new file mode 100644
index 0000000..52644a5
--- /dev/null
+++ b/ldap/servers/slapd/openldapber.h
@@ -0,0 +1,25 @@
+/*
+ * openldap lber library does not provide an API which returns the ber size
+ * (ber->ber_len) when the ber tag is LBER_DEFAULT or LBER_OVERFLOW.
+ * The ber size is useful when issuing an error message to indicate how
+ * large the maxbersize needs to be set.
+ * Borrowed from liblber/lber-int.h
+ */
+struct lber_options {
+ short lbo_valid;
+ unsigned short lbo_options;
+ int lbo_debug;
+};
+struct berelement {
+ struct lber_options ber_opts;
+ ber_tag_t ber_tag;
+ ber_len_t ber_len;
+ ber_tag_t ber_usertag;
+ char *ber_buf;
+ char *ber_ptr;
+ char *ber_end;
+ char *ber_sos_ptr;
+ char *ber_rwptr;
+ void *ber_memctx;
+};
+typedef struct berelement OLBerElement;
--
1.8.1.4