From c0cb15445c1434b3d317b1c06ab1a0ba8dbc6f04 Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Tue, 19 May 2020 15:11:53 -0400
Subject: [PATCH 06/12] Issue 51102 - RFE - ds-replcheck - make online timeout
configurable
Bug Description: When doing an online check with replicas that are very
far apart the connection can time out as the hardcoded
timeout is 5 seconds.
Fix Description: Change the default timeout to never timeout, and add an
CLI option to specify a specific timeout.
Also caught all the possible LDAP exceptions so we can
cleanly "fail". Fixed some python syntax issues, and
improved the entry inconsistency report
relates: https://pagure.io/389-ds-base/issue/51102
Reviewed by: firstyear & spichugi(Thanks!)
---
ldap/admin/src/scripts/ds-replcheck | 90 ++++++++++++++++++-----------
1 file changed, 57 insertions(+), 33 deletions(-)
diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck
index 30bcfd65d..5bb7dfce3 100755
--- a/ldap/admin/src/scripts/ds-replcheck
+++ b/ldap/admin/src/scripts/ds-replcheck
@@ -1,7 +1,7 @@
#!/usr/bin/python3
# --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2018 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
# All rights reserved.
#
# License: GPL (version 3 or any later version).
@@ -21,10 +21,9 @@ import getpass
import signal
from ldif import LDIFRecordList
from ldap.ldapobject import SimpleLDAPObject
-from ldap.cidict import cidict
from ldap.controls import SimplePagedResultsControl
from lib389._entry import Entry
-from lib389.utils import ensure_str, ensure_list_str, ensure_int
+from lib389.utils import ensure_list_str, ensure_int
VERSION = "2.0"
RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
@@ -185,11 +184,11 @@ def report_conflict(entry, attr, opts):
report = True
if 'nscpentrywsi' in entry.data:
- found = False
for val in entry.data['nscpentrywsi']:
if val.lower().startswith(attr + ';'):
if (opts['starttime'] - extract_time(val)) <= opts['lag']:
report = False
+ break
return report
@@ -321,6 +320,9 @@ def ldif_search(LDIF, dn):
count = 0
ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']
val = ""
+ attr = ""
+ state_attr = ""
+ part_dn = ""
result['entry'] = None
result['conflict'] = None
result['tombstone'] = False
@@ -570,6 +572,7 @@ def cmp_entry(mentry, rentry, opts):
if val.lower().startswith(mattr + ';'):
if not found:
diff['diff'].append(" Master:")
+ diff['diff'].append(" - Value: %s" % (val.split(':')[1].lstrip()))
diff['diff'].append(" - State Info: %s" % (val))
diff['diff'].append(" - Date: %s\n" % (time.ctime(extract_time(val))))
found = True
@@ -588,6 +591,7 @@ def cmp_entry(mentry, rentry, opts):
if val.lower().startswith(mattr + ';'):
if not found:
diff['diff'].append(" Replica:")
+ diff['diff'].append(" - Value: %s" % (val.split(':')[1].lstrip()))
diff['diff'].append(" - State Info: %s" % (val))
diff['diff'].append(" - Date: %s\n" % (time.ctime(extract_time(val))))
found = True
@@ -654,7 +658,6 @@ def do_offline_report(opts, output_file=None):
rconflicts = []
rtombstones = 0
mtombstones = 0
- idx = 0
# Open LDIF files
try:
@@ -926,7 +929,7 @@ def validate_suffix(ldapnode, suffix, hostname):
:return - True if suffix exists, otherwise False
"""
try:
- master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE )
+ ldapnode.search_s(suffix, ldap.SCOPE_BASE)
except ldap.NO_SUCH_OBJECT:
print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix))
return False
@@ -968,12 +971,12 @@ def connect_to_replicas(opts):
replica = SimpleLDAPObject(ruri)
# Set timeouts
- master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
- master.set_option(ldap.OPT_TIMEOUT,5.0)
- replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
- replica.set_option(ldap.OPT_TIMEOUT,5.0)
+ master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+ master.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
+ replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+ replica.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
- # Setup Secure Conenction
+ # Setup Secure Connection
if opts['certdir'] is not None:
# Setup Master
if opts['mprotocol'] != LDAPI:
@@ -1003,7 +1006,7 @@ def connect_to_replicas(opts):
try:
master.simple_bind_s(opts['binddn'], opts['bindpw'])
except ldap.SERVER_DOWN as e:
- print("Cannot connect to %r" % muri)
+ print(f"Cannot connect to {muri} ({str(e)})")
sys.exit(1)
except ldap.LDAPError as e:
print("Error: Failed to authenticate to Master: ({}). "
@@ -1014,7 +1017,7 @@ def connect_to_replicas(opts):
try:
replica.simple_bind_s(opts['binddn'], opts['bindpw'])
except ldap.SERVER_DOWN as e:
- print("Cannot connect to %r" % ruri)
+ print(f"Cannot connect to {ruri} ({str(e)})")
sys.exit(1)
except ldap.LDAPError as e:
print("Error: Failed to authenticate to Replica: ({}). "
@@ -1218,7 +1221,6 @@ def do_online_report(opts, output_file=None):
"""
m_done = False
r_done = False
- done = False
report = {}
report['diff'] = []
report['m_missing'] = []
@@ -1257,15 +1259,22 @@ def do_online_report(opts, output_file=None):
# Read the results and start comparing
while not m_done or not r_done:
- if not m_done:
- m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
- elif not r_done:
- m_rdata = []
-
- if not r_done:
- r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
- elif not m_done:
- r_rdata = []
+ try:
+ if not m_done:
+ m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
+ elif not r_done:
+ m_rdata = []
+ except ldap.LDAPError as e:
+ print("Error: Problem getting the results from the master: %s", str(e))
+ sys.exit(1)
+ try:
+ if not r_done:
+ r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
+ elif not m_done:
+ r_rdata = []
+ except ldap.LDAPError as e:
+ print("Error: Problem getting the results from the replica: %s", str(e))
+ sys.exit(1)
# Convert entries
mresult = convert_entries(m_rdata)
@@ -1291,11 +1300,15 @@ def do_online_report(opts, output_file=None):
]
if m_pctrls:
if m_pctrls[0].cookie:
- # Copy cookie from response control to request control
- req_pr_ctrl.cookie = m_pctrls[0].cookie
- master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
- "(|(objectclass=*)(objectclass=ldapsubentry))",
- ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+ try:
+ # Copy cookie from response control to request control
+ req_pr_ctrl.cookie = m_pctrls[0].cookie
+ master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+ "(|(objectclass=*)(objectclass=ldapsubentry))",
+ ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+ except ldap.LDAPError as e:
+ print("Error: Problem searching the master: %s", str(e))
+ sys.exit(1)
else:
m_done = True # No more pages available
else:
@@ -1311,11 +1324,15 @@ def do_online_report(opts, output_file=None):
if r_pctrls:
if r_pctrls[0].cookie:
- # Copy cookie from response control to request control
- req_pr_ctrl.cookie = r_pctrls[0].cookie
- replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
- "(|(objectclass=*)(objectclass=ldapsubentry))",
- ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+ try:
+ # Copy cookie from response control to request control
+ req_pr_ctrl.cookie = r_pctrls[0].cookie
+ replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+ "(|(objectclass=*)(objectclass=ldapsubentry))",
+ ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+ except ldap.LDAPError as e:
+ print("Error: Problem searching the replica: %s", str(e))
+ sys.exit(1)
else:
r_done = True # No more pages available
else:
@@ -1426,6 +1443,9 @@ def init_online_params(args):
# prompt for password
opts['bindpw'] = getpass.getpass('Enter password: ')
+ # lastly handle the timeout
+ opts['timeout'] = int(args.timeout)
+
return opts
@@ -1553,6 +1573,8 @@ def main():
state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None)
state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections',
dest='certdir', default=None)
+ state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections. Default is no timeout.',
+ type=int, dest='timeout', default=-1)
# Online mode
online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences")
@@ -1577,6 +1599,8 @@ def main():
online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)',
dest='pagesize', default=500)
online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None)
+ online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections. Default is no timeout.',
+ type=int, dest='timeout', default=-1)
# Offline LDIF mode
offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')")
--
2.26.2