Blame SOURCES/0084-Ticket-49576-Update-ds-replcheck-for-new-conflict-en.patch

b663b9
From 19945c4807f6b3269fb65100ddaea5f596f68e72 Mon Sep 17 00:00:00 2001
b663b9
From: Mark Reynolds <mreynolds@redhat.com>
b663b9
Date: Fri, 18 May 2018 07:29:11 -0400
b663b9
Subject: [PATCH 1/6] Ticket 49576 - Update ds-replcheck for new conflict
b663b9
 entries
b663b9
b663b9
Description:  This patch addresses the recvent changes to conflict
b663b9
              entries and tombstones.
b663b9
b663b9
https://pagure.io/389-ds-base/issue/49576
b663b9
b663b9
Reviewed by: tbordaz(Thanks!)
b663b9
b663b9
(cherry picked from commit 53e58cdbfb2a2672ac21cd9b6d59f8b345478324)
b663b9
---
b663b9
 ldap/admin/src/scripts/ds-replcheck | 456 +++++++++++++++++++---------
b663b9
 1 file changed, 312 insertions(+), 144 deletions(-)
b663b9
b663b9
diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck
b663b9
index 45c4670a3..b801ccaa8 100755
b663b9
--- a/ldap/admin/src/scripts/ds-replcheck
b663b9
+++ b/ldap/admin/src/scripts/ds-replcheck
b663b9
@@ -1,7 +1,7 @@
b663b9
 #!/usr/bin/python
b663b9
 
b663b9
 # --- BEGIN COPYRIGHT BLOCK ---
b663b9
-# Copyright (C) 2017 Red Hat, Inc.
b663b9
+# Copyright (C) 2018 Red Hat, Inc.
b663b9
 # All rights reserved.
b663b9
 #
b663b9
 # License: GPL (version 3 or any later version).
b663b9
@@ -9,6 +9,7 @@
b663b9
 # --- END COPYRIGHT BLOCK ---
b663b9
 #
b663b9
 
b663b9
+import os
b663b9
 import re
b663b9
 import time
b663b9
 import ldap
b663b9
@@ -20,7 +21,7 @@ from ldap.ldapobject import SimpleLDAPObject
b663b9
 from ldap.cidict import cidict
b663b9
 from ldap.controls import SimplePagedResultsControl
b663b9
 
b663b9
-VERSION = "1.2"
b663b9
+VERSION = "1.3"
b663b9
 RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
b663b9
 LDAP = 'ldap'
b663b9
 LDAPS = 'ldaps'
b663b9
@@ -36,6 +37,7 @@ class Entry(object):
b663b9
     ''' This is a stripped down version of Entry from python-lib389.
b663b9
     Once python-lib389 is released on RHEL this class will go away.
b663b9
     '''
b663b9
+
b663b9
     def __init__(self, entrydata):
b663b9
         if entrydata:
b663b9
             self.dn = entrydata[0]
b663b9
@@ -51,7 +53,7 @@ class Entry(object):
b663b9
 
b663b9
 
b663b9
 def get_entry(entries, dn):
b663b9
-    ''' Loop over enties looking for a matching dn
b663b9
+    ''' Loop over a list of enties looking for a matching dn
b663b9
     '''
b663b9
     for entry in entries:
b663b9
         if entry.dn == dn:
b663b9
@@ -60,7 +62,7 @@ def get_entry(entries, dn):
b663b9
 
b663b9
 
b663b9
 def remove_entry(rentries, dn):
b663b9
-    ''' Remove an entry from the array of entries
b663b9
+    ''' Remove an entry from the list of entries
b663b9
     '''
b663b9
     for entry in rentries:
b663b9
         if entry.dn == dn:
b663b9
@@ -69,7 +71,7 @@ def remove_entry(rentries, dn):
b663b9
 
b663b9
 
b663b9
 def extract_time(stateinfo):
b663b9
-    ''' Take the nscpEntryWSI attribute and get the most recent timestamp from
b663b9
+    ''' Take the nscpEntryWSI(state info) attribute and get the most recent timestamp from
b663b9
     one of the csns (vucsn, vdcsn, mdcsn, adcsn)
b663b9
 
b663b9
     Return the timestamp in decimal
b663b9
@@ -87,7 +89,7 @@ def extract_time(stateinfo):
b663b9
 
b663b9
 
b663b9
 def convert_timestamp(timestamp):
b663b9
-    ''' Convert createtimestamp to ctime: 20170405184656Z -> Wed Apr  5 19:46:56 2017
b663b9
+    ''' Convert createtimestamp to ctime: 20170405184656Z ----> Wed Apr  5 19:46:56 2017
b663b9
     '''
b663b9
     time_tuple = (int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]),
b663b9
                   int(timestamp[8:10]), int(timestamp[10:12]), int(timestamp[12:14]),
b663b9
@@ -97,27 +99,43 @@ def convert_timestamp(timestamp):
b663b9
 
b663b9
 
b663b9
 def convert_entries(entries):
b663b9
-    '''Convert and normalize the ldap entries.  Take note of conflicts and tombstones
b663b9
-    '''
b663b9
+    '''For online report.  Convert and normalize the ldap entries.  Take note of
b663b9
+    conflicts and tombstones '''
b663b9
     new_entries = []
b663b9
     conflict_entries = []
b663b9
+    glue_entries = []
b663b9
     result = {}
b663b9
     tombstones = 0
b663b9
+
b663b9
     for entry in entries:
b663b9
         new_entry = Entry(entry)
b663b9
         new_entry.data = {k.lower(): v for k, v in list(new_entry.data.items())}
b663b9
-        if 'nsds5replconflict' in new_entry.data:
b663b9
+        if new_entry.dn.endswith("cn=mapping tree,cn=config"):
b663b9
+            '''Skip replica entry (ldapsearch brings this in because the filter
b663b9
+            we use triggers an internal operation to return the config entry - so
b663b9
+            it must be skipped
b663b9
+            '''
b663b9
+            continue
b663b9
+        if ('nsds5replconflict' in new_entry.data and 'nsTombstone' not in new_entry.data['objectclass'] and
b663b9
+            'nstombstone' not in new_entry.data['objectclass']):
b663b9
+            # This is a conflict entry that is NOT a tombstone entry (should this be reconsidered?)
b663b9
             conflict_entries.append(new_entry)
b663b9
+            if 'glue' in new_entry.data['objectclass']:
b663b9
+                # A glue entry here is not necessarily a glue entry there.  Keep track of
b663b9
+                # them for when we check missing entries
b663b9
+                glue_entries.append(new_entry)
b663b9
         else:
b663b9
             new_entries.append(new_entry)
b663b9
 
b663b9
         if 'nstombstonecsn' in new_entry.data:
b663b9
+            # Maintain tombstone count
b663b9
             tombstones += 1
b663b9
     del entries
b663b9
 
b663b9
     result['entries'] = new_entries
b663b9
     result['conflicts'] = conflict_entries
b663b9
     result['tombstones'] = tombstones
b663b9
+    result['glue'] = glue_entries
b663b9
 
b663b9
     return result
b663b9
 
b663b9
@@ -174,20 +192,60 @@ def get_ruv_report(opts):
b663b9
     return report
b663b9
 
b663b9
 
b663b9
+def remove_attr_state_info(attr):
b663b9
+    state_attr = None
b663b9
+    idx = attr.find(';')
b663b9
+    if idx > 0:
b663b9
+        state_attr = attr  # preserve state info for diff report
b663b9
+        if ";deleted" in attr:
b663b9
+            # Ignore this attribute it was deleted
b663b9
+            return None, state_attr
b663b9
+        attr = attr[:idx]
b663b9
+
b663b9
+    return attr.lower(), state_attr
b663b9
+
b663b9
+def add_attr_entry(entry, val, attr, state_attr):
b663b9
+    ''' Offline mode (ldif comparision) Add the attr to the entry, and if there
b663b9
+    is state info add nscpentrywsi attr - we need consistency with online mode
b663b9
+    to make code simpler '''
b663b9
+    if attr is not None:
b663b9
+        if attr in entry:
b663b9
+            entry[attr].append(val)
b663b9
+        else:
b663b9
+            entry[attr] = [val]
b663b9
+
b663b9
+    # Handle state info for diff report
b663b9
+    if state_attr is not None:
b663b9
+        state_attr = state_attr + ": " + val
b663b9
+        if 'nscpentrywsi' in entry:
b663b9
+            entry['nscpentrywsi'].append(state_attr)
b663b9
+        else:
b663b9
+            entry['nscpentrywsi'] = [state_attr]
b663b9
+    val = ""
b663b9
+
b663b9
+
b663b9
 #
b663b9
 # Offline mode helper functions
b663b9
 #
b663b9
-def ldif_search(LDIF, dn, conflicts=False):
b663b9
-    ''' Search ldif by DN
b663b9
+def ldif_search(LDIF, dn):
b663b9
+    ''' Offline mode -  Search ldif for a single DN.  We need to factor in that
b663b9
+    DN's and attribute values can wrap lines and are identified by a leading
b663b9
+    white space.  So we can't fully process an attribute until we get to the
b663b9
+    next attribute.
b663b9
     '''
b663b9
     result = {}
b663b9
     data = {}
b663b9
     found_conflict = False
b663b9
+    found_subentry = False
b663b9
     found_part_dn = False
b663b9
+    found_part_val = False
b663b9
+    found_attr = False
b663b9
+    found_tombstone = False
b663b9
+    found_glue = False
b663b9
     found = False
b663b9
-    reset_line = False
b663b9
     count = 0
b663b9
-
b663b9
+    ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']
b663b9
+    val = ""
b663b9
     result['entry'] = None
b663b9
     result['conflict'] = None
b663b9
     result['tombstone'] = False
b663b9
@@ -195,54 +253,132 @@ def ldif_search(LDIF, dn, conflicts=False):
b663b9
     for line in LDIF:
b663b9
         count += 1
b663b9
         line = line.rstrip()
b663b9
-        if reset_line:
b663b9
-            reset_line = False
b663b9
-            line = prev_line
b663b9
+
b663b9
         if found:
b663b9
+            # We found our entry, now build up the entry (account from line wrap)
b663b9
             if line == "":
b663b9
-                # End of entry
b663b9
+                # End of entry - update entry's last attribute value and break out
b663b9
+                add_attr_entry(data, val, attr, state_attr)
b663b9
+                val = ""
b663b9
+                # Done!
b663b9
                 break
b663b9
 
b663b9
             if line[0] == ' ':
b663b9
-                # continuation line
b663b9
-                prev = data[attr][len(data[attr]) - 1]
b663b9
-                data[attr][len(data[attr]) - 1] = prev + line.strip()
b663b9
+                # continuation line (wrapped value)
b663b9
+                val += line[1:]
b663b9
+                found_part_val = True
b663b9
                 continue
b663b9
+            elif found_part_val:
b663b9
+                # We have the complete value now (it was wrapped)
b663b9
+                found_part_val = False
b663b9
+                found_attr = False
b663b9
+                add_attr_entry(data, val, attr, state_attr)
b663b9
+
b663b9
+                # Now that the value is added to the entry lets process the new attribute...
b663b9
+                value_set = line.split(":", 1)
b663b9
+                attr, state_attr = remove_attr_state_info(value_set[0])
b663b9
+
b663b9
+                if attr in ignore_list or (attr is None and state_attr is None):
b663b9
+                    # Skip it
b663b9
+                    found_attr = False
b663b9
+                    attr = None
b663b9
+                    continue
b663b9
 
b663b9
-            value_set = line.split(":", 1)
b663b9
-            attr = value_set[0].lower()
b663b9
-            if attr.startswith('nsds5replconflict'):
b663b9
-                found_conflict = True
b663b9
-            if attr.startswith('nstombstonecsn'):
b663b9
-                result['tombstone'] = True
b663b9
-
b663b9
-            if attr in data:
b663b9
-                data[attr].append(value_set[1].strip())
b663b9
+                val = value_set[1].strip()
b663b9
+                found_attr = True
b663b9
+
b663b9
+                if attr is not None:
b663b9
+                    # Set the entry type flags
b663b9
+                    if attr.startswith('nsds5replconflict'):
b663b9
+                        found_conflict = True
b663b9
+                    if attr.startswith("objectclass") and val == "ldapsubentry":
b663b9
+                        found_subentry = True
b663b9
+                    if attr.startswith('nstombstonecsn'):
b663b9
+                        result['tombstone'] = True
b663b9
+                        found_tombstone = True
b663b9
+                continue
b663b9
             else:
b663b9
-                data[attr] = [value_set[1].strip()]
b663b9
+                # New attribute...
b663b9
+                if found_attr:
b663b9
+                    # But first we have to add the previous complete attr value to the entry data
b663b9
+                    add_attr_entry(data, val, attr, state_attr)
b663b9
+
b663b9
+                # Process new attribute
b663b9
+                value_set = line.split(":", 1)
b663b9
+                attr, state_attr = remove_attr_state_info(value_set[0])
b663b9
+                if attr is None or attr in ignore_list:
b663b9
+                    # Skip it (its deleted)
b663b9
+                    found_attr = False
b663b9
+                    attr = None
b663b9
+                    continue
b663b9
+
b663b9
+                val = value_set[1].strip()
b663b9
+                found_attr = True
b663b9
+
b663b9
+                # Set the entry type flags
b663b9
+                if attr.startswith('nsds5replconflict'):
b663b9
+                    found_conflict = True
b663b9
+                if attr.startswith("objectclass") and (val == "ldapsubentry" or val == "glue"):
b663b9
+                    if val == "glue":
b663b9
+                        found_glue = True
b663b9
+                    found_subentry = True
b663b9
+                if attr.startswith('nstombstonecsn'):
b663b9
+                    result['tombstone'] = True
b663b9
+                    found_tombstone = True
b663b9
+                continue
b663b9
+
b663b9
         elif found_part_dn:
b663b9
             if line[0] == ' ':
b663b9
+                # DN is still wrapping, keep building up the dn value
b663b9
                 part_dn += line[1:].lower()
b663b9
             else:
b663b9
-                # We have the full dn
b663b9
+                # We now have the full dn
b663b9
                 found_part_dn = False
b663b9
-                reset_line = True
b663b9
-                prev_line = line
b663b9
                 if part_dn == dn:
b663b9
+                    # We found our entry
b663b9
                     found = True
b663b9
+
b663b9
+                    # But now we have a new attribute to process
b663b9
+                    value_set = line.split(":", 1)
b663b9
+                    attr, state_attr = remove_attr_state_info(value_set[0])
b663b9
+                    if attr is None or attr in ignore_list:
b663b9
+                        # Skip it (its deleted)
b663b9
+                        found_attr = False
b663b9
+                        attr = None
b663b9
+                        continue
b663b9
+
b663b9
+                    val = value_set[1].strip()
b663b9
+                    found_attr = True
b663b9
+
b663b9
+                    if attr.startswith('nsds5replconflict'):
b663b9
+                        found_conflict = True
b663b9
+                    if attr.startswith("objectclass") and val == "ldapsubentry":
b663b9
+                        found_subentry = True
b663b9
+
b663b9
+                    if attr.startswith('nstombstonecsn'):
b663b9
+                        result['tombstone'] = True
b663b9
+                        found_tombstone = True
b663b9
                     continue
b663b9
+
b663b9
         if line.startswith('dn: '):
b663b9
             if line[4:].lower() == dn:
b663b9
+                # We got our full DN, now process the entry
b663b9
                 found = True
b663b9
                 continue
b663b9
             else:
b663b9
+                # DN wraps the line, keep looping until we get the whole value
b663b9
                 part_dn = line[4:].lower()
b663b9
                 found_part_dn = True
b663b9
 
b663b9
+    # Keep track of entry index - we use this later when searching the LDIF again
b663b9
     result['idx'] = count
b663b9
-    if found_conflict:
b663b9
+
b663b9
+    result['glue'] = None
b663b9
+    if found_conflict and found_subentry and found_tombstone is False:
b663b9
         result['entry'] = None
b663b9
         result['conflict'] = Entry([dn, data])
b663b9
+        if found_glue:
b663b9
+            result['glue'] = result['conflict']
b663b9
     elif found:
b663b9
         result['conflict'] = None
b663b9
         result['entry'] = Entry([dn, data])
b663b9
@@ -251,7 +387,7 @@ def ldif_search(LDIF, dn, conflicts=False):
b663b9
 
b663b9
 
b663b9
 def get_dns(LDIF, opts):
b663b9
-    ''' Get all the DN's
b663b9
+    ''' Get all the DN's from an LDIF file
b663b9
     '''
b663b9
     dns = []
b663b9
     found = False
b663b9
@@ -275,7 +411,7 @@ def get_dns(LDIF, opts):
b663b9
 
b663b9
 
b663b9
 def get_ldif_ruv(LDIF, opts):
b663b9
-    ''' Search the ldif and get the ruv entry
b663b9
+    ''' Search the LDIF and get the ruv entry
b663b9
     '''
b663b9
     LDIF.seek(0)
b663b9
     result = ldif_search(LDIF, opts['ruv_dn'])
b663b9
@@ -283,7 +419,7 @@ def get_ldif_ruv(LDIF, opts):
b663b9
 
b663b9
 
b663b9
 def cmp_entry(mentry, rentry, opts):
b663b9
-    ''' Compare the two entries, and return a diff map
b663b9
+    ''' Compare the two entries, and return a "diff map"
b663b9
     '''
b663b9
     diff = {}
b663b9
     diff['dn'] = mentry['dn']
b663b9
@@ -307,6 +443,7 @@ def cmp_entry(mentry, rentry, opts):
b663b9
                 diff['missing'].append(" - Replica missing attribute: \"%s\"" % (mattr))
b663b9
                 diff_count += 1
b663b9
                 if 'nscpentrywsi' in mentry.data:
b663b9
+                    # Great we have state info so we can provide details about the missing attribute
b663b9
                     found = False
b663b9
                     for val in mentry.data['nscpentrywsi']:
b663b9
                         if val.lower().startswith(mattr + ';'):
b663b9
@@ -316,6 +453,7 @@ def cmp_entry(mentry, rentry, opts):
b663b9
                             diff['missing'].append(" - Master's State Info: %s" % (val))
b663b9
                             diff['missing'].append(" - Date: %s\n" % (time.ctime(extract_time(val))))
b663b9
                 else:
b663b9
+                    # No state info, just move on
b663b9
                     diff['missing'].append("")
b663b9
 
b663b9
         elif mentry.data[mattr] != rentry.data[mattr]:
b663b9
@@ -335,6 +473,9 @@ def cmp_entry(mentry, rentry, opts):
b663b9
                     if not found:
b663b9
                         diff['diff'].append("      Master: ")
b663b9
                         for val in mentry.data[mattr]:
b663b9
+                            # This is an "origin" value which means it's never been
b663b9
+                            # updated since replication was set up.  So its the
b663b9
+                            # original value
b663b9
                             diff['diff'].append("        - Origin value: %s" % (val))
b663b9
                         diff['diff'].append("")
b663b9
 
b663b9
@@ -350,10 +491,13 @@ def cmp_entry(mentry, rentry, opts):
b663b9
                     if not found:
b663b9
                         diff['diff'].append("      Replica: ")
b663b9
                         for val in rentry.data[mattr]:
b663b9
+                            # This is an "origin" value which means it's never been
b663b9
+                            # updated since replication was set up.  So its the
b663b9
+                            # original value
b663b9
                             diff['diff'].append("        - Origin value: %s" % (val))
b663b9
                         diff['diff'].append("")
b663b9
                 else:
b663b9
-                    # no state info
b663b9
+                    # no state info, report what we got
b663b9
                     diff['diff'].append("      Master: ")
b663b9
                     for val in mentry.data[mattr]:
b663b9
                         diff['diff'].append("        - %s: %s" % (mattr, val))
b663b9
@@ -436,40 +580,62 @@ def do_offline_report(opts, output_file=None):
b663b9
     MLDIF.seek(idx)
b663b9
     RLDIF.seek(idx)
b663b9
 
b663b9
-    # Compare the master entries with the replica's
b663b9
+    """ Compare the master entries with the replica's.  Take our list of dn's from
b663b9
+    the master ldif and get that entry( dn) from the master and replica ldif.  In
b663b9
+    this phase we keep keep track of conflict/tombstone counts, and we check for
b663b9
+    missing entries and entry differences.   We only need to do the entry diff
b663b9
+    checking in this phase - we do not need to do it when process the replica dn's
b663b9
+    because if the entry exists in both LDIF's then we already checked or diffs
b663b9
+    while processing the master dn's.
b663b9
+    """
b663b9
     print ("Comparing Master to Replica...")
b663b9
     missing = False
b663b9
     for dn in master_dns:
b663b9
-        mresult = ldif_search(MLDIF, dn, True)
b663b9
-        rresult = ldif_search(RLDIF, dn, True)
b663b9
+        mresult = ldif_search(MLDIF, dn)
b663b9
+        rresult = ldif_search(RLDIF, dn)
b663b9
+
b663b9
+        if dn in replica_dns:
b663b9
+            if (rresult['entry'] is not None or rresult['glue'] is not None or
b663b9
+                rresult['conflict'] is not None or rresult['tombstone']):
b663b9
+                """ We can safely remove this DN from the replica dn list as it
b663b9
+                does not need to be checked again.  This also speeds things up
b663b9
+                when doing the replica vs master phase.
b663b9
+                """
b663b9
+                replica_dns.remove(dn)
b663b9
 
b663b9
         if mresult['tombstone']:
b663b9
             mtombstones += 1
b663b9
+            # continue
b663b9
+        if rresult['tombstone']:
b663b9
+            rtombstones += 1
b663b9
 
b663b9
         if mresult['conflict'] is not None or rresult['conflict'] is not None:
b663b9
+            # If either entry is a conflict we still process it here
b663b9
             if mresult['conflict'] is not None:
b663b9
                 mconflicts.append(mresult['conflict'])
b663b9
+            if rresult['conflict'] is not None:
b663b9
+                rconflicts.append(rresult['conflict'])
b663b9
         elif rresult['entry'] is None:
b663b9
-            # missing entry - restart the search from beginning
b663b9
+            # missing entry - restart the search from beginning in case it got skipped
b663b9
             RLDIF.seek(0)
b663b9
             rresult = ldif_search(RLDIF, dn)
b663b9
-            if rresult['entry'] is None:
b663b9
-                # missing entry in rentries
b663b9
-                RLDIF.seek(mresult['idx'])  # Set the cursor to the last good line
b663b9
+            if rresult['entry'] is None and rresult['glue'] is None:
b663b9
+                # missing entry in Replica(rentries)
b663b9
+                RLDIF.seek(mresult['idx'])  # Set the LDIF cursor/index to the last good line
b663b9
                 if not missing:
b663b9
-                    missing_report += ('Replica is missing entries:\n')
b663b9
+                    missing_report += ('  Entries missing on Replica:\n')
b663b9
                     missing = True
b663b9
                 if mresult['entry'] and 'createtimestamp' in mresult['entry'].data:
b663b9
-                    missing_report += ('  - %s  (Master\'s creation date:  %s)\n' %
b663b9
+                    missing_report += ('   - %s  (Created on Master at: %s)\n' %
b663b9
                                        (dn, convert_timestamp(mresult['entry'].data['createtimestamp'][0])))
b663b9
                 else:
b663b9
                     missing_report += ('  - %s\n' % dn)
b663b9
-            else:
b663b9
+            elif mresult['tombstone'] is False:
b663b9
                 # Compare the entries
b663b9
                 diff = cmp_entry(mresult['entry'], rresult['entry'], opts)
b663b9
                 if diff:
b663b9
                     diff_report.append(format_diff(diff))
b663b9
-        else:
b663b9
+        elif mresult['tombstone'] is False:
b663b9
             # Compare the entries
b663b9
             diff = cmp_entry(mresult['entry'], rresult['entry'], opts)
b663b9
             if diff:
b663b9
@@ -478,7 +644,10 @@ def do_offline_report(opts, output_file=None):
b663b9
     if missing:
b663b9
         missing_report += ('\n')
b663b9
 
b663b9
-    # Search Replica, and look for missing entries only.  Count entries as well
b663b9
+    """ Search Replica, and look for missing entries only.  We already did the
b663b9
+    diff checking, so its only missing entries we are worried about. Count the
b663b9
+    remaining conflict & tombstone entries as well.
b663b9
+    """
b663b9
     print ("Comparing Replica to Master...")
b663b9
     MLDIF.seek(0)
b663b9
     RLDIF.seek(0)
b663b9
@@ -486,26 +655,26 @@ def do_offline_report(opts, output_file=None):
b663b9
     for dn in replica_dns:
b663b9
         rresult = ldif_search(RLDIF, dn)
b663b9
         mresult = ldif_search(MLDIF, dn)
b663b9
-
b663b9
         if rresult['tombstone']:
b663b9
             rtombstones += 1
b663b9
-        if mresult['entry'] is not None or rresult['conflict'] is not None:
b663b9
-            if rresult['conflict'] is not None:
b663b9
-                rconflicts.append(rresult['conflict'])
b663b9
+            # continue
b663b9
+
b663b9
+        if rresult['conflict'] is not None:
b663b9
+            rconflicts.append(rresult['conflict'])
b663b9
         elif mresult['entry'] is None:
b663b9
             # missing entry
b663b9
             MLDIF.seek(0)
b663b9
             mresult = ldif_search(MLDIF, dn)
b663b9
-            if mresult['entry'] is None and mresult['conflict'] is not None:
b663b9
-                MLDIF.seek(rresult['idx'])  # Set the cursor to the last good line
b663b9
+            if mresult['entry'] is None and mresult['glue'] is None:
b663b9
+                MLDIF.seek(rresult['idx'])  # Set the LDIF cursor/index to the last good line
b663b9
                 if not missing:
b663b9
-                    missing_report += ('Master is missing entries:\n')
b663b9
+                    missing_report += ('  Entries missing on Master:\n')
b663b9
                     missing = True
b663b9
-                if 'createtimestamp' in rresult['entry'].data:
b663b9
-                    missing_report += ('  - %s  (Replica\'s creation date:  %s)\n' %
b663b9
+                if rresult['entry'] and 'createtimestamp' in rresult['entry'].data:
b663b9
+                    missing_report += ('   - %s  (Created on Replica at: %s)\n' %
b663b9
                                        (dn, convert_timestamp(rresult['entry'].data['createtimestamp'][0])))
b663b9
                 else:
b663b9
-                    missing_report += ('  - %s\n')
b663b9
+                    missing_report += ('  - %s\n' % dn)
b663b9
     if missing:
b663b9
         missing_report += ('\n')
b663b9
 
b663b9
@@ -553,8 +722,8 @@ def do_offline_report(opts, output_file=None):
b663b9
         print(final_report)
b663b9
 
b663b9
 
b663b9
-def check_for_diffs(mentries, rentries, report, opts):
b663b9
-    ''' Check for diffs, return the updated report
b663b9
+def check_for_diffs(mentries, mglue, rentries, rglue, report, opts):
b663b9
+    ''' Online mode only - Check for diffs, return the updated report
b663b9
     '''
b663b9
     diff_report = []
b663b9
     m_missing = []
b663b9
@@ -569,18 +738,26 @@ def check_for_diffs(mentries, rentries, report, opts):
b663b9
     for mentry in mentries:
b663b9
         rentry = get_entry(rentries, mentry.dn)
b663b9
         if rentry:
b663b9
-            diff = cmp_entry(mentry, rentry, opts)
b663b9
-            if diff:
b663b9
-                diff_report.append(format_diff(diff))
b663b9
+            if 'nsTombstone' not in rentry.data['objectclass'] and 'nstombstone' not in rentry.data['objectclass']:
b663b9
+                diff = cmp_entry(mentry, rentry, opts)
b663b9
+                if diff:
b663b9
+                    diff_report.append(format_diff(diff))
b663b9
             # Now remove the rentry from the rentries so we can find stragglers
b663b9
             remove_entry(rentries, rentry.dn)
b663b9
         else:
b663b9
-            # Add missing entry in Replica
b663b9
-            r_missing.append(mentry)
b663b9
+            rentry = get_entry(rglue, mentry.dn)
b663b9
+            if rentry:
b663b9
+                # Glue entry nothing to compare
b663b9
+                remove_entry(rentries, rentry.dn)
b663b9
+            else:
b663b9
+                # Add missing entry in Replica
b663b9
+                r_missing.append(mentry)
b663b9
 
b663b9
     for rentry in rentries:
b663b9
         # We should not have any entries if we are sync
b663b9
-        m_missing.append(rentry)
b663b9
+        mentry = get_entry(mglue, rentry.dn)
b663b9
+        if mentry is None:
b663b9
+            m_missing.append(rentry)
b663b9
 
b663b9
     if len(diff_report) > 0:
b663b9
         report['diff'] += diff_report
b663b9
@@ -609,6 +786,12 @@ def connect_to_replicas(opts):
b663b9
         ruri = "%s://%s:%s/" % (opts['rprotocol'], opts['rhost'], opts['rport'])
b663b9
     replica = SimpleLDAPObject(ruri)
b663b9
 
b663b9
+    # Set timeouts
b663b9
+    master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
b663b9
+    master.set_option(ldap.OPT_TIMEOUT,5.0)
b663b9
+    replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
b663b9
+    replica.set_option(ldap.OPT_TIMEOUT,5.0)
b663b9
+
b663b9
     # Setup Secure Conenction
b663b9
     if opts['certdir'] is not None:
b663b9
         # Setup Master
b663b9
@@ -620,7 +803,7 @@ def connect_to_replicas(opts):
b663b9
                 try:
b663b9
                     master.start_tls_s()
b663b9
                 except ldap.LDAPError as e:
b663b9
-                    print('TLS negotiation failed on Master: %s' % str(e))
b663b9
+                    print('TLS negotiation failed on Master: {}'.format(str(e)))
b663b9
                     exit(1)
b663b9
 
b663b9
         # Setup Replica
b663b9
@@ -632,7 +815,7 @@ def connect_to_replicas(opts):
b663b9
                 try:
b663b9
                     replica.start_tls_s()
b663b9
                 except ldap.LDAPError as e:
b663b9
-                    print('TLS negotiation failed on Master: %s' % str(e))
b663b9
+                    print('TLS negotiation failed on Master: {}'.format(str(e)))
b663b9
                     exit(1)
b663b9
 
b663b9
     # Open connection to master
b663b9
@@ -642,7 +825,8 @@ def connect_to_replicas(opts):
b663b9
         print("Cannot connect to %r" % muri)
b663b9
         exit(1)
b663b9
     except ldap.LDAPError as e:
b663b9
-        print("Error: Failed to authenticate to Master: %s", str(e))
b663b9
+        print("Error: Failed to authenticate to Master: ({}).  "
b663b9
+              "Please check your credentials and LDAP urls are correct.".format(str(e)))
b663b9
         exit(1)
b663b9
 
b663b9
     # Open connection to replica
b663b9
@@ -652,7 +836,8 @@ def connect_to_replicas(opts):
b663b9
         print("Cannot connect to %r" % ruri)
b663b9
         exit(1)
b663b9
     except ldap.LDAPError as e:
b663b9
-        print("Error: Failed to authenticate to Replica: %s", str(e))
b663b9
+        print("Error: Failed to authenticate to Replica: ({}).  "
b663b9
+              "Please check your credentials and LDAP urls are correct.".format(str(e)))
b663b9
         exit(1)
b663b9
 
b663b9
     # Get the RUVs
b663b9
@@ -665,7 +850,7 @@ def connect_to_replicas(opts):
b663b9
             print("Error: Master does not have an RUV entry")
b663b9
             exit(1)
b663b9
     except ldap.LDAPError as e:
b663b9
-        print("Error: Failed to get Master RUV entry: %s", str(e))
b663b9
+        print("Error: Failed to get Master RUV entry: {}".format(str(e)))
b663b9
         exit(1)
b663b9
 
b663b9
     print ("Gathering Replica's RUV...")
b663b9
@@ -678,7 +863,7 @@ def connect_to_replicas(opts):
b663b9
             exit(1)
b663b9
 
b663b9
     except ldap.LDAPError as e:
b663b9
-        print("Error: Failed to get Replica RUV entry: %s", str(e))
b663b9
+        print("Error: Failed to get Replica RUV entry: {}".format(str(e)))
b663b9
         exit(1)
b663b9
 
b663b9
     return (master, replica, opts)
b663b9
@@ -687,6 +872,7 @@ def connect_to_replicas(opts):
b663b9
 def print_online_report(report, opts, output_file):
b663b9
     ''' Print the online report
b663b9
     '''
b663b9
+
b663b9
     print ('Preparing final report...')
b663b9
     m_missing = len(report['m_missing'])
b663b9
     r_missing = len(report['r_missing'])
b663b9
@@ -711,22 +897,23 @@ def print_online_report(report, opts, output_file):
b663b9
         missing = True
b663b9
         final_report += ('\nMissing Entries\n')
b663b9
         final_report += ('=====================================================\n\n')
b663b9
-        if m_missing > 0:
b663b9
-            final_report += ('  Entries missing on Master:\n')
b663b9
-            for entry in report['m_missing']:
b663b9
+
b663b9
+        if r_missing > 0:
b663b9
+            final_report += ('  Entries missing on Replica:\n')
b663b9
+            for entry in report['r_missing']:
b663b9
                 if 'createtimestamp' in entry.data:
b663b9
-                    final_report += ('   - %s  (Created on Replica at: %s)\n' %
b663b9
+                    final_report += ('   - %s  (Created on Master at: %s)\n' %
b663b9
                                      (entry.dn, convert_timestamp(entry.data['createtimestamp'][0])))
b663b9
                 else:
b663b9
                     final_report += ('   - %s\n' % (entry.dn))
b663b9
 
b663b9
-        if r_missing > 0:
b663b9
-            if m_missing > 0:
b663b9
+        if m_missing > 0:
b663b9
+            if r_missing > 0:
b663b9
                 final_report += ('\n')
b663b9
-            final_report += ('  Entries missing on Replica:\n')
b663b9
-            for entry in report['r_missing']:
b663b9
+            final_report += ('  Entries missing on Master:\n')
b663b9
+            for entry in report['m_missing']:
b663b9
                 if 'createtimestamp' in entry.data:
b663b9
-                    final_report += ('   - %s  (Created on Master at: %s)\n' %
b663b9
+                    final_report += ('   - %s  (Created on Replica at: %s)\n' %
b663b9
                                      (entry.dn, convert_timestamp(entry.data['createtimestamp'][0])))
b663b9
                 else:
b663b9
                     final_report += ('   - %s\n' % (entry.dn))
b663b9
@@ -751,7 +938,8 @@ def print_online_report(report, opts, output_file):
b663b9
 def remove_state_info(entry):
b663b9
     ''' Remove the state info for the attributes used in the conflict report
b663b9
     '''
b663b9
-    attrs = ['objectclass', 'nsds5replconflict', 'createtimestamp']
b663b9
+    attrs = ['objectclass', 'nsds5replconflict', 'createtimestamp' , 'modifytimestamp']
b663b9
+    # attrs = ['createtimestamp']
b663b9
     for key, val in list(entry.data.items()):
b663b9
         for attr in attrs:
b663b9
             if key.lower().startswith(attr):
b663b9
@@ -766,9 +954,6 @@ def get_conflict_report(mentries, rentries, verbose, format_conflicts=False):
b663b9
     r_conflicts = []
b663b9
 
b663b9
     for entry in mentries:
b663b9
-        if format_conflicts:
b663b9
-            remove_state_info(entry)
b663b9
-
b663b9
         if 'glue' in entry.data['objectclass']:
b663b9
             m_conflicts.append({'dn': entry.dn, 'conflict': entry.data['nsds5replconflict'][0],
b663b9
                                 'date': entry.data['createtimestamp'][0], 'glue': 'yes'})
b663b9
@@ -776,9 +961,6 @@ def get_conflict_report(mentries, rentries, verbose, format_conflicts=False):
b663b9
             m_conflicts.append({'dn': entry.dn, 'conflict': entry.data['nsds5replconflict'][0],
b663b9
                                 'date': entry.data['createtimestamp'][0], 'glue': 'no'})
b663b9
     for entry in rentries:
b663b9
-        if format_conflicts:
b663b9
-            remove_state_info(entry)
b663b9
-
b663b9
         if 'glue' in entry.data['objectclass']:
b663b9
             r_conflicts.append({'dn': entry.dn, 'conflict': entry.data['nsds5replconflict'][0],
b663b9
                                 'date': entry.data['createtimestamp'][0], 'glue': 'yes'})
b663b9
@@ -790,7 +972,7 @@ def get_conflict_report(mentries, rentries, verbose, format_conflicts=False):
b663b9
         report = "\n\nConflict Entries\n"
b663b9
         report += "=====================================================\n\n"
b663b9
         if len(m_conflicts) > 0:
b663b9
-            report += ('Master Conflict Entries: %d\n' % (len(m_conflicts)))
b663b9
+            report += ('Master Conflict Entries:  %d\n' % (len(m_conflicts)))
b663b9
             if verbose:
b663b9
                 for entry in m_conflicts:
b663b9
                     report += ('\n - %s\n' % (entry['dn']))
b663b9
@@ -799,7 +981,7 @@ def get_conflict_report(mentries, rentries, verbose, format_conflicts=False):
b663b9
                     report += ('    - Created:    %s\n' % (convert_timestamp(entry['date'])))
b663b9
 
b663b9
         if len(r_conflicts) > 0:
b663b9
-            if len(m_conflicts) > 0:
b663b9
+            if len(m_conflicts) > 0 and verbose:
b663b9
                 report += "\n"  # add spacer
b663b9
             report += ('Replica Conflict Entries: %d\n' % (len(r_conflicts)))
b663b9
             if verbose:
b663b9
@@ -814,46 +996,6 @@ def get_conflict_report(mentries, rentries, verbose, format_conflicts=False):
b663b9
         return ""
b663b9
 
b663b9
 
b663b9
-def get_tombstones(replica, opts):
b663b9
-    ''' Return the number of tombstones
b663b9
-    '''
b663b9
-    paged_ctrl = SimplePagedResultsControl(True, size=opts['pagesize'], cookie='')
b663b9
-    controls = [paged_ctrl]
b663b9
-    req_pr_ctrl = controls[0]
b663b9
-    count = 0
b663b9
-
b663b9
-    try:
b663b9
-        msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
-                                   '(&(objectclass=nstombstone)(nstombstonecsn=*))',
b663b9
-                                   ['dn'], serverctrls=controls)
b663b9
-    except ldap.LDAPError as e:
b663b9
-        print("Error: Failed to get tombstone entries: %s", str(e))
b663b9
-        exit(1)
b663b9
-
b663b9
-    done = False
b663b9
-    while not done:
b663b9
-        rtype, rdata, rmsgid, rctrls = replica.result3(msgid)
b663b9
-        count += len(rdata)
b663b9
-
b663b9
-        pctrls = [
b663b9
-                c
b663b9
-                for c in rctrls
b663b9
-                if c.controlType == SimplePagedResultsControl.controlType
b663b9
-                ]
b663b9
-        if pctrls:
b663b9
-            if pctrls[0].cookie:
b663b9
-                # Copy cookie from response control to request control
b663b9
-                req_pr_ctrl.cookie = pctrls[0].cookie
b663b9
-                msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
-                                           '(&(objectclass=nstombstone)(nstombstonecsn=*))',
b663b9
-                                           ['dn'], serverctrls=controls)
b663b9
-            else:
b663b9
-                done = True  # No more pages available
b663b9
-        else:
b663b9
-            done = True
b663b9
-    return count
b663b9
-
b663b9
-
b663b9
 def do_online_report(opts, output_file=None):
b663b9
     ''' Check for differences between two replicas
b663b9
     '''
b663b9
@@ -880,7 +1022,7 @@ def do_online_report(opts, output_file=None):
b663b9
     req_pr_ctrl = controls[0]
b663b9
     try:
b663b9
         master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
-                                         "(|(objectclass=*)(objectclass=ldapsubentry))",
b663b9
+                                         "(|(objectclass=*)(objectclass=ldapsubentry)(objectclass=nstombstone))",
b663b9
                                          ['*', 'createtimestamp', 'nscpentrywsi', 'nsds5replconflict'],
b663b9
                                          serverctrls=controls)
b663b9
     except ldap.LDAPError as e:
b663b9
@@ -888,7 +1030,7 @@ def do_online_report(opts, output_file=None):
b663b9
         exit(1)
b663b9
     try:
b663b9
         replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
-                                           "(|(objectclass=*)(objectclass=ldapsubentry))",
b663b9
+                                           "(|(objectclass=*)(objectclass=ldapsubentry)(objectclass=nstombstone))",
b663b9
                                            ['*', 'createtimestamp', 'nscpentrywsi', 'nsds5replconflict'],
b663b9
                                            serverctrls=controls)
b663b9
     except ldap.LDAPError as e:
b663b9
@@ -918,7 +1060,9 @@ def do_online_report(opts, output_file=None):
b663b9
         rconflicts += rresult['conflicts']
b663b9
 
b663b9
         # Check for diffs
b663b9
-        report = check_for_diffs(mresult['entries'], rresult['entries'], report, opts)
b663b9
+        report = check_for_diffs(mresult['entries'], mresult['glue'],
b663b9
+                                 rresult['entries'], rresult['glue'],
b663b9
+                                 report, opts)
b663b9
 
b663b9
         if not m_done:
b663b9
             # Master
b663b9
@@ -933,7 +1077,7 @@ def do_online_report(opts, output_file=None):
b663b9
                     req_pr_ctrl.cookie = m_pctrls[0].cookie
b663b9
                     master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
                         "(|(objectclass=*)(objectclass=ldapsubentry))",
b663b9
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'nsds5replconflict'], serverctrls=controls)
b663b9
+                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
b663b9
                 else:
b663b9
                     m_done = True  # No more pages available
b663b9
             else:
b663b9
@@ -953,7 +1097,7 @@ def do_online_report(opts, output_file=None):
b663b9
                     req_pr_ctrl.cookie = r_pctrls[0].cookie
b663b9
                     replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
b663b9
                         "(|(objectclass=*)(objectclass=ldapsubentry))",
b663b9
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'nsds5replconflict'], serverctrls=controls)
b663b9
+                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
b663b9
                 else:
b663b9
                     r_done = True  # No more pages available
b663b9
             else:
b663b9
@@ -961,10 +1105,8 @@ def do_online_report(opts, output_file=None):
b663b9
 
b663b9
     # Get conflicts & tombstones
b663b9
     report['conflict'] = get_conflict_report(mconflicts, rconflicts, opts['conflicts'])
b663b9
-    report['mtombstones'] = get_tombstones(master, opts)
b663b9
-    report['rtombstones'] = get_tombstones(replica, opts)
b663b9
-    report['m_count'] += report['mtombstones']
b663b9
-    report['r_count'] += report['rtombstones']
b663b9
+    report['mtombstones'] = mresult['tombstones']
b663b9
+    report['rtombstones'] = rresult['tombstones']
b663b9
 
b663b9
     # Do the final report
b663b9
     print_online_report(report, opts, output_file)
b663b9
@@ -1027,11 +1169,16 @@ def main():
b663b9
 
b663b9
     # Parse the ldap URLs
b663b9
     if args.murl is not None and args.rurl is not None:
b663b9
+        # Make sure the URLs are different
b663b9
+        if args.murl == args.rurl:
b663b9
+            print("Master and Replica LDAP URLs are the same, they must be different")
b663b9
+            exit(1)
b663b9
+
b663b9
         # Parse Master url
b663b9
-        murl = ldapurl.LDAPUrl(args.murl)
b663b9
         if not ldapurl.isLDAPUrl(args.murl):
b663b9
             print("Master LDAP URL is invalid")
b663b9
             exit(1)
b663b9
+        murl = ldapurl.LDAPUrl(args.murl)
b663b9
         if murl.urlscheme in VALID_PROTOCOLS:
b663b9
             opts['mprotocol'] = murl.urlscheme
b663b9
         else:
b663b9
@@ -1052,10 +1199,10 @@ def main():
b663b9
             opts['mport'] = parts[1]
b663b9
 
b663b9
         # Parse Replica url
b663b9
-        rurl = ldapurl.LDAPUrl(args.rurl)
b663b9
         if not ldapurl.isLDAPUrl(args.rurl):
b663b9
             print("Replica LDAP URL is invalid")
b663b9
             exit(1)
b663b9
+        rurl = ldapurl.LDAPUrl(args.rurl)
b663b9
         if rurl.urlscheme in VALID_PROTOCOLS:
b663b9
             opts['rprotocol'] = rurl.urlscheme
b663b9
         else:
b663b9
@@ -1075,11 +1222,19 @@ def main():
b663b9
             opts['rhost'] = parts[0]
b663b9
             opts['rport'] = parts[1]
b663b9
 
b663b9
+    # Validate certdir
b663b9
+    opts['certdir'] = None
b663b9
+    if args.certdir:
b663b9
+        if os.path.exists() and os.path.isdir(certdir):
b663b9
+            opts['certdir'] = args.certdir
b663b9
+        else:
b663b9
+            print("certificate directory ({}) does not exist or is not a directory".format(args.certdir))
b663b9
+            exit(1)
b663b9
+
b663b9
     # Initialize the options
b663b9
     opts['binddn'] = args.binddn
b663b9
     opts['bindpw'] = args.bindpw
b663b9
     opts['suffix'] = args.suffix
b663b9
-    opts['certdir'] = args.certdir
b663b9
     opts['starttime'] = int(time.time())
b663b9
     opts['verbose'] = args.verbose
b663b9
     opts['mldif'] = args.mldif
b663b9
@@ -1109,6 +1264,18 @@ def main():
b663b9
 
b663b9
     if opts['mldif'] is not None and opts['rldif'] is not None:
b663b9
         print ("Performing offline report...")
b663b9
+
b663b9
+        # Validate LDIF files, must exist and not be empty
b663b9
+        for ldif_dir in [opts['mldif'], opts['rldif']]:
b663b9
+            if not os.path.exists(ldif_dir):
b663b9
+                print ("LDIF file ({}) does not exist".format(ldif_dir))
b663b9
+                exit(1)
b663b9
+            if os.path.getsize(ldif_dir) == 0:
b663b9
+                print ("LDIF file ({}) is empty".format(ldif_dir))
b663b9
+                exit(1)
b663b9
+        if opts['mldif'] == opts['rldif']:
b663b9
+            print("The Master and Replica LDIF files must be different")
b663b9
+            exit(1)
b663b9
         do_offline_report(opts, OUTPUT_FILE)
b663b9
     else:
b663b9
         print ("Performing online report...")
b663b9
@@ -1118,5 +1285,6 @@ def main():
b663b9
         print('Finished writing report to "%s"' % (args.file))
b663b9
         OUTPUT_FILE.close()
b663b9
 
b663b9
+
b663b9
 if __name__ == '__main__':
b663b9
     main()
b663b9
-- 
b663b9
2.17.0
b663b9