From b161c9731acb0677712f9a0f34c9a314f7245795 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Nov 03 2015 07:24:03 +0000 Subject: import 389-ds-base-1.3.3.1-23.el7_1 --- diff --git a/SOURCES/0076-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch b/SOURCES/0076-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch new file mode 100644 index 0000000..cf4cac8 --- /dev/null +++ b/SOURCES/0076-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch @@ -0,0 +1,40 @@ +From b9771a9a3202b4d3a8562ed7359c824f8922b4fe Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Wed, 15 Oct 2014 16:20:51 -0700 +Subject: [PATCH 76/84] Ticket #47553 - Enhance ACIs to have more control over + MODRDN operations + +Description: Macro SLAPI_ACL_ALL does not contain SLAPI_ACL_MODDN. +Thus, even though all operations are allowed by "allow (all)", just +modrdn fails with "Insufficient access (50)". + +https://fedorahosted.org/389/ticket/47553 + +Reviewed by tbordaz@redhat.com (Thank you, Thierry!!) + +(cherry picked from commit 4aafe7444d983c08b16a84b7c23c8d303de45dc6) +--- + ldap/servers/slapd/slapi-plugin.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h +index dfe75eb..5a7af5e 100644 +--- a/ldap/servers/slapd/slapi-plugin.h ++++ b/ldap/servers/slapd/slapi-plugin.h +@@ -237,12 +237,12 @@ NSPR_API(PRUint32) PR_fprintf(struct PRFileDesc* fd, const char *fmt, ...) + #define SLAPI_ACL_ADD 0x20 + #define SLAPI_ACL_SELF 0x40 + #define SLAPI_ACL_PROXY 0x80 +-#define SLAPI_ACL_ALL 0x7f + /* Values 0x200 and 0x400 are booked (acl.h) by + * ACLPB_SLAPI_ACL_WRITE_ADD + * ACLPB_SLAPI_ACL_WRITE_DEL + */ + #define SLAPI_ACL_MODDN 0x0800 ++#define SLAPI_ACL_ALL 0x087f + + + /* +-- +1.9.3 + diff --git a/SOURCES/0077-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch b/SOURCES/0077-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch new file mode 100644 index 0000000..c56fbc8 --- /dev/null +++ b/SOURCES/0077-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch @@ -0,0 +1,676 @@ +From b19dc5b453751ca63545e39537bfb87707cf867e Mon Sep 17 00:00:00 2001 +From: "Thierry bordaz (tbordaz)" +Date: Thu, 16 Oct 2014 17:24:15 +0200 +Subject: [PATCH 77/84] Ticket 47553: Enhance ACIs to have more control over + MODRDN operations + +Bug Description: + The ticket 47553 introduces a new aci right: moddn. + This rights allows/deny to do a MODDN from a part of the DIT to an other. + Before the right allow/deny to do a MODDN was granted if the subject had 'write' access to the rdn attribute. + To switch from the previous mode to the new one, there is a toggle 'nsslapd-moddn-aci'. + The getEffectiveRight control, should report the MODDN right ('n') according to + the acis and the value of this toggle + +Fix Description: + test 'nsslapd-moddn-aci' in the geteffectiveright code + +https://fedorahosted.org/389/ticket/47553 + +Reviewed by: Noriko (Thanks !) + +Platforms tested: F17 + +Flag Day: no + +Doc impact: no + +(cherry picked from commit be67f8128d4a19edec95a6b1b022fd9262710e74) +--- + dirsrvtests/tickets/ticket47553_ger.py | 553 ++++++++++++++++++++++++++ + ldap/servers/plugins/acl/acleffectiverights.c | 67 ++-- + 2 files changed, 590 insertions(+), 30 deletions(-) + create mode 100644 dirsrvtests/tickets/ticket47553_ger.py + +diff --git a/dirsrvtests/tickets/ticket47553_ger.py b/dirsrvtests/tickets/ticket47553_ger.py +new file mode 100644 +index 0000000..d688c70 +--- /dev/null ++++ b/dirsrvtests/tickets/ticket47553_ger.py +@@ -0,0 +1,553 @@ ++''' ++Created on Nov 7, 2013 ++ ++@author: tbordaz ++''' ++import os ++import sys ++import time ++import ldap ++import logging ++import socket ++import time ++import logging ++import pytest ++import re ++from lib389 import DirSrv, Entry, tools ++from lib389.tools import DirSrvTools ++from lib389._constants import * ++from lib389.properties import * ++from constants import * ++from lib389._constants import REPLICAROLE_MASTER ++from ldap.controls.simple import GetEffectiveRightsControl ++ ++logging.getLogger(__name__).setLevel(logging.DEBUG) ++log = logging.getLogger(__name__) ++ ++# ++# important part. We can deploy Master1 and Master2 on different versions ++# ++installation1_prefix = None ++installation2_prefix = None ++ ++TEST_REPL_DN = "cn=test_repl, %s" % SUFFIX ++ ++STAGING_CN = "staged user" ++PRODUCTION_CN = "accounts" ++EXCEPT_CN = "excepts" ++ ++STAGING_DN = "cn=%s,%s" % (STAGING_CN, SUFFIX) ++PRODUCTION_DN = "cn=%s,%s" % (PRODUCTION_CN, SUFFIX) ++PROD_EXCEPT_DN = "cn=%s,%s" % (EXCEPT_CN, PRODUCTION_DN) ++ ++STAGING_PATTERN = "cn=%s*,%s" % (STAGING_CN[:2], SUFFIX) ++PRODUCTION_PATTERN = "cn=%s*,%s" % (PRODUCTION_CN[:2], SUFFIX) ++BAD_STAGING_PATTERN = "cn=bad*,%s" % (SUFFIX) ++BAD_PRODUCTION_PATTERN = "cn=bad*,%s" % (SUFFIX) ++ ++BIND_CN = "bind_entry" ++BIND_DN = "cn=%s,%s" % (BIND_CN, SUFFIX) ++BIND_PW = "password" ++ ++NEW_ACCOUNT = "new_account" ++MAX_ACCOUNTS = 20 ++ ++CONFIG_MODDN_ACI_ATTR = "nsslapd-moddn-aci" ++ ++class TopologyMaster1Master2(object): ++ def __init__(self, master1, master2): ++ master1.open() ++ self.master1 = master1 ++ ++ master2.open() ++ self.master2 = master2 ++ ++ ++@pytest.fixture(scope="module") ++def topology(request): ++ ''' ++ This fixture is used to create a replicated topology for the 'module'. ++ The replicated topology is MASTER1 <-> Master2. ++ At the beginning, It may exists a master2 instance and/or a master2 instance. ++ It may also exists a backup for the master1 and/or the master2. ++ ++ Principle: ++ If master1 instance exists: ++ restart it ++ If master2 instance exists: ++ restart it ++ If backup of master1 AND backup of master2 exists: ++ create or rebind to master1 ++ create or rebind to master2 ++ ++ restore master1 from backup ++ restore master2 from backup ++ else: ++ Cleanup everything ++ remove instances ++ remove backups ++ Create instances ++ Initialize replication ++ Create backups ++ ''' ++ global installation1_prefix ++ global installation2_prefix ++ ++ # allocate master1 on a given deployement ++ master1 = DirSrv(verbose=False) ++ if installation1_prefix: ++ args_instance[SER_DEPLOYED_DIR] = installation1_prefix ++ ++ # Args for the master1 instance ++ args_instance[SER_HOST] = HOST_MASTER_1 ++ args_instance[SER_PORT] = PORT_MASTER_1 ++ args_instance[SER_SERVERID_PROP] = SERVERID_MASTER_1 ++ args_master = args_instance.copy() ++ master1.allocate(args_master) ++ ++ # allocate master1 on a given deployement ++ master2 = DirSrv(verbose=False) ++ if installation2_prefix: ++ args_instance[SER_DEPLOYED_DIR] = installation2_prefix ++ ++ # Args for the consumer instance ++ args_instance[SER_HOST] = HOST_MASTER_2 ++ args_instance[SER_PORT] = PORT_MASTER_2 ++ args_instance[SER_SERVERID_PROP] = SERVERID_MASTER_2 ++ args_master = args_instance.copy() ++ master2.allocate(args_master) ++ ++ ++ # Get the status of the backups ++ backup_master1 = master1.checkBackupFS() ++ backup_master2 = master2.checkBackupFS() ++ ++ # Get the status of the instance and restart it if it exists ++ instance_master1 = master1.exists() ++ if instance_master1: ++ master1.stop(timeout=10) ++ master1.start(timeout=10) ++ ++ instance_master2 = master2.exists() ++ if instance_master2: ++ master2.stop(timeout=10) ++ master2.start(timeout=10) ++ ++ if backup_master1 and backup_master2: ++ # The backups exist, assuming they are correct ++ # we just re-init the instances with them ++ if not instance_master1: ++ master1.create() ++ # Used to retrieve configuration information (dbdir, confdir...) ++ master1.open() ++ ++ if not instance_master2: ++ master2.create() ++ # Used to retrieve configuration information (dbdir, confdir...) ++ master2.open() ++ ++ # restore master1 from backup ++ master1.stop(timeout=10) ++ master1.restoreFS(backup_master1) ++ master1.start(timeout=10) ++ ++ # restore master2 from backup ++ master2.stop(timeout=10) ++ master2.restoreFS(backup_master2) ++ master2.start(timeout=10) ++ else: ++ # We should be here only in two conditions ++ # - This is the first time a test involve master-consumer ++ # so we need to create everything ++ # - Something weird happened (instance/backup destroyed) ++ # so we discard everything and recreate all ++ ++ # Remove all the backups. So even if we have a specific backup file ++ # (e.g backup_master) we clear all backups that an instance my have created ++ if backup_master1: ++ master1.clearBackupFS() ++ if backup_master2: ++ master2.clearBackupFS() ++ ++ # Remove all the instances ++ if instance_master1: ++ master1.delete() ++ if instance_master2: ++ master2.delete() ++ ++ # Create the instances ++ master1.create() ++ master1.open() ++ master2.create() ++ master2.open() ++ ++ # ++ # Now prepare the Master-Consumer topology ++ # ++ # First Enable replication ++ master1.replica.enableReplication(suffix=SUFFIX, role=REPLICAROLE_MASTER, replicaId=REPLICAID_MASTER_1) ++ master2.replica.enableReplication(suffix=SUFFIX, role=REPLICAROLE_MASTER, replicaId=REPLICAID_MASTER_2) ++ ++ # Initialize the supplier->consumer ++ ++ properties = {RA_NAME: r'meTo_$host:$port', ++ RA_BINDDN: defaultProperties[REPLICATION_BIND_DN], ++ RA_BINDPW: defaultProperties[REPLICATION_BIND_PW], ++ RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD], ++ RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]} ++ repl_agreement = master1.agreement.create(suffix=SUFFIX, host=master2.host, port=master2.port, properties=properties) ++ ++ if not repl_agreement: ++ log.fatal("Fail to create a replica agreement") ++ sys.exit(1) ++ ++ log.debug("%s created" % repl_agreement) ++ ++ properties = {RA_NAME: r'meTo_$host:$port', ++ RA_BINDDN: defaultProperties[REPLICATION_BIND_DN], ++ RA_BINDPW: defaultProperties[REPLICATION_BIND_PW], ++ RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD], ++ RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]} ++ master2.agreement.create(suffix=SUFFIX, host=master1.host, port=master1.port, properties=properties) ++ ++ master1.agreement.init(SUFFIX, HOST_MASTER_2, PORT_MASTER_2) ++ master1.waitForReplInit(repl_agreement) ++ ++ # Check replication is working fine ++ master1.add_s(Entry((TEST_REPL_DN, { ++ 'objectclass': "top person".split(), ++ 'sn': 'test_repl', ++ 'cn': 'test_repl'}))) ++ loop = 0 ++ while loop <= 10: ++ try: ++ ent = master2.getEntry(TEST_REPL_DN, ldap.SCOPE_BASE, "(objectclass=*)") ++ break ++ except ldap.NO_SUCH_OBJECT: ++ time.sleep(1) ++ loop += 1 ++ ++ # Time to create the backups ++ master1.stop(timeout=10) ++ master1.backupfile = master1.backupFS() ++ master1.start(timeout=10) ++ ++ master2.stop(timeout=10) ++ master2.backupfile = master2.backupFS() ++ master2.start(timeout=10) ++ ++ # clear the tmp directory ++ master1.clearTmpDir(__file__) ++ ++ # ++ # Here we have two instances master and consumer ++ # with replication working. Either coming from a backup recovery ++ # or from a fresh (re)init ++ # Time to return the topology ++ return TopologyMaster1Master2(master1, master2) ++ ++ ++ ++def _bind_manager(topology): ++ topology.master1.log.info("Bind as %s " % DN_DM) ++ topology.master1.simple_bind_s(DN_DM, PASSWORD) ++ ++def _bind_normal(topology): ++ # bind as bind_entry ++ topology.master1.log.info("Bind as %s" % BIND_DN) ++ topology.master1.simple_bind_s(BIND_DN, BIND_PW) ++ ++def _moddn_aci_deny_tree(topology, mod_type=None, target_from=STAGING_DN, target_to=PROD_EXCEPT_DN): ++ ''' ++ It denies the access moddn_to in cn=except,cn=accounts,SUFFIX ++ ''' ++ assert mod_type != None ++ ++ ACI_TARGET_FROM = "" ++ ACI_TARGET_TO = "" ++ if target_from: ++ ACI_TARGET_FROM = "(target_from = \"ldap:///%s\")" % (target_from) ++ if target_to: ++ ACI_TARGET_TO = "(target_to = \"ldap:///%s\")" % (target_to) ++ ++ ACI_ALLOW = "(version 3.0; acl \"Deny MODDN to prod_except\"; deny (moddn)" ++ ACI_SUBJECT = " userdn = \"ldap:///%s\";)" % BIND_DN ++ ACI_BODY = ACI_TARGET_TO + ACI_TARGET_FROM + ACI_ALLOW + ACI_SUBJECT ++ mod = [(mod_type, 'aci', ACI_BODY)] ++ #topology.master1.modify_s(SUFFIX, mod) ++ topology.master1.log.info("Add a DENY aci under %s " % PROD_EXCEPT_DN) ++ topology.master1.modify_s(PROD_EXCEPT_DN, mod) ++ ++def _moddn_aci_staging_to_production(topology, mod_type=None, target_from=STAGING_DN, target_to=PRODUCTION_DN): ++ assert mod_type != None ++ ++ ++ ACI_TARGET_FROM = "" ++ ACI_TARGET_TO = "" ++ if target_from: ++ ACI_TARGET_FROM = "(target_from = \"ldap:///%s\")" % (target_from) ++ if target_to: ++ ACI_TARGET_TO = "(target_to = \"ldap:///%s\")" % (target_to) ++ ++ ACI_ALLOW = "(version 3.0; acl \"MODDN from staging to production\"; allow (moddn)" ++ ACI_SUBJECT = " userdn = \"ldap:///%s\";)" % BIND_DN ++ ACI_BODY = ACI_TARGET_FROM + ACI_TARGET_TO + ACI_ALLOW + ACI_SUBJECT ++ mod = [(mod_type, 'aci', ACI_BODY)] ++ topology.master1.modify_s(SUFFIX, mod) ++ ++def _moddn_aci_from_production_to_staging(topology, mod_type=None): ++ assert mod_type != None ++ ++ ACI_TARGET = "(target_from = \"ldap:///%s\") (target_to = \"ldap:///%s\")" % (PRODUCTION_DN, STAGING_DN) ++ ACI_ALLOW = "(version 3.0; acl \"MODDN from production to staging\"; allow (moddn)" ++ ACI_SUBJECT = " userdn = \"ldap:///%s\";)" % BIND_DN ++ ACI_BODY = ACI_TARGET + ACI_ALLOW + ACI_SUBJECT ++ mod = [(mod_type, 'aci', ACI_BODY)] ++ topology.master1.modify_s(SUFFIX, mod) ++ ++ ++def test_ticket47553_init(topology): ++ """ ++ Creates ++ - a staging DIT ++ - a production DIT ++ - add accounts in staging DIT ++ - enable ACL logging (commented for performance reason) ++ ++ """ ++ ++ topology.master1.log.info("\n\n######################### INITIALIZATION ######################\n") ++ ++ # entry used to bind with ++ topology.master1.log.info("Add %s" % BIND_DN) ++ topology.master1.add_s(Entry((BIND_DN, { ++ 'objectclass': "top person".split(), ++ 'sn': BIND_CN, ++ 'cn': BIND_CN, ++ 'userpassword': BIND_PW}))) ++ ++ # DIT for staging ++ topology.master1.log.info("Add %s" % STAGING_DN) ++ topology.master1.add_s(Entry((STAGING_DN, { ++ 'objectclass': "top organizationalRole".split(), ++ 'cn': STAGING_CN, ++ 'description': "staging DIT"}))) ++ ++ # DIT for production ++ topology.master1.log.info("Add %s" % PRODUCTION_DN) ++ topology.master1.add_s(Entry((PRODUCTION_DN, { ++ 'objectclass': "top organizationalRole".split(), ++ 'cn': PRODUCTION_CN, ++ 'description': "production DIT"}))) ++ ++ # DIT for production/except ++ topology.master1.log.info("Add %s" % PROD_EXCEPT_DN) ++ topology.master1.add_s(Entry((PROD_EXCEPT_DN, { ++ 'objectclass': "top organizationalRole".split(), ++ 'cn': EXCEPT_CN, ++ 'description': "production except DIT"}))) ++ ++ # enable acl error logging ++ #mod = [(ldap.MOD_REPLACE, 'nsslapd-errorlog-level', '128')] ++ #topology.master1.modify_s(DN_CONFIG, mod) ++ #topology.master2.modify_s(DN_CONFIG, mod) ++ ++ ++ ++ ++ ++ # add dummy entries in the staging DIT ++ for cpt in range(MAX_ACCOUNTS): ++ name = "%s%d" % (NEW_ACCOUNT, cpt) ++ topology.master1.add_s(Entry(("cn=%s,%s" % (name, STAGING_DN), { ++ 'objectclass': "top person".split(), ++ 'sn': name, ++ 'cn': name}))) ++ ++ ++def test_ticket47553_mode_default_add_deny(topology): ++ ''' ++ This test case checks that the ADD operation fails (no ADD aci on production) ++ ''' ++ ++ topology.master1.log.info("\n\n######################### mode moddn_aci : ADD (should fail) ######################\n") ++ ++ _bind_normal(topology) ++ ++ # ++ # First try to add an entry in production => INSUFFICIENT_ACCESS ++ # ++ try: ++ topology.master1.log.info("Try to add %s" % PRODUCTION_DN) ++ name = "%s%d" % (NEW_ACCOUNT, 0) ++ topology.master1.add_s(Entry(("cn=%s,%s" % (name, PRODUCTION_DN), { ++ 'objectclass': "top person".split(), ++ 'sn': name, ++ 'cn': name}))) ++ assert 0 # this is an error, we should not be allowed to add an entry in production ++ except Exception as e: ++ topology.master1.log.info("Exception (expected): %s" % type(e).__name__) ++ assert isinstance(e, ldap.INSUFFICIENT_ACCESS) ++ ++def test_ticket47553_mode_default_ger_no_moddn(topology): ++ topology.master1.log.info("\n\n######################### mode moddn_aci : GER no moddn ######################\n") ++ request_ctrl = GetEffectiveRightsControl(criticality=True,authzId="dn: " + BIND_DN) ++ msg_id = topology.master1.search_ext(PRODUCTION_DN, ldap.SCOPE_SUBTREE, "objectclass=*", serverctrls=[request_ctrl]) ++ rtype,rdata,rmsgid,response_ctrl = topology.master1.result3(msg_id) ++ ger={} ++ value='' ++ for dn, attrs in rdata: ++ topology.master1.log.info ("dn: %s" % dn) ++ value = attrs['entryLevelRights'][0] ++ ++ topology.master1.log.info ("############### entryLevelRights: %r" % value) ++ assert 'n' not in value ++ ++def test_ticket47553_mode_default_ger_with_moddn(topology): ++ ''' ++ This test case adds the moddn aci and check ger contains 'n' ++ ''' ++ ++ topology.master1.log.info("\n\n######################### mode moddn_aci: GER with moddn ######################\n") ++ ++ # successfull MOD with the ACI ++ _bind_manager(topology) ++ _moddn_aci_staging_to_production(topology, mod_type=ldap.MOD_ADD, target_from=STAGING_DN, target_to=PRODUCTION_DN) ++ _bind_normal(topology) ++ ++ request_ctrl = GetEffectiveRightsControl(criticality=True,authzId="dn: " + BIND_DN) ++ msg_id = topology.master1.search_ext(PRODUCTION_DN, ldap.SCOPE_SUBTREE, "objectclass=*", serverctrls=[request_ctrl]) ++ rtype,rdata,rmsgid,response_ctrl = topology.master1.result3(msg_id) ++ ger={} ++ value = '' ++ for dn, attrs in rdata: ++ topology.master1.log.info ("dn: %s" % dn) ++ value = attrs['entryLevelRights'][0] ++ ++ topology.master1.log.info ("############### entryLevelRights: %r" % value) ++ assert 'n' in value ++ ++ # successfull MOD with the both ACI ++ _bind_manager(topology) ++ _moddn_aci_staging_to_production(topology, mod_type=ldap.MOD_DELETE, target_from=STAGING_DN, target_to=PRODUCTION_DN) ++ _bind_normal(topology) ++ ++def test_ticket47553_mode_switch_default_to_legacy(topology): ++ ''' ++ This test switch the server from default mode to legacy ++ ''' ++ topology.master1.log.info("\n\n######################### Disable the moddn aci mod ######################\n" ) ++ _bind_manager(topology) ++ mod = [(ldap.MOD_REPLACE, CONFIG_MODDN_ACI_ATTR, 'off')] ++ topology.master1.modify_s(DN_CONFIG, mod) ++ ++def test_ticket47553_mode_legacy_ger_no_moddn1(topology): ++ topology.master1.log.info("\n\n######################### mode legacy 1: GER no moddn ######################\n") ++ request_ctrl = GetEffectiveRightsControl(criticality=True,authzId="dn: " + BIND_DN) ++ msg_id = topology.master1.search_ext(PRODUCTION_DN, ldap.SCOPE_SUBTREE, "objectclass=*", serverctrls=[request_ctrl]) ++ rtype,rdata,rmsgid,response_ctrl = topology.master1.result3(msg_id) ++ ger={} ++ value='' ++ for dn, attrs in rdata: ++ topology.master1.log.info ("dn: %s" % dn) ++ value = attrs['entryLevelRights'][0] ++ ++ topology.master1.log.info ("############### entryLevelRights: %r" % value) ++ assert 'n' not in value ++ ++def test_ticket47553_mode_legacy_ger_no_moddn2(topology): ++ topology.master1.log.info("\n\n######################### mode legacy 2: GER no moddn ######################\n") ++ # successfull MOD with the ACI ++ _bind_manager(topology) ++ _moddn_aci_staging_to_production(topology, mod_type=ldap.MOD_ADD, target_from=STAGING_DN, target_to=PRODUCTION_DN) ++ _bind_normal(topology) ++ ++ request_ctrl = GetEffectiveRightsControl(criticality=True,authzId="dn: " + BIND_DN) ++ msg_id = topology.master1.search_ext(PRODUCTION_DN, ldap.SCOPE_SUBTREE, "objectclass=*", serverctrls=[request_ctrl]) ++ rtype,rdata,rmsgid,response_ctrl = topology.master1.result3(msg_id) ++ ger={} ++ value='' ++ for dn, attrs in rdata: ++ topology.master1.log.info ("dn: %s" % dn) ++ value = attrs['entryLevelRights'][0] ++ ++ topology.master1.log.info ("############### entryLevelRights: %r" % value) ++ assert 'n' not in value ++ ++ # successfull MOD with the both ACI ++ _bind_manager(topology) ++ _moddn_aci_staging_to_production(topology, mod_type=ldap.MOD_DELETE, target_from=STAGING_DN, target_to=PRODUCTION_DN) ++ _bind_normal(topology) ++ ++def test_ticket47553_mode_legacy_ger_with_moddn(topology): ++ topology.master1.log.info("\n\n######################### mode legacy : GER with moddn ######################\n") ++ ++ # being allowed to read/write the RDN attribute use to allow the RDN ++ ACI_TARGET = "(target = \"ldap:///%s\")(targetattr=\"cn\")" % (PRODUCTION_DN) ++ ACI_ALLOW = "(version 3.0; acl \"MODDN production changing the RDN attribute\"; allow (read,search,write)" ++ ACI_SUBJECT = " userdn = \"ldap:///%s\";)" % BIND_DN ++ ACI_BODY = ACI_TARGET + ACI_ALLOW + ACI_SUBJECT ++ ++ # successfull MOD with the ACI ++ _bind_manager(topology) ++ mod = [(ldap.MOD_ADD, 'aci', ACI_BODY)] ++ topology.master1.modify_s(SUFFIX, mod) ++ _bind_normal(topology) ++ ++ request_ctrl = GetEffectiveRightsControl(criticality=True,authzId="dn: " + BIND_DN) ++ msg_id = topology.master1.search_ext(PRODUCTION_DN, ldap.SCOPE_SUBTREE, "objectclass=*", serverctrls=[request_ctrl]) ++ rtype,rdata,rmsgid,response_ctrl = topology.master1.result3(msg_id) ++ ger={} ++ value='' ++ for dn, attrs in rdata: ++ topology.master1.log.info ("dn: %s" % dn) ++ value = attrs['entryLevelRights'][0] ++ ++ topology.master1.log.info ("############### entryLevelRights: %r" % value) ++ assert 'n' in value ++ ++ # successfull MOD with the both ACI ++ _bind_manager(topology) ++ mod = [(ldap.MOD_DELETE, 'aci', ACI_BODY)] ++ topology.master1.modify_s(SUFFIX, mod) ++ _bind_normal(topology) ++ ++ ++def test_ticket47553_final(topology): ++ topology.master1.stop(timeout=10) ++ topology.master2.stop(timeout=10) ++ ++def run_isolated(): ++ ''' ++ run_isolated is used to run these test cases independently of a test scheduler (xunit, py.test..) ++ To run isolated without py.test, you need to ++ - edit this file and comment '@pytest.fixture' line before 'topology' function. ++ - set the installation prefix ++ - run this program ++ ''' ++ global installation1_prefix ++ global installation2_prefix ++ installation1_prefix = None ++ installation2_prefix = None ++ ++ topo = topology(True) ++ topo.master1.log.info("\n\n######################### Ticket 47553 ######################\n") ++ test_ticket47553_init(topo) ++ ++ # Check that without appropriate aci we are not allowed to add/delete ++ test_ticket47553_mode_default_add_deny(topo) ++ test_ticket47553_mode_default_ger_no_moddn(topo) ++ test_ticket47553_mode_default_ger_with_moddn(topo) ++ test_ticket47553_mode_switch_default_to_legacy(topo) ++ test_ticket47553_mode_legacy_ger_no_moddn1(topo) ++ test_ticket47553_mode_legacy_ger_no_moddn2(topo) ++ test_ticket47553_mode_legacy_ger_with_moddn(topo) ++ ++ test_ticket47553_final(topo) ++ ++ ++ ++ ++if __name__ == '__main__': ++ run_isolated() ++ +diff --git a/ldap/servers/plugins/acl/acleffectiverights.c b/ldap/servers/plugins/acl/acleffectiverights.c +index b9e2055..3c7d635 100644 +--- a/ldap/servers/plugins/acl/acleffectiverights.c ++++ b/ldap/servers/plugins/acl/acleffectiverights.c +@@ -456,38 +456,45 @@ _ger_get_entry_rights ( + entryrights |= SLAPI_ACL_DELETE; + _append_gerstr(gerstr, gerstrsize, gerstrcap, "d", NULL); + } +- /* +- * Some limitation/simplification applied here: +- * - The modrdn right requires the rights to delete the old rdn and +- * the new one. However we have no knowledge of what the new rdn +- * is going to be. +- * - In multi-valued RDN case, we check the right on +- * the first rdn type only for now. +- */ +- rdn = slapi_rdn_new_dn ( slapi_entry_get_ndn (e) ); +- slapi_rdn_get_first(rdn, &rdntype, &rdnvalue); +- if ( NULL != rdntype ) { +- slapi_log_error (SLAPI_LOG_ACL, plugin_name, +- "_ger_get_entry_rights: SLAPI_ACL_WRITE_DEL & _ADD %s\n", rdntype ); +- if (acl_access_allowed(gerpb, e, rdntype, NULL, +- ACLPB_SLAPI_ACL_WRITE_DEL) == LDAP_SUCCESS && +- acl_access_allowed(gerpb, e, rdntype, NULL, +- ACLPB_SLAPI_ACL_WRITE_ADD) == LDAP_SUCCESS) +- { +- /* n - rename e */ +- entryrights |= SLAPI_ACL_WRITE; +- _append_gerstr(gerstr, gerstrsize, gerstrcap, "n", NULL); +- } +- } +- slapi_rdn_free ( &rdn ); +- +- if (acl_access_allowed(gerpb, e, NULL, NULL, SLAPI_ACL_MODDN) == LDAP_SUCCESS) { +- slapi_log_error (SLAPI_LOG_ACL, plugin_name, +- "_ger_get_entry_rights: SLAPI_ACL_MODDN %s\n", slapi_entry_get_ndn (e) ); ++ ++ if (config_get_moddn_aci()) { ++ /* The server enforces the new MODDN aci right. ++ * So the status 'n' is set if this right is granted. ++ * Opposed to the legacy mode where this flag is set if ++ * WRITE was granted on rdn attrbibute ++ */ ++ if (acl_access_allowed(gerpb, e, NULL, NULL, SLAPI_ACL_MODDN) == LDAP_SUCCESS) { ++ slapi_log_error(SLAPI_LOG_ACL, plugin_name, ++ "_ger_get_entry_rights: SLAPI_ACL_MODDN %s\n", slapi_entry_get_ndn(e)); ++ /* n - rename e */ ++ entryrights |= SLAPI_ACL_MODDN; ++ _append_gerstr(gerstr, gerstrsize, gerstrcap, "n", NULL); ++ } ++ } else { ++ /* ++ * Some limitation/simplification applied here: ++ * - The modrdn right requires the rights to delete the old rdn and ++ * the new one. However we have no knowledge of what the new rdn ++ * is going to be. ++ * - In multi-valued RDN case, we check the right on ++ * the first rdn type only for now. ++ */ ++ rdn = slapi_rdn_new_dn(slapi_entry_get_ndn(e)); ++ slapi_rdn_get_first(rdn, &rdntype, &rdnvalue); ++ if (NULL != rdntype) { ++ slapi_log_error(SLAPI_LOG_ACL, plugin_name, ++ "_ger_get_entry_rights: SLAPI_ACL_WRITE_DEL & _ADD %s\n", rdntype); ++ if (acl_access_allowed(gerpb, e, rdntype, NULL, ++ ACLPB_SLAPI_ACL_WRITE_DEL) == LDAP_SUCCESS && ++ acl_access_allowed(gerpb, e, rdntype, NULL, ++ ACLPB_SLAPI_ACL_WRITE_ADD) == LDAP_SUCCESS) { + /* n - rename e */ +- entryrights |= SLAPI_ACL_MODDN; +- _append_gerstr(gerstr, gerstrsize, gerstrcap, "n", NULL); ++ entryrights |= SLAPI_ACL_WRITE; ++ _append_gerstr(gerstr, gerstrsize, gerstrcap, "n", NULL); ++ } + } ++ slapi_rdn_free(&rdn); ++ } + if ( entryrights == 0 ) + { + _append_gerstr(gerstr, gerstrsize, gerstrcap, "none", NULL); +-- +1.9.3 + diff --git a/SOURCES/0078-Ticket-48265-Complex-filter-in-a-search-request-doen.patch b/SOURCES/0078-Ticket-48265-Complex-filter-in-a-search-request-doen.patch new file mode 100644 index 0000000..9b9e18f --- /dev/null +++ b/SOURCES/0078-Ticket-48265-Complex-filter-in-a-search-request-doen.patch @@ -0,0 +1,55 @@ +From 924a6ec6ca3131ff7233fd664ede9051907d96b1 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Wed, 2 Sep 2015 14:28:27 -0700 +Subject: [PATCH 78/84] Ticket #48265 - Complex filter in a search request + doen't work as expected. (regression) + +Description: commit c2658c14802783d0a8919783aa7123be9e749c18 to fix +Ticket 47521 - Complex filter in a search request doen't work as expected. +regressed this case: + "(&(&(|(l=A)(l=B)(l=C))(|(C=D)(c=E)))(|(uid=*test*)(cn=*test*))(o=X))" +in which a simple filter follows a complex filter which choice is +different from the outer choice. I.e., '|' for (uid=...)(cn=...) +is different from the first '&'. + +The fix for 47521 solves this case: + "(&(&(uid=A)(cn=B))(&(givenname=C))(mail=D)(&(description=E)))" +in this case, (mail=D) used to be dropped from the filter in the +function index_subsys_flatten_filter. + +The 47521 fix saved the simple filter "(mail=D)" in the 2nd example, +but it forced to skip the complex filter with the different choice +and converted the 1st example to: + "(&(&(|(l=A)(l=B)(l=C))(|(C=D)(c=E)))(o=X))" +This patch saves such a complex filter, as well. + +https://fedorahosted.org/389/ticket/48265 + +Reviewed by mreynolds@redhat.com (Thank you, Mark!!) + +(cherry picked from commit 8c3d3e4648fbb5229e329e2154d46f1ae808ba02) +(cherry picked from commit 3d9dbf2d441e551495a1f3169dc2020324c484b4) +(cherry picked from commit c03a9f7c121355aefadc92ed67bcb6f400196017) +--- + ldap/servers/slapd/index_subsystem.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/ldap/servers/slapd/index_subsystem.c b/ldap/servers/slapd/index_subsystem.c +index eff908e..d0ba1bd 100644 +--- a/ldap/servers/slapd/index_subsystem.c ++++ b/ldap/servers/slapd/index_subsystem.c +@@ -441,6 +441,11 @@ static void index_subsys_flatten_filter(Slapi_Filter *flist) + } + else + { ++ /* don't loose a nested filter having a different choice */ ++ if (flast) { ++ flast->f_next = f; ++ flast = f; ++ } + fprev = f; + f = f->f_next; + } +-- +1.9.3 + diff --git a/SOURCES/0079-Ticket-47912-Proper-handling-of-No-original_tombston.patch b/SOURCES/0079-Ticket-47912-Proper-handling-of-No-original_tombston.patch new file mode 100644 index 0000000..8fe2078 --- /dev/null +++ b/SOURCES/0079-Ticket-47912-Proper-handling-of-No-original_tombston.patch @@ -0,0 +1,89 @@ +From 8f54c897c404164c618c808435d3887e62d32915 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Tue, 7 Oct 2014 14:02:20 -0700 +Subject: [PATCH 79/84] Ticket #47912 - Proper handling of "No + original_tombstone for changenumber" errors + +Bug Description: As analyzed by Ludwig Krispen in + https://fedorahosted.org/389/ticket/47912#comment:1, +an error message "No original_tombstone for ..." is always logged +if original_tombstone does not exist. It should be just for the +case create_tombstone_entry. + +Fix Description: This patch place the original_tombstone handling +in "if (create_tombstone_entry)" clause. + +https://fedorahosted.org/389/ticket/47912 + +Reviewed by lkrispen@redhat.com (Thank you, Ludwig!!) + +(cherry picked from commit 36381c120773872d3d4d2cb2417f155e6ac790a6) +--- + ldap/servers/slapd/back-ldbm/ldbm_delete.c | 48 +++++++++++++++--------------- + 1 file changed, 24 insertions(+), 24 deletions(-) + +diff --git a/ldap/servers/slapd/back-ldbm/ldbm_delete.c b/ldap/servers/slapd/back-ldbm/ldbm_delete.c +index 56ea3df..3de8efa 100644 +--- a/ldap/servers/slapd/back-ldbm/ldbm_delete.c ++++ b/ldap/servers/slapd/back-ldbm/ldbm_delete.c +@@ -225,33 +225,33 @@ ldbm_back_delete( Slapi_PBlock *pb ) + free_delete_existing_entry = 1; /* must free the dup */ + if (create_tombstone_entry) { + slapi_sdn_set_ndn_byval(&nscpEntrySDN, slapi_sdn_get_ndn(slapi_entry_get_sdn(e->ep_entry))); +- } + +- /* reset tombstone entry */ +- if (original_tombstone) { +- /* must duplicate tombstone before returning it to cache, +- * which could free the entry. */ +- if ( (tmptombstone = backentry_dup( original_tombstone )) == NULL ) { +- ldap_result_code= LDAP_OPERATIONS_ERROR; +- goto error_return; +- } +- if (cache_is_in_cache(&inst->inst_cache, tombstone)) { +- CACHE_REMOVE(&inst->inst_cache, tombstone); +- } +- CACHE_RETURN(&inst->inst_cache, &tombstone); +- if (tombstone) { ++ /* reset tombstone entry */ ++ if (original_tombstone) { ++ /* must duplicate tombstone before returning it to cache, ++ * which could free the entry. */ ++ if ( (tmptombstone = backentry_dup( original_tombstone )) == NULL ) { ++ ldap_result_code= LDAP_OPERATIONS_ERROR; ++ goto error_return; ++ } ++ if (cache_is_in_cache(&inst->inst_cache, tombstone)) { ++ CACHE_REMOVE(&inst->inst_cache, tombstone); ++ } ++ CACHE_RETURN(&inst->inst_cache, &tombstone); ++ if (tombstone) { ++ slapi_log_error(SLAPI_LOG_FATAL, "ldbm_back_delete", ++ "conn=%lu op=%d [retry: %d] tombstone %s is not freed!!! refcnt %d, state %d\n", ++ conn_id, op_id, retry_count, slapi_entry_get_dn(tombstone->ep_entry), ++ tombstone->ep_refcnt, tombstone->ep_state); ++ } ++ tombstone = original_tombstone; ++ original_tombstone = tmptombstone; ++ tmptombstone = NULL; ++ } else { + slapi_log_error(SLAPI_LOG_FATAL, "ldbm_back_delete", +- "conn=%lu op=%d [retry: %d] tombstone %s is not freed!!! refcnt %d, state %d\n", +- conn_id, op_id, retry_count, slapi_entry_get_dn(tombstone->ep_entry), +- tombstone->ep_refcnt, tombstone->ep_state); ++ "conn=%lu op=%d [retry: %d] No original_tombstone for %s!!\n", ++ conn_id, op_id, retry_count, slapi_entry_get_dn(e->ep_entry)); + } +- tombstone = original_tombstone; +- original_tombstone = tmptombstone; +- tmptombstone = NULL; +- } else { +- slapi_log_error(SLAPI_LOG_FATAL, "ldbm_back_delete", +- "conn=%lu op=%d [retry: %d] No original_tombstone for %s!!\n", +- conn_id, op_id, retry_count, slapi_entry_get_dn(e->ep_entry)); + } + if (ruv_c_init) { + /* reset the ruv txn stuff */ +-- +1.9.3 + diff --git a/SOURCES/0080-Ticket-48208-CleanAllRUV-should-completely-purge-cha.patch b/SOURCES/0080-Ticket-48208-CleanAllRUV-should-completely-purge-cha.patch new file mode 100644 index 0000000..2ae359c --- /dev/null +++ b/SOURCES/0080-Ticket-48208-CleanAllRUV-should-completely-purge-cha.patch @@ -0,0 +1,809 @@ +From d32a172a4bc927a5eb72acecfe07ba7fa8ea3a55 Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Wed, 8 Jul 2015 11:48:27 -0400 +Subject: [PATCH 80/84] Ticket 48208 - CleanAllRUV should completely purge + changelog + +Bug Description: After cleanAllRUV finishes, the changelog still + contains entries from the cleaned rid. Under certain + conditions this can allow the RUV to get polluted + again, and the ruv element will be missing the replica + url. + +Fix Description: At the end of the cleaning task, fire of a thread to + to completely purge the changelog of all entries + containing the cleaned rid. + + Also, improved the cleanAllRUV task when dealing + with a server shutdown - previously if the timing is + right the task can "delay/hang" the shutdown process. + +https://fedorahosted.org/389/ticket/48208 + +Reviewed by: nhosoi(Thanks!) + +(cherry picked from commit ff1c34538b0600259dba4801da2b2f0993fa5404) +(cherry picked from commit 9e4cf12cfbfde0761325b75c3fd5a8b39223760a) +(cherry picked from commit 46cd28db8402517febf0c5db4f2f869c491c41c0) +--- + ldap/servers/plugins/replication/cl5_api.c | 447 ++++++++++++++++++--- + ldap/servers/plugins/replication/cl5_api.h | 5 +- + .../plugins/replication/repl5_replica_config.c | 44 +- + 3 files changed, 430 insertions(+), 66 deletions(-) + +diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c +index 42e52ae..c5840b5 100644 +--- a/ldap/servers/plugins/replication/cl5_api.c ++++ b/ldap/servers/plugins/replication/cl5_api.c +@@ -353,14 +353,17 @@ static void _cl5TrimCleanup (); + static int _cl5TrimMain (void *param); + static void _cl5DoTrimming (ReplicaId rid); + static void _cl5CompactDBs(); +-static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid); ++static void _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid); ++static int _cl5PurgeGetFirstEntry (Object *obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key); ++static int _cl5PurgeGetNextEntry (CL5Entry *entry, void *iterator, DBT *key); ++static void _cl5TrimFile (Object *obj, long *numToTrim); + static PRBool _cl5CanTrim (time_t time, long *numToTrim); + static int _cl5ReadRUV (const char *replGen, Object *obj, PRBool purge); + static int _cl5WriteRUV (CL5DBFile *file, PRBool purge); + static int _cl5ConstructRUV (const char *replGen, Object *obj, PRBool purge); + static int _cl5UpdateRUV (Object *obj, CSN *csn, PRBool newReplica, PRBool purge); + static int _cl5GetRUV2Purge2 (Object *fileObj, RUV **ruv); +-void trigger_cl_trimming_thread(void *rid); ++void trigger_cl_purging_thread(void *rid); + + /* bakup/recovery, import/export */ + static int _cl5LDIF2Operation (char *ldifEntry, slapi_operation_parameters *op, +@@ -3499,9 +3502,17 @@ static void _cl5DoTrimming (ReplicaId rid) + trimmed more often than other. We might have to fix that by, for + example, randomizing starting point */ + obj = objset_first_obj (s_cl5Desc.dbFiles); +- while (obj && _cl5CanTrim ((time_t)0, &numToTrim)) ++ while (obj && (_cl5CanTrim ((time_t)0, &numToTrim) || rid)) + { +- _cl5TrimFile (obj, &numToTrim, rid); ++ if (rid){ ++ /* ++ * We are cleaning an invalid rid, and need to strip it ++ * from the changelog. ++ */ ++ _cl5PurgeRID (obj, rid); ++ } else { ++ _cl5TrimFile (obj, &numToTrim); ++ } + obj = objset_next_obj (s_cl5Desc.dbFiles, obj); + } + +@@ -3578,12 +3589,351 @@ bail: + return; + } + ++/* ++ * If the rid is not set it is the very first iteration of the changelog. ++ * If the rid is set, we are doing another pass, and we have a key as our ++ * starting point. ++ */ ++static int ++_cl5PurgeGetFirstEntry(Object *obj, CL5Entry *entry, void **iterator, DB_TXN *txnid, int rid, DBT *key) ++{ ++ DBC *cursor = NULL; ++ DBT data = {0}; ++ CL5Iterator *it; ++ CL5DBFile *file; ++ int rc; ++ ++ file = (CL5DBFile*)object_get_data (obj); ++ ++ /* create cursor */ ++ rc = file->db->cursor(file->db, txnid, &cursor, 0); ++ if (rc != 0) ++ { ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeGetFirstEntry: failed to create cursor; db error - %d %s\n", rc, db_strerror(rc)); ++ rc = CL5_DB_ERROR; ++ goto done; ++ } ++ ++ key->flags = DB_DBT_MALLOC; ++ data.flags = DB_DBT_MALLOC; ++ while ((rc = cursor->c_get(cursor, key, &data, rid?DB_SET:DB_NEXT)) == 0) ++ { ++ /* skip service entries on the first pass (rid == 0)*/ ++ if (!rid && cl5HelperEntry ((char*)key->data, NULL)) ++ { ++ slapi_ch_free(&key->data); ++ slapi_ch_free(&(data.data)); ++ continue; ++ } ++ ++ /* format entry */ ++ rc = cl5DBData2Entry(data.data, data.size, entry); ++ slapi_ch_free(&(data.data)); ++ if (rc != 0) ++ { ++ slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, ++ "_cl5PurgeGetFirstEntry: failed to format entry: %d\n", rc); ++ goto done; ++ } ++ ++ it = (CL5Iterator*)slapi_ch_malloc(sizeof (CL5Iterator)); ++ it->cursor = cursor; ++ object_acquire (obj); ++ it->file = obj; ++ *(CL5Iterator**)iterator = it; ++ ++ return CL5_SUCCESS; ++ } ++ ++ slapi_ch_free(&key->data); ++ slapi_ch_free(&(data.data)); ++ ++ /* walked of the end of the file */ ++ if (rc == DB_NOTFOUND) ++ { ++ rc = CL5_NOTFOUND; ++ goto done; ++ } ++ ++ /* db error occured while iterating */ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeGetFirstEntry: failed to get entry; db error - %d %s\n", ++ rc, db_strerror(rc)); ++ rc = CL5_DB_ERROR; ++ ++done: ++ /* ++ * We didn't success in assigning this cursor to the iterator, ++ * so we need to free the cursor here. ++ */ ++ if (cursor) ++ cursor->c_close(cursor); ++ ++ return rc; ++} ++ ++/* ++ * Get the next entry. If we get a lock error we will restart the process ++ * starting at the current key. ++ */ ++static int ++_cl5PurgeGetNextEntry (CL5Entry *entry, void *iterator, DBT *key) ++{ ++ CL5Iterator *it; ++ DBT data={0}; ++ int rc; ++ ++ it = (CL5Iterator*) iterator; ++ ++ key->flags = DB_DBT_MALLOC; ++ data.flags = DB_DBT_MALLOC; ++ while ((rc = it->cursor->c_get(it->cursor, key, &data, DB_NEXT)) == 0) ++ { ++ if (cl5HelperEntry ((char*)key->data, NULL)) ++ { ++ slapi_ch_free(&key->data); ++ slapi_ch_free(&(data.data)); ++ continue; ++ } ++ ++ /* format entry */ ++ rc = cl5DBData2Entry (data.data, data.size, entry); ++ slapi_ch_free (&(data.data)); ++ if (rc != 0) ++ { ++ if (rc != CL5_DB_LOCK_ERROR){ ++ /* Not a lock error, free the key */ ++ slapi_ch_free(&key->data); ++ } ++ slapi_log_error(rc == CL5_DB_LOCK_ERROR?SLAPI_LOG_REPL:SLAPI_LOG_FATAL, ++ repl_plugin_name_cl, ++ "_cl5PurgeGetNextEntry: failed to format entry: %d\n", ++ rc); ++ ++ } ++ ++ return rc; ++ } ++ slapi_ch_free(&(data.data)); ++ ++ /* walked of the end of the file or entry is out of range */ ++ if (rc == 0 || rc == DB_NOTFOUND){ ++ slapi_ch_free(&key->data); ++ return CL5_NOTFOUND; ++ } ++ if (rc != CL5_DB_LOCK_ERROR){ ++ /* Not a lock error, free the key */ ++ slapi_ch_free(&key->data); ++ } ++ ++ /* cursor operation failed */ ++ slapi_log_error(rc == CL5_DB_LOCK_ERROR?SLAPI_LOG_REPL:SLAPI_LOG_FATAL, ++ repl_plugin_name_cl, ++ "_cl5PurgeGetNextEntry: failed to get entry; db error - %d %s\n", ++ rc, db_strerror(rc)); ++ ++ return rc; ++} ++ ++#define MAX_RETRIES 10 ++/* ++ * _cl5PurgeRID(Object *obj, ReplicaId cleaned_rid) ++ * ++ * Clean the entire changelog of updates from the "cleaned rid" via CLEANALLRUV ++ * Delete entries in batches so we don't consume too many db locks, and we don't ++ * lockup the changelog during the entire purging process using one transaction. ++ * We save the key from the last iteration so we don't have to start from the ++ * beginning for each new iteration. ++ */ ++static void ++_cl5PurgeRID(Object *obj, ReplicaId cleaned_rid) ++{ ++ slapi_operation_parameters op = {0}; ++ ReplicaId csn_rid; ++ CL5Entry entry; ++ DB_TXN *txnid = NULL; ++ DBT key = {0}; ++ void *iterator = NULL; ++ long totalTrimmed = 0; ++ long trimmed = 0; ++ char *starting_key = NULL; ++ int batch_count = 0; ++ int db_lock_retry_count = 0; ++ int first_pass = 1; ++ int finished = 0; ++ int rc = 0; ++ ++ PR_ASSERT (obj); ++ entry.op = &op; ++ ++ /* ++ * Keep processing the changelog until we are done, shutting down, or we ++ * maxed out on the db lock retries. ++ */ ++ while (!finished && db_lock_retry_count < MAX_RETRIES && !slapi_is_shutting_down()){ ++ trimmed = 0; ++ ++ /* ++ * Sleep a bit to allow others to use the changelog - we can't hog the ++ * changelog for the entire purge. ++ */ ++ DS_Sleep(PR_MillisecondsToInterval(100)); ++ ++ rc = TXN_BEGIN(s_cl5Desc.dbEnv, NULL, &txnid, 0); ++ if (rc != 0){ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: failed to begin transaction; db error - %d %s. " ++ "Changelog was not purged of rid(%d)\n", ++ rc, db_strerror(rc), cleaned_rid); ++ return; ++ } ++ ++ /* ++ * Check every changelog entry for the cleaned rid ++ */ ++ rc = _cl5PurgeGetFirstEntry(obj, &entry, &iterator, txnid, first_pass?0:cleaned_rid, &key); ++ first_pass = 0; ++ while (rc == CL5_SUCCESS && !slapi_is_shutting_down()) { ++ /* ++ * Store the new starting key - we need this starting key in case ++ * we run out of locks and have to start the transaction over. ++ */ ++ slapi_ch_free_string(&starting_key); ++ starting_key = slapi_ch_strdup((char*)key.data); ++ ++ if(trimmed == 10000 || (batch_count && trimmed == batch_count)){ ++ /* ++ * Break out, and commit these deletes. Do not free the key, ++ * we need it for the next pass. ++ */ ++ cl5_operation_parameters_done (&op); ++ db_lock_retry_count = 0; /* reset the retry count */ ++ break; ++ } ++ if(op.csn){ ++ csn_rid = csn_get_replicaid (op.csn); ++ if (csn_rid == cleaned_rid){ ++ rc = _cl5CurrentDeleteEntry (iterator); ++ if (rc != CL5_SUCCESS){ ++ /* log error */ ++ cl5_operation_parameters_done (&op); ++ if (rc == CL5_DB_LOCK_ERROR){ ++ /* ++ * Ran out of locks, need to restart the transaction. ++ * Reduce the the batch count and reset the key to ++ * the starting point ++ */ ++ slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, ++ "_cl5PurgeRID: Ran out of db locks deleting entry. " ++ "Reduce the batch value and restart.\n"); ++ batch_count = trimmed - 10; ++ if (batch_count < 10){ ++ batch_count = 10; ++ } ++ trimmed = 0; ++ slapi_ch_free(&(key.data)); ++ key.data = starting_key; ++ starting_key = NULL; ++ db_lock_retry_count++; ++ break; ++ } else { ++ /* fatal error */ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: fatal error (%d)\n", rc); ++ slapi_ch_free(&(key.data)); ++ finished = 1; ++ break; ++ } ++ } ++ trimmed++; ++ } ++ } ++ slapi_ch_free(&(key.data)); ++ cl5_operation_parameters_done (&op); ++ ++ rc = _cl5PurgeGetNextEntry (&entry, iterator, &key); ++ if (rc == CL5_DB_LOCK_ERROR){ ++ /* ++ * Ran out of locks, need to restart the transaction. ++ * Reduce the the batch count and reset the key to the starting ++ * point. ++ */ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: Ran out of db locks getting the next entry. " ++ "Reduce the batch value and restart.\n"); ++ batch_count = trimmed - 10; ++ if (batch_count < 10){ ++ batch_count = 10; ++ } ++ trimmed = 0; ++ cl5_operation_parameters_done (&op); ++ slapi_ch_free(&(key.data)); ++ key.data = starting_key; ++ starting_key = NULL; ++ db_lock_retry_count++; ++ break; ++ } ++ } ++ ++ if (rc == CL5_NOTFOUND){ ++ /* Scanned the entire changelog, we're done */ ++ finished = 1; ++ } ++ ++ /* Destroy the iterator before we finish with the txn */ ++ cl5DestroyIterator (iterator); ++ ++ /* ++ * Commit or abort the txn ++ */ ++ if (rc == CL5_SUCCESS || rc == CL5_NOTFOUND){ ++ rc = TXN_COMMIT (txnid, 0); ++ if (rc != 0){ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: failed to commit transaction; db error - %d %s. " ++ "Changelog was not completely purged of rid (%d)\n", ++ rc, db_strerror(rc), cleaned_rid); ++ break; ++ } else if (finished){ ++ /* We're done */ ++ totalTrimmed += trimmed; ++ break; ++ } else { ++ /* Not done yet */ ++ totalTrimmed += trimmed; ++ trimmed = 0; ++ } ++ } else { ++ rc = TXN_ABORT (txnid); ++ if (rc != 0){ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: failed to abort transaction; db error - %d %s. " ++ "Changelog was not completely purged of rid (%d)\n", ++ rc, db_strerror(rc), cleaned_rid); ++ } ++ if (batch_count == 0){ ++ /* This was not a retry. Fatal error, break out */ ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5PurgeRID: Changelog was not purged of rid (%d)\n", ++ cleaned_rid); ++ break; ++ } ++ } ++ } ++ slapi_ch_free_string(&starting_key); ++ ++ slapi_log_error (SLAPI_LOG_REPL, repl_plugin_name_cl, ++ "_cl5PurgeRID: Removed (%ld entries) that originated from rid (%d)\n", ++ totalTrimmed, cleaned_rid); ++} ++ + /* Note that each file contains changes for a single replicated area. + trimming algorithm: + */ + #define CL5_TRIM_MAX_PER_TRANSACTION 10 + +-static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) ++static void _cl5TrimFile (Object *obj, long *numToTrim) + { + DB_TXN *txnid; + RUV *ruv = NULL; +@@ -3606,7 +3956,6 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + } + + entry.op = &op; +- + while ( !finished && !slapi_is_shutting_down() ) + { + it = NULL; +@@ -3627,7 +3976,7 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + } + + finished = _cl5GetFirstEntry (obj, &entry, &it, txnid); +- while ( !finished ) ++ while ( !finished && !slapi_is_shutting_down()) + { + /* + * This change can be trimmed if it exceeds purge +@@ -3641,11 +3990,12 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + continue; + } + csn_rid = csn_get_replicaid (op.csn); ++ + if ( (*numToTrim > 0 || _cl5CanTrim (entry.time, numToTrim)) && + ruv_covers_csn_strict (ruv, op.csn) ) + { + rc = _cl5CurrentDeleteEntry (it); +- if ( rc == CL5_SUCCESS && cleaned_rid != csn_rid) ++ if ( rc == CL5_SUCCESS) + { + rc = _cl5UpdateRUV (obj, op.csn, PR_FALSE, PR_TRUE); + } +@@ -3659,7 +4009,6 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + /* The above two functions have logged the error */ + abort = PR_TRUE; + } +- + } + else + { +@@ -3716,7 +4065,7 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + rc = TXN_ABORT (txnid); + if (rc != 0) + { +- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5TrimFile: failed to abort transaction; db error - %d %s\n", + rc, db_strerror(rc)); + } +@@ -3727,7 +4076,7 @@ static void _cl5TrimFile (Object *obj, long *numToTrim, ReplicaId cleaned_rid) + if (rc != 0) + { + finished = 1; +- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5TrimFile: failed to commit transaction; db error - %d %s\n", + rc, db_strerror(rc)); + } +@@ -4751,9 +5100,9 @@ static int _cl5WriteOperationTxn(const char *replName, const char *replGen, + goto done; + } + #endif +- /* back off */ ++ /* back off */ + interval = PR_MillisecondsToInterval(slapi_rand() % 100); +- DS_Sleep(interval); ++ DS_Sleep(interval); + } + #if USE_DB_TXN + /* begin transaction */ +@@ -4799,19 +5148,19 @@ static int _cl5WriteOperationTxn(const char *replName, const char *replGen, + } + cnt ++; + } +- ++ + if (rc == 0) /* we successfully added entry */ + { + #if USE_DB_TXN + rc = TXN_COMMIT (txnid, 0); + #endif + } +- else ++ else + { +- char s[CSN_STRSIZE]; +- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ char s[CSN_STRSIZE]; ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5WriteOperationTxn: failed to write entry with csn (%s); " +- "db error - %d %s\n", csn_as_string(op->csn,PR_FALSE,s), ++ "db error - %d %s\n", csn_as_string(op->csn,PR_FALSE,s), + rc, db_strerror(rc)); + #if USE_DB_TXN + rc = TXN_ABORT (txnid); +@@ -4832,7 +5181,7 @@ static int _cl5WriteOperationTxn(const char *replName, const char *replGen, + /* update purge vector if we have not seen any changes from this replica before */ + _cl5UpdateRUV (file_obj, op->csn, PR_TRUE, PR_TRUE); + +- slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name_cl, ++ slapi_log_error(SLAPI_LOG_PLUGIN, repl_plugin_name_cl, + "cl5WriteOperationTxn: successfully written entry with csn (%s)\n", csnStr); + rc = CL5_SUCCESS; + done: +@@ -4846,7 +5195,7 @@ done: + return rc; + } + +-static int _cl5WriteOperation(const char *replName, const char *replGen, ++static int _cl5WriteOperation(const char *replName, const char *replGen, + const slapi_operation_parameters *op, PRBool local) + { + return _cl5WriteOperationTxn(replName, replGen, op, local, NULL); +@@ -4897,7 +5246,7 @@ static int _cl5GetFirstEntry (Object *obj, CL5Entry *entry, void **iterator, DB_ + goto done; + } + +- it = (CL5Iterator*)slapi_ch_malloc (sizeof (CL5Iterator)); ++ it = (CL5Iterator*)slapi_ch_malloc(sizeof (CL5Iterator)); + it->cursor = cursor; + object_acquire (obj); + it->file = obj; +@@ -4972,7 +5321,7 @@ static int _cl5GetNextEntry (CL5Entry *entry, void *iterator) + slapi_ch_free (&(data.data)); + if (rc != 0) + { +- slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, + "_cl5GetNextEntry: failed to format entry: %d\n", rc); + } + +@@ -5001,38 +5350,42 @@ static int _cl5GetNextEntry (CL5Entry *entry, void *iterator) + } + + /* cursor operation failed */ +- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, +- "_cl5GetNextEntry: failed to get entry; db error - %d %s\n", +- rc, db_strerror(rc)); ++ slapi_log_error(rc == CL5_DB_LOCK_ERROR?SLAPI_LOG_REPL:SLAPI_LOG_FATAL, ++ repl_plugin_name_cl, ++ "_cl5GetNextEntry: failed to get entry; db error - %d %s\n", ++ rc, db_strerror(rc)); + +- return CL5_DB_ERROR; ++ return rc; + } + + static int _cl5CurrentDeleteEntry (void *iterator) + { + int rc; + CL5Iterator *it; +- CL5DBFile *file; ++ CL5DBFile *file; + +- PR_ASSERT (iterator); ++ PR_ASSERT (iterator); + + it = (CL5Iterator*)iterator; + + rc = it->cursor->c_del (it->cursor, 0); + + if (rc == 0) { +- /* decrement entry count */ +- file = (CL5DBFile*)object_get_data (it->file); +- PR_AtomicDecrement (&file->entryCount); +- return CL5_SUCCESS; +- } else { +- slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, +- "_cl5CurrentDeleteEntry failed, err=%d %s\n", +- rc, db_strerror(rc)); +- /* We don't free(close) the cursor here, as the caller will free it by a call to cl5DestroyIterator */ +- /* Freeing it here is a potential bug, as the cursor can't be referenced later once freed */ +- return CL5_DB_ERROR; +- } ++ /* decrement entry count */ ++ file = (CL5DBFile*)object_get_data (it->file); ++ PR_AtomicDecrement (&file->entryCount); ++ return CL5_SUCCESS; ++ } else { ++ slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, ++ "_cl5CurrentDeleteEntry failed, err=%d %s\n", ++ rc, db_strerror(rc)); ++ /* ++ * We don't free(close) the cursor here, as the caller will free it by ++ * a call to cl5DestroyIterator. Freeing it here is a potential bug, ++ * as the cursor can't be referenced later once freed. ++ */ ++ return rc; ++ } + } + + static PRBool _cl5IsValidIterator (const CL5Iterator *iterator) +@@ -6304,7 +6657,7 @@ static int _cl5ExportFile (PRFileDesc *prFile, Object *obj) + slapi_write_buffer (prFile, "\n", strlen("\n")); + + entry.op = &op; +- rc = _cl5GetFirstEntry (obj, &entry, &iterator, NULL); ++ rc = _cl5GetFirstEntry (obj, &entry, &iterator, NULL); + while (rc == CL5_SUCCESS) + { + rc = _cl5Operation2LDIF (&op, file->replGen, &buff, &len); +@@ -6725,16 +7078,16 @@ cl5CleanRUV(ReplicaId rid){ + slapi_rwlock_unlock (s_cl5Desc.stLock); + } + +-void trigger_cl_trimming(ReplicaId rid){ ++void trigger_cl_purging(ReplicaId rid){ + PRThread *trim_tid = NULL; + +- slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "trigger_cl_trimming: rid (%d)\n",(int)rid); +- trim_tid = PR_CreateThread(PR_USER_THREAD, (VFP)(void*)trigger_cl_trimming_thread, ++ slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "trigger_cl_purging: rid (%d)\n",(int)rid); ++ trim_tid = PR_CreateThread(PR_USER_THREAD, (VFP)(void*)trigger_cl_purging_thread, + (void *)&rid, PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD, + PR_UNJOINABLE_THREAD, DEFAULT_THREAD_STACKSIZE); + if (NULL == trim_tid){ + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, +- "trigger_cl_trimming: failed to create trimming " ++ "trigger_cl_purging: failed to create trimming " + "thread; NSPR error - %d\n", PR_GetError ()); + } else { + /* need a little time for the thread to get started */ +@@ -6743,7 +7096,7 @@ void trigger_cl_trimming(ReplicaId rid){ + } + + void +-trigger_cl_trimming_thread(void *arg){ ++trigger_cl_purging_thread(void *arg){ + ReplicaId rid = *(ReplicaId *)arg; + + /* make sure we have a change log, and we aren't closing it */ +@@ -6752,7 +7105,7 @@ trigger_cl_trimming_thread(void *arg){ + } + if (CL5_SUCCESS != _cl5AddThread()) { + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, +- "trigger_cl_trimming: failed to increment thread count " ++ "trigger_cl_purging: failed to increment thread count " + "NSPR error - %d\n", PR_GetError ()); + } + _cl5DoTrimming(rid); +diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h +index ba9eb32..5dcc8e2 100644 +--- a/ldap/servers/plugins/replication/cl5_api.h ++++ b/ldap/servers/plugins/replication/cl5_api.h +@@ -145,6 +145,9 @@ enum + CL5_CSN_ERROR, /* CSN API failed */ + CL5_RUV_ERROR, /* RUV API failed */ + CL5_OBJSET_ERROR, /* namedobjset api failed */ ++ CL5_DB_LOCK_ERROR, /* bdb returns error 12 when the db runs out of locks, ++ this var needs to be in slot 12 of the list. ++ Do not re-order enum above! */ + CL5_PURGED_DATA, /* requested data has been purged */ + CL5_MISSING_DATA, /* data should be in the changelog, but is missing */ + CL5_UNKNOWN_ERROR, /* unclassified error */ +@@ -492,6 +495,6 @@ int cl5WriteRUV(); + int cl5DeleteRUV(); + void cl5CleanRUV(ReplicaId rid); + void cl5NotifyCleanup(int rid); +-void trigger_cl_trimming(ReplicaId rid); ++void trigger_cl_purging(ReplicaId rid); + + #endif +diff --git a/ldap/servers/plugins/replication/repl5_replica_config.c b/ldap/servers/plugins/replication/repl5_replica_config.c +index 1570ba7..974778c 100644 +--- a/ldap/servers/plugins/replication/repl5_replica_config.c ++++ b/ldap/servers/plugins/replication/repl5_replica_config.c +@@ -1468,6 +1468,11 @@ replica_execute_cleanruv_task (Object *r, ReplicaId rid, char *returntext /* not + */ + cl5CleanRUV(rid); + ++ /* ++ * Now purge the changelog ++ */ ++ trigger_cl_purging(rid); ++ + if (rc != RUV_SUCCESS){ + slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name, "cleanruv_task: task failed(%d)\n",rc); + return LDAP_OPERATIONS_ERROR; +@@ -1867,7 +1872,7 @@ replica_cleanallruv_thread(void *arg) + /* no agmts, just clean this replica */ + break; + } +- while (agmt_obj){ ++ while (agmt_obj && !slapi_is_shutting_down()){ + agmt = (Repl_Agmt*)object_get_data (agmt_obj); + if(!agmt_is_enabled(agmt) || get_agmt_agreement_type(agmt) == REPLICA_TYPE_WINDOWS){ + agmt_obj = agmtlist_get_next_agreement_for_replica (data->replica, agmt_obj); +@@ -1947,13 +1952,15 @@ replica_cleanallruv_thread(void *arg) + break; + } + /* +- * need to sleep between passes ++ * Need to sleep between passes unless we are shutting down + */ +- cleanruv_log(data->task, CLEANALLRUV_ID, "Replicas have not been cleaned yet, " +- "retrying in %d seconds", interval); +- PR_Lock( notify_lock ); +- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) ); +- PR_Unlock( notify_lock ); ++ if (!slapi_is_shutting_down()){ ++ cleanruv_log(data->task, CLEANALLRUV_ID, "Replicas have not been cleaned yet, " ++ "retrying in %d seconds", interval); ++ PR_Lock( notify_lock ); ++ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) ); ++ PR_Unlock( notify_lock ); ++ } + + if(interval < 14400){ /* 4 hour max */ + interval = interval * 2; +@@ -1964,10 +1971,9 @@ replica_cleanallruv_thread(void *arg) + + done: + /* +- * If the replicas are cleaned, release the rid, and trim the changelog ++ * If the replicas are cleaned, release the rid + */ + if(!aborted){ +- trigger_cl_trimming(data->rid); + delete_cleaned_rid_config(data); + /* make sure all the replicas have been "pre_cleaned" before finishing */ + check_replicas_are_done_cleaning(data); +@@ -1977,7 +1983,7 @@ done: + /* + * Shutdown or abort + */ +- if(!is_task_aborted(data->rid)){ ++ if(!is_task_aborted(data->rid) || slapi_is_shutting_down()){ + cleanruv_log(data->task, CLEANALLRUV_ID,"Server shutting down. Process will resume at server startup"); + } else { + cleanruv_log(data->task, CLEANALLRUV_ID,"Task aborted for rid(%d).",data->rid); +@@ -2212,7 +2218,7 @@ check_agmts_are_caught_up(cleanruv_data *data, char *maxcsn) + not_all_caughtup = 0; + break; + } +- while (agmt_obj){ ++ while (agmt_obj && !slapi_is_shutting_down()){ + agmt = (Repl_Agmt*)object_get_data (agmt_obj); + if(!agmt_is_enabled(agmt) || get_agmt_agreement_type(agmt) == REPLICA_TYPE_WINDOWS){ + agmt_obj = agmtlist_get_next_agreement_for_replica (data->replica, agmt_obj); +@@ -2269,7 +2275,7 @@ check_agmts_are_alive(Replica *replica, ReplicaId rid, Slapi_Task *task) + not_all_alive = 0; + break; + } +- while (agmt_obj){ ++ while (agmt_obj && !slapi_is_shutting_down()){ + agmt = (Repl_Agmt*)object_get_data (agmt_obj); + if(!agmt_is_enabled(agmt) || get_agmt_agreement_type(agmt) == REPLICA_TYPE_WINDOWS){ + agmt_obj = agmtlist_get_next_agreement_for_replica (replica, agmt_obj); +@@ -3034,12 +3040,14 @@ replica_abort_task_thread(void *arg) + break; + } + /* +- * need to sleep between passes ++ * Need to sleep between passes. unless we are shutting down + */ +- cleanruv_log(data->task, ABORT_CLEANALLRUV_ID,"Retrying in %d seconds",interval); +- PR_Lock( notify_lock ); +- PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) ); +- PR_Unlock( notify_lock ); ++ if (!slapi_is_shutting_down()){ ++ cleanruv_log(data->task, ABORT_CLEANALLRUV_ID,"Retrying in %d seconds",interval); ++ PR_Lock( notify_lock ); ++ PR_WaitCondVar( notify_cvar, PR_SecondsToInterval(interval) ); ++ PR_Unlock( notify_lock ); ++ } + + if(interval < 14400){ /* 4 hour max */ + interval = interval * 2; +@@ -3057,7 +3065,7 @@ done: + * Wait for this server to stop its cleanallruv task(which removes the rid from the cleaned list) + */ + cleanruv_log(data->task, ABORT_CLEANALLRUV_ID, "Waiting for CleanAllRUV task to abort..."); +- while(is_cleaned_rid(data->rid)){ ++ while(is_cleaned_rid(data->rid) && !slapi_is_shutting_down()){ + DS_Sleep(PR_SecondsToInterval(1)); + count++; + if(count == 60){ /* it should not take this long */ +-- +1.9.3 + diff --git a/SOURCES/0081-Ticket-47931-memberOf-retrocl-deadlocks.patch b/SOURCES/0081-Ticket-47931-memberOf-retrocl-deadlocks.patch new file mode 100644 index 0000000..c5ce9dc --- /dev/null +++ b/SOURCES/0081-Ticket-47931-memberOf-retrocl-deadlocks.patch @@ -0,0 +1,1317 @@ +From 570d9af5ceed18e600bbf040e48752f923718a01 Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Tue, 4 Aug 2015 12:19:31 -0400 +Subject: [PATCH 81/84] Ticket 47931 - memberOf & retrocl deadlocks + +Bug Description: When concurrently updating multiple backends the + memberOf and retrocl plugins can deadlock on each + other. This is caused by the required retrocl lock, + and the db lock on the changenumber index in the + retrocl db. + +Fix Description: Added scoping to the retrocl that allows subtrees/suffixes + to be included or excluded. Also moved the existing + memberOf scoping outside of its global lock. + + Also improved the memberOf config copying to be consistent + and more efficient. Improved the memberOf scoping attributes + to be multivalued. And, properly valdiated new config + settings in the preop valdiation function, instead of the + "apply config" function. + +https://fedorahosted.org/389/ticket/47931 + +Valgrind: passed + +Reviewed by: nhosoi(Thanks!) + +(cherry picked from commit fd959ac864d6d86d24928bc2c6f097d1a6031ecd) +(cherry picked from commit d8108476d3bedbcc03f6c61bfb3d50e921faaf42) +(cherry picked from commit 9ba3240a177c156e365f22c721432321bb0a679e) +--- + ldap/servers/plugins/memberof/memberof.c | 247 ++++++++++++++--------- + ldap/servers/plugins/memberof/memberof.h | 8 +- + ldap/servers/plugins/memberof/memberof_config.c | 253 +++++++++++++++++------- + ldap/servers/plugins/retrocl/retrocl.c | 183 +++++++++++++++-- + ldap/servers/plugins/retrocl/retrocl.h | 4 + + ldap/servers/plugins/retrocl/retrocl_po.c | 41 +++- + 6 files changed, 536 insertions(+), 200 deletions(-) + +diff --git a/ldap/servers/plugins/memberof/memberof.c b/ldap/servers/plugins/memberof/memberof.c +index 14bad98..1840e34 100644 +--- a/ldap/servers/plugins/memberof/memberof.c ++++ b/ldap/servers/plugins/memberof/memberof.c +@@ -148,7 +148,7 @@ static int memberof_compare(MemberOfConfig *config, const void *a, const void *b + static int memberof_qsort_compare(const void *a, const void *b); + static void memberof_load_array(Slapi_Value **array, Slapi_Attr *attr); + static int memberof_del_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN *sdn); +-static int memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, ++static int memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, MemberOfConfig *config, + char **types, plugin_search_entry_callback callback, void *callback_data); + static int memberof_is_direct_member(MemberOfConfig *config, Slapi_Value *groupdn, + Slapi_Value *memberdn); +@@ -176,7 +176,7 @@ static const char *fetch_attr(Slapi_Entry *e, const char *attrname, + static void memberof_fixup_task_thread(void *arg); + static int memberof_fix_memberof(MemberOfConfig *config, char *dn, char *filter_str); + static int memberof_fix_memberof_callback(Slapi_Entry *e, void *callback_data); +- ++static int memberof_entry_in_scope(MemberOfConfig *config, Slapi_DN *sdn); + + /*** implementation ***/ + +@@ -521,7 +521,8 @@ memberof_get_plugin_area() + int memberof_postop_del(Slapi_PBlock *pb) + { + int ret = SLAPI_PLUGIN_SUCCESS; +- MemberOfConfig configCopy = {0, 0, 0, 0}; ++ MemberOfConfig *mainConfig = NULL; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + Slapi_DN *sdn; + void *caller_id = NULL; + +@@ -541,12 +542,13 @@ int memberof_postop_del(Slapi_PBlock *pb) + struct slapi_entry *e = NULL; + + slapi_pblock_get( pb, SLAPI_ENTRY_PRE_OP, &e ); +- +- /* We need to get the config lock first. Trying to get the +- * config lock after we already hold the op lock can cause +- * a deadlock. */ + memberof_rlock_config(); +- /* copy config so it doesn't change out from under us */ ++ mainConfig = memberof_get_config(); ++ if(!memberof_entry_in_scope(mainConfig, slapi_entry_get_sdn(e))){ ++ /* The entry is not in scope, bail...*/ ++ memberof_unlock_config(); ++ goto bail; ++ } + memberof_copy_config(&configCopy, memberof_get_config()); + memberof_unlock_config(); + +@@ -561,7 +563,6 @@ int memberof_postop_del(Slapi_PBlock *pb) + "memberof_postop_del: error deleting dn (%s) from group. Error (%d)\n", + slapi_sdn_get_dn(sdn),ret); + memberof_unlock(); +- memberof_free_config(&configCopy); + goto bail; + } + +@@ -586,10 +587,10 @@ int memberof_postop_del(Slapi_PBlock *pb) + } + } + memberof_unlock(); ++bail: + memberof_free_config(&configCopy); + } + +-bail: + if(ret){ + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); + ret = SLAPI_PLUGIN_FAILURE; +@@ -623,7 +624,7 @@ memberof_del_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, Slapi_DN * + + groupattrs[0] = config->groupattrs[i]; + +- rc = memberof_call_foreach_dn(pb, sdn, groupattrs, ++ rc = memberof_call_foreach_dn(pb, sdn, config, groupattrs, + memberof_del_dn_type_callback, &data); + } + +@@ -673,6 +674,20 @@ memberof_del_dn_type_callback(Slapi_Entry *e, void *callback_data) + return rc; + } + ++/* Check if the the entry include scope is a child of the sdn */ ++static Slapi_DN* ++memberof_scope_is_child_of_dn(MemberOfConfig *config, Slapi_DN *sdn) ++{ ++ int i = 0; ++ ++ while(config->entryScopes && config->entryScopes[i]){ ++ if(slapi_sdn_issuffix(config->entryScopes[i], sdn)){ ++ return config->entryScopes[i]; ++ } ++ i++; ++ } ++ return NULL; ++} + /* + * Does a callback search of "type=dn" under the db suffix that "dn" is in, + * unless all_backends is set, then we look at all the backends. If "dn" +@@ -681,7 +696,7 @@ memberof_del_dn_type_callback(Slapi_Entry *e, void *callback_data) + */ + int + memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, +- char **types, plugin_search_entry_callback callback, void *callback_data) ++ MemberOfConfig *config, char **types, plugin_search_entry_callback callback, void *callback_data) + { + Slapi_PBlock *search_pb = NULL; + Slapi_DN *base_sdn = NULL; +@@ -689,9 +704,7 @@ memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, + char *escaped_filter_val; + char *filter_str = NULL; + char *cookie = NULL; +- int all_backends = memberof_config_get_all_backends(); +- Slapi_DN *entry_scope = memberof_config_get_entry_scope(); +- Slapi_DN *entry_scope_exclude_subtree = memberof_config_get_entry_scope_exclude_subtree(); ++ int all_backends = config->allBackends; + int types_name_len = 0; + int num_types = 0; + int dn_len = slapi_sdn_get_ndn_len(sdn); +@@ -699,11 +712,7 @@ memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, + int rc = 0; + int i = 0; + +- if (entry_scope && !slapi_sdn_issuffix(sdn, entry_scope)) { +- return (rc); +- } +- +- if (entry_scope_exclude_subtree && slapi_sdn_issuffix(sdn, entry_scope_exclude_subtree)) { ++ if (!memberof_entry_in_scope(config, sdn)) { + return (rc); + } + +@@ -760,6 +769,8 @@ memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, + search_pb = slapi_pblock_new(); + be = slapi_get_first_backend(&cookie); + while(be){ ++ Slapi_DN *scope_sdn = NULL; ++ + if(!all_backends){ + be = slapi_be_select(sdn); + if(be == NULL){ +@@ -775,13 +786,14 @@ memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, + continue; + } + } +- if (entry_scope) { +- if (slapi_sdn_issuffix(base_sdn, entry_scope)) { ++ ++ if (config->entryScopes || config->entryScopeExcludeSubtrees) { ++ if (memberof_entry_in_scope(config, base_sdn)) { + /* do nothing, entry scope is spanning + * multiple suffixes, start at suffix */ +- } else if (slapi_sdn_issuffix(entry_scope, base_sdn)) { ++ } else if ((scope_sdn = memberof_scope_is_child_of_dn(config, base_sdn))) { + /* scope is below suffix, set search base */ +- base_sdn = entry_scope; ++ base_sdn = scope_sdn; + } else if(!all_backends){ + break; + } else { +@@ -799,7 +811,6 @@ memberof_call_foreach_dn(Slapi_PBlock *pb, Slapi_DN *sdn, + break; + } + +- + if(!all_backends){ + break; + } +@@ -824,10 +835,7 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + { + int ret = SLAPI_PLUGIN_SUCCESS; + void *caller_id = NULL; +- Slapi_DN *entry_scope = NULL; +- Slapi_DN *entry_scope_exclude_subtree = memberof_config_get_entry_scope_exclude_subtree(); + +- entry_scope = memberof_config_get_entry_scope(); + slapi_log_error( SLAPI_LOG_TRACE, MEMBEROF_PLUGIN_SUBSYSTEM, + "--> memberof_postop_modrdn\n" ); + +@@ -842,7 +850,7 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + if(memberof_oktodo(pb)) + { + MemberOfConfig *mainConfig = 0; +- MemberOfConfig configCopy = {0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + struct slapi_entry *pre_e = NULL; + struct slapi_entry *post_e = NULL; + Slapi_DN *pre_sdn = 0; +@@ -850,7 +858,6 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + + slapi_pblock_get( pb, SLAPI_ENTRY_PRE_OP, &pre_e ); + slapi_pblock_get( pb, SLAPI_ENTRY_POST_OP, &post_e ); +- + if(pre_e && post_e) + { + pre_sdn = slapi_entry_get_sdn(pre_e); +@@ -863,11 +870,19 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + memberof_copy_config(&configCopy, mainConfig); + memberof_unlock_config(); + ++ /* Need to check both the pre/post entries */ ++ if((pre_sdn && !memberof_entry_in_scope(&configCopy, pre_sdn)) && ++ (post_sdn && !memberof_entry_in_scope(&configCopy, post_sdn))) ++ { ++ /* The entry is not in scope */ ++ goto bail; ++ } ++ + memberof_lock(); + + /* update any downstream members */ + if(pre_sdn && post_sdn && configCopy.group_filter && +- 0 == slapi_filter_test_simple(post_e, configCopy.group_filter)) ++ 0 == slapi_filter_test_simple(post_e, configCopy.group_filter)) + { + int i = 0; + Slapi_Attr *attr = 0; +@@ -879,7 +894,7 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + if(0 == slapi_entry_attr_find(post_e, configCopy.groupattrs[i], &attr)) + { + if((ret = memberof_moddn_attr_list(pb, &configCopy, pre_sdn, +- post_sdn, attr) != 0)) ++ post_sdn, attr) != 0)) + { + slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, + "memberof_postop_modrdn - update failed for (%s), error (%d)\n", +@@ -894,49 +909,49 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + * of other group entries. We need to update any member + * attributes to refer to the new name. */ + if (ret == LDAP_SUCCESS && pre_sdn && post_sdn) { +- if ((entry_scope && !slapi_sdn_issuffix(post_sdn, entry_scope)) || +- (entry_scope_exclude_subtree && slapi_sdn_issuffix(post_sdn, entry_scope_exclude_subtree))) { ++ if (!memberof_entry_in_scope(&configCopy, post_sdn)){ + if((ret = memberof_del_dn_from_groups(pb, &configCopy, pre_sdn))){ + slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, + "memberof_postop_modrdn - delete dn failed for (%s), error (%d)\n", + slapi_sdn_get_dn(pre_sdn), ret); + } + if(ret == LDAP_SUCCESS && pre_e && configCopy.group_filter && +- 0 == slapi_filter_test_simple(pre_e, configCopy.group_filter)) { ++ 0 == slapi_filter_test_simple(pre_e, configCopy.group_filter)) ++ { + /* is the entry of interest as a group? */ +- int i = 0; +- Slapi_Attr *attr = 0; +- +- /* Loop through to find each grouping attribute separately. */ +- for (i = 0; configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { +- if (0 == slapi_entry_attr_find(pre_e, configCopy.groupattrs[i], &attr)) { +- if((ret = memberof_del_attr_list(pb, &configCopy, pre_sdn, attr))){ +- slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modrdn: error deleting attr list - dn (%s). Error (%d)\n", +- slapi_sdn_get_dn(pre_sdn),ret); +- } ++ int i = 0; ++ Slapi_Attr *attr = 0; + ++ /* Loop through to find each grouping attribute separately. */ ++ for (i = 0; configCopy.groupattrs[i] && ret == LDAP_SUCCESS; i++) { ++ if (0 == slapi_entry_attr_find(pre_e, configCopy.groupattrs[i], &attr)) { ++ if((ret = memberof_del_attr_list(pb, &configCopy, pre_sdn, attr))){ ++ slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modrdn: error deleting attr list - dn (%s). Error (%d)\n", ++ slapi_sdn_get_dn(pre_sdn),ret); + } ++ + } +- } ++ } ++ } + if(ret == LDAP_SUCCESS) { +- memberof_del_dn_data del_data = {0, configCopy.memberof_attr}; +- if((ret = memberof_del_dn_type_callback(post_e, &del_data))){ +- slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modrdn - delete dn callback failed for (%s), error (%d)\n", +- slapi_entry_get_dn(post_e), ret); +- } ++ memberof_del_dn_data del_data = {0, configCopy.memberof_attr}; ++ if((ret = memberof_del_dn_type_callback(post_e, &del_data))){ ++ slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, ++ "memberof_postop_modrdn - delete dn callback failed for (%s), error (%d)\n", ++ slapi_entry_get_dn(post_e), ret); + } ++ } + } else { + if((ret = memberof_replace_dn_from_groups(pb, &configCopy, pre_sdn, post_sdn))){ + slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "memberof_postop_modrdn - replace dne failed for (%s), error (%d)\n", ++ "memberof_postop_modrdn - replace dn failed for (%s), error (%d)\n", + slapi_sdn_get_dn(pre_sdn), ret); + } + } + } +- + memberof_unlock(); ++bail: + memberof_free_config(&configCopy); + } + +@@ -978,7 +993,7 @@ memberof_replace_dn_from_groups(Slapi_PBlock *pb, MemberOfConfig *config, + + groupattrs[0] = config->groupattrs[i]; + +- if((ret = memberof_call_foreach_dn(pb, pre_sdn, groupattrs, ++ if((ret = memberof_call_foreach_dn(pb, pre_sdn, config, groupattrs, + memberof_replace_dn_type_callback, + &data))) + { +@@ -1096,12 +1111,11 @@ int memberof_postop_modify(Slapi_PBlock *pb) + goto done; + } + +- +- if(memberof_oktodo(pb) && (sdn = memberof_getsdn(pb))) ++ if(memberof_oktodo(pb)) + { + int config_copied = 0; + MemberOfConfig *mainConfig = 0; +- MemberOfConfig configCopy = {0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + /* get the mod set */ + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); +@@ -1120,19 +1134,22 @@ int memberof_postop_modify(Slapi_PBlock *pb) + * only copy the config the first time it's needed so + * it remains the same for all mods in the operation, + * despite any config changes that may be made. */ +- if (!config_copied) +- { ++ if (!config_copied){ + memberof_rlock_config(); + mainConfig = memberof_get_config(); + + if (memberof_is_grouping_attr(type, mainConfig)) + { + interested = 1; ++ if (!memberof_entry_in_scope(mainConfig, sdn)){ ++ /* Entry is not in scope */ ++ memberof_unlock_config(); ++ goto bail; ++ } + /* copy config so it doesn't change out from under us */ + memberof_copy_config(&configCopy, mainConfig); + config_copied = 1; + } +- + memberof_unlock_config(); + } else { + if (memberof_is_grouping_attr(type, &configCopy)) +@@ -1229,8 +1246,7 @@ int memberof_postop_modify(Slapi_PBlock *pb) + } + + bail: +- if (config_copied) +- { ++ if (config_copied){ + memberof_free_config(&configCopy); + } + +@@ -1276,22 +1292,25 @@ int memberof_postop_add(Slapi_PBlock *pb) + + if(memberof_oktodo(pb) && (sdn = memberof_getsdn(pb))) + { +- MemberOfConfig *mainConfig = 0; +- MemberOfConfig configCopy = {0, 0, 0, 0}; + struct slapi_entry *e = NULL; +- ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++ MemberOfConfig *mainConfig; + slapi_pblock_get( pb, SLAPI_ENTRY_POST_OP, &e ); +- + + /* is the entry of interest? */ + memberof_rlock_config(); + mainConfig = memberof_get_config(); + if(e && mainConfig && mainConfig->group_filter && + 0 == slapi_filter_test_simple(e, mainConfig->group_filter)) ++ + { + interested = 1; +- /* copy config so it doesn't change out from under us */ +- memberof_copy_config(&configCopy, mainConfig); ++ if(!memberof_entry_in_scope(mainConfig, slapi_entry_get_sdn(e))){ ++ /* Entry is not in scope */ ++ memberof_unlock_config(); ++ goto bail; ++ } ++ memberof_copy_config(&configCopy, memberof_get_config()); + } + memberof_unlock_config(); + +@@ -1316,11 +1335,11 @@ int memberof_postop_add(Slapi_PBlock *pb) + } + + memberof_unlock(); +- + memberof_free_config(&configCopy); + } + } + ++bail: + if(ret){ + slapi_pblock_set(pb, SLAPI_RESULT_CODE, &ret); + ret = SLAPI_PLUGIN_FAILURE; +@@ -1358,26 +1377,61 @@ int memberof_oktodo(Slapi_PBlock *pb) + } + + if(slapi_pblock_get(pb, SLAPI_PLUGIN_OPRETURN, &oprc) != 0) +- { ++ { + slapi_log_error( SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, + "memberof_postop_oktodo: could not get parameters\n" ); + ret = -1; + } + +- /* this plugin should only execute if the operation succeeded +- */ +- if(oprc != 0) ++ /* this plugin should only execute if the operation succeeded */ ++ if(oprc != 0) + { + ret = 0; + } +- ++ ++bail: + slapi_log_error( SLAPI_LOG_TRACE, MEMBEROF_PLUGIN_SUBSYSTEM, + "<-- memberof_postop_oktodo\n" ); + +-bail: + return ret; + } + ++/* ++ * Return 1 if the entry is in the scope. ++ * For MODRDN the caller should check both the preop ++ * and postop entries. If we are moving out of, or ++ * into scope, we should process it. ++ */ ++static int ++memberof_entry_in_scope(MemberOfConfig *config, Slapi_DN *sdn) ++{ ++ if (config->entryScopeExcludeSubtrees){ ++ int i = 0; ++ ++ /* check the excludes */ ++ while(config->entryScopeExcludeSubtrees[i]){ ++ if (slapi_sdn_issuffix(sdn, config->entryScopeExcludeSubtrees[i])){ ++ return 0; ++ } ++ i++; ++ } ++ } ++ if (config->entryScopes){ ++ int i = 0; ++ ++ /* check the excludes */ ++ while(config->entryScopes[i]){ ++ if (slapi_sdn_issuffix(sdn, config->entryScopes[i])){ ++ return 1; ++ } ++ i++; ++ } ++ return 0; ++ } ++ ++ return 1; ++} ++ + static Slapi_DN * + memberof_getsdn(Slapi_PBlock *pb) + { +@@ -2045,7 +2099,7 @@ memberof_get_groups_r(MemberOfConfig *config, Slapi_DN *member_sdn, + { + /* Search for any grouping attributes that point to memberdn. + * For each match, add it to the list, recurse and do same search */ +- return memberof_call_foreach_dn(NULL, member_sdn, config->groupattrs, ++ return memberof_call_foreach_dn(NULL, member_sdn, config, config->groupattrs, + memberof_get_groups_callback, data); + } + +@@ -2057,12 +2111,12 @@ int memberof_get_groups_callback(Slapi_Entry *e, void *callback_data) + { + Slapi_DN *group_sdn = slapi_entry_get_sdn(e); + char *group_ndn = slapi_entry_get_ndn(e); +- char *group_dn = slapi_entry_get_dn(e); ++ char *group_dn = slapi_entry_get_dn(e); + Slapi_Value *group_ndn_val = 0; +- Slapi_Value *group_dn_val = 0; ++ Slapi_Value *group_dn_val = 0; + Slapi_ValueSet *groupvals = *((memberof_get_groups_data*)callback_data)->groupvals; +- Slapi_ValueSet *group_norm_vals = *((memberof_get_groups_data*)callback_data)->group_norm_vals; +- Slapi_DN *entry_scope_exclude_subtree = memberof_config_get_entry_scope_exclude_subtree(); ++ Slapi_ValueSet *group_norm_vals = *((memberof_get_groups_data*)callback_data)->group_norm_vals; ++ MemberOfConfig *config = ((memberof_get_groups_data*)callback_data)->config; + int rc = 0; + + if(slapi_is_shutting_down()){ +@@ -2116,18 +2170,19 @@ int memberof_get_groups_callback(Slapi_Entry *e, void *callback_data) + goto bail; + } + +- /* if the group does not belong to an excluded subtree, adds it to the valueset */ +- if (!(entry_scope_exclude_subtree && slapi_sdn_issuffix(group_sdn, entry_scope_exclude_subtree))) { +- /* Push group_dn_val into the valueset. This memory is now owned +- * by the valueset. */ +- group_dn_val = slapi_value_new_string(group_dn); +- slapi_valueset_add_value_ext(groupvals, group_dn_val, SLAPI_VALUE_FLAG_PASSIN); +- slapi_valueset_add_value_ext(group_norm_vals, group_ndn_val, SLAPI_VALUE_FLAG_PASSIN); +- } +- +- /* now recurse to find parent groups of e */ +- memberof_get_groups_r(((memberof_get_groups_data*)callback_data)->config, +- group_sdn, callback_data); ++ /* if the group does not belong to an excluded subtree, adds it to the valueset */ ++ if (memberof_entry_in_scope(config, group_sdn)) { ++ /* Push group_dn_val into the valueset. This memory is now owned ++ * by the valueset. */ ++ group_dn_val = slapi_value_new_string(group_dn); ++ slapi_valueset_add_value_ext(groupvals, group_dn_val, SLAPI_VALUE_FLAG_PASSIN); ++ slapi_valueset_add_value_ext(group_norm_vals, group_ndn_val, SLAPI_VALUE_FLAG_PASSIN); ++ } ++ if(!config->skip_nested || config->fixup_task){ ++ /* now recurse to find parent groups of e */ ++ memberof_get_groups_r(((memberof_get_groups_data*)callback_data)->config, ++ group_sdn, callback_data); ++ } + + bail: + return rc; +@@ -2218,8 +2273,8 @@ memberof_test_membership(Slapi_PBlock *pb, MemberOfConfig *config, + { + char *attrs[2] = {config->memberof_attr, 0}; + +- return memberof_call_foreach_dn(pb, group_sdn, attrs, +- memberof_test_membership_callback , config); ++ return memberof_call_foreach_dn(pb, group_sdn, config, attrs, ++ memberof_test_membership_callback, config); + } + + /* +diff --git a/ldap/servers/plugins/memberof/memberof.h b/ldap/servers/plugins/memberof/memberof.h +index 59029d7..4516387 100644 +--- a/ldap/servers/plugins/memberof/memberof.h ++++ b/ldap/servers/plugins/memberof/memberof.h +@@ -80,8 +80,10 @@ typedef struct memberofconfig { + char **groupattrs; + char *memberof_attr; + int allBackends; +- Slapi_DN *entryScope; +- Slapi_DN *entryScopeExcludeSubtree; ++ Slapi_DN **entryScopes; ++ int entryScopeCount; ++ Slapi_DN **entryScopeExcludeSubtrees; ++ int entryExcludeScopeCount; + Slapi_Filter *group_filter; + Slapi_Attr **group_slapiattrs; + } MemberOfConfig; +@@ -100,8 +102,6 @@ void memberof_rlock_config(); + void memberof_wlock_config(); + void memberof_unlock_config(); + int memberof_config_get_all_backends(); +-Slapi_DN * memberof_config_get_entry_scope(); +-Slapi_DN * memberof_config_get_entry_scope_exclude_subtree(); + void memberof_set_config_area(Slapi_DN *sdn); + Slapi_DN * memberof_get_config_area(); + void memberof_set_plugin_area(Slapi_DN *sdn); +diff --git a/ldap/servers/plugins/memberof/memberof_config.c b/ldap/servers/plugins/memberof/memberof_config.c +index eb83bd0..5fc1314 100644 +--- a/ldap/servers/plugins/memberof/memberof_config.c ++++ b/ldap/servers/plugins/memberof/memberof_config.c +@@ -77,7 +77,7 @@ static int memberof_search (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_En + /* This is the main configuration which is updated from dse.ldif. The + * config will be copied when it is used by the plug-in to prevent it + * being changed out from under a running memberOf operation. */ +-static MemberOfConfig theConfig = {NULL, NULL,0, NULL, NULL, NULL, NULL}; ++static MemberOfConfig theConfig = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + static Slapi_RWLock *memberof_config_lock = 0; + static int inited = 0; + +@@ -89,6 +89,19 @@ static int dont_allow_that(Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Ent + return SLAPI_DSE_CALLBACK_ERROR; + } + ++static void ++memberof_free_scope(Slapi_DN **scopes, int *count) ++{ ++ int i = 0; ++ ++ while(scopes && scopes[i]){ ++ slapi_sdn_free(&scopes[i]); ++ i++; ++ } ++ slapi_ch_free((void**)&scopes); ++ *count = 0; ++} ++ + /* + * memberof_config() + * +@@ -184,16 +197,22 @@ memberof_release_config() + * + * Validate the pending changes in the e entry. + */ +-static int ++int + memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* e, + int *returncode, char *returntext, void *arg) + { + Slapi_Attr *memberof_attr = NULL; + Slapi_Attr *group_attr = NULL; + Slapi_DN *config_sdn = NULL; ++ Slapi_DN **include_dn = NULL; ++ Slapi_DN **exclude_dn = NULL; + char *syntaxoid = NULL; + char *config_dn = NULL; ++ char *skip_nested = NULL; ++ char **entry_scopes = NULL; ++ char **entry_exclude_scopes = NULL; + int not_dn_syntax = 0; ++ int num_vals = 0; + + *returncode = LDAP_UNWILLING_TO_PERFORM; /* be pessimistic */ + +@@ -301,8 +320,112 @@ memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr + *returncode = LDAP_UNWILLING_TO_PERFORM; + } + } ++ /* ++ * Check the entry scopes ++ */ ++ entry_scopes = slapi_entry_attr_get_charray_ext(e, MEMBEROF_ENTRY_SCOPE_ATTR, &num_vals); ++ if(entry_scopes){ ++ int i = 0; ++ ++ /* Validate the syntax before we create our DN array */ ++ for (i = 0;i < num_vals; i++){ ++ if(slapi_dn_syntax_check(pb, entry_scopes[i], 1)){ ++ /* invalid dn syntax */ ++ PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, ++ "%s: Invalid DN (%s) for include suffix.", ++ MEMBEROF_PLUGIN_SUBSYSTEM, entry_scopes[i]); ++ slapi_ch_array_free(entry_scopes); ++ theConfig.entryScopeCount = 0; ++ *returncode = LDAP_UNWILLING_TO_PERFORM; ++ goto done; ++ } ++ } ++ /* Now create our SDN array for conflict checking */ ++ include_dn = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *), num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ include_dn[i] = slapi_sdn_new_dn_passin(entry_scopes[i]); ++ } ++ } ++ /* ++ * Check and process the entry exclude scopes ++ */ ++ entry_exclude_scopes = ++ slapi_entry_attr_get_charray_ext(e, MEMBEROF_ENTRY_SCOPE_EXCLUDE_SUBTREE, &num_vals); ++ if(entry_exclude_scopes){ ++ int i = 0; ++ ++ /* Validate the syntax before we create our DN array */ ++ for (i = 0;i < num_vals; i++){ ++ if(slapi_dn_syntax_check(pb, entry_exclude_scopes[i], 1)){ ++ /* invalid dn syntax */ ++ PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, ++ "%s: Invalid DN (%s) for exclude suffix.", ++ MEMBEROF_PLUGIN_SUBSYSTEM, entry_scopes[i]); ++ slapi_ch_array_free(entry_exclude_scopes); ++ *returncode = LDAP_UNWILLING_TO_PERFORM; ++ goto done; ++ } ++ } ++ /* Now create our SDN array for conflict checking */ ++ exclude_dn = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ exclude_dn[i] = slapi_sdn_new_dn_passin(entry_exclude_scopes[i]); ++ } ++ } ++ /* ++ * Need to do conflict checking ++ */ ++ if(include_dn && exclude_dn){ ++ /* ++ * Make sure we haven't mixed the same suffix, and there are no ++ * conflicts between the includes and excludes ++ */ ++ int i = 0; ++ ++ while(include_dn[i]){ ++ int x = 0; ++ while(exclude_dn[x]){ ++ if(slapi_sdn_compare(include_dn[i], exclude_dn[x] ) == 0) ++ { ++ /* we have a conflict */ ++ PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, ++ "%s: include suffix (%s) is also listed as an exclude suffix list", ++ MEMBEROF_PLUGIN_SUBSYSTEM, slapi_sdn_get_dn(include_dn[i])); ++ *returncode = LDAP_UNWILLING_TO_PERFORM; ++ goto done; ++ } ++ x++; ++ } ++ i++; ++ } ++ ++ /* Check for parent/child conflicts */ ++ i = 0; ++ while(include_dn[i]){ ++ int x = 0; ++ while(exclude_dn[x]){ ++ if(slapi_sdn_issuffix(include_dn[i], exclude_dn[x])) ++ { ++ /* we have a conflict */ ++ PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, ++ "%s: include suffix (%s) is a child of the exclude suffix(%s)", ++ MEMBEROF_PLUGIN_SUBSYSTEM, ++ slapi_sdn_get_dn(include_dn[i]), ++ slapi_sdn_get_dn(exclude_dn[i])); ++ *returncode = LDAP_UNWILLING_TO_PERFORM; ++ goto done; ++ } ++ x++; ++ } ++ i++; ++ } ++ } + + done: ++ memberof_free_scope(exclude_dn, &num_vals); ++ memberof_free_scope(include_dn, &num_vals); ++ slapi_ch_free((void**)&entry_scopes); ++ slapi_ch_free((void**)&entry_exclude_scopes); + slapi_sdn_free(&config_sdn); + slapi_ch_free_string(&config_dn); + +@@ -316,7 +439,6 @@ done: + } + } + +- + /* + * memberof_apply_config() + * +@@ -335,9 +457,11 @@ memberof_apply_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* + int num_groupattrs = 0; + int groupattr_name_len = 0; + char *allBackends = NULL; +- char *entryScope = NULL; +- char *entryScopeExcludeSubtree = NULL; ++ char **entryScopes = NULL; ++ char **entryScopeExcludeSubtrees = NULL; + char *sharedcfg = NULL; ++ char *skip_nested = NULL; ++ int num_vals = 0; + + *returncode = LDAP_SUCCESS; + +@@ -369,8 +493,7 @@ memberof_apply_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* + groupattrs = slapi_entry_attr_get_charray(e, MEMBEROF_GROUP_ATTR); + memberof_attr = slapi_entry_attr_get_charptr(e, MEMBEROF_ATTR); + allBackends = slapi_entry_attr_get_charptr(e, MEMBEROF_BACKEND_ATTR); +- entryScope = slapi_entry_attr_get_charptr(e, MEMBEROF_ENTRY_SCOPE_ATTR); +- entryScopeExcludeSubtree = slapi_entry_attr_get_charptr(e, MEMBEROF_ENTRY_SCOPE_EXCLUDE_SUBTREE); ++ skip_nested = slapi_entry_attr_get_charptr(e, MEMBEROF_SKIP_NESTED_ATTR); + + /* + * We want to be sure we don't change the config in the middle of +@@ -487,49 +610,39 @@ memberof_apply_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* + theConfig.allBackends = 0; + } + +- slapi_sdn_free(&theConfig.entryScope); +- if (entryScope) +- { +- if (slapi_dn_syntax_check(NULL, entryScope, 1) == 1) { +- slapi_log_error(SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "Error: Ignoring invalid DN used as plugin entry scope: [%s]\n", +- entryScope); +- theConfig.entryScope = NULL; +- slapi_ch_free_string(&entryScope); +- } else { +- theConfig.entryScope = slapi_sdn_new_dn_passin(entryScope); ++ /* ++ * Check and process the entry scopes ++ */ ++ memberof_free_scope(theConfig.entryScopes, &theConfig.entryScopeCount); ++ entryScopes = slapi_entry_attr_get_charray_ext(e, MEMBEROF_ENTRY_SCOPE_ATTR, &num_vals); ++ if(entryScopes){ ++ int i = 0; ++ ++ /* Validation has already been performed in preop, just build the DN's */ ++ theConfig.entryScopes = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *), num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ theConfig.entryScopes[i] = slapi_sdn_new_dn_passin(entryScopes[i]); + } +- } else { +- theConfig.entryScope = NULL; ++ theConfig.entryScopeCount = num_vals; /* shortcut for config copy */ + } +- +- slapi_sdn_free(&theConfig.entryScopeExcludeSubtree); +- if (entryScopeExcludeSubtree) +- { +- if (theConfig.entryScope == NULL) { +- slapi_log_error(SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "Error: Ignoring ExcludeSubtree (%s) because entryScope is not define\n", +- entryScopeExcludeSubtree); +- theConfig.entryScopeExcludeSubtree = NULL; +- slapi_ch_free_string(&entryScopeExcludeSubtree); +- } else if (slapi_dn_syntax_check(NULL, entryScopeExcludeSubtree, 1) == 1) { +- slapi_log_error(SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "Error: Ignoring invalid DN used as plugin entry exclude subtree: [%s]\n", +- entryScopeExcludeSubtree); +- theConfig.entryScopeExcludeSubtree = NULL; +- slapi_ch_free_string(&entryScopeExcludeSubtree); +- } else { +- theConfig.entryScopeExcludeSubtree = slapi_sdn_new_dn_passin(entryScopeExcludeSubtree); ++ /* ++ * Check and process the entry exclude scopes ++ */ ++ memberof_free_scope(theConfig.entryScopeExcludeSubtrees, ++ &theConfig.entryExcludeScopeCount); ++ entryScopeExcludeSubtrees = ++ slapi_entry_attr_get_charray_ext(e, MEMBEROF_ENTRY_SCOPE_EXCLUDE_SUBTREE, &num_vals); ++ if(entryScopeExcludeSubtrees){ ++ int i = 0; ++ ++ /* Validation has already been performed in preop, just build the DN's */ ++ theConfig.entryScopeExcludeSubtrees = ++ (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ theConfig.entryScopeExcludeSubtrees[i] = ++ slapi_sdn_new_dn_passin(entryScopeExcludeSubtrees[i]); + } +- } else { +- theConfig.entryScopeExcludeSubtree = NULL; +- } +- if (theConfig.entryScopeExcludeSubtree && theConfig.entryScope && !slapi_sdn_issuffix(theConfig.entryScopeExcludeSubtree, theConfig.entryScope)) { +- slapi_log_error(SLAPI_LOG_FATAL, MEMBEROF_PLUGIN_SUBSYSTEM, +- "Error: Ignoring ExcludeSubtree (%s) that is out of the scope (%s)\n", +- slapi_sdn_get_dn(theConfig.entryScopeExcludeSubtree), +- slapi_sdn_get_dn(theConfig.entryScope)); +- slapi_sdn_free(&theConfig.entryScopeExcludeSubtree); ++ theConfig.entryExcludeScopeCount = num_vals; /* shortcut for config copy */ + } + + /* release the lock */ +@@ -542,6 +655,9 @@ done: + slapi_ch_free_string(&sharedcfg); + slapi_ch_free_string(&memberof_attr); + slapi_ch_free_string(&allBackends); ++ slapi_ch_free_string(&skip_nested); ++ slapi_ch_free((void **)&entryScopes); ++ slapi_ch_free((void **)&entryScopeExcludeSubtrees); + + if (*returncode != LDAP_SUCCESS) + { +@@ -618,6 +734,23 @@ memberof_copy_config(MemberOfConfig *dest, MemberOfConfig *src) + { + dest->allBackends = src->allBackends; + } ++ ++ if(src->entryScopes){ ++ int num_vals = 0; ++ ++ dest->entryScopes = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),src->entryScopeCount+1); ++ for(num_vals = 0; src->entryScopes[num_vals]; num_vals++){ ++ dest->entryScopes[num_vals] = slapi_sdn_dup(src->entryScopes[num_vals]); ++ } ++ } ++ if(src->entryScopeExcludeSubtrees){ ++ int num_vals = 0; ++ ++ dest->entryScopeExcludeSubtrees = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),src->entryExcludeScopeCount+1); ++ for(num_vals = 0; src->entryScopes[num_vals]; num_vals++){ ++ dest->entryScopeExcludeSubtrees[num_vals] = slapi_sdn_dup(src->entryScopeExcludeSubtrees[num_vals]); ++ } ++ } + } + } + +@@ -643,6 +776,8 @@ memberof_free_config(MemberOfConfig *config) + slapi_ch_free((void **)&config->group_slapiattrs); + + slapi_ch_free_string(&config->memberof_attr); ++ memberof_free_scope(config->entryScopes, &config->entryScopeCount); ++ memberof_free_scope(config->entryScopeExcludeSubtrees, &config->entryExcludeScopeCount); + } + } + +@@ -708,30 +843,6 @@ memberof_config_get_all_backends() + return all_backends; + } + +-Slapi_DN * +-memberof_config_get_entry_scope() +-{ +- Slapi_DN *entry_scope; +- +- slapi_rwlock_rdlock(memberof_config_lock); +- entry_scope = theConfig.entryScope; +- slapi_rwlock_unlock(memberof_config_lock); +- +- return entry_scope; +-} +- +-Slapi_DN * +-memberof_config_get_entry_scope_exclude_subtree() +-{ +- Slapi_DN *entry_exclude_subtree; +- +- slapi_rwlock_rdlock(memberof_config_lock); +- entry_exclude_subtree = theConfig.entryScopeExcludeSubtree; +- slapi_rwlock_unlock(memberof_config_lock); +- +- return entry_exclude_subtree; +-} +- + /* + * Check if we are modifying the config, or changing the shared config entry + */ +diff --git a/ldap/servers/plugins/retrocl/retrocl.c b/ldap/servers/plugins/retrocl/retrocl.c +index 8a0f350..7679c29 100644 +--- a/ldap/servers/plugins/retrocl/retrocl.c ++++ b/ldap/servers/plugins/retrocl/retrocl.c +@@ -82,6 +82,9 @@ char **retrocl_attributes = NULL; + char **retrocl_aliases = NULL; + int retrocl_log_deleted = 0; + ++static Slapi_DN **retrocl_includes = NULL; ++static Slapi_DN **retrocl_excludes = NULL; ++ + /* ----------------------------- Retrocl Plugin */ + + static Slapi_PluginDesc retrocldesc = {"retrocl", VENDOR, DS_PACKAGE_VERSION, "Retrocl Plugin"}; +@@ -386,6 +389,8 @@ static int retrocl_start (Slapi_PBlock *pb) + int rc = 0; + Slapi_Entry *e = NULL; + char **values = NULL; ++ int num_vals = 0; ++ int i = 0; + + retrocl_rootdse_init(pb); + +@@ -406,6 +411,87 @@ static int retrocl_start (Slapi_PBlock *pb) + return -1; + } + ++ /* Get the exclude suffixes */ ++ values = slapi_entry_attr_get_charray_ext(e, CONFIG_CHANGELOG_EXCLUDE_SUFFIX, &num_vals); ++ if(values){ ++ /* Validate the syntax before we create our DN array */ ++ for (i = 0;i < num_vals; i++){ ++ if(slapi_dn_syntax_check(pb, values[i], 1)){ ++ /* invalid dn syntax */ ++ slapi_log_error(SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, ++ "Invalid DN (%s) for exclude suffix.\n", values[i] ); ++ slapi_ch_array_free(values); ++ return -1; ++ } ++ } ++ /* Now create our SDN array */ ++ retrocl_excludes = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ retrocl_excludes[i] = slapi_sdn_new_dn_byval(values[i]); ++ } ++ slapi_ch_array_free(values); ++ } ++ /* Get the include suffixes */ ++ values = slapi_entry_attr_get_charray_ext(e, CONFIG_CHANGELOG_INCLUDE_SUFFIX, &num_vals); ++ if(values){ ++ for (i = 0;i < num_vals; i++){ ++ /* Validate the syntax before we create our DN array */ ++ if(slapi_dn_syntax_check(pb, values[i], 1)){ ++ /* invalid dn syntax */ ++ slapi_log_error(SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, ++ "Invalid DN (%s) for include suffix.\n", values[i] ); ++ slapi_ch_array_free(values); ++ return -1; ++ } ++ } ++ /* Now create our SDN array */ ++ retrocl_includes = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),num_vals+1); ++ for (i = 0;i < num_vals; i++){ ++ retrocl_includes[i] = slapi_sdn_new_dn_byval(values[i]); ++ } ++ slapi_ch_array_free(values); ++ } ++ if(retrocl_includes && retrocl_excludes){ ++ /* ++ * Make sure we haven't mixed the same suffix, and there are no ++ * conflicts between the includes and excludes ++ */ ++ int i = 0; ++ ++ while(retrocl_includes[i]){ ++ int x = 0; ++ while(retrocl_excludes[x]){ ++ if(slapi_sdn_compare(retrocl_includes[i], retrocl_excludes[x] ) == 0){ ++ /* we have a conflict */ ++ slapi_log_error(SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, ++ "include suffix (%s) is also listed in exclude suffix list\n", ++ slapi_sdn_get_dn(retrocl_includes[i])); ++ return -1; ++ } ++ x++; ++ } ++ i++; ++ } ++ ++ /* Check for parent/child conflicts */ ++ i = 0; ++ while(retrocl_includes[i]){ ++ int x = 0; ++ while(retrocl_excludes[x]){ ++ if(slapi_sdn_issuffix(retrocl_includes[i], retrocl_excludes[x])){ ++ /* we have a conflict */ ++ slapi_log_error(SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, ++ "include suffix (%s) is a child of the exclude suffix(%s)\n", ++ slapi_sdn_get_dn(retrocl_includes[i]), ++ slapi_sdn_get_dn(retrocl_excludes[i])); ++ return -1; ++ } ++ x++; ++ } ++ i++; ++ } ++ } ++ + values = slapi_entry_attr_get_charray(e, "nsslapd-attribute"); + if (values != NULL) { + int n = 0; +@@ -471,6 +557,49 @@ static int retrocl_start (Slapi_PBlock *pb) + } + + /* ++ * Check if an entry is in the configured scope. ++ * Return 1 if entry is in the scope, or 0 otherwise. ++ * For MODRDN the caller should check both the preop ++ * and postop entries. If we are moving out of, or ++ * into scope, we should record it. ++ */ ++int ++retrocl_entry_in_scope(Slapi_Entry *e) ++{ ++ Slapi_DN *sdn = slapi_entry_get_sdn(e); ++ ++ if (e == NULL){ ++ return 1; ++ } ++ ++ if (retrocl_excludes){ ++ int i = 0; ++ ++ /* check the excludes */ ++ while(retrocl_excludes[i]){ ++ if (slapi_sdn_issuffix(sdn, retrocl_excludes[i])){ ++ return 0; ++ } ++ i++; ++ } ++ } ++ if (retrocl_includes){ ++ int i = 0; ++ ++ /* check the excludes */ ++ while(retrocl_includes[i]){ ++ if (slapi_sdn_issuffix(sdn, retrocl_includes[i])){ ++ return 1; ++ } ++ i++; ++ } ++ return 0; ++ } ++ ++ return 1; ++} ++ ++/* + * Function: retrocl_stop + * + * Returns: 0 +@@ -483,26 +612,40 @@ static int retrocl_start (Slapi_PBlock *pb) + + static int retrocl_stop (Slapi_PBlock *pb) + { +- int rc = 0; +- +- slapi_ch_array_free(retrocl_attributes); +- retrocl_attributes = NULL; +- slapi_ch_array_free(retrocl_aliases); +- retrocl_aliases = NULL; +- +- retrocl_stop_trimming(); +- retrocl_be_changelog = NULL; +- retrocl_forget_changenumbers(); +- PR_DestroyLock(retrocl_internal_lock); +- retrocl_internal_lock = NULL; +- slapi_destroy_rwlock(retrocl_cn_lock); +- retrocl_cn_lock = NULL; +- legacy_initialised = 0; +- +- slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, "", +- LDAP_SCOPE_BASE,"(objectclass=*)", retrocl_rootdse_search); +- +- return rc; ++ int rc = 0; ++ int i = 0; ++ ++ slapi_ch_array_free(retrocl_attributes); ++ retrocl_attributes = NULL; ++ slapi_ch_array_free(retrocl_aliases); ++ retrocl_aliases = NULL; ++ ++ while(retrocl_excludes && retrocl_excludes[i]){ ++ slapi_sdn_free(&retrocl_excludes[i]); ++ i++; ++ } ++ slapi_ch_free((void**)&retrocl_excludes); ++ i = 0; ++ ++ while(retrocl_includes && retrocl_includes[i]){ ++ slapi_sdn_free(&retrocl_includes[i]); ++ i++; ++ } ++ slapi_ch_free((void**)&retrocl_includes); ++ ++ retrocl_stop_trimming(); ++ retrocl_be_changelog = NULL; ++ retrocl_forget_changenumbers(); ++ PR_DestroyLock(retrocl_internal_lock); ++ retrocl_internal_lock = NULL; ++ slapi_destroy_rwlock(retrocl_cn_lock); ++ retrocl_cn_lock = NULL; ++ legacy_initialised = 0; ++ ++ slapi_config_remove_callback(SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, "", ++ LDAP_SCOPE_BASE,"(objectclass=*)", retrocl_rootdse_search); ++ ++ return rc; + } + + /* +diff --git a/ldap/servers/plugins/retrocl/retrocl.h b/ldap/servers/plugins/retrocl/retrocl.h +index 27f55dc..fe8bf84 100644 +--- a/ldap/servers/plugins/retrocl/retrocl.h ++++ b/ldap/servers/plugins/retrocl/retrocl.h +@@ -96,6 +96,8 @@ typedef struct _cnumRet { + /* was originally changelogmaximumage */ + #define CONFIG_CHANGELOG_MAXAGE_ATTRIBUTE "nsslapd-changelogmaxage" + #define CONFIG_CHANGELOG_DIRECTORY_ATTRIBUTE "nsslapd-changelogdir" ++#define CONFIG_CHANGELOG_INCLUDE_SUFFIX "nsslapd-include-suffix" ++#define CONFIG_CHANGELOG_EXCLUDE_SUFFIX "nsslapd-exclude-suffix" + + #define RETROCL_CHANGELOG_DN "cn=changelog" + #define RETROCL_MAPPINGTREE_DN "cn=\"cn=changelog\",cn=mapping tree,cn=config" +@@ -169,4 +171,6 @@ extern void retrocl_init_trimming(void); + extern void retrocl_stop_trimming(void); + extern char *retrocl_get_config_str(const char *attrt); + ++int retrocl_entry_in_scope(Slapi_Entry *e); ++ + #endif /* _H_RETROCL */ +diff --git a/ldap/servers/plugins/retrocl/retrocl_po.c b/ldap/servers/plugins/retrocl/retrocl_po.c +index 3f8af81..8010db0 100644 +--- a/ldap/servers/plugins/retrocl/retrocl_po.c ++++ b/ldap/servers/plugins/retrocl/retrocl_po.c +@@ -169,6 +169,7 @@ write_replog_db( + int flag, + time_t curtime, + Slapi_Entry *log_e, ++ Slapi_Entry *post_entry, + const char *newrdn, + LDAPMod **modrdn_mods, + const char *newsuperior +@@ -185,11 +186,26 @@ write_replog_db( + int err = 0; + int ret = LDAP_SUCCESS; + int i; ++ int mark = 0; + + if (!dn) { + slapi_log_error( SLAPI_LOG_PLUGIN, RETROCL_PLUGIN_NAME, "write_replog_db: NULL dn\n"); + return ret; + } ++ mark = (post_entry && retrocl_entry_in_scope(post_entry)); ++ slapi_log_error( SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, "post in scope (%d)\n",mark); ++ ++ if (post_entry){ ++ if(!retrocl_entry_in_scope(log_e) && !retrocl_entry_in_scope(post_entry)){ ++ /* modrdn: entry not in scope, just return... */ ++ return ret; ++ } ++ } else { ++ if(!retrocl_entry_in_scope(log_e)){ ++ /* entry not in scope, just return... */ ++ return ret; ++ } ++ } + + PR_Lock(retrocl_internal_lock); + changenum = retrocl_assign_changenumber(); +@@ -348,7 +364,7 @@ write_replog_db( + break; + + case OP_DELETE: +- if (log_e) { ++ if (retrocl_log_deleted) { + /* we have to log the full entry */ + if ( entry2reple( e, log_e, OP_DELETE ) != 0 ) { + err = SLAPI_PLUGIN_FAILURE; +@@ -588,7 +604,8 @@ int retrocl_postob (Slapi_PBlock *pb, int optype) + char *dn; + LDAPMod **log_m = NULL; + int flag = 0; +- Slapi_Entry *te = NULL; ++ Slapi_Entry *entry = NULL; ++ Slapi_Entry *post_entry = NULL; + Slapi_Operation *op = NULL; + LDAPMod **modrdn_mods = NULL; + char *newrdn = NULL; +@@ -649,7 +666,12 @@ int retrocl_postob (Slapi_PBlock *pb, int optype) + LDAPDebug0Args(LDAP_DEBUG_TRACE,"not applying change for nsTombstone entries\n"); + return SLAPI_PLUGIN_SUCCESS; + } +- ++ /* ++ * Start by grabbing the preop entry, ADD will replace it as needed. Getting the entry ++ * allows up to perform scoping in write_replog_db() for all op types. ++ */ ++ (void)slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &entry); ++ + switch ( optype ) { + case OP_MODIFY: + (void)slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &log_m ); +@@ -659,14 +681,14 @@ int retrocl_postob (Slapi_PBlock *pb, int optype) + * For adds, we want the unnormalized dn, so we can preserve + * spacing, case, when replicating it. + */ +- (void)slapi_pblock_get( pb, SLAPI_ADD_ENTRY, &te ); +- if ( NULL != te ) { +- dn = slapi_entry_get_dn( te ); ++ (void)slapi_pblock_get( pb, SLAPI_ADD_ENTRY, &entry ); ++ if ( NULL != entry ) { ++ dn = slapi_entry_get_dn( entry ); + } + break; + case OP_DELETE: + if (retrocl_log_deleted) +- (void)slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &te); ++ (void)slapi_pblock_get(pb, SLAPI_ENTRY_PRE_OP, &entry); + break; + case OP_MODRDN: + /* newrdn is used just for logging; no need to be normalized */ +@@ -674,13 +696,14 @@ int retrocl_postob (Slapi_PBlock *pb, int optype) + (void)slapi_pblock_get( pb, SLAPI_MODRDN_DELOLDRDN, &flag ); + (void)slapi_pblock_get( pb, SLAPI_MODIFY_MODS, &modrdn_mods ); + (void)slapi_pblock_get( pb, SLAPI_MODRDN_NEWSUPERIOR_SDN, &newsuperior ); ++ (void)slapi_pblock_get(pb, SLAPI_ENTRY_POST_OP, &post_entry); + break; + } + + /* check if we should log change to retro changelog, and + * if so, do it here */ +- if((rc = write_replog_db( pb, optype, dn, log_m, flag, curtime, te, +- newrdn, modrdn_mods, slapi_sdn_get_dn(newsuperior) ))) ++ if((rc = write_replog_db( pb, optype, dn, log_m, flag, curtime, entry, ++ post_entry, newrdn, modrdn_mods, slapi_sdn_get_dn(newsuperior) ))) + { + slapi_log_error(SLAPI_LOG_FATAL, "retrocl-plugin", + "retrocl_postob: operation failure [%d]\n", rc); +-- +1.9.3 + diff --git a/SOURCES/0082-Ticket-47931-Fix-coverity-issues.patch b/SOURCES/0082-Ticket-47931-Fix-coverity-issues.patch new file mode 100644 index 0000000..eeaab2e --- /dev/null +++ b/SOURCES/0082-Ticket-47931-Fix-coverity-issues.patch @@ -0,0 +1,58 @@ +From 476ffd9a01ac5bb55d32edad64c68eebd8773861 Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Mon, 10 Aug 2015 10:42:40 -0400 +Subject: [PATCH 82/84] Ticket 47931 - Fix coverity issues + +Description: Fix coverity issues in memberof_config.c + + 13316 - double free + 13315 - Dereference after null check + 13314 - Dereference after null check + 13313 - copy/paste error + +https://fedorahosted.org/389/ticket/47931 + +Reviewed by: rmeggins(Thanks!) + +(cherry picked from commit 5daea973e4526584ee41d7b9f4b1b4993b4de6f1) +(cherry picked from commit 9a0047ef75f6dbeb1980ac77fab5d62865c77e6a) +(cherry picked from commit a389bc3bafccb1f7bd9917a734230680e382af91) +--- + ldap/servers/plugins/memberof/memberof_config.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/ldap/servers/plugins/memberof/memberof_config.c b/ldap/servers/plugins/memberof/memberof_config.c +index 5fc1314..c335cab 100644 +--- a/ldap/servers/plugins/memberof/memberof_config.c ++++ b/ldap/servers/plugins/memberof/memberof_config.c +@@ -335,6 +335,7 @@ memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr + "%s: Invalid DN (%s) for include suffix.", + MEMBEROF_PLUGIN_SUBSYSTEM, entry_scopes[i]); + slapi_ch_array_free(entry_scopes); ++ entry_scopes = NULL; + theConfig.entryScopeCount = 0; + *returncode = LDAP_UNWILLING_TO_PERFORM; + goto done; +@@ -360,8 +361,9 @@ memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr + /* invalid dn syntax */ + PR_snprintf(returntext, SLAPI_DSE_RETURNTEXT_SIZE, + "%s: Invalid DN (%s) for exclude suffix.", +- MEMBEROF_PLUGIN_SUBSYSTEM, entry_scopes[i]); ++ MEMBEROF_PLUGIN_SUBSYSTEM, entry_exclude_scopes[i]); + slapi_ch_array_free(entry_exclude_scopes); ++ entry_exclude_scopes = NULL; + *returncode = LDAP_UNWILLING_TO_PERFORM; + goto done; + } +@@ -747,7 +749,7 @@ memberof_copy_config(MemberOfConfig *dest, MemberOfConfig *src) + int num_vals = 0; + + dest->entryScopeExcludeSubtrees = (Slapi_DN **)slapi_ch_calloc(sizeof(Slapi_DN *),src->entryExcludeScopeCount+1); +- for(num_vals = 0; src->entryScopes[num_vals]; num_vals++){ ++ for(num_vals = 0; src->entryScopeExcludeSubtrees[num_vals]; num_vals++){ + dest->entryScopeExcludeSubtrees[num_vals] = slapi_sdn_dup(src->entryScopeExcludeSubtrees[num_vals]); + } + } +-- +1.9.3 + diff --git a/SOURCES/0083-Ticket-47831-remove-debug-logging-from-retro-cl.patch b/SOURCES/0083-Ticket-47831-remove-debug-logging-from-retro-cl.patch new file mode 100644 index 0000000..c426060 --- /dev/null +++ b/SOURCES/0083-Ticket-47831-remove-debug-logging-from-retro-cl.patch @@ -0,0 +1,41 @@ +From cbab8a3051994db662348a3c9a1a19d56a8545ca Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Wed, 19 Aug 2015 10:03:50 -0400 +Subject: [PATCH 83/84] Ticket 47831 - remove debug logging from retro cl + +Description: Instrumented debug logging was accidentally left in the source. + This logging is being removed. + +https://fedorahosted.org/389/ticket/47931 + +Reviewed by: mreynolds + +(cherry picked from commit db7153f89bf3dda935e6ef4f175697bda32fe720) +(cherry picked from commit 1781280f133c4877f83949400294641a558f5406) +(cherry picked from commit 0e44c819b72dfad40a7f9eea6067f6060fa9c35b) +--- + ldap/servers/plugins/retrocl/retrocl_po.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/ldap/servers/plugins/retrocl/retrocl_po.c b/ldap/servers/plugins/retrocl/retrocl_po.c +index 8010db0..d0cc5e8 100644 +--- a/ldap/servers/plugins/retrocl/retrocl_po.c ++++ b/ldap/servers/plugins/retrocl/retrocl_po.c +@@ -186,14 +186,11 @@ write_replog_db( + int err = 0; + int ret = LDAP_SUCCESS; + int i; +- int mark = 0; + + if (!dn) { + slapi_log_error( SLAPI_LOG_PLUGIN, RETROCL_PLUGIN_NAME, "write_replog_db: NULL dn\n"); + return ret; + } +- mark = (post_entry && retrocl_entry_in_scope(post_entry)); +- slapi_log_error( SLAPI_LOG_FATAL, RETROCL_PLUGIN_NAME, "post in scope (%d)\n",mark); + + if (post_entry){ + if(!retrocl_entry_in_scope(log_e) && !retrocl_entry_in_scope(post_entry)){ +-- +1.9.3 + diff --git a/SOURCES/0084-Ticket-47831-remove-debug-logging-from-retro-cl.patch b/SOURCES/0084-Ticket-47831-remove-debug-logging-from-retro-cl.patch new file mode 100644 index 0000000..69426c4 --- /dev/null +++ b/SOURCES/0084-Ticket-47831-remove-debug-logging-from-retro-cl.patch @@ -0,0 +1,115 @@ +From f5b9a4be65641b29e37e9a0f9a15fee91db2a1e6 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Tue, 8 Sep 2015 12:20:33 -0700 +Subject: [PATCH 84/84] Ticket 47831 - remove debug logging from retro cl + +Description: 47831 patch expects "skip_nested" which is not supposed +to be in the branch rhel-7.1. +--- + ldap/servers/plugins/memberof/memberof.c | 16 +++++++--------- + ldap/servers/plugins/memberof/memberof_config.c | 6 +----- + 2 files changed, 8 insertions(+), 14 deletions(-) + +diff --git a/ldap/servers/plugins/memberof/memberof.c b/ldap/servers/plugins/memberof/memberof.c +index 1840e34..4d79cf6 100644 +--- a/ldap/servers/plugins/memberof/memberof.c ++++ b/ldap/servers/plugins/memberof/memberof.c +@@ -522,7 +522,7 @@ int memberof_postop_del(Slapi_PBlock *pb) + { + int ret = SLAPI_PLUGIN_SUCCESS; + MemberOfConfig *mainConfig = NULL; +- MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + Slapi_DN *sdn; + void *caller_id = NULL; + +@@ -850,7 +850,7 @@ int memberof_postop_modrdn(Slapi_PBlock *pb) + if(memberof_oktodo(pb)) + { + MemberOfConfig *mainConfig = 0; +- MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + struct slapi_entry *pre_e = NULL; + struct slapi_entry *post_e = NULL; + Slapi_DN *pre_sdn = 0; +@@ -1115,7 +1115,7 @@ int memberof_postop_modify(Slapi_PBlock *pb) + { + int config_copied = 0; + MemberOfConfig *mainConfig = 0; +- MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + + /* get the mod set */ + slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods); +@@ -1293,7 +1293,7 @@ int memberof_postop_add(Slapi_PBlock *pb) + if(memberof_oktodo(pb) && (sdn = memberof_getsdn(pb))) + { + struct slapi_entry *e = NULL; +- MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++ MemberOfConfig configCopy = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + MemberOfConfig *mainConfig; + slapi_pblock_get( pb, SLAPI_ENTRY_POST_OP, &e ); + +@@ -2178,11 +2178,9 @@ int memberof_get_groups_callback(Slapi_Entry *e, void *callback_data) + slapi_valueset_add_value_ext(groupvals, group_dn_val, SLAPI_VALUE_FLAG_PASSIN); + slapi_valueset_add_value_ext(group_norm_vals, group_ndn_val, SLAPI_VALUE_FLAG_PASSIN); + } +- if(!config->skip_nested || config->fixup_task){ +- /* now recurse to find parent groups of e */ +- memberof_get_groups_r(((memberof_get_groups_data*)callback_data)->config, +- group_sdn, callback_data); +- } ++ /* now recurse to find parent groups of e */ ++ memberof_get_groups_r(((memberof_get_groups_data*)callback_data)->config, ++ group_sdn, callback_data); + + bail: + return rc; +diff --git a/ldap/servers/plugins/memberof/memberof_config.c b/ldap/servers/plugins/memberof/memberof_config.c +index c335cab..6be7f3c 100644 +--- a/ldap/servers/plugins/memberof/memberof_config.c ++++ b/ldap/servers/plugins/memberof/memberof_config.c +@@ -77,7 +77,7 @@ static int memberof_search (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_En + /* This is the main configuration which is updated from dse.ldif. The + * config will be copied when it is used by the plug-in to prevent it + * being changed out from under a running memberOf operation. */ +-static MemberOfConfig theConfig = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++static MemberOfConfig theConfig = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + static Slapi_RWLock *memberof_config_lock = 0; + static int inited = 0; + +@@ -208,7 +208,6 @@ memberof_validate_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entr + Slapi_DN **exclude_dn = NULL; + char *syntaxoid = NULL; + char *config_dn = NULL; +- char *skip_nested = NULL; + char **entry_scopes = NULL; + char **entry_exclude_scopes = NULL; + int not_dn_syntax = 0; +@@ -462,7 +461,6 @@ memberof_apply_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* + char **entryScopes = NULL; + char **entryScopeExcludeSubtrees = NULL; + char *sharedcfg = NULL; +- char *skip_nested = NULL; + int num_vals = 0; + + *returncode = LDAP_SUCCESS; +@@ -495,7 +493,6 @@ memberof_apply_config (Slapi_PBlock *pb, Slapi_Entry* entryBefore, Slapi_Entry* + groupattrs = slapi_entry_attr_get_charray(e, MEMBEROF_GROUP_ATTR); + memberof_attr = slapi_entry_attr_get_charptr(e, MEMBEROF_ATTR); + allBackends = slapi_entry_attr_get_charptr(e, MEMBEROF_BACKEND_ATTR); +- skip_nested = slapi_entry_attr_get_charptr(e, MEMBEROF_SKIP_NESTED_ATTR); + + /* + * We want to be sure we don't change the config in the middle of +@@ -657,7 +654,6 @@ done: + slapi_ch_free_string(&sharedcfg); + slapi_ch_free_string(&memberof_attr); + slapi_ch_free_string(&allBackends); +- slapi_ch_free_string(&skip_nested); + slapi_ch_free((void **)&entryScopes); + slapi_ch_free((void **)&entryScopeExcludeSubtrees); + +-- +1.9.3 + diff --git a/SOURCES/0085-Ticket-48195-Slow-replication-when-deleting-large-qu.patch b/SOURCES/0085-Ticket-48195-Slow-replication-when-deleting-large-qu.patch new file mode 100644 index 0000000..f36fa21 --- /dev/null +++ b/SOURCES/0085-Ticket-48195-Slow-replication-when-deleting-large-qu.patch @@ -0,0 +1,45 @@ +From e9faf2091c545c8967db41961ef3d14f449e6c8a Mon Sep 17 00:00:00 2001 +From: Ludwig Krispenz +Date: Thu, 18 Jun 2015 15:22:54 +0200 +Subject: [PATCH 85/86] Ticket 48195 - Slow replication when deleting large + quantities of multi-valued attributes + +https://fedorahosted.org/389/ticket/48195 + +In update resoultion for entry deletion, there is still use of valuearray_find() to find an existingvalue to update its csn. +with the fix for ticket #346 there exists slapi_valueset_find() which uses the possibility to do a binary search on the +values. +Fix: do not use valuearray_find + +Review: Rich, Thanks +(cherry picked from commit 09ab8c799fc3d87db7a5b3aa07eccf9b41ea43d5) +(cherry picked from commit a980b795ac03200fd01a2d05ce568691681d50ef) +(cherry picked from commit 229ee77872d34acd53707e92c4d0861fa80b3d1f) +--- + ldap/servers/slapd/valueset.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/ldap/servers/slapd/valueset.c b/ldap/servers/slapd/valueset.c +index 1378bd1..9d77b0c 100644 +--- a/ldap/servers/slapd/valueset.c ++++ b/ldap/servers/slapd/valueset.c +@@ -1440,12 +1440,12 @@ valueset_update_csn_for_valuearray_ext(Slapi_ValueSet *vs, const Slapi_Attr *a, + int del_index = -1, del_count = 0; + for (i=0;valuestoupdate[i]!=NULL;++i) + { +- int index= valuearray_find(a, vs->va, valuestoupdate[i]); +- if(index!=-1) ++ Slapi_Value *v = slapi_valueset_find(a, vs, valuestoupdate[i]); ++ if(v) + { +- value_update_csn(vs->va[index],t,csn); ++ value_update_csn(v,t,csn); + if (csnref_updated) +- valuestoupdate[i]->v_csnset = (CSNSet *)value_get_csnset(vs->va[index]); ++ valuestoupdate[i]->v_csnset = (CSNSet *)value_get_csnset(v); + valuearrayfast_add_value_passin(&vaf_valuesupdated,valuestoupdate[i]); + valuestoupdate[i]= NULL; + del_count++; +-- +1.9.3 + diff --git a/SOURCES/0086-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch b/SOURCES/0086-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch new file mode 100644 index 0000000..e5ed2ee --- /dev/null +++ b/SOURCES/0086-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch @@ -0,0 +1,44 @@ +From 4c82238f0c4273dbf7cf945eebc2ede66f47f680 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Thu, 16 Jul 2015 10:34:47 -0700 +Subject: [PATCH 86/86] Ticket #48226 - In MMR, double free coould occur under + some special condition + +Bug description: + In a replicated topology, a authenticated user that have write access + on an entry can send a series of operations that crash the server. + The crash is due to an access to a already freed buffer. +Fix description: + To avoid the double free, duplicate a CSNSet and assign it to the + Slapi_Value. + +https://fedorahosted.org/389/ticket/48226 + +Reviewed by rmeggins@redhat.com (Thank you, Rich!!) + +(cherry picked from commit a0f8e0f981a046882db299a7a6d6d1c01bc19571) +(cherry picked from commit bdbc81e62eb8d7b8dfb298c7ba983cf86353fe66) +(cherry picked from commit 413414c98313a076111d8e40a7a10fa369433e6e) +--- + ldap/servers/slapd/valueset.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/ldap/servers/slapd/valueset.c b/ldap/servers/slapd/valueset.c +index 9d77b0c..fb7a99b 100644 +--- a/ldap/servers/slapd/valueset.c ++++ b/ldap/servers/slapd/valueset.c +@@ -1444,8 +1444,9 @@ valueset_update_csn_for_valuearray_ext(Slapi_ValueSet *vs, const Slapi_Attr *a, + if(v) + { + value_update_csn(v,t,csn); +- if (csnref_updated) +- valuestoupdate[i]->v_csnset = (CSNSet *)value_get_csnset(v); ++ if (csnref_updated) { ++ valuestoupdate[i]->v_csnset = csnset_dup(value_get_csnset(v)); ++ } + valuearrayfast_add_value_passin(&vaf_valuesupdated,valuestoupdate[i]); + valuestoupdate[i]= NULL; + del_count++; +-- +1.9.3 + diff --git a/SOURCES/0087-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch b/SOURCES/0087-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch new file mode 100644 index 0000000..c6cc32e --- /dev/null +++ b/SOURCES/0087-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch @@ -0,0 +1,77 @@ +From 92af3c54655cdac45271d2111a53ba6bd6400052 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Tue, 15 Sep 2015 18:25:02 -0700 +Subject: [PATCH] Ticket #48226 - In MMR, double free coould occur under some + special condition + +Description: commit a0f8e0f981a046882db299a7a6d6d1c01bc19571 introduced +a memory leak in the case of resolve_attribute_state_present_to_deleted. +In the case, csnset is not consumed. Thus, it has to be freed by csnset_ +free. + +https://fedorahosted.org/389/ticket/48226 + +Reviewed by mreynolds@redhat.com (Thank you, Mark!!) + +(cherry picked from commit b26ec6762fe2b5d37ade59243086cfd2308e8f0a) +(cherry picked from commit 4a3efc3330a034fa485f33e453054758561d4cea) +(cherry picked from commit 14e08bde4a48a8e8b56edc817b5d1e3d56b96c72) +--- + ldap/servers/slapd/entrywsi.c | 22 +++++++++++----------- + ldap/servers/slapd/valueset.c | 1 + + 2 files changed, 12 insertions(+), 11 deletions(-) + +diff --git a/ldap/servers/slapd/entrywsi.c b/ldap/servers/slapd/entrywsi.c +index 41afe1a..6773b9f 100644 +--- a/ldap/servers/slapd/entrywsi.c ++++ b/ldap/servers/slapd/entrywsi.c +@@ -1305,23 +1305,23 @@ resolve_attribute_state_present_to_deleted(Slapi_Entry *e, Slapi_Attr *a, Slapi_ + const CSN *adcsn= attr_get_deletion_csn(a); + int i; + if ( valuestoupdate != NULL && valuestoupdate[0] != NULL ) { +- for (i=0;valuestoupdate[i]!=NULL;++i) { +- /* This call ensures that the value does not contain a deletion_csn +- * which is before the presence_csn or distinguished_csn of the value. +- */ +- purge_attribute_state_multi_valued(a, valuestoupdate[i]); +- vdcsn= value_get_csn(valuestoupdate[i], CSN_TYPE_VALUE_DELETED); +- vucsn= value_get_csn(valuestoupdate[i], CSN_TYPE_VALUE_UPDATED); +- deletedcsn= csn_max(vdcsn, adcsn); ++ for (i=0;valuestoupdate[i]!=NULL;++i) { ++ /* This call ensures that the value does not contain a deletion_csn ++ * which is before the presence_csn or distinguished_csn of the value. ++ */ ++ purge_attribute_state_multi_valued(a, valuestoupdate[i]); ++ vdcsn= value_get_csn(valuestoupdate[i], CSN_TYPE_VALUE_DELETED); ++ vucsn= value_get_csn(valuestoupdate[i], CSN_TYPE_VALUE_UPDATED); ++ deletedcsn= csn_max(vdcsn, adcsn); + if(csn_compare(vucsn,deletedcsn)<0) + { +- if(!value_distinguished_at_csn(e, a, valuestoupdate[i], deletedcsn)) ++ if(!value_distinguished_at_csn(e, a, valuestoupdate[i], deletedcsn)) + { + entry_present_value_to_deleted_value(a,valuestoupdate[i]); + } + } +- valuestoupdate[i]->v_csnset = NULL; +- } ++ csnset_free(&valuestoupdate[i]->v_csnset); ++ } + } + } + +diff --git a/ldap/servers/slapd/valueset.c b/ldap/servers/slapd/valueset.c +index fb7a99b..7b5fa01 100644 +--- a/ldap/servers/slapd/valueset.c ++++ b/ldap/servers/slapd/valueset.c +@@ -1445,6 +1445,7 @@ valueset_update_csn_for_valuearray_ext(Slapi_ValueSet *vs, const Slapi_Attr *a, + { + value_update_csn(v,t,csn); + if (csnref_updated) { ++ csnset_free(&valuestoupdate[i]->v_csnset); + valuestoupdate[i]->v_csnset = csnset_dup(value_get_csnset(v)); + } + valuearrayfast_add_value_passin(&vaf_valuesupdated,valuestoupdate[i]); +-- +1.9.3 + diff --git a/SPECS/389-ds-base.spec b/SPECS/389-ds-base.spec index 9a9aba4..76f89d2 100644 --- a/SPECS/389-ds-base.spec +++ b/SPECS/389-ds-base.spec @@ -25,7 +25,7 @@ Summary: 389 Directory Server (base) Name: 389-ds-base Version: 1.3.3.1 -Release: %{?relprefix}20%{?prerel}%{?dist} +Release: %{?relprefix}23%{?prerel}%{?dist} License: GPLv2 with exceptions URL: http://port389.org/ Group: System Environment/Daemons @@ -190,6 +190,18 @@ Patch72: 0072-Ticket-48192-Individual-abandoned-simple-paged-resul.patc Patch73: 0073-Ticket-48194-nsSSL3Ciphers-preference-not-enforced-s.patch Patch74: 0074-Ticket-48192-Individual-abandoned-simple-paged-resul.patch Patch75: 0075-Ticket-48223-Winsync-fails-when-AD-users-have-multip.patch +Patch76: 0076-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch +Patch77: 0077-Ticket-47553-Enhance-ACIs-to-have-more-control-over-.patch +Patch78: 0078-Ticket-48265-Complex-filter-in-a-search-request-doen.patch +Patch79: 0079-Ticket-47912-Proper-handling-of-No-original_tombston.patch +Patch80: 0080-Ticket-48208-CleanAllRUV-should-completely-purge-cha.patch +Patch81: 0081-Ticket-47931-memberOf-retrocl-deadlocks.patch +Patch82: 0082-Ticket-47931-Fix-coverity-issues.patch +Patch83: 0083-Ticket-47831-remove-debug-logging-from-retro-cl.patch +Patch84: 0084-Ticket-47831-remove-debug-logging-from-retro-cl.patch +Patch85: 0085-Ticket-48195-Slow-replication-when-deleting-large-qu.patch +Patch86: 0086-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch +Patch87: 0087-Ticket-48226-In-MMR-double-free-coould-occur-under-s.patch %description 389 Directory Server is an LDAPv3 compliant server. The base package includes @@ -316,6 +328,18 @@ cp %{SOURCE2} README.devel %patch73 -p1 %patch74 -p1 %patch75 -p1 +%patch76 -p1 +%patch77 -p1 +%patch78 -p1 +%patch79 -p1 +%patch80 -p1 +%patch81 -p1 +%patch82 -p1 +%patch83 -p1 +%patch84 -p1 +%patch85 -p1 +%patch86 -p1 +%patch87 -p1 %build %if %{use_openldap} @@ -476,6 +500,22 @@ fi %{_libdir}/%{pkgname}/libns-dshttpd.so* %changelog +* Thu Sep 24 2015 Noriko Hosoi - 1.3.3.1-23 +- release 1.3.3.1-23 +- Resolves: bug 1262363 - In MMR, double free coould occur under some special condition (DS 48226) + +* Fri Sep 11 2015 Noriko Hosoi - 1.3.3.1-22 +- release 1.3.3.1-22 +- Resolves: bug 1262363 - In MMR, double free coould occur under some special condition (DS 48226) + +* Tue Sep 8 2015 Noriko Hosoi - 1.3.3.1-21 +- release 1.3.3.1-21 +- Resolves: bug 1258318 - Deadlock with retrochangelog, memberof plugin (DS 47931) +- Resolves: bug 1259466 - Enhance ACIs to have more control over MODRDN operations (DS 47553) +- Resolves: bug 1259999 - Some filters in RHDS10 are not working fine. (DS 48265) +- Resolves: bug 1260000 - handling of "No original_tombstone for changenumber" errors (DS 47912) +- Resolves: bug 1260001 - cleanallruv should completely clean changelog (DS 48208) + * Thu Jul 16 2015 Noriko Hosoi - 1.3.3.1-20 - release 1.3.3.1-20 - Resolves: bug 1243718 - Winsync fails when AD users have multiple spaces (two)inside the value of the rdn attribute (DS 48223)