|
 |
3d0f51 |
From 8b89bf22dea16956e4a21174f28ec11f32fc2db4 Mon Sep 17 00:00:00 2001
|
|
|
3d0f51 |
From: tbordaz <tbordaz@redhat.com>
|
|
|
3d0f51 |
Date: Mon, 21 Nov 2022 11:41:15 +0100
|
|
|
3d0f51 |
Subject: [PATCH 1/3] Issue 3729 - (cont) RFE Extend log of operations
|
|
|
3d0f51 |
statistics in access log (#5538)
|
|
|
3d0f51 |
|
|
|
3d0f51 |
Bug description:
|
|
|
3d0f51 |
This is a continuation of the #3729
|
|
|
3d0f51 |
The previous fix did not manage internal SRCH, so
|
|
|
3d0f51 |
statistics of internal SRCH were not logged
|
|
|
3d0f51 |
|
|
|
3d0f51 |
Fix description:
|
|
|
3d0f51 |
For internal operation log_op_stat uses
|
|
|
3d0f51 |
connid/op_id/op_internal_id/op_nested_count that have been
|
|
|
3d0f51 |
computed log_result
|
|
|
3d0f51 |
|
|
|
3d0f51 |
For direct operation log_op_stat uses info from the
|
|
|
3d0f51 |
operation itself (o_connid and o_opid)
|
|
|
3d0f51 |
|
|
|
3d0f51 |
log_op_stat relies on operation_type rather than
|
|
|
3d0f51 |
o_tag that is not available for internal operation
|
|
|
3d0f51 |
|
|
|
3d0f51 |
relates: #3729
|
|
|
3d0f51 |
|
|
|
3d0f51 |
Reviewed by: Pierre Rogier
|
|
|
3d0f51 |
---
|
|
|
3d0f51 |
.../tests/suites/ds_logs/ds_logs_test.py | 90 ++++++++++++++++++-
|
|
|
3d0f51 |
ldap/servers/slapd/proto-slap.h | 2 +-
|
|
|
3d0f51 |
ldap/servers/slapd/result.c | 74 +++++++++------
|
|
|
3d0f51 |
3 files changed, 136 insertions(+), 30 deletions(-)
|
|
|
3d0f51 |
|
|
|
3d0f51 |
diff --git a/dirsrvtests/tests/suites/ds_logs/ds_logs_test.py b/dirsrvtests/tests/suites/ds_logs/ds_logs_test.py
|
|
|
3d0f51 |
index 865a6d0a3..67605438b 100644
|
|
|
3d0f51 |
--- a/dirsrvtests/tests/suites/ds_logs/ds_logs_test.py
|
|
|
3d0f51 |
+++ b/dirsrvtests/tests/suites/ds_logs/ds_logs_test.py
|
|
|
3d0f51 |
@@ -13,7 +13,7 @@ import pytest
|
|
|
3d0f51 |
import subprocess
|
|
|
3d0f51 |
from lib389._mapped_object import DSLdapObject
|
|
|
3d0f51 |
from lib389.topologies import topology_st
|
|
|
3d0f51 |
-from lib389.plugins import AutoMembershipPlugin, ReferentialIntegrityPlugin, AutoMembershipDefinitions
|
|
|
3d0f51 |
+from lib389.plugins import AutoMembershipPlugin, ReferentialIntegrityPlugin, AutoMembershipDefinitions, MemberOfPlugin
|
|
|
3d0f51 |
from lib389.idm.user import UserAccounts
|
|
|
3d0f51 |
from lib389.idm.group import Groups
|
|
|
3d0f51 |
from lib389.idm.organizationalunit import OrganizationalUnits
|
|
|
3d0f51 |
@@ -1254,6 +1254,94 @@ def test_stat_index(topology_st, request):
|
|
|
3d0f51 |
|
|
|
3d0f51 |
request.addfinalizer(fin)
|
|
|
3d0f51 |
|
|
|
3d0f51 |
+def test_stat_internal_op(topology_st, request):
|
|
|
3d0f51 |
+ """Check that statistics can also be collected for internal operations
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ :id: 19f393bd-5866-425a-af7a-4dade06d5c77
|
|
|
3d0f51 |
+ :setup: Standalone Instance
|
|
|
3d0f51 |
+ :steps:
|
|
|
3d0f51 |
+ 1. Check that nsslapd-statlog-level is 0 (default)
|
|
|
3d0f51 |
+ 2. Enable memberof plugins
|
|
|
3d0f51 |
+ 3. Create a user
|
|
|
3d0f51 |
+ 4. Remove access log (to only detect new records)
|
|
|
3d0f51 |
+ 5. Enable statistic logging nsslapd-statlog-level=1
|
|
|
3d0f51 |
+ 6. Check that on direct SRCH there is no 'Internal' Stat records
|
|
|
3d0f51 |
+ 7. Remove access log (to only detect new records)
|
|
|
3d0f51 |
+ 8. Add group with the user, so memberof triggers internal search
|
|
|
3d0f51 |
+ and check it exists 'Internal' Stat records
|
|
|
3d0f51 |
+ :expectedresults:
|
|
|
3d0f51 |
+ 1. Success
|
|
|
3d0f51 |
+ 2. Success
|
|
|
3d0f51 |
+ 3. Success
|
|
|
3d0f51 |
+ 4. Success
|
|
|
3d0f51 |
+ 5. Success
|
|
|
3d0f51 |
+ 6. Success
|
|
|
3d0f51 |
+ 7. Success
|
|
|
3d0f51 |
+ 8. Success
|
|
|
3d0f51 |
+ """
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ inst = topology_st.standalone
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 1
|
|
|
3d0f51 |
+ log.info("Assert nsslapd-statlog-level is by default 0")
|
|
|
3d0f51 |
+ assert topology_st.standalone.config.get_attr_val_int("nsslapd-statlog-level") == 0
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 2
|
|
|
3d0f51 |
+ memberof = MemberOfPlugin(inst)
|
|
|
3d0f51 |
+ memberof.enable()
|
|
|
3d0f51 |
+ inst.restart()
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 3 Add setup entries
|
|
|
3d0f51 |
+ users = UserAccounts(inst, DEFAULT_SUFFIX, rdn=None)
|
|
|
3d0f51 |
+ user = users.create(properties={'uid': 'test_1',
|
|
|
3d0f51 |
+ 'cn': 'test_1',
|
|
|
3d0f51 |
+ 'sn': 'test_1',
|
|
|
3d0f51 |
+ 'description': 'member',
|
|
|
3d0f51 |
+ 'uidNumber': '1000',
|
|
|
3d0f51 |
+ 'gidNumber': '2000',
|
|
|
3d0f51 |
+ 'homeDirectory': '/home/testuser'})
|
|
|
3d0f51 |
+ # Step 4 reset accesslog
|
|
|
3d0f51 |
+ topology_st.standalone.stop()
|
|
|
3d0f51 |
+ lpath = topology_st.standalone.ds_access_log._get_log_path()
|
|
|
3d0f51 |
+ os.unlink(lpath)
|
|
|
3d0f51 |
+ topology_st.standalone.start()
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 5 enable statistics
|
|
|
3d0f51 |
+ log.info("Set nsslapd-statlog-level: 1 to enable indexing statistics")
|
|
|
3d0f51 |
+ topology_st.standalone.config.set("nsslapd-statlog-level", "1")
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 6 for direct SRCH only non internal STAT records
|
|
|
3d0f51 |
+ entries = topology_st.standalone.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "uid=test_1")
|
|
|
3d0f51 |
+ topology_st.standalone.stop()
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*STAT read index.*')
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*STAT read index: attribute.*')
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*STAT read index: duration.*')
|
|
|
3d0f51 |
+ assert not topology_st.standalone.ds_access_log.match('.*Internal.*STAT.*')
|
|
|
3d0f51 |
+ topology_st.standalone.start()
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 7 reset accesslog
|
|
|
3d0f51 |
+ topology_st.standalone.stop()
|
|
|
3d0f51 |
+ lpath = topology_st.standalone.ds_access_log._get_log_path()
|
|
|
3d0f51 |
+ os.unlink(lpath)
|
|
|
3d0f51 |
+ topology_st.standalone.start()
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ # Step 8 trigger internal searches and check internal stat records
|
|
|
3d0f51 |
+ groups = Groups(inst, DEFAULT_SUFFIX, rdn=None)
|
|
|
3d0f51 |
+ group = groups.create(properties={'cn': 'mygroup',
|
|
|
3d0f51 |
+ 'member': 'uid=test_1,%s' % DEFAULT_SUFFIX,
|
|
|
3d0f51 |
+ 'description': 'group'})
|
|
|
3d0f51 |
+ topology_st.standalone.restart()
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*Internal.*STAT read index.*')
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*Internal.*STAT read index: attribute.*')
|
|
|
3d0f51 |
+ assert topology_st.standalone.ds_access_log.match('.*Internal.*STAT read index: duration.*')
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ def fin():
|
|
|
3d0f51 |
+ log.info('Deleting user/group')
|
|
|
3d0f51 |
+ user.delete()
|
|
|
3d0f51 |
+ group.delete()
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
+ request.addfinalizer(fin)
|
|
|
3d0f51 |
+
|
|
|
3d0f51 |
if __name__ == '__main__':
|
|
|
3d0f51 |
# Run isolated
|
|
|
3d0f51 |
# -s for DEBUG mode
|
|
|
3d0f51 |
diff --git a/ldap/servers/slapd/proto-slap.h b/ldap/servers/slapd/proto-slap.h
|
|
|
3d0f51 |
index 77832797b..c63ad8e74 100644
|
|
|
3d0f51 |
--- a/ldap/servers/slapd/proto-slap.h
|
|
|
3d0f51 |
+++ b/ldap/servers/slapd/proto-slap.h
|
|
|
3d0f51 |
@@ -515,7 +515,7 @@ long long config_get_pw_minage(void);
|
|
|
3d0f51 |
long long config_get_pw_warning(void);
|
|
|
3d0f51 |
int config_get_errorlog_level(void);
|
|
|
3d0f51 |
int config_get_accesslog_level(void);
|
|
|
3d0f51 |
-int config_get_statlog_level();
|
|
|
3d0f51 |
+int config_get_statlog_level(void);
|
|
|
3d0f51 |
int config_get_securitylog_level(void);
|
|
|
3d0f51 |
int config_get_auditlog_logging_enabled(void);
|
|
|
3d0f51 |
int config_get_auditfaillog_logging_enabled(void);
|
|
|
3d0f51 |
diff --git a/ldap/servers/slapd/result.c b/ldap/servers/slapd/result.c
|
|
|
3d0f51 |
index c8b363cce..2ba205e04 100644
|
|
|
3d0f51 |
--- a/ldap/servers/slapd/result.c
|
|
|
3d0f51 |
+++ b/ldap/servers/slapd/result.c
|
|
|
3d0f51 |
@@ -33,7 +33,7 @@ static long current_conn_count;
|
|
|
3d0f51 |
static PRLock *current_conn_count_mutex;
|
|
|
3d0f51 |
static int flush_ber(Slapi_PBlock *pb, Connection *conn, Operation *op, BerElement *ber, int type);
|
|
|
3d0f51 |
static char *notes2str(unsigned int notes, char *buf, size_t buflen);
|
|
|
3d0f51 |
-static void log_op_stat(Slapi_PBlock *pb);
|
|
|
3d0f51 |
+static void log_op_stat(Slapi_PBlock *pb, uint64_t connid, int32_t op_id, int32_t op_internal_id, int32_t op_nested_count);
|
|
|
3d0f51 |
static void log_result(Slapi_PBlock *pb, Operation *op, int err, ber_tag_t tag, int nentries);
|
|
|
3d0f51 |
static void log_entry(Operation *op, Slapi_Entry *e);
|
|
|
3d0f51 |
static void log_referral(Operation *op);
|
|
|
3d0f51 |
@@ -2000,65 +2000,82 @@ notes2str(unsigned int notes, char *buf, size_t buflen)
|
|
|
3d0f51 |
return (buf);
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
|
|
|
3d0f51 |
+#define STAT_LOG_CONN_OP_FMT_INT_INT "conn=Internal(%" PRIu64 ") op=%d(%d)(%d)"
|
|
|
3d0f51 |
+#define STAT_LOG_CONN_OP_FMT_EXT_INT "conn=%" PRIu64 " (Internal) op=%d(%d)(%d)"
|
|
|
3d0f51 |
static void
|
|
|
3d0f51 |
-log_op_stat(Slapi_PBlock *pb)
|
|
|
3d0f51 |
+log_op_stat(Slapi_PBlock *pb, uint64_t connid, int32_t op_id, int32_t op_internal_id, int32_t op_nested_count)
|
|
|
3d0f51 |
{
|
|
|
3d0f51 |
-
|
|
|
3d0f51 |
- Connection *conn = NULL;
|
|
|
3d0f51 |
Operation *op = NULL;
|
|
|
3d0f51 |
Op_stat *op_stat;
|
|
|
3d0f51 |
struct timespec duration;
|
|
|
3d0f51 |
char stat_etime[ETIME_BUFSIZ] = {0};
|
|
|
3d0f51 |
+ int internal_op;
|
|
|
3d0f51 |
|
|
|
3d0f51 |
if (config_get_statlog_level() == 0) {
|
|
|
3d0f51 |
return;
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
|
|
|
3d0f51 |
- slapi_pblock_get(pb, SLAPI_CONNECTION, &conn;;
|
|
|
3d0f51 |
slapi_pblock_get(pb, SLAPI_OPERATION, &op);
|
|
|
3d0f51 |
+ internal_op = operation_is_flag_set(op, OP_FLAG_INTERNAL);
|
|
|
3d0f51 |
op_stat = op_stat_get_operation_extension(pb);
|
|
|
3d0f51 |
|
|
|
3d0f51 |
- if (conn == NULL || op == NULL || op_stat == NULL) {
|
|
|
3d0f51 |
+ if (op == NULL || op_stat == NULL) {
|
|
|
3d0f51 |
return;
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
/* process the operation */
|
|
|
3d0f51 |
- switch (op->o_tag) {
|
|
|
3d0f51 |
- case LDAP_REQ_BIND:
|
|
|
3d0f51 |
- case LDAP_REQ_UNBIND:
|
|
|
3d0f51 |
- case LDAP_REQ_ADD:
|
|
|
3d0f51 |
- case LDAP_REQ_DELETE:
|
|
|
3d0f51 |
- case LDAP_REQ_MODRDN:
|
|
|
3d0f51 |
- case LDAP_REQ_MODIFY:
|
|
|
3d0f51 |
- case LDAP_REQ_COMPARE:
|
|
|
3d0f51 |
+ switch (operation_get_type(op)) {
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_BIND:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_UNBIND:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_ADD:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_DELETE:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_MODRDN:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_MODIFY:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_COMPARE:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_EXTENDED:
|
|
|
3d0f51 |
break;
|
|
|
3d0f51 |
- case LDAP_REQ_SEARCH:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_SEARCH:
|
|
|
3d0f51 |
if ((LDAP_STAT_READ_INDEX & config_get_statlog_level()) &&
|
|
|
3d0f51 |
op_stat->search_stat) {
|
|
|
3d0f51 |
struct component_keys_lookup *key_info;
|
|
|
3d0f51 |
for (key_info = op_stat->search_stat->keys_lookup; key_info; key_info = key_info->next) {
|
|
|
3d0f51 |
- slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
- "conn=%" PRIu64 " op=%d STAT read index: attribute=%s key(%s)=%s --> count %d\n",
|
|
|
3d0f51 |
- op->o_connid, op->o_opid,
|
|
|
3d0f51 |
- key_info->attribute_type, key_info->index_type, key_info->key,
|
|
|
3d0f51 |
- key_info->id_lookup_cnt);
|
|
|
3d0f51 |
+ if (internal_op) {
|
|
|
3d0f51 |
+ slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
+ connid == 0 ? STAT_LOG_CONN_OP_FMT_INT_INT "STAT read index: attribute=%s key(%s)=%s --> count %d\n":
|
|
|
3d0f51 |
+ STAT_LOG_CONN_OP_FMT_EXT_INT "STAT read index: attribute=%s key(%s)=%s --> count %d\n",
|
|
|
3d0f51 |
+ connid, op_id, op_internal_id, op_nested_count,
|
|
|
3d0f51 |
+ key_info->attribute_type, key_info->index_type, key_info->key,
|
|
|
3d0f51 |
+ key_info->id_lookup_cnt);
|
|
|
3d0f51 |
+ } else {
|
|
|
3d0f51 |
+ slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
+ "conn=%" PRIu64 " op=%d STAT read index: attribute=%s key(%s)=%s --> count %d\n",
|
|
|
3d0f51 |
+ connid, op_id,
|
|
|
3d0f51 |
+ key_info->attribute_type, key_info->index_type, key_info->key,
|
|
|
3d0f51 |
+ key_info->id_lookup_cnt);
|
|
|
3d0f51 |
+ }
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
|
|
|
3d0f51 |
/* total elapsed time */
|
|
|
3d0f51 |
slapi_timespec_diff(&op_stat->search_stat->keys_lookup_end, &op_stat->search_stat->keys_lookup_start, &duration);
|
|
|
3d0f51 |
snprintf(stat_etime, ETIME_BUFSIZ, "%" PRId64 ".%.09" PRId64 "", (int64_t)duration.tv_sec, (int64_t)duration.tv_nsec);
|
|
|
3d0f51 |
- slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
- "conn=%" PRIu64 " op=%d STAT read index: duration %s\n",
|
|
|
3d0f51 |
- op->o_connid, op->o_opid, stat_etime);
|
|
|
3d0f51 |
+ if (internal_op) {
|
|
|
3d0f51 |
+ slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
+ connid == 0 ? STAT_LOG_CONN_OP_FMT_INT_INT "STAT read index: duration %s\n":
|
|
|
3d0f51 |
+ STAT_LOG_CONN_OP_FMT_EXT_INT "STAT read index: duration %s\n",
|
|
|
3d0f51 |
+ connid, op_id, op_internal_id, op_nested_count, stat_etime);
|
|
|
3d0f51 |
+ } else {
|
|
|
3d0f51 |
+ slapi_log_stat(LDAP_STAT_READ_INDEX,
|
|
|
3d0f51 |
+ "conn=%" PRIu64 " op=%d STAT read index: duration %s\n",
|
|
|
3d0f51 |
+ op->o_connid, op->o_opid, stat_etime);
|
|
|
3d0f51 |
+ }
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
break;
|
|
|
3d0f51 |
- case LDAP_REQ_ABANDON_30:
|
|
|
3d0f51 |
- case LDAP_REQ_ABANDON:
|
|
|
3d0f51 |
+ case SLAPI_OPERATION_ABANDON:
|
|
|
3d0f51 |
break;
|
|
|
3d0f51 |
|
|
|
3d0f51 |
default:
|
|
|
3d0f51 |
slapi_log_err(SLAPI_LOG_ERR,
|
|
|
3d0f51 |
- "log_op_stat", "Ignoring unknown LDAP request (conn=%" PRIu64 ", tag=0x%lx)\n",
|
|
|
3d0f51 |
- conn->c_connid, op->o_tag);
|
|
|
3d0f51 |
+ "log_op_stat", "Ignoring unknown LDAP request (conn=%" PRIu64 ", op_type=0x%lx)\n",
|
|
|
3d0f51 |
+ connid, operation_get_type(op));
|
|
|
3d0f51 |
break;
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
@@ -2218,7 +2235,7 @@ log_result(Slapi_PBlock *pb, Operation *op, int err, ber_tag_t tag, int nentries
|
|
|
3d0f51 |
} else {
|
|
|
3d0f51 |
ext_str = "";
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
- log_op_stat(pb);
|
|
|
3d0f51 |
+ log_op_stat(pb, op->o_connid, op->o_opid, 0, 0);
|
|
|
3d0f51 |
slapi_log_access(LDAP_DEBUG_STATS,
|
|
|
3d0f51 |
"conn=%" PRIu64 " op=%d RESULT err=%d"
|
|
|
3d0f51 |
" tag=%" BERTAG_T " nentries=%d wtime=%s optime=%s etime=%s%s%s%s\n",
|
|
|
3d0f51 |
@@ -2233,6 +2250,7 @@ log_result(Slapi_PBlock *pb, Operation *op, int err, ber_tag_t tag, int nentries
|
|
|
3d0f51 |
}
|
|
|
3d0f51 |
} else {
|
|
|
3d0f51 |
int optype;
|
|
|
3d0f51 |
+ log_op_stat(pb, connid, op_id, op_internal_id, op_nested_count);
|
|
|
3d0f51 |
#define LOG_MSG_FMT " tag=%" BERTAG_T " nentries=%d wtime=%s optime=%s etime=%s%s%s\n"
|
|
|
3d0f51 |
slapi_log_access(LDAP_DEBUG_ARGS,
|
|
|
3d0f51 |
connid == 0 ? LOG_CONN_OP_FMT_INT_INT LOG_MSG_FMT :
|
|
|
3d0f51 |
--
|
|
|
3d0f51 |
2.38.1
|
|
|
3d0f51 |
|