From a6a52365df26edd4f6b0028056395d943344d787 Mon Sep 17 00:00:00 2001

From: Mark Reynolds <mreynolds@redhat.com>

Date: Thu, 11 Jun 2020 15:30:28 -0400

Subject: [PATCH] Issue 50791 - Healthcheck should look for notes=A/F in access

 log

Description:  Add checks for notes=A (fully unindexed search) and

              notes=F (Unknown attribute in search filter) in the

              current access log.

relates: https://pagure.io/389-ds-base/issue/50791

Reviewed by: firstyear(Thanks!)

---

 src/lib389/lib389/cli_ctl/health.py |  4 +-

 src/lib389/lib389/dirsrv_log.py     | 72 +++++++++++++++++++++++++++--

 src/lib389/lib389/lint.py           | 26 ++++++++++-

 3 files changed, 96 insertions(+), 6 deletions(-)

diff --git a/src/lib389/lib389/cli_ctl/health.py b/src/lib389/lib389/cli_ctl/health.py

index 6333a753a..89484a11b 100644

--- a/src/lib389/lib389/cli_ctl/health.py

+++ b/src/lib389/lib389/cli_ctl/health.py

@@ -1,5 +1,5 @@

 # --- BEGIN COPYRIGHT BLOCK ---

-# Copyright (C) 2019 Red Hat, Inc.

+# Copyright (C) 2020 Red Hat, Inc.

 # All rights reserved.

 #

 # License: GPL (version 3 or any later version).

@@ -18,6 +18,7 @@ from lib389.monitor import MonitorDiskSpace

 from lib389.replica import Replica, Changelog5

 from lib389.nss_ssl import NssSsl

 from lib389.dseldif import FSChecks, DSEldif

+from lib389.dirsrv_log import DirsrvAccessLog

 from lib389 import lint

 from lib389 import plugins

 from lib389._constants import DSRC_HOME

@@ -37,6 +38,7 @@ CHECK_OBJECTS = [

     Changelog5,

     DSEldif,

     NssSsl,

+    DirsrvAccessLog,

 ]

diff --git a/src/lib389/lib389/dirsrv_log.py b/src/lib389/lib389/dirsrv_log.py

index baac2a3c9..7bed4bb17 100644

--- a/src/lib389/lib389/dirsrv_log.py

+++ b/src/lib389/lib389/dirsrv_log.py

@@ -1,5 +1,5 @@

 # --- BEGIN COPYRIGHT BLOCK ---

-# Copyright (C) 2016 Red Hat, Inc.

+# Copyright (C) 2020 Red Hat, Inc.

 # All rights reserved.

 #

 # License: GPL (version 3 or any later version).

@@ -9,12 +9,17 @@

 """Helpers for managing the directory server internal logs.

 """

+import copy

 import re

 import gzip

 from dateutil.parser import parse as dt_parse

 from glob import glob

 from lib389.utils import ensure_bytes

-

+from lib389._mapped_object_lint import DSLint

+from lib389.lint import (

+    DSLOGNOTES0001,  # Unindexed search

+    DSLOGNOTES0002,  # Unknown attr in search filter

+)

 # Because many of these settings can change live, we need to check for certain

 # attributes all the time.

@@ -35,7 +40,7 @@ MONTH_LOOKUP = {

 }

-class DirsrvLog(object):

+class DirsrvLog(DSLint):

     """Class of functions to working with the various DIrectory Server logs

     """

     def __init__(self, dirsrv):

@@ -189,6 +194,67 @@ class DirsrvAccessLog(DirsrvLog):

         self.full_regexs = [self.prog_m1, self.prog_con, self.prog_discon]

         self.result_regexs = [self.prog_notes, self.prog_repl,

                               self.prog_result]

+    @classmethod

+    def lint_uid(cls):

+        return 'logs'

+

+    def _log_get_search_stats(self, conn, op):

+        lines = self.match(f".* conn={conn} op={op} SRCH base=.*")

+        if len(lines) != 1:

+            return None

+

+        quoted_vals = re.findall('"([^"]*)"', lines[0])

+        return {

+            'base': quoted_vals[0],

+            'filter': quoted_vals[1],

+            'timestamp': re.findall('\[(.*)\]', lines[0])[0],

+            'scope': lines[0].split(' scope=', 1)[1].split(' ',1)[0]

+        }

+

+    def _lint_notes(self):

+        """

+        Check for notes=A (fully unindexed searches), and

+        notes=F (unknown attribute in filter)

+        """

+        for pattern, lint_report in [(".* notes=A", DSLOGNOTES0001), (".* notes=F", DSLOGNOTES0002)]:

+            lines = self.match(pattern)

+            if len(lines) > 0:

+                count = 0

+                searches = []

+                for line in lines:

+                    if ' RESULT err=' in line:

+                        # Looks like a valid notes=A/F

+                        conn = line.split(' conn=', 1)[1].split(' ',1)[0]

+                        op = line.split(' op=', 1)[1].split(' ',1)[0]

+                        etime = line.split(' etime=', 1)[1].split(' ',1)[0]

+                        stats = self._log_get_search_stats(conn, op)

+                        if stats is not None:

+                            timestamp = stats['timestamp']

+                            base = stats['base']

+                            scope = stats['scope']

+                            srch_filter = stats['filter']

+                            count += 1

+                            if lint_report == DSLOGNOTES0001:

+                                searches.append(f'\n  [{count}] Unindexed Search\n'

+                                                f'      - date:    {timestamp}\n'

+                                                f'      - conn/op: {conn}/{op}\n'

+                                                f'      - base:    {base}\n'

+                                                f'      - scope:   {scope}\n'

+                                                f'      - filter:  {srch_filter}\n'

+                                                f'      - etime:   {etime}\n')

+                            else:

+                                searches.append(f'\n  [{count}] Invalid Attribute in Filter\n'

+                                                f'      - date:    {timestamp}\n'

+                                                f'      - conn/op: {conn}/{op}\n'

+                                                f'      - filter:  {srch_filter}\n')

+                if len(searches) > 0:

+                    report = copy.deepcopy(lint_report)

+                    report['items'].append(self._get_log_path())

+                    report['detail'] = report['detail'].replace('NUMBER', str(count))

+                    for srch in searches:

+                        report['detail'] += srch

+                    yield report

+

     def _get_log_path(self):

         """Return the current log file location"""

diff --git a/src/lib389/lib389/lint.py b/src/lib389/lib389/lint.py

index a103feec7..4b1700b92 100644

--- a/src/lib389/lib389/lint.py

+++ b/src/lib389/lib389/lint.py

@@ -1,5 +1,5 @@

 # --- BEGIN COPYRIGHT BLOCK ---

-# Copyright (C) 2019 Red Hat, Inc.

+# Copyright (C) 2020 Red Hat, Inc.

 # All rights reserved.

 #

 # License: GPL (version 3 or any later version).

@@ -253,7 +253,7 @@ can use the CLI tool "dsconf" to resolve the conflict.  Here is an example:

     Remove conflict entry and keep only the original/counterpart entry:

-        # dsconf slapd-YOUR_INSTANCE  repl-conflict remove <DN of conflict entry>

+        # dsconf slapd-YOUR_INSTANCE  repl-conflict delete <DN of conflict entry>

     Replace the original/counterpart entry with the conflict entry:

@@ -418,3 +418,25 @@ until the time issues have been resolved:

 Also look at https://access.redhat.com/documentation/en-us/red_hat_directory_server/11/html/administration_guide/managing_replication-troubleshooting_replication_related_problems

 and find the paragraph "Too much time skew"."""

 }

+

+DSLOGNOTES0001 = {

+    'dsle': 'DSLOGNOTES0001',

+    'severity': 'Medium',

+    'description': 'Unindexed Search',

+    'items': ['Performance'],

+    'detail': """Found NUMBER fully unindexed searches in the current access log.

+Unindexed searches can cause high CPU and slow down the entire server's performance.\n""",

+    'fix': """Examine the searches that are unindexed, and either properly index the attributes

+in the filter, increase the nsslapd-idlistscanlimit, or stop using that filter."""

+}

+

+DSLOGNOTES0002 = {

+    'dsle': 'DSLOGNOTES0002',

+    'severity': 'Medium',

+    'description': 'Unknown Attribute In Filter',

+    'items': ['Possible Performance Impact'],

+    'detail': """Found NUMBER searches in the current access log that are using an

+unknown attribute in the search filter.\n""",

+    'fix': """Stop using this these unknown attributes in the filter, or add the schema

+to the server and make sure it's properly indexed."""

+}

-- 

2.26.2