From 5c3d35055fa16a319600c83bd9cacd040b63ac39 Mon Sep 17 00:00:00 2001 From: Mark Reynolds Date: Wed, 28 May 2014 14:38:03 -0400 Subject: [PATCH 216/225] Ticket 47446 - logconv.pl memory continually grows Bug Description: Running logconv.pl without any special options continually consumes more memory. Fix Description: The main "leak" was a file descriptor array that was constantly growing with every connection, but the array served no purpose - so it was removed. Some other minor changes were made: output formating improvements, making sure verbose hashes were not being updated if the verbose optoin was not enabled, and not calling the CSV report fiunctions if a report was not being asked for. https://fedorahosted.org/389/ticket/47446 Reviewed by: nhosoi(Thanks!) (cherry picked from commit 805386caff210ac07a428b38e99ea7b853bece1f) (cherry picked from commit 519c58e8082ef093e676eb7a1aaaec03a739ea69) (cherry picked from commit bf4cbf12875cf46d4eecd67a579c857c997e1ce9) --- ldap/admin/src/logconv.pl | 133 ++++++++++++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 53 deletions(-) diff --git a/ldap/admin/src/logconv.pl b/ldap/admin/src/logconv.pl index ea33544..edc5fd1 100755 --- a/ldap/admin/src/logconv.pl +++ b/ldap/admin/src/logconv.pl @@ -56,7 +56,7 @@ Getopt::Long::Configure ("bundling"); Getopt::Long::Configure ("permute"); if ($#ARGV < 0){; -&displayUsage; + &displayUsage; } ####################################### @@ -78,8 +78,8 @@ my $dataLocation = "/tmp"; my $startTLSoid = "1.3.6.1.4.1.1466.20037"; my @statnames=qw(last last_str results srch add mod modrdn moddn cmp del abandon conns sslconns bind anonbind unbind notesA notesU etime); -my $s_stats = new_stats_block( ); -my $m_stats = new_stats_block( ); +my $s_stats; +my $m_stats; my $verb = "no"; my @excludeIP; my $xi = 0; @@ -99,7 +99,6 @@ my %bindReport; my @vlvconn; my @vlvop; my @fds; -my $fdds = 0; my $reportBinds = "no"; my $rootDN = ""; my $needCleanup = 0; @@ -392,34 +391,28 @@ sub statusreport { ########################################## if ($files[$#files] =~ m/access.rotationinfo/) { $file_count--; } +$logCount = $file_count; + +print "Processing $logCount Access Log(s)...\n"; -print "Processing $file_count Access Log(s)...\n\n"; #print "Filename\t\t\t Total Lines\n"; #print "--------------------------------------------------\n"; my $skipFirstFile = 0; -if ($file_count > 1 && $files[0] =~ /\/access$/){ - $files[$file_count] = $files[0]; - $file_count++; - $skipFirstFile = 1; +if ($logCount > 1 && $files[0] =~ /\/access$/){ + $files[$logCount] = $files[0]; + $skipFirstFile = 1; + $file_count++; } -$logCount = $file_count; + my $logline; my $totalLineCount = 0; for (my $count=0; $count < $file_count; $count++){ # we moved access to the end of the list, so if its the first file skip it - if($file_count > 1 && $count == 0 && $skipFirstFile == 1){ - next; - } - if($file_count > 1 && $count == 0 && $skipFirstFile == 1){ - next; - } - if (-z $files[$count]){ - # access log is empty - print "Skipping empty access log ($files[$count])...\n"; + if($logCount > 1 && $count == 0 && $skipFirstFile == 1){ next; } $linesProcessed = 0; $lineBlockCount = 0; @@ -428,6 +421,12 @@ for (my $count=0; $count < $file_count; $count++){ $atime,$mtime,$ctime,$blksize,$blocks) = stat($files[$count]); print sprintf "[%03d] %-30s\tsize (bytes): %12s\n",$logCount, $files[$count], $cursize; $logCount--; + if ($cursize == 0){ + # access log is empty + print "Skipping empty access log ($files[$count])...\n"; + next; + } + my $logCountStr; if($logCount < 10 ){ # add a zero for formatting purposes @@ -612,6 +611,12 @@ if($reportBinds eq "yes"){ &displayBindReport(); } +if($reportStats ne ""){ + # No need to print the general report, so exit here + removeDataFiles(); + exit (0); +} + # # Continue with standard report # @@ -1366,9 +1371,9 @@ if ($usage =~ /j/i || $verb eq "yes"){ # We're done, clean up the data files # removeDataFiles(); - exit (0); + ####################### # # # Display Usage # @@ -1524,7 +1529,11 @@ parseLineBind { $bindReport{$dn}{"failedBind"} = 0; } if ($_ =~ /conn= *([0-9A-Z]+)/i) { - $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; + if($bindReport{$dn}{"conn"}){ + $bindReport{$dn}{"conn"} = $bindReport{$dn}{"conn"} . " $1 "; + } else { + $bindReport{$dn}{"conn"} = $1; + } } return; } @@ -1645,7 +1654,7 @@ sub parseLineNormal # Additional performance stats my ($time, $tzone) = split (' ', $_); - if (($reportStats or ($verb eq "yes") || ($usage =~ /y/)) && (!defined($last_tm) or ($time ne $last_tm))) + if (($reportStats || ($verb eq "yes") || ($usage =~ /y/)) && (!defined($last_tm) or ($time ne $last_tm))) { $last_tm = $time; $time =~ s/\[//; @@ -1685,12 +1694,16 @@ sub parseLineNormal if($reportStats){ inc_stats('srch',$s_stats,$m_stats); } if ($_ =~ / attrs=\"(.*)\"/i){ $anyAttrs++; - my $attr = $hashes->{attr}; - map { $attr->{$_}++ } split /\s/, $1; + if ($usage =~ /r/i || $verb eq "yes"){ + my $attr = $hashes->{attr}; + map { $attr->{$_}++ } split /\s/, $1; + } } if (/ attrs=ALL/){ - my $attr = $hashes->{attr}; - $attr->{"All Attributes"}++; + if ($usage =~ /r/i || $verb eq "yes"){ + my $attr = $hashes->{attr}; + $attr->{"All Attributes"}++; + } $anyAttrs++; } if ($verb eq "yes"){ @@ -1768,16 +1781,20 @@ sub parseLineNormal $bindCount++; if($reportStats){ inc_stats('bind',$s_stats,$m_stats); } if ($1 ne ""){ - $tmpp = $1; - $tmpp =~ tr/A-Z/a-z/; - $hashes->{bindlist}->{$tmpp}++; if($1 eq $rootDN){ $rootDNBindCount++; } + if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){ + $tmpp = $1; + $tmpp =~ tr/A-Z/a-z/; + $hashes->{bindlist}->{$tmpp}++; + } } else { $anonymousBindCount++; - $hashes->{bindlist}->{"Anonymous Binds"}++; - inc_stats('anonbind',$s_stats,$m_stats); + if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){ + $hashes->{bindlist}->{"Anonymous Binds"}++; + if($reportStats){ inc_stats('anonbind',$s_stats,$m_stats); } + } } } if (m/ connection from/){ @@ -1796,15 +1813,20 @@ sub parseLineNormal if ($simConnection > $maxsimConnection) { $maxsimConnection = $simConnection; } - ($connID) = $_ =~ /conn=(\d*)\s/; - $openConnection{$connID} = $ip; - if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { - $hashes->{start_time_of_connection}->{$connID} = $gmtime; + if ($verb eq "yes" || $usage =~ /p/ || $reportStats){ + ($connID) = $_ =~ /conn=(\d*)\s/; + $openConnection{$connID} = $ip; + if ($reportStats or ($verb eq "yes") || ($usage =~ /y/)) { + $hashes->{start_time_of_connection}->{$connID} = $gmtime; + } } } if (m/ SSL client bound as /){$sslClientBindCount++;} if (m/ SSL failed to map client certificate to LDAP DN/){$sslClientFailedCount++;} - if (m/ fd=/ && m/slot=/){$fdTaken++} + if (m/slot=/ && $_ =~ /fd= *([0-9]+)/i) { + $fdTaken++; + if ($1 > $highestFdTaken){ $highestFdTaken = $1; } + } if (m/ fd=/ && m/closed/){ ($connID) = $_ =~ /conn=(\d*)\s/; handleConnClose($connID); @@ -1819,20 +1841,20 @@ sub parseLineNormal $op = $2; } if ($binddn ne ""){ - if($binddn eq $rootDN){$rootDNBindCount++;} - $tmpp = $binddn; - $tmpp =~ tr/A-Z/a-z/; - $hashes->{bindlist}->{$tmpp}++; - if($usage =~ /f/ || $verb eq "yes"){ + if($binddn eq $rootDN){ $rootDNBindCount++; } + if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){ + $tmpp = $binddn; + $tmpp =~ tr/A-Z/a-z/; + $hashes->{bindlist}->{$tmpp}++; $hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = $tmpp; } } else { $anonymousBindCount++; - $hashes->{bindlist}->{"Anonymous Binds"}++; - if($usage =~ /f/ || $verb eq "yes"){ + if($usage =~ /f/ || $usage =~ /u/ || $usage =~ /U/ || $usage =~ /b/ || $verb eq "yes"){ + $hashes->{bindlist}->{"Anonymous Binds"}++; $hashes->{bind_conn_op}->{"$serverRestartCount,$conn,$op"} = ""; } - inc_stats('anonbind',$s_stats,$m_stats); + if($reportStats){ inc_stats('anonbind',$s_stats,$m_stats); } } } if (m/ UNBIND/){ @@ -2109,20 +2131,30 @@ sub parseLineNormal } } if ($_ =~ /err= *([0-9]+)/i){ - $errorCode[$1]++; + if ($usage =~ /e/i || $verb eq "yes"){ $errorCode[$1]++; } if ($1 ne "0"){ $errorCount++;} else { $successCount++;} } - if ($_ =~ /etime= *([0-9.]+)/ ) { $hashes->{etime}->{$1}++; inc_stats_val('etime',$1,$s_stats,$m_stats); } + if ($_ =~ /etime= *([0-9.]+)/ ) { + my $etime_val = $1; + if ($usage =~ /t/i || $verb eq "yes"){ $hashes->{etime}->{$etime_val}++; } + if ($reportStats){ inc_stats_val('etime',$etime_val,$s_stats,$m_stats); } + } if ($_ =~ / tag=101 / || $_ =~ / tag=111 / || $_ =~ / tag=100 / || $_ =~ / tag=115 /){ - if ($_ =~ / nentries= *([0-9]+)/i ){ $hashes->{nentries}->{$1}++; } + if ($_ =~ / nentries= *([0-9]+)/i ){ + if ($usage =~ /n/i || $verb eq "yes"){ + $hashes->{nentries}->{$1}++; + } + } } if (m/objectclass=\*/i || m/objectclass=top/i ){ if (m/ scope=2 /){ $objectclassTopCount++;} } if (m/ EXT oid=/){ $extopCount++; - if ($_ =~ /oid=\" *([0-9\.]+)/i ){ $hashes->{oid}->{$1}++; } + if ($_ =~ /oid=\" *([0-9\.]+)/i ){ + if ($usage =~ /x/i || $verb eq "yes"){$hashes->{oid}->{$1}++; } + } if ($1 && $1 eq $startTLSoid){$startTLSCount++;} if ($verb eq "yes"){ if ($_ =~ /conn= *([0-9A-Z]+) +op= *([0-9\-]+)/i){ $hashes->{ext_conn_op}->{"$serverRestartCount,$1,$2"}++;} @@ -2182,11 +2214,6 @@ sub parseLineNormal $scopeCount++; } } - if ($_ =~ /fd= *([0-9]+)/i ) { - $fds[$fdds] = $1; - if ($fds[$fdds] > $highestFdTaken) {$highestFdTaken = $fds[$fdds];} - $fdds++; - } if ($usage =~ /f/ || $verb eq "yes"){ if (/ err=49 tag=/ && / dn=\"/){ if ($_ =~ /dn=\"(.*)\"/i ){ -- 1.8.1.4