Added the general preprocessing framework

logparser/PreDrain/PreDrain.py

@@ -0,0 +1,95 @@
+"""
+Description : This file implements the Drain algorithm for log parsing
+Author      : LogPAI team
+License     : MIT
+"""
+
+import os
+from datetime import datetime
+from logparser.Drain import Logcluster, Node
+import logparser.Drain
+from logparser.utils.preprocessing import preprocess
+
+class LogParser(logparser.Drain.LogParser):
+    def __init__(self, log_format, estimate=2000, **kwargs):
+        """
+            estimate: The number of logs used for regex usage estimation
+        """
+        super().__init__(log_format, **kwargs)
+        self.estimate = estimate
+
+    #seq1 is template
+    def seqDist(self, seq1, seq2):
+        assert len(seq1) == len(seq2)
+        simTokens = 0
+        numOfPar = 0
+        # Add fast return if they are the same
+        if seq1 == seq2:
+            for t in seq1:
+                if t == "<*>": numOfPar += 1
+            return 1, numOfPar
+
+        for token1, token2 in zip(seq1, seq2):
+            if token1 == '<*>':
+                numOfPar += 1
+                continue
+            if token1 == token2:
+                simTokens += 1 
+
+        retVal = float(simTokens) / len(seq1)
+
+        return retVal, numOfPar
+
+    def parse(self, logName):
+        print('Parsing file: ' + os.path.join(self.path, logName))
+        start_time = datetime.now()
+        self.logName = logName
+        rootNode = Node()
+        logCluL = []
+
+        self.load_data()
+
+        count = 0
+        matched_types = []
+        use_sequence = []
+        for idx, line in self.df_log.iterrows():
+            logID = line['LineId']
+
+            # Add preprocess
+            if idx < self.estimate:
+                logmessageL, sequence, matched = preprocess(line['Content'], estimation_stage=True)
+                logmessageL = logmessageL.strip().split()
+                matched_types.extend(matched)
+                use_sequence = sequence
+            else:
+                if idx == self.estimate:
+                    matched_types = set(matched_types)
+                    use_sequence = [i for i in use_sequence if i in matched_types]
+                logmessageL = preprocess(line['Content'], estimation_stage=False, use_sequence=use_sequence).strip().split()
+
+            matchCluster = self.treeSearch(rootNode, logmessageL)
+
+            #Match no existing log cluster
+            if matchCluster is None:
+                newCluster = Logcluster(logTemplate=logmessageL, logIDL=[logID])
+                logCluL.append(newCluster)
+                self.addSeqToPrefixTree(rootNode, newCluster)
+
+            #Add the new log message to the existing cluster
+            else:
+                newTemplate = self.getTemplate(logmessageL, matchCluster.logTemplate)
+                matchCluster.logIDL.append(logID)
+                if ' '.join(newTemplate) != ' '.join(matchCluster.logTemplate): 
+                    matchCluster.logTemplate = newTemplate
+
+            count += 1
+            if count % 1000 == 0 or count == len(self.df_log):
+                print('Processed {0:.1f}% of log lines.'.format(count * 100.0 / len(self.df_log)))
+
+
+        if not os.path.exists(self.savePath):
+            os.makedirs(self.savePath)
+
+        self.outputResult(logCluL)
+
+        print('Parsing done. [Time taken: {!s}]'.format(datetime.now() - start_time))

logparser/PreDrain/README.md

@@ -0,0 +1,94 @@
+# Drain + "Preprocessing is All You Need"
+
+## What Are the New Framework Features?
+
+Getting tired of low parsing accuracies? Our log preprocessing framework is here to save your day! Go to ```./benchmark/logparser/utils/preprocessing.py``` to check the implementation details.
+
+### More Regexes
+Our study identified several categories of variables that are not matched by the default Loghub regexes but can be identified using **consistent and generalizable** regexes. Therefore, we enriched the regex set used for log preprocessing. The regexes used in our new framework are introduced in the following table: 
+
+| Semantic       | Regex                                                                                                         | Introduction                                                           |
+|----------------|---------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------|
+| IPv4_port      | r'(/\|)(\d+\.){3}\d+(:\d+)?'                                                                                  | IPv4 addresses (optional: with port).                                  |
+| host_port      | r'([\w-]+\.)+[\w-]+\:\d+'                                                                                     | Domain host names with port.                                           |
+| package_host   | r'([\w-]+\.){2,}[\w-]+(\$[\w-]+)*(\@[\w-]+)?'                                                                 | Package (optional: with port and node)/Domain host names without port. |
+| Mac_address    | r'^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$'                                                                  | MAC addresses.                                                         |
+| IPv6           | r'(([0-9a-fA-F]{1,4}:){7}([0-9a-fA-F]{1,4}\|:)\|(([0-9a-fA-F]{1,4}:){1,7}\|:):((:[0-9a-fA-F]{1,4}){1,7}\|:))' | IPv6 addresses.                                                        |
+| path           | r'(/\|)(([\w.-]+\|\<\*\>)/)+([\w.-]+\|\<\*\>)'                                                                | File paths.                                                            |
+| size           | r'\b\d+\.?\d*\s?([KGTMkgtm]?(B\|b)\|([KGTMkgtm]))\b'                                                          | Memory sizes.                                                          |
+| duration       | r'\b\<?\d+\s?(sec\|s\|ms)\b'                                                                                    | Time duration.                                                         |
+| block          | r'blk\_\-?\d+'                                                                                                | (System specific) Block identifier.                                    |
+| time           | r'\b\d{2}:\d{2}(:\d{2}\|:\d{2},\d+)?\b'                                                                       | Time information.                                                      |
+| date           | r'\b(\d{4}-\d{2}-\d{2})\|\d{4}/\d{2}/\d{2}\b'                                                                 | Date information.                                                      |
+| numerical      | r'\b(\-?\+?\d+\.?\d*)\b\|\b0[Xx][a-fA-F\d]+\b\|\b[a-fA-F\d]{4,}\b'                                            | Numerical values: integers, floats, or hexidecimal.                    |
+| url            | r'\bhttps?:\/\/(www\.)?[a-zA-Z0-9-]+(\.[a-zA-Z]{2,})+(:[0-9]{1,5})?(\/[^\s]*)?\b'                             | URL.                                                                   |
+| weekday_months | r'\b(%s)\b' % '\|'.join(weekday_abb+weekday+month_abb+months)                                                 | Weekdays or months (full names or abbreviations).                      |
+
+### Well Organized Orders
+The variable identification in preprocessing is done in a sequence: a token will be converted into a placeholder once it has been identified as a variable. Therefore, a poorly organized identification order may cause problems in parsing and lead to parsing accuracy decrement. For example, if time variables are detected before MAC addresses, then a MAC address "00:00:00:12:34:56" will be replaced as "<\*>:<\*>" instead of the correct form "<\*>." Therefore, we carefully organized the detection sequence as:
+```
+'url', 'IPv4_port', 'host_port', 'package_host', 'IPv6', 'Mac_address', 'time', 'path', 'block', 'date', 'duration', 'size', 'numerical', 'weekday_months'
+```
+
+### Customizable Masks
+Our framework allows users to customize the masks for variables. For example, an IPv4 address with port can be masked as either the finegrained "<\*>:<\*>" or the standard form "<\*>". The framework leverages "<\*>" for default parsing, but customizable masks can be managed using the ```regex_map``` dictionary and enabled in parsers.
+
+### Easy Knowledge Management
+Have some domain specific regexes in your mind? Add it to the regex set! Update the ```regex_match``` dictionary and ```sequence``` list to preprocess your log.
+
+## Know Your Targets (the variables)
+Loghub provides various regexes for log preprocessing. These regexes were selected based on the domain knowledge for each system. We summarized the regexes and de-duplicated them as follows: 
+![image](./plots/default-regex.png?raw=true)
+
+According to RQ1, we found that using all these default regexes is insufficient for variable detection in the preprocessing stage. Hence, we carried out a study on the non-matchable variables from Loghub-2k and manually categorized them. The two authors independently labeled a small subset and discussed the category range. Leveraging the range, the two authors then labeled the remaining variables independently and discussed the final labels. The labeled variables can be checked at ``not_matched_variables.csv``. A summary of the variable types and their ratios is available here:
+
+![image](./plots/non-matchable.png?raw=true)
+
+Our framework aims to reduce the not-matching number of generalizable (i.e., not customized or system-specific) variables (e.g., IPv6 addresses).
+
+## Dataset
+We used the smaller-scale dataset ``Loghub-2k`` for variable extraction and categorization; the framework is developed based on the findings in this dataset. To replicate the log parsing process and test the generalizability of our findings, we used the ``Loghub 2.0`` dataset for framework impact evaluation. The two datasets contain labeled log messages from 14 different systems. Both 2k and the full 2.0 version log data, along with their detailed introductions, can be found at https://github.com/logpai/loghub-2.0.
+
+## Parsing Tools
+Our work focuses on improving the performance of statistic-based parsers with **manageable, interpretable, and generalizable** knowledge provided in the preprocessing stage. According to the Loghub 2.0 results, only four statistic-based log parsers (i.e., Drain, IPLoM, LFA, and LogCluster) can parse all the full-sized log files in 12 hours. Considering the applicability of these four tools in real-life usage, we only evaluated them in our study. The implementation codes are inherited from the Loghub 2.0 repository. 
+
+
+
+## Replicate the Results
+Result replication is made easy! 
+
+### Overall Performance
+Run the following commands to obtain the parsing result and evaluations (GA, PA, FGA, FTA) on all log messages:
+
+```
+cd benchmark/
+./run_all_full.sh
+```
+
+We illustrate the evaluation results of the four statistic-based parsers in the following box plot. The blue boxes indicate the parsers with the original preprocessing function, while the yellow boxes show the results of parsers with the new preprocessing framework. The red lines show the medians and the green arrows indicate the means.
+
+![image](./plots/comparison_full.png?raw=true)
+
+### Performance on Different Complexity Subgroups
+The log messages are divided into three subgroups according to the number of variables in the message: ``#Param=0 (complexity=1)``, ``0<#Param<5 (complexity=2)``, and ``#Param>=5 (complexity=3)``. Run the following commands to obtain the parsing result and evaluations (GA, PA, FGA, FTA) on log messages in different subgroups:
+
+```
+cd benchmark/
+./run_complexity_full.sh
+```
+
+The following plot visualizes the average evaluation results of log parsers on logs with different numbers of variables. The red dot lines illustrate the original results obtained with the previous preprocessing function.
+
+![image](./plots/complexity_full_all.png?raw=true)
+
+### Performance on Different Frequency Subgroups
+We extract the messages with the most frequent 10\% and the least frequent 10\% templates and evaluate the impact brought by our framework. Run the following commands to obtain the parsing result and evaluations (GA, PA, FGA, FTA) on log messages in different subgroups:
+
+```
+cd benchmark/
+./run_frequency_full.sh
+```
+
+The following plot visualizes the average evaluation results of log parsers on logs with different frequencies (i.e., the most frequent 10% and the least frequent 10%.) The red dot lines illustrate the original results obtained with the previous preprocessing function.
+
+![image](./plots/frequency_full_all.png?raw=true)

logparser/PreDrain/__init__.py

@@ -0,0 +1 @@
+from .PreDrain import *

logparser/PreDrain/benchmark.py

@@ -0,0 +1,155 @@
+# =========================================================================
+# Copyright (C) 2016-2023 LOGPAI (https://github.com/logpai).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========================================================================
+
+
+import sys
+sys.path.append("../../")
+from logparser.PreDrain import LogParser
+from logparser.utils import evaluator
+import os
+import pandas as pd
+
+
+input_dir = "../../data/loghub_2k/"  # The input directory of log file
+output_dir = "PreDrain_result/"  # The output directory of parsing results
+
+
+benchmark_settings = {
+    "HDFS": {
+        "log_file": "HDFS/HDFS_2k.log",
+        "log_format": "<Date> <Time> <Pid> <Level> <Component>: <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Hadoop": {
+        "log_file": "Hadoop/Hadoop_2k.log",
+        "log_format": "<Date> <Time> <Level> \[<Process>\] <Component>: <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Spark": {
+        "log_file": "Spark/Spark_2k.log",
+        "log_format": "<Date> <Time> <Level> <Component>: <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Zookeeper": {
+        "log_file": "Zookeeper/Zookeeper_2k.log",
+        "log_format": "<Date> <Time> - <Level>  \[<Node>:<Component>@<Id>\] - <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "BGL": {
+        "log_file": "BGL/BGL_2k.log",
+        "log_format": "<Label> <Timestamp> <Date> <Node> <Time> <NodeRepeat> <Type> <Component> <Level> <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "HPC": {
+        "log_file": "HPC/HPC_2k.log",
+        "log_format": "<LogId> <Node> <Component> <State> <Time> <Flag> <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Thunderbird": {
+        "log_file": "Thunderbird/Thunderbird_2k.log",
+        "log_format": "<Label> <Timestamp> <Date> <User> <Month> <Day> <Time> <Location> <Component>(\[<PID>\])?: <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Windows": {
+        "log_file": "Windows/Windows_2k.log",
+        "log_format": "<Date> <Time>, <Level>                  <Component>    <Content>",
+        "st": 0.7,
+        "depth": 5,
+    },
+    "Linux": {
+        "log_file": "Linux/Linux_2k.log",
+        "log_format": "<Month> <Date> <Time> <Level> <Component>(\[<PID>\])?: <Content>",
+        "st": 0.39,
+        "depth": 6,
+    },
+    "Android": {
+        "log_file": "Android/Android_2k.log",
+        "log_format": "<Date> <Time>  <Pid>  <Tid> <Level> <Component>: <Content>",
+        "st": 0.2,
+        "depth": 6,
+    },
+    "HealthApp": {
+        "log_file": "HealthApp/HealthApp_2k.log",
+        "log_format": "<Time>\|<Component>\|<Pid>\|<Content>",
+        "st": 0.2,
+        "depth": 4,
+    },
+    "Apache": {
+        "log_file": "Apache/Apache_2k.log",
+        "log_format": "\[<Time>\] \[<Level>\] <Content>",
+        "st": 0.5,
+        "depth": 4,
+    },
+    "Proxifier": {
+        "log_file": "Proxifier/Proxifier_2k.log",
+        "log_format": "\[<Time>\] <Program> - <Content>",
+        "st": 0.6,
+        "depth": 3,
+    },
+    "OpenSSH": {
+        "log_file": "OpenSSH/OpenSSH_2k.log",
+        "log_format": "<Date> <Day> <Time> <Component> sshd\[<Pid>\]: <Content>",
+        "st": 0.6,
+        "depth": 5,
+    },
+    "OpenStack": {
+        "log_file": "OpenStack/OpenStack_2k.log",
+        "log_format": "<Logrecord> <Date> <Time> <Pid> <Level> <Component> \[<ADDR>\] <Content>",
+        "st": 0.5,
+        "depth": 5,
+    },
+    "Mac": {
+        "log_file": "Mac/Mac_2k.log",
+        "log_format": "<Month>  <Date> <Time> <User> <Component>\[<PID>\]( \(<Address>\))?: <Content>",
+        "st": 0.7,
+        "depth": 6,
+    },
+}
+
+bechmark_result = []
+for dataset, setting in benchmark_settings.items():
+    print("\n=== Evaluation on %s ===" % dataset)
+    indir = os.path.join(input_dir, os.path.dirname(setting["log_file"]))
+    log_file = os.path.basename(setting["log_file"])
+
+    parser = LogParser(
+        log_format=setting["log_format"],
+        indir=indir,
+        outdir=output_dir,
+        depth=setting["depth"],
+        st=setting["st"],
+    )
+    parser.parse(log_file)
+
+    F1_measure, accuracy = evaluator.evaluate(
+        groundtruth=os.path.join(indir, log_file + "_structured.csv"),
+        parsedresult=os.path.join(output_dir, log_file + "_structured.csv"),
+    )
+    bechmark_result.append([dataset, F1_measure, accuracy])
+
+
+print("\n=== Overall evaluation results ===")
+df_result = pd.DataFrame(bechmark_result, columns=["Dataset", "F1_measure", "Accuracy"])
+df_result.set_index("Dataset", inplace=True)
+print(df_result)
+df_result.to_csv("PreDrain_bechmark_result.csv", float_format="%.6f")

logparser/PreDrain/demo.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import sys
+sys.path.append('../../')
+from logparser.PreDrain import LogParser
+
+input_dir  = '../../data/loghub_2k/HDFS/' # The input directory of log file
+output_dir = 'demo_result/'  # The output directory of parsing results
+log_file   = 'HDFS_2k.log'  # The input log file name
+log_format = '<Date> <Time> <Pid> <Level> <Component>: <Content>'  # HDFS log format
+st         = 0.5  # Similarity threshold
+depth      = 4  # Depth of all leaf nodes
+
+parser = LogParser(log_format, indir=input_dir, outdir=output_dir,  depth=depth, st=st)
+parser.parse(log_file)