fix: fix CISE_Alarm messages parsing (#2609)

wojtekzyla · web-flow · commit 3ee8fa068538 · 2024-10-21T16:31:52.000+02:00
diff --git a/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf b/package/etc/conf.d/conflib/post-filter/app-postfilter-cisco_ise.conf
@@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() {
 application app-postfilter-cisco_ise[sc4s-finalfilter] {
 	filter {
         program('CISE_' type(string) flags(prefix))
-        and "${.values.num}" != 1;
+        and "${.values.num}" != 1
+        and not program('CISE_Alarm');
     };
     parser { app-postfilter-cisco_ise(); };
 };
diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf b/package/etc/conf.d/conflib/syslog/app-syslog-cisco_ise.conf
@@ -18,6 +18,27 @@ parser ise_event_time {
 block parser app-syslog-cisco_ise() {
 
     channel {
+
+        if {
+            parser {
+                regexp-parser(
+                    template("${MESSAGE}")
+                    patterns("^(?<real_day>\\d{2}) (?<real_hour>\\d{2}:\\d{2}:\\d{2}) (?<real_host>[^ ]+) (?<real_program>[^ ]+) (?<rest_of_message>.*)")
+                    prefix(".parsed.")
+                );
+
+                date-parser-nofilter(
+                    format('%b %d %H:%M:%S')
+                    template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}")
+                );
+            };
+            rewrite {
+                set("${.parsed.real_host}" value("HOST"));
+                set("${.parsed.real_program}" value("PROGRAM"));
+                set("${.parsed.rest_of_message}" value("MESSAGE"));
+            };
+        };
+
         parser {
             csv-parser(
                 columns(serial, num, seq, message)
@@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() {
                 product('ise')
             );
         };
-
-
-   };
+    };
 };
+
 application app-syslog-cisco_ise[sc4s-syslog-pgm] {
 	filter {
-        program('CISE_' type(string) flags(prefix));
+        program('CISE_' type(string) flags(prefix))
+        or message('CISE_' type(string) flags(substring));
     };
     parser { app-syslog-cisco_ise(); };
 };
diff --git a/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf b/package/lite/etc/addons/cisco/app-postfilter-cisco_ise.conf
@@ -57,7 +57,8 @@ block parser app-postfilter-cisco_ise() {
 application app-postfilter-cisco_ise[sc4s-finalfilter] {
 	filter {
         program('CISE_' type(string) flags(prefix))
-        and "${.values.num}" != 1;
+        and "${.values.num}" != 1
+        and not program('CISE_Alarm');
     };
     parser { app-postfilter-cisco_ise(); };
 };
diff --git a/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf b/package/lite/etc/addons/cisco/app-syslog-cisco_ise.conf
@@ -18,6 +18,27 @@ parser ise_event_time {
 block parser app-syslog-cisco_ise() {
 
     channel {
+
+        if {
+            parser {
+                regexp-parser(
+                    template("${MESSAGE}")
+                    patterns("^(?<real_day>\\d{2}) (?<real_hour>\\d{2}:\\d{2}:\\d{2}) (?<real_host>[^ ]+) (?<real_program>[^ ]+) (?<rest_of_message>.*)")
+                    prefix(".parsed.")
+                );
+
+                date-parser-nofilter(
+                    format('%b %d %H:%M:%S')
+                    template("${PROGRAM} ${.parsed.real_day} ${.parsed.real_hour}")
+                );
+            };
+            rewrite {
+                set("${.parsed.real_host}" value("HOST"));
+                set("${.parsed.real_program}" value("PROGRAM"));
+                set("${.parsed.rest_of_message}" value("MESSAGE"));
+            };
+        };
+
         parser {
             csv-parser(
                 columns(serial, num, seq, message)
@@ -44,13 +65,13 @@ block parser app-syslog-cisco_ise() {
                 product('ise')
             );
         };
-
-
-   };
+    };
 };
+
 application app-syslog-cisco_ise[sc4s-syslog-pgm] {
 	filter {
-        program('CISE_' type(string) flags(prefix));
+        program('CISE_' type(string) flags(prefix))
+        or message('CISE_' type(string) flags(substring));
     };
     parser { app-syslog-cisco_ise(); };
 };
diff --git a/tests/test_cisco_ise.py b/tests/test_cisco_ise.py
@@ -207,7 +207,7 @@ def test_cisco_ise_cise_alarm_single(
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
     st = env.from_string(
-        'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "Server=10.0.0.5"'
+        'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_Alarm WARN: RADIUS Authentication Request dropped : Server=10.0.0.5;"'
     )
     search = st.render(epoch=epoch, host=host)
 
@@ -218,3 +218,102 @@ def test_cisco_ise_cise_alarm_single(
     record_property("message", message)
 
     assert result_count == 1
+
+@pytest.mark.addons("cisco")
+def test_cisco_ise_double_timestamp_and_hostname(
+    record_property,  setup_splunk, setup_sc4s
+):
+    host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}"
+
+    dt = datetime.datetime.now()
+    _, bsd, time, date, tzoffset, _, epoch = time_operations(dt)
+
+    # Tune time functions for Cisco ISE
+    time = time[:-3]
+    tzoffset = tzoffset[0:3] + ":" + tzoffset[3:]
+    epoch = epoch[:-3]
+
+    mt = env.from_string(
+        "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 1 4 2020-01-01 10:00:00.000000 +00:00 0000015291 70501 NOTICE System-Stats: ISE Counters, ConfigVersionId=1, OperationCounters=Counter=1_LocalEndPointReads:1]\n"
+    )
+    message = mt.render(
+        mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
+    )
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" "CISE_System_Statistics: 0000001313 1 4 2020-01-01 10:00:00.000000"'
+    )
+    search = st.render(epoch=epoch, host=host)
+
+    result_count, _ = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", result_count)
+    record_property("message", message)
+
+    assert result_count == 1
+
+@pytest.mark.addons("cisco")
+def test_cisco_ise_double_timestamp_and_hostname_sequence_eq_0(
+    record_property,  setup_splunk, setup_sc4s
+):
+    host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}"
+
+    dt = datetime.datetime.now()
+    _, bsd, time, date, tzoffset, _, epoch = time_operations(dt)
+
+    # Tune time functions for Cisco ISE
+    time = time[:-3]
+    tzoffset = tzoffset[0:3] + ":" + tzoffset[3:]
+    epoch = epoch[:-3]
+
+    mt = env.from_string(
+        "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 0 {{ date }} {{ time }} {{ tzoffset }} 0000015291 70501 NOTICE System-Stats: part one,\n"
+    )
+
+    message = mt.render(
+        mark="<165>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
+    )
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    # Generate new datetime for subsequent messages; not used in log path parser so actually could be anything
+    dt = datetime.datetime.now() + datetime.timedelta(seconds=1)
+    bsd = dt.strftime("%b %d %H:%M:%S")
+
+    mt = env.from_string(
+        "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 1  part two,\n"
+    )
+    message = mt.render(
+        mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
+    )
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    mt = env.from_string(
+        "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 2  part three,\n"
+    )
+    message = mt.render(
+        mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
+    )
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    mt = env.from_string(
+        "{{ mark }}{{ bsd }} wrong_host {{ bsd }} {{ host }} CISE_System_Statistics 0000001313 4 3  part four,\n"
+    )
+    message = mt.render(
+        mark="<111>", bsd=bsd, host=host, date=date, time=time, tzoffset=tzoffset
+    )
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search _time={{ epoch }} index=netauth host="{{ host }}" sourcetype="cisco:ise:syslog" one two three four'
+    )
+    search = st.render(epoch=epoch, host=host)
+
+    result_count, _ = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", result_count)
+    record_property("message", message)
+
+    assert result_count == 1
