Skip to content

Commit 6862ada

Browse files
mjabascal10mjabascal10
committed
feat: add Liquibase changelogs for pfSense and SonicWall filter updates
1 parent 23adb5f commit 6862ada

4 files changed

Lines changed: 756 additions & 1 deletion

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22

33
-- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section.
44
-- Added TLS connection options and setup steps for secure Syslog integration
5-
-- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries.
5+
-- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries.
6+
–- Improved correlation rule handling for pfSense and SonicWall data sources to enhance detection accuracy and event normalization.

‎backend/src/main/resources/config/liquibase/changelog/20251107001_update_filter_pfsense.xml‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<databaseChangeLog
3+
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
6+
7+
<changeSet id="20251107001" author="JocLRojas">
8+
9+
<sql dbms="postgresql" splitStatements="true" stripComments="true">
10+
<![CDATA[
11+
12+
UPDATE public.utm_logstash_filter
13+
SET filter_version='2.1.0',
14+
logstash_filter='filter {
15+
16+
# pfSense filter version 2.1.0
17+
# Based on https://docs.netgate.com/pfsense/en/latest/monitoring/logs/raw-filter-format.html
18+
19+
split {
20+
field => "message"
21+
terminator => "<utm-log-separator>"
22+
}
23+
24+
#Looking for datasource generated by an agent and parse original message
25+
if [message] =~ /\[utm_stack_agent_ds=(.+)\]-(.+)/ {
26+
grok {
27+
match => {
28+
"message" => "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}"
29+
}
30+
}
31+
}
32+
if [original_log_message] {
33+
mutate {
34+
update => { "message" => "%{[original_log_message]}"
35+
}
36+
}
37+
}
38+
39+
if ![dataType] {
40+
# The log destination is already identified by the agent so, don''t need an entry point
41+
#......................................................................#
42+
#Generating dataType field required by CurrelationRulesEngine
43+
mutate {
44+
add_field => { "dataType" => "firewall-pfsense" }
45+
}
46+
47+
#......................................................................#
48+
#Using grok to parse header of the message
49+
grok {
50+
match => {
51+
"message" => [
52+
# Format with syslog header: <priority>version timestamp host program[pid]: data
53+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:msg_all}",
54+
# Format with syslog header: <priority>version timestamp host program pid - - data
55+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:msg_all}",
56+
# Format with syslog header: <priority>version timestamp host program pid: data
57+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:msg_all}",
58+
# Format with syslog header: <priority>version timestamp host program: data (no pid)
59+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}: %{GREEDYDATA:msg_all}",
60+
# Original pattern (legacy support): optional number at start + timestamp
61+
"(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})? %{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}",
62+
# Fallback: just timestamp and host for logs without syslog headers
63+
"%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}",
64+
# Direct CSV payload without any header
65+
"%{IPORHOST:syslog_host} %{DATA:program}.*? %{GREEDYDATA:msg_all}"
66+
]
67+
}
68+
tag_on_failure => ["_grok_header_fail"]
69+
}
70+
71+
#......................................................................#
72+
#Checking that the msg_all field exists
73+
if [msg_all] {
74+
if [message] =~ / filterlog(.+),(match|\w+),(block|pass),(in|out),(4|6)/ {
75+
grok {
76+
match => {
77+
"msg_all" => "%{WORD:event_type}(\s)?(\[)?(%{NUMBER:pid})?(\])?( - -|:) %{GREEDYDATA:csv_msg}"
78+
}
79+
}
80+
#.....................................................................#
81+
#Check if csv_msg exists and parsing it
82+
if [csv_msg] {
83+
#Changing the empty fields by X0X and then eliminating them
84+
mutate {
85+
gsub => [
86+
"csv_msg", ",,", ",X0X,"
87+
]
88+
}
89+
#The gsub is repeated, because the first time it leaves some intermediate fields empty
90+
mutate {
91+
gsub => [
92+
"csv_msg", ",,", ",X0X,"
93+
]
94+
}
95+
}
96+
} else if [message] =~ / [a-z-_\.]+( \d+)? - - / {
97+
grok {
98+
match => {
99+
"msg_all" => "%{DATA:event_type}( %{NUMBER:pid})? - - (- )?%{GREEDYDATA:msg}"
100+
}
101+
}
102+
} else if [message] =~ /\/(.+?)( \d+)? - - \((.+?)\) [A-Z]+ \((.+?)\)/ {
103+
grok {
104+
match => {
105+
"msg_all" => "%{PATH:process}( %{NUMBER:pid})? - - \(%{DATA:user}\) %{WORD:command_action} \(%{PATH:process_path}\)"
106+
}
107+
}
108+
}
109+
}
110+
111+
#......................................................................#
112+
# Rename other fields
113+
mutate {
114+
rename => {
115+
"event_type" => "[logx][pfsense][event_type]"
116+
"msg" => "[logx][pfsense][msg]"
117+
"pid" => "[logx][pfsense][pid]"
118+
"priority" => "[logx][pfsense][priority]"
119+
"message" => "[logx][pfsense][message]"
120+
}
121+
}
122+
123+
#.....................................................................#
124+
#Generating dataSource field required by Correlation Engine
125+
if [syslog_host] {
126+
mutate {
127+
rename => { "syslog_host" => "[logx][pfsense][syslog_host]" }
128+
update => { "dataSource" => "%{[logx][pfsense][syslog_host]}" }
129+
}
130+
}
131+
if (![dataSource]){
132+
mutate {
133+
add_field => { "dataSource" => "%{host}" }
134+
}
135+
}
136+
137+
if [logx][pfsense][message] and [logx][pfsense][message] =~ /^</ {
138+
grok {
139+
match => {
140+
"[logx][pfsense][message]" => [
141+
"<%{NUMBER:priority}>%{INT:syslog_version} %{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}(?: %{NUMBER:pid})? - - %{GREEDYDATA:payload}",
142+
143+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:payload}",
144+
145+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:payload}",
146+
147+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:payload}",
148+
149+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}: %{GREEDYDATA:payload}",
150+
151+
"<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}.*? %{GREEDYDATA:payload}"
152+
]
153+
}
154+
tag_on_failure => ["_grok2_fail"]
155+
}
156+
if !("_grok2_fail" in [tags]) and [payload] {
157+
mutate {
158+
replace => { "[logx][pfsense][message]" => "%{[payload]}" }
159+
}
160+
}
161+
}
162+
#......................................................................#
163+
#Remove unnecessary fields
164+
mutate {
165+
remove_field => [
166+
"@version","timestamp","payload","deviceTime","path","type","syslog_version","syslog_host","not_defined",
167+
"pid","priority","program","event","csv_msg","csv_field","msg_all","original_log_message","tags","headers"
168+
]
169+
}
170+
}
171+
172+
if [logx][pfsense][message] {
173+
174+
#......................................................................#
175+
#IPv4 branches
176+
if [logx][pfsense][message] =~ /,4,/ {
177+
178+
#......................................................................#
179+
#IPv4 ICMP: map ICMP-specific tail
180+
if [logx][pfsense][message] =~ /,icmp,/ {
181+
csv {
182+
source => "[logx][pfsense][message]"
183+
target => "[logx][pfsense]"
184+
separator => ","
185+
skip_empty_columns => true
186+
columns => [
187+
"rule_number","sub_rule_number","anchor","tracker","interface","reason",
188+
"action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl",
189+
"ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length",
190+
"src_ip","dest_ip",
191+
"icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5"
192+
]
193+
}
194+
195+
#......................................................................#
196+
#IPv4 TCP/UDP: share list; TCP fills extra fields, UDP leaves them nil
197+
} else {
198+
csv {
199+
source => "[logx][pfsense][message]"
200+
target => "[logx][pfsense]"
201+
separator => ","
202+
skip_empty_columns => true
203+
columns => [
204+
"rule_number","sub_rule_number","anchor","tracker","interface","reason",
205+
"action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl",
206+
"ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length",
207+
"src_ip","dest_ip",
208+
"src_port","dest_port","data_length",
209+
"tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options"
210+
]
211+
}
212+
}
213+
}
214+
215+
#......................................................................#
216+
#IPv6 branches
217+
else if [logx][pfsense][message] =~ /,6,/ {
218+
219+
#......................................................................#
220+
#IPv6 ICMP
221+
if [logx][pfsense][message] =~ /,icmp,/ {
222+
csv {
223+
source => "[logx][pfsense][message]"
224+
target => "[logx][pfsense]"
225+
separator => ","
226+
skip_empty_columns => true
227+
columns => [
228+
"rule_number","sub_rule_number","anchor","tracker","interface","reason",
229+
"action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit",
230+
"proto","ipv6_protocol_id","ip_length","src_ip","dest_ip",
231+
"icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5"
232+
]
233+
}
234+
235+
#......................................................................#
236+
# IPv6 TCP/UDP
237+
} else {
238+
csv {
239+
source => "[logx][pfsense][message]"
240+
target => "[logx][pfsense]"
241+
separator => ","
242+
skip_empty_columns => true
243+
columns => [
244+
"rule_number","sub_rule_number","anchor","tracker","interface","reason",
245+
"action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit",
246+
"proto","ipv6_protocol_id","ip_length","src_ip","dest_ip",
247+
"src_port","dest_port","data_length",
248+
"tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options"
249+
]
250+
}
251+
}
252+
}
253+
254+
#......................................................................#
255+
#Generating action field for established connections
256+
if [logx][pfsense][action] == "pass" {
257+
mutate { add_field => { "[logx][utm][action]" => "Success" } }
258+
}
259+
}
260+
mutate {
261+
remove_field => "[logx][pfsense][message]"
262+
}
263+
}'
264+
WHERE id=1522;
265+
]]>
266+
</sql>
267+
</changeSet>
268+
</databaseChangeLog>

0 commit comments

Comments
 (0)