Monitoring Guide

wget -O splunk-10.2.2-80b90d638de6-linux-amd64.deb \
  "https://download.splunk.com/products/splunk/releases/10.2.2/linux/splunk-10.2.2-80b90d638de6-linux-amd64.deb"

sudo dpkg -i splunk-10.2.2-80b90d638de6-linux-amd64.deb

sudo /opt/splunk/bin/splunk start --accept-license

sudo /opt/splunk/bin/splunk enable boot-start -user splunk
# or as root without a dedicated user:
sudo /opt/splunk/bin/splunk enable boot-start

sudo ufw allow 8000/tcp
sudo ufw allow 9997/tcp
sudo ufw allow 8089/tcp

# Copy the wget command from the download page above, then:
sudo dpkg -i splunkforwarder-<version>-linux-amd64.deb

sudo /opt/splunkforwarder/bin/splunk start --accept-license --answer-yes

# Add the Splunk server as the receiver (adjust the IP!)
sudo /opt/splunkforwarder/bin/splunk add forward-server \
  <SPLUNK-SERVER-IP>:9997 -auth admin:changeme

# Syslog
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/syslog \
  -index main -sourcetype syslog

# Auth log (SSH logins, sudo etc.)
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/auth.log \
  -index main -sourcetype linux_secure

# Kernel log
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/kern.log \
  -index main -sourcetype syslog

sudo /opt/splunkforwarder/bin/splunk restart

# MSI in the same directory as the PS script
msiexec.exe /i splunkforwarder-10.2.2-x64.msi AGREETOLICENSE=Yes `
  SPLUNKUSERNAME=admin SPLUNKPASSWORD=changeme `
  RECEIVING_INDEXER="<SPLUNK-SERVER-IP>:9997" `
  /quiet /L*v splunk_install.log

# Add Windows Event Logs
& "C:\Program Files\SplunkUniversalForwarder\bin\splunk.exe" `
  add monitor "C:\Windows\System32\winevt\Logs\Security.evtx" `
  -index main -sourcetype WinEventLog:Security

& "C:\Program Files\SplunkUniversalForwarder\bin\splunk.exe" `
  add monitor "C:\Windows\System32\winevt\Logs\System.evtx" `
  -index main -sourcetype WinEventLog:System

sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/apache2/access.log \
  -index web -sourcetype apache_combined

sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/nginx/access.log \
  -index web -sourcetype nginx_access

sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/nginx/error.log \
  -index web -sourcetype nginx_error

# MySQL slow query + error log
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/mysql/error.log \
  -index db -sourcetype mysql:error

# PostgreSQL
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/postgresql/ \
  -index db -sourcetype postgresql

# Syslog
[monitor:///var/log/syslog]
index = main
sourcetype = syslog
disabled = false

# Auth
[monitor:///var/log/auth.log]
index = main
sourcetype = linux_secure
disabled = false

# Apache
[monitor:///var/log/apache2/access.log]
index = web
sourcetype = apache_combined
disabled = false

[tcpout]
defaultGroup = my_indexers

[tcpout:my_indexers]
server = <SPLUNK-SERVER-IP>:9997

# Name          Content
main             Linux syslog, auth · general
web              Apache, Nginx access/error logs
db               MySQL, PostgreSQL logs
wineventlog      Windows Event Logs

index=* earliest=-15m

index=main sourcetype=linux_secure "Failed password"
| stats count by src_ip, user
| sort -count

index=wineventlog EventCode=4625
| stats count by Account_Name, src_ip
| sort -count

index=* earliest=-1h
| stats count by host, sourcetype
| sort -count
| head 10

title: LSASS Memory Access by Non-System Process
status: experimental
logsource:
  product: windows
  category: process_access
detection:
  selection:
    TargetImage|endswith: '\lsass.exe'
    GrantedAccess|contains:
      - '0x1010'
      - '0x1410'
      - '0x1438'
      - '0x143a'
  filter:
    SourceImage|startswith:
      - 'C:\Windows\System32\'
      - 'C:\Windows\SysWOW64\'
  condition: selection and not filter
falsepositives:
  - AV/EDR solutions
level: critical
tags:
  - attack.credential_access
  - attack.t1003.001

title: Kerberoasting RC4 Downgrade
status: stable
logsource:
  product: windows
  service: security
detection:
  selection:
    EventID: 4769
    ServiceName|not-startswith: '$'
    TicketEncryptionType: '0x17'
    TicketOptions: '0x40810000'
  condition: selection
falsepositives:
  - Legacy systems requiring RC4
level: high
tags:
  - attack.credential_access
  - attack.t1558.003

title: Pass-the-Hash NTLM Logon
status: stable
logsource:
  product: windows
  service: security
detection:
  selection:
    EventID: 4624
    LogonType: 3
    AuthenticationPackageName: 'NTLM'
    LogonProcessName: 'NtLmSsp'
  filter:
    AccountName|endswith: '$'
  condition: selection and not filter
falsepositives:
  - Legacy NTLM authentication
level: high
tags:
  - attack.lateral_movement
  - attack.t1550.002

title: Golden Ticket - krbtgt Account Activity
status: experimental
logsource:
  product: windows
  service: security
detection:
  selection_tgt:
    EventID: 4769
    ServiceName: 'krbtgt'
  selection_anomaly:
    EventID: 4624
    LogonType: 3
    KeyLength: 0
  condition: selection_tgt or selection_anomaly
level: critical
tags:
  - attack.privilege_escalation
  - attack.t1558.001

title: Metasploit Default Reverse Shell Port
status: experimental
logsource:
  product: windows
  category: network_connection
detection:
  selection:
    Initiated: 'true'
    DestinationPort:
      - 4444
      - 4445
      - 4443
  filter:
    Image|startswith:
      - 'C:\Windows\'
      - 'C:\Program Files\'
  condition: selection and not filter
level: high
tags:
  - attack.command_and_control
  - attack.t1571

title: Suspicious PowerShell Execution
status: stable
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    Image|endswith: '\powershell.exe'
    CommandLine|contains:
      - '-EncodedCommand'
      - '-enc '
      - '-NoProfile'
      - 'IEX'
      - 'Invoke-Expression'
      - 'DownloadString'
  condition: selection
level: high
tags:
  - attack.execution
  - attack.t1059.001

title: NTDS Dump via ntdsutil or vssadmin
status: stable
logsource:
  product: windows
  category: process_creation
detection:
  selection_ntdsutil:
    Image|endswith: '\ntdsutil.exe'
    CommandLine|contains: 'ac i ntds'
  selection_vss:
    Image|endswith: '\vssadmin.exe'
    CommandLine|contains:
      - 'create shadow'
      - 'list shadows'
  condition: selection_ntdsutil or selection_vss
level: critical
tags:
  - attack.credential_access
  - attack.t1003.003

title: Ruby Script Execution from Non-Standard Path
status: experimental
logsource:
  product: windows
  category: process_creation
detection:
  selection:
    Image|endswith: '\ruby.exe'
    CommandLine|contains: '.rb'
  filter:
    CommandLine|contains:
      - 'C:\Ruby'
      - 'C:\Program Files\Ruby'
  condition: selection and not filter
level: medium
tags:
  - attack.execution
  - attack.t1059

tcp.port == 4444 or tcp.port == 4445 or tcp.port == 4443

(http.request or http.response) and
(http.user_agent contains "Mozilla/4.0 (compatible; MSIE 6.0" or
 http.user_agent contains "Mozilla/5.0 (Windows NT 6.1; Trident")

smb2.cmd == 0x0005 and smb2.flags.response == 0

kerberos.etype == 23

ldap and ldap.searchRequest and frame.len > 200

dns.qry.name matches "[a-z0-9]{20,}\." and dns.flags.response == 0

ntlmssp.auth.ntresponse and not ntlmssp.negotiate

http.request and not (ip.dst == <your_internal_range>)

tcp.flags == 0x002 and ip.dst != <internal_range>

sudo /opt/splunkforwarder/bin/splunk add monitor /opt/zeek/logs/current/ \
  -index network -sourcetype zeek

# Suricata EVE JSON
sudo /opt/splunkforwarder/bin/splunk add monitor /var/log/suricata/eve.json \
  -index ids -sourcetype suricata:eve

index=wineventlog EventCode=4688
ParentImage IN ("*\\cmd.exe", "*\\powershell.exe", "*\\wscript.exe", "*\\cscript.exe")
NewProcessName IN ("*\\net.exe", "*\\whoami.exe", "*\\ipconfig.exe", "*\\systeminfo.exe")
| table _time, Computer, ParentImage, NewProcessName, CommandLine

index=wineventlog EventCode=4104
ScriptBlockText IN ("*Invoke-Mimikatz*", "*Invoke-ReflectivePEInjection*",
                    "*Get-GPPPassword*", "*Invoke-Shellcode*")
| table _time, Computer, ScriptBlockText

index=wineventlog EventCode IN (4720, 4728, 4732, 4756)
| eval action=case(
    EventCode=4720, "User account created",
    EventCode=4728, "Added to global privileged group",
    EventCode=4732, "Added to local privileged group",
    EventCode=4756, "Added to universal group"
  )
| table _time, Computer, SubjectUserName, TargetUserName, action

index=wineventlog EventCode=4688
NewProcessName IN ("*\\ntdsutil.exe", "*\\vssadmin.exe")
| eval alert=case(
    like(CommandLine, "%ac i ntds%"), "NTDS dump via ntdsutil · CRITICAL",
    like(CommandLine, "%create shadow%"), "VSS Shadow Copy created · HIGH",
    like(CommandLine, "%list shadows%"), "VSS enumeration · MEDIUM",
    true(), "Check manually"
  )
| table _time, Computer, CommandLine, alert