diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index ee3c7b5..724ace7 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -5,14 +5,18 @@ jobs: name: Windows Test runs-on: windows-latest steps: - - name: Set up Go 1.17 - uses: actions/setup-go@v2 + - name: Set up Go 1.23 + uses: actions/setup-go@v4 with: - go-version: 1.17 + go-version: 1.23 + + # Caching seems to really slow down the build due to the time + # taken to save the cache. + cache: false id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build if: always() diff --git a/cmd/extract_windows.go b/cmd/extract_windows.go index 0b973d1..ae7c97b 100644 --- a/cmd/extract_windows.go +++ b/cmd/extract_windows.go @@ -161,7 +161,7 @@ func doExtract() { } defer fd.Close() - fmt.Printf("Openning message table file %v\n", message_table) + fmt.Printf("Opening message table file %v\n", message_table) reader, err := reader.NewPagedReader(fd, 4096, 100) if err != nil { diff --git a/cmd/parse.go b/cmd/parse.go index 264b5c2..dbea889 100644 --- a/cmd/parse.go +++ b/cmd/parse.go @@ -21,6 +21,9 @@ var ( parse_file_message_file = parse.Flag("messagedb", "Path to messages database."). String() + parse_file_disable_message = parse.Flag("disable_messages", "Disable message resolver."). + Bool() + start_record_id = parse.Flag("start", "First EventID to dump").Int() number_of_records = parse.Flag("number", "How many records to print"). Default("99999999").Int() @@ -78,6 +81,10 @@ func (self *parsingContext) Parse() { } func NewParsingContext() *parsingContext { + if *parse_file_disable_message { + return &parsingContext{evtx.NullResolver{}} + } + if *parse_file_message_file != "" { resolver, err := evtx.NewDBResolver(*parse_file_message_file) kingpin.FatalIfError(err, " %v", err) diff --git a/debug.go b/debug.go index f5a46bf..574839f 100644 --- a/debug.go +++ b/debug.go @@ -11,3 +11,5 @@ func debug(format string, args ...interface{}) { fmt.Printf(format, args...) } } + +func DlvBreak() {} diff --git a/evtx.go b/evtx.go index 41824a1..b442da7 100644 --- a/evtx.go +++ b/evtx.go @@ -39,6 +39,8 @@ const ( EVTX_EVENT_RECORD_MAGIC = "\x2a\x2a\x00\x00" EVTX_EVENT_RECORD_SIZE = 24 + + TemplateContext = true ) type EvtxGUID struct { @@ -65,6 +67,7 @@ type EVTXHeader struct { MinorVersion uint16 MajorVersion uint16 HeaderBlockSize uint16 + ChunkCount uint16 _ [76]byte FileFlags uint32 CheckSum uint32 @@ -84,7 +87,7 @@ type EventRecord struct { func (self *EventRecord) Parse(ctx *ParseContext) { template := ctx.NewTemplate(0) - ParseBinXML(ctx) + ParseBinXML(ctx, !TemplateContext) self.Event = template.Expand(nil) } @@ -111,6 +114,11 @@ type ChunkHeader struct { FirstEventRecID uint64 LastEventRecID uint64 HeaderSize uint32 + LastEventRecOffset uint32 + _ [4]byte + EventRecordCheckSum uint32 + _ [68]byte + CheckSum uint32 } type Chunk struct { @@ -300,7 +308,7 @@ type ParseContext struct { chunk *Chunk // A lookup table of templates we already saw in this - // chunk. Further events in the chunk well reuse the same + // chunk. Further events in the chunk will reuse the same // templates by id. knownIDs map[int]*TemplateNode } @@ -617,6 +625,7 @@ func ReadPrefixedUnicodeString(ctx *ParseContext, is_null_terminated bool) strin buffer := ctx.ConsumeBytes(count * 2) result := UTF16LEToUTF8(buffer) debug("ReadPrefixedUnicodeString exit: %x %s\n", ctx.Offset(), string(result)) + return string(result) } @@ -640,13 +649,22 @@ func ReadName(ctx *ParseContext) string { } // This is called when we open a new XML Tag. e.g. " According to [MS-EVEN6] the dependency identifier is not present + > when the element start is used in a substitution token with value + > type: Binary XML (0x21). + */ + ParseBinXML(new_ctx, !TemplateContext) ctx.SkipBytes(arg.argLen) arg_values[idx] = new_ctx.CurrentTemplate().Expand(nil) @@ -900,7 +928,10 @@ func ParseOptionalSubstitution(ctx *ParseContext) bool { return true } -func ParseBinXML(ctx *ParseContext) { +// When parsing the XML inside a template the elements has a slightly +// different structure. +// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc#414-element-start +func ParseBinXML(ctx *ParseContext, template_context bool) { debug("ParseBinXML\n") keep_going := true @@ -912,9 +943,9 @@ func ParseBinXML(ctx *ParseContext) { keep_going = false case 0x01 /* OpenStartElementToken */ : - keep_going = ParseOpenStartElement(ctx, false) + keep_going = ParseOpenStartElement(ctx, false, template_context) case 0x41: - keep_going = ParseOpenStartElement(ctx, true) + keep_going = ParseOpenStartElement(ctx, true, template_context) case 0x02: /* CloseStartElementToken */ keep_going = ParseCloseStartElement(ctx) case 0x03 /* CloseEmptyElementToken */, 0x04: /* CloseElementToken */ @@ -930,6 +961,7 @@ func ParseBinXML(ctx *ParseContext) { case 0x0B /* PIDataToken */ : case 0x0C /* TemplateInstanceToken */ : keep_going = ParseTemplateInstance(ctx) + case 0x0D /* NormalSubstitutionToken */, 0x0E: /* OptionalSubstitutionToken */ keep_going = ParseOptionalSubstitution(ctx) @@ -973,7 +1005,7 @@ func GetChunks(fd io.ReadSeeker) ([]*Chunk, error) { for offset := int64(header.HeaderBlockSize); true; offset += EVTX_CHUNK_SIZE { chunk, err := NewChunk(fd, offset) if err != nil { - if errors.Cause(err) == io.EOF { + if errors.Is(err, io.EOF) || errors.Is(err, os.ErrNotExist) { break } continue diff --git a/fixtures/CAPI2_Operational.golden b/fixtures/CAPI2_Operational.golden new file mode 100644 index 0000000..90881eb --- /dev/null +++ b/fixtures/CAPI2_Operational.golden @@ -0,0 +1,56 @@ +{ + "System": { + "Provider": { + "Name": "Microsoft-Windows-CAPI2", + "Guid": "{5bbca4a8-b209-48dc-a8c7-b23d3e5216fb}" + }, + "EventID": { + "Value": 70 + }, + "Version": 0, + "Level": 4, + "Task": 70, + "Opcode": 0, + "Keywords": 4611686018427388032, + "TimeCreated": { + "SystemTime": 1751191789.1541903 + }, + "EventRecordID": 707, + "Correlation": { + "ActivityID": "0E2A62B5-0688-42DE-9AE1-A2AD54A8CE40" + }, + "Execution": { + "ProcessID": 23708, + "ThreadID": 17980 + }, + "Channel": "Microsoft-Windows-CAPI2/Operational", + "Computer": "ILDHBZJST3", + "Security": { + "UserID": "S-1-5-21-1332095402-2705258134-1427695613-1001" + } + }, + "UserData": { + "CryptAcquireCertificatePrivateKey": { + "Certificate": { + "fileRef": "3AA55F503B946700761B3990B569E6F248C3D31D.cer", + "subjectName": "5ae3f7e8-6eef-4e9e-8fc9-94f5d0c0862b" + }, + "Flags": { + "value": "10040", + "CRYPT_ACQUIRE_SILENT_FLAG": "true", + "CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG": "true" + }, + "EventAuxInfo": { + "ProcessName": "RuntimeBroker.exe" + }, + "CorrelationAuxInfo": { + "TaskId": "{59A34F27-7A62-4C7D-9DE1-9E1F0E2382D5}", + "SeqNumber": "2" + }, + "Result": { + "value": "0" + } + } + }, + "Message": "" + } diff --git a/fixtures/Event4624_linux.golden b/fixtures/Event4624_linux.golden index 84227cd..bf76f8d 100644 --- a/fixtures/Event4624_linux.golden +++ b/fixtures/Event4624_linux.golden @@ -55,5 +55,6 @@ "VirtualAccount": "%%1843", "TargetLinkedLogonId": 0, "ElevatedToken": "%%1843" - } + }, + "Message": "" } diff --git a/fixtures/Event4624_windows.golden b/fixtures/Event4624_windows.golden index 5de312c..865dfac 100644 --- a/fixtures/Event4624_windows.golden +++ b/fixtures/Event4624_windows.golden @@ -56,5 +56,5 @@ "TargetLinkedLogonId": 0, "ElevatedToken": "%%1843" }, - "Message": "An account was successfully logged on.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1001\n\tAccount Name:\t\ttest\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t170334\n\nLogon Type:\t\t\t2\n\nImpersonation Level:\t\tImpersonation\r\n\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1002\n\tAccount Name:\t\tuser\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t6003213\n\tLogon GUID:\t\t00000000-0000-0000-0000-000000000000\n\nProcess Information:\n\tProcess ID:\t\t4764\n\tProcess Name:\t\tC:\\Windows\\System32\\svchost.exe\n\nNetwork Information:\n\tWorkstation Name:\tTESTCOMPUTER\n\tSource Network Address:\t::1\n\tSource Port:\t\t0\n\nDetailed Authentication Information:\n\tLogon Process:\t\tseclogo\n\tAuthentication Package:\tNegotiate\n\tTransited Services:\t-\n\tPackage Name (NTLM only):\t-\n\tKey Length:\t\t0\n\nThis event is generated when a logon session is created. It is generated on the computer that was accessed.\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThe network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases.\n\nThe impersonation level field indicates the extent to which a process in the logon session can impersonate.\n\nThe authentication information fields provide detailed information about this specific logon request.\n\t- Logon GUID is a unique identifier that can be used to correlate this event with a KDC event.\n\t- Transited services indicate which intermediate services have participated in this logon request.\n\t- Package name indicates which sub-protocol was used among the NTLM protocols.\n\t- Key length indicates the length of the generated session key. This will be 0 if no session key was requested.\r\n" + "Message": "An account was successfully logged on.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1001\n\tAccount Name:\t\ttest\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t170334\n\nLogon Information:\n\tLogon Type:\t\t2\n\tRestricted Admin Mode:\t-\n\tVirtual Account:\t\tNo\r\n\n\tElevated Token:\t\tNo\r\n\n\nImpersonation Level:\t\tImpersonation\r\n\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1002\n\tAccount Name:\t\tuser\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t6003213\n\tLinked Logon ID:\t\t0\n\tNetwork Account Name:\t-\n\tNetwork Account Domain:\t-\n\tLogon GUID:\t\t00000000-0000-0000-0000-000000000000\n\nProcess Information:\n\tProcess ID:\t\t4764\n\tProcess Name:\t\tC:\\Windows\\System32\\svchost.exe\n\nNetwork Information:\n\tWorkstation Name:\tTESTCOMPUTER\n\tSource Network Address:\t::1\n\tSource Port:\t\t0\n\nDetailed Authentication Information:\n\tLogon Process:\t\tseclogo\n\tAuthentication Package:\tNegotiate\n\tTransited Services:\t-\n\tPackage Name (NTLM only):\t-\n\tKey Length:\t\t0\n\nThis event is generated when a logon session is created. It is generated on the computer that was accessed.\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThe network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases.\n\nThe impersonation level field indicates the extent to which a process in the logon session can impersonate.\n\nThe authentication information fields provide detailed information about this specific logon request.\n\t- Logon GUID is a unique identifier that can be used to correlate this event with a KDC event.\n\t- Transited services indicate which intermediate services have participated in this logon request.\n\t- Package name indicates which sub-protocol was used among the NTLM protocols.\n\t- Key length indicates the length of the generated session key. This will be 0 if no session key was requested.\r\n" } diff --git a/go.mod b/go.mod index 8aa3eb4..c2c37f9 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,36 @@ module github.com/refractionPOINT/evtx require ( - github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844 + github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d github.com/alecthomas/assert v1.0.0 github.com/davecgh/go-spew v1.1.1 - github.com/hashicorp/golang-lru v0.5.4 - github.com/mattn/go-sqlite3 v1.14.10 + github.com/hashicorp/golang-lru v1.0.2 + github.com/mattn/go-sqlite3 v1.14.24 github.com/pkg/errors v0.9.1 github.com/sebdah/goldie v1.0.0 - github.com/sebdah/goldie/v2 v2.5.3 // indirect - github.com/stretchr/objx v0.3.0 // indirect - github.com/stretchr/testify v1.7.0 - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.29.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 - www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14 - www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de + www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e + www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b +) + +require ( + github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect + github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a // indirect + github.com/Velocidex/yaml/v2 v2.2.8 // indirect + github.com/alecthomas/colour v0.1.0 // indirect + github.com/alecthomas/repr v0.1.1 // indirect + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) // replace www.velocidex.com/golang/go-pe => /home/mic/projects/go-pe/ //replace github.com/Velocidex/ordereddict => /home/mic/projects/ordereddict -go 1.13 +go 1.20 diff --git a/go.sum b/go.sum index 33c54a3..01d65bb 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,20 @@ -github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844 h1:mzLkFvHqUgpZJ6M+TF4oCybAvkEEZxkQBH71BDeyHW4= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844/go.mod h1:Y5Tfx5SKGOzkulpqfonrdILSPIuNg+GqKE/DhVJgnpg= -github.com/Velocidex/pkcs7 v0.0.0-20210524015001-8d1eee94a157 h1:cNRL6O5MZdKi4i0aQxW6+7RoT34QMHFuRKpigCIHBG8= -github.com/Velocidex/pkcs7 v0.0.0-20210524015001-8d1eee94a157/go.mod h1:/fy/Eg4TQz9KkJduvZfGCnbWTQ/LKaknS2wYB52cU6c= +github.com/Velocidex/ordereddict v0.0.0-20221110130714-6a7cb85851cd/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= +github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d h1:fn372EqKyazBxYUP5HPpBi3jId4MXuppEypEALGfvEk= +github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= +github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a h1:H7dVazNcaE80V8cy99TF7LPpwzvr1uJ4I2nDjb5ek7E= +github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a/go.mod h1:/fy/Eg4TQz9KkJduvZfGCnbWTQ/LKaknS2wYB52cU6c= github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= +github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= @@ -18,8 +23,9 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -27,8 +33,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= -github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -42,19 +48,47 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= -github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -63,10 +97,12 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14 h1:jaV3eZOQvjY1JD3Plxxj11Lu/Oc0uQj/oRipFggTONM= www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14/go.mod h1:Q/J/huOyH6IlY2aShigY1CnZnw5EO0+FZJgnGEBrT5Q= -www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de h1:0jvTb9AaocZVjkqVpE/Qa8PgAW+yfp9o3KKSwWcoOr4= -www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de/go.mod h1:j9Xy8Z9wxzY2SCB8CqDkkoSzy+eUwevnOrRm/XM2q/A= +www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e h1:uf1AsYiIzUMJMIdFsVdrIw/BjrGzZbrsnz9xmeZmlYU= +www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e/go.mod h1:jk+uZGukrJZWgnNH6q9tJLUnbugHEDPCQdIOmBBMXY4= +www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b h1:hOxQYDyETh4wdnCbM9Il4X+6LwonGdLnsoznqvzw48A= +www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA= diff --git a/message_sets.go b/message_sets.go new file mode 100644 index 0000000..d2ce13d --- /dev/null +++ b/message_sets.go @@ -0,0 +1,79 @@ +package evtx + +import ( + "regexp" + "strconv" + "sync" +) + +var ( + expansionRegex = regexp.MustCompile("%[0-9]+") +) + +type MessageSet struct { + mu sync.Mutex + Provider string + Channel string + Messages map[int]string + Parameters map[int]string +} + +func (self *MessageSet) AddMessage(event_id int, message string) { + self.mu.Lock() + defer self.mu.Unlock() + + number_of_expansions := self.getLargestExpansion(message) + key := event_id<<16 | number_of_expansions + + self.Messages[key] = message +} + +func (self *MessageSet) AddParameter(event_id int, message string) { + self.mu.Lock() + defer self.mu.Unlock() + + self.Parameters[event_id] = message +} + +func (self *MessageSet) GetParameter(id int) string { + self.mu.Lock() + defer self.mu.Unlock() + + res, _ := self.Parameters[id] + return res +} + +// Calculates the largest expansion number from the message string. +func (self *MessageSet) getLargestExpansion(message string) int { + res := 0 + + for _, m := range expansionRegex.FindAllString(message, -1) { + val, err := strconv.Atoi(m[1:]) + if err == nil { + val-- + if val > res { + res = val + } + } + } + + return res +} + +// Sometimes a number of message strings are generated for each event +// id. This function finds the most appropriate message string with +// the most expansions relevant for this event. +func (self *MessageSet) GetBestMessage( + event_id, number_of_expansions int) string { + self.mu.Lock() + defer self.mu.Unlock() + + for i := number_of_expansions; i > 0; i-- { + key := event_id<<16 | i + res, pres := self.Messages[key] + if pres { + return res + } + } + return "" +} diff --git a/messages.go b/messages.go index 2eef78f..3c76517 100644 --- a/messages.go +++ b/messages.go @@ -18,14 +18,14 @@ var ( ) type MessageResolver interface { - GetMessage(provider, channel string, event_id int) string + GetMessage(provider, channel string, event_id int, number_of_expansions int) string GetParameter(provider, channel string, parameter_id int) string Close() } type NullResolver struct{} -func (self NullResolver) GetMessage(provider, channel string, event_id int) string { +func (self NullResolver) GetMessage(provider, channel string, event_id, number_of_expansions int) string { return "" } @@ -37,6 +37,9 @@ func (self NullResolver) Close() {} func flatten(dict *ordereddict.Dict) []interface{} { result := []interface{}{} + if dict == nil { + return result + } for _, k := range dict.Keys() { value, _ := dict.Get(k) @@ -80,6 +83,13 @@ func maybeExpandObjects(provider, channel string, func ExpandMessage( event *ordereddict.Dict, resolver MessageResolver) string { + // Now get and flatten the user data or event data + data, pres := ordereddict.GetMap(event, "UserData") + if !pres { + data, _ = ordereddict.GetMap(event, "EventData") + } + expansions := flatten(data) + provider, _ := ordereddict.GetString(event, "System.Provider.Name") provider_guid, _ := ordereddict.GetString(event, "System.Provider.Guid") channel, _ := ordereddict.GetString(event, "System.Channel") @@ -87,9 +97,9 @@ func ExpandMessage( // Get the raw message. First try using the GUID then using the // name if possible. - message := resolver.GetMessage(provider_guid, channel, event_id) + message := resolver.GetMessage(provider_guid, channel, event_id, len(expansions)) if message == "" { - message = resolver.GetMessage(provider, channel, event_id) + message = resolver.GetMessage(provider, channel, event_id, len(expansions)) if message == "" { // No raw message string, just return. return message @@ -99,16 +109,6 @@ func ExpandMessage( provider = provider_guid } - // Now get and flatten the user data or event data - data, pres := ordereddict.GetMap(event, "UserData") - if !pres { - data, pres = ordereddict.GetMap(event, "EventData") - if !pres { - return message - } - } - expansions := flatten(data) - // Replace expansions in the message with the user data. return expansion_re.ReplaceAllStringFunc(message, func(match string) string { switch match { diff --git a/messages_database.go b/messages_database.go index a558ed0..ad63037 100644 --- a/messages_database.go +++ b/messages_database.go @@ -11,7 +11,7 @@ type DBResolver struct { // TODO: What is happening with the channel here? func (self *DBResolver) GetMessage( - provider, channel string, event_id int) string { + provider, channel string, event_id, number_of_expansions int) string { rows, err := self.query.Query(provider, event_id) if err != nil { return "" diff --git a/messages_windows.go b/messages_windows.go index 6b55585..88dbbf5 100644 --- a/messages_windows.go +++ b/messages_windows.go @@ -6,6 +6,7 @@ package evtx import ( "os" "path/filepath" + "sort" "strings" lru "github.com/hashicorp/golang-lru" @@ -23,11 +24,21 @@ func NewWindowsMessageResolver() *WindowsMessageResolver { return &WindowsMessageResolver{ // string->MessageSet cache: cache, + + // MUI files can be found in the SxS directory - we cache that + // periodically. + mui_cache: make(map[string][]string), } } type WindowsMessageResolver struct { cache *lru.Cache + + mui_cache map[string][]string +} + +func (self *WindowsMessageResolver) buildSxScache() { + self.mui_cache = make(map[string][]string) } func (self *WindowsMessageResolver) getMessageSets( @@ -39,10 +50,10 @@ func (self *WindowsMessageResolver) getMessageSets( message_set_any, pres := self.cache.Get(key) if !pres { var err error - message_set_any, err = GetMessagesByGUID(provider, channel) + message_set_any, err = self.GetMessagesByGUID(provider, channel) if err != nil { // Try to get the messages by provider name - message_set_any, err = GetMessages(provider, channel) + message_set_any, err = self.GetMessages(provider, channel) if err != nil { // Cache the failure by storing nil in the map self.cache.Add(key, nil) @@ -61,7 +72,7 @@ func (self *WindowsMessageResolver) getMessageSets( } func (self *WindowsMessageResolver) GetMessage( - provider, channel string, event_id int) string { + provider, channel string, event_id, number_of_expansions int) string { message_set, err := self.getMessageSets(provider, channel) if err != nil { @@ -69,11 +80,7 @@ func (self *WindowsMessageResolver) GetMessage( } // Get the event if it is there - res, pres := message_set.Messages[event_id] - if pres { - return res.Message - } - return "" + return message_set.GetBestMessage(event_id, number_of_expansions) } func (self *WindowsMessageResolver) GetParameter( @@ -88,22 +95,11 @@ func (self *WindowsMessageResolver) GetParameter( return "" } - res, pres := message_set.Parameters[parameter_id] - if pres { - return res.Message - } - return "" + return message_set.GetParameter(parameter_id) } func (self *WindowsMessageResolver) Close() {} -type MessageSet struct { - Provider string - Channel string - Messages map[int]*pe.Message - Parameters map[int]*pe.Message -} - // ExpandLocations Produces a list of possible locations the message // file may be. We process all of them because sometimes event // messages are split across multiple dlls. For example, a generic @@ -175,11 +171,16 @@ func ExpandLocations(message_file string) []string { } // Message file values may be separated by ; - return include_muis(split_system32(replace_env_vars( + res := include_muis(split_system32(replace_env_vars( strings.Split(message_file, ";")))) + sort.Slice(res, func(i, j int) bool { + return len(res[i]) > len(res[j]) + }) + return res } -func GetMessagesByGUID(provider_guid, channel string) (*MessageSet, error) { +func (self *WindowsMessageResolver) GetMessagesByGUID( + provider_guid, channel string) (*MessageSet, error) { key_path := `Software\Microsoft\Windows\CurrentVersion\WinEVT\Publishers\{` + provider_guid + "}" provider_key, err := registry.OpenKey(registry.LOCAL_MACHINE, key_path, registry.READ|registry.ENUMERATE_SUB_KEYS|registry.WOW64_64KEY) @@ -212,19 +213,19 @@ func expandLocations( result := &MessageSet{ Provider: provider, Channel: channel, - Messages: make(map[int]*pe.Message), - Parameters: make(map[int]*pe.Message), + Messages: make(map[int]string), + Parameters: make(map[int]string), } - populateMessages(message_files, result.Messages) + populateMessages(message_files, result.AddMessage) if parameter_files != "" { - populateMessages(parameter_files, result.Parameters) + populateMessages(parameter_files, result.AddParameter) } return result, nil } -func populateMessages(message_files string, set map[int]*pe.Message) { +func populateMessages(message_files string, adder func(event_id int, message string)) { for _, message_file := range ExpandLocations(message_files) { fd, err := os.Open(message_file) if err != nil { @@ -249,12 +250,13 @@ func populateMessages(message_files string, set map[int]*pe.Message) { } for _, msg := range messages { - set[msg.EventId] = msg + adder(msg.EventId, msg.Message) } } } -func GetMessages(provider, channel string) (*MessageSet, error) { +func (self *WindowsMessageResolver) GetMessages( + provider, channel string) (*MessageSet, error) { root_key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Services\EventLog`, registry.READ|registry.ENUMERATE_SUB_KEYS|registry.WOW64_64KEY) diff --git a/parser_test.go b/parser_test.go index 602b2fe..57aa7eb 100644 --- a/parser_test.go +++ b/parser_test.go @@ -44,6 +44,22 @@ func (self *EVTXTestSuite) TestCollector() { goldie.Assert(self.T(), fixture_name, out) } +func (self *EVTXTestSuite) TestTemplates() { + cmdline := []string{ + "parse", "testdata/Microsoft-Windows-CAPI2_Operational_EventID70.evtx", + "--disable_messages", + } + cmd := exec.Command(self.binary, cmdline...) + out, err := cmd.CombinedOutput() + assert.NoError(self.T(), err) + + out = bytes.ReplaceAll(out, []byte{'\r', '\n'}, []byte{'\n'}) + fixture_name := "CAPI2_Operational" + fmt.Printf("Testing fixture %v\n", fixture_name) + + goldie.Assert(self.T(), fixture_name, out) +} + func TestEvtx(t *testing.T) { suite.Run(t, &EVTXTestSuite{}) } diff --git a/testdata/Microsoft-Windows-CAPI2_Operational_EventID70.evtx b/testdata/Microsoft-Windows-CAPI2_Operational_EventID70.evtx new file mode 100644 index 0000000..686234c Binary files /dev/null and b/testdata/Microsoft-Windows-CAPI2_Operational_EventID70.evtx differ