Sync with upstream/master (2025-08-12)

.github/workflows/windows.yaml

@@ -5,14 +5,18 @@ jobs:
     name: Windows Test
     runs-on: windows-latest
     steps:
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v2
+    - name: Set up Go 1.23
+      uses: actions/setup-go@v4
       with:
-        go-version: 1.17
+        go-version: 1.23
+
+        # Caching seems to really slow down the build due to the time
+        # taken to save the cache.
+        cache: false
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Build
       if: always()

cmd/extract_windows.go

@@ -14,7 +14,7 @@
 	"golang.org/x/sys/windows/registry"
 	kingpin "gopkg.in/alecthomas/kingpin.v2"
 	"www.velocidex.com/golang/binparsergen/reader"
 	"www.velocidex.com/golang/evtx"
 	pe "www.velocidex.com/golang/go-pe"
 )
 
@@ -161,7 +161,7 @@
 		}
 		defer fd.Close()
 
-		fmt.Printf("Openning message table file %v\n", message_table)
+		fmt.Printf("Opening message table file %v\n", message_table)
 
 		reader, err := reader.NewPagedReader(fd, 4096, 100)
 		if err != nil {

cmd/parse.go

@@ -21,6 +21,9 @@ var (
 	parse_file_message_file = parse.Flag("messagedb", "Path to messages database.").
 				String()
 
+	parse_file_disable_message = parse.Flag("disable_messages", "Disable message resolver.").
+					Bool()
+
 	start_record_id   = parse.Flag("start", "First EventID to dump").Int()
 	number_of_records = parse.Flag("number", "How many records to print").
 				Default("99999999").Int()
@@ -78,6 +81,10 @@ func (self *parsingContext) Parse() {
 }
 
 func NewParsingContext() *parsingContext {
+	if *parse_file_disable_message {
+		return &parsingContext{evtx.NullResolver{}}
+	}
+
 	if *parse_file_message_file != "" {
 		resolver, err := evtx.NewDBResolver(*parse_file_message_file)
 		kingpin.FatalIfError(err, " %v", err)

debug.go

@@ -11,3 +11,5 @@ func debug(format string, args ...interface{}) {
 		fmt.Printf(format, args...)
 	}
 }
+
+func DlvBreak() {}

evtx.go

@@ -39,6 +39,8 @@ const (
 
 	EVTX_EVENT_RECORD_MAGIC = "\x2a\x2a\x00\x00"
 	EVTX_EVENT_RECORD_SIZE  = 24
+
+	TemplateContext = true
 )
 
 type EvtxGUID struct {
@@ -65,6 +67,7 @@ type EVTXHeader struct {
 	MinorVersion    uint16
 	MajorVersion    uint16
 	HeaderBlockSize uint16
+	ChunkCount      uint16
 	_               [76]byte
 	FileFlags       uint32
 	CheckSum        uint32
@@ -84,7 +87,7 @@ type EventRecord struct {
 
 func (self *EventRecord) Parse(ctx *ParseContext) {
 	template := ctx.NewTemplate(0)
-	ParseBinXML(ctx)
+	ParseBinXML(ctx, !TemplateContext)
 
 	self.Event = template.Expand(nil)
 }
@@ -111,6 +114,11 @@ type ChunkHeader struct {
 	FirstEventRecID     uint64
 	LastEventRecID      uint64
 	HeaderSize          uint32
+	LastEventRecOffset  uint32
+	_                   [4]byte
+	EventRecordCheckSum uint32
+	_                   [68]byte
+	CheckSum            uint32
 }
 
 type Chunk struct {
@@ -300,7 +308,7 @@ type ParseContext struct {
 	chunk *Chunk
 
 	// A lookup table of templates we already saw in this
-	// chunk. Further events in the chunk well reuse the same
+	// chunk. Further events in the chunk will reuse the same
 	// templates by id.
 	knownIDs map[int]*TemplateNode
 }
@@ -617,6 +625,7 @@ func ReadPrefixedUnicodeString(ctx *ParseContext, is_null_terminated bool) strin
 	buffer := ctx.ConsumeBytes(count * 2)
 	result := UTF16LEToUTF8(buffer)
 	debug("ReadPrefixedUnicodeString exit: %x %s\n", ctx.Offset(), string(result))
+
 	return string(result)
 }
 
@@ -640,13 +649,22 @@ func ReadName(ctx *ParseContext) string {
 }
 
 // This is called when we open a new XML Tag. e.g. "<EventData".
-func ParseOpenStartElement(ctx *ParseContext, has_attr bool) bool {
+func ParseOpenStartElement(ctx *ParseContext,
+	has_attr bool, template_instance bool) bool {
+
 	debug("ParseOpenStartElement Enter: %x\n", ctx.Offset())
 	/*
 		dependencyID := ctx.ConsumeUint16()
-		elementLength := ctx.ConsumeUint32()
 	*/
-	ctx.SkipBytes(2 + 4)
+	// ctx.SkipBytes(2 + 4)
+
+	if template_instance {
+		ctx.SkipBytes(2)
+	}
+
+	elementLength := ctx.ConsumeUint32()
+	debug("ParseOpenStartElement elementLength: %x\n", elementLength)
+
 	nameBuffer := ReadName(ctx)
 
 	attributeListLength := uint32(0)
@@ -722,7 +740,8 @@ func ParseTemplateInstance(ctx *ParseContext) bool {
 	/*
 		tempResLen := ctx.ConsumeUint32()
 	*/
-	ctx.SkipBytes(4)
+	template_definition_data := int(ctx.ConsumeUint32())
+	debug("ParseTemplateInstance template_definition_data %x\n", template_definition_data)
 
 	// Template arguments should not be unreasonable here. Just cap
 	// them at a reasonable size.
@@ -735,13 +754,13 @@ func ParseTemplateInstance(ctx *ParseContext) bool {
 
 	template, pres := ctx.GetTemplateByID(short_id)
 	if !pres {
-		// longID := ctx.ConsumeBytes(16)
+		// longGUID := ctx.ConsumeBytes(16)
 		ctx.SkipBytes(16)
 		templateBodyLen := int(ctx.ConsumeUint32())
 
 		tmp_ctx := ctx.Copy()
 		template = tmp_ctx.NewTemplate(short_id)
-		ParseBinXML(tmp_ctx)
+		ParseBinXML(tmp_ctx, TemplateContext)
 
 		ctx.SkipBytes(templateBodyLen)
 		numArguments = ctx.ConsumeUint32()
@@ -837,9 +856,18 @@ func ParseTemplateInstance(ctx *ParseContext) bool {
 				str += fmt.Sprintf("-%d", ctx.ConsumeUint32())
 			}
 			arg_values[idx] = str
+
 		case 0x21: // BinXml
 			new_ctx := ctx.Copy()
-			ParseBinXML(new_ctx)
+
+			// Substitution args are not encoded in template context.
+			/*
+			 https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc#token_types
+			 > According to [MS-EVEN6] the dependency identifier is not present
+			 > when the element start is used in a substitution token with value
+			 > type: Binary XML (0x21).
+			*/
+			ParseBinXML(new_ctx, !TemplateContext)
 			ctx.SkipBytes(arg.argLen)
 
 			arg_values[idx] = new_ctx.CurrentTemplate().Expand(nil)
@@ -900,7 +928,10 @@ func ParseOptionalSubstitution(ctx *ParseContext) bool {
 	return true
 }
 
-func ParseBinXML(ctx *ParseContext) {
+// When parsing the XML inside a template the elements has a slightly
+// different structure.
+// https://github.com/libyal/libevtx/blob/main/documentation/Windows%20XML%20Event%20Log%20(EVTX).asciidoc#414-element-start
+func ParseBinXML(ctx *ParseContext, template_context bool) {
 	debug("ParseBinXML\n")
 	keep_going := true
 
@@ -912,9 +943,9 @@ func ParseBinXML(ctx *ParseContext) {
 			keep_going = false
 
 		case 0x01 /* OpenStartElementToken */ :
-			keep_going = ParseOpenStartElement(ctx, false)
+			keep_going = ParseOpenStartElement(ctx, false, template_context)
 		case 0x41:
-			keep_going = ParseOpenStartElement(ctx, true)
+			keep_going = ParseOpenStartElement(ctx, true, template_context)
 		case 0x02: /* CloseStartElementToken */
 			keep_going = ParseCloseStartElement(ctx)
 		case 0x03 /*  CloseEmptyElementToken */, 0x04: /*  CloseElementToken */
@@ -930,6 +961,7 @@ func ParseBinXML(ctx *ParseContext) {
 		case 0x0B /*  PIDataToken */ :
 		case 0x0C /*  TemplateInstanceToken */ :
 			keep_going = ParseTemplateInstance(ctx)
+
 		case 0x0D /*  NormalSubstitutionToken */, 0x0E: /*  OptionalSubstitutionToken */
 			keep_going = ParseOptionalSubstitution(ctx)
 
@@ -973,7 +1005,7 @@ func GetChunks(fd io.ReadSeeker) ([]*Chunk, error) {
 	for offset := int64(header.HeaderBlockSize); true; offset += EVTX_CHUNK_SIZE {
 		chunk, err := NewChunk(fd, offset)
 		if err != nil {
-			if errors.Cause(err) == io.EOF {
+			if errors.Is(err, io.EOF) || errors.Is(err, os.ErrNotExist) {
 				break
 			}
 			continue

fixtures/CAPI2_Operational.golden

@@ -0,0 +1,56 @@
+{
+  "System": {
+   "Provider": {
+    "Name": "Microsoft-Windows-CAPI2",
+    "Guid": "{5bbca4a8-b209-48dc-a8c7-b23d3e5216fb}"
+   },
+   "EventID": {
+    "Value": 70
+   },
+   "Version": 0,
+   "Level": 4,
+   "Task": 70,
+   "Opcode": 0,
+   "Keywords": 4611686018427388032,
+   "TimeCreated": {
+    "SystemTime": 1751191789.1541903
+   },
+   "EventRecordID": 707,
+   "Correlation": {
+    "ActivityID": "0E2A62B5-0688-42DE-9AE1-A2AD54A8CE40"
+   },
+   "Execution": {
+    "ProcessID": 23708,
+    "ThreadID": 17980
+   },
+   "Channel": "Microsoft-Windows-CAPI2/Operational",
+   "Computer": "ILDHBZJST3",
+   "Security": {
+    "UserID": "S-1-5-21-1332095402-2705258134-1427695613-1001"
+   }
+  },
+  "UserData": {
+   "CryptAcquireCertificatePrivateKey": {
+    "Certificate": {
+     "fileRef": "3AA55F503B946700761B3990B569E6F248C3D31D.cer",
+     "subjectName": "5ae3f7e8-6eef-4e9e-8fc9-94f5d0c0862b"
+    },
+    "Flags": {
+     "value": "10040",
+     "CRYPT_ACQUIRE_SILENT_FLAG": "true",
+     "CRYPT_ACQUIRE_ALLOW_NCRYPT_KEY_FLAG": "true"
+    },
+    "EventAuxInfo": {
+     "ProcessName": "RuntimeBroker.exe"
+    },
+    "CorrelationAuxInfo": {
+     "TaskId": "{59A34F27-7A62-4C7D-9DE1-9E1F0E2382D5}",
+     "SeqNumber": "2"
+    },
+    "Result": {
+     "value": "0"
+    }
+   }
+  },
+  "Message": ""
+ }

fixtures/Event4624_linux.golden

@@ -55,5 +55,6 @@
    "VirtualAccount": "%%1843",
    "TargetLinkedLogonId": 0,
    "ElevatedToken": "%%1843"
-  }
+  },
+  "Message": ""
  }

fixtures/Event4624_windows.golden

@@ -56,5 +56,5 @@
    "TargetLinkedLogonId": 0,
    "ElevatedToken": "%%1843"
   },
-  "Message": "An account was successfully logged on.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1001\n\tAccount Name:\t\ttest\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t170334\n\nLogon Type:\t\t\t2\n\nImpersonation Level:\t\tImpersonation\r\n\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1002\n\tAccount Name:\t\tuser\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t6003213\n\tLogon GUID:\t\t00000000-0000-0000-0000-000000000000\n\nProcess Information:\n\tProcess ID:\t\t4764\n\tProcess Name:\t\tC:\\Windows\\System32\\svchost.exe\n\nNetwork Information:\n\tWorkstation Name:\tTESTCOMPUTER\n\tSource Network Address:\t::1\n\tSource Port:\t\t0\n\nDetailed Authentication Information:\n\tLogon Process:\t\tseclogo\n\tAuthentication Package:\tNegotiate\n\tTransited Services:\t-\n\tPackage Name (NTLM only):\t-\n\tKey Length:\t\t0\n\nThis event is generated when a logon session is created. It is generated on the computer that was accessed.\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThe network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases.\n\nThe impersonation level field indicates the extent to which a process in the logon session can impersonate.\n\nThe authentication information fields provide detailed information about this specific logon request.\n\t- Logon GUID is a unique identifier that can be used to correlate this event with a KDC event.\n\t- Transited services indicate which intermediate services have participated in this logon request.\n\t- Package name indicates which sub-protocol was used among the NTLM protocols.\n\t- Key length indicates the length of the generated session key. This will be 0 if no session key was requested.\r\n"
+  "Message": "An account was successfully logged on.\n\nSubject:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1001\n\tAccount Name:\t\ttest\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t170334\n\nLogon Information:\n\tLogon Type:\t\t2\n\tRestricted Admin Mode:\t-\n\tVirtual Account:\t\tNo\r\n\n\tElevated Token:\t\tNo\r\n\n\nImpersonation Level:\t\tImpersonation\r\n\n\nNew Logon:\n\tSecurity ID:\t\tS-1-5-21-546003962-2713609280-610790815-1002\n\tAccount Name:\t\tuser\n\tAccount Domain:\t\tTESTCOMPUTER\n\tLogon ID:\t\t6003213\n\tLinked Logon ID:\t\t0\n\tNetwork Account Name:\t-\n\tNetwork Account Domain:\t-\n\tLogon GUID:\t\t00000000-0000-0000-0000-000000000000\n\nProcess Information:\n\tProcess ID:\t\t4764\n\tProcess Name:\t\tC:\\Windows\\System32\\svchost.exe\n\nNetwork Information:\n\tWorkstation Name:\tTESTCOMPUTER\n\tSource Network Address:\t::1\n\tSource Port:\t\t0\n\nDetailed Authentication Information:\n\tLogon Process:\t\tseclogo\n\tAuthentication Package:\tNegotiate\n\tTransited Services:\t-\n\tPackage Name (NTLM only):\t-\n\tKey Length:\t\t0\n\nThis event is generated when a logon session is created. It is generated on the computer that was accessed.\n\nThe subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe.\n\nThe logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network).\n\nThe New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on.\n\nThe network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases.\n\nThe impersonation level field indicates the extent to which a process in the logon session can impersonate.\n\nThe authentication information fields provide detailed information about this specific logon request.\n\t- Logon GUID is a unique identifier that can be used to correlate this event with a KDC event.\n\t- Transited services indicate which intermediate services have participated in this logon request.\n\t- Package name indicates which sub-protocol was used among the NTLM protocols.\n\t- Key length indicates the length of the generated session key. This will be 0 if no session key was requested.\r\n"
  }

go.mod

@@ -1,23 +1,36 @@
 module github.com/refractionPOINT/evtx
 
 require (
-	github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844
+	github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d
 	github.com/alecthomas/assert v1.0.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/hashicorp/golang-lru v0.5.4
-	github.com/mattn/go-sqlite3 v1.14.10
+	github.com/hashicorp/golang-lru v1.0.2
+	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/pkg/errors v0.9.1
 	github.com/sebdah/goldie v1.0.0
-	github.com/sebdah/goldie/v2 v2.5.3 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/sys v0.29.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	www.velocidex.com/golang/binparsergen v0.1.1-0.20220107080050-ae6122c5ed14
-	www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de
+	www.velocidex.com/golang/binparsergen v0.1.1-0.20240404114946-8f66c7cf586e
+	www.velocidex.com/golang/go-pe v0.1.1-0.20250101153735-7a925ba8334b
+)
+
+require (
+	github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect
+	github.com/Velocidex/pkcs7 v0.0.0-20230220112103-d4ed02e1862a // indirect
+	github.com/Velocidex/yaml/v2 v2.2.8 // indirect
+	github.com/alecthomas/colour v0.1.0 // indirect
+	github.com/alecthomas/repr v0.1.1 // indirect
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sergi/go-diff v1.2.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 // replace www.velocidex.com/golang/go-pe => /home/mic/projects/go-pe/
 //replace github.com/Velocidex/ordereddict => /home/mic/projects/ordereddict
 
-go 1.13
+go 1.20