diff --git a/README.md b/README.md index f0949ff..7b3bc0d 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,12 @@ Want to add the OpenGraph schema to your JSON document? ```json { - "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/payload-schema.json" + "$schema": "https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/schema.json" } ``` Most editors will ask you to trust the schema's source. Be sure to add the following URL to your trusted domains ```text -https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/validator/jsonschema/ +https://raw.githubusercontent.com/SpecterOps/chow/refs/heads/main/pkg/payload/jsonschema/ ``` diff --git a/go.mod b/go.mod index df22e5a..a2e7b10 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,10 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/text v0.34.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + golang.org/x/text v0.35.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6116b92..b5a1cf9 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,29 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index da5c228..fb8451f 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,7 @@ import ( "os" "strings" - "github.com/specterops/chow/pkg/validator" + "github.com/specterops/chow/pkg/payload" ) var ( @@ -40,13 +40,13 @@ func main() { } defer reader.Close() - jsonSchema, err := validator.LoadIngestSchema() + jsonSchema, err := payload.LoadSchema() if err != nil { slog.Error("Failed to load ingest schema", slog.String("err", err.Error())) os.Exit(1) } - v := validator.NewValidator(reader, jsonSchema) + v := payload.NewValidator(reader, jsonSchema) _, report, err := v.ParseAndValidate() validationFailed := err != nil @@ -76,7 +76,7 @@ func main() { } } -func outputReport(w io.WriteCloser, report validator.ValidationReport) error { +func outputReport(w io.WriteCloser, report payload.ValidationReport) error { for _, e := range report.CriticalErrors { _, err := w.Write([]byte(formatCriticalError(e))) if err != nil { @@ -108,11 +108,11 @@ func outputReport(w io.WriteCloser, report validator.ValidationReport) error { return nil } -func formatCriticalError(e validator.CriticalError) string { +func formatCriticalError(e payload.CriticalError) string { return fmt.Sprintf("CRITICAL ERROR:\n%s\n%v", e.Message, e.Error) } -func formatValidationError(valErr validator.ValidationError) (string, error) { +func formatValidationError(valErr payload.ValidationError) (string, error) { var ( sb strings.Builder objBytes bytes.Buffer diff --git a/pkg/validator/jsonschema/edge.json b/pkg/payload/jsonschema/edge.json similarity index 96% rename from pkg/validator/jsonschema/edge.json rename to pkg/payload/jsonschema/edge.json index 8e9937e..417b1ae 100644 --- a/pkg/validator/jsonschema/edge.json +++ b/pkg/payload/jsonschema/edge.json @@ -57,6 +57,9 @@ }, "kind": { "type": "string", + "not": { + "pattern": "^[Tt][Aa][Gg](?:_|$)" + }, "description": "Optional kind filter; the referenced node must have this kind." } }, @@ -105,7 +108,10 @@ "kind": { "type": "string", "description": "Edge kind name must contain only alphanumeric characters and underscores.", - "pattern": "^[A-Za-z0-9_]+$" + "pattern": "^[A-Za-z0-9_]+$", + "not": { + "pattern": "^[Tt][Aa][Gg](?:_|$)" + } }, "properties": { "$ref": "#/$defs/property_map" diff --git a/pkg/validator/jsonschema/metadata.json b/pkg/payload/jsonschema/metadata.json similarity index 100% rename from pkg/validator/jsonschema/metadata.json rename to pkg/payload/jsonschema/metadata.json diff --git a/pkg/validator/jsonschema/node.json b/pkg/payload/jsonschema/node.json similarity index 94% rename from pkg/validator/jsonschema/node.json rename to pkg/payload/jsonschema/node.json index ba87bbc..e347abd 100644 --- a/pkg/validator/jsonschema/node.json +++ b/pkg/payload/jsonschema/node.json @@ -36,7 +36,12 @@ }, "kinds": { "type": ["array"], - "items": { "type": "string" }, + "items": { + "type": "string", + "not": { + "pattern": "^[Tt][Aa][Gg](?:_|$)" + } + }, "minItems": 0, "maxItems": 3, "description": "An array of kind labels for the node. The first element is treated as the node's primary kind and is used to determine which icon to display in the graph UI. This primary kind is only used for visual representation and has no semantic significance for data processing." diff --git a/pkg/validator/jsonschema/payload-schema.json b/pkg/payload/jsonschema/schema.json similarity index 100% rename from pkg/validator/jsonschema/payload-schema.json rename to pkg/payload/jsonschema/schema.json diff --git a/pkg/validator/schema.go b/pkg/payload/schema.go similarity index 94% rename from pkg/validator/schema.go rename to pkg/payload/schema.go index bfa5709..171a121 100644 --- a/pkg/validator/schema.go +++ b/pkg/payload/schema.go @@ -13,7 +13,7 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -package validator +package payload import ( "bytes" @@ -26,14 +26,14 @@ import ( //go:embed jsonschema var schemaFiles embed.FS -type IngestSchema struct { +type Schema struct { NodeSchema *jsonschema.Schema EdgeSchema *jsonschema.Schema MetaSchema *jsonschema.Schema } -func LoadIngestSchema() (IngestSchema, error) { - var schema IngestSchema +func LoadSchema() (Schema, error) { + var schema Schema if nodeSchema, err := loadSchema("node.json"); err != nil { return schema, err } else if edgeSchema, err := loadSchema("edge.json"); err != nil { diff --git a/pkg/validator/validator.go b/pkg/payload/validator.go similarity index 86% rename from pkg/validator/validator.go rename to pkg/payload/validator.go index c98bbd1..fc9c533 100644 --- a/pkg/validator/validator.go +++ b/pkg/payload/validator.go @@ -13,7 +13,7 @@ // limitations under the License. // // SPDX-License-Identifier: Apache-2.0 -package validator +package payload import ( "encoding/json" @@ -43,7 +43,7 @@ type Validator struct { decoder *json.Decoder depth int - schema IngestSchema + schema Schema originalData originalData opengraphData opengraphData @@ -72,7 +72,7 @@ type opengraphData struct { EdgesValidated int } -func NewValidator(reader io.Reader, schema IngestSchema) Validator { +func NewValidator(reader io.Reader, schema Schema) Validator { return Validator{ reader: reader, decoder: json.NewDecoder(reader), @@ -104,6 +104,31 @@ type ValidationError struct { Errors []ValidationErrorDetail } +func (s ValidationError) Error() string { + var ( + details = make([]string, 0, len(s.Errors)) + message = "validation error" + ) + + if s.Location != "" { + message = fmt.Sprintf("%s at %s", message, s.Location) + } + + for _, validationErrorDetail := range s.Errors { + if validationErrorDetail.Location != "" { + details = append(details, fmt.Sprintf("%s: %s", validationErrorDetail.Location, validationErrorDetail.Error)) + } else if validationErrorDetail.Error != "" { + details = append(details, validationErrorDetail.Error) + } + } + + if len(details) > 0 { + message = fmt.Sprintf("%s: %s", message, strings.Join(details, "; ")) + } + + return message +} + type ValidationErrorDetail struct { Location string Error string @@ -122,8 +147,11 @@ type ParsedOpenGraphData struct { EdgesValidated int } -// buildParsedData() aggregates data collected during ParseAndValidate() into the ParsedData struct -func (v *Validator) buildParsedData() ParsedData { +// buildValidatedData() aggregates data collected during ParseAndValidate() into the ParsedData struct. +// It is specific to the validation path and relies on signals (GraphFound, NodesValidated, etc.) +// that are only populated by validationLoop. ParseMetadata() builds its result inline rather than +// using this helper. +func (v *Validator) buildValidatedData() ParsedData { p := ParsedData{} if (v.opengraphData.GraphFound || v.opengraphData.MetadataFound) && (v.originalData.MetadataFound || v.originalData.DataFound) { @@ -159,7 +187,7 @@ func (v *Validator) buildValidationReport() ValidationReport { // result() is a helper for returning the current parsed data, validation report, and provided error. func (v *Validator) result(err error) (ParsedData, ValidationReport, error) { - return v.buildParsedData(), v.buildValidationReport(), err + return v.buildValidatedData(), v.buildValidationReport(), err } // Error Helper functions ------------------------------------------------------------------------- @@ -262,6 +290,32 @@ func (v *Validator) ParseAndValidate() (ParsedData, ValidationReport, error) { return v.result(v.finalizeParse()) } +// ParseMetadata() walks the top-level JSON object and extracts metadata (either legacy "meta" or +// opengraph "metadata") without performing schema validation of the payload body. It returns as soon +// as a metadata tag is successfully decoded; the remainder of the reader is not consumed. +func (v *Validator) ParseMetadata() (ParsedData, error) { + if err := v.enterObject(); err != nil { + v.reportCriticalError("failed to enter json object", err) + return ParsedData{}, err + } + + err := v.parseLoop() + + p := ParsedData{} + switch { + case v.originalData.MetadataFound: + p.PayloadType = v.originalData.Metadata.Type + p.LegacyMetadata = v.originalData.Metadata + case v.opengraphData.MetadataFound: + p.PayloadType = ingest.DataTypeOpenGraph + p.OpengraphData.Metadata = v.opengraphData.Metadata + case v.opengraphData.GraphFound: + p.PayloadType = ingest.DataTypeOpenGraph + } + + return p, err +} + // readToEnd() checks for trailing input if validation succeeded, then consumes all remaining bytes from the decoder // buffer and reader while preserving any existing loop error. func (v *Validator) readToEnd(loopErr error) error { @@ -303,7 +357,7 @@ func (v *Validator) finalizeParse() error { return nil } -// Validation Loop functions ---------------------------------------------------------------------- +// Loop functions ---------------------------------------------------------------------- // validationLoop() is the primary driver behind the file validation. It walks through the file and directs to // child validation functions @@ -383,6 +437,47 @@ func (v *Validator) validationLoop() error { } } +// parseLoop() walks the top-level object looking for tags that identify the payload shape +// ("meta", "metadata"), decoding any metadata tag into the Validator's internal state and +// returning as soon as a tag that uniquely identifies the payload type is found or the +// top-level object is exited. +func (v *Validator) parseLoop() error { + for { + if tag, exitedBlock, err := v.nextTagAtDepth(1); err != nil { + v.reportCriticalError("failed parsing top level tag", err) + return err + } else if exitedBlock { + return nil + } else { + switch tag { + case "meta": + var metadata ingest.OriginalMetadata + if err := v.decoder.Decode(&metadata); err != nil { + v.reportCriticalError("failed to decode original metadata", err) + return err + } + + v.originalData.MetadataFound = true + v.originalData.Metadata = metadata + return nil + case "metadata": + var metadata ingest.OpengraphMetadata + if err := v.decoder.Decode(&metadata); err != nil { + v.reportCriticalError("failed to decode opengraph metadata", err) + return err + } + + v.opengraphData.MetadataFound = true + v.opengraphData.Metadata = metadata + return nil + case "graph": + v.opengraphData.GraphFound = true + default: + } + } + } +} + // handleOriginalMetadata() parses and validates original metadata after a "meta" tag is found at the top level func (v *Validator) handleOriginalMetadata() (ingest.OriginalMetadata, error) { var originalMetadata ingest.OriginalMetadata diff --git a/pkg/payload/validator_test.go b/pkg/payload/validator_test.go new file mode 100644 index 0000000..7d772ec --- /dev/null +++ b/pkg/payload/validator_test.go @@ -0,0 +1,587 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 +package payload_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/specterops/chow/pkg/ingest" + "github.com/specterops/chow/pkg/payload" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var emptyValidationReport = payload.ValidationReport{CriticalErrors: []payload.CriticalError{}, ValidationErrors: []payload.ValidationError{}} + +type parseAndValidateAssertion struct { + name string + payload string + expectedParsedData payload.ParsedData + errValidationFunc func(t *testing.T, report payload.ValidationReport, err error) +} + +func Test_ParseAndValidate(t *testing.T) { + assertions := []parseAndValidateAssertion{ + // OpenGraph payload tests + { + name: "successful opengraph payload", + payload: `{"metadata":{},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "successful opengraph payload with no metadata", + payload: `{"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "successful opengraph metadata", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "successful opengraph payload with $schema", + payload: `{"$schema":"test","metadata":{"source_kind":"hellobase"},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "successful opengraph payload with node", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User"],"properties":{"items":["hi"]}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "unsuccessful opengraph payload, node id validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":1,"kinds":["User"]}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/nodes[0]", + RawObject: `{"id":1,"kinds":["User"]}`, + Errors: []payload.ValidationErrorDetail{{Location: "/id", Error: "got number, want string"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, node kinds validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User", 1]}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/nodes[0]", + RawObject: `{"id":"TESTNODE","kinds":["User", 1]}`, + Errors: []payload.ValidationErrorDetail{{Location: "/kinds/1", Error: "got number, want string"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, node kind tag prefix validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["Tag_Admin"]}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/nodes[0]", + RawObject: `{"id":"TESTNODE","kinds":["Tag_Admin"]}`, + Errors: []payload.ValidationErrorDetail{{Location: "/kinds/0", Error: "'not' failed"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, node kind standalone tag validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["tAg"]}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/nodes[0]", + RawObject: `{"id":"TESTNODE","kinds":["tAg"]}`, + Errors: []payload.ValidationErrorDetail{{Location: "/kinds/0", Error: "'not' failed"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, node properties validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User"],"properties":{"items":{}}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/nodes[0]", + RawObject: `{"id":"TESTNODE","kinds":["User"],"properties":{"items":{}}}`, + Errors: []payload.ValidationErrorDetail{{Location: "/properties/items", Error: "invalid type"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, node multiple validation errors", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":1,"kinds":["User"],"properties":{"items":{}}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + require.Len(t, report.ValidationErrors, 1) + require.Equal(t, "/graph/nodes[0]", report.ValidationErrors[0].Location) + require.Equal(t, `{"id":1,"kinds":["User"],"properties":{"items":{}}}`, report.ValidationErrors[0].RawObject) + assert.ElementsMatch(t, report.ValidationErrors[0].Errors, []payload.ValidationErrorDetail{{Location: "/id", Error: "got number, want string"}, {Location: "/properties/items", Error: "invalid type"}}) + }, + }, + { + name: "unsuccessful opengraph payload, exceeds max validation errors", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"1","kinds":["A","A","A","A"]},` + + `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + + `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + + `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + + `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 15}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrMaxValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + {Location: "/graph/nodes[0]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[1]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[2]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[3]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[4]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[5]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[6]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[7]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[8]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[9]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[10]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[11]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[12]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[13]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + {Location: "/graph/nodes[14]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []payload.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, + }) + }, + }, + { + name: "successful opengraph payload with edge", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "successful opengraph payload with edge property matching", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"ROHAN"}]},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "unsuccessful opengraph payload, edge properties validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":{}}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":{}}}`, + Errors: []payload.ValidationErrorDetail{{Location: "/properties/items", Error: "invalid type"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, edge id validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":1},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"value":1},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}`, + Errors: []payload.ValidationErrorDetail{{Location: "/start/value", Error: "got number, want string"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, edge kind tag prefix validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"TAG_Admin"}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"TAG_Admin"}`, + Errors: []payload.ValidationErrorDetail{{Location: "/kind", Error: "'not' failed"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, edge kind standalone tag validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"TaG"}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"TaG"}`, + Errors: []payload.ValidationErrorDetail{{Location: "/kind", Error: "'not' failed"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, edge endpoint kind tag prefix validation error", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"edges":[{"start":{"value":"TESTNODE","kind":"tag_Admin"},"end":{"value":"TESTNODE2"},"kind":"RELATED"}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"value":"TESTNODE","kind":"tag_Admin"},"end":{"value":"TESTNODE2"},"kind":"RELATED"}`, + Errors: []payload.ValidationErrorDetail{{Location: "/start/kind", Error: "'not' failed"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph payload, invalid edge property matching", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"match_by":"property","property_matchers":{"key":"prop_1","operator":"equals","value":"ROHAN"}},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: payload.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrValidationErrors) + + assert.ElementsMatch(t, report.ValidationErrors, []payload.ValidationError{ + { + Location: "/graph/edges[0]", + RawObject: `{"start":{"match_by":"property","property_matchers":{"key":"prop_1","operator":"equals","value":"ROHAN"}},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}`, + Errors: []payload.ValidationErrorDetail{{Location: "/start/property_matchers", Error: "got object, want array"}}, + }, + }) + }, + }, + { + name: "unsuccessful opengraph metadata", + payload: `{"metadata":{"source_kind":1},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrOpengraphMetadataValidation) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "opengraph metadata failed validation", Error: payload.ErrOpengraphMetadataValidation}}) + }, + }, + { + name: "unsuccessful opengraph no child tags", + payload: `{"graph":{}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "graph tag requires child nodes or edges tag", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful opengraph metadata, invalid field", + payload: `{"metadata":{"random field":"hello"},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrOpengraphMetadataValidation) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "opengraph metadata failed validation", Error: payload.ErrOpengraphMetadataValidation}}) + }, + }, + // Original payload tests + { + name: "successful original payload", + payload: `{"meta":{"methods": 0,"type":"sessions","count": 0,"version": 5},"data":[]}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "unsuccessful original payload, no data tag", + payload: `{"meta":{"methods": 0,"type":"sessions","count": 0,"version":5}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "no data tag found to match original metadata tag", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful original payload, no meta tag", + payload: `{"data":[]}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "no meta tag found to match original data tag", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful original payload, duplicate meta tag", + payload: `{"meta":{"methods":0,"type":"sessions","count":0,"version":5},"meta":0,"data":[]}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "duplicate top level meta tag found", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful original payload, invalid meta", + payload: `{"data":[],"meta":0}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + require.Len(t, report.CriticalErrors, 1) + var ( + criticalError = report.CriticalErrors[0] + unmarshalErr = &json.UnmarshalTypeError{} + ) + + assert.Equal(t, "failed to decode original metadata", criticalError.Message) + assert.ErrorAs(t, criticalError.Error, &unmarshalErr) + assert.ErrorAs(t, err, &unmarshalErr) + }, + }, + { + name: "swapped order", + payload: `{"data":[],"meta":{"methods":0,"type":"sessions","count":0,"version":5}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.Equal(t, emptyValidationReport, report) + assert.NoError(t, err) + }, + }, + { + name: "unsuccessful original payload, invalid type", + payload: `{"data":[],"meta":{"methods":0,"type":"invalid","count":0,"version":5}}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidDataType) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "invalid original metadata data type", Error: payload.ErrInvalidDataType}}) + }, + }, + // Invalid payload tests + { + name: "unsuccessful payload, no valid tags", + payload: `{}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "no tags found", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "enforce mutual exclusivity", + payload: `{"data":[],"graph":{}}`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "cannot have both original data tag and opengraph graph tag", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful payload, unrecognized top level tag", + payload: `{"graph":{"nodes":[]},"pants":{}}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorIs(t, err, payload.ErrInvalidFileConfiguration) + + assert.ElementsMatch(t, report.CriticalErrors, []payload.CriticalError{{Message: "unrecognized top level tag: pants", Error: payload.ErrInvalidFileConfiguration}}) + }, + }, + { + name: "unsuccessful payload, trailing data after object", + payload: `{"graph":{"nodes":[]}}{}`, + expectedParsedData: payload.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, + errValidationFunc: func(t *testing.T, report payload.ValidationReport, err error) { + assert.ErrorContains(t, err, "expected EOF, instead got token: {") + require.Len(t, report.CriticalErrors, 1) + assert.Equal(t, "expected to hit the end of the file", report.CriticalErrors[0].Message) + assert.ErrorContains(t, report.CriticalErrors[0].Error, "expected EOF, instead got token: {") + }, + }, + } + + schema, err := payload.LoadSchema() + require.NoError(t, err) + + for _, assertion := range assertions { + t.Run(assertion.name, func(t *testing.T) { + v := payload.NewValidator(strings.NewReader(assertion.payload), schema) + + parsedData, validationReport, err := v.ParseAndValidate() + assert.Equal(t, assertion.expectedParsedData, parsedData) + assertion.errValidationFunc(t, validationReport, err) + }) + } +} + +type parseMetadataAssertion struct { + name string + payload string + expectedParsedData payload.ParsedData + errValidationFunc func(t *testing.T, err error) +} + +func Test_ParseMetadata(t *testing.T) { + assertions := []parseMetadataAssertion{ + { + name: "legacy metadata", + payload: `{"meta":{"methods":0,"type":"sessions","count":0,"version":5},"data":[]}`, + expectedParsedData: payload.ParsedData{ + PayloadType: ingest.DataTypeSession, + LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}, + }, + errValidationFunc: func(t *testing.T, err error) { + assert.NoError(t, err) + }, + }, + { + name: "legacy metadata after data", + payload: `{"data":[],"meta":{"methods":0,"type":"sessions","count":0,"version":5}}`, + expectedParsedData: payload.ParsedData{ + PayloadType: ingest.DataTypeSession, + LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}, + }, + errValidationFunc: func(t *testing.T, err error) { + assert.NoError(t, err) + }, + }, + { + name: "opengraph metadata", + payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{ + PayloadType: ingest.DataTypeOpenGraph, + OpengraphData: payload.ParsedOpenGraphData{ + Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, + }, + }, + errValidationFunc: func(t *testing.T, err error) { + assert.NoError(t, err) + }, + }, + { + name: "opengraph graph only", + payload: `{"graph":{"nodes":[]}}`, + expectedParsedData: payload.ParsedData{ + PayloadType: ingest.DataTypeOpenGraph, + }, + errValidationFunc: func(t *testing.T, err error) { + assert.NoError(t, err) + }, + }, + { + name: "opengraph metadata after graph", + payload: `{"graph":{"nodes":[]},"metadata":{"source_kind":"hellobase"}}`, + expectedParsedData: payload.ParsedData{ + PayloadType: ingest.DataTypeOpenGraph, + OpengraphData: payload.ParsedOpenGraphData{ + Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, + }, + }, + errValidationFunc: func(t *testing.T, err error) { + assert.NoError(t, err) + }, + }, + { + name: "invalid top level json", + payload: `[]`, + expectedParsedData: payload.ParsedData{}, + errValidationFunc: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "expected open bracket") + }, + }, + } + + schema, err := payload.LoadSchema() + require.NoError(t, err) + + for _, assertion := range assertions { + t.Run(assertion.name, func(t *testing.T) { + v := payload.NewValidator(strings.NewReader(assertion.payload), schema) + + parsedData, err := v.ParseMetadata() + assert.Equal(t, assertion.expectedParsedData, parsedData) + assertion.errValidationFunc(t, err) + }) + } +} + +func TestValidationError_Error(t *testing.T) { + validationErr := payload.ValidationError{ + Location: "/graph/nodes[0]", + Errors: []payload.ValidationErrorDetail{ + {Location: "/id", Error: "got number, want string"}, + {Location: "/properties/items", Error: "invalid type"}, + }, + } + + assert.Equal(t, "validation error at /graph/nodes[0]: /id: got number, want string; /properties/items: invalid type", validationErr.Error()) +} diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go deleted file mode 100644 index f352373..0000000 --- a/pkg/validator/validator_test.go +++ /dev/null @@ -1,404 +0,0 @@ -// Copyright 2026 Specter Ops, Inc. -// -// Licensed under the Apache License, Version 2.0 -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 -package validator_test - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/specterops/chow/pkg/ingest" - validator "github.com/specterops/chow/pkg/validator" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -var emptyValidationReport = validator.ValidationReport{CriticalErrors: []validator.CriticalError{}, ValidationErrors: []validator.ValidationError{}} - -type parseAndValidateAssertion struct { - name string - payload string - expectedParsedData validator.ParsedData - errValidationFunc func(t *testing.T, report validator.ValidationReport, err error) -} - -func Test_ParseAndValidate(t *testing.T) { - assertions := []parseAndValidateAssertion{ - // OpenGraph payload tests - { - name: "successful opengraph payload", - payload: `{"metadata":{},"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "successful opengraph payload with no metadata", - payload: `{"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "successful opengraph metadata", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "successful opengraph payload with $schema", - payload: `{"$schema":"test","metadata":{"source_kind":"hellobase"},"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "successful opengraph payload with node", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User"],"properties":{"items":["hi"]}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "unsuccessful opengraph payload, node id validation error", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":1,"kinds":["User"]}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/nodes[0]", - RawObject: `{"id":1,"kinds":["User"]}`, - Errors: []validator.ValidationErrorDetail{{Location: "/id", Error: "got number, want string"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph payload, node kinds validation error", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User", 1]}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/nodes[0]", - RawObject: `{"id":"TESTNODE","kinds":["User", 1]}`, - Errors: []validator.ValidationErrorDetail{{Location: "/kinds/1", Error: "got number, want string"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph payload, node properties validation error", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"TESTNODE","kinds":["User"],"properties":{"items":{}}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/nodes[0]", - RawObject: `{"id":"TESTNODE","kinds":["User"],"properties":{"items":{}}}`, - Errors: []validator.ValidationErrorDetail{{Location: "/properties/items", Error: "invalid type"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph payload, node multiple validation errors", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":1,"kinds":["User"],"properties":{"items":{}}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - require.Len(t, report.ValidationErrors, 1) - require.Equal(t, "/graph/nodes[0]", report.ValidationErrors[0].Location) - require.Equal(t, `{"id":1,"kinds":["User"],"properties":{"items":{}}}`, report.ValidationErrors[0].RawObject) - assert.ElementsMatch(t, report.ValidationErrors[0].Errors, []validator.ValidationErrorDetail{{Location: "/id", Error: "got number, want string"}, {Location: "/properties/items", Error: "invalid type"}}) - }, - }, - { - name: "unsuccessful opengraph payload, exceeds max validation errors", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[{"id":"1","kinds":["A","A","A","A"]},` + - `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + - `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + - `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},` + - `{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]},{"id":"1","kinds":["A","A","A","A"]}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, NodesValidated: 15}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrMaxValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - {Location: "/graph/nodes[0]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[1]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[2]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[3]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[4]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[5]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[6]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[7]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[8]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[9]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[10]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[11]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[12]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[13]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - {Location: "/graph/nodes[14]", RawObject: `{"id":"1","kinds":["A","A","A","A"]}`, Errors: []validator.ValidationErrorDetail{{Location: "/kinds", Error: "maxItems: got 4, want 3"}}}, - }) - }, - }, - { - name: "successful opengraph payload with edge", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "successful opengraph payload with edge property matching", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"ROHAN"}]},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "unsuccessful opengraph payload, edge properties validation error", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":{}}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/edges[0]", - RawObject: `{"start":{"value":"TESTNODE"},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":{}}}`, - Errors: []validator.ValidationErrorDetail{{Location: "/properties/items", Error: "invalid type"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph payload, edge id validation error", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"value":1},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/edges[0]", - RawObject: `{"start":{"value":1},"end":{"value":"TESTNODE2"},"kind":"RELATED","properties":{"items":["hi"]}}`, - Errors: []validator.ValidationErrorDetail{{Location: "/start/value", Error: "got number, want string"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph payload, invalid edge property matching", - payload: `{"metadata":{"source_kind":"hellobase"},"graph":{"nodes":[],"edges":[{"start":{"match_by":"property","property_matchers":{"key":"prop_1","operator":"equals","value":"ROHAN"}},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}]}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph, OpengraphData: validator.ParsedOpenGraphData{Metadata: ingest.OpengraphMetadata{SourceKind: "hellobase"}, EdgesValidated: 1}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrValidationErrors) - - assert.ElementsMatch(t, report.ValidationErrors, []validator.ValidationError{ - { - Location: "/graph/edges[0]", - RawObject: `{"start":{"match_by":"property","property_matchers":{"key":"prop_1","operator":"equals","value":"ROHAN"}},"end":{"match_by":"property","property_matchers":[{"key":"prop_1","operator":"equals","value":"WES"}]},"kind":"RELATED","properties":{"items":["hi"]}}`, - Errors: []validator.ValidationErrorDetail{{Location: "/start/property_matchers", Error: "got object, want array"}}, - }, - }) - }, - }, - { - name: "unsuccessful opengraph metadata", - payload: `{"metadata":{"source_kind":1},"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrOpengraphMetadataValidation) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "opengraph metadata failed validation", Error: validator.ErrOpengraphMetadataValidation}}) - }, - }, - { - name: "unsuccessful opengraph no child tags", - payload: `{"graph":{}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "graph tag requires child nodes or edges tag", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful opengraph metadata, invalid field", - payload: `{"metadata":{"random field":"hello"},"graph":{"nodes":[]}}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrOpengraphMetadataValidation) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "opengraph metadata failed validation", Error: validator.ErrOpengraphMetadataValidation}}) - }, - }, - // Original payload tests - { - name: "successful original payload", - payload: `{"meta":{"methods": 0,"type":"sessions","count": 0,"version": 5},"data":[]}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "unsuccessful original payload, no data tag", - payload: `{"meta":{"methods": 0,"type":"sessions","count": 0,"version":5}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "no data tag found to match original metadata tag", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful original payload, no meta tag", - payload: `{"data":[]}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "no meta tag found to match original data tag", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful original payload, duplicate meta tag", - payload: `{"meta":{"methods":0,"type":"sessions","count":0,"version":5},"meta":0,"data":[]}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "duplicate top level meta tag found", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful original payload, invalid meta", - payload: `{"data":[],"meta":0}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - require.Len(t, report.CriticalErrors, 1) - var ( - criticalError = report.CriticalErrors[0] - unmarshalErr = &json.UnmarshalTypeError{} - ) - - assert.Equal(t, "failed to decode original metadata", criticalError.Message) - assert.ErrorAs(t, criticalError.Error, &unmarshalErr) - assert.ErrorAs(t, err, &unmarshalErr) - }, - }, - { - name: "swapped order", - payload: `{"data":[],"meta":{"methods":0,"type":"sessions","count":0,"version":5}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeSession, LegacyMetadata: ingest.OriginalMetadata{Type: ingest.DataTypeSession, Methods: 0, Version: 5}}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.Equal(t, emptyValidationReport, report) - assert.NoError(t, err) - }, - }, - { - name: "unsuccessful original payload, invalid type", - payload: `{"data":[],"meta":{"methods":0,"type":"invalid","count":0,"version":5}}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidDataType) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "invalid original metadata data type", Error: validator.ErrInvalidDataType}}) - }, - }, - // Invalid payload tests - { - name: "unsuccessful payload, no valid tags", - payload: `{}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "no tags found", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "enforce mutual exclusivity", - payload: `{"data":[],"graph":{}}`, - expectedParsedData: validator.ParsedData{}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "cannot have both original data tag and opengraph graph tag", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful payload, unrecognized top level tag", - payload: `{"graph":{"nodes":[]},"pants":{}}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorIs(t, err, validator.ErrInvalidFileConfiguration) - - assert.ElementsMatch(t, report.CriticalErrors, []validator.CriticalError{{Message: "unrecognized top level tag: pants", Error: validator.ErrInvalidFileConfiguration}}) - }, - }, - { - name: "unsuccessful payload, trailing data after object", - payload: `{"graph":{"nodes":[]}}{}`, - expectedParsedData: validator.ParsedData{PayloadType: ingest.DataTypeOpenGraph}, - errValidationFunc: func(t *testing.T, report validator.ValidationReport, err error) { - assert.ErrorContains(t, err, "expected EOF, instead got token: {") - require.Len(t, report.CriticalErrors, 1) - assert.Equal(t, "expected to hit the end of the file", report.CriticalErrors[0].Message) - assert.ErrorContains(t, report.CriticalErrors[0].Error, "expected EOF, instead got token: {") - }, - }, - } - - schema, err := validator.LoadIngestSchema() - require.NoError(t, err) - - for _, assertion := range assertions { - t.Run(assertion.name, func(t *testing.T) { - v := validator.NewValidator(strings.NewReader(assertion.payload), schema) - - parsedData, validationReport, err := v.ParseAndValidate() - assert.Equal(t, assertion.expectedParsedData, parsedData) - assertion.errValidationFunc(t, validationReport, err) - }) - } -}