parseNestedDicts

[RagingRedRiot]

Description of the change

There are some stringified JSON that gets returned by some enterprise-license features that aren't being parsed into JSON. Specifically, the Data field is returned when Automated Investigation & Response (AIR) generates an AI generated report. Without this nested JSON field being parsed, it's difficult to use the data in D&R and FP logic. This change recursively searches parsed fields for stringified JSON and parses them further if found. The recursion is limited to only fields that are parsed, meaning the recursion doesn't recheck all the values every time its ran and has a finite limit to as many nested stringified JSON values there are.

Type of change

• Bug fix (non-breaking change that fixes an issue)
• New feature (non-breaking change that adds functionality)
• Breaking change (fix or feature that would cause existing functionality to not work as expected)

Related issues

https://github.com/BHISSOC/tracking/issues/302

[RagingRedRiot]

I have tested the code change to ensure it does not break shipping logs.

I used the following unit test to ensure that the JSON parsing logic behaves as expected.

package usp_o365

import (
	"encoding/json"
	"testing"

	"github.com/refractionPOINT/usp-adapters/utils"
)

func TestMain(t *testing.T) {
        // Test data that represents what we want to parse
	var testData = `{
		"String": "Not nested string value",
		"Integer": 3,
		"Data": "{\"nestedData\":\"This is my nested value\"}",
		"Data2": "{\"nestedData2\":\"{\\\"Data3\\\":\\\"This is my double nested value\\\"}\"}"
	}`

	var dict utils.Dict // Represents the current JSON output
	if err := json.Unmarshal([]byte(testData), &dict); err != nil {
		t.Fatal("failed to unmarshal test data: %w", err)
	}

	updatedDict := parseNestedDicts(dict) // Represents the updated JSON output

	// dict doesn't parse nested stringified JSON
	// String - Expected to be parsed
	_, ok := dict["String"].(string)
	if !ok {
		t.Fatalf("String is missing or is not a string; got: %#v", dict["String"])
	}

	// Integer - Expected to be parsed
	rawInt, ok := dict["Integer"].(uint64)
	if !ok {
		t.Fatalf("Integer is missing or is not a uint64; got: %#v", dict["Integer"])
	}
	if rawInt != 3 {
		t.Fatalf("Integer has wrong value; want 3, got %d", rawInt)
	}

	// Data - Does not parse beyond a string
	_, ok = dict["Data"].(string)
	if !ok {
		t.Fatalf("Data is missing or is not a string; got: %#v", dict["Data"])
	}

	// Data2 - Does not parse beyond a string
	_, ok = dict["Data2"].(string)
	if !ok {
		t.Fatalf("Data2 is missing or is not a string; got: %#v", dict["Data2"])
	}

	// updatedDict parses nested stringified JSON
	// String - Same as dict
	_, ok = updatedDict["String"].(string)
	if !ok {
		t.Fatalf("String is missing or is not a string; got: %#v", updatedDict["String"])
	}

	// Integer - Same as dict
	rawInt, ok = updatedDict["Integer"].(uint64)
	if !ok {
		t.Fatalf("Integer is missing or is not a uint64; got: %#v", updatedDict["Integer"])
	}
	if rawInt != 3 {
		t.Fatalf("Integer has wrong value; want 3, got %d", rawInt)
	}

	// Data → nestedData
	dataDictUpdated, ok := updatedDict["Data"].(utils.Dict)
	if !ok {
		t.Fatalf("Data is missing or is not a utils.Dict; got: %#v", updatedDict["Data"])
	}
	nested, ok := dataDictUpdated["nestedData"].(string)
	if !ok {
		t.Fatalf("nestedData is missing or not a string; got: %#v", dataDictUpdated["nestedData"])
	}
	if nested != "This is my nested value" {
		t.Errorf("nestedData has wrong value; want %q, got %q", "This is my nested value", nested)
	}

	// Data2 → nestedData2 → Data3
	data2DictUpdated, ok := updatedDict["Data2"].(utils.Dict)
	if !ok {
		t.Fatalf("Data2 is missing or is not a utils.Dict; got: %#v", updatedDict["Data2"])
	}
	nested2Dict, ok := data2DictUpdated["nestedData2"].(utils.Dict)
	if !ok {
		t.Fatalf("nestedData2 is missing or is not a utils.Dict; got: %#v", data2DictUpdated["nestedData2"])
	}
	deepVal, ok := nested2Dict["Data3"].(string)
	if !ok {
		t.Fatalf("Data3 is missing or is not a string; got: %#v", nested2Dict["Data3"])
	}
	if deepVal != "This is my double nested value" {
		t.Errorf("Data3 has wrong value; want %q, got %q", "This is my double nested value", deepVal)
	}
}