From 5bf1836bb625e42a6687d6c34c03464df676bdf7 Mon Sep 17 00:00:00 2001 From: Bryan Font Date: Mon, 13 Jul 2026 14:24:42 -0400 Subject: [PATCH] Handle ambiguous Codex lineage evidence --- .../Generated/CodexParserHash.generated.swift | 2 +- .../Codex/CodexLineageDiscovery.swift | 102 +++++--- .../Providers/Codex/CodexLineageLedger.swift | 246 +++++++++++++++++- .../Providers/Codex/CodexLineageShadow.swift | 44 +++- .../Vendored/CostUsage/CostUsageScanner.swift | 31 ++- .../CodexLineageDiscoveryTests.swift | 101 ++++++- .../CodexLineageLedgerTests.swift | 164 ++++++++++++ .../CodexLineageShadowTests.swift | 13 +- 8 files changed, 645 insertions(+), 58 deletions(-) diff --git a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift index d57f41412c..90a5d3ff38 100644 --- a/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift +++ b/Sources/CodexBarCore/Generated/CodexParserHash.generated.swift @@ -1,5 +1,5 @@ // Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand. enum CodexParserHash { - static let value = "07457314442f300f" + static let value = "4c426503aec9cb49" } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift index 0b75a82e87..8d61444a76 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift @@ -5,7 +5,7 @@ enum CodexLineageDiscovery { struct Report: Equatable, Sendable { let documents: [CodexLineageLedger.Document] let referencedParentDocumentCount: Int - let unresolvedParentIDs: Set + let unresolvedParents: Set } static func discover( @@ -15,20 +15,24 @@ enum CodexLineageDiscovery { { var locator = ParentFileLocator(roots: roots, checkCancellation: checkCancellation) var documents: [CodexLineageLedger.Document] = [] - var knownIDs: Set = [] + var knownIDs: Set = [] var seenPaths: Set = [] - var pendingParentIDs: [String] = [] - var unresolvedParentIDs: Set = [] + var pendingParentIDs: [ScopedIdentity] = [] + var unresolvedParents: Set = [] var referencedParentDocumentCount = 0 func remember(_ document: CodexLineageLedger.Document) { documents.append(document) - knownIDs.insert(Self.canonicalSessionID(document.ownerID)) + knownIDs.insert(.init(scopeID: document.scopeID, sessionID: Self.canonicalSessionID(document.ownerID))) if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) { - knownIDs.insert(Self.canonicalSessionID(metadataSessionID)) + knownIDs.insert(.init( + scopeID: document.scopeID, + sessionID: Self.canonicalSessionID(metadataSessionID))) } if let parentSessionID = Self.nonEmpty(document.parentSessionID) { - pendingParentIDs.append(Self.canonicalSessionID(parentSessionID)) + pendingParentIDs.append(.init( + scopeID: document.scopeID, + sessionID: Self.canonicalSessionID(parentSessionID))) } } @@ -44,35 +48,46 @@ enum CodexLineageDiscovery { var nextParent = 0 while nextParent < pendingParentIDs.count { try checkCancellation?() - let parentID = pendingParentIDs[nextParent] + let parentIdentity = pendingParentIDs[nextParent] nextParent += 1 - guard !knownIDs.contains(parentID), !unresolvedParentIDs.contains(parentID) else { continue } - guard let parentURL = try locator.fileURL(for: parentID) else { - unresolvedParentIDs.insert(parentID) + let unresolvedIdentity = CodexLineageLedger.ParentIdentity( + scopeID: parentIdentity.scopeID, + sessionID: parentIdentity.sessionID) + guard !knownIDs.contains(parentIdentity), !unresolvedParents.contains(unresolvedIdentity) else { continue } - let path = parentURL.standardizedFileURL.path - guard seenPaths.insert(path).inserted else { - unresolvedParentIDs.insert(parentID) + guard let parentURLs = try locator.fileURLs( + for: parentIdentity.sessionID, + scopeID: parentIdentity.scopeID) + else { + unresolvedParents.insert(unresolvedIdentity) continue } - let parent = try CostUsageScanner.parseCodexLineageDocument( - fileURL: parentURL, - checkCancellation: checkCancellation) - let ownerID = Self.canonicalSessionID(parent.ownerID) - let metadataSessionID = parent.metadataSessionID.map(Self.canonicalSessionID) - guard ownerID == parentID || metadataSessionID == parentID else { - unresolvedParentIDs.insert(parentID) - continue + var foundParent = false + for parentURL in parentURLs { + let path = parentURL.standardizedFileURL.path + guard seenPaths.insert(path).inserted else { continue } + let parent = try CostUsageScanner.parseCodexLineageDocument( + fileURL: parentURL, + checkCancellation: checkCancellation) + let ownerID = Self.canonicalSessionID(parent.ownerID) + let metadataSessionID = parent.metadataSessionID.map(Self.canonicalSessionID) + guard ownerID == parentIdentity.sessionID || metadataSessionID == parentIdentity.sessionID else { + continue + } + referencedParentDocumentCount += 1 + foundParent = true + remember(parent) + } + if !foundParent { + unresolvedParents.insert(unresolvedIdentity) } - referencedParentDocumentCount += 1 - remember(parent) } return Report( documents: documents, referencedParentDocumentCount: referencedParentDocumentCount, - unresolvedParentIDs: unresolvedParentIDs) + unresolvedParents: unresolvedParents) } private static func nonEmpty(_ value: String?) -> String? { @@ -84,21 +99,38 @@ enum CodexLineageDiscovery { UUID(uuidString: value)?.uuidString.lowercased() ?? value } + private struct ScopedIdentity: Equatable, Hashable { + let scopeID: String + let sessionID: String + } + private struct ParentFileLocator { let roots: [URL] let checkCancellation: CostUsageScanner.CancellationCheck? var didIndexRoots = false - var filesByID: [String: URL] = [:] + var filesByID: [ScopedIdentity: Set] = [:] - mutating func fileURL(for sessionID: String) throws -> URL? { + mutating func fileURLs(for sessionID: String, scopeID: String) throws -> [URL]? { let canonicalID = CodexLineageDiscovery.canonicalSessionID(sessionID) - if let known = self.filesByID[canonicalID] { - return known + let key = ScopedIdentity(scopeID: scopeID, sessionID: canonicalID) + if let known = self.filesByID[key] { + return Self.unambiguousMatches(known) } if !self.didIndexRoots { try self.indexRoots() } - return self.filesByID[canonicalID] + guard let matches = self.filesByID[key] else { return nil } + return Self.unambiguousMatches(matches) + } + + private static func unambiguousMatches(_ matches: Set) -> [URL]? { + guard !matches.isEmpty else { return nil } + if matches.count == 1 { + return Array(matches) + } + let ownerIDs = Set(matches.compactMap(CostUsageScanner.codexRolloutOwnerID(fileURL:))) + guard ownerIDs.count == 1 else { return nil } + return matches.sorted { $0.path < $1.path } } private mutating func indexRoots() throws { @@ -122,14 +154,20 @@ enum CodexLineageDiscovery { try self.checkCancellation?() if let ownerID = CostUsageScanner.codexRolloutOwnerID(fileURL: fileURL) { let canonicalID = CodexLineageDiscovery.canonicalSessionID(ownerID) - self.filesByID[canonicalID] = self.filesByID[canonicalID] ?? fileURL + let key = ScopedIdentity( + scopeID: CostUsageScanner.codexLineageScopeID(fileURL: fileURL), + sessionID: canonicalID) + self.filesByID[key, default: []].insert(fileURL) } if let metadataID = try CostUsageScanner.parseCodexSessionIdentifier( fileURL: fileURL, checkCancellation: self.checkCancellation) { let canonicalID = CodexLineageDiscovery.canonicalSessionID(metadataID) - self.filesByID[canonicalID] = self.filesByID[canonicalID] ?? fileURL + let key = ScopedIdentity( + scopeID: CostUsageScanner.codexLineageScopeID(fileURL: fileURL), + sessionID: canonicalID) + self.filesByID[key, default: []].insert(fileURL) } } } diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift index bf59ca377a..7152a3aab0 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageLedger.swift @@ -41,6 +41,60 @@ enum CodexLineageLedger { let metadataSessionID: String? let parentSessionID: String? let observations: [Observation] + let scopeID: String + let incompleteObservationCount: Int + + init( + ownerID: String, + metadataSessionID: String?, + parentSessionID: String?, + observations: [Observation], + scopeID: String = "", + incompleteObservationCount: Int = 0) + { + self.ownerID = ownerID + self.metadataSessionID = metadataSessionID + self.parentSessionID = parentSessionID + self.observations = observations + self.scopeID = scopeID + self.incompleteObservationCount = incompleteObservationCount + } + } + + enum ContainmentReason: String, CaseIterable, Equatable, Hashable, Sendable { + case malformedTimestamp + case incompleteObservation + case conflictingOwnerIdentity + case identityCollision + case ancestryCycle + } + + enum FamilyQuality: Equatable, Sendable { + case primary + case incompleteProvenance + case contained(Set) + } + + struct FamilyDisposition: Equatable, Sendable { + let scopeID: String + let ownerIDs: Set + let quality: FamilyQuality + } + + struct ParentIdentity: Equatable, Hashable, Sendable { + let scopeID: String + let sessionID: String + + init(scopeID: String = "", sessionID: String) { + self.scopeID = scopeID + self.sessionID = sessionID + } + } + + struct ConservativeReport: Equatable, Sendable { + let primary: Report + let families: [FamilyDisposition] + let containedDocuments: [Document] } struct Report: Equatable, Sendable { @@ -78,12 +132,13 @@ enum CodexLineageLedger { for document in documents { try checkCancellation?() guard !document.ownerID.isEmpty else { throw LedgerError.emptyOwnerID } - graph.insert(document.ownerID) + let ownerID = Self.scoped(document.ownerID, document: document) + graph.insert(ownerID) if let metadataSessionID = Self.nonEmpty(document.metadataSessionID) { - graph.union(document.ownerID, metadataSessionID) + graph.union(ownerID, Self.scoped(metadataSessionID, document: document)) } if let parentSessionID = Self.nonEmpty(document.parentSessionID) { - graph.union(document.ownerID, parentSessionID) + graph.union(ownerID, Self.scoped(parentSessionID, document: document)) } } @@ -91,7 +146,7 @@ enum CodexLineageLedger { var physicalObservationCount = 0 for document in documents { try checkCancellation?() - let componentID = graph.find(document.ownerID) + let componentID = graph.find(Self.scoped(document.ownerID, document: document)) var accepted = acceptedByComponent[componentID] ?? [:] for observation in document.observations { try checkCancellation?() @@ -138,11 +193,81 @@ enum CodexLineageLedger { localDays: localDays, utcRows: Self.dailyRows(from: utcRows), localRows: Self.dailyRows(from: localRows), - componentCount: Set(documents.map { graph.find($0.ownerID) }).count, + componentCount: Set(documents.map { graph.find(Self.scoped($0.ownerID, document: $0)) }).count, acceptedObservationCount: acceptedObservationCount, duplicateObservationCount: physicalObservationCount - acceptedObservationCount) } + /// Routes every lineage family to either primary ledger accounting or explicit containment. + /// Contained documents are returned to the caller for a future family-scoped fallback; they never + /// contribute to `primary`, which prevents double accounting by construction. + static func reconcileConservatively( + documents: [Document], + unresolvedParents: Set = [], + localTimeZone: TimeZone, + checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> ConservativeReport + { + var graph = DisjointSet() + for document in documents { + try checkCancellation?() + let owner = Self.scoped(document.ownerID, document: document) + graph.insert(owner) + if let metadata = Self.nonEmpty(document.metadataSessionID) { + graph.union(owner, Self.scoped(metadata, document: document)) + } + if let parent = Self.nonEmpty(document.parentSessionID) { + graph.union(owner, Self.scoped(parent, document: document)) + } + } + + var documentsByFamily: [String: [Document]] = [:] + for document in documents { + try checkCancellation?() + let family = graph.find(Self.scoped(document.ownerID, document: document)) + documentsByFamily[family, default: []].append(document) + } + + var primaryDocuments: [Document] = [] + var containedDocuments: [Document] = [] + var families: [FamilyDisposition] = [] + for familyDocuments in documentsByFamily.values { + try checkCancellation?() + let reasons = Self.containmentReasons(in: familyDocuments) + let owners = Set(familyDocuments.map(\.ownerID)) + let unresolved = familyDocuments.contains { document in + guard let parent = Self.nonEmpty(document.parentSessionID) else { return false } + return unresolvedParents.contains(.init( + scopeID: document.scopeID, + sessionID: Self.canonicalIdentity(parent))) + } + let quality: FamilyQuality + if reasons.isEmpty { + quality = unresolved ? .incompleteProvenance : .primary + primaryDocuments.append(contentsOf: familyDocuments) + } else { + quality = .contained(reasons) + containedDocuments.append(contentsOf: familyDocuments) + } + families.append(FamilyDisposition( + scopeID: familyDocuments.first?.scopeID ?? "", + ownerIDs: owners, + quality: quality)) + } + + families.sort { + ($0.scopeID, $0.ownerIDs.sorted().joined(separator: "\u{0}")) + < ($1.scopeID, $1.ownerIDs.sorted().joined(separator: "\u{0}")) + } + containedDocuments.sort(by: Self.documentComesBefore) + return try ConservativeReport( + primary: Self.reconcile( + documents: primaryDocuments, + localTimeZone: localTimeZone, + checkCancellation: checkCancellation), + families: families, + containedDocuments: containedDocuments) + } + private struct Fingerprint: Equatable, Hashable { let last: Totals let total: Totals @@ -170,6 +295,117 @@ enum CodexLineageLedger { return value } + private static func scoped(_ identity: String, document: Document) -> String { + document.scopeID + "\u{0}" + self.canonicalIdentity(identity) + } + + private static func canonicalIdentity(_ identity: String) -> String { + UUID(uuidString: identity)?.uuidString.lowercased() ?? identity + } + + private static func documentComesBefore(_ lhs: Document, _ rhs: Document) -> Bool { + let lhsKey = [ + lhs.scopeID, + lhs.ownerID, + lhs.metadataSessionID ?? "", + lhs.parentSessionID ?? "", + String(lhs.incompleteObservationCount), + lhs.observations.map(Self.observationSortKey).joined(separator: "\u{1}"), + ].joined(separator: "\u{0}") + let rhsKey = [ + rhs.scopeID, + rhs.ownerID, + rhs.metadataSessionID ?? "", + rhs.parentSessionID ?? "", + String(rhs.incompleteObservationCount), + rhs.observations.map(Self.observationSortKey).joined(separator: "\u{1}"), + ].joined(separator: "\u{0}") + return lhsKey < rhsKey + } + + private static func observationSortKey(_ observation: Observation) -> String { + [ + observation.timestamp, + observation.model, + String(observation.last.input), + String(observation.last.cached), + String(observation.last.output), + String(observation.total.input), + String(observation.total.cached), + String(observation.total.output), + ].joined(separator: "\u{0}") + } + + private static func containmentReasons(in documents: [Document]) -> Set { + var reasons: Set = [] + if documents.contains(where: { $0.incompleteObservationCount > 0 }) { + reasons.insert(.incompleteObservation) + } + if documents.flatMap(\.observations) + .contains(where: { CostUsageScanner.dateFromTimestamp($0.timestamp) == nil }) + { + reasons.insert(.malformedTimestamp) + } + let ownerGroups = Dictionary(grouping: documents, by: \.ownerID) + if ownerGroups.values.contains(where: { group in + let metadataIDs = Set(group.compactMap { Self.nonEmpty($0.metadataSessionID) }) + let parentIDs = Set(group.compactMap { Self.nonEmpty($0.parentSessionID) }) + return metadataIDs.count > 1 || parentIDs.count > 1 + }) { + reasons.insert(.conflictingOwnerIdentity) + } + let metadataGroups = Dictionary(grouping: documents.compactMap { document in + Self.nonEmpty(document.metadataSessionID).map { (Self.canonicalIdentity($0), document) } + }, by: \.0) + if metadataGroups.contains(where: { metadataID, entries in + let documents = entries.map(\.1) + let owners = Set(documents.map { Self.canonicalIdentity($0.ownerID) }) + guard owners.count > 1, !owners.contains(metadataID) else { return false } + return !documents.allSatisfy { + $0.parentSessionID.map(Self.canonicalIdentity) == metadataID + } + }) { + reasons.insert(.identityCollision) + } + if Self.hasAncestryCycle(documents) { + reasons.insert(.ancestryCycle) + } + return reasons + } + + private static func hasAncestryCycle(_ documents: [Document]) -> Bool { + let metadataOwners = Dictionary(grouping: documents.compactMap { document in + document.metadataSessionID.map { ($0, document.ownerID) } + }, by: \.0).mapValues { Set($0.map(\.1)) } + var parents: [String: Set] = [:] + for document in documents { + guard let parent = Self.nonEmpty(document.parentSessionID) else { continue } + var targets = metadataOwners[parent] ?? [parent] + if parent != document.ownerID { + targets.remove(document.ownerID) + } + parents[document.ownerID, default: []].formUnion(targets) + } + var visiting: Set = [] + var visited: Set = [] + func visit(_ owner: String) -> Bool { + if visiting.contains(owner) { + return true + } + if visited.contains(owner) { + return false + } + visiting.insert(owner) + for parent in parents[owner] ?? [] where visit(parent) { + return true + } + visiting.remove(owner) + visited.insert(owner) + return false + } + return Set(documents.map(\.ownerID)).contains(where: visit) + } + /// Duplicate copies can carry different model evidence. Keep the earliest physical copy, /// then resolve equal-time ties deterministically while preferring attributable evidence. private static func shouldPreferModel(_ candidate: String, over existing: String) -> Bool { diff --git a/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift b/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift index a6938113e0..aae024629e 100644 --- a/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift +++ b/Sources/CodexBarCore/Providers/Codex/CodexLineageShadow.swift @@ -23,6 +23,10 @@ enum CodexLineageShadow { let referencedParentDocumentCount: Int let unresolvedParentCount: Int let rejectedObservationCount: Int + let primaryFamilyCount: Int + let incompleteProvenanceFamilyCount: Int + let containedFamilyCount: Int + let containmentReasonCounts: [CodexLineageLedger.ContainmentReason: Int] } static func run( @@ -42,27 +46,27 @@ enum CodexLineageShadow { documents.reserveCapacity(discovery.documents.count) for document in discovery.documents { try checkCancellation?() - var observations: [CodexLineageLedger.Observation] = [] - observations.reserveCapacity(document.observations.count) for observation in document.observations { try checkCancellation?() let accepted = CostUsageScanner.dateFromTimestamp(observation.timestamp) != nil if !accepted { rejectedObservationCount += 1 - } else { - observations.append(observation) } } documents.append(CodexLineageLedger.Document( ownerID: document.ownerID, metadataSessionID: document.metadataSessionID, parentSessionID: document.parentSessionID, - observations: observations)) + observations: document.observations, + scopeID: document.scopeID, + incompleteObservationCount: document.incompleteObservationCount)) } - let ledger = try CodexLineageLedger.reconcile( + let conservative = try CodexLineageLedger.reconcileConservatively( documents: documents, + unresolvedParents: discovery.unresolvedParents, localTimeZone: localTimeZone, checkCancellation: checkCancellation) + let ledger = conservative.primary let legacy = Self.legacyTotalsByDay(legacyDays) let dayKeys = Set(legacy.keys).union(ledger.localDays.keys) .filter(dayRange.contains) @@ -76,8 +80,32 @@ enum CodexLineageShadow { duplicateObservationCount: ledger.duplicateObservationCount, componentCount: ledger.componentCount, referencedParentDocumentCount: discovery.referencedParentDocumentCount, - unresolvedParentCount: discovery.unresolvedParentIDs.count, - rejectedObservationCount: rejectedObservationCount) + unresolvedParentCount: discovery.unresolvedParents.count, + rejectedObservationCount: rejectedObservationCount, + primaryFamilyCount: conservative.families.count { $0.quality == .primary }, + incompleteProvenanceFamilyCount: conservative.families.count { + $0.quality == .incompleteProvenance + }, + containedFamilyCount: conservative.families.count { + if case .contained = $0.quality { + return true + } + return false + }, + containmentReasonCounts: Self.containmentReasonCounts(conservative.families)) + } + + private static func containmentReasonCounts( + _ families: [CodexLineageLedger.FamilyDisposition]) -> [CodexLineageLedger.ContainmentReason: Int] + { + var counts: [CodexLineageLedger.ContainmentReason: Int] = [:] + for family in families { + guard case let .contained(reasons) = family.quality else { continue } + for reason in reasons { + counts[reason, default: 0] += 1 + } + } + return counts } private static func legacyTotalsByDay( diff --git a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift index 08c25b83e3..8514a9bcb2 100644 --- a/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift +++ b/Sources/CodexBarCore/Vendored/CostUsage/CostUsageScanner.swift @@ -103,6 +103,7 @@ enum CostUsageScanner { let forkedFromId: String? let snapshots: [CodexTimestampedTotals] let observations: [CodexLineageLedger.Observation] + let incompleteObservationCount: Int } enum CodexForkBaseline { @@ -1685,7 +1686,7 @@ enum CostUsageScanner { return nil } - // swiftlint:disable:next cyclomatic_complexity + // swiftlint:disable:next cyclomatic_complexity function_body_length private static func parseCodexTokenSnapshots( fileURL: URL, checkCancellation: CancellationCheck? = nil) throws -> CodexParsedTokenEvidence @@ -1696,6 +1697,7 @@ enum CostUsageScanner { var accumulator = CodexSnapshotAccumulator() var snapshots: [CodexTimestampedTotals] = [] var observations: [CodexLineageLedger.Observation] = [] + var incompleteObservationCount = 0 var warnedAboutUnparsedTimestamp = false func parsedSnapshotDate(timestamp: String) -> Date? { @@ -1716,6 +1718,9 @@ enum CostUsageScanner { total: CostUsageCodexTotals?) { guard last != nil || total != nil else { return } + if last == nil || total == nil { + incompleteObservationCount += 1 + } let counted = accumulator.apply(last: last, total: total) snapshots.append(CodexTimestampedTotals( timestamp: timestamp, @@ -1739,7 +1744,13 @@ enum CostUsageScanner { prefixBytes: 512 * 1024, checkCancellation: checkCancellation, onLine: { line in - guard !line.bytes.isEmpty, !line.wasTruncated else { return } + guard !line.bytes.isEmpty else { return } + if line.wasTruncated { + if line.bytes.range(of: Data(#""token_count""#.utf8)) != nil { + incompleteObservationCount += 1 + } + return + } if let fastLine = Self.parseCodexFastLine(line.bytes) { switch fastLine { case let .sessionMeta(metadata): @@ -1843,7 +1854,8 @@ enum CostUsageScanner { sessionId: sessionId, forkedFromId: forkedFromId, snapshots: snapshots, - observations: observations) + observations: observations, + incompleteObservationCount: incompleteObservationCount) } static func parseCodexLineageDocument( @@ -1857,7 +1869,18 @@ enum CostUsageScanner { ownerID: Self.codexRolloutOwnerID(fileURL: fileURL) ?? parsed.sessionId ?? fileURL.standardizedFileURL.path, metadataSessionID: parsed.sessionId, parentSessionID: parsed.forkedFromId, - observations: parsed.observations) + observations: parsed.observations, + scopeID: Self.codexLineageScopeID(fileURL: fileURL), + incompleteObservationCount: parsed.incompleteObservationCount) + } + + static func codexLineageScopeID(fileURL: URL) -> String { + let standardized = fileURL.standardizedFileURL + let components = standardized.pathComponents + if let index = components.lastIndex(where: { $0 == "sessions" || $0 == "archived_sessions" }) { + return NSString.path(withComponents: Array(components[.. CodexLineageLedger.Totals { diff --git a/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift b/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift index 9ab3126144..c3c40dc4e7 100644 --- a/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift +++ b/Tests/CodexBarTests/CodexLineageDiscoveryTests.swift @@ -41,7 +41,7 @@ struct CodexLineageDiscoveryTests { #expect(Set(report.documents.map(\.ownerID)) == [childID, parentID, grandparentID]) #expect(report.documents.map(\.ownerID).contains(unrelatedID) == false) #expect(report.referencedParentDocumentCount == 2) - #expect(report.unresolvedParentIDs.isEmpty) + #expect(report.unresolvedParents.isEmpty) #expect(FileManager.default.fileExists(atPath: parent.path)) #expect(FileManager.default.fileExists(atPath: grandparent.path)) } @@ -63,7 +63,9 @@ struct CodexLineageDiscoveryTests { #expect(report.documents.count == 1) #expect(report.referencedParentDocumentCount == 0) - #expect(report.unresolvedParentIDs == ["66666666-6666-4666-8666-666666666666"]) + #expect(report.unresolvedParents == [.init( + scopeID: environment.codexSessionsRoot.deletingLastPathComponent().path, + sessionID: "66666666-6666-4666-8666-666666666666")]) } @Test @@ -89,7 +91,100 @@ struct CodexLineageDiscoveryTests { #expect(report.documents.map(\.ownerID).contains(parentID)) #expect(report.referencedParentDocumentCount == 1) - #expect(report.unresolvedParentIDs.isEmpty) + #expect(report.unresolvedParents.isEmpty) + } + + @Test + func `parent lookup never crosses normalized Codex homes`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let parentID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + let otherHomeSessions = environment.root + .appendingPathComponent("other-home", isDirectory: true) + .appendingPathComponent("sessions", isDirectory: true) + _ = try Self.writeRollout( + root: otherHomeSessions, + relativeDirectory: "2026/07/09", + ownerID: parentID, + metadataID: parentID) + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + metadataID: "child", + parentID: parentID) + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, otherHomeSessions]) + + #expect(report.documents.map(\.ownerID) == ["bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"]) + #expect(report.unresolvedParents == [.init( + scopeID: environment.codexSessionsRoot.deletingLastPathComponent().path, + sessionID: parentID)]) + } + + @Test + func `ambiguous parent identity is unresolved instead of path-order dependent`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let parentAlias = "shared-parent-alias" + _ = try Self.writeRollout( + root: environment.codexArchivedSessionsRoot, + relativeDirectory: "", + ownerID: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + metadataID: parentAlias) + _ = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2025/01/01", + ownerID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + metadataID: parentAlias) + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + metadataID: "child", + parentID: parentAlias) + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, environment.codexArchivedSessionsRoot]) + + #expect(report.documents.count == 1) + #expect(report.unresolvedParents == [.init( + scopeID: environment.codexSessionsRoot.deletingLastPathComponent().path, + sessionID: parentAlias)]) + } + + @Test + func `compatible active and archived copies of one parent are both retained`() throws { + let environment = try CostUsageTestEnvironment() + defer { environment.cleanup() } + let parentID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + _ = try Self.writeRollout( + root: environment.codexArchivedSessionsRoot, + relativeDirectory: "", + ownerID: parentID, + metadataID: parentID) + _ = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2025/01/01", + ownerID: parentID, + metadataID: parentID) + let child = try Self.writeRollout( + root: environment.codexSessionsRoot, + relativeDirectory: "2026/07/09", + ownerID: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + metadataID: "child", + parentID: parentID) + + let report = try CodexLineageDiscovery.discover( + includedFiles: [child], + roots: [environment.codexSessionsRoot, environment.codexArchivedSessionsRoot]) + + #expect(report.documents.count == 3) + #expect(report.referencedParentDocumentCount == 2) + #expect(report.unresolvedParents.isEmpty) } private static func writeRollout( diff --git a/Tests/CodexBarTests/CodexLineageLedgerTests.swift b/Tests/CodexBarTests/CodexLineageLedgerTests.swift index aca6368d34..4d23eb192d 100644 --- a/Tests/CodexBarTests/CodexLineageLedgerTests.swift +++ b/Tests/CodexBarTests/CodexLineageLedgerTests.swift @@ -32,6 +32,8 @@ struct CodexLineageLedgerTests { #expect(document.metadataSessionID == "metadata-id") #expect(document.parentSessionID == "parent-id") #expect(document.observations.count == 2) + #expect(document.incompleteObservationCount == 1) + #expect(document.scopeID == environment.root.path) #expect(document.observations[0].model == "gpt-5.4") #expect(document.observations[0].last == .init(input: 100, cached: 40, output: 10)) #expect(document.observations[1].total == .init(input: 150, cached: 60, output: 15)) @@ -309,6 +311,168 @@ struct CodexLineageLedgerTests { #expect(forward.utcDays["2026-07-10"] == nil) } + @Test + func `incomplete evidence contains the entire family without primary contribution`() throws { + let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100) + let parent = Self.document(owner: "parent", observations: [observation]) + let child = CodexLineageLedger.Document( + ownerID: "child", + metadataSessionID: "child", + parentSessionID: "parent", + observations: [observation], + incompleteObservationCount: 1) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [parent, child], + localTimeZone: .gmt) + + #expect(report.primary.utcDays.isEmpty) + #expect(report.containedDocuments.count == 2) + #expect(report.families == [.init( + scopeID: "", + ownerIDs: ["parent", "child"], + quality: .contained([.incompleteObservation]))]) + } + + @Test + func `missing ancestry lowers provenance without discarding unique descendant usage`() throws { + let child = Self.document( + owner: "child", + parent: "missing-parent", + observations: [Self.observation( + timestamp: "2026-07-09T12:00:00Z", + input: 100, + totalInput: 100)]) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [child], + unresolvedParents: [.init(sessionID: "missing-parent")], + localTimeZone: .gmt) + + #expect(report.primary.utcDays["2026-07-09"]?.input == 100) + #expect(report.containedDocuments.isEmpty) + #expect(report.families.first?.quality == .incompleteProvenance) + } + + @Test + func `malformed timestamps and ancestry cycles route families to containment deterministically`() throws { + let malformed = Self.observation(timestamp: "not-a-time", input: 20, totalInput: 20) + let first = Self.document(owner: "first", parent: "second", observations: [malformed]) + let second = Self.document(owner: "second", parent: "first", observations: []) + + let forward = try CodexLineageLedger.reconcileConservatively( + documents: [first, second], + localTimeZone: .gmt) + let reversed = try CodexLineageLedger.reconcileConservatively( + documents: [second, first], + localTimeZone: .gmt) + + #expect(forward == reversed) + #expect(forward.primary.utcDays.isEmpty) + #expect(forward.families.first?.quality == .contained([.malformedTimestamp, .ancestryCycle])) + } + + @Test + func `identical identities in separate Codex homes remain additive`() throws { + let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100) + let first = CodexLineageLedger.Document( + ownerID: "same-owner", + metadataSessionID: "same-metadata", + parentSessionID: nil, + observations: [observation], + scopeID: "/first-home") + let second = CodexLineageLedger.Document( + ownerID: "same-owner", + metadataSessionID: "same-metadata", + parentSessionID: nil, + observations: [observation], + scopeID: "/second-home") + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [first, second], + localTimeZone: .gmt) + + #expect(report.primary.utcDays["2026-07-09"]?.input == 200) + #expect(report.primary.componentCount == 2) + #expect(report.families.count == 2) + } + + @Test + func `conflicting physical copies of one owner are contained`() throws { + let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100) + let first = Self.document(owner: "same-owner", metadata: "first", observations: [observation]) + let second = Self.document(owner: "same-owner", metadata: "second", observations: [observation]) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [first, second], + localTimeZone: .gmt) + + #expect(report.primary.utcDays.isEmpty) + #expect(report.families.first?.quality == .contained([.conflictingOwnerIdentity])) + } + + @Test + func `missing optional identity fields remain compatible with a complete copy`() throws { + let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100) + let complete = Self.document( + owner: "same-owner", + metadata: "metadata", + parent: "parent", + observations: [observation]) + let partial = Self.document(owner: "same-owner", observations: [observation]) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [partial, complete], + localTimeZone: .gmt) + + #expect(report.primary.utcDays["2026-07-09"]?.input == 100) + #expect(report.containedDocuments.isEmpty) + #expect(report.families.first?.quality == .primary) + } + + @Test + func `retained ancestor metadata does not manufacture an ancestry cycle`() throws { + let observation = Self.observation(timestamp: "2026-07-09T12:00:00Z", input: 100, totalInput: 100) + let root = Self.document(owner: "root", metadata: "root", observations: [observation]) + let child = Self.document( + owner: "child", + metadata: "root", + parent: "root", + observations: [observation]) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [root, child], + localTimeZone: .gmt) + + #expect(report.primary.utcDays["2026-07-09"]?.input == 100) + #expect(report.families.first?.quality == .primary) + } + + @Test + func `unrelated owners sharing an ungrounded metadata identity are contained`() throws { + let first = Self.document( + owner: "first", + metadata: "shared", + observations: [Self.observation( + timestamp: "2026-07-09T12:00:00Z", + input: 100, + totalInput: 100)]) + let second = Self.document( + owner: "second", + metadata: "shared", + observations: [Self.observation( + timestamp: "2026-07-09T12:01:00Z", + input: 200, + totalInput: 200)]) + + let report = try CodexLineageLedger.reconcileConservatively( + documents: [first, second], + localTimeZone: .gmt) + + #expect(report.primary.utcDays.isEmpty) + #expect(report.families.first?.quality == .contained([.identityCollision])) + } + private static func document( owner: String, metadata: String? = nil, diff --git a/Tests/CodexBarTests/CodexLineageShadowTests.swift b/Tests/CodexBarTests/CodexLineageShadowTests.swift index 51eb59adfc..b6c5454070 100644 --- a/Tests/CodexBarTests/CodexLineageShadowTests.swift +++ b/Tests/CodexBarTests/CodexLineageShadowTests.swift @@ -36,13 +36,16 @@ struct CodexLineageShadowTests { #expect(report.days == [.init( day: "2026-07-09", legacy: .init(input: 150, cached: 20, output: 10), - ledger: .init(input: 100, cached: 0, output: 0))]) - #expect(report.days[0].delta == .init(input: -50, cached: -20, output: -10)) - #expect(report.acceptedObservationCount == 1) - #expect(report.duplicateObservationCount == 1) - #expect(report.componentCount == 1) + ledger: .zero)]) + #expect(report.days[0].delta == .init(input: -150, cached: -20, output: -10)) + #expect(report.acceptedObservationCount == 0) + #expect(report.duplicateObservationCount == 0) + #expect(report.componentCount == 0) #expect(report.rejectedObservationCount == 1) #expect(report.unresolvedParentCount == 0) + #expect(report.primaryFamilyCount == 0) + #expect(report.containedFamilyCount == 1) + #expect(report.containmentReasonCounts == [.malformedTimestamp: 1]) } private static func writeRollout(