Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Generated by Scripts/regenerate-codex-parser-hash.sh. Do not edit by hand.

enum CodexParserHash {
static let value = "cdef6eb9658a43e2"
static let value = "2f24930e5c06fb86"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import Foundation

enum CodexLineageAccountingMode: String, CaseIterable, Sendable {
case legacy
case shadow
case lineage

static let defaultMode: Self = .legacy
static let schemaVersion = 1

var producerKeySuffix: String {
"lineage-accounting:\(self.rawValue):v\(Self.schemaVersion)"
}
}

/// Selects one whole-scan authority while preserving a permanent family-scoped containment route.
enum CodexLineageAccountingSelector {
typealias PackedDays = [String: [String: [Int]]]

enum SelectionError: Error, Equatable {
case missingContainedFamilyEvidence
}

struct ContainedDocument: Equatable, Sendable {
/// Stable logical rollout identity. Physical active/archive copies share this value.
let identity: String
let days: PackedDays
}

struct ContainedFamily: Equatable, Sendable {
/// Legacy file contributions belonging only to this contained family.
let documents: [ContainedDocument]
}

struct Selection: Equatable, Sendable {
let days: PackedDays
let usedLineageAuthority: Bool
let containedFamilyCount: Int
}

static func select(
mode: CodexLineageAccountingMode,
authorization: CodexLineagePromotionEvaluator.Authorization? = nil,
legacyDays: PackedDays,
primaryRows: [CodexLineageLedger.DailyRow],
containedFamilies: [ContainedFamily]) -> Selection
{
guard mode == .lineage, authorization != nil else {
return Selection(days: legacyDays, usedLineageAuthority: false, containedFamilyCount: 0)
}
var days = Self.days(from: primaryRows)
for family in containedFamilies {
Self.add(Self.containedDays(family.documents), to: &days)
}
return Selection(
days: days,
usedLineageAuthority: true,
containedFamilyCount: containedFamilies.count)
}

/// A contained family contributes once. Physical copies of one logical rollout use an envelope;
/// distinct siblings remain additive so containment does not discard their independent work.
private static func containedDays(_ documents: [ContainedDocument]) -> PackedDays {
var documentsByIdentity: [String: [PackedDays]] = [:]
for document in documents {
documentsByIdentity[document.identity, default: []].append(document.days)
}
var result: PackedDays = [:]
for copies in documentsByIdentity.values {
var envelope: PackedDays = [:]
for copy in copies {
for (day, models) in copy {
for (model, packed) in models {
let existing = envelope[day]?[model] ?? [0, 0, 0]
envelope[day, default: [:]][model] = (0..<3).map { index in
max(existing[safe: index] ?? 0, packed[safe: index] ?? 0)
}
}
}
}
Self.add(envelope, to: &result)
}
return result
}

private static func days(from rows: [CodexLineageLedger.DailyRow]) -> PackedDays {
var days: PackedDays = [:]
for row in rows {
var packed = days[row.day]?[row.model] ?? [0, 0, 0]
packed[0] += row.totals.input
packed[1] += row.totals.cached
packed[2] += row.totals.output
days[row.day, default: [:]][row.model] = packed
}
return days
}

private static func add(_ source: PackedDays, to destination: inout PackedDays) {
for (day, models) in source {
for (model, packed) in models {
var value = destination[day]?[model] ?? [0, 0, 0]
for index in 0..<3 {
value[index] += packed[safe: index] ?? 0
}
destination[day, default: [:]][model] = value
}
}
}
}
159 changes: 159 additions & 0 deletions Sources/CodexBarCore/Providers/Codex/CodexLineageDiscovery.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
import Foundation

/// Expands an already bounded rollout-file selection with only the ancestors its documents reference.
enum CodexLineageDiscovery {
struct Report: Equatable, Sendable {
let documents: [CodexLineageLedger.Document]
let referencedParentDocumentCount: Int
let unresolvedParents: Set<CodexLineageLedger.ParentIdentity>
}

static func discover(
includedFiles: [URL],
roots: [URL],
checkCancellation: CostUsageScanner.CancellationCheck? = nil) throws -> Report
{
var locator = ParentFileLocator(roots: roots, checkCancellation: checkCancellation)
var documents: [CodexLineageLedger.Document] = []
var knownIDs: Set<ScopedIdentity> = []
var seenPaths: Set<String> = []
var pendingParentIDs: [ScopedIdentity] = []
var unresolvedParents: Set<CodexLineageLedger.ParentIdentity> = []
var referencedParentDocumentCount = 0

func remember(_ document: CodexLineageLedger.Document) {
documents.append(document)
knownIDs.insert(.init(scopeID: document.scopeID, sessionID: Self.canonicalSessionID(document.ownerID)))
if let parentSessionID = Self.nonEmpty(document.parentSessionID) {
pendingParentIDs.append(.init(
scopeID: document.scopeID,
sessionID: Self.canonicalSessionID(parentSessionID)))
}
}

for fileURL in includedFiles.sorted(by: { $0.path < $1.path }) {
try checkCancellation?()
let path = fileURL.standardizedFileURL.path
guard seenPaths.insert(path).inserted else { continue }
try remember(CostUsageScanner.parseCodexLineageDocument(
fileURL: fileURL,
checkCancellation: checkCancellation))
}

var nextParent = 0
while nextParent < pendingParentIDs.count {
try checkCancellation?()
let parentIdentity = pendingParentIDs[nextParent]
nextParent += 1
let unresolvedIdentity = CodexLineageLedger.ParentIdentity(
scopeID: parentIdentity.scopeID,
sessionID: parentIdentity.sessionID)
guard !knownIDs.contains(parentIdentity), !unresolvedParents.contains(unresolvedIdentity) else {
continue
}
guard let parentURLs = try locator.fileURLs(
for: parentIdentity.sessionID,
scopeID: parentIdentity.scopeID)
else {
unresolvedParents.insert(unresolvedIdentity)
continue
}
var foundParent = false
for parentURL in parentURLs {
let path = parentURL.standardizedFileURL.path
guard seenPaths.insert(path).inserted else { continue }
let parent = try CostUsageScanner.parseCodexLineageDocument(
fileURL: parentURL,
checkCancellation: checkCancellation)
let ownerID = Self.canonicalSessionID(parent.ownerID)
guard ownerID == parentIdentity.sessionID else {
continue
}
referencedParentDocumentCount += 1
foundParent = true
remember(parent)
}
if !foundParent {
unresolvedParents.insert(unresolvedIdentity)
}
}

return Report(
documents: documents,
referencedParentDocumentCount: referencedParentDocumentCount,
unresolvedParents: unresolvedParents)
}

private static func nonEmpty(_ value: String?) -> String? {
guard let value, !value.isEmpty else { return nil }
return value
}

private static func canonicalSessionID(_ value: String) -> String {
UUID(uuidString: value)?.uuidString.lowercased() ?? value
}

private struct ScopedIdentity: Equatable, Hashable {
let scopeID: String
let sessionID: String
}

private struct ParentFileLocator {
let roots: [URL]
let checkCancellation: CostUsageScanner.CancellationCheck?
var didIndexRoots = false
var filesByID: [ScopedIdentity: Set<URL>] = [:]

mutating func fileURLs(for sessionID: String, scopeID: String) throws -> [URL]? {
let canonicalID = CodexLineageDiscovery.canonicalSessionID(sessionID)
let key = ScopedIdentity(scopeID: scopeID, sessionID: canonicalID)
if let known = self.filesByID[key] {
return Self.unambiguousMatches(known)
}
if !self.didIndexRoots {
try self.indexRoots()
}
guard let matches = self.filesByID[key] else { return nil }
return Self.unambiguousMatches(matches)
}

private static func unambiguousMatches(_ matches: Set<URL>) -> [URL]? {
guard !matches.isEmpty else { return nil }
if matches.count == 1 {
return Array(matches)
}
let ownerIDs = Set(matches.compactMap(CostUsageScanner.codexRolloutOwnerID(fileURL:)))
guard ownerIDs.count == 1 else { return nil }
return matches.sorted { $0.path < $1.path }
}

private mutating func indexRoots() throws {
self.didIndexRoots = true
var files: [URL] = []
for root in self.roots {
try self.checkCancellation?()
guard let enumerator = FileManager.default.enumerator(
at: root,
includingPropertiesForKeys: [.isRegularFileKey],
options: [.skipsHiddenFiles, .skipsPackageDescendants])
else { continue }
while let fileURL = enumerator.nextObject() as? URL {
try self.checkCancellation?()
guard fileURL.pathExtension.lowercased() == "jsonl" else { continue }
files.append(fileURL)
}
}

for fileURL in files.sorted(by: { $0.path < $1.path }) {
try self.checkCancellation?()
if let ownerID = CostUsageScanner.codexRolloutOwnerID(fileURL: fileURL) {
let canonicalID = CodexLineageDiscovery.canonicalSessionID(ownerID)
let key = ScopedIdentity(
scopeID: CostUsageScanner.codexLineageScopeID(fileURL: fileURL),
sessionID: canonicalID)
self.filesByID[key, default: []].insert(fileURL)
}
}
}
}
}
Loading