graphify-notes/tiktoken_drift.json at main · ipunithgowda/graphify-notes · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
{
  "n": 15,
  "encoder_used": "cl100k_base (OpenAI GPT-4/3.5; close proxy for Claude)",
  "graphify_heuristic": "words \u00d7 1.33",
  "stats": {
    "abs_drift_pct": {
      "min": 0.3,
      "median": 6.1,
      "max": 74.2,
      "mean": 22.3
    }
  },
  "per_query": [
    {
      "q": "how does authentication work",
      "heuristic": 54,
      "tiktoken": 53,
      "drift_pct": -1.9
    },
    {
      "q": "what is the main entry point",
      "heuristic": 628,
      "tiktoken": 189,
      "drift_pct": -69.9
    },
    {
      "q": "how are errors handled",
      "heuristic": 441,
      "tiktoken": 164,
      "drift_pct": -62.8
    },
    {
      "q": "what connects the data layer to the api",
      "heuristic": 706,
      "tiktoken": 189,
      "drift_pct": -73.2
    },
    {
      "q": "what are the core abstractions",
      "heuristic": 388,
      "tiktoken": 100,
      "drift_pct": -74.2
    },
    {
      "q": "how is the audit chain constructed",
      "heuristic": 3228,
      "tiktoken": 3651,
      "drift_pct": 13.1
    },
    {
      "q": "where are database queries defined",
      "heuristic": 3270,
      "tiktoken": 3400,
      "drift_pct": 4.0
    },
    {
      "q": "how is tenant isolation enforced",
      "heuristic": 229,
      "tiktoken": 227,
      "drift_pct": -0.9
    },
    {
      "q": "where is state mutation tracked",
      "heuristic": 134,
      "tiktoken": 143,
      "drift_pct": 6.7
    },
    {
      "q": "which modules handle export",
      "heuristic": 342,
      "tiktoken": 363,
      "drift_pct": 6.1
    },
    {
      "q": "which tests cover the correlation engine",
      "heuristic": 426,
      "tiktoken": 447,
      "drift_pct": 4.9
    },
    {
      "q": "how is the severity score computed",
      "heuristic": 369,
      "tiktoken": 370,
      "drift_pct": 0.3
    },
    {
      "q": "where is the configuration loaded",
      "heuristic": 3499,
      "tiktoken": 3400,
      "drift_pct": -2.8
    },
    {
      "q": "how does the incident workflow transition states",
      "heuristic": 306,
      "tiktoken": 332,
      "drift_pct": 8.5
    },
    {
      "q": "what defines a playbook",
      "heuristic": 117,
      "tiktoken": 123,
      "drift_pct": 5.1
    }
  ]
}