Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions examples/ai-token-stream-expr-usage.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import { type AITokenUsagePayload, mul, inputTokens, outputTokens } from "@scrawn/core";
import {
type AITokenUsagePayload,
mul,
inputTokens,
outputTokens,
} from "@scrawn/core";
import { biller } from "./scrawn/biller.ts";
import { config } from "dotenv";
config({ path: ".env.local" });

async function* tokenUsageFromAIStream(): AsyncGenerator<AITokenUsagePayload<"PREMIUM_CALL" | "EXTRA_FEE">> {
async function* tokenUsageFromAIStream(): AsyncGenerator<
AITokenUsagePayload<"PREMIUM_CALL" | "EXTRA_FEE">
> {
const userId = "c0971bcb-b901-4c3e-a191-c9a97871c39f";

yield {
Expand All @@ -12,16 +19,22 @@ async function* tokenUsageFromAIStream(): AsyncGenerator<AITokenUsagePayload<"PR
inputTokens: 150,
outputTokens: 0,
inputDebit: { expr: biller.expr("PER_TOKEN_INPUT") },
outputDebit: { expr: biller.expr(mul(biller.tag("EXTRA_FEE"), outputTokens())) },
outputDebit: {
expr: biller.expr(mul(biller.tag("EXTRA_FEE"), outputTokens())),
},
};

yield {
userId,
model: "gpt-4",
inputTokens: 0,
outputTokens: 75,
inputDebit: { expr: biller.expr(mul(biller.tag("PREMIUM_CALL"), inputTokens())) },
outputDebit: { expr: biller.expr(mul(biller.tag("EXTRA_FEE"), outputTokens())) },
inputDebit: {
expr: biller.expr(mul(biller.tag("PREMIUM_CALL"), inputTokens())),
},
outputDebit: {
expr: biller.expr(mul(biller.tag("EXTRA_FEE"), outputTokens())),
},
};
}

Expand Down
45 changes: 28 additions & 17 deletions examples/analytics-usage.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,14 @@
import { Analytics, eq, neq, gt, and, asc, desc, sum, count } from "@scrawn/analytics";
import {
Analytics,
eq,
neq,
gt,
and,
asc,
desc,
sum,
count,
} from "@scrawn/analytics";
import { biller } from "./scrawn/biller.ts";
import { config } from "dotenv";
config({ path: ".env.local" });
Expand All @@ -21,14 +31,19 @@ async function main() {

// Middleware events with high debit
const expensiveMiddleware = await sdkEvent
.where(and(
eq(sdkEvent.fields.basicUsageType, "MIDDLEWARE_CALL"),
gt(sdkEvent.fields.debitAmount, 100),
))
.where(
and(
eq(sdkEvent.fields.basicUsageType, "MIDDLEWARE_CALL"),
gt(sdkEvent.fields.debitAmount, 100)
)
)
.orderBy(desc(sdkEvent.fields.debitAmount))
.limit(5)
.execute();
console.log("Expensive middleware calls:", JSON.stringify(expensiveMiddleware, null, 2));
console.log(
"Expensive middleware calls:",
JSON.stringify(expensiveMiddleware, null, 2)
);

// AI token usage for a specific model
const gpt4Usage = await aiToken
Expand All @@ -48,36 +63,32 @@ async function main() {
console.log("Total debit by user:", JSON.stringify(totalByUser, null, 2));

// Count of payment events
const paymentCount = await payment
.aggregate(count())
.execute();
const paymentCount = await payment.aggregate(count()).execute();
console.log("Payment events:", JSON.stringify(paymentCount, null, 2));

// ── Data Queries ──

// List production users
const prodUsers = await users
.where(and(
eq(users.fields.mode, "production"),
))
.where(and(eq(users.fields.mode, "production")))
.orderBy(asc(users.fields.id))
.limit(10)
.execute();
console.log("Production users:", JSON.stringify(prodUsers, null, 2));

// List all tags
const allTags = await tags
.orderBy(asc(tags.fields.key))
.limit(50)
.execute();
const allTags = await tags.orderBy(asc(tags.fields.key)).limit(50).execute();
console.log("Tags:", JSON.stringify(allTags, null, 2));

// Unprocessed sessions
const unprocessedSessions = await sessions
.where(eq(sessions.fields.processed, "false"))
.limit(10)
.execute();
console.log("Unprocessed sessions:", JSON.stringify(unprocessedSessions, null, 2));
console.log(
"Unprocessed sessions:",
JSON.stringify(unprocessedSessions, null, 2)
);
}

main().catch(console.error);
20 changes: 8 additions & 12 deletions examples/basic-usage.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
import { biller } from "./scrawn/biller.ts";

async function main() {
await biller.basicUsageEventConsumer(
{
userId: "c0971bcb-b901-4c3e-a191-c9a97871c39f",
debitAmount: 3000,
}
);
await biller.basicUsageEventConsumer({
userId: "c0971bcb-b901-4c3e-a191-c9a97871c39f",
debitAmount: 3000,
});

await biller.basicUsageEventConsumer(
{
userId: "c0971bcb-b901-4c3e-a191-c9a97871c39f",
debitTag: "PREMIUM_CALL",
}
);
await biller.basicUsageEventConsumer({
userId: "c0971bcb-b901-4c3e-a191-c9a97871c39f",
debitTag: "PREMIUM_CALL",
});

console.log("Basic usage events consumed successfully");
}
Expand Down
70 changes: 52 additions & 18 deletions packages/analytics/scripts/gen-fields.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
/**
* Codegen script: reads proto .d.ts files and generates field definitions.
*
*
* Run: bun run scripts/gen-fields.ts
*
*
* For event queries: reads EventRow.AsObject to get field names + types
* For data queries: reads per-table proto enums to get field names
* Maps proto camelCase property names → FieldRef instances
Expand All @@ -19,11 +19,20 @@ type FieldEntry = { name: string; type: string; protoName: string };

// ── Parse EventRow interface from query.ts ──

function parseQueryFields(): { sdkCall: FieldEntry[]; aiToken: FieldEntry[]; payment: FieldEntry[] } {
const src = fs.readFileSync(path.join(GEN_DIR, "query", "v1", "query.ts"), "utf-8");
function parseQueryFields(): {
sdkCall: FieldEntry[];
aiToken: FieldEntry[];
payment: FieldEntry[];
} {
const src = fs.readFileSync(
path.join(GEN_DIR, "query", "v1", "query.ts"),
"utf-8"
);

// Extract EventRow interface properties
const eventRowMatch = src.match(/export interface EventRow \{\s*([^}]+)\s*\}/s);
const eventRowMatch = src.match(
/export interface EventRow \{\s*([^}]+)\s*\}/s
);
if (!eventRowMatch) throw new Error("Could not find EventRow in query.ts");

const props = eventRowMatch[1];
Expand All @@ -33,16 +42,22 @@ function parseQueryFields(): { sdkCall: FieldEntry[]; aiToken: FieldEntry[]; pay
while ((m = propRe.exec(props)) !== null) {
const [, name, type] = m;
if (name === "basicUsageType" || name === "debitAmount") continue; // handled below
if (fields.some(f => f.name === name)) continue;
if (fields.some((f) => f.name === name)) continue;
const tsType = type === "number" ? "number" : "string";
fields.push({ name, type: tsType, protoName: name });
}

// Map fields to specific groups based on what we know:
// Common fields across all event types
const common: FieldEntry[] = [
...fields.filter(f =>
["eventId", "eventType", "userId", "reportedTimestamp", "ingestedTimestamp"].includes(f.name)
...fields.filter((f) =>
[
"eventId",
"eventType",
"userId",
"reportedTimestamp",
"ingestedTimestamp",
].includes(f.name)
),
// apiKeyId returned by backend but missing from proto EventRow — TODO: add to proto
{ name: "apiKeyId", type: "string", protoName: "api_key_id" },
Expand All @@ -52,18 +67,31 @@ function parseQueryFields(): { sdkCall: FieldEntry[]; aiToken: FieldEntry[]; pay
...common,
{ name: "basicUsageType", type: "string", protoName: "basic_usage_type" },
{ name: "debitAmount", type: "number", protoName: "debit_amount" },
...fields.filter(f => f.name === "metadata"),
...fields.filter((f) => f.name === "metadata"),
];

const aiToken: FieldEntry[] = [
...common,
{ name: "model", type: "string", protoName: "model" },
{ name: "inputTokens", type: "number", protoName: "input_tokens" },
{ name: "outputTokens", type: "number", protoName: "output_tokens" },
{ name: "inputDebitAmount", type: "number", protoName: "input_debit_amount" },
{ name: "outputDebitAmount", type: "number", protoName: "output_debit_amount" },
...fields.filter(f =>
["provider", "inputCacheTokens", "inputCacheDebitAmount", "metadata"].includes(f.name)
{
name: "inputDebitAmount",
type: "number",
protoName: "input_debit_amount",
},
{
name: "outputDebitAmount",
type: "number",
protoName: "output_debit_amount",
},
...fields.filter((f) =>
[
"provider",
"inputCacheTokens",
"inputCacheDebitAmount",
"metadata",
].includes(f.name)
),
];

Expand All @@ -83,10 +111,13 @@ function parseTableName(enumName: string): string {
}

function parseDataFields(): Record<string, FieldEntry[]> {
const src = fs.readFileSync(path.join(GEN_DIR, "data", "v1", "data.ts"), "utf-8");
const src = fs.readFileSync(
path.join(GEN_DIR, "data", "v1", "data.ts"),
"utf-8"
);

const tableFields: Record<string, FieldEntry[]> = {};

// Find all Field enums
const enumRe = /export enum (\w+Field) \{\s*([^}]+)\s*\}/gs;
let m: RegExpExecArray | null;
Expand All @@ -101,7 +132,7 @@ function parseDataFields(): Record<string, FieldEntry[]> {
while ((mm = memberRe.exec(body)) !== null) {
const protoMember = mm[1];
if (protoMember.endsWith("_UNSPECIFIED")) continue;

// Strip prefix: USERS_LAST_BILLED_TIMESTAMP → last_billed_timestamp
const prefix = tableName.toUpperCase() + "_";
const stripped = protoMember.startsWith(prefix)
Expand All @@ -122,7 +153,10 @@ function parseDataFields(): Record<string, FieldEntry[]> {

// ── Generate output ──

function generateFieldsFile(fields: Record<string, FieldEntry[]>, fileName: string): void {
function generateFieldsFile(
fields: Record<string, FieldEntry[]>,
fileName: string
): void {
const lines: string[] = [
"// AUTO-GENERATED by scripts/gen-fields.ts — do not edit",
'import { FieldRef } from "../fieldRef.js";',
Expand Down Expand Up @@ -153,7 +187,7 @@ generateFieldsFile(
aiToken: queryFields.aiToken,
payment: queryFields.payment,
},
"query/fields.ts",
"query/fields.ts"
);

const dataFields = parseDataFields();
Expand Down
3 changes: 0 additions & 3 deletions packages/analytics/src/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,3 @@ export class Analytics {
};
}
}



15 changes: 9 additions & 6 deletions packages/analytics/src/data/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export abstract class BaseDataBuilder<TFields> {

constructor(
public readonly fields: TFields,
public readonly tableName: string,
public readonly tableName: string
) {}

where(filter: FilterCondition | FilterGroup): this {
Expand Down Expand Up @@ -46,19 +46,22 @@ export abstract class BaseDataBuilder<TFields> {
};
}

protected unwrap(res: { columns?: string[]; rows?: Array<{ values?: string[] }>; total?: number }): DataQueryResult<TFields> {
protected unwrap(res: {
columns?: string[];
rows?: Array<{ values?: string[] }>;
total?: number;
}): DataQueryResult<TFields> {
const cols = res.columns ?? [];
const rows = (res.rows ?? []).map((r) => {
const vals = r.values ?? [];
const obj: Record<string, string> = {};
cols.forEach((c, i) => { obj[c] = vals[i] ?? ""; });
cols.forEach((c, i) => {
obj[c] = vals[i] ?? "";
});
return obj as unknown as InferRow<TFields>;
});
return { columns: cols, rows, total: res.total ?? 0 };
}

abstract execute(): Promise<DataQueryResult<TFields>>;
}



11 changes: 9 additions & 2 deletions packages/analytics/src/data/tables.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,20 @@ export class TagsBuilder extends BaseDataBuilder<typeof tagsFields> {
}
}

export class ExpressionsBuilder extends BaseDataBuilder<typeof expressionsFields> {
export class ExpressionsBuilder extends BaseDataBuilder<
typeof expressionsFields
> {
constructor(private grpc: GrpcClient, private apiKey: string) {
super(expressionsFields, "expressions");
}
async execute(): Promise<DataQueryResult<typeof expressionsFields>> {
const params = this.buildParams();
const res = await callDataQuery(this.grpc, this.apiKey, "expressions", params);
const res = await callDataQuery(
this.grpc,
this.apiKey,
"expressions",
params
);
return this.unwrap(res);
}
}
Expand Down
3 changes: 0 additions & 3 deletions packages/analytics/src/fieldRef.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,3 @@ export class FieldRef<T> {
export type InferRow<TFields> = {
[K in keyof TFields]: TFields[K] extends FieldRef<infer V> ? V : never;
};



Loading
Loading