Skip to content

Commit 6939f66

Browse files
mayinghanmayinghan
committed
update
1 parent acbd83c commit 6939f66

1 file changed

Lines changed: 122 additions & 123 deletions

File tree

‎eval_protocol/pytest/handle_persist_flow.py‎

Lines changed: 122 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -82,133 +82,132 @@ def handle_persist_flow(all_results: list[list[EvaluationRow]], test_func_name:
8282
f.write("\n")
8383

8484
should_upload = os.getenv("EP_NO_UPLOAD") != "1"
85-
8685
if not should_upload:
8786
continue
8887

89-
def get_auth_value(key: str) -> str | None:
90-
"""Get auth value from config file or environment."""
91-
try:
92-
config_path = Path.home() / ".fireworks" / "auth.ini"
93-
if config_path.exists():
94-
config = configparser.ConfigParser() # noqa: F821
95-
config.read(config_path)
96-
for section in ["DEFAULT", "auth"]:
97-
if config.has_section(section) and config.has_option(section, key):
98-
return config.get(section, key)
99-
except Exception:
100-
pass
101-
return os.getenv(key)
102-
103-
fireworks_api_key = get_auth_value("FIREWORKS_API_KEY")
104-
fireworks_account_id = get_auth_value("FIREWORKS_ACCOUNT_ID")
105-
106-
if not fireworks_api_key and not fireworks_account_id:
107-
store_experiment_link(
108-
experiment_id,
109-
"No Fireworks API key AND account ID found",
110-
"failure",
111-
)
112-
continue
113-
elif not fireworks_api_key:
114-
store_experiment_link(
115-
experiment_id,
116-
"No Fireworks API key found",
117-
"failure",
118-
)
119-
continue
120-
elif not fireworks_account_id:
121-
store_experiment_link(
122-
experiment_id,
123-
"No Fireworks account ID found",
124-
"failure",
125-
)
126-
continue
127-
128-
headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
129-
130-
# Make dataset first
131-
dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
132-
133-
dataset_payload = { # pyright: ignore[reportUnknownVariableType]
134-
"dataset": {
135-
"displayName": dataset_name,
136-
"evalProtocol": {},
137-
"format": "FORMAT_UNSPECIFIED",
138-
"exampleCount": f"{len(exp_rows)}",
139-
},
140-
"datasetId": dataset_name,
141-
}
142-
143-
dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers) # pyright: ignore[reportUnknownArgumentType]
144-
145-
# Skip if dataset creation failed
146-
if dataset_response.status_code not in [200, 201]:
147-
store_experiment_link(
148-
experiment_id,
149-
f"Dataset creation failed: {dataset_response.status_code} {dataset_response.text}",
150-
"failure",
151-
)
152-
continue
153-
154-
dataset_data: dict[str, Any] = dataset_response.json() # pyright: ignore[reportAny, reportExplicitAny]
155-
dataset_id = dataset_data.get("datasetId", dataset_name) # pyright: ignore[reportAny]
156-
157-
# Upload the JSONL file content
158-
upload_url = (
159-
f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
88+
def get_auth_value(key: str) -> str | None:
89+
"""Get auth value from config file or environment."""
90+
try:
91+
config_path = Path.home() / ".fireworks" / "auth.ini"
92+
if config_path.exists():
93+
config = configparser.ConfigParser() # noqa: F821
94+
config.read(config_path)
95+
for section in ["DEFAULT", "auth"]:
96+
if config.has_section(section) and config.has_option(section, key):
97+
return config.get(section, key)
98+
except Exception:
99+
pass
100+
return os.getenv(key)
101+
102+
fireworks_api_key = get_auth_value("FIREWORKS_API_KEY")
103+
fireworks_account_id = get_auth_value("FIREWORKS_ACCOUNT_ID")
104+
105+
if not fireworks_api_key and not fireworks_account_id:
106+
store_experiment_link(
107+
experiment_id,
108+
"No Fireworks API key AND account ID found",
109+
"failure",
110+
)
111+
continue
112+
elif not fireworks_api_key:
113+
store_experiment_link(
114+
experiment_id,
115+
"No Fireworks API key found",
116+
"failure",
117+
)
118+
continue
119+
elif not fireworks_account_id:
120+
store_experiment_link(
121+
experiment_id,
122+
"No Fireworks account ID found",
123+
"failure",
124+
)
125+
continue
126+
127+
headers = {"Authorization": f"Bearer {fireworks_api_key}", "Content-Type": "application/json"}
128+
129+
# Make dataset first
130+
dataset_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets"
131+
132+
dataset_payload = { # pyright: ignore[reportUnknownVariableType]
133+
"dataset": {
134+
"displayName": dataset_name,
135+
"evalProtocol": {},
136+
"format": "FORMAT_UNSPECIFIED",
137+
"exampleCount": f"{len(exp_rows)}",
138+
},
139+
"datasetId": dataset_name,
140+
}
141+
142+
dataset_response = requests.post(dataset_url, json=dataset_payload, headers=headers) # pyright: ignore[reportUnknownArgumentType]
143+
144+
# Skip if dataset creation failed
145+
if dataset_response.status_code not in [200, 201]:
146+
store_experiment_link(
147+
experiment_id,
148+
f"Dataset creation failed: {dataset_response.status_code} {dataset_response.text}",
149+
"failure",
150+
)
151+
continue
152+
153+
dataset_data: dict[str, Any] = dataset_response.json() # pyright: ignore[reportAny, reportExplicitAny]
154+
dataset_id = dataset_data.get("datasetId", dataset_name) # pyright: ignore[reportAny]
155+
156+
# Upload the JSONL file content
157+
upload_url = (
158+
f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/datasets/{dataset_id}:upload"
159+
)
160+
upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
161+
162+
with open(exp_file, "rb") as f:
163+
files = {"file": f}
164+
upload_response = requests.post(upload_url, files=files, headers=upload_headers)
165+
166+
# Skip if upload failed
167+
if upload_response.status_code not in [200, 201]:
168+
store_experiment_link(
169+
experiment_id,
170+
f"File upload failed: {upload_response.status_code} {upload_response.text}",
171+
"failure",
172+
)
173+
continue
174+
175+
# Create evaluation job (optional - don't skip experiment if this fails)
176+
eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
177+
# Truncate job ID to fit 63 character limit
178+
job_id_base = f"{dataset_name}-job"
179+
if len(job_id_base) > 63:
180+
# Keep the "-job" suffix and truncate the dataset_name part
181+
max_dataset_name_len = 63 - 4 # 4 = len("-job")
182+
truncated_dataset_name = dataset_name[:max_dataset_name_len]
183+
job_id_base = f"{truncated_dataset_name}-job"
184+
185+
eval_job_payload = {
186+
"evaluationJobId": job_id_base,
187+
"evaluationJob": {
188+
"evaluator": f"accounts/{fireworks_account_id}/evaluators/dummy",
189+
"inputDataset": f"accounts/{fireworks_account_id}/datasets/dummy",
190+
"outputDataset": f"accounts/{fireworks_account_id}/datasets/{dataset_id}",
191+
},
192+
}
193+
194+
eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
195+
196+
if eval_response.status_code in [200, 201]:
197+
eval_job_data = eval_response.json() # pyright: ignore[reportAny]
198+
job_id = eval_job_data.get("evaluationJobId", job_id_base) # pyright: ignore[reportAny]
199+
200+
store_experiment_link(
201+
experiment_id,
202+
f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}",
203+
"success",
204+
)
205+
else:
206+
store_experiment_link(
207+
experiment_id,
208+
f"Job creation failed: {eval_response.status_code} {eval_response.text}",
209+
"failure",
160210
)
161-
upload_headers = {"Authorization": f"Bearer {fireworks_api_key}"}
162-
163-
with open(exp_file, "rb") as f:
164-
files = {"file": f}
165-
upload_response = requests.post(upload_url, files=files, headers=upload_headers)
166-
167-
# Skip if upload failed
168-
if upload_response.status_code not in [200, 201]:
169-
store_experiment_link(
170-
experiment_id,
171-
f"File upload failed: {upload_response.status_code} {upload_response.text}",
172-
"failure",
173-
)
174-
continue
175-
176-
# Create evaluation job (optional - don't skip experiment if this fails)
177-
eval_job_url = f"https://api.fireworks.ai/v1/accounts/{fireworks_account_id}/evaluationJobs"
178-
# Truncate job ID to fit 63 character limit
179-
job_id_base = f"{dataset_name}-job"
180-
if len(job_id_base) > 63:
181-
# Keep the "-job" suffix and truncate the dataset_name part
182-
max_dataset_name_len = 63 - 4 # 4 = len("-job")
183-
truncated_dataset_name = dataset_name[:max_dataset_name_len]
184-
job_id_base = f"{truncated_dataset_name}-job"
185-
186-
eval_job_payload = {
187-
"evaluationJobId": job_id_base,
188-
"evaluationJob": {
189-
"evaluator": f"accounts/{fireworks_account_id}/evaluators/dummy",
190-
"inputDataset": f"accounts/{fireworks_account_id}/datasets/dummy",
191-
"outputDataset": f"accounts/{fireworks_account_id}/datasets/{dataset_id}",
192-
},
193-
}
194-
195-
eval_response = requests.post(eval_job_url, json=eval_job_payload, headers=headers)
196-
197-
if eval_response.status_code in [200, 201]:
198-
eval_job_data = eval_response.json() # pyright: ignore[reportAny]
199-
job_id = eval_job_data.get("evaluationJobId", job_id_base) # pyright: ignore[reportAny]
200-
201-
store_experiment_link(
202-
experiment_id,
203-
f"https://app.fireworks.ai/dashboard/evaluation-jobs/{job_id}",
204-
"success",
205-
)
206-
else:
207-
store_experiment_link(
208-
experiment_id,
209-
f"Job creation failed: {eval_response.status_code} {eval_response.text}",
210-
"failure",
211-
)
212211

213212
except Exception as e:
214213
# Do not fail evaluation if experiment JSONL writing fails

0 commit comments

Comments
 (0)