Skip to content

Commit 8d2dd36

Browse files
skydjoltony-ducrocq-saagieJulien Bignon
authored
🚑 fix/project import (#228)
* Change logs api graphql by log-proxy * use log-proxy for job, app and conditions logs * 🚑 fix/project import * 🚑 Upgrade python version in unit test. * 🚑 Remove externalTechnologies --------- Co-authored-by: Tony Ducrocq <tony.ducrocq@saagie.com> Co-authored-by: Julien Bignon <julien.bignon@saagie.com>
1 parent 159d3cc commit 8d2dd36

20 files changed

Lines changed: 102 additions & 688 deletions

‎.github/workflows/build-and-test-pr.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
runs-on: ubuntu-latest
4545
strategy:
4646
matrix:
47-
python-version: [ "3.8", "3.9","3.10" ]
47+
python-version: [ "3.9","3.10", "3.11", "3.12", "3.13" ]
4848
steps:
4949
- uses: actions/checkout@v3
5050
- name: Set up Python ${{ matrix.python-version }}

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,6 @@ _build
1010
venv
1111
.vscode
1212
.coverage
13-
cov.xml
13+
api-saagie.iml
14+
out/
15+
.env

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ pip install saagieapi==<version>
4444
| >= 2023.05 | >= 2.10.0 |
4545
| >= 2024.01 | >= 2.11.0 |
4646
| >= 2024.02 | >= 2.12.0 |
47+
| >= 2024.03 | >= 2.13.0 |
48+
| >= 2024.05 | >= 2.14.0 |
4749

4850
## Contributing
4951

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "saagieapi"
3-
version = "2.13.0"
3+
version = "2.14.0"
44
description = "Python API to interact with Saagie"
55
authors = ["Saagie"]
66
license = "GLWTPL"

‎saagieapi/apps/apps.py‎

Lines changed: 31 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1467,17 +1467,19 @@ def count_history_statuses(self, history_id, version_number, start_time):
14671467

14681468
def get_logs(
14691469
self,
1470+
project_id: str,
14701471
app_id: str,
14711472
app_execution_id: str,
14721473
limit: int = None,
14731474
skip: int = None,
14741475
log_stream: str = None,
1475-
start_at: str = None,
14761476
):
14771477
"""Get logs of the app
14781478
14791479
Parameters
14801480
----------
1481+
project_id : str
1482+
UUID of your project
14811483
app_id : str
14821484
UUID of your app
14831485
app_execution_id : str
@@ -1490,9 +1492,6 @@ def get_logs(
14901492
Stream of logs to follow. Values accepted :
14911493
[ENVVARS_STDOUT, ENVVARS_STDERR, ORCHESTRATION_STDOUT, ORCHESTRATION_STDERR, STDERR, STDOUT]
14921494
By default, all the streams are retrieved
1493-
start_at: str, optional
1494-
Get logs since a specific datetime.
1495-
Following formats accepted : "2024-04-09 10:00:00" and "2024-04-09T10:00:00"
14961495
14971496
Returns
14981497
-------
@@ -1508,43 +1507,35 @@ def get_logs(
15081507
... skip=5,
15091508
... start_at="2024-04-09 10:00:00"
15101509
... )
1510+
15111511
{
1512-
"appLogs": {
1513-
"count": 25,
1514-
"content": [
1515-
{
1516-
"index": 5,
1517-
"value": "[I 2024-04-09 13:38:36.982 ServerApp] jupyterlab_git | extension was successfully linked.",
1518-
"containerId": "d7104fa7371c5ed6ef540fa8b0620a654a0e02c57136e29f0fcc03d16e36d74f",
1519-
"stream": "STDERR",
1520-
"recordAt": "2024-04-09T13:38:36.982473892Z"
1521-
},
1522-
{
1523-
"index": 6,
1524-
"value": "[W 2024-04-09 13:38:36.987 NotebookApp] 'ip' has moved from NotebookApp to ServerApp. This config will be passed to ServerApp. Be sure to update your config before our next release.",
1525-
"containerId": "d7104fa7371c5ed6ef540fa8b0620a654a0e02c57136e29f0fcc03d16e36d74f",
1526-
"stream": "STDERR",
1527-
"recordAt": "2024-04-09T13:38:36.987400105Z"
1528-
}
1529-
]
1512+
"logs": [
1513+
{
1514+
"index": 0,
1515+
"stream": "STDERR",
1516+
"time": "2024-12-11T14:27:42.858298425Z",
1517+
"value": "[I 2024-04-09 13:38:36.982 ServerApp] jupyterlab_git | extension was successfully linked.",
1518+
},
1519+
{
1520+
"index": 1,
1521+
"stream": "STDERR",
1522+
"time": "2024-12-11T14:27:42.859697094Z",
1523+
"value": "AH00558: httpd: Could not reliably determine the server's fully qualified domain name, using 10.4.3.20. Set the 'ServerName' directive globally to suppress this message"
15301524
}
1525+
],
1526+
"limit": 10000,
1527+
"total": 5,
1528+
"order": "asc",
1529+
"source": "elastic"
15311530
}
15321531
"""
1533-
params = {
1534-
"appId": app_id,
1535-
"appExecutionId": app_execution_id,
1536-
}
1537-
1538-
if limit:
1539-
params["limit"] = limit
1540-
1541-
if skip:
1542-
params["skip"] = skip
1543-
1544-
if log_stream:
1545-
params["stream"] = log_stream
1546-
1547-
if start_at:
1548-
params["recordAt"] = start_at
1549-
1550-
return self.saagie_api.client.execute(query=gql(GQL_GET_APP_LOG), variable_values=params)
1532+
if limit is None:
1533+
limit = 10000
1534+
if skip is None:
1535+
skip = 0
1536+
if log_stream is None:
1537+
log_stream = "ENVVARS_STDOUT,ENVVARS_STDERR,ORCHESTRATION_STDOUT,ORCHESTRATION_STDERR,STDERR,STDOUT"
1538+
url = f"{self.saagie_api.url_saagie}log-proxy/api/logs/{self.saagie_api.realm}/platform/{self.saagie_api.platform}/project/{project_id}/app_execution/{app_execution_id}?limit={limit}&skip={skip}&streams={log_stream}"
1539+
response = self.saagie_api.request_client.send(method="GET", url=url, raise_for_status=True)
1540+
1541+
return response.json()

‎saagieapi/apps/gql_queries.py‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -452,19 +452,3 @@
452452
countAppHistoryStatuses(appHistoryId: $appHistoryId, versionNumber: $versionNumber, startTime: $startTime)
453453
}
454454
"""
455-
456-
GQL_GET_APP_LOG = """
457-
query appLogs($appId: UUID!, $appExecutionId: UUID!, $limit: Int, $skip: Int, $stream: LogStream, $recordAt: String) {
458-
appLogs(appId: $appId, appExecutionId: $appExecutionId, limit: $limit, skip: $skip, stream: $stream, recordAt: $recordAt)
459-
{
460-
count
461-
content {
462-
index
463-
value
464-
containerId
465-
stream
466-
recordAt
467-
}
468-
}
469-
}
470-
"""

‎saagieapi/gql_queries.py‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -132,59 +132,3 @@
132132
)
133133
}
134134
"""
135-
136-
GQL_COUNT_CONDITION_LOGS = """
137-
query conditionPipelineCountFilteredLogs($conditionInstanceId: UUID!,
138-
$projectID: UUID!,
139-
$streams: [LogStream]!) {
140-
conditionPipelineCountFilteredLogs (
141-
conditionInstanceID: $conditionInstanceId,
142-
projectID: $projectID,
143-
streams: $streams
144-
)
145-
}
146-
"""
147-
148-
GQL_GET_CONDITION_LOGS_BY_CONDITION = """
149-
query conditionPipelineByNodeIdFilteredLogs($pipelineInstanceID: UUID!,
150-
$conditionNodeID: UUID!,
151-
$projectID: UUID!,
152-
$streams: [LogStream]!) {
153-
conditionPipelineByNodeIdFilteredLogs(
154-
pipelineInstanceID: $pipelineInstanceID,
155-
conditionNodeID: $conditionNodeID,
156-
projectID: $projectID,
157-
streams: $streams
158-
) {
159-
count
160-
content {
161-
index
162-
value
163-
stream
164-
}
165-
}
166-
}
167-
"""
168-
169-
GQL_GET_CONDITION_LOGS_BY_INSTANCE = """
170-
query conditionPipelineFilteredLogs($conditionInstanceId: UUID!,
171-
$projectId: UUID!,
172-
$limit: Int,
173-
$skip: Int,
174-
$streams: [LogStream]!) {
175-
conditionPipelineFilteredLogs(
176-
conditionInstanceID: $conditionInstanceId,
177-
projectID:$projectId,
178-
limit: $limit,
179-
skip: $skip,
180-
streams: $streams
181-
) {
182-
count
183-
content {
184-
index
185-
value
186-
stream
187-
}
188-
}
189-
}
190-
"""

‎saagieapi/jobs/jobs.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1457,10 +1457,8 @@ def __launch_request(self, file: str, payload_str: str, params: Dict) -> Dict:
14571457
Dict of the request response
14581458
"""
14591459
if file:
1460-
file_info = Path(file)
1461-
os.chdir(file_info.parent)
1462-
file = Path(file_info.name)
1463-
with file.open(mode="rb") as file_content:
1460+
file_info = Path(file).absolute()
1461+
with file_info.open(mode="rb") as file_content:
14641462
params["file"] = file_content
14651463
try:
14661464
req = self.saagie_api.client.execute(
@@ -1612,8 +1610,7 @@ def import_from_json(
16121610

16131611
if version_path.exists():
16141612
if path_to_package := next(version_path.iterdir(), None):
1615-
os.chdir(path_to_package.parent)
1616-
file_name = path_to_package.name
1613+
file_name = path_to_package.absolute()
16171614
else:
16181615
file_name = ""
16191616

‎saagieapi/repositories/gql_queries.py‎

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -206,15 +206,6 @@
206206
}
207207
lastReversibleId
208208
}
209-
connectionTypes {
210-
id
211-
label
212-
actions {
213-
checkConnection {
214-
scriptId
215-
}
216-
}
217-
}
218209
technologies {
219210
id
220211
technologyId
@@ -258,35 +249,6 @@
258249
lastUpdate
259250
}
260251
}
261-
... on ExtJobTechnology {
262-
iconUrl
263-
contexts {
264-
id
265-
label
266-
available
267-
missingFacets
268-
description
269-
recommended
270-
trustLevel
271-
deprecationDate
272-
lastUpdate
273-
connectionTypeUUID
274-
actions {
275-
getStatus {
276-
scriptId
277-
}
278-
start {
279-
scriptId
280-
}
281-
stop {
282-
scriptId
283-
}
284-
getLogs {
285-
scriptId
286-
}
287-
}
288-
}
289-
}
290252
... on SparkTechnology {
291253
contexts {
292254
id

‎saagieapi/repositories/repositories.py‎

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -168,17 +168,6 @@ def get_info(
168168
],
169169
'lastReversibleId': 'a17c73ed-fca1-4f25-a343-914c7ac23bae'
170170
},
171-
'connectionTypes': [
172-
{
173-
'id': '5b4b8ffb-9228-4f7a-9d39-67fd3c2862d3',
174-
'label': 'AWS Connection',
175-
'actions': {
176-
'checkConnection': {
177-
'scriptId': '9359e392-58a0-42db-9ce9-b68679aa9131'
178-
}
179-
}
180-
}
181-
],
182171
'technologies': [
183172
{
184173
'id': '1bf79f1d-7e2d-4daf-976d-8702114ab507',
@@ -204,43 +193,6 @@ def get_info(
204193
}
205194
]
206195
},
207-
{
208-
'id': 'db34c9b9-47c7-4dc6-8c3c-2d8ccf5afa11',
209-
'technologyId': 'aws-lambda',
210-
'label': 'AWS Lambda',
211-
'icon': 'aws-lambda',
212-
'repositoryId': '9fcbddfe-a7b7-4d25-807c-ad030782c923',
213-
'available': True,
214-
'missingFacets': [],
215-
'description': 'Run code without thinking about servers. Pay only for the compute time you consume',
216-
'iconUrl': None,
217-
'contexts': [
218-
{
219-
'id': 'functions',
220-
'label': 'Functions',
221-
'available': True,
222-
'missingFacets': [],
223-
'description': 'AWS Lambda Functions',
224-
'recommended': False,
225-
'trustLevel': 'Experimental',
226-
'deprecationDate': None,
227-
'lastUpdate': '2022-08-31T13:05:32.031Z',
228-
'connectionTypeUUID': '5b4b8ffb-9228-4f7a-9d39-67fd3c2862d3',
229-
'actions': {
230-
'getStatus': {
231-
'scriptId': '50794533-091b-4d66-9463-96f0ce255785'
232-
},
233-
'start': {
234-
'scriptId': '50794533-091b-4d66-9463-96f0ce255785'
235-
},
236-
'stop': None,
237-
'getLogs': {
238-
'scriptId': '50794533-091b-4d66-9463-96f0ce255785'
239-
}
240-
}
241-
}
242-
]
243-
}
244196
]
245197
}
246198
}

0 commit comments

Comments
 (0)