From a81549a6286ef0a9014421aad327475784ad8a93 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Fri, 14 Feb 2025 08:05:46 -0600 Subject: [PATCH 01/16] Start of 1.8.3, updates to /metadata/tml/export for all the new flags that have been included, especially around obj_id --- setup.cfg | 2 +- src/thoughtspot_rest_api_v1/_version.py | 2 +- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 25 +++++++++++++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9ed18e4..0e360a0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = thoughtspot_rest_api_v1 -version = 1.8.2 +version = 1.8.3 description = Library implementing the ThoughtSpot V1 REST API long_description = file: README.md long_description_content_type = text/markdown diff --git a/src/thoughtspot_rest_api_v1/_version.py b/src/thoughtspot_rest_api_v1/_version.py index aa1a8c4..cfe6447 100644 --- a/src/thoughtspot_rest_api_v1/_version.py +++ b/src/thoughtspot_rest_api_v1/_version.py @@ -1 +1 @@ -__version__ = '1.8.2' +__version__ = '1.8.3' diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index accd3f9..37b6728 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -483,9 +483,19 @@ def metadata_tml_async_status(self, request: Dict): # Out of convenience, providing a simple List[str] input for getting these by GUID. metadata_request will override # if you need the deeper functionality with names / types - def metadata_tml_export(self, metadata_ids: List[str], export_associated: bool = False, export_fqn: bool = False, - edoc_format: Optional[str] = None, export_schema_version: Optional[str] = None, - metadata_request: Optional[List[Dict]] = None): + def metadata_tml_export(self, + metadata_ids: List[str], + export_associated: bool = False, + export_fqn: bool = False, + edoc_format: Optional[str] = None, + export_schema_version: Optional[str] = None, + metadata_request: Optional[List[Dict]] = None, + export_dependent: Optional[bool] = None, + export_connection_as_dependent: Optional[bool] = None, + all_orgs_override: Optional[bool] = None, + export_options: Optional[Dict] = None + ): + endpoint = 'metadata/tml/export' request = { @@ -505,6 +515,15 @@ def metadata_tml_export(self, metadata_ids: List[str], export_associated: bool = for i in metadata_ids: metadata_list.append({'identifier': i}) request['metadata'] = metadata_list + if export_dependent is not None: + request['export_dependent'] = export_dependent + if export_connection_as_dependent is not None: + request['export_connection_as_dependent'] = export_connection_as_dependent + if all_orgs_override is not None: + request['all_orgs_override'] = all_orgs_override + if export_options is not None: + request['export_options'] = export_options + return self.post_request(endpoint=endpoint, request=request) def metadata_tml_export_batch(self, request: Dict): From 14e55a76aa6ebf64f975f4c4845f3e6a30e5bbce Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Fri, 14 Feb 2025 13:54:03 -0600 Subject: [PATCH 02/16] All updates and additions to metadata endpoints available in 10.6 --- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 77 ++++++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index 37b6728..0f18400 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -3,6 +3,7 @@ import json import requests +from charset_normalizer.utils import identify_sig_or_bom from requests_toolbelt.adapters.socket_options import TCPKeepAliveAdapter class ReportTypes: @@ -457,24 +458,56 @@ def metadata_answer_sql(self, answer_identifier: str): } return self.post_request(endpoint=endpoint, request=request) - def metadata_tml_import(self, metadata_tmls: List[str], import_policy: str = 'PARTIAL', create_new: bool = False): + def metadata_tml_import(self, + metadata_tmls: List[str], + import_policy: str = 'PARTIAL', + create_new: bool = False, + all_orgs_context: Optional[bool] = None, + skip_cdw_validation_for_tables: Optional[bool] = None, + skip_diff_check: Optional[bool] = None, + enable_large_metadata_validation: Optional[bool] = None + ): endpoint = 'metadata/tml/import' request = { 'metadata_tmls': metadata_tmls, 'import_policy': import_policy, 'create_new': create_new } - return self.post_request(endpoint=endpoint, request=request) - - def metadata_tml_async_import(self, metadata_tmls: List[str], import_policy: str = 'PARTIAL', - create_new: bool = False, all_orgs_context: bool = False, - skip_cdw_validation_for_tables: bool = False): + if all_orgs_context is not None: + request['all_orgs_context'] = all_orgs_context + if skip_cdw_validation_for_tables is not None: + request['cdw_validation_for_tables'] = skip_cdw_validation_for_tables + if skip_diff_check is not None: + request['skip_diff_check'] = skip_diff_check + if enable_large_metadata_validation is not None: + request['enable_large_metadata_validation'] = enable_large_metadata_validation + + + return self.post_request(endpoint=endpoint, request=request) + + def metadata_tml_async_import(self, metadata_tmls: List[str], + import_policy: str = 'PARTIAL', + create_new: bool = False, + all_orgs_context: Optional[bool] = None, + skip_cdw_validation_for_tables: Optional[bool] = None, + skip_diff_check: Optional[bool] = None, + enable_large_metadata_validation: Optional[bool] = None + ): endpoint = 'metadata/tml/async/import' request = { 'metadata_tmls': metadata_tmls, 'import_policy': import_policy, 'create_new': create_new } + if all_orgs_context is not None: + request['all_orgs_context'] = all_orgs_context + if skip_cdw_validation_for_tables is not None: + request['cdw_validation_for_tables'] = skip_cdw_validation_for_tables + if skip_diff_check is not None: + request['skip_diff_check'] = skip_diff_check + if enable_large_metadata_validation is not None: + request['enable_large_metadata_validation'] = enable_large_metadata_validation + return self.post_request(endpoint=endpoint, request=request) def metadata_tml_async_status(self, request: Dict): @@ -547,6 +580,38 @@ def metadata_delete(self, metadata_ids: List[str], delete_disabled_objects: bool request['metadata'] = metadata_list return self.post_request(endpoint=endpoint, request=request) + def metadata_copyobject(self, identifier: str, object_type: Optional[str] = None, + title: Optional[str] = None, description: Optional[str] = None): + endpoint = 'metadata/copyobject' + request = { + identifier : identifier + } + if object_type is not None: + request['type'] = object_type + if title is not None: + request['title'] = title + if description is not None: + request['description'] = description + + return self.post_request(endpoint=endpoint, request=request) + + def metadata_worksheets_convert(self, worksheet_ids: Optional[List[str]] = None, + exclude_worksheet_ids: Optional[List[str]] = None, + convert_all: bool = False, + apply_changes: bool = False): + endpoint = 'metadata/worksheets/convert' + request = { + 'convert_all': convert_all, + 'apply_changes': apply_changes + } + if worksheet_ids is not None: + request['worksheet_ids'] = worksheet_ids + if exclude_worksheet_ids is not None: + request['exclude_worksheet_ids'] = exclude_worksheet_ids + + return self.post_request(endpoint=endpoint, request=request) + + # # /reports/ endpoints # From 94d8a1504b18e1519baeff5194ec7b6f8178f8bd Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 17 Feb 2025 08:01:10 -0600 Subject: [PATCH 03/16] Authentication endpoints updated to reflect latest --- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index 0f18400..2e73414 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -210,6 +210,57 @@ def auth_token_object(self, username: str, object_id: str, password: Optional[st response.raise_for_status() return response.json() + # V2 API Bearer token can be used with V1 /session/login/token for Trusted Auth flow + # or used with each API call (no session object) or used with V2 /auth/session/login to create session + # additional_request_parameters allows setting new arbitrary keys and data structures to + # be appended along with the existing defined method arguments + def auth_token_custom(self, username: str, password: Optional[str] = None, org_id: Optional[int] = None, + secret_key: Optional[str] = None, validity_time_in_sec: int = 300, + persist_option: str = 'NONE', reset_option: Optional[List[str]] = None, + auto_create: bool = False, display_name: Optional[str] = None, + email: Optional[str] = None, groups: Optional[List[str]] = None, + additional_request_parameters: Optional[Dict] = None) -> Dict: + endpoint = 'auth/token/custom' + + url = self.base_url + endpoint + + json_post_data = { + 'username': username, + 'validity_time_in_sec': validity_time_in_sec + } + + if secret_key is not None: + json_post_data['secret_key'] = secret_key + + elif username is not None and password is not None: + json_post_data['password'] = password + else: + raise Exception("If using username/password, must include both") + + if org_id is not None: + json_post_data['org_id'] = org_id + + # User provisioning options + if auto_create is True: + if display_name is not None and email is not None: + json_post_data['auto_create'] = True + json_post_data['display_name'] = display_name + json_post_data['email'] = email + if groups is not None: + group_objects = {} + json_post_data['group_identifiers'] = group_identifiers + else: + raise Exception("If using auto_create=True, must include display_name and email") + + if additional_request_parameters is not None: + for param in additional_request_parameters: + json_post_data[param] = additional_request_parameters[param] + + response = self.requests_session.post(url=url, json=json_post_data) + + response.raise_for_status() + return response.json() + def auth_token_revoke(self) -> bool: endpoint = 'auth/token/revoke' @@ -220,6 +271,10 @@ def auth_token_revoke(self) -> bool: response.raise_for_status() return True + def auth_token_validate(self, token: str): + endpoint = 'auth/token/validate' + self.post_request(endpoint=endpoint, request={"token": token}) + # # Generic wrappers for the basic HTTP methods # Theoretically, you can just get bearer token and issue any command with endpoint and request @@ -272,6 +327,10 @@ def auth_session_user(self): endpoint = 'auth/session/user' return self.get_request(endpoint=endpoint) + def auth_session_token(self): + endpoint = 'auth/session/token' + return self.get_request(endpoint=endpoint) + # # /users/ endpoints # @@ -864,3 +923,8 @@ def ai_answer_create(self, metadata_identifier: str, query: str): return self.post_request(endpoint=endpoint, request=request) + def ai_analytical_questions(self, request: Dict): + endpoint = 'ai/analytical-questions' + return self.post_request(endpoint=endpoint, request=request) + + From b4151a217f93b0f47d8f6de58c3e6df4413c2774 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 17 Feb 2025 08:12:08 -0600 Subject: [PATCH 04/16] Added auth_token_direct_request to allow passing Dict to any of the token endpoints --- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index 2e73414..f30166a 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -247,8 +247,10 @@ def auth_token_custom(self, username: str, password: Optional[str] = None, org_i json_post_data['display_name'] = display_name json_post_data['email'] = email if groups is not None: - group_objects = {} - json_post_data['group_identifiers'] = group_identifiers + group_objects = [] + for group in groups: + group_objects.append({'identifier': group}) + json_post_data['groups'] = group_objects else: raise Exception("If using auto_create=True, must include display_name and email") @@ -261,6 +263,13 @@ def auth_token_custom(self, username: str, password: Optional[str] = None, org_i response.raise_for_status() return response.json() + # If you want to use a request object rather than the hardcoded Python arguments + # of the other methods above + def auth_token_direct_request(self, token_type: str, request: Dict): + endpoint = 'auth/token/' + token_type.lower() + + self.post_request(endpoint=endpoint, request=request) + def auth_token_revoke(self) -> bool: endpoint = 'auth/token/revoke' From 030396a4cdaf67f114ff91f9f8fb755161ab2520 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 17 Feb 2025 09:10:02 -0600 Subject: [PATCH 05/16] Updated with all the new connection endpoints --- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index f30166a..3764f37 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -810,6 +810,22 @@ def connection_update(self, request: Dict): endpoint = 'connection/update' return self.post_request(endpoint=endpoint, request=request) + def connection_delete_v2(self, connection_identifier: str): + endpoint = 'connection/{}/delete'.format(connection_identifier) + return self.post_request(endpoint=endpoint) + + def connection_update_v2(self, connection_identifier: str, request: Dict): + endpoint = 'connection/{}/update'.format(connection_identifier) + return self.post_request(endpoint=endpoint, request=request) + + def connection_download_connection_metadata_changes(self, connection_identifier: str): + endpoint = 'connections/download-connection-metadata-changes/{}'.format(connection_identifier) + return self.post_request(endpoint=endpoint) + + def connection_fetch_connection_diff_status(self, connection_identifier: str): + endpoint = 'connections/fetch-connection-diff-status/{}'.format(connection_identifier) + return self.post_request(endpoint=endpoint) + # # /roles/ endpoints # From 6b87bb14e9cf9123b55e1d626539c4a83b749811 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 17 Feb 2025 10:36:45 -0600 Subject: [PATCH 06/16] Added users_activate() and users_deactivate() --- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index 3764f37..8681991 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -387,6 +387,28 @@ def users_force_logout(self, user_identifiers: List[str]): } return self.post_request(endpoint=endpoint, request=request) + def users_activate(self, user_identifier: str, auth_token: str, password: str, properties: Optional[str] = None): + endpoint = 'users/activate' + request = { + 'user_identifier': user_identifier, + 'auth_token': auth_token, + 'password': password + } + if properties is not None: + request['properties'] = properties + + return self.post_request(endpoint=endpoint, request=request) + + def users_deactivate(self, user_identifier: str, base_url: str): + endpoint = 'users/deactivate' + request = { + 'user_identifier': user_identifier, + 'base_url': base_url + } + + return self.post_request(endpoint=endpoint, request=request) + + # # /system/ endpoints # @@ -679,6 +701,10 @@ def metadata_worksheets_convert(self, worksheet_ids: Optional[List[str]] = None, return self.post_request(endpoint=endpoint, request=request) + # EA in 10.6 for obj_id + def metadata_headers_update(self, request: Dict): + endpoint = 'metadata/headers/update' + return self.post_request(endpoint=endpoint, request=request) # # /reports/ endpoints From a9e76bf3f427abd88cce5047b56f9ece171bf7a2 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Tue, 18 Feb 2025 05:19:11 -0600 Subject: [PATCH 07/16] 1.8.4 is slight update to fix one of the new endpoints --- setup.cfg | 2 +- src/thoughtspot_rest_api_v1/_version.py | 2 +- src/thoughtspot_rest_api_v1/tsrestapiv2.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0e360a0..a3110c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = thoughtspot_rest_api_v1 -version = 1.8.3 +version = 1.8.4 description = Library implementing the ThoughtSpot V1 REST API long_description = file: README.md long_description_content_type = text/markdown diff --git a/src/thoughtspot_rest_api_v1/_version.py b/src/thoughtspot_rest_api_v1/_version.py index cfe6447..fa2822c 100644 --- a/src/thoughtspot_rest_api_v1/_version.py +++ b/src/thoughtspot_rest_api_v1/_version.py @@ -1 +1 @@ -__version__ = '1.8.3' +__version__ = '1.8.4' diff --git a/src/thoughtspot_rest_api_v1/tsrestapiv2.py b/src/thoughtspot_rest_api_v1/tsrestapiv2.py index 8681991..77a7da0 100644 --- a/src/thoughtspot_rest_api_v1/tsrestapiv2.py +++ b/src/thoughtspot_rest_api_v1/tsrestapiv2.py @@ -837,11 +837,11 @@ def connection_update(self, request: Dict): return self.post_request(endpoint=endpoint, request=request) def connection_delete_v2(self, connection_identifier: str): - endpoint = 'connection/{}/delete'.format(connection_identifier) + endpoint = 'connections/{}/delete'.format(connection_identifier) return self.post_request(endpoint=endpoint) def connection_update_v2(self, connection_identifier: str, request: Dict): - endpoint = 'connection/{}/update'.format(connection_identifier) + endpoint = 'connections/{}/update'.format(connection_identifier) return self.post_request(endpoint=endpoint, request=request) def connection_download_connection_metadata_changes(self, connection_identifier: str): From ae421b908e5f7c7f3b78cb77b262fff92b09ae3c Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Tue, 25 Feb 2025 13:40:48 +0000 Subject: [PATCH 08/16] Added set_obj_id.py but it is just a preview since some APIs may finalize after 10.6 --- examples_v2/set_obj_id.py | 104 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 examples_v2/set_obj_id.py diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py new file mode 100644 index 0000000..f34f32e --- /dev/null +++ b/examples_v2/set_obj_id.py @@ -0,0 +1,104 @@ +import json +import os +import requests.exceptions +from typing import Optional, Dict, List + +from src.thoughtspot_rest_api_v1 import TSRestApiV2, TSTypesV2, ReportTypes, TSRestApiV1 + +username = os.getenv('username') # or type in yourself +password = os.getenv('password') # or type in yourself +server = os.getenv('server') # or type in yourself +org_id = 0 + +ts = TSRestApiV2(server_url=server) +try: + auth_token_response = ts.auth_token_full(username=username, password=password, + validity_time_in_sec=3000, org_id=org_id) + ts.bearer_token = auth_token_response['token'] +except requests.exceptions.HTTPError as e: + print(e) + print(e.response.content) + exit() + +def create_obj_id_update_request(guid: str, obj_id: str): + update_req = { + "headers_update": + ( + {'identifier': guid, + 'attributes': ( + { + 'name': 'obj_id', + 'value': obj_id + } + ) + } + ) + } + return update_req + +# { 'guid' : 'obj_id' } +def create_multi_obj_id_update_request(guid_obj_id_map: Dict): + update_req = { + "headers_update": [] + } + for guid in guid_obj_id_map: + header_item = { + 'identifier': guid, + 'attributes': ( + { + 'name': 'obj_id', + 'value': guid_obj_id_map[guid] + } + ) + } + update_req["headers_update"].append(header_item) + + return update_req + +def set_one_object(): + # Simple example of setting a Table object to have a Full Qualified Name as the obj_id + update_req = create_obj_id_update_request(guid='43ab8a16-473a-44dc-9c78-4346eeb51f6c', obj_id='Conn.DB_Name.TableName') + + resp = ts.metadata_headers_update(request=update_req) + print(json.dumps(resp, indent=2)) + + +def export_tml_with_obj_id(guid:Optional[str] = None, obj_id: Optional[str] = None): + # Example of metadata search using obj_identifier (the property may be updated?) + if obj_id is not None: + search_req = { + "metadata": ( + {'obj_identifier': obj_id} + ), + "sort_options": { + "field_name": "CREATED", + "order": "DESC" + } + } + + tables = ts.metadata_search(request=search_req) + if len(tables) == 1: + guid = tables[0]['metadata_id'] + + # print(json.dumps(log_tables, indent=2)) + + if guid is None: + raise Exception() + + # export_options allow shifting TML export to obj_id, without any guid references + exp_opt = { + "include_obj_id_ref": True, + "include_guid": False, + "include_obj_id": True + } + + + yaml_tml = ts.metadata_tml_export(metadata_ids=[guid], edoc_format='YAML', + export_options=exp_opt) + print(yaml_tml[0]['edoc']) + print("-------") + + # Save the file with {obj_id}.{type}.{tml} + filename = "{}.table.tml".format(table['metadata_header']['objId']) + with open(file=filename, mode='w') as f: + f.write(yaml_tml[0]['edoc']) \ No newline at end of file From 2fe7e91a6b615ecde517027f0c7f71c430cab4b5 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Fri, 28 Feb 2025 11:15:49 +0000 Subject: [PATCH 09/16] Finished export function --- examples_v2/set_obj_id.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index f34f32e..8160955 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -63,7 +63,9 @@ def set_one_object(): print(json.dumps(resp, indent=2)) -def export_tml_with_obj_id(guid:Optional[str] = None, obj_id: Optional[str] = None): +def export_tml_with_obj_id(guid:Optional[str] = None, + obj_id: Optional[str] = None, + save_to_disk=True): # Example of metadata search using obj_identifier (the property may be updated?) if obj_id is not None: search_req = { @@ -79,6 +81,7 @@ def export_tml_with_obj_id(guid:Optional[str] = None, obj_id: Optional[str] = No tables = ts.metadata_search(request=search_req) if len(tables) == 1: guid = tables[0]['metadata_id'] + obj_id = tables[0]['metadata_header']['objId'] # print(json.dumps(log_tables, indent=2)) @@ -95,10 +98,14 @@ def export_tml_with_obj_id(guid:Optional[str] = None, obj_id: Optional[str] = No yaml_tml = ts.metadata_tml_export(metadata_ids=[guid], edoc_format='YAML', export_options=exp_opt) - print(yaml_tml[0]['edoc']) - print("-------") - # Save the file with {obj_id}.{type}.{tml} - filename = "{}.table.tml".format(table['metadata_header']['objId']) - with open(file=filename, mode='w') as f: - f.write(yaml_tml[0]['edoc']) \ No newline at end of file + if save_to_disk is True: + print(yaml_tml[0]['edoc']) + print("-------") + + # Save the file with {obj_id}.{type}.{tml} + filename = "{}.table.tml".format(obj_id) + with open(file=filename, mode='w') as f: + f.write(yaml_tml[0]['edoc']) + + return yaml_tml From cee3b861569f32dbe352ca99f22d0736d9a0b47b Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Fri, 28 Feb 2025 14:31:35 +0000 Subject: [PATCH 10/16] First request functions to get the objects to name --- examples_v2/set_obj_id.py | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index 8160955..0b0c7b3 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -109,3 +109,46 @@ def export_tml_with_obj_id(guid:Optional[str] = None, f.write(yaml_tml[0]['edoc']) return yaml_tml + +def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: Optional[int] = None): + + if org_id is None: + org_req = { + "org_identifier": org_name + } + org_resp = ts.orgs_search(request=org_req) + if len(org_resp) == 1: + org_id = org_resp[0]["id"] + else: + raise Exception("No org with that org_name was found, please try again or provide org_id") + + ts2 = TSRestApiV2(server_url=server) + try: + auth_token_response = ts2.auth_token_full(username=username, password=password, + validity_time_in_sec=3000, org_id=org_id) + ts.bearer_token = auth_token_response['token'] + except requests.exceptions.HTTPError as e: + print(e) + print(e.response.content) + exit() + + types = ["LOGICAL_TABLE", "LIVEBOARD", "ANSWER"] + search_req = { + "metadata": { + "record_offset": 0, + "record_size": -1, + "include_headers": True, + "metadata":[ + { + "type": "LOGICAL_TABLE" + } + ] + }, + "sort_options": { + "field_name": "CREATED", + "order": "DESC" + } + } + + search_resp = ts2.metadata_search(request=search_req) + print(json.dumps(search_resp, indent=2)) From de7ab00d2d5030e0410f914c364c3a70d691a8c7 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Fri, 28 Feb 2025 16:39:56 +0000 Subject: [PATCH 11/16] First round of data object auto- obj_id generation. Can keep iterating on naming pattern --- examples_v2/set_obj_id.py | 54 ++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index 0b0c7b3..da5738b 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -134,21 +134,51 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: types = ["LOGICAL_TABLE", "LIVEBOARD", "ANSWER"] search_req = { - "metadata": { - "record_offset": 0, - "record_size": -1, - "include_headers": True, - "metadata":[ - { - "type": "LOGICAL_TABLE" - } - ] - }, + "record_offset": 0, + "record_size": -1, + "include_headers": True, + "include_details": True, + "metadata":[ + { + "type": "LOGICAL_TABLE" + } + ] + , "sort_options": { "field_name": "CREATED", "order": "DESC" } } - search_resp = ts2.metadata_search(request=search_req) - print(json.dumps(search_resp, indent=2)) + # Tables - split out Worksheets/ Models / Views from actual Table Objects + try: + tables_resp = ts2.metadata_search(request=search_req) + except requests.exceptions.HTTPError as e: + print(e) + print(e.response.content) + exit() + + # print(json.dumps(tables_resp, indent=2)) + + final_guid_obj_id_map = {} + + for table in tables_resp: + ds_type = table["metadata_header"]["type"] + + guid = table["metadata_id"] + # Real tables + if ds_type in ['ONE_TO_ONE_LOGICAL']: + detail = table["metadata_detail"] + db_table_details = detail["logicalTableContent"]["tableMappingInfo"] + + obj_id = "{}__{}__{}".format(db_table_details["databaseName"], db_table_details["schemaName"], + db_table_details["tableName"]) + else: + obj_id = table["metadata_name"].replace(" ", "") # Need more transformation + + final_guid_obj_id_map[guid] = obj_id + + # print(json.dumps(final_guid_obj_id_map, indent=2)) + + return final_guid_obj_id_map + From c69608caa1dbbe75ff9b5fe237e8b2a00a112916 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 3 Mar 2025 10:57:34 +0000 Subject: [PATCH 12/16] Updated the automatic obj_id generation and adding functions for determining duplicates before attempting to set --- examples_v2/set_obj_id.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index da5738b..75901e5 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -2,6 +2,9 @@ import os import requests.exceptions from typing import Optional, Dict, List +from urllib import parse +import re +from collections import Counter from src.thoughtspot_rest_api_v1 import TSRestApiV2, TSTypesV2, ReportTypes, TSRestApiV1 @@ -166,15 +169,30 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: ds_type = table["metadata_header"]["type"] guid = table["metadata_id"] + + # Special property for certain system items that exist across all orgs - skip, cannot reset except in Org 0 + if "belongToAllOrgs" in table["metadata_header"]: + if table["metadata_header"]["belongToAllOrgs"] is True: + continue + # Real tables if ds_type in ['ONE_TO_ONE_LOGICAL']: detail = table["metadata_detail"] db_table_details = detail["logicalTableContent"]["tableMappingInfo"] + # Assumes a "{db}__{schema}__{tableName}" naming convention, but + # {tsConnection}__{table} may make more sense across a number of Orgs with identical 'schemas' with + # differing names + # Essentially you want identical, unique obj_id for "the same table" across Orgs obj_id = "{}__{}__{}".format(db_table_details["databaseName"], db_table_details["schemaName"], db_table_details["tableName"]) else: - obj_id = table["metadata_name"].replace(" ", "") # Need more transformation + # For non-table objects, obj_ids just need to be URL safe strings + # This is an example of a basic transformation from the Display Name to a URL safe string + obj_id = table["metadata_name"].replace(" ", "_") # Need more transformation + obj_id = parse.quote(obj_id) + # After parse quoting, there characters are in form %XX , replace with _ or blank space + obj_id = re.sub(r"%..", "", obj_id) final_guid_obj_id_map[guid] = obj_id @@ -182,3 +200,14 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: return final_guid_obj_id_map +def list_duplicate_obj_ids(initial_map: Dict): + cnt = Counter(initial_map.values()) + + if len(initial_map) == len(cnt): + return [] + else: + duplicate_obj_ids = [] + for c in cnt: + if cnt[c] > 1: + duplicate_obj_ids.append(c) + return duplicate_obj_ids From 2fa85bad4fded8dd447833c58cd8d3fa9e3bd075 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 3 Mar 2025 11:03:42 +0000 Subject: [PATCH 13/16] Return map with any duplicates so you can manually adjust --- examples_v2/set_obj_id.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index 75901e5..ee54b87 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -200,7 +200,7 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: return final_guid_obj_id_map -def list_duplicate_obj_ids(initial_map: Dict): +def find_duplicate_obj_ids(initial_map: Dict) -> Dict: cnt = Counter(initial_map.values()) if len(initial_map) == len(cnt): @@ -210,4 +210,9 @@ def list_duplicate_obj_ids(initial_map: Dict): for c in cnt: if cnt[c] > 1: duplicate_obj_ids.append(c) - return duplicate_obj_ids + dup_map = {} + + for m in initial_map: + if initial_map[m] in duplicate_obj_ids: + dup_map[m] = initial_map[m] + return dup_map From 5ea3708d955e3aa8b0d456dec9923e3b146522e4 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Mon, 3 Mar 2025 11:04:56 +0000 Subject: [PATCH 14/16] Return map with any duplicates so you can manually adjust --- examples_v2/set_obj_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index ee54b87..f310ed3 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -204,7 +204,7 @@ def find_duplicate_obj_ids(initial_map: Dict) -> Dict: cnt = Counter(initial_map.values()) if len(initial_map) == len(cnt): - return [] + return {} else: duplicate_obj_ids = [] for c in cnt: From 1ea06b413df272b1bb7852cb78dde0fdd7b59eeb Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Wed, 5 Mar 2025 20:00:13 +0000 Subject: [PATCH 15/16] Added a call to get connection names if you want to use them in naming schemes --- examples_v2/set_obj_id.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/examples_v2/set_obj_id.py b/examples_v2/set_obj_id.py index f310ed3..f61139e 100644 --- a/examples_v2/set_obj_id.py +++ b/examples_v2/set_obj_id.py @@ -142,8 +142,23 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: "include_headers": True, "include_details": True, "metadata":[ + {"type": "LOGICAL_TABLE"},{"type": "LIVEBOARD"},{"type": "ANSWER"} + ] + , + "sort_options": { + "field_name": "CREATED", + "order": "DESC" + } + } + + conn_req = { + "record_offset": 0, + "record_size": -1, + "include_headers": False, + "include_details": False, + "metadata": [ { - "type": "LOGICAL_TABLE" + "type": "CONNECTION" } ] , @@ -161,6 +176,26 @@ def retrieve_dev_org_objects_for_mapping(org_name: Optional[str] = None, org_id: print(e.response.content) exit() + # Connection names for mapping and use in obj_id naming schema for Tables + try: + conn_resp = ts2.metadata_search(request=conn_req) + except requests.exceptions.HTTPError as e: + print(e) + print(e.response.content) + exit() + + conn_map = {} + for conn in conn_resp: + # Create URL safe portion of obj_id for Connection + # Assuming No Duplicate Connection Names in Org (fix this first if you don't have) + # Define whatever automatic transformations to create URL safe + # but aesthetically pleasing first transform from the existing object names + c_obj_id = conn["metadata_name"].replace(" ", "") + c_obj_id = parse.quote(c_obj_id) + # obj_id = obj_id.replace("%3A", "_") + # After parse quoting, there characters are in form %XX , replace with _ or blank space + c_obj_id = re.sub(r"%..", "", c_obj_id) + conn_map[conn["metadata_id"]] = c_obj_id # print(json.dumps(tables_resp, indent=2)) final_guid_obj_id_map = {} From a4476bf6e336666a803f5534b47d7d705f19afd5 Mon Sep 17 00:00:00 2001 From: "bryant.howell" Date: Wed, 26 Mar 2025 14:13:50 -0500 Subject: [PATCH 16/16] Added requirements.txt and updated setup.cfg to remove the YAML packages which are not used by this library anymore --- requirements.txt | 2 ++ setup.cfg | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c281cfa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +requests-toolbelt \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a3110c6..f36fb9e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,6 @@ packages = find: python_requires = >=3.6 install_requires = requests - PyYAML - oyaml requests_toolbelt