From 333e16bbf8be7191958973f2e89eb1881317390b Mon Sep 17 00:00:00 2001 From: Hariprasad Kothuru Date: Tue, 18 Jun 2024 17:09:55 +0530 Subject: [PATCH 01/10] azure openAI support --- .../README.md | 6 + .../chunks_extraction.py | 180 ++++++++++++++++-- .../variables.py | 6 + 3 files changed, 176 insertions(+), 16 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index add4fdb2..d23b8a18 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -31,6 +31,12 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit ```plaintext OPENAI_KEY = "sk-***" + #Azure Open Ai + apiKey= "***" + userSubDomain= "***" + deployment= "***" + Apiversion= "***" + # Access Token hostUrl = "***" clientId = "**" diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index 427cbdaf..350056fc 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -3,12 +3,12 @@ import copy import os import json -import html import time import re import traceback import ingest_data as data_ingestion import aiofiles +import aiohttp import html import urllib.parse from bs4 import BeautifulSoup, Tag @@ -20,6 +20,12 @@ import variables as config logger = get_logger() OPENAI_KEY = config.OPENAI_KEY +Azure_key= config.apiKey + +if OPENAI_KEY: + llm_used="openai" +elif Azure_key: + llm_used="azure" open_ai_conf = { "API_BASE":"https://api.openai.com/v1/", @@ -31,19 +37,79 @@ "timeout":150, "max_retries":0 } +azure_open_ai_conf = { + "API_BASE": "https://koreopenai.openai.azure.com", + "deployment_id": config.deployment, + "api_version": config.Apiversion, + "API_KEY": Azure_key, + "model": "gpt-3.5-turbo-16k", + "max_tokens": 7000, + "temperature": 0.5, + "top_p": 1, + "timeout": 150, + "max_retries": 0 +} + prompt_json = [ { "role": "system", "content": "{{instruction_msg}}\n--------------------------------------------------" "----------------------------------------------------------------------\n\nContent:-{{content}}" } - ] +] + +azure_prompt=[ + { + "role": "system", + "content": "{{instruction_msg}}" + }, + { + "role": "user", + "content": "{{content}}" + } +] + instruction_msg = "Given an markdown for a possible table of content for a document, understand the text and identify table of contents with correct heading and subheadings. Please note that there may be some extra info in the input text, which may not be needed for the index, use you knowledge to decide what to include in the index.\nFigure out add the parents of each subheading and return a lookup table map, which maps a heading to its hierarchical heading with all the parents appended as prefix. \nCHILD HEADING SHOULD BE PRESENT IN THE VALUE ALWAYS.\nMap should be like:-\n{\n\"Child heading 1\": \"prefix1 prefix2 Child heading 1\",\n\"Child heading 2\": \" \" Date: Wed, 19 Jun 2024 13:38:17 +0530 Subject: [PATCH 02/10] variable name changes --- .../chunks_extraction.py | 10 +++++----- .../SalesforceCustomExtractionUtility/variables.py | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index 350056fc..c0d7b0e9 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -20,7 +20,7 @@ import variables as config logger = get_logger() OPENAI_KEY = config.OPENAI_KEY -Azure_key= config.apiKey +Azure_key= config.AZURE_OPENAI_KEY if OPENAI_KEY: llm_used="openai" @@ -28,8 +28,8 @@ llm_used="azure" open_ai_conf = { - "API_BASE":"https://api.openai.com/v1/", - "model": "gpt-3.5-turbo-16k", + "API_BASE":config.openAI_apibase, + "model": config.openAI_model, "API_KEY":OPENAI_KEY, "max_tokens":9000, "temperature":0.5, @@ -38,11 +38,11 @@ "max_retries":0 } azure_open_ai_conf = { - "API_BASE": "https://koreopenai.openai.azure.com", + "API_BASE": config.API_BASE, "deployment_id": config.deployment, "api_version": config.Apiversion, "API_KEY": Azure_key, - "model": "gpt-3.5-turbo-16k", + "model": config.Azure_model, "max_tokens": 7000, "temperature": 0.5, "top_p": 1, diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index c3d08e9b..419da13f 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -4,10 +4,14 @@ load_dotenv() # OpenAI Configuration -OPENAI_KEY = os.environ.get('OPENAI_KEY') +OPENAI_KEY =os.environ.get("open_ai_key") +openAI_model=os.environ.get("openai_model") +openAI_apibase=os.environ.get("openAI_apibase") #azure open ai -apiKey= os.environ.get("apiKey") +AZURE_OPENAI_KEY= os.environ.get("AZURE_OPENAI_KEY") +API_BASE=os.environ.get("API_BASE") +Azure_model=os.environ.get("azure_model") userSubDomain=os.environ.get("userSubDomain") deployment= os.environ.get("deployment") Apiversion=os.environ.get("Apiversion") From 9705fb2882c26f20485f98bb579036a505b12cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Crohanchaurasia-Kore=E2=80=9D?= <“rohan.chaurasia@kore.com”> Date: Wed, 19 Jun 2024 16:49:53 +0530 Subject: [PATCH 03/10] Updated readme --- .../README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index d23b8a18..2084c4ee 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -29,13 +29,20 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit 4. **Create a `.env` file and configure the following environment variables**: ```plaintext - OPENAI_KEY = "sk-***" - #Azure Open Ai - apiKey= "***" - userSubDomain= "***" - deployment= "***" - Apiversion= "***" + # open ai + open_ai_key="" + openAI_apibase="" + openai_model="" + + + #Azure open ai + AZURE_OPENAI_KEY="" + API_BASE="" + azure_model="" + userSubDomain= "" + deployment="" + Apiversion="" # Access Token hostUrl = "***" From 0e65310f35b6354a758653a69dc893e616162a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Crohanchaurasia-Kore=E2=80=9D?= <“rohan.chaurasia@kore.com”> Date: Wed, 19 Jun 2024 17:14:34 +0530 Subject: [PATCH 04/10] Updated requirement.txt --- Extraction/SalesforceCustomExtractionUtility/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Extraction/SalesforceCustomExtractionUtility/requirements.txt b/Extraction/SalesforceCustomExtractionUtility/requirements.txt index c7da6471..f5e5db3a 100644 --- a/Extraction/SalesforceCustomExtractionUtility/requirements.txt +++ b/Extraction/SalesforceCustomExtractionUtility/requirements.txt @@ -3,6 +3,7 @@ beautifulsoup4==4.12.3 markdownify==0.12.1 openai==1.25.2 dirtyjson==1.0.8 +urllib3==1.26.15 python-dotenv requests pandas From 3b0dea663f9bf097e4929c16e1382bba3025e084 Mon Sep 17 00:00:00 2001 From: MATHIVATHANI B Date: Tue, 25 Jun 2024 16:35:00 +0530 Subject: [PATCH 05/10] Proxy changes --- .../chunks_extraction.py | 29 ++-- .../ingest_data.py | 5 +- .../salesforceData_extraction.py | 124 ++++++++++-------- .../variables.py | 13 +- 4 files changed, 96 insertions(+), 75 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index c0d7b0e9..d555eb4e 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -60,11 +60,11 @@ azure_prompt=[ { - "role": "system", + "role": "system", "content": "{{instruction_msg}}" }, { - "role": "user", + "role": "user", "content": "{{content}}" } ] @@ -93,7 +93,7 @@ async def fetch_azure_response(self, **request_payload): async with aiohttp.ClientSession() as session: for _ in range(self.max_retries + 1): try: - async with session.post(url, headers=headers, json=request_payload) as response: + async with session.post(url, headers=headers, json=request_payload, ssl=False) as response: if response.status == 200: return await response.json() else: @@ -130,7 +130,7 @@ async def parse_html(file_path): except UnicodeDecodeError as e: print(f"Error decoding file {file_path} with Latin-1 encoding: {e}") # Reading the file - + index = data return index @@ -142,6 +142,10 @@ def replace_backslashes(input_string): def clean_json(json_str): data = dict(dirtyjson.loads(json_str)) json_string = data.get("raw_data", "") + # json_string = json_string.replace("\n","").replace("\t","") + # json_string = re.sub(r'(? Date: Mon, 1 Jul 2024 12:47:47 +0530 Subject: [PATCH 06/10] Updated code which reads csv input --- .../README.md | 8 +- .../chunks_extraction.py | 20 +- .../salesforceData_extraction.py | 191 ++++++++++++------ .../variables.py | 4 +- 4 files changed, 148 insertions(+), 75 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index 2084c4ee..cacf21df 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -70,13 +70,17 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit /data/Python-3.9.7/bin/python3.9 -m venv py3.9.7 source /data/py3.9.7/bin/activate ``` +6. **Replace Proxy url** + + In variables.py add the proxy urls under #proxies then update the url in salesforceData_extraction (line- 193). . + -6. **Install the required packages**: +7. **Install the required packages**: ```bash pip install -r requirements.txt ``` -7. **Run the utility**: +8. **Run the utility**: ```bash python main.py ``` diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index d555eb4e..bbf47386 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -172,13 +172,14 @@ async def parse_json(file_path): raw_data = cleaned_json.get("raw_data", {}) layout_items = raw_data.get("layoutItems",[]) lastPublisheddate= raw_data.get("lastPublishedDate","") + knowledge_article_urls=raw_data.get("url") html_string = "" for item in layout_items: if item.get("type","") in ["RICH_TEXT_AREA","TEXT"] : if item.get("value", ""): html_string += item.get("value") unescaped_string = html.unescape(html_string) - return unescaped_string,lastPublisheddate + return unescaped_string,lastPublisheddate,knowledge_article_urls def check_tag(tag, heading_ids): @@ -514,7 +515,7 @@ def split_into_chunks(text, heading_start, heading_end): chunks.append(dict(heading = heading, content = content)) return chunks -async def convert_to_SA_format(chunks,lastupdatedDate, **kwargs): +async def convert_to_SA_format(chunks,lastupdatedDate,unique_ArticleIDs_url, **kwargs): data_list = list() for chunk in chunks: title = chunk.get("heading") @@ -527,7 +528,8 @@ async def convert_to_SA_format(chunks,lastupdatedDate, **kwargs): "content": chunk.get("content"), "html": urllib.parse.quote(chunk.get("content")), "content_markdown" : markdown, - "url": kwargs.get("url",""), + # "url": kwargs.get("url",""), + "url": unique_ArticleIDs_url, "meta_data":kwargs.get("meta_data",{}), "doc_name" : kwargs.get("filename",""), "lastModifiedDate": lastupdatedDate @@ -539,7 +541,7 @@ def convert_chunks_to_markdown(chunks): if chunk.get("content",""): chunk['content_markdown'] = md(chunk.get('content')) return chunks -async def extract_chunks(input_html,lastupdatedDate,**kwargs): +async def extract_chunks(input_html,lastupdatedDate,unique_ArticleIDs_url,**kwargs): try: soup = BeautifulSoup(input_html, 'html.parser') soup_for_toc = copy.deepcopy(soup) @@ -554,7 +556,7 @@ async def extract_chunks(input_html,lastupdatedDate,**kwargs): markdown_chunks = convert_chunks_to_markdown(extracted_chunks) # Split the html into chunk if failed to extract using the above approach # chunks = split_into_chunks(html_as_markdown, heading_start, heading_end) - sa_structured_data = await convert_to_SA_format(markdown_chunks,lastupdatedDate,**kwargs) + sa_structured_data = await convert_to_SA_format(markdown_chunks,lastupdatedDate,unique_ArticleIDs_url,**kwargs) except Exception as e: print("Error in extracting Chunks for the given file : {}. Sending empty data".format(kwargs.get("filename"))) print(traceback.format_exc()) @@ -571,6 +573,7 @@ async def save_json(output_file_path, store_chunks): async def helper(input_directory_path, output_directory_path): structure_Data_Count=0 updatedDates={} + articleID_urls={} # html_directory_path="./html" os.makedirs(output_directory_path, exist_ok=True) input_html_directory_path = os.path.join(input_directory_path, "html") @@ -590,7 +593,7 @@ async def helper(input_directory_path, output_directory_path): else: # Open the file in write mode - parsed_html,updatedDate = await parse_json(file_path) + parsed_html,updatedDate,knowledge_article_urls = await parse_json(file_path) # print(parsed_html) with open(html_input_file_path, 'w') as file: # Write the HTML string to the file @@ -598,6 +601,7 @@ async def helper(input_directory_path, output_directory_path): with open(html_input_file_path, 'w', encoding='utf-8') as output_file: output_file.write(parsed_html) updatedDates.update({html_input_file_path:updatedDate}) + articleID_urls.update({html_input_file_path:knowledge_article_urls}) print(f"Parsed HTML saved to {html_input_file_path}") except UnicodeEncodeError as e: print(f"Error encoding parsed HTML to UTF-8: {e}") @@ -612,7 +616,9 @@ async def helper(input_directory_path, output_directory_path): print(f'Staring Extraction for {filename}') if html_input_file_path in updatedDates: lastupdatedDate= updatedDates[html_input_file_path] - json_output = await extract_chunks(input_html,lastupdatedDate,**kwargs) + if html_input_file_path in articleID_urls: + unique_ArticleIDs_url= articleID_urls[html_input_file_path] + json_output = await extract_chunks(input_html,lastupdatedDate,unique_ArticleIDs_url,**kwargs) html_path = os.path.join(input_html_directory_path, f"{os.path.splitext(filename)[0]}_chunks.json") with open(html_path, 'w') as file: json.dump(json_output, file, indent=4) diff --git a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py index 9b461a03..cb2548a3 100644 --- a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py @@ -4,14 +4,14 @@ import aiohttp import chunks_extraction as sa_utility import os -import shutil import ssl -from utils.logger import get_logger, write_json_to_separate_file +import shutil +import csv +from utils.logger import get_logger, write_json_to_separate_file,write_refresh_token logger = get_logger() #access token generation def generate_access_token(): try: - # print("PROXY: ",config.proxies) url = config.accessTokenUrl payload = { 'grant_type': config.accessTokenGrantType, @@ -19,14 +19,13 @@ def generate_access_token(): 'client_id': config.accessTokenClientId, 'redirect_uri':config.redirectUri, 'code': config.accessTokenAuthCode - } + } headers = { 'Content-Type': 'application/x-www-form-urlencoded' } - # proxy= {"http": "http://vz-proxy.pncint.net:80","https": "http://vz-proxy.pncint.net:80"} response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=False) - response.raise_for_status() - print("Response",response) + response.raise_for_status() + #getting access token and instance_url; access_token = response.json().get('access_token') instance_url=response.json().get('instance_url') @@ -44,16 +43,16 @@ def generate_refresh_token(refresh_token) : 'grant_type': config.refreshTokenGrantType, 'client_secret': config.refreshTokenClientSecret, 'client_id': config.refreshTokenClientId, - 'refresh_token' :refresh_token + 'refresh_token' :refresh_token } headers = { 'Content-Type': 'application/x-www-form-urlencoded' } response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=False) - response.raise_for_status() + response.raise_for_status() + #getting access token and instance_url; access_token = response.json().get('access_token') - if not access_token: raise ValueError("Access token not found in response") return access_token @@ -65,73 +64,140 @@ def generate_refresh_token(refresh_token) : def make_api_call(url, headers=None): try: response = requests.get(url, headers=headers,proxies=config.proxies,verify=False) - response.raise_for_status() - logger.info(f"API call successful for URL: {url}") + response.raise_for_status() return response.json() except requests.HTTPError as e: logger.error(f"HTTP error occurred for URL {url}: {e}, status code: {response.status_code}", exc_info=True) except Exception as e: logger.error(f"An error occurred during API call for URL {url}: {e}", exc_info=True) -#getting id -def get_articles_list_id(access_token): +#getting id +async def get_articles_list_id(access_token): try: - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" - headers = { + urlNames=[] + item_ids=[] + emptyUrlNames=[] + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + print(row) + urlNames.append(row[0]) + print(len(urlNames)) + #splitting into batches + urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] + # print(urlBatches) + count=0 + for eachBatch in urlBatches: + count=count+1; + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + for url_name in eachBatch: + + # print(url_name) + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" + # print(lists_url) + headers = { "Authorization": f"Bearer {access_token}", "Accept-Language": "en-US", } - expectedItemIds=config.itemIds - if expectedItemIds!="" : - item_ids= expectedItemIds - else: - lists_response = make_api_call(lists_url, headers) - item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] - list_url_all=[] - # item_ids = ["kA01L000000EMkqSAG","kA04p000000dzUHCAY","kA01L000000ELC6SAO","kA01L000000ENNPSA4","kA01L000000ELokSAG","kA01L0000001hbKSAQ","kA04p0000006UQYCA2","kA01L000000ENH7SAO","kA04p0000001ClcCAE","kA01L000000EMipSAG"] - print(item_ids) - - #getting all the other page urls and the article ids in them - if 'nextRecordsUrl' in lists_response: - for page in lists_response['nextRecordsUrl']: - if lists_response['nextRecordsUrl']: - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US" - } - lists_url_2 = f"{lists_url}{lists_response['nextRecordsUrl']}" - lists_responses = make_api_call(lists_url_2, headers) - item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] - - item_ids=item_ids + item_id_all - lists_response=lists_responses - list_url_all.append(lists_url_2) + listResponse = make_api_call(lists_url,headers) + if listResponse['totalSize']==0 or listResponse['records'] == None : + emptyUrlNames.append(url_name) + + else : + itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] + # print(itemId) + item_ids.append(itemId[0]) + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + # print(f"API calls for fetching item ids are successful for batch {count} .") + emptyUrlNameslist = [[item] for item in emptyUrlNames] + print("Empty URLS",emptyUrlNameslist) + + with open('emptyUrl.csv', 'w', newline='') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerows(emptyUrlNameslist) + # expectedItemIds=config.itemIds + # if expectedItemIds!="" : + # item_ids= expectedItemIds + # else: + # lists_response = make_api_call(lists_url, headers) + # item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] + # list_url_all=[] + # # item_ids = ["kA01L000000EMkqSAG","kA04p000000dzUHCAY","kA01L000000ELC6SAO","kA01L000000ENNPSA4","kA01L000000ELokSAG","kA01L0000001hbKSAQ","kA04p0000006UQYCA2","kA01L000000ENH7SAO","kA04p0000001ClcCAE","kA01L000000EMipSAG"] + # print(item_ids) + + # #getting all the other page urls and the article ids in them + # if 'nextRecordsUrl' in lists_response: + # for page in lists_response['nextRecordsUrl']: + # if lists_response['nextRecordsUrl']: + # headers = { + # "Authorization": f"Bearer {access_token}", + # "Accept-Language": "en-US" + # } + # lists_url_2 = f"{lists_url}{lists_response['nextRecordsUrl']}" + # lists_responses = make_api_call(lists_url_2, headers) + # item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] + + # item_ids=item_ids + item_id_all + # lists_response=lists_responses + # list_url_all.append(lists_url_2) + # lists_url = f"https://koreai56-dev-ed.my.salesforce.com/services/data/v57.0/support/knowledgeArticles?pageNumber=1" + + # headers = { + # "Authorization": f"Bearer {access_token}", + # "Accept-Language": "en-US", + # } + # expectedItemIds=config.itemIds + # if expectedItemIds!="" : + # item_ids= expectedItemIds + # else: + # lists_response = make_api_call(lists_url, headers) + # item_ids = [item["id"] for item in lists_response.get("articles", [])] + # list_url_all=[] + # # item_ids = ["kA01L000000EMkqSAG","kA04p000000dzUHCAY","kA01L000000ELC6SAO","kA01L000000ENNPSA4","kA01L000000ELokSAG","kA01L0000001hbKSAQ","kA04p0000006UQYCA2","kA01L000000ENH7SAO","kA04p0000001ClcCAE","kA01L000000EMipSAG"] + # print(item_ids) + # #getting all the other page urls and the article ids in them + # if 'nextPageUrl' in lists_response: + # for page in lists_response['nextPageUrl']: + # if lists_response['nextPageUrl']: + # headers = { + # "Authorization": f"Bearer {access_token}", + # "Accept-Language": "en-US" + # } + # lists_url_2 = f"https://koreai56-dev-ed.my.salesforce.com/{lists_response['nextPageUrl']}" + # lists_responses = make_api_call(lists_url_2, headers) + # item_id_all = [item["id"] for item in lists_responses.get("articles", [])] + # item_ids=item_ids + item_id_all + # lists_response=lists_responses + # list_url_all.append(lists_url_2) + # print(item_ids) except Exception as e: logger.error(f"An error occurred while retrieving 'Articles' list ID: {e}", exc_info=True) + logger.error(f"API calls for fetching item ids failed at batch {count} .") return item_ids #getting details async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): - item_url = f"{instance_url}/services/data/v57.0/support/knowledgeArticles/{item_id}" + item_url = f"{instance_url}/services/data/v57.0/support/knowledgeArticles/{item_id}" headers = { "Authorization": f"Bearer {new_access_token}", "Accept-Language": "en-US" - } - try: + } + try: ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE - async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session: - async with session.get(item_url, headers=headers, timeout=20,proxy="http://vz-proxy.pncint.net:80") as response: - response.raise_for_status() - item_response = await response.json() - + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session: + async with session.get(item_url, headers=headers, timeout=20,proxy="client proxy url") as response: + response.raise_for_status() + item_response = await response.json() + item_response["url"]= item_url + return { 'raw_data': item_response - } + } except aiohttp.ClientError as e: logger.error(f"An error occurred during item details fetch: {e}", exc_info=True) except asyncio.TimeoutError: @@ -142,23 +208,21 @@ async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,proxy_url): logger.info(f"Making API call for retreiving all the item details") batches = [itemIds[i:i+int(config.eachBatchCount)] for i in range(0, len(itemIds), int(config.eachBatchCount))] - print(batches) + try: - - results_final=[]; - x=1 + + results_final=[]; + x=1 for eachBatch in batches: - tasks = [] - print(len(eachBatch)) - # print("Next Batch") + tasks = [] new_access_token=generate_refresh_token(refresh_token) print("New Access Token",new_access_token) for item_id in eachBatch: task = fetch_item_details(new_access_token,instance_url,item_id,proxy_url) - tasks.append(task) + tasks.append(task) results = await asyncio.gather(*tasks) - results_final.append(results); - print("results", results) + results_final.append(results); + # x=1 for individual_batch in results_final : for individual_response in individual_batch : @@ -173,7 +237,7 @@ async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,pr os.makedirs(config.input_path) return filtered_results - + except aiohttp.ClientError as e: logger.error(f"An error occurred during API call for retrieving item details: {e}", exc_info=True) @@ -182,18 +246,15 @@ async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,pr async def extractData(): try: access_token,instance_url,refresh_token = generate_access_token() - - print("Access Token",access_token,instance_url) logger.info("Access token generated successfully.") logger.debug(f"Access token: {access_token}") logger.info("Access token generated successfully.") logger.debug(f"Access token: {access_token}") - itemIds = get_articles_list_id(access_token) + itemIds = await get_articles_list_id(access_token) logger.debug(f"Instance Url: {itemIds}") proxy_url=config.proxies - details= await make_list_api_call(access_token,instance_url,itemIds,refresh_token,proxy_url) logger.debug(f"Details: {details}") diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index cfcbabd3..5989f6ed 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -42,8 +42,10 @@ #paths input_path=os.environ.get('input_path') output_path=os.environ.get('output_path') + #proxies -proxies= {"http": "http://vz-proxy.pncint.net:80","https": "http://vz-proxy.pncint.net:80"} +#Replace "PROXY URL" +proxies= {"http": "PROXY URL","https": "PROXY URL"} #itemIds itemIds="" \ No newline at end of file From 65e688c1e3a0ba8dd63774065f4bec6fe23d6267 Mon Sep 17 00:00:00 2001 From: Mathivathani Date: Wed, 3 Jul 2024 16:29:56 +0530 Subject: [PATCH 07/10] updated code with csv input --- .../README.md | 27 ++- .../chunks_extraction.py | 5 +- .../salesforceData_extraction.py | 176 ++++++++---------- .../variables.py | 20 +- 4 files changed, 113 insertions(+), 115 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index cacf21df..b2d36dce 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -29,6 +29,9 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit 4. **Create a `.env` file and configure the following environment variables**: ```plaintext + + # Specify the language model provider: "openai" or "azure" + llm_used="" # open ai open_ai_key="" @@ -44,6 +47,10 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit deployment="" Apiversion="" + #update testing environment + # if cloud then test_env="login" for sandbox="test" + test_env="" + # Access Token hostUrl = "***" clientId = "**" @@ -64,23 +71,31 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit output_path = ".data/output/" ``` + #Proxy URLs for network access + proxies= {"http": "PROXY URL","https": "PROXY URL"} + + #Specifies whether data fetching uses URL names or Item IDs + #inputFormat="urlnames" or inputFormat="itemids" + inputFormat ="" + + #SSL Verification + #Set to "False" if SSL Verification is not needed, otherwise set to "True" + ssl="" + + 5. **Create and activate a virtual environment**: ```bash cd /data /data/Python-3.9.7/bin/python3.9 -m venv py3.9.7 source /data/py3.9.7/bin/activate ``` -6. **Replace Proxy url** - - In variables.py add the proxy urls under #proxies then update the url in salesforceData_extraction (line- 193). . - -7. **Install the required packages**: +6. **Install the required packages**: ```bash pip install -r requirements.txt ``` -8. **Run the utility**: +7. **Run the utility**: ```bash python main.py ``` diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index bbf47386..d1c04d76 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -22,10 +22,7 @@ OPENAI_KEY = config.OPENAI_KEY Azure_key= config.AZURE_OPENAI_KEY -if OPENAI_KEY: - llm_used="openai" -elif Azure_key: - llm_used="azure" +llm_used=config.llm_used open_ai_conf = { "API_BASE":config.openAI_apibase, diff --git a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py index cb2548a3..a8aa2a7a 100644 --- a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py @@ -23,7 +23,7 @@ def generate_access_token(): headers = { 'Content-Type': 'application/x-www-form-urlencoded' } - response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=False) + response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=config.sslVerify) response.raise_for_status() #getting access token and instance_url; @@ -48,7 +48,7 @@ def generate_refresh_token(refresh_token) : headers = { 'Content-Type': 'application/x-www-form-urlencoded' } - response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=False) + response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=config.sslVerify) response.raise_for_status() #getting access token and instance_url; @@ -63,7 +63,7 @@ def generate_refresh_token(refresh_token) : # Make an API call to the specified URL with optional headers. def make_api_call(url, headers=None): try: - response = requests.get(url, headers=headers,proxies=config.proxies,verify=False) + response = requests.get(url, headers=headers,proxies=config.proxies,verify=config.sslVerify) response.raise_for_status() return response.json() except requests.HTTPError as e: @@ -74,104 +74,80 @@ def make_api_call(url, headers=None): #getting id async def get_articles_list_id(access_token): try: - urlNames=[] - item_ids=[] - emptyUrlNames=[] - with open('input.csv', 'r') as csvfile: - csvreader = csv.reader(csvfile) - for row in csvreader: - print(row) - urlNames.append(row[0]) - print(len(urlNames)) - #splitting into batches - urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] - # print(urlBatches) - count=0 - for eachBatch in urlBatches: - count=count+1; - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - for url_name in eachBatch: - - # print(url_name) - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" - # print(lists_url) - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US", - } - listResponse = make_api_call(lists_url,headers) - if listResponse['totalSize']==0 or listResponse['records'] == None : - emptyUrlNames.append(url_name) - - else : - itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] - # print(itemId) - item_ids.append(itemId[0]) - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - # print(f"API calls for fetching item ids are successful for batch {count} .") - emptyUrlNameslist = [[item] for item in emptyUrlNames] - print("Empty URLS",emptyUrlNameslist) + if config.inputFormat=="urlnames": + urlNames=[] + item_ids=[] + emptyUrlNames=[] + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + print(row) + urlNames.append(row[0]) + print(len(urlNames)) + #splitting into batches + urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] + # print(urlBatches) + count=0 + for eachBatch in urlBatches: + count=count+1; + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + for url_name in eachBatch: + + # print(url_name) + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" + # print(lists_url) + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } + listResponse = make_api_call(lists_url,headers) + if listResponse['totalSize']==0 or listResponse['records'] == None : + emptyUrlNames.append(url_name) - with open('emptyUrl.csv', 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile) - csvwriter.writerows(emptyUrlNameslist) - # expectedItemIds=config.itemIds - # if expectedItemIds!="" : - # item_ids= expectedItemIds - # else: - # lists_response = make_api_call(lists_url, headers) - # item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] - # list_url_all=[] - # # item_ids = ["kA01L000000EMkqSAG","kA04p000000dzUHCAY","kA01L000000ELC6SAO","kA01L000000ENNPSA4","kA01L000000ELokSAG","kA01L0000001hbKSAQ","kA04p0000006UQYCA2","kA01L000000ENH7SAO","kA04p0000001ClcCAE","kA01L000000EMipSAG"] - # print(item_ids) - - # #getting all the other page urls and the article ids in them - # if 'nextRecordsUrl' in lists_response: - # for page in lists_response['nextRecordsUrl']: - # if lists_response['nextRecordsUrl']: - # headers = { - # "Authorization": f"Bearer {access_token}", - # "Accept-Language": "en-US" - # } - # lists_url_2 = f"{lists_url}{lists_response['nextRecordsUrl']}" - # lists_responses = make_api_call(lists_url_2, headers) - # item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] - - # item_ids=item_ids + item_id_all - # lists_response=lists_responses - # list_url_all.append(lists_url_2) - - # lists_url = f"https://koreai56-dev-ed.my.salesforce.com/services/data/v57.0/support/knowledgeArticles?pageNumber=1" - - # headers = { - # "Authorization": f"Bearer {access_token}", - # "Accept-Language": "en-US", - # } - # expectedItemIds=config.itemIds - # if expectedItemIds!="" : - # item_ids= expectedItemIds - # else: - # lists_response = make_api_call(lists_url, headers) - # item_ids = [item["id"] for item in lists_response.get("articles", [])] - # list_url_all=[] - # # item_ids = ["kA01L000000EMkqSAG","kA04p000000dzUHCAY","kA01L000000ELC6SAO","kA01L000000ENNPSA4","kA01L000000ELokSAG","kA01L0000001hbKSAQ","kA04p0000006UQYCA2","kA01L000000ENH7SAO","kA04p0000001ClcCAE","kA01L000000EMipSAG"] - # print(item_ids) + else : + itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] + # print(itemId) + item_ids.append(itemId[0]) + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + # print(f"API calls for fetching item ids are successful for batch {count} .") + emptyUrlNameslist = [[item] for item in emptyUrlNames] + print("Empty URLS",emptyUrlNameslist) + + with open('emptyUrl.csv', 'w', newline='') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerows(emptyUrlNameslist) + elif config.inputFormat=="itemids": + expectedItemIds=config.itemIds + if expectedItemIds!="" : + item_ids= expectedItemIds + else: + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } + lists_response = make_api_call(lists_url, headers) + item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] + list_url_all=[] + # print(item_ids) + + #getting all the other page urls and the article ids in them + if 'nextRecordsUrl' in lists_response: + for page in lists_response['nextRecordsUrl']: + if lists_response['nextRecordsUrl']: + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US" + } + lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}" + lists_responses = make_api_call(lists_url_2, headers) + item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] + + item_ids=item_ids + item_id_all + lists_response=lists_responses + list_url_all.append(lists_url_2) - # #getting all the other page urls and the article ids in them - # if 'nextPageUrl' in lists_response: - # for page in lists_response['nextPageUrl']: - # if lists_response['nextPageUrl']: - # headers = { - # "Authorization": f"Bearer {access_token}", - # "Accept-Language": "en-US" - # } - # lists_url_2 = f"https://koreai56-dev-ed.my.salesforce.com/{lists_response['nextPageUrl']}" - # lists_responses = make_api_call(lists_url_2, headers) - # item_id_all = [item["id"] for item in lists_responses.get("articles", [])] - # item_ids=item_ids + item_id_all - # lists_response=lists_responses - # list_url_all.append(lists_url_2) # print(item_ids) except Exception as e: logger.error(f"An error occurred while retrieving 'Articles' list ID: {e}", exc_info=True) @@ -187,10 +163,10 @@ async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): } try: ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False + ssl_context.check_hostname = config.sslVerify ssl_context.verify_mode = ssl.CERT_NONE async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session: - async with session.get(item_url, headers=headers, timeout=20,proxy="client proxy url") as response: + async with session.get(item_url, headers=headers, timeout=20,proxy=config.proxies["http"]) as response: response.raise_for_status() item_response = await response.json() item_response["url"]= item_url diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index 5989f6ed..776fb917 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -3,6 +3,10 @@ load_dotenv() +#LLM +llm_used=os.environ("llm_used") + + # OpenAI Configuration OPENAI_KEY =os.environ.get("open_ai_key") openAI_model=os.environ.get("openai_model") @@ -17,7 +21,7 @@ Apiversion=os.environ.get("Apiversion") # Access Token Configuration -accessTokenUrl = "https://test.salesforce.com/services/oauth2/token" +accessTokenUrl = f"https://{os.environ('test_env')}.salesforce.com/services/oauth2/token" hostUrl = os.environ.get('hostUrl') accessTokenGrantType = "authorization_code" accessTokenClientId = os.environ.get('clientId') @@ -26,7 +30,7 @@ accessTokenAuthCode =os.environ.get('accessTokenAuthCode') #Refresh Token Configuration -refreshTokenUrl = "https://test.salesforce.com/services/oauth2/token" +refreshTokenUrl = f"https://{os.environ('test_env')}test.salesforce.com/services/oauth2/token" refreshTokenGrantType = "refresh_token" refreshTokenClientId = os.environ.get('clientId') refreshTokenClientSecret = os.environ.get('clientSecret') @@ -44,8 +48,14 @@ output_path=os.environ.get('output_path') #proxies -#Replace "PROXY URL" -proxies= {"http": "PROXY URL","https": "PROXY URL"} +proxies=os.environ.get('proxies') + +#Fetching definiton +inputFormat=os.environ.get('inputFormat') + +#set SSL verfication +sslVerify=os.environ.get('ssl') #itemIds -itemIds="" \ No newline at end of file +itemIds="" + From de395c9f2a5ae26163be83fda55c585cba7f8ef5 Mon Sep 17 00:00:00 2001 From: Mathivathani Date: Thu, 4 Jul 2024 17:38:27 +0530 Subject: [PATCH 08/10] updated code after handling input formats and output file --- .../README.md | 4 +-- .../chunks_extraction.py | 34 ++++++++++++++++--- .../ingest_data.py | 2 +- .../salesforceData_extraction.py | 23 +++++++------ .../variables.py | 6 ++-- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index b2d36dce..a371636c 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -69,7 +69,7 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit # Input and Output Paths input_path = ".data/input/" output_path = ".data/output/" - ``` + #Proxy URLs for network access proxies= {"http": "PROXY URL","https": "PROXY URL"} @@ -81,7 +81,7 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit #SSL Verification #Set to "False" if SSL Verification is not needed, otherwise set to "True" ssl="" - + ``` 5. **Create and activate a virtual environment**: ```bash diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index d1c04d76..78ea64b6 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -560,13 +560,37 @@ async def extract_chunks(input_html,lastupdatedDate,unique_ArticleIDs_url,**kwar sa_structured_data = list() return sa_structured_data +# async def save_json(output_file_path, store_chunks): +# async with aiofiles.open(output_file_path, 'w') as json_file: +# await json_file.write(json.dumps(store_chunks, indent=2)) +# print("data ingestion started") +# ingest=data_ingestion.ingest_new_data(store_chunks) +# print(ingest) + async def save_json(output_file_path, store_chunks): - async with aiofiles.open(output_file_path, 'w') as json_file: - await json_file.write(json.dumps(store_chunks, indent=2)) - print("data ingestion started") - ingest=data_ingestion.ingest_new_data(store_chunks) - print(ingest) +#starting here + if os.path.exists(output_file_path): + # File exists, read current data + async with aiofiles.open(output_file_path, 'r') as json_file: + file_content = await json_file.read() + if file_content.strip(): + data = json.loads(file_content) + else: + data = [] + print("data",data) + else: + data = [] + data.extend(store_chunks) +#End here + # print(data) + async with aiofiles.open(output_file_path, 'w') as json_file: + await json_file.write(json.dumps(data, indent=2)) + + # Example of data ingestion (replace with your own logic) + print("Data ingestion started") + ingest = data_ingestion.ingest_new_data(store_chunks) + print(ingest) async def helper(input_directory_path, output_directory_path): structure_Data_Count=0 updatedDates={} diff --git a/Extraction/SalesforceCustomExtractionUtility/ingest_data.py b/Extraction/SalesforceCustomExtractionUtility/ingest_data.py index 772713cf..883a7ea6 100644 --- a/Extraction/SalesforceCustomExtractionUtility/ingest_data.py +++ b/Extraction/SalesforceCustomExtractionUtility/ingest_data.py @@ -16,7 +16,7 @@ def ingest_new_data(json_file): 'Content-Type': 'application/json', 'Auth': Auth } - response = requests.post(urls, headers=headers, data=payload,proxies=config.proxies,verify=False) + response = requests.post(urls, headers=headers, data=payload,proxies=config.proxies,verify=bool(config.sslVerify)) if response.status_code == 200: logger.info("data ingested successfully") print("data ingested") diff --git a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py index a8aa2a7a..ed19a1a2 100644 --- a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py @@ -23,7 +23,7 @@ def generate_access_token(): headers = { 'Content-Type': 'application/x-www-form-urlencoded' } - response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=config.sslVerify) + response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=bool(config.sslVerify)) response.raise_for_status() #getting access token and instance_url; @@ -38,6 +38,7 @@ def generate_access_token(): def generate_refresh_token(refresh_token) : try: + # print(config.accessTokenUrl) url = config.accessTokenUrl payload = { 'grant_type': config.refreshTokenGrantType, @@ -48,7 +49,7 @@ def generate_refresh_token(refresh_token) : headers = { 'Content-Type': 'application/x-www-form-urlencoded' } - response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=config.sslVerify) + response = requests.post(url, headers=headers, data=payload,proxies=config.proxies,verify=bool(config.sslVerify)) response.raise_for_status() #getting access token and instance_url; @@ -63,7 +64,7 @@ def generate_refresh_token(refresh_token) : # Make an API call to the specified URL with optional headers. def make_api_call(url, headers=None): try: - response = requests.get(url, headers=headers,proxies=config.proxies,verify=config.sslVerify) + response = requests.get(url, headers=headers,proxies=config.proxies,verify=bool(config.sslVerify)) response.raise_for_status() return response.json() except requests.HTTPError as e: @@ -74,20 +75,20 @@ def make_api_call(url, headers=None): #getting id async def get_articles_list_id(access_token): try: + count=0 if config.inputFormat=="urlnames": urlNames=[] item_ids=[] emptyUrlNames=[] with open('input.csv', 'r') as csvfile: csvreader = csv.reader(csvfile) - for row in csvreader: - print(row) - urlNames.append(row[0]) + for row in csvreader: + # print(row) + urlNames.append(row[0]) print(len(urlNames)) #splitting into batches urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] # print(urlBatches) - count=0 for eachBatch in urlBatches: count=count+1; logger.info(f"API calls for fetching item ids are successful for batch {count} .") @@ -127,6 +128,7 @@ async def get_articles_list_id(access_token): "Accept-Language": "en-US", } lists_response = make_api_call(lists_url, headers) + # print("lists_response",lists_response) item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] list_url_all=[] # print(item_ids) @@ -163,10 +165,11 @@ async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): } try: ssl_context = ssl.create_default_context() - ssl_context.check_hostname = config.sslVerify + ssl_context.check_hostname = bool(config.sslVerify) ssl_context.verify_mode = ssl.CERT_NONE - async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session: - async with session.get(item_url, headers=headers, timeout=20,proxy=config.proxies["http"]) as response: + async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context)) as session: + # async with aiohttp.ClientSession() as session: + async with session.get(item_url, headers=headers, timeout=20,proxies=config.proxies['http']) as response: response.raise_for_status() item_response = await response.json() item_response["url"]= item_url diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index 776fb917..0d46a462 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -4,7 +4,7 @@ load_dotenv() #LLM -llm_used=os.environ("llm_used") +llm_used=os.environ.get("llm_used") # OpenAI Configuration @@ -21,7 +21,7 @@ Apiversion=os.environ.get("Apiversion") # Access Token Configuration -accessTokenUrl = f"https://{os.environ('test_env')}.salesforce.com/services/oauth2/token" +accessTokenUrl = f"https://{os.environ.get('test_env')}.salesforce.com/services/oauth2/token" hostUrl = os.environ.get('hostUrl') accessTokenGrantType = "authorization_code" accessTokenClientId = os.environ.get('clientId') @@ -30,7 +30,7 @@ accessTokenAuthCode =os.environ.get('accessTokenAuthCode') #Refresh Token Configuration -refreshTokenUrl = f"https://{os.environ('test_env')}test.salesforce.com/services/oauth2/token" +refreshTokenUrl = f"https://{os.environ.get('test_env')}.salesforce.com/services/oauth2/token" refreshTokenGrantType = "refresh_token" refreshTokenClientId = os.environ.get('clientId') refreshTokenClientSecret = os.environ.get('clientSecret') From 6a5d1a3cf590332e7136a17ccca095ed68f885db Mon Sep 17 00:00:00 2001 From: Mathivathani Date: Thu, 4 Jul 2024 20:06:39 +0530 Subject: [PATCH 09/10] updated code after suggested changes --- Extraction/SalesforceCustomExtractionUtility/README.md | 4 ++++ .../SalesforceCustomExtractionUtility/chunks_extraction.py | 2 +- Extraction/SalesforceCustomExtractionUtility/variables.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index a371636c..d8312b89 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -81,6 +81,10 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit #SSL Verification #Set to "False" if SSL Verification is not needed, otherwise set to "True" ssl="" + + #If you need the details of specific documents you can enter their article ids or leave it as empty + itemIds="" + ``` 5. **Create and activate a virtual environment**: diff --git a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py index 78ea64b6..932a9c78 100644 --- a/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/chunks_extraction.py @@ -90,7 +90,7 @@ async def fetch_azure_response(self, **request_payload): async with aiohttp.ClientSession() as session: for _ in range(self.max_retries + 1): try: - async with session.post(url, headers=headers, json=request_payload, ssl=False) as response: + async with session.post(url, headers=headers, json=request_payload, verify=bool(config.sslVerify)) as response: if response.status == 200: return await response.json() else: diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index 0d46a462..31aaf39a 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -57,5 +57,5 @@ sslVerify=os.environ.get('ssl') #itemIds -itemIds="" +itemIds=os.environ.get('itemIds') From c83d3c2bcec6276f413db648c8077fb1a87f2473 Mon Sep 17 00:00:00 2001 From: Mathivathani Date: Mon, 29 Jul 2024 16:25:54 +0530 Subject: [PATCH 10/10] "Updated Code with all input formats" --- .../README.md | 11 +- .../emptyUrl.csv | 0 .../input.csv | 0 .../requirements.txt | 3 +- .../salesforceData_extraction.py | 158 +++++++++--------- .../variables.py | 9 +- 6 files changed, 96 insertions(+), 85 deletions(-) create mode 100644 Extraction/SalesforceCustomExtractionUtility/emptyUrl.csv create mode 100644 Extraction/SalesforceCustomExtractionUtility/input.csv diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index d8312b89..063d2f85 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -74,16 +74,17 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit #Proxy URLs for network access proxies= {"http": "PROXY URL","https": "PROXY URL"} - #Specifies whether data fetching uses URL names or Item IDs - #inputFormat="urlnames" or inputFormat="itemids" - inputFormat ="" + + #input type = "articleId" [If we need to fetch data from selected article ids] or "urlNames" [If we need to fetch data using csv file containing UrlNames] or ''[empty incase of fetching the whole data] + inputType="articleId" + + #itemIds format incase of selected article ids ['kA0J1000000oLlhKAE','kA0J1000000oLlOKAU','kA0J1000000oM5LKAU'] or '' incase the article ids are mentioned in csv file + itemIds='' #SSL Verification #Set to "False" if SSL Verification is not needed, otherwise set to "True" ssl="" - #If you need the details of specific documents you can enter their article ids or leave it as empty - itemIds="" ``` diff --git a/Extraction/SalesforceCustomExtractionUtility/emptyUrl.csv b/Extraction/SalesforceCustomExtractionUtility/emptyUrl.csv new file mode 100644 index 00000000..e69de29b diff --git a/Extraction/SalesforceCustomExtractionUtility/input.csv b/Extraction/SalesforceCustomExtractionUtility/input.csv new file mode 100644 index 00000000..e69de29b diff --git a/Extraction/SalesforceCustomExtractionUtility/requirements.txt b/Extraction/SalesforceCustomExtractionUtility/requirements.txt index f5e5db3a..3948201a 100644 --- a/Extraction/SalesforceCustomExtractionUtility/requirements.txt +++ b/Extraction/SalesforceCustomExtractionUtility/requirements.txt @@ -7,4 +7,5 @@ urllib3==1.26.15 python-dotenv requests pandas -aiohttp \ No newline at end of file +aiohttp +ast \ No newline at end of file diff --git a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py index ed19a1a2..54b204a0 100644 --- a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py @@ -7,6 +7,7 @@ import ssl import shutil import csv +import ast from utils.logger import get_logger, write_json_to_separate_file,write_refresh_token logger = get_logger() #access token generation @@ -75,86 +76,91 @@ def make_api_call(url, headers=None): #getting id async def get_articles_list_id(access_token): try: - count=0 - if config.inputFormat=="urlnames": - urlNames=[] - item_ids=[] - emptyUrlNames=[] - with open('input.csv', 'r') as csvfile: - csvreader = csv.reader(csvfile) - for row in csvreader: - # print(row) - urlNames.append(row[0]) - print(len(urlNames)) - #splitting into batches - urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] - # print(urlBatches) - for eachBatch in urlBatches: - count=count+1; - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - for url_name in eachBatch: - - # print(url_name) - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" - # print(lists_url) - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US", - } - listResponse = make_api_call(lists_url,headers) - if listResponse['totalSize']==0 or listResponse['records'] == None : - emptyUrlNames.append(url_name) - - else : - itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] - # print(itemId) - item_ids.append(itemId[0]) - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - # print(f"API calls for fetching item ids are successful for batch {count} .") - emptyUrlNameslist = [[item] for item in emptyUrlNames] - print("Empty URLS",emptyUrlNameslist) + urlNames=[] + item_ids=[] + emptyUrlNames=[] + ItemIds=config.itemIds + # print(expectedItemIds) + if config.inputType=='': + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } - with open('emptyUrl.csv', 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile) - csvwriter.writerows(emptyUrlNameslist) - elif config.inputFormat=="itemids": - expectedItemIds=config.itemIds - if expectedItemIds!="" : - item_ids= expectedItemIds - else: - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US", - } - lists_response = make_api_call(lists_url, headers) - # print("lists_response",lists_response) - item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] - list_url_all=[] - # print(item_ids) - - #getting all the other page urls and the article ids in them - if 'nextRecordsUrl' in lists_response: - for page in lists_response['nextRecordsUrl']: - if lists_response['nextRecordsUrl']: - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US" - } - lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}" - lists_responses = make_api_call(lists_url_2, headers) - item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] - - item_ids=item_ids + item_id_all - lists_response=lists_responses - list_url_all.append(lists_url_2) - - + lists_response = make_api_call(lists_url, headers) + item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] + list_url_all=[] + # print(item_ids) + + #getting all the other page urls and the article ids in them + if 'nextRecordsUrl' in lists_response: + for page in lists_response['nextRecordsUrl']: + if lists_response['nextRecordsUrl']: + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US" + } + lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}" + lists_responses = make_api_call(lists_url_2, headers) + item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] + + item_ids=item_ids + item_id_all + lists_response=lists_responses + list_url_all.append(lists_url_2) + + elif config.inputType=='articleId' and ItemIds!="": + expectedItemIds = ast.literal_eval(ItemIds) + item_ids= expectedItemIds + elif config.inputType=='articleId' and ItemIds=="": + + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + urlNames.append(row[0]) + print(len(urlNames)) + item_ids=urlNames + + elif config.inputType=='urlNames': + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + urlNames.append(row[0]) + print(len(urlNames)) + + urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] + count=0 + for eachBatch in urlBatches: + count=count+1; + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + for url_name in eachBatch: + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" + # print(lists_url) + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } + listResponse = make_api_call(lists_url,headers) + if listResponse['totalSize']==0 or listResponse['records'] == None : + emptyUrlNames.append(url_name) + + else : + itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] + # print(itemId) + item_ids.append(itemId[0]) + # print(f"API calls for fetching item ids are successful for batch {count} .") + emptyUrlNameslist = [[item] for item in emptyUrlNames] + print("Empty URLS",emptyUrlNameslist) + + with open('emptyUrl.csv', 'w', newline='') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerows(emptyUrlNameslist) + # print(item_ids) except Exception as e: logger.error(f"An error occurred while retrieving 'Articles' list ID: {e}", exc_info=True) logger.error(f"API calls for fetching item ids failed at batch {count} .") - + return item_ids #getting details async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): @@ -224,7 +230,9 @@ async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,pr #salesforce data extraction async def extractData(): try: + access_token,instance_url,refresh_token = generate_access_token() + logger.info("Access token generated successfully.") logger.debug(f"Access token: {access_token}") diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index 31aaf39a..8e513ac6 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -49,13 +49,14 @@ #proxies proxies=os.environ.get('proxies') - -#Fetching definiton -inputFormat=os.environ.get('inputFormat') - #set SSL verfication sslVerify=os.environ.get('ssl') +#input type +inputType=os.environ.get('inputType') + #itemIds itemIds=os.environ.get('itemIds') + +