Updated code with all the input formats

Extraction/SalesforceCustomExtractionUtility/README.md

@@ -74,16 +74,17 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit
     #Proxy URLs for network access
     proxies= {"http": "PROXY URL","https": "PROXY URL"}
 
-    #Specifies whether data fetching uses URL names or Item IDs
-    #inputFormat="urlnames" or inputFormat="itemids"
-    inputFormat =""
+
+    #input type = "articleId" [If we need to fetch data from selected article ids] or "urlNames" [If we need to fetch data using csv file containing UrlNames] or ''[empty incase of fetching the whole data]
+    inputType="articleId"
+
+    #itemIds format incase of selected article ids ['kA0J1000000oLlhKAE','kA0J1000000oLlOKAU','kA0J1000000oM5LKAU'] or '' incase the article ids are mentioned in csv file
+    itemIds=''
 
     #SSL Verification
     #Set to "False" if SSL Verification is not needed, otherwise set to "True"
     ssl=""
 
-    #If you need the details of specific documents you can enter their article ids or leave it as empty
-    itemIds=""
 
     ```
 

Extraction/SalesforceCustomExtractionUtility/requirements.txt

@@ -7,4 +7,5 @@ urllib3==1.26.15
 python-dotenv
 requests
 pandas
-aiohttp
+aiohttp
+ast

Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py

@@ -7,6 +7,7 @@
 import ssl
 import shutil
 import csv
+import ast
 from utils.logger import get_logger, write_json_to_separate_file,write_refresh_token
 logger = get_logger()
 #access token generation
@@ -75,86 +76,91 @@ def make_api_call(url, headers=None):
 #getting id 
 async def get_articles_list_id(access_token):
     try:
-        count=0
-        if config.inputFormat=="urlnames":
-            urlNames=[]
-            item_ids=[]
-            emptyUrlNames=[]
-            with open('input.csv', 'r') as csvfile:
-             csvreader = csv.reader(csvfile)
-             for row in csvreader:
-                #  print(row)
-                 urlNames.append(row[0]) 
-            print(len(urlNames))
-            #splitting into batches
-            urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))]
-            # print(urlBatches)
-            for eachBatch in urlBatches:
-                count=count+1;
-            logger.info(f"API calls for fetching item ids are successful for batch {count} .")
-            for url_name in eachBatch:
-
-                # print(url_name)
-                lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" 
-                # print(lists_url)
-                headers = {
-                    "Authorization": f"Bearer {access_token}",
-                    "Accept-Language": "en-US",
-                    }
-                listResponse =   make_api_call(lists_url,headers)
-                if listResponse['totalSize']==0 or listResponse['records'] == None :
-                    emptyUrlNames.append(url_name)
-
-                else :
-                    itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])]
-                    #  print(itemId)
-                    item_ids.append(itemId[0])
-            logger.info(f"API calls for fetching item ids are successful for batch {count} .")
-            # print(f"API calls for fetching item ids are successful for batch {count} .")
-            emptyUrlNameslist = [[item] for item in emptyUrlNames]
-            print("Empty URLS",emptyUrlNameslist)  
+        urlNames=[]
+        item_ids=[]
+        emptyUrlNames=[]
+        ItemIds=config.itemIds
+        # print(expectedItemIds)
+        if config.inputType=='':
+            lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" 
+            headers = {
+                   "Authorization": f"Bearer {access_token}",
+                   "Accept-Language": "en-US",
+                  }
 
-            with open('emptyUrl.csv', 'w', newline='') as csvfile:
-                csvwriter = csv.writer(csvfile)
-                csvwriter.writerows(emptyUrlNameslist)
-        elif config.inputFormat=="itemids":
-            expectedItemIds=config.itemIds
-            if expectedItemIds!="" :
-                item_ids= expectedItemIds 
-            else:
-                lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" 
-                headers = {
-                    "Authorization": f"Bearer {access_token}",
-                    "Accept-Language": "en-US",
-                    }
-                lists_response = make_api_call(lists_url, headers)
-                # print("lists_response",lists_response)
-                item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])]
-                list_url_all=[]
-                # print(item_ids)
-
-                #getting all the other page urls and the article ids in them
-                if 'nextRecordsUrl' in lists_response:
-                    for page in lists_response['nextRecordsUrl']:
-                        if lists_response['nextRecordsUrl']:
-                            headers = {
-                                "Authorization": f"Bearer {access_token}",
-                                "Accept-Language": "en-US"
-                                }
-                            lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}"
-                            lists_responses = make_api_call(lists_url_2, headers)
-                            item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])]
-
-                            item_ids=item_ids + item_id_all
-                            lists_response=lists_responses
-                            list_url_all.append(lists_url_2)
-
-
+            lists_response = make_api_call(lists_url, headers)
+            item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])]
+            list_url_all=[]
+            # print(item_ids)
+
+            #getting all the other page urls and the article ids in them
+            if 'nextRecordsUrl' in lists_response:
+                for page in lists_response['nextRecordsUrl']:
+                    if lists_response['nextRecordsUrl']:
+                        headers = {
+                            "Authorization": f"Bearer {access_token}",
+                            "Accept-Language": "en-US"
+                            }
+                        lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}"
+                        lists_responses = make_api_call(lists_url_2, headers)
+                        item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])]
+
+                        item_ids=item_ids + item_id_all
+                        lists_response=lists_responses
+                        list_url_all.append(lists_url_2)
+
+        elif config.inputType=='articleId' and ItemIds!="":
+         expectedItemIds = ast.literal_eval(ItemIds)
+         item_ids= expectedItemIds 
+        elif config.inputType=='articleId' and ItemIds=="":
+
+          with open('input.csv', 'r') as csvfile:
+           csvreader = csv.reader(csvfile)
+           for row in csvreader:
+            urlNames.append(row[0]) 
+          print(len(urlNames))
+          item_ids=urlNames
+
+        elif config.inputType=='urlNames':
+          with open('input.csv', 'r') as csvfile:
+           csvreader = csv.reader(csvfile)
+           for row in csvreader:
+            urlNames.append(row[0]) 
+          print(len(urlNames))
+
+          urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))]
+          count=0
+          for eachBatch in urlBatches:
+           count=count+1;
+           logger.info(f"API calls for fetching item ids are successful for batch {count} .")
+           for url_name in eachBatch:
+            lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" 
+            # print(lists_url)
+            headers = {
+                   "Authorization": f"Bearer {access_token}",
+                   "Accept-Language": "en-US",
+                  }
+            listResponse =   make_api_call(lists_url,headers)
+            if listResponse['totalSize']==0 or listResponse['records'] == None :
+                  emptyUrlNames.append(url_name)
+
+            else :
+                 itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])]
+                #  print(itemId)
+                 item_ids.append(itemId[0])
+        # print(f"API calls for fetching item ids are successful for batch {count} .")
+        emptyUrlNameslist = [[item] for item in emptyUrlNames]
+        print("Empty URLS",emptyUrlNameslist)  
+
+        with open('emptyUrl.csv', 'w', newline='') as csvfile:
+         csvwriter = csv.writer(csvfile)
+         csvwriter.writerows(emptyUrlNameslist)
+
         # print(item_ids)
     except Exception as e:
         logger.error(f"An error occurred while retrieving 'Articles' list ID: {e}", exc_info=True)
         logger.error(f"API calls for fetching item ids failed at batch {count} .")
-
+    
     return item_ids
 #getting details
 async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url):
@@ -224,6 +230,7 @@ async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,pr
 #salesforce data extraction
 async def extractData():
     try:
+
         access_token,instance_url,refresh_token = generate_access_token()
         logger.info("Access token generated successfully.")
         logger.debug(f"Access token: {access_token}")

Extraction/SalesforceCustomExtractionUtility/variables.py

@@ -49,13 +49,14 @@
 
 #proxies
 proxies=os.environ.get('proxies')
-
-#Fetching definiton
-inputFormat=os.environ.get('inputFormat')
-
 #set SSL verfication
 sslVerify=os.environ.get('ssl')
 
+#input type 
+inputType=os.environ.get('inputType') 
+
 #itemIds
 itemIds=os.environ.get('itemIds')
 
+
+