Add --force option to bypass fingerprint check during import

Author: tacxou

Makefile

@@ -7,6 +7,7 @@ APP_NAME = "sesame-taiga_crawler"
 PLATFORM = "linux/amd64"
 STC_RUN = "all"
 STC_IMPORTS = "all"
+STC_FORCE = "0"
 include .env
 
 .DEFAULT_GOAL := help
@@ -30,7 +31,7 @@ run-crawler-docker: ## Lance le crawler Sesame - Taiga avec python !
 		--platform $(PLATFORM) \
 		--name $(APP_NAME) \
 		-v $(CURDIR):/data \
-		$(IMG_NAME) python main.py --run=$(STC_RUN) --imports=$(STC_IMPORTS)
+		$(IMG_NAME) python main.py --run=$(STC_RUN) --imports=$(STC_IMPORTS) --force $(STC_FORCE)
 
 run-crawler: ## Lance le crawler Sesame - Taiga avec python !
 	@python3 main.py

main.py

@@ -118,7 +118,8 @@ async def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--run', help='all | taiga | sesame',default='all')
     parser.add_argument('--imports', help='all | ind | pictures',default='all')
-    parser.add_argument('--an', help='Année universitaire à importer ',default="0")
+    parser.add_argument('--an', help='Année universitaire à importer',default="0")
+    parser.add_argument('--force', help="Force l'import et bypass le check du fingerprint",default="0")
     args = parser.parse_args()
     if args.an != 0:
         print(f"Import pour l'annee {args.an}")
@@ -148,7 +149,7 @@ async def main():
         print(f"Imports: {args.imports}")
         start_time = datetime.now()
         if (args.imports == 'ind' or args.imports == 'all'):
-            await import_ind()
+            await import_ind(args.force)
         if (args.imports == 'pictures' or args.imports == 'all'):
             await import_pictures()
         end_time = datetime.now()

src/import_ind.py

@@ -28,7 +28,7 @@ async def read_response(response):
     jsonMessage = json.loads(message)
     print(jsonMessage)
 
-async def send_request(session, url,exclusions, json):
+async def send_request(session, url,exclusions, json, force):
     if (json.get('inetOrgPerson', {}).get('employeeNumber') == None and json.get('$setOnInsert', {}).get('inetOrgPerson', {}).get('employeeNumber') == None):
         print(f"MISSING employeeNumber -> $set: {json.get('inetOrgPerson', {})}, $setOnInsert: {json.get('$setOnInsert', {}).get('inetOrgPerson', {})}")
         return
@@ -40,6 +40,7 @@ async def send_request(session, url,exclusions, json):
     params = {
     #     "filters[inetOrgPerson.employeeNumber]": f"{json.get('inetOrgPerson', {}).get('employeeNumber')}",
         "filters[inetOrgPerson.employeeType]": "TAIGA",
+        "force": force,
     }
 
     employeeNumber = json.get('inetOrgPerson', {}).get('employeeNumber') or json.get('$setOnInsert', {}).get('inetOrgPerson', {}).get('employeeNumber')
@@ -91,21 +92,21 @@ async def get_data(data, config):
 
 
 
-async def process_data(data, config, file, session):
+async def process_data(data, config, file, session, force):
     print(f"Processing {file}")
     result = await get_data(data, config)
     with open(f'./data/{file}', 'w', encoding='utf-8') as fichier:
         json.dump(result, fichier, ensure_ascii=False, indent=4)
     exclude=config.get('exclude',[])
-    tasks = [send_request(session, f'{sesame_api_baseurl}/management/identities/upsert',config.get('exclude',[]),entry) for entry in result]
+    tasks = [send_request(session, f'{sesame_api_baseurl}/management/identities/upsert',config.get('exclude',[]),entry, force) for entry in result]
     await gather_with_concurrency(sesame_import_parallels_files, tasks)
     print(f"Processed {file}")
 
 async def load_config():
     with open('./config.yml', 'r', encoding='utf-8') as fichier:
         return yaml.load(fichier, Loader=yaml.FullLoader)
 
-async def import_ind():
+async def import_ind(force: bool):
     configs = await load_config()
     cache_files = os.listdir('./cache')
     datas = {}
@@ -116,5 +117,5 @@ async def import_ind():
               datas[file] = json.load(fichier).get('data')
 
     async with aiohttp.ClientSession() as session:
-        tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files if file in configs.keys()]
+        tasks = [process_data(datas[file], configs[file], file, session, force) for file in cache_files if file in configs.keys()]
         await gather_with_concurrency(sesame_import_parallels_files, tasks)