diff --git a/.gitignore b/.gitignore index e43b0f9..be47843 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .DS_Store +.env diff --git a/ASL word gestures/main.py b/ASL_word_gestures/main.py similarity index 100% rename from ASL word gestures/main.py rename to ASL_word_gestures/main.py diff --git a/ASL word gestures/modeltrainer.py b/ASL_word_gestures/modeltrainer.py similarity index 100% rename from ASL word gestures/modeltrainer.py rename to ASL_word_gestures/modeltrainer.py diff --git a/ASL word gestures/proto1.py b/ASL_word_gestures/proto1.py similarity index 100% rename from ASL word gestures/proto1.py rename to ASL_word_gestures/proto1.py diff --git a/deps.txt b/deps.txt index 93728a9..eb20b28 100644 --- a/deps.txt +++ b/deps.txt @@ -6,3 +6,4 @@ tensorflow numpy comtypes pandas +pyspellchecker==0.7.2 diff --git a/prototype 1/main.py b/prototype_1/main.py similarity index 100% rename from prototype 1/main.py rename to prototype_1/main.py diff --git a/prototype 1/proto1/proto1/proto1.ino b/prototype_1/proto1/proto1/proto1.ino similarity index 100% rename from prototype 1/proto1/proto1/proto1.ino rename to prototype_1/proto1/proto1/proto1.ino diff --git a/prototype 2/AI model + Prediction/ai_trainer.py b/prototype_2/AI_model_and_Prediction/ai_trainer.py similarity index 100% rename from prototype 2/AI model + Prediction/ai_trainer.py rename to prototype_2/AI_model_and_Prediction/ai_trainer.py diff --git a/prototype 2/AI model + Prediction/asl_cnn_model_rel.h5 b/prototype_2/AI_model_and_Prediction/asl_cnn_model_rel.h5 similarity index 100% rename from prototype 2/AI model + Prediction/asl_cnn_model_rel.h5 rename to prototype_2/AI_model_and_Prediction/asl_cnn_model_rel.h5 diff --git a/prototype 2/AI model + Prediction/testmodel.py b/prototype_2/AI_model_and_Prediction/testmodel.py similarity index 64% rename from prototype 2/AI model + Prediction/testmodel.py rename to prototype_2/AI_model_and_Prediction/testmodel.py index d9fe1ca..e9f21b8 100644 --- a/prototype 2/AI model + Prediction/testmodel.py +++ b/prototype_2/AI_model_and_Prediction/testmodel.py @@ -6,8 +6,13 @@ import queue import platform import subprocess +from spellchecker import SpellChecker +import os +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +spell = SpellChecker() +current_word = ""; -model = tf.keras.models.load_model("asl_cnn_model_rel.h5") +model = tf.keras.models.load_model(os.path.join(BASE_DIR, "../AI_model_and_Prediction/asl_cnn_model_rel.h5")) class_names = [ 'A','B','C','D','E','F','G','H','I','J','K','L','M', @@ -15,6 +20,14 @@ 'del','nothing','space' ] +def auto_correct(word): + # words like i and a: + if len(word) <= 1: + return word + # correcting the letters + corrected = spell.correction(word) + return corrected if corrected else word + lastspoken = None label_lock = threading.Lock() landmark_queue = queue.Queue(maxsize=5) @@ -26,23 +39,36 @@ single_lower_limit_index = double_letter_frames-first_letter_frames upper_limit_index = double_letter_frames + 1 -def speak(label): +def speak(text): system = platform.system() if system == "Darwin": - subprocess.Popen(["say", label]) + subprocess.Popen(["say", text]) elif system == "Windows": import comtypes.client speaker = comtypes.client.CreateObject("SAPI.SpVoice") - speaker.Speak(label) + speaker.Speak(text) else: - subprocess.Popen(["spd-say", label]) + subprocess.Popen(["spd-say", text]) + +def process_label(label): + global current_word + if label == "space": + if current_word != "": + corrected = auto_correct(current_word) + speak(corrected) + print("Typed:", current_word, "Corrected:", corrected) + current_word = "" + elif label == "del": + current_word = current_word[:-1] + elif label != "nothing": + current_word += label.lower() BaseOptions = mp.tasks.BaseOptions HandLandmarker = mp.tasks.vision.HandLandmarker HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions VisionRunningMode = mp.tasks.vision.RunningMode -MODEL_PATH = "../hand_landmarker.task" +MODEL_PATH = os.path.join(BASE_DIR, "../hand_landmarker.task") def hand_callback(result, output_image, timestamp_ms): if not result.hand_landmarks: @@ -93,17 +119,20 @@ def hand_callback(result, output_image, timestamp_ms): labelbuffer.pop(0) if len(labelbuffer) < upper_limit_index: continue - elif len(set(labelbuffer[single_lower_limit_index:upper_limit_index])) == 1 and labelbuffer[single_lower_limit_index] != labelbuffer[single_lower_limit_index - 1]: + stablelabel = labelbuffer[double_letter_frames] + if len(set(labelbuffer[single_lower_limit_index:upper_limit_index])) == 1 and labelbuffer[single_lower_limit_index] != labelbuffer[single_lower_limit_index - 1]: if lastspoken != labelbuffer[double_letter_frames]: - speak(labelbuffer[double_letter_frames]) - lastspoken = labelbuffer[double_letter_frames] + process_label(stablelabel) + lastspoken = stablelabel elif len(set(labelbuffer[1:upper_limit_index])) == 1 and labelbuffer[1] != labelbuffer[0]: # allow a duplicate letter here - speak(labelbuffer[double_letter_frames]) - lastspoken = labelbuffer[double_letter_frames] + process_label(stablelabel) + lastspoken = stablelabel except queue.Empty: pass - + cv2.putText(frame, current_word, (50,50), + cv2.FONT_HERSHEY_SIMPLEX, 1, + (0,255,0), 2) cv2.imshow("Hand Sign Prediction", frame) if cv2.waitKey(5) & 0xFF == 27: break diff --git a/prototype 2/AI model + Prediction/tfjs_convertor.py b/prototype_2/AI_model_and_Prediction/tfjs_convertor.py similarity index 100% rename from prototype 2/AI model + Prediction/tfjs_convertor.py rename to prototype_2/AI_model_and_Prediction/tfjs_convertor.py diff --git a/prototype 2/MegaDataset/DataSetMaker.py b/prototype_2/MegaDataset/DataSetMaker.py similarity index 99% rename from prototype 2/MegaDataset/DataSetMaker.py rename to prototype_2/MegaDataset/DataSetMaker.py index cdcdd7f..d8606c5 100644 --- a/prototype 2/MegaDataset/DataSetMaker.py +++ b/prototype_2/MegaDataset/DataSetMaker.py @@ -60,7 +60,7 @@ def UpdateLabelsAndOrientation(): global target_label, target_orientation try: target_label = letterhash[input("Enter target label: ").upper()] - except KeyError: + except: print("Invalid label, might be on Tryjobs, if not on TryJobs, restart the program and give it another try.") quit() target_orientation = int(input( diff --git a/prototype 2/MegaDataset/Important info.html b/prototype_2/MegaDataset/Important info.html similarity index 100% rename from prototype 2/MegaDataset/Important info.html rename to prototype_2/MegaDataset/Important info.html diff --git a/prototype 2/MegaDataset/hand_landmarker.task b/prototype_2/MegaDataset/hand_landmarker.task similarity index 100% rename from prototype 2/MegaDataset/hand_landmarker.task rename to prototype_2/MegaDataset/hand_landmarker.task diff --git a/prototype 2/hand_landmarker.task b/prototype_2/hand_landmarker.task similarity index 100% rename from prototype 2/hand_landmarker.task rename to prototype_2/hand_landmarker.task diff --git a/prototype 2/main.py b/prototype_2/main.py similarity index 100% rename from prototype 2/main.py rename to prototype_2/main.py diff --git a/prototype 2/mediapipe_handtracking_demo/mediapipeht.py b/prototype_2/mediapipe_handtracking_demo/mediapipeht.py similarity index 100% rename from prototype 2/mediapipe_handtracking_demo/mediapipeht.py rename to prototype_2/mediapipe_handtracking_demo/mediapipeht.py diff --git a/prototype 3/Gerber_Project_Hearthstone_Speaker_PCB_Project_Hearthstone_Speaker_2026-03-13.zip b/prototype_3/Gerber_Project_Hearthstone_Speaker_PCB_Project_Hearthstone_Speaker_2026-03-13.zip similarity index 100% rename from prototype 3/Gerber_Project_Hearthstone_Speaker_PCB_Project_Hearthstone_Speaker_2026-03-13.zip rename to prototype_3/Gerber_Project_Hearthstone_Speaker_PCB_Project_Hearthstone_Speaker_2026-03-13.zip diff --git a/prototype 3/main.py b/prototype_3/main.py similarity index 100% rename from prototype 3/main.py rename to prototype_3/main.py