HandTrackerCode/main.py at main · Kenneth-Choothakan/HandTrackerCode · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import cv2
import mediapipe as mp
import tkinter as tk

from Movements.Rotate import pinch2Rotate
from Movements.WheelMove import WheelMove
from Movements.PointAndClick import pointClick

from Mousing.MouseHover import mouseHover
from Mousing.LeftClick import leftClick
from Mousing.RightClick import rightClick

import keyboard
import speech_recognition as sr
import soundfile as sf
import sounddevice as sd
import os


#Get measurements of monitor
root = tk.Tk()
screen_width = root.winfo_screenwidth()
screen_height = root.winfo_screenheight()
root.destroy()

#Video capture stuff
cap = cv2.VideoCapture(0)
mpHands = mp.solutions.hands
hands = mpHands.Hands(static_image_mode=False, max_num_hands=1, min_detection_confidence=0.7)
mp_draw = mp.solutions.drawing_utils

if not cap.isOpened():
    print("Error: Could not open camera.")
    exit()

print("Camera opened successfully.")

jarvis = sr.Recognizer()

def findDistance(distance1, distance2, distance3, distance4):
    return (distance1**2 + distance2**2) / (distance3**2 + distance4**2)

start = True
signing = False
modes = ["Movement", "Mousing"]
modeI = 0
mode = modes[0]
def checkMode(hand_landmarks):
    global modes
    global signing
    global modeI
    global mode
    dx_8_12 = hand_landmarks.landmark[8].x - hand_landmarks.landmark[12].x
    dy_8_12 = hand_landmarks.landmark[8].y - hand_landmarks.landmark[12].y
    dx_8_5 = hand_landmarks.landmark[8].x - hand_landmarks.landmark[5].x
    dy_8_5 = hand_landmarks.landmark[8].y - hand_landmarks.landmark[5].y

    dx_4_14 = hand_landmarks.landmark[4].x - hand_landmarks.landmark[14].x
    dy_4_14 = hand_landmarks.landmark[4].y - hand_landmarks.landmark[14].y
    dx_4_15 = hand_landmarks.landmark[4].x - hand_landmarks.landmark[15].x
    dy_4_15 = hand_landmarks.landmark[4].y - hand_landmarks.landmark[15].y

    dy_12_9 = hand_landmarks.landmark[12].y - hand_landmarks.landmark[9].y
    distance_ratio_barrel = findDistance(dx_8_12,dy_8_12,dx_8_5,dy_8_5)
    distance_ratio_4_14 = findDistance(dx_4_14,dy_4_14,dx_4_15,dy_4_15)

    if distance_ratio_barrel < 0.2 and distance_ratio_4_14 < 0.3 and dy_8_5 < 0 and dy_12_9 < 0:
        if signing == False:
            if (modeI < len(modes)-1):
                modeI += 1
            else:
                modeI = 0
            mode = modes[modeI]
            signing = True
    else:
        signing = False


while True:
    success, img = cap.read()
    if not success:
        break

    img = cv2.flip(img, 1)
    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    results = hands.process(img_rgb)
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            mp_draw.draw_landmarks(img, hand_landmarks, mpHands.HAND_CONNECTIONS)

            checkMode(hand_landmarks)

            if mode == "Movement":
                pinch2Rotate(hand_landmarks, screen_width, screen_height)
                WheelMove(hand_landmarks, screen_width, screen_height)
                #pointClick(hand_landmarks, screen_width, screen_height)

            if mode == "Mousing":
                mouseHover(hand_landmarks, screen_width, screen_height)
                leftClick(hand_landmarks)
                rightClick(hand_landmarks)

    cv2.imshow("Image", img)

    if (cv2.waitKey(1) and keyboard.is_pressed("ESC")):
        break


cap.release()
cv2.destroyAllWindows()