Skip to content

Improve remote screen streaming sync #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ajeybhat47 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Improve remote screen streaming sync #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 29 additions & 9 deletions client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,17 @@

import psutil
from scapy.all import ARP, Ether, srp
import time

# define the MAC address of the device you're looking for
target_mac = "00:00:00:00:00:00"
target_mac2 = "00:00:00:00:00:00"

HOST = 'localhost'

# these will be updated once we connect to the server
SERVER_WIDTH = 1920
SERVER_HEIGHT = 1080

with os.popen('arp -a') as f:
Arp_data = f.read()
Arp_table = []
Expand Down Expand Up @@ -83,6 +86,14 @@
s.connect((HOST, PORT))
print('Connected to server')

# receive screen resolution from server
width_bytes = s.recv(4)
height_bytes = s.recv(4)
if width_bytes and height_bytes:
SERVER_WIDTH = int.from_bytes(width_bytes, byteorder='big')
SERVER_HEIGHT = int.from_bytes(height_bytes, byteorder='big')
print(f'Server screen size: {SERVER_WIDTH}x{SERVER_HEIGHT}')


STOP_BUTTON = 0
Button_state = 0 # 0-no click 1-left click 2-right click
Expand All @@ -101,6 +112,7 @@ def send_mouse_coords(conn):

global STOP_BUTTON
global Button_state
global SERVER_WIDTH, SERVER_HEIGHT
global win_x
global win_y
global win_width
Expand All @@ -115,10 +127,10 @@ def send_mouse_coords(conn):
x, y = pyautogui.position()

try:
msg = "|"+str((x-win_x)/(win_width/1920))+":" + \
str((y-win_y)/(win_height/1080))+":"+str(Button_state)+"|"
# print(msg)
except:
remote_x = (x - win_x) * SERVER_WIDTH / win_width
remote_y = (y - win_y) * SERVER_HEIGHT / win_height
msg = f"|{remote_x}:{remote_y}:{Button_state}|"
except Exception:
pass
# print(msg.encode())
s.sendall(msg.encode())
Expand All @@ -131,13 +143,15 @@ def send_mouse_coords(conn):
def clicking(conn):
global STOP_BUTTON
global Button_state
global SERVER_WIDTH, SERVER_HEIGHT

if STOP_BUTTON:
return

def on_click(x, y, button, pressed):
global STOP_BUTTON
global Button_state
global SERVER_WIDTH, SERVER_HEIGHT

if STOP_BUTTON:
return
Expand All @@ -159,10 +173,11 @@ def on_click(x, y, button, pressed):
Button_state = 0

try:
msg = "|"+str((x-win_x)/(win_width/1920))+":" + \
str((y-win_y)/(win_height/1080))+":"+str(Button_state)+"|"
remote_x = (x - win_x) * SERVER_WIDTH / win_width
remote_y = (y - win_y) * SERVER_HEIGHT / win_height
msg = f"|{remote_x}:{remote_y}:{Button_state}|"
print(msg)
except:
except Exception:
pass
# print(msg.encode())
s.sendall(msg.encode())
Expand All @@ -177,12 +192,14 @@ def on_click(x, y, button, pressed):
def receive_video_data(conn):
# Create a buffer to hold the received video data
global STOP_BUTTON
global SERVER_WIDTH, SERVER_HEIGHT

global win_x
global win_y
global win_width
global win_height
cv2.namedWindow('Screen', cv2.WINDOW_NORMAL)
cv2.resizeWindow('Screen', SERVER_WIDTH, SERVER_HEIGHT)

win_x = cv2.getWindowImageRect("Screen")[0]
win_y = cv2.getWindowImageRect("Screen")[1]
Expand Down Expand Up @@ -215,8 +232,11 @@ def receive_video_data(conn):
img = cv2.imdecode(np.frombuffer(
buffer, dtype=np.uint8), cv2.IMREAD_COLOR)

# Resize to current window size for smoother display
display_img = cv2.resize(img, (win_width, win_height))

# Display the image
cv2.imshow('Screen', img)
cv2.imshow('Screen', display_img)
# cv2.waitKey(1)

if cv2.waitKey(1) & 0xFF == ord('q'):
Expand Down
62 changes: 40 additions & 22 deletions server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@
HOST = '0.0.0.0' # IP address of the server
PORT = 8000 # Port to listen on

# default screen dimensions will be updated after connection
SCREEN_WIDTH = 1920
SCREEN_HEIGHT = 1080


# pyautogui.FAILSAFE = False
pyautogui.PAUSE = 0
Expand All @@ -32,11 +36,21 @@
conn, addr = s.accept()
print(f'Connected by {addr}')

# Determine screen resolution and send it to the client
with mss.mss() as monitor:
MON_INFO = monitor.monitors[1]
SCREEN_WIDTH = MON_INFO['width']
SCREEN_HEIGHT = MON_INFO['height']

conn.sendall(SCREEN_WIDTH.to_bytes(4, byteorder='big'))
conn.sendall(SCREEN_HEIGHT.to_bytes(4, byteorder='big'))

# Create a monitor instance for screen recording


def send_screen(conn):

global SCREEN_WIDTH, SCREEN_HEIGHT
with mss.mss() as monitor:

while True:
Expand All @@ -47,7 +61,9 @@ def send_screen(conn):
img = np.array(screenshot)

# Convert the image to JPEG format for compression
encoded, buffer = cv2.imencode('.jpg', img)
# reduce quality slightly to improve bandwidth usage
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 50]
encoded, buffer = cv2.imencode('.jpg', img, encode_param)

# Send the size of the JPEG buffer to the client
size = len(buffer)
Expand All @@ -58,35 +74,37 @@ def send_screen(conn):


def receive_mouse_input(conn):
global SCREEN_WIDTH, SCREEN_HEIGHT
buffer = ''
while True:
# receive mouse input coordinates from client
data = conn.recv(1024).decode()
if not data:
break
print(data)
data = data.split("|")
data = data[1]
try:
x, y, z = data.split(':')
x = float(x)
y = float(y)
z = int(z)
if x > 0 and x < 1920:
if y > 0 and y < 1080:
pass
print("moved to", x, y)
buffer += data

while '|' in buffer:
start = buffer.find('|')
end = buffer.find('|', start + 1)
if end == -1:
break
chunk = buffer[start + 1:end]
buffer = buffer[end + 1:]
try:
x, y, z = chunk.split(':')
x = float(x)
y = float(y)
z = int(z)
if 0 <= x <= SCREEN_WIDTH and 0 <= y <= SCREEN_HEIGHT:
pyautogui.moveTo(round(x), round(y))
if (z == 0):
if z == 0:
pyautogui.mouseUp(button="left")
# pyautogui.mouseUp(button="right")
elif (z == 1):
# print("click")
elif z == 1:
pyautogui.mouseDown(button="left")
elif (z == 2):
pyautogui.click(
button="right", clicks=1, interval=0.25)
except:
print("error in coordinates")
elif z == 2:
pyautogui.click(button="right", clicks=1, interval=0.25)
except Exception as e:
print("error in coordinates", e)


# function to handle client connections
Expand Down