How to get touch coordinates?

[zhuraromdev]

Hello, how can I store coordinates of mouse?

I was trying to make like this, however numbers seems to be wrong:

def get_touch_coordinates():
    touch_coordinates = []
    process = subprocess.Popen(['adb', 'shell', 'getevent', '-lt'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    for line in process.stdout:
        line = line.decode('utf-8').strip()
        if 'ABS_MT_POSITION_X' in line or 'ABS_MT_POSITION_Y' in line:
            coords = line.split()[-1]
            touch_coordinates.append(int(coords, 16))
            if len(touch_coordinates) == 2:
                x, y = touch_coordinates
                touch_coordinates = []
                return x, y

Example of output:

timestamp,event_type,x,y,button,screenshot_path,screen_width,screen_height,application_name,trajectory
2024-08-06_14-49-11_281052,click,3623,8631,,and/screenshot_2024-08-06_14-49-11_281052.png,2208,1840,and,
2024-08-06_14-49-13_687736,click,4007,9671,,and/screenshot_2024-08-06_14-49-13_687736.png,2208,1840,and,
2024-08-06_14-49-16_482075,click,4167,5807,,and/screenshot_2024-08-06_14-49-16_482075.png,2208,1840,and,
2024-08-06_14-49-19_139051,click,2807,9719,,and/screenshot_2024-08-06_14-49-19_139051.png,2208,1840,and,
2024-08-06_14-49-21_497052,click,3839,13479,,and/screenshot_2024-08-06_14-49-21_497052.png,2208,1840,and,

Also sending the full code, for better understanding:

from PIL import Image, ImageDraw
import datetime
import os
import pandas as pd
import threading
import adbutils
import subprocess

# Connect to Android emulator
def connect_device():
    adb = adbutils.AdbClient(host="127.0.0.1", port=5037)
    devices = adb.device_list()
    if not devices:
        raise adbutils.errors.AdbError("Can't find any android device/emulator")
    return devices[0]

try:
    device = connect_device()
except adbutils.errors.AdbError as e:
    print(e)
    print("Ensure your emulator is running and connected.")
    exit(1)

# Initialize a DataFrame to store logs
df_columns = ["timestamp", "event_type", "x", "y", "button", "screenshot_path", "screen_width", "screen_height", "application_name", "trajectory"]
df = pd.DataFrame(columns=df_columns)

# Function to get touch coordinates from the emulator
def get_touch_coordinates():
    touch_coordinates = []
    process = subprocess.Popen(['adb', 'shell', 'getevent', '-lt'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    for line in process.stdout:
        line = line.decode('utf-8').strip()
        if 'ABS_MT_POSITION_X' in line or 'ABS_MT_POSITION_Y' in line:
            coords = line.split()[-1]
            touch_coordinates.append(int(coords, 16))
            if len(touch_coordinates) == 2:
                x, y = touch_coordinates
                touch_coordinates = []
                return x, y

# Function to get the screen size of the emulator
def get_screen_size():
    output = subprocess.check_output(['adb', 'shell', 'wm', 'size']).decode('utf-8')
    dimensions = output.split()[-1]
    width, height = map(int, dimensions.split('x'))
    return width, height

# Function to log the event and take a screenshot from Android emulator
def log_and_screenshot(event_type, x, y, button=None):
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S_%f")
    screenshot_path = os.path.join(screenshot_dir, f"screenshot_{timestamp}.png")

    # Capture screenshot from Android emulator
    temp_screenshot_path = "/sdcard/screenshot.png"
    device.shell(['screencap', '-p', temp_screenshot_path])
    device.sync.pull(temp_screenshot_path, screenshot_path)
    device.shell(['rm', temp_screenshot_path])

    screenshot = Image.open(screenshot_path)
    draw = ImageDraw.Draw(screenshot)
    cursor_size = 10

    # Scale the raw touch coordinates to match the screen size
    x = int(x * screen_width / 4096)  # Adjust based on your device resolution and maximum coordinate range
    y = int(y * screen_height / 4096)

    # Draw the cursor position with a green cross
    draw.line((x - cursor_size, y, x + cursor_size, y), fill="green", width=2)
    draw.line((x, y - cursor_size, x, y + cursor_size), fill="green", width=2)

    screenshot.save(screenshot_path, format='PNG')

    # Log the event
    event_data = {
        "timestamp": timestamp,
        "event_type": event_type,
        "x": x,
        "y": y,
        "button": button,
        "screenshot_path": screenshot_path,
        "screen_width": screen_width,
        "screen_height": screen_height,
        "application_name": application_name,
        "trajectory": ""
    }
    global df
    df = pd.concat([df, pd.DataFrame([event_data])], ignore_index=True)
    df.to_csv(log_file_csv, index=False)

    with open(log_file_txt, "a") as f:
        f.write(f"{timestamp} - {event_type} - ({x}, {y}) - {button} - {screenshot_path} - {screen_width}x{screen_height} - {application_name}\n")

# Function to listen for touch events and log them
def listen_for_touch_events():
    while True:
        coordinates = get_touch_coordinates()
        if coordinates:
            x, y = coordinates
            log_and_screenshot("click", x, y)

# Main function to handle user choice
def main():
    global application_name, screenshot_dir, log_file_csv, log_file_txt, screen_width, screen_height
    application_name = input("Enter the application, which you want to record: ")

    # Create a new directory for the screenshots based on the application name
    screenshot_dir = application_name
    os.makedirs(screenshot_dir, exist_ok=True)

    # Define the log files in the new directory
    log_file_csv = os.path.join(screenshot_dir, "mouse_log.csv")
    log_file_txt = os.path.join(screenshot_dir, "mouse_log.txt")

    # Get screen size from the Android emulator
    screen_width, screen_height = get_screen_size()

    print("Choose an option:")
    print("1. Automatic mode")
    print("2. Manual mode")
    choice = input("Enter your choice (1 or 2): ")

    if choice == '1':
        # Start performing random actions in a separate thread
        threading.Thread(target=perform_random_actions).start()
    elif choice == '2':
        # Start listening for touch events
        threading.Thread(target=listen_for_touch_events, daemon=True).start()
        print("Manual mode activated. Click in the emulator to log events.")
    else:
        print("Invalid choice. Exiting.")
        return

    # Keep the script running to listen for events
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        print("Script terminated.")

if __name__ == "__main__":
    main()

[Flojomojo]

This issue is not with adbutils, as you just execute a adb command (you don't even use device.shell()).

But your code is almost right, but the 4096 you divide by is probably wrong (and should be adjusted as the comment next to it suggests)
Here is some code that automatically does that (adjusted to your coding style but added some comments on how to better integrate adbutils)

from PIL import Image, ImageDraw
import datetime
import os
import pandas as pd
import threading
import adbutils
import subprocess
import time
import re


# Connect to Android emulator
def connect_device():
    adb = adbutils.AdbClient(host="127.0.0.1", port=5037)
    devices = adb.device_list()
    if not devices:
        raise adbutils.errors.AdbError("Can't find any android device/emulator")
    return devices[1]


try:
    device = connect_device()
except adbutils.errors.AdbError as e:
    print(e)
    print("Ensure your emulator is running and connected.")
    exit(1)

# Initialize a DataFrame to store logs
df_columns = [
    "timestamp",
    "event_type",
    "x",
    "y",
    "button",
    "screenshot_path",
    "screen_width",
    "screen_height",
    "application_name",
    "trajectory",
]
df = pd.DataFrame(columns=df_columns)


# Function to get touch coordinates from the emulator
def get_touch_coordinates():
    touch_coordinates = []
    # NOTE: should just use device.shell()
    process = subprocess.Popen(
        ["adb", "shell", "getevent", "-lt"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    for line in process.stdout:
        line = line.decode("utf-8").strip()
        if "ABS_MT_POSITION_X" in line or "ABS_MT_POSITION_Y" in line:
            coords = line.split()[-1]
            touch_coordinates.append(int(coords, 16))
            if len(touch_coordinates) == 2:
                x, y = touch_coordinates
                touch_coordinates = []
                return x, y


# Function to get the screen size of the emulator
def get_screen_size():
    # NOTE: should just use device.window_size() instead
    output = subprocess.check_output(
        ["adb", "shell", "wm", "size"]
    ).decode("utf-8")
    dimensions = output.split()[-1]
    width, height = map(int, dimensions.split("x"))
    return width, height

# Get the max touch coordinates
def get_max_touch_coords():
    # NOTE: should just use device.shell()
    output = subprocess.check_output(
        ["adb", "shell", "getevent", "-il"]
    ).decode("utf-8")

    max_x = None
    max_y = None

    # find 'max <number>'
    max_regex = re.compile(r'max\s+(\d+)')

    for line in output.splitlines():
        if "ABS_MT_POSITION_X" in line:
            match = max_regex.search(line)
            if match:
                max_x = int(match.group(1))
        elif "ABS_MT_POSITION_Y" in line:
            match = max_regex.search(line)
            if match:
                max_y = int(match.group(1))

        if max_x is not None and max_y is not None:
            # Found both, no need to keep parsing
            break

    # Use 4095 (range 4096) as a fallback
    if max_x is None:
        print("Could not determine max X from getevent. Defaulting to 4095.")
        max_x = 4095
    if max_y is None:
        print("Could not determine max Y from getevent. Defaulting to 4095.")
        max_y = 4095

    # range is 0 to max, so the number of points is (max + 1)
    return max_x + 1, max_y + 1


# Function to log the event and take a screenshot from Android emulator
def log_and_screenshot(event_type, x, y, button=None):
    timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S_%f")
    screenshot_path = os.path.join(screenshot_dir, f"screenshot_{timestamp}.png")

    # Capture screenshot from Android emulator
    temp_screenshot_path = "/sdcard/screenshot.png"
    device.shell(["screencap", "-p", temp_screenshot_path])
    device.sync.pull(temp_screenshot_path, screenshot_path)
    device.shell(["rm", temp_screenshot_path])

    screenshot = Image.open(screenshot_path)
    draw = ImageDraw.Draw(screenshot)
    cursor_size = 10

    # Scale the raw touch coordinates to match the screen size
    x = int(
        x * screen_width / max_touch_x
    )  # Adjust based on your device resolution and maximum coordinate range
    y = int(y * screen_height / max_touch_y)

    # Draw the cursor position with a green cross
    draw.line((x - cursor_size, y, x + cursor_size, y), fill="green", width=2)
    draw.line((x, y - cursor_size, x, y + cursor_size), fill="green", width=2)

    screenshot.save(screenshot_path, format="PNG")

    # Log the event
    event_data = {
        "timestamp": timestamp,
        "event_type": event_type,
        "x": x,
        "y": y,
        "button": button,
        "screenshot_path": screenshot_path,
        "screen_width": screen_width,
        "screen_height": screen_height,
        "application_name": application_name,
        "trajectory": "",
    }
    global df
    df = pd.concat([df, pd.DataFrame([event_data])], ignore_index=True)
    df.to_csv(log_file_csv, index=False)

    with open(log_file_txt, "a") as f:
        f.write(
            f"{timestamp} - {event_type} - ({x}, {y}) - {button} - {screenshot_path} - {screen_width}x{screen_height} - {application_name}\n"
        )


# Function to listen for touch events and log them
def listen_for_touch_events():
    while True:
        coordinates = get_touch_coordinates()
        if coordinates:
            x, y = coordinates
            log_and_screenshot("click", x, y)


# Main function to handle user choice
def main():
    global \
        application_name, \
        screenshot_dir, \
        log_file_csv, \
        log_file_txt, \
        screen_width, \
        screen_height, \
        max_touch_x, \
        max_touch_y
    application_name = input("Enter the application, which you want to record: ")

    # Create a new directory for the screenshots based on the application name
    screenshot_dir = application_name
    os.makedirs(screenshot_dir, exist_ok=True)

    # Define the log files in the new directory
    log_file_csv = os.path.join(screenshot_dir, "mouse_log.csv")
    log_file_txt = os.path.join(screenshot_dir, "mouse_log.txt")

    # Get screen size from the Android emulator
    screen_width, screen_height = get_screen_size()

    max_touch_x, max_touch_y = get_max_touch_coords()

    print("Choose an option:")
    print("1. Automatic mode")
    print("2. Manual mode")
    choice = input("Enter your choice (1 or 2): ")

    if choice == "1":
        # Start performing random actions in a separate thread
        threading.Thread(target=perform_random_actions).start()
    elif choice == "2":
        # Start listening for touch events
        threading.Thread(target=listen_for_touch_events, daemon=True).start()
        print("Manual mode activated. Click in the emulator to log events.")
    else:
        print("Invalid choice. Exiting.")
        return

    # Keep the script running to listen for events
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        print("Script terminated.")


if __name__ == "__main__":
    main()