Tokenizer.java

import java.util.*;

/**
 * Tokenizer converts raw text into clean searchable tokens.
 */
public class Tokenizer {

    /**
     * Converts text into a list of normalized words.
     * Example:
     *  "Java is GREAT!" -> ["java", "is", "great"]
     */
    public static List<String> tokenize(String text) {

        if (text == null || text.isEmpty()) {
            return Collections.emptyList();
        }

        // Convert to lowercase
        text = text.toLowerCase();

        // Remove punctuation & special characters
        text = text.replaceAll("[^a-z ]", " ");

        // Split by whitespace
        String[] tokens = text.split("\\s+");

        List<String> words = new ArrayList<>();
        for (String token : tokens) {
            if (!token.isEmpty()) {
                words.add(token);
            }
        }

        return words;
    }
}