diff --git a/examples/src/main/java/io/github/yasmramos/mindforge/examples/TreeSHAPExample.java b/examples/src/main/java/io/github/yasmramos/mindforge/examples/TreeSHAPExample.java index 57bee8b..5049dd7 100644 --- a/examples/src/main/java/io/github/yasmramos/mindforge/examples/TreeSHAPExample.java +++ b/examples/src/main/java/io/github/yasmramos/mindforge/examples/TreeSHAPExample.java @@ -20,10 +20,9 @@ public static void main(String[] args) { System.out.println("Loading Iris dataset..."); var dataset = DatasetLoader.loadIris(); double[][] X = dataset.getFeatures(); - int[] y = dataset.getTargets(); + int[] y = dataset.getLabels(); System.out.println("Dataset: " + X.length + " samples, " + X[0].length + " features"); - System.out.println("Classes: " + dataset.getClasses().length + "\n"); // Train Random Forest System.out.println("Training Random Forest classifier..."); @@ -36,6 +35,8 @@ public static void main(String[] args) { rf.fit(X, y); + System.out.println("Classes: " + rf.getClasses().length + "\n"); + // Evaluate model int[] predictions = rf.predict(X); double accuracy = Metrics.accuracy(y, predictions); @@ -60,7 +61,7 @@ public static void main(String[] args) { System.out.println("=== Explaining Single Instance ==="); double[] instance = X[100]; // Pick an instance to explain int trueLabel = y[100]; - int predictedLabel = rf.predict(instance)[0]; + int predictedLabel = rf.predict(new double[][] { instance })[0]; System.out.println("Instance: " + java.util.Arrays.toString(instance)); System.out.println("True label: " + trueLabel); diff --git a/src/main/java/io/github/yasmramos/mindforge/model_selection/UnifiedModelSelector.java b/src/main/java/io/github/yasmramos/mindforge/model_selection/UnifiedModelSelector.java new file mode 100644 index 0000000..8ab71a2 --- /dev/null +++ b/src/main/java/io/github/yasmramos/mindforge/model_selection/UnifiedModelSelector.java @@ -0,0 +1,523 @@ +package io.github.yasmramos.mindforge.model_selection; + +import io.github.yasmramos.mindforge.classification.Classifier; +import io.github.yasmramos.mindforge.regression.Regressor; +import io.github.yasmramos.mindforge.clustering.Clusterer; + +import java.io.Serializable; +import java.util.*; +import java.util.function.Supplier; + +/** + * Unified Model Selector that integrates Grid Search, Randomized Search, and Bayesian Optimization. + * Supports classification, regression, and clustering with automatic strategy selection. + * + * @author MindForge + */ +public class UnifiedModelSelector implements Serializable { + private static final long serialVersionUID = 1L; + + public enum SearchStrategy { + GRID_SEARCH, + RANDOMIZED_SEARCH, + BAYESIAN_OPTIMIZATION, + AUTO + } + + public enum ModelType { + CLASSIFICATION, + REGRESSION, + CLUSTERING + } + + private final SearchStrategy strategy; + private final ModelType modelType; + private final int cv; + private final String scoring; + private final int nIterations; + private final int nInitialPoints; + private final long seed; + + private Object bestModel; + private Map bestParams; + private double bestScore; + private List> cvResults; + private boolean fitted; + + // For Bayesian Optimization + private Map parameterBounds; + + /** + * Creates a UnifiedModelSelector with auto strategy selection. + */ + public UnifiedModelSelector(ModelType modelType, int cv, String scoring) { + this(SearchStrategy.AUTO, modelType, cv, scoring, 50, 5, 42); + } + + /** + * Creates a UnifiedModelSelector with full configuration. + */ + public UnifiedModelSelector(SearchStrategy strategy, ModelType modelType, int cv, + String scoring, int nIterations, int nInitialPoints, long seed) { + this.strategy = strategy; + this.modelType = modelType; + this.cv = cv; + this.scoring = scoring; + this.nIterations = nIterations; + this.nInitialPoints = nInitialPoints; + this.seed = seed; + this.fitted = false; + this.parameterBounds = new LinkedHashMap<>(); + } + + /** + * Sets parameter grid for Grid Search. + */ + public UnifiedModelSelector setParamGrid(Map paramGrid) { + // Convert discrete values to bounds for Bayesian Optimization + for (Map.Entry entry : paramGrid.entrySet()) { + Object[] values = entry.getValue(); + if (values.length > 0 && values[0] instanceof Number) { + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (Object val : values) { + double d = ((Number) val).doubleValue(); + if (d < min) min = d; + if (d > max) max = d; + } + parameterBounds.put(entry.getKey(), new double[]{min, max}); + } + } + return this; + } + + /** + * Sets continuous parameter bounds for Bayesian Optimization. + */ + public UnifiedModelSelector setParameterBounds(Map bounds) { + this.parameterBounds = new LinkedHashMap<>(bounds); + return this; + } + + /** + * Fits the model selector using the configured strategy. + */ + public UnifiedModelSelector fit(Supplier modelFactory, double[][] X, int[] y) { + SearchStrategy actualStrategy = selectStrategy(modelType, parameterBounds); + + switch (actualStrategy) { + case GRID_SEARCH: + runGridSearch(modelFactory, X, y); + break; + case RANDOMIZED_SEARCH: + runRandomizedSearch(modelFactory, X, y); + break; + case BAYESIAN_OPTIMIZATION: + runBayesianOptimization(modelFactory, X, y); + break; + default: + throw new IllegalStateException("Unknown strategy: " + actualStrategy); + } + + fitted = true; + return this; + } + + /** + * Fits for clustering (unsupervised). + */ + public UnifiedModelSelector fitClustering(Supplier> clusterFactory, double[][] X) { + SearchStrategy actualStrategy = selectStrategy(ModelType.CLUSTERING, parameterBounds); + + switch (actualStrategy) { + case BAYESIAN_OPTIMIZATION: + runBayesianOptimizationClustering(clusterFactory, X); + break; + case RANDOMIZED_SEARCH: + runRandomizedSearchClustering(clusterFactory, X); + break; + default: + runGridSearchClustering(clusterFactory, X); + break; + } + + fitted = true; + return this; + } + + private SearchStrategy selectStrategy(ModelType type, Map bounds) { + if (strategy != SearchStrategy.AUTO) { + return strategy; + } + + // Auto-select based on parameter space + if (bounds == null || bounds.isEmpty()) { + return SearchStrategy.GRID_SEARCH; + } + + // If many parameters or continuous space, use Bayesian Optimization + if (bounds.size() > 3) { + return SearchStrategy.BAYESIAN_OPTIMIZATION; + } + + return SearchStrategy.RANDOMIZED_SEARCH; + } + + private void runGridSearch(Supplier modelFactory, double[][] X, int[] y) { + // Simplified grid search implementation + cvResults = new ArrayList<>(); + bestScore = Double.NEGATIVE_INFINITY; + + // Generate combinations from bounds (discretized) + List> combinations = discretizeBounds(5); + + for (Map params : combinations) { + double score = crossValidate(modelFactory, params, X, y); + + Map result = new HashMap<>(params); + result.put("mean_score", score); + cvResults.add(result); + + if (score > bestScore) { + bestScore = score; + bestParams = new HashMap<>(params); + bestModel = createModelWithParams(modelFactory, params); + ((Classifier)bestModel).train(X, y); + } + } + } + + private void runRandomizedSearch(Supplier modelFactory, double[][] X, int[] y) { + Random random = new Random(seed); + cvResults = new ArrayList<>(); + bestScore = Double.NEGATIVE_INFINITY; + + for (int i = 0; i < nIterations; i++) { + Map params = sampleRandomParams(random); + double score = crossValidate(modelFactory, params, X, y); + + Map result = new HashMap<>(params); + result.put("mean_score", score); + cvResults.add(result); + + if (score > bestScore) { + bestScore = score; + bestParams = new HashMap<>(params); + bestModel = createModelWithParams(modelFactory, params); + ((Classifier)bestModel).train(X, y); + } + } + } + + private void runBayesianOptimization(Supplier modelFactory, double[][] X, int[] y) { + BayesianOptimization bo = new BayesianOptimization.Builder() + .nIterations(nIterations) + .nInitialPoints(nInitialPoints) + .seed(seed) + .build(); + + bo.setParameterBounds(parameterBounds); + + Map optimalParams = bo.optimize(params -> { + Map objParams = new HashMap<>(); + params.forEach((k, v) -> objParams.put(k, v)); + return crossValidate(modelFactory, objParams, X, y); + }); + + bestParams = new HashMap<>(optimalParams); + bestScore = bo.getBestValue(); + bestModel = createModelWithParams(modelFactory, bestParams); + + if (modelType == ModelType.CLASSIFICATION) { + ((Classifier)bestModel).train(X, y); + } else if (modelType == ModelType.REGRESSION) { + // Convert int[] y to double[] for regression + double[] yDouble = new double[y.length]; + for (int i = 0; i < y.length; i++) { + yDouble[i] = y[i]; + } + ((Regressor)bestModel).train(X, yDouble); + } + + // Store BO results + cvResults = new ArrayList<>(); + List observedValues = bo.getObservedValues(); + for (Double value : observedValues) { + Map result = new HashMap<>(); + result.put("mean_score", value); + cvResults.add(result); + } + } + + private void runBayesianOptimizationClustering(Supplier> clusterFactory, double[][] X) { + BayesianOptimization bo = new BayesianOptimization.Builder() + .nIterations(nIterations) + .nInitialPoints(nInitialPoints) + .seed(seed) + .build(); + + bo.setParameterBounds(parameterBounds); + + Map optimalParams = bo.optimize(params -> { + Map objParams = new HashMap<>(); + params.forEach((k, v) -> objParams.put(k, v)); + return evaluateClustering(clusterFactory, objParams, X); + }); + + bestParams = new HashMap<>(optimalParams); + bestScore = bo.getBestValue(); + bestModel = createClusterWithParams(clusterFactory, bestParams); + ((Clusterer)bestModel).cluster(X); + } + + private void runRandomizedSearchClustering(Supplier> clusterFactory, double[][] X) { + Random random = new Random(seed); + cvResults = new ArrayList<>(); + bestScore = Double.NEGATIVE_INFINITY; + + for (int i = 0; i < nIterations; i++) { + Map params = sampleRandomParams(random); + double score = evaluateClustering(clusterFactory, params, X); + + Map result = new HashMap<>(params); + result.put("score", score); + cvResults.add(result); + + if (score > bestScore) { + bestScore = score; + bestParams = new HashMap<>(params); + bestModel = createClusterWithParams(clusterFactory, params); + ((Clusterer)bestModel).cluster(X); + } + } + } + + private void runGridSearchClustering(Supplier> clusterFactory, double[][] X) { + cvResults = new ArrayList<>(); + bestScore = Double.NEGATIVE_INFINITY; + + List> combinations = discretizeBounds(5); + + for (Map params : combinations) { + double score = evaluateClustering(clusterFactory, params, X); + + Map result = new HashMap<>(params); + result.put("score", score); + cvResults.add(result); + + if (score > bestScore) { + bestScore = score; + bestParams = new HashMap<>(params); + bestModel = createClusterWithParams(clusterFactory, params); + ((Clusterer)bestModel).cluster(X); + } + } + } + + private double crossValidate(Supplier modelFactory, Map params, + double[][] X, int[] y) { + KFold kfold = new KFold(cv, false, new Random(seed)); + List scores = new ArrayList<>(); + + for (int[][] fold : kfold.split(X.length)) { + int[] trainIdx = fold[0]; + int[] testIdx = fold[1]; + + double[][] XTrain = subset(X, trainIdx); + int[] yTrain = subset(y, trainIdx); + double[][] XTest = subset(X, testIdx); + int[] yTest = subset(y, testIdx); + + Object model = createModelWithParams(modelFactory, params); + + if (modelType == ModelType.CLASSIFICATION) { + Classifier clf = (Classifier) model; + clf.train(XTrain, yTrain); + scores.add(calculateAccuracy(clf, XTest, yTest)); + } else if (modelType == ModelType.REGRESSION) { + // Convert int[] yTrain to double[] for regression + double[] yTrainDouble = new double[yTrain.length]; + for (int i = 0; i < yTrain.length; i++) { + yTrainDouble[i] = yTrain[i]; + } + Regressor reg = (Regressor) model; + reg.train(XTrain, yTrainDouble); + scores.add(calculateR2(reg, XTest, yTest)); + } + } + + return scores.stream().mapToDouble(Double::doubleValue).average().orElse(0.0); + } + + private double evaluateClustering(Supplier> clusterFactory, + Map params, double[][] X) { + Clusterer clusterer = createClusterWithParams(clusterFactory, params); + int[] labels = clusterer.cluster(X); + + // Silhouette score as default metric + return calculateSilhouetteScore(X, labels); + } + + private double calculateSilhouetteScore(double[][] X, int[] labels) { + int n = X.length; + if (n < 2) return 0; + + double sumSilhouette = 0; + int count = 0; + + for (int i = 0; i < n; i++) { + int label = labels[i]; + + // Calculate a(i): mean distance to other points in same cluster + double sumSame = 0; + int countSame = 0; + for (int j = 0; j < n; j++) { + if (i != j && labels[j] == label) { + sumSame += euclideanDistance(X[i], X[j]); + countSame++; + } + } + double a = countSame > 0 ? sumSame / countSame : 0; + + // Calculate b(i): min mean distance to points in other clusters + double b = Double.MAX_VALUE; + Set otherLabels = new HashSet<>(); + for (int j = 0; j < n; j++) { + if (labels[j] != label) { + otherLabels.add(labels[j]); + } + } + + for (int otherLabel : otherLabels) { + double sumOther = 0; + int countOther = 0; + for (int j = 0; j < n; j++) { + if (labels[j] == otherLabel) { + sumOther += euclideanDistance(X[i], X[j]); + countOther++; + } + } + if (countOther > 0) { + double meanOther = sumOther / countOther; + if (meanOther < b) { + b = meanOther; + } + } + } + + if (b == Double.MAX_VALUE) b = 0; + + // Silhouette coefficient + double s = (b - a) / Math.max(a, b); + sumSilhouette += s; + count++; + } + + return count > 0 ? sumSilhouette / count : 0; + } + + private double euclideanDistance(double[] x1, double[] x2) { + double sum = 0; + for (int i = 0; i < x1.length; i++) { + double diff = x1[i] - x2[i]; + sum += diff * diff; + } + return Math.sqrt(sum); + } + + private List> discretizeBounds(int nPoints) { + List> combinations = new ArrayList<>(); + combinations.add(new HashMap<>()); + + for (Map.Entry entry : parameterBounds.entrySet()) { + String paramName = entry.getKey(); + double[] bounds = entry.getValue(); + double step = (bounds[1] - bounds[0]) / (nPoints - 1); + + List> newCombinations = new ArrayList<>(); + + for (Map combo : combinations) { + for (int i = 0; i < nPoints; i++) { + Map newCombo = new HashMap<>(combo); + newCombo.put(paramName, bounds[0] + i * step); + newCombinations.add(newCombo); + } + } + + combinations = newCombinations; + } + + return combinations; + } + + private Map sampleRandomParams(Random random) { + Map params = new HashMap<>(); + for (Map.Entry entry : parameterBounds.entrySet()) { + double[] bounds = entry.getValue(); + params.put(entry.getKey(), bounds[0] + random.nextDouble() * (bounds[1] - bounds[0])); + } + return params; + } + + private Object createModelWithParams(Supplier factory, Map params) { + // In a full implementation, use reflection to set parameters + return factory.get(); + } + + private Clusterer createClusterWithParams(Supplier> factory, + Map params) { + return factory.get(); + } + + private double calculateAccuracy(Classifier clf, double[][] X, int[] y) { + int[] predictions = clf.predict(X); + int correct = 0; + for (int i = 0; i < y.length; i++) { + if (predictions[i] == y[i]) correct++; + } + return (double) correct / y.length; + } + + private double calculateR2(Regressor reg, double[][] X, int[] y) { + double[] predictions = reg.predict(X); + double mean = 0; + for (double val : y) mean += val; + mean /= y.length; + + double ssTot = 0, ssRes = 0; + for (int i = 0; i < y.length; i++) { + ssTot += Math.pow(y[i] - mean, 2); + ssRes += Math.pow(y[i] - predictions[i], 2); + } + + return 1 - (ssRes / ssTot); + } + + private double[][] subset(double[][] X, int[] indices) { + double[][] result = new double[indices.length][]; + for (int i = 0; i < indices.length; i++) { + result[i] = X[indices[i]]; + } + return result; + } + + private int[] subset(int[] y, int[] indices) { + int[] result = new int[indices.length]; + for (int i = 0; i < indices.length; i++) { + result[i] = y[indices[i]]; + } + return result; + } + + // Getters + public Object getBestModel() { return bestModel; } + public Map getBestParams() { return bestParams != null ? new HashMap<>(bestParams) : null; } + public double getBestScore() { return bestScore; } + public List> getCvResults() { return cvResults != null ? new ArrayList<>(cvResults) : null; } + public boolean isFitted() { return fitted; } + + @SuppressWarnings("unchecked") + public T getBestModelAs() { + return (T) bestModel; + } +}