UI automation to get the Direct Text

Tests/CaptureLanguageUtilitiesTests.cs

@@ -0,0 +1,84 @@
+using Text_Grab.Models;
+using Text_Grab.Utilities;
+
+namespace Tests;
+
+public class CaptureLanguageUtilitiesTests
+{
+    [Fact]
+    public void MatchesPersistedLanguage_MatchesByLanguageTag()
+    {
+        UiAutomationLang language = new();
+
+        bool matches = CaptureLanguageUtilities.MatchesPersistedLanguage(language, UiAutomationLang.Tag);
+
+        Assert.True(matches);
+    }
+
+    [Fact]
+    public void MatchesPersistedLanguage_MatchesLegacyTesseractDisplayName()
+    {
+        TessLang language = new("eng");
+
+        bool matches = CaptureLanguageUtilities.MatchesPersistedLanguage(language, language.CultureDisplayName);
+
+        Assert.True(matches);
+    }
+
+    [Fact]
+    public void FindPreferredLanguageIndex_PrefersPersistedMatchBeforeFallbackLanguage()
+    {
+        List<Text_Grab.Interfaces.ILanguage> languages =
+        [
+            new UiAutomationLang(),
+            new WindowsAiLang(),
+            new GlobalLang("en-US")
+        ];
+
+        int index = CaptureLanguageUtilities.FindPreferredLanguageIndex(
+            languages,
+            UiAutomationLang.Tag,
+            new GlobalLang("en-US"));
+
+        Assert.Equal(0, index);
+    }
+
+    [Fact]
+    public void SupportsTableOutput_ReturnsFalseForUiAutomation()
+    {
+        Assert.False(CaptureLanguageUtilities.SupportsTableOutput(new UiAutomationLang()));
+    }
+
+    [Fact]
+    public void RequiresLiveUiAutomationSource_ReturnsTrueForStaticUiAutomationWithoutSnapshot()
+    {
+        bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
+            new UiAutomationLang(),
+            isStaticImageSource: true,
+            hasFrozenUiAutomationSnapshot: false);
+
+        Assert.True(requiresLiveSource);
+    }
+
+    [Fact]
+    public void RequiresLiveUiAutomationSource_ReturnsFalseWhenFrozenSnapshotExists()
+    {
+        bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
+            new UiAutomationLang(),
+            isStaticImageSource: true,
+            hasFrozenUiAutomationSnapshot: true);
+
+        Assert.False(requiresLiveSource);
+    }
+
+    [Fact]
+    public void RequiresLiveUiAutomationSource_ReturnsFalseForOcrLanguageOnStaticImage()
+    {
+        bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
+            new GlobalLang("en-US"),
+            isStaticImageSource: true,
+            hasFrozenUiAutomationSnapshot: false);
+
+        Assert.False(requiresLiveSource);
+    }
+}

Tests/ImageMethodsTests.cs

@@ -0,0 +1,49 @@
+using System.Drawing;
+using System.Windows;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using Text_Grab;
+
+namespace Tests;
+
+public class ImageMethodsTests
+{
+    [WpfFact]
+    public void ImageSourceToBitmap_ConvertsBitmapSourceDerivedImages()
+    {
+        byte[] pixels =
+        [
+            0, 0, 255, 255,
+            0, 255, 0, 255,
+            255, 0, 0, 255,
+            255, 255, 255, 255
+        ];
+
+        BitmapSource source = BitmapSource.Create(
+            2,
+            2,
+            96,
+            96,
+            PixelFormats.Bgra32,
+            null,
+            pixels,
+            8);
+        CroppedBitmap cropped = new(source, new Int32Rect(1, 0, 1, 2));
+
+        using Bitmap? bitmap = ImageMethods.ImageSourceToBitmap(cropped);
+
+        Assert.NotNull(bitmap);
+        Assert.Equal(1, bitmap!.Width);
+        Assert.Equal(2, bitmap.Height);
+    }
+
+    [WpfFact]
+    public void ImageSourceToBitmap_ReturnsNullForNonBitmapImageSources()
+    {
+        DrawingImage drawingImage = new();
+
+        Bitmap? bitmap = ImageMethods.ImageSourceToBitmap(drawingImage);
+
+        Assert.Null(bitmap);
+    }
+}

Tests/LanguageServiceTests.cs

@@ -34,6 +34,16 @@ public void GetLanguageTag_WithWindowsAiLang_ReturnsWinAI()
         Assert.Equal("WinAI", tag);
     }
 
+    [Fact]
+    public void GetLanguageTag_WithUiAutomationLang_ReturnsUiAutomationTag()
+    {
+        UiAutomationLang uiAutomationLang = new();
+
+        string tag = LanguageService.GetLanguageTag(uiAutomationLang);
+
+        Assert.Equal(UiAutomationLang.Tag, tag);
+    }
+
     [Fact]
     public void GetLanguageTag_WithTessLang_ReturnsRawTag()
     {
@@ -86,6 +96,16 @@ public void GetLanguageKind_WithWindowsAiLang_ReturnsWindowsAi()
         Assert.Equal(LanguageKind.WindowsAi, kind);
     }
 
+    [Fact]
+    public void GetLanguageKind_WithUiAutomationLang_ReturnsUiAutomation()
+    {
+        UiAutomationLang uiAutomationLang = new();
+
+        LanguageKind kind = LanguageService.GetLanguageKind(uiAutomationLang);
+
+        Assert.Equal(LanguageKind.UiAutomation, kind);
+    }
+
     [Fact]
     public void GetLanguageKind_WithTessLang_ReturnsTesseract()
     {
@@ -149,4 +169,16 @@ public void LanguageUtilities_DelegatesTo_LanguageService()
         Assert.Equal("en-US", tag);
         Assert.Equal(LanguageKind.Global, kind);
     }
+
+    [Fact]
+    public void HistoryInfo_OcrLanguage_RehydratesUiAutomationLanguage()
+    {
+        HistoryInfo historyInfo = new()
+        {
+            LanguageTag = UiAutomationLang.Tag,
+            LanguageKind = LanguageKind.UiAutomation,
+        };
+
+        Assert.IsType<UiAutomationLang>(historyInfo.OcrLanguage);
+    }
 }

Tests/UiAutomationUtilitiesTests.cs

@@ -0,0 +1,165 @@
+using System.Linq;
+using System.Windows;
+using System.Windows.Automation;
+using Text_Grab.Models;
+using Text_Grab.Utilities;
+
+namespace Tests;
+
+public class UiAutomationUtilitiesTests
+{
+    [Fact]
+    public void NormalizeText_TrimsWhitespaceAndCollapsesEmptyLines()
+    {
+        string normalized = UIAutomationUtilities.NormalizeText("  Hello   world \r\n\r\n Second\tline ");
+
+        Assert.Equal($"Hello world{Environment.NewLine}Second line", normalized);
+    }
+
+    [Fact]
+    public void TryAddUniqueText_DeduplicatesNormalizedValues()
+    {
+        HashSet<string> seen = [];
+        List<string> output = [];
+
+        bool addedFirst = UIAutomationUtilities.TryAddUniqueText(" Hello   world ", seen, output);
+        bool addedSecond = UIAutomationUtilities.TryAddUniqueText("Hello world", seen, output);
+
+        Assert.True(addedFirst);
+        Assert.False(addedSecond);
+        Assert.Single(output);
+    }
+
+    [Fact]
+    public void FindTargetWindowCandidate_PrefersCenterPointHit()
+    {
+        WindowSelectionCandidate first = new((nint)1, new Rect(0, 0, 80, 80), "First", 1);
+        WindowSelectionCandidate second = new((nint)2, new Rect(90, 0, 80, 80), "Second", 2);
+
+        WindowSelectionCandidate? candidate = UIAutomationUtilities.FindTargetWindowCandidate(
+            new Rect(100, 10, 20, 20),
+            [first, second]);
+
+        Assert.Same(second, candidate);
+    }
+
+    [Fact]
+    public void FindTargetWindowCandidate_FallsBackToLargestIntersection()
+    {
+        WindowSelectionCandidate first = new((nint)1, new Rect(0, 0, 50, 50), "First", 1);
+        WindowSelectionCandidate second = new((nint)2, new Rect(60, 0, 80, 80), "Second", 2);
+
+        WindowSelectionCandidate? candidate = UIAutomationUtilities.FindTargetWindowCandidate(
+            new Rect(40, 40, 30, 30),
+            [first, second]);
+
+        Assert.Same(second, candidate);
+    }
+
+    [Fact]
+    public void ShouldUseNameFallback_SkipsStructuralControls()
+    {
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Window));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Group));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Pane));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Custom));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Button));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.SplitButton));
+        Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.ComboBox));
+    }
+
+    [Fact]
+    public void ShouldUseNameFallback_AllowsVisibleTextContainers()
+    {
+        Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Text));
+        Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.ListItem));
+        Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.MenuItem));
+        Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.TabItem));
+    }
+
+    [Fact]
+    public void GetSamplePoints_UsesCenterPointForSmallSelections()
+    {
+        IReadOnlyList<Point> samplePoints = UIAutomationUtilities.GetSamplePoints(new Rect(10, 20, 40, 30));
+
+        Point samplePoint = Assert.Single(samplePoints);
+        Assert.Equal(new Point(30, 35), samplePoint);
+    }
+
+    [Fact]
+    public void GetSamplePoints_UsesGridForLargerSelections()
+    {
+        IReadOnlyList<Point> samplePoints = UIAutomationUtilities.GetSamplePoints(new Rect(0, 0, 100, 100));
+
+        Assert.Equal(9, samplePoints.Count);
+        Assert.Contains(new Point(50, 50), samplePoints);
+        Assert.Contains(new Point(20, 20), samplePoints);
+        Assert.Contains(new Point(80, 80), samplePoints);
+    }
+
+    [Fact]
+    public void GetPointProbePoints_ReturnsCenterThenCrosshairNeighbors()
+    {
+        IReadOnlyList<Point> probePoints = UIAutomationUtilities.GetPointProbePoints(new Point(25, 40));
+
+        Assert.Equal(5, probePoints.Count);
+        Assert.Equal(new Point(25, 40), probePoints[0]);
+        Assert.Contains(new Point(23, 40), probePoints);
+        Assert.Contains(new Point(27, 40), probePoints);
+        Assert.Contains(new Point(25, 38), probePoints);
+        Assert.Contains(new Point(25, 42), probePoints);
+    }
+
+    [Fact]
+    public void TryClipBounds_ReturnsIntersectionForOverlappingRects()
+    {
+        bool clipped = UIAutomationUtilities.TryClipBounds(
+            new Rect(10, 10, 50, 50),
+            new Rect(30, 25, 50, 50),
+            out Rect result);
+
+        Assert.True(clipped);
+        Assert.Equal(new Rect(30, 25, 30, 35), result);
+    }
+
+    [Fact]
+    public void TryClipBounds_ReturnsFalseWhenBoundsDoNotIntersect()
+    {
+        bool clipped = UIAutomationUtilities.TryClipBounds(
+            new Rect(10, 10, 20, 20),
+            new Rect(100, 100, 20, 20),
+            out Rect result);
+
+        Assert.False(clipped);
+        Assert.Equal(Rect.Empty, result);
+    }
+
+    [Fact]
+    public void TryAddUniqueOverlayItem_DeduplicatesNormalizedTextAndBounds()
+    {
+        HashSet<string> seen = [];
+        List<UiAutomationOverlayItem> output = [];
+        UiAutomationOverlayItem first = new(" Hello   world ", new Rect(10.01, 20.01, 30.01, 40.01), UiAutomationOverlaySource.ElementBounds);
+        UiAutomationOverlayItem second = new("Hello world", new Rect(10.04, 20.04, 30.04, 40.04), UiAutomationOverlaySource.VisibleTextRange);
+
+        bool addedFirst = UIAutomationUtilities.TryAddUniqueOverlayItem(first, seen, output);
+        bool addedSecond = UIAutomationUtilities.TryAddUniqueOverlayItem(second, seen, output);
+
+        Assert.True(addedFirst);
+        Assert.False(addedSecond);
+        Assert.Single(output);
+    }
+
+    [Fact]
+    public void SortOverlayItems_OrdersTopThenLeft()
+    {
+        IReadOnlyList<UiAutomationOverlayItem> sorted = UIAutomationUtilities.SortOverlayItems(
+        [
+            new UiAutomationOverlayItem("Bottom", new Rect(40, 30, 10, 10), UiAutomationOverlaySource.ElementBounds),
+            new UiAutomationOverlayItem("Right", new Rect(25, 10, 10, 10), UiAutomationOverlaySource.ElementBounds),
+            new UiAutomationOverlayItem("Left", new Rect(10, 10, 10, 10), UiAutomationOverlaySource.ElementBounds),
+        ]);
+
+        Assert.Equal(["Left", "Right", "Bottom"], sorted.Select(item => item.Text));
+    }
+}

Text-Grab/App.config

@@ -163,6 +163,9 @@
             <setting name="FsgDefaultMode" serializeAs="String">
                 <value>Default</value>
             </setting>
+            <setting name="FsgSelectionStyle" serializeAs="String">
+                <value>Region</value>
+            </setting>
             <setting name="FsgShadeOverlay" serializeAs="String">
                 <value>True</value>
             </setting>
@@ -196,6 +199,21 @@
             <setting name="OverrideAiArchCheck" serializeAs="String">
                 <value>False</value>
             </setting>
+            <setting name="UiAutomationEnabled" serializeAs="String">
+                <value>True</value>
+            </setting>
+            <setting name="UiAutomationFallbackToOcr" serializeAs="String">
+                <value>True</value>
+            </setting>
+            <setting name="UiAutomationTraversalMode" serializeAs="String">
+                <value>Balanced</value>
+            </setting>
+            <setting name="UiAutomationIncludeOffscreen" serializeAs="String">
+                <value>False</value>
+            </setting>
+            <setting name="UiAutomationPreferFocusedElement" serializeAs="String">
+                <value>True</value>
+            </setting>
             <setting name="GrabFrameTranslationEnabled" serializeAs="String">
                 <value>False</value>
             </setting>

Text-Grab/Controls/LanguagePicker.xaml

@@ -19,7 +19,7 @@
                                Mode=TwoWay}"
         SelectionChanged="MainComboBox_SelectionChanged">
         <ComboBox.ItemTemplate>
-            <DataTemplate DataType="global:Language">
+            <DataTemplate>
                 <TextBlock Text="{Binding DisplayName}" />
             </DataTemplate>
         </ComboBox.ItemTemplate>