Skip to content

Commit 76ff856

Browse files
TheJoeFinTheJoeFin
authored
Merge pull request #630 from TheJoeFin/ui-automation
UI automation to get the Direct Text
2 parents 68504b6 + 4036c0d commit 76ff856

30 files changed

Lines changed: 2682 additions & 296 deletions

‎Tests/CaptureLanguageUtilitiesTests.cs‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
using Text_Grab.Models;
2+
using Text_Grab.Utilities;
3+
4+
namespace Tests;
5+
6+
public class CaptureLanguageUtilitiesTests
7+
{
8+
[Fact]
9+
public void MatchesPersistedLanguage_MatchesByLanguageTag()
10+
{
11+
UiAutomationLang language = new();
12+
13+
bool matches = CaptureLanguageUtilities.MatchesPersistedLanguage(language, UiAutomationLang.Tag);
14+
15+
Assert.True(matches);
16+
}
17+
18+
[Fact]
19+
public void MatchesPersistedLanguage_MatchesLegacyTesseractDisplayName()
20+
{
21+
TessLang language = new("eng");
22+
23+
bool matches = CaptureLanguageUtilities.MatchesPersistedLanguage(language, language.CultureDisplayName);
24+
25+
Assert.True(matches);
26+
}
27+
28+
[Fact]
29+
public void FindPreferredLanguageIndex_PrefersPersistedMatchBeforeFallbackLanguage()
30+
{
31+
List<Text_Grab.Interfaces.ILanguage> languages =
32+
[
33+
new UiAutomationLang(),
34+
new WindowsAiLang(),
35+
new GlobalLang("en-US")
36+
];
37+
38+
int index = CaptureLanguageUtilities.FindPreferredLanguageIndex(
39+
languages,
40+
UiAutomationLang.Tag,
41+
new GlobalLang("en-US"));
42+
43+
Assert.Equal(0, index);
44+
}
45+
46+
[Fact]
47+
public void SupportsTableOutput_ReturnsFalseForUiAutomation()
48+
{
49+
Assert.False(CaptureLanguageUtilities.SupportsTableOutput(new UiAutomationLang()));
50+
}
51+
52+
[Fact]
53+
public void RequiresLiveUiAutomationSource_ReturnsTrueForStaticUiAutomationWithoutSnapshot()
54+
{
55+
bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
56+
new UiAutomationLang(),
57+
isStaticImageSource: true,
58+
hasFrozenUiAutomationSnapshot: false);
59+
60+
Assert.True(requiresLiveSource);
61+
}
62+
63+
[Fact]
64+
public void RequiresLiveUiAutomationSource_ReturnsFalseWhenFrozenSnapshotExists()
65+
{
66+
bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
67+
new UiAutomationLang(),
68+
isStaticImageSource: true,
69+
hasFrozenUiAutomationSnapshot: true);
70+
71+
Assert.False(requiresLiveSource);
72+
}
73+
74+
[Fact]
75+
public void RequiresLiveUiAutomationSource_ReturnsFalseForOcrLanguageOnStaticImage()
76+
{
77+
bool requiresLiveSource = CaptureLanguageUtilities.RequiresLiveUiAutomationSource(
78+
new GlobalLang("en-US"),
79+
isStaticImageSource: true,
80+
hasFrozenUiAutomationSnapshot: false);
81+
82+
Assert.False(requiresLiveSource);
83+
}
84+
}

‎Tests/ImageMethodsTests.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System.Drawing;
2+
using System.Windows;
3+
using System.Windows.Media;
4+
using System.Windows.Media.Imaging;
5+
using Text_Grab;
6+
7+
namespace Tests;
8+
9+
public class ImageMethodsTests
10+
{
11+
[WpfFact]
12+
public void ImageSourceToBitmap_ConvertsBitmapSourceDerivedImages()
13+
{
14+
byte[] pixels =
15+
[
16+
0, 0, 255, 255,
17+
0, 255, 0, 255,
18+
255, 0, 0, 255,
19+
255, 255, 255, 255
20+
];
21+
22+
BitmapSource source = BitmapSource.Create(
23+
2,
24+
2,
25+
96,
26+
96,
27+
PixelFormats.Bgra32,
28+
null,
29+
pixels,
30+
8);
31+
CroppedBitmap cropped = new(source, new Int32Rect(1, 0, 1, 2));
32+
33+
using Bitmap? bitmap = ImageMethods.ImageSourceToBitmap(cropped);
34+
35+
Assert.NotNull(bitmap);
36+
Assert.Equal(1, bitmap!.Width);
37+
Assert.Equal(2, bitmap.Height);
38+
}
39+
40+
[WpfFact]
41+
public void ImageSourceToBitmap_ReturnsNullForNonBitmapImageSources()
42+
{
43+
DrawingImage drawingImage = new();
44+
45+
Bitmap? bitmap = ImageMethods.ImageSourceToBitmap(drawingImage);
46+
47+
Assert.Null(bitmap);
48+
}
49+
}

‎Tests/LanguageServiceTests.cs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@ public void GetLanguageTag_WithWindowsAiLang_ReturnsWinAI()
3434
Assert.Equal("WinAI", tag);
3535
}
3636

37+
[Fact]
38+
public void GetLanguageTag_WithUiAutomationLang_ReturnsUiAutomationTag()
39+
{
40+
UiAutomationLang uiAutomationLang = new();
41+
42+
string tag = LanguageService.GetLanguageTag(uiAutomationLang);
43+
44+
Assert.Equal(UiAutomationLang.Tag, tag);
45+
}
46+
3747
[Fact]
3848
public void GetLanguageTag_WithTessLang_ReturnsRawTag()
3949
{
@@ -86,6 +96,16 @@ public void GetLanguageKind_WithWindowsAiLang_ReturnsWindowsAi()
8696
Assert.Equal(LanguageKind.WindowsAi, kind);
8797
}
8898

99+
[Fact]
100+
public void GetLanguageKind_WithUiAutomationLang_ReturnsUiAutomation()
101+
{
102+
UiAutomationLang uiAutomationLang = new();
103+
104+
LanguageKind kind = LanguageService.GetLanguageKind(uiAutomationLang);
105+
106+
Assert.Equal(LanguageKind.UiAutomation, kind);
107+
}
108+
89109
[Fact]
90110
public void GetLanguageKind_WithTessLang_ReturnsTesseract()
91111
{
@@ -149,4 +169,16 @@ public void LanguageUtilities_DelegatesTo_LanguageService()
149169
Assert.Equal("en-US", tag);
150170
Assert.Equal(LanguageKind.Global, kind);
151171
}
172+
173+
[Fact]
174+
public void HistoryInfo_OcrLanguage_RehydratesUiAutomationLanguage()
175+
{
176+
HistoryInfo historyInfo = new()
177+
{
178+
LanguageTag = UiAutomationLang.Tag,
179+
LanguageKind = LanguageKind.UiAutomation,
180+
};
181+
182+
Assert.IsType<UiAutomationLang>(historyInfo.OcrLanguage);
183+
}
152184
}

‎Tests/UiAutomationUtilitiesTests.cs‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
using System.Linq;
2+
using System.Windows;
3+
using System.Windows.Automation;
4+
using Text_Grab.Models;
5+
using Text_Grab.Utilities;
6+
7+
namespace Tests;
8+
9+
public class UiAutomationUtilitiesTests
10+
{
11+
[Fact]
12+
public void NormalizeText_TrimsWhitespaceAndCollapsesEmptyLines()
13+
{
14+
string normalized = UIAutomationUtilities.NormalizeText(" Hello world \r\n\r\n Second\tline ");
15+
16+
Assert.Equal($"Hello world{Environment.NewLine}Second line", normalized);
17+
}
18+
19+
[Fact]
20+
public void TryAddUniqueText_DeduplicatesNormalizedValues()
21+
{
22+
HashSet<string> seen = [];
23+
List<string> output = [];
24+
25+
bool addedFirst = UIAutomationUtilities.TryAddUniqueText(" Hello world ", seen, output);
26+
bool addedSecond = UIAutomationUtilities.TryAddUniqueText("Hello world", seen, output);
27+
28+
Assert.True(addedFirst);
29+
Assert.False(addedSecond);
30+
Assert.Single(output);
31+
}
32+
33+
[Fact]
34+
public void FindTargetWindowCandidate_PrefersCenterPointHit()
35+
{
36+
WindowSelectionCandidate first = new((nint)1, new Rect(0, 0, 80, 80), "First", 1);
37+
WindowSelectionCandidate second = new((nint)2, new Rect(90, 0, 80, 80), "Second", 2);
38+
39+
WindowSelectionCandidate? candidate = UIAutomationUtilities.FindTargetWindowCandidate(
40+
new Rect(100, 10, 20, 20),
41+
[first, second]);
42+
43+
Assert.Same(second, candidate);
44+
}
45+
46+
[Fact]
47+
public void FindTargetWindowCandidate_FallsBackToLargestIntersection()
48+
{
49+
WindowSelectionCandidate first = new((nint)1, new Rect(0, 0, 50, 50), "First", 1);
50+
WindowSelectionCandidate second = new((nint)2, new Rect(60, 0, 80, 80), "Second", 2);
51+
52+
WindowSelectionCandidate? candidate = UIAutomationUtilities.FindTargetWindowCandidate(
53+
new Rect(40, 40, 30, 30),
54+
[first, second]);
55+
56+
Assert.Same(second, candidate);
57+
}
58+
59+
[Fact]
60+
public void ShouldUseNameFallback_SkipsStructuralControls()
61+
{
62+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Window));
63+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Group));
64+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Pane));
65+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Custom));
66+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Button));
67+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.SplitButton));
68+
Assert.False(UIAutomationUtilities.ShouldUseNameFallback(ControlType.ComboBox));
69+
}
70+
71+
[Fact]
72+
public void ShouldUseNameFallback_AllowsVisibleTextContainers()
73+
{
74+
Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.Text));
75+
Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.ListItem));
76+
Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.MenuItem));
77+
Assert.True(UIAutomationUtilities.ShouldUseNameFallback(ControlType.TabItem));
78+
}
79+
80+
[Fact]
81+
public void GetSamplePoints_UsesCenterPointForSmallSelections()
82+
{
83+
IReadOnlyList<Point> samplePoints = UIAutomationUtilities.GetSamplePoints(new Rect(10, 20, 40, 30));
84+
85+
Point samplePoint = Assert.Single(samplePoints);
86+
Assert.Equal(new Point(30, 35), samplePoint);
87+
}
88+
89+
[Fact]
90+
public void GetSamplePoints_UsesGridForLargerSelections()
91+
{
92+
IReadOnlyList<Point> samplePoints = UIAutomationUtilities.GetSamplePoints(new Rect(0, 0, 100, 100));
93+
94+
Assert.Equal(9, samplePoints.Count);
95+
Assert.Contains(new Point(50, 50), samplePoints);
96+
Assert.Contains(new Point(20, 20), samplePoints);
97+
Assert.Contains(new Point(80, 80), samplePoints);
98+
}
99+
100+
[Fact]
101+
public void GetPointProbePoints_ReturnsCenterThenCrosshairNeighbors()
102+
{
103+
IReadOnlyList<Point> probePoints = UIAutomationUtilities.GetPointProbePoints(new Point(25, 40));
104+
105+
Assert.Equal(5, probePoints.Count);
106+
Assert.Equal(new Point(25, 40), probePoints[0]);
107+
Assert.Contains(new Point(23, 40), probePoints);
108+
Assert.Contains(new Point(27, 40), probePoints);
109+
Assert.Contains(new Point(25, 38), probePoints);
110+
Assert.Contains(new Point(25, 42), probePoints);
111+
}
112+
113+
[Fact]
114+
public void TryClipBounds_ReturnsIntersectionForOverlappingRects()
115+
{
116+
bool clipped = UIAutomationUtilities.TryClipBounds(
117+
new Rect(10, 10, 50, 50),
118+
new Rect(30, 25, 50, 50),
119+
out Rect result);
120+
121+
Assert.True(clipped);
122+
Assert.Equal(new Rect(30, 25, 30, 35), result);
123+
}
124+
125+
[Fact]
126+
public void TryClipBounds_ReturnsFalseWhenBoundsDoNotIntersect()
127+
{
128+
bool clipped = UIAutomationUtilities.TryClipBounds(
129+
new Rect(10, 10, 20, 20),
130+
new Rect(100, 100, 20, 20),
131+
out Rect result);
132+
133+
Assert.False(clipped);
134+
Assert.Equal(Rect.Empty, result);
135+
}
136+
137+
[Fact]
138+
public void TryAddUniqueOverlayItem_DeduplicatesNormalizedTextAndBounds()
139+
{
140+
HashSet<string> seen = [];
141+
List<UiAutomationOverlayItem> output = [];
142+
UiAutomationOverlayItem first = new(" Hello world ", new Rect(10.01, 20.01, 30.01, 40.01), UiAutomationOverlaySource.ElementBounds);
143+
UiAutomationOverlayItem second = new("Hello world", new Rect(10.04, 20.04, 30.04, 40.04), UiAutomationOverlaySource.VisibleTextRange);
144+
145+
bool addedFirst = UIAutomationUtilities.TryAddUniqueOverlayItem(first, seen, output);
146+
bool addedSecond = UIAutomationUtilities.TryAddUniqueOverlayItem(second, seen, output);
147+
148+
Assert.True(addedFirst);
149+
Assert.False(addedSecond);
150+
Assert.Single(output);
151+
}
152+
153+
[Fact]
154+
public void SortOverlayItems_OrdersTopThenLeft()
155+
{
156+
IReadOnlyList<UiAutomationOverlayItem> sorted = UIAutomationUtilities.SortOverlayItems(
157+
[
158+
new UiAutomationOverlayItem("Bottom", new Rect(40, 30, 10, 10), UiAutomationOverlaySource.ElementBounds),
159+
new UiAutomationOverlayItem("Right", new Rect(25, 10, 10, 10), UiAutomationOverlaySource.ElementBounds),
160+
new UiAutomationOverlayItem("Left", new Rect(10, 10, 10, 10), UiAutomationOverlaySource.ElementBounds),
161+
]);
162+
163+
Assert.Equal(["Left", "Right", "Bottom"], sorted.Select(item => item.Text));
164+
}
165+
}

‎Text-Grab/App.config‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,9 @@
163163
<setting name="FsgDefaultMode" serializeAs="String">
164164
<value>Default</value>
165165
</setting>
166+
<setting name="FsgSelectionStyle" serializeAs="String">
167+
<value>Region</value>
168+
</setting>
166169
<setting name="FsgShadeOverlay" serializeAs="String">
167170
<value>True</value>
168171
</setting>
@@ -196,6 +199,21 @@
196199
<setting name="OverrideAiArchCheck" serializeAs="String">
197200
<value>False</value>
198201
</setting>
202+
<setting name="UiAutomationEnabled" serializeAs="String">
203+
<value>True</value>
204+
</setting>
205+
<setting name="UiAutomationFallbackToOcr" serializeAs="String">
206+
<value>True</value>
207+
</setting>
208+
<setting name="UiAutomationTraversalMode" serializeAs="String">
209+
<value>Balanced</value>
210+
</setting>
211+
<setting name="UiAutomationIncludeOffscreen" serializeAs="String">
212+
<value>False</value>
213+
</setting>
214+
<setting name="UiAutomationPreferFocusedElement" serializeAs="String">
215+
<value>True</value>
216+
</setting>
199217
<setting name="GrabFrameTranslationEnabled" serializeAs="String">
200218
<value>False</value>
201219
</setting>

‎Text-Grab/Controls/LanguagePicker.xaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
Mode=TwoWay}"
2020
SelectionChanged="MainComboBox_SelectionChanged">
2121
<ComboBox.ItemTemplate>
22-
<DataTemplate DataType="global:Language">
22+
<DataTemplate>
2323
<TextBlock Text="{Binding DisplayName}" />
2424
</DataTemplate>
2525
</ComboBox.ItemTemplate>

0 commit comments

Comments
 (0)