this repo has no description
Add tap by accessibility id/label
+190
-40
+6
README.md
+6
README.md
···
123
123
124
124
# Basic interactions
125
125
axe tap -x 100 -y 200 --udid $UDID
126
+
axe tap --id "Safari" --udid $UDID
127
+
axe tap --label "Safari" --udid $UDID
126
128
axe type 'Hello World!' --udid $UDID
127
129
axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid $UDID
128
130
axe button home --udid $UDID
···
144
146
# Tap at coordinates
145
147
axe tap -x 100 -y 200 --udid SIMULATOR_UDID
146
148
axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID
149
+
150
+
# Tap by accessibility element (uses describe-ui accessibility tree)
151
+
axe tap --id "Safari" --udid SIMULATOR_UDID
152
+
axe tap --label "Safari" --udid SIMULATOR_UDID
147
153
148
154
# Swipe gestures
149
155
axe swipe --start-x 100 --start-y 300 --end-x 300 --end-y 100 --udid SIMULATOR_UDID
+5
-1
Sources/AXe/Commands/DescribeUI.swift
+5
-1
Sources/AXe/Commands/DescribeUI.swift
···
37
37
}
38
38
39
39
// Fetch accessibility information
40
-
try await AccessibilityFetcher.fetchAccessibilityInfo(for: simulatorUDID, logger: logger)
40
+
let jsonData = try await AccessibilityFetcher.fetchAccessibilityInfoJSONData(for: simulatorUDID, logger: logger)
41
+
guard let jsonString = String(data: jsonData, encoding: .utf8) else {
42
+
throw CLIError(errorDescription: "Failed to convert accessibility info to JSON string.")
43
+
}
44
+
print(jsonString)
41
45
}
42
46
}
+87
-9
Sources/AXe/Commands/Tap.swift
+87
-9
Sources/AXe/Commands/Tap.swift
···
5
5
6
6
struct Tap: AsyncParsableCommand {
7
7
static let configuration = CommandConfiguration(
8
-
abstract: "Tap on a specific point on the screen."
8
+
abstract: "Tap on a specific point on the screen, or locate an element by accessibility and tap its center."
9
9
)
10
10
11
11
@Option(name: .customShort("x"), help: "The X coordinate of the point to tap.")
12
-
var pointX: Double
12
+
var pointX: Double?
13
13
14
14
@Option(name: .customShort("y"), help: "The Y coordinate of the point to tap.")
15
-
var pointY: Double
15
+
var pointY: Double?
16
+
17
+
@Option(name: [.customLong("id")], help: "Tap the center of the element matching AXUniqueId (accessibilityIdentifier). Ignored if -x and -y are provided.")
18
+
var elementID: String?
19
+
20
+
@Option(name: [.customLong("label")], help: "Tap the center of the element matching AXLabel (accessibilityLabel). Ignored if -x and -y are provided.")
21
+
var elementLabel: String?
16
22
17
23
@Option(name: .customLong("pre-delay"), help: "Delay before tapping in seconds.")
18
24
var preDelay: Double?
···
24
30
var simulatorUDID: String
25
31
26
32
func validate() throws {
27
-
// Validate coordinates are non-negative
28
-
guard pointX >= 0, pointY >= 0 else {
29
-
throw ValidationError("Coordinates must be non-negative values.")
33
+
if pointX != nil || pointY != nil {
34
+
guard let pointX, let pointY else {
35
+
throw ValidationError("Both -x and -y must be provided together.")
36
+
}
37
+
guard pointX >= 0, pointY >= 0 else {
38
+
throw ValidationError("Coordinates must be non-negative values.")
39
+
}
40
+
} else {
41
+
if elementID == nil && elementLabel == nil {
42
+
throw ValidationError("Either provide both -x/-y, or use --id/--label to tap an element.")
43
+
}
44
+
if elementID != nil && elementLabel != nil {
45
+
throw ValidationError("Use only one of --id or --label.")
46
+
}
47
+
if let elementID, elementID.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
48
+
throw ValidationError("--id must not be empty.")
49
+
}
50
+
if let elementLabel, elementLabel.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
51
+
throw ValidationError("--label must not be empty.")
52
+
}
30
53
}
31
54
32
55
// Validate delays if provided
···
49
72
50
73
try await performGlobalSetup(logger: logger)
51
74
52
-
logger.info().log("Tapping at (\(pointX), \(pointY))")
75
+
let resolvedPoint: (x: Double, y: Double)
76
+
let resolvedDescription: String
77
+
78
+
if let pointX, let pointY {
79
+
resolvedPoint = (x: pointX, y: pointY)
80
+
resolvedDescription = "(\(pointX), \(pointY))"
81
+
} else {
82
+
let roots = try await AccessibilityFetcher.fetchAccessibilityElements(for: simulatorUDID, logger: logger)
83
+
let element: AccessibilityElement = try locateElement(in: roots)
84
+
85
+
guard let frame = element.frame else {
86
+
print("Warning: Matched element has no frame. No tap performed.", to: &standardError)
87
+
throw CLIError(errorDescription: "Matched element has no frame.")
88
+
}
89
+
guard frame.width > 0, frame.height > 0 else {
90
+
print("Warning: Matched element has an invalid frame size (\(frame.width)x\(frame.height)). No tap performed.", to: &standardError)
91
+
throw CLIError(errorDescription: "Matched element has an invalid frame size.")
92
+
}
93
+
94
+
let centerX = frame.x + (frame.width / 2.0)
95
+
let centerY = frame.y + (frame.height / 2.0)
96
+
resolvedPoint = (x: centerX, y: centerY)
97
+
resolvedDescription = "center of matched element at (\(centerX), \(centerY))"
98
+
}
99
+
100
+
logger.info().log("Tapping at \(resolvedDescription)")
53
101
54
102
// Create tap events with timing controls
55
103
var events: [FBSimulatorHIDEvent] = []
···
61
109
}
62
110
63
111
// Add the main tap event
64
-
let tapEvent = FBSimulatorHIDEvent.tapAt(x: pointX, y: pointY)
112
+
let tapEvent = FBSimulatorHIDEvent.tapAt(x: resolvedPoint.x, y: resolvedPoint.y)
65
113
events.append(tapEvent)
66
114
67
115
// Add post-delay if specified
···
84
132
logger.info().log("Tap completed successfully")
85
133
86
134
// Output success message to stdout
87
-
print("✓ Tap at (\(pointX), \(pointY)) completed successfully")
135
+
print("✓ Tap at (\(resolvedPoint.x), \(resolvedPoint.y)) completed successfully")
136
+
}
137
+
138
+
private func locateElement(in roots: [AccessibilityElement]) throws -> AccessibilityElement {
139
+
let allElements = roots.flatMap { $0.flattened() }
140
+
141
+
if let elementID {
142
+
let query = elementID.trimmingCharacters(in: .whitespacesAndNewlines)
143
+
let matches = allElements.filter { $0.normalizedUniqueId == query }
144
+
return try selectUniqueMatch(matches, kind: "--id", value: elementID)
145
+
}
146
+
147
+
if let elementLabel {
148
+
let query = elementLabel.trimmingCharacters(in: .whitespacesAndNewlines)
149
+
let matches = allElements.filter { $0.normalizedLabel == query }
150
+
return try selectUniqueMatch(matches, kind: "--label", value: elementLabel)
151
+
}
152
+
153
+
throw CLIError(errorDescription: "Unexpected state: no coordinates and no element query.")
154
+
}
155
+
156
+
private func selectUniqueMatch(_ matches: [AccessibilityElement], kind: String, value: String) throws -> AccessibilityElement {
157
+
guard !matches.isEmpty else {
158
+
print("Warning: No accessibility element matched \(kind) '\(value)'. No tap performed.", to: &standardError)
159
+
throw CLIError(errorDescription: "No accessibility element matched \(kind) '\(value)'.")
160
+
}
161
+
guard matches.count == 1 else {
162
+
print("Warning: Multiple (\(matches.count)) accessibility elements matched \(kind) '\(value)'. No tap performed.", to: &standardError)
163
+
throw CLIError(errorDescription: "Multiple accessibility elements matched \(kind) '\(value)'.")
164
+
}
165
+
return matches[0]
88
166
}
89
167
}
+33
Sources/AXe/Utilities/AccessibilityElement.swift
+33
Sources/AXe/Utilities/AccessibilityElement.swift
···
1
+
import Foundation
2
+
3
+
struct AccessibilityElement: Decodable {
4
+
struct Frame: Decodable {
5
+
let x: Double
6
+
let y: Double
7
+
let width: Double
8
+
let height: Double
9
+
}
10
+
11
+
let type: String?
12
+
let frame: Frame?
13
+
let children: [AccessibilityElement]?
14
+
15
+
let AXLabel: String?
16
+
let AXUniqueId: String?
17
+
18
+
var normalizedLabel: String? {
19
+
AXLabel?.trimmingCharacters(in: .whitespacesAndNewlines)
20
+
}
21
+
22
+
var normalizedUniqueId: String? {
23
+
AXUniqueId?.trimmingCharacters(in: .whitespacesAndNewlines)
24
+
}
25
+
26
+
func flattened() -> [AccessibilityElement] {
27
+
var result: [AccessibilityElement] = [self]
28
+
if let children {
29
+
result.append(contentsOf: children.flatMap { $0.flattened() })
30
+
}
31
+
return result
32
+
}
33
+
}
+20
-29
Sources/AXe/Utilities/AccessibilityFetcher.swift
+20
-29
Sources/AXe/Utilities/AccessibilityFetcher.swift
···
5
5
// MARK: - Accessibility Fetcher
6
6
@MainActor
7
7
struct AccessibilityFetcher {
8
-
static func fetchAccessibilityInfo(for simulatorUDID: String, logger: AxeLogger) async throws {
9
-
logger.info().log("Accessibility Info Fetcher started for simulator UDID: \(simulatorUDID)")
10
-
8
+
static func fetchAccessibilityInfoJSONData(for simulatorUDID: String, logger: AxeLogger) async throws -> Data {
11
9
let simulatorSet = try await getSimulatorSet(deviceSetPath: nil, logger: logger, reporter: EmptyEventReporter.shared)
12
-
logger.info().log("FBSimulatorSet obtained.")
13
-
10
+
14
11
guard let target = simulatorSet.allSimulators.first(where: { $0.udid == simulatorUDID }) else {
15
12
throw CLIError(errorDescription: "Simulator with UDID \(simulatorUDID) not found in set.")
16
13
}
17
-
logger.info().log("Target (FBSimulator) obtained: \(target.udid) - \(target.name), State: \(FBiOSTargetStateStringFromState(target.state))")
18
14
19
-
logger.info().log("Fetching accessibility info directly from FBSimulator...")
20
-
// FBSimulator conforms to FBAccessibilityCommands, which has accessibilityElementsWithNestedFormat:
21
-
// It returns FBFuture<id> which becomes FBFuture<AnyObject> in Swift.
15
+
// FBSimulator conforms to FBAccessibilityCommands.
22
16
let accessibilityInfoFuture: FBFuture<AnyObject> = target.accessibilityElements(withNestedFormat: true)
23
-
24
17
let infoAnyObject: AnyObject = try await FutureBridge.value(accessibilityInfoFuture)
25
-
logger.info().log("Accessibility info raw object (AnyObject) received.")
26
18
27
-
// Check if it's NSDictionary or NSArray, as both are valid top-level JSON structures.
28
-
let jsonData: Data
29
19
if let nsDict = infoAnyObject as? NSDictionary {
30
-
logger.info().log("Successfully cast to NSDictionary.")
31
-
jsonData = try JSONSerialization.data(withJSONObject: nsDict, options: [.prettyPrinted])
32
-
} else if let nsArray = infoAnyObject as? NSArray {
33
-
logger.info().log("Successfully cast to NSArray.")
34
-
jsonData = try JSONSerialization.data(withJSONObject: nsArray, options: [.prettyPrinted])
35
-
} else {
36
-
logger.error().log("Accessibility info was not an NSDictionary or NSArray as expected. Type: \(type(of: infoAnyObject))")
37
-
throw CLIError(errorDescription: "Accessibility info was not a dictionary or array as expected.")
20
+
return try JSONSerialization.data(withJSONObject: nsDict, options: [.prettyPrinted])
21
+
}
22
+
if let nsArray = infoAnyObject as? NSArray {
23
+
return try JSONSerialization.data(withJSONObject: nsArray, options: [.prettyPrinted])
38
24
}
39
-
40
-
if let jsonString = String(data: jsonData, encoding: .utf8) {
41
-
print(jsonString)
42
-
} else {
43
-
logger.error().log("Failed to convert accessibility info to JSON string.")
44
-
throw CLIError(errorDescription: "Failed to convert accessibility info to JSON string.")
25
+
26
+
throw CLIError(errorDescription: "Accessibility info was not a dictionary or array as expected.")
27
+
}
28
+
29
+
static func fetchAccessibilityElements(for simulatorUDID: String, logger: AxeLogger) async throws -> [AccessibilityElement] {
30
+
let jsonData = try await fetchAccessibilityInfoJSONData(for: simulatorUDID, logger: logger)
31
+
let decoder = JSONDecoder()
32
+
33
+
if let roots = try? decoder.decode([AccessibilityElement].self, from: jsonData) {
34
+
return roots
45
35
}
46
-
47
-
logger.info().log("Accessibility Info Fetcher finished successfully.")
36
+
37
+
let root = try decoder.decode(AccessibilityElement.self, from: jsonData)
38
+
return [root]
48
39
}
49
40
}
+34
Tests/TapTests.swift
+34
Tests/TapTests.swift
···
19
19
#expect(tapCountElement?.label == "Tap Count: 1", "Tap count should be 1")
20
20
#expect(tapLocationElement?.label == "Tap Location: (200, 400)", "Tap location should be (200, 400)")
21
21
}
22
+
23
+
@Test("Tap by AXUniqueId navigates back to home")
24
+
func tapByIDNavigatesBack() async throws {
25
+
// Arrange
26
+
try await TestHelpers.launchPlaygroundApp(to: "tap-test")
27
+
28
+
// Act
29
+
try await TestHelpers.runAxeCommand("tap --id BackButton", simulatorUDID: defaultSimulatorUDID)
30
+
try await Task.sleep(nanoseconds: 1_000_000_000)
31
+
32
+
// Assert
33
+
let uiState = try await TestHelpers.getUIState()
34
+
let homeMarker = UIStateParser.findElementContainingLabel(in: uiState, containing: "Touch & Gestures")
35
+
let tapTestMarker = UIStateParser.findElementContainingLabel(in: uiState, containing: "Tap Count:")
36
+
#expect(homeMarker != nil)
37
+
#expect(tapTestMarker == nil)
38
+
}
39
+
40
+
@Test("Tap by AXLabel navigates back to home")
41
+
func tapByLabelNavigatesBack() async throws {
42
+
// Arrange
43
+
try await TestHelpers.launchPlaygroundApp(to: "tap-test")
44
+
45
+
// Act
46
+
try await TestHelpers.runAxeCommand("tap --label 'AXe Playground'", simulatorUDID: defaultSimulatorUDID)
47
+
try await Task.sleep(nanoseconds: 1_000_000_000)
48
+
49
+
// Assert
50
+
let uiState = try await TestHelpers.getUIState()
51
+
let homeMarker = UIStateParser.findElementContainingLabel(in: uiState, containing: "Touch & Gestures")
52
+
let tapTestMarker = UIStateParser.findElementContainingLabel(in: uiState, containing: "Tap Count:")
53
+
#expect(homeMarker != nil)
54
+
#expect(tapTestMarker == nil)
55
+
}
22
56
23
57
@Test("Multiple taps register correct count")
24
58
func multipleTaps() async throws {
+5
-1
USAGE_EXAMPLES.md
+5
-1
USAGE_EXAMPLES.md
···
41
41
# Simple tap
42
42
axe tap -x 100 -y 200 --udid SIMULATOR_UDID
43
43
44
+
# Tap by accessibility element (uses describe-ui accessibility tree)
45
+
axe tap --id "Safari" --udid SIMULATOR_UDID
46
+
axe tap --label "Safari" --udid SIMULATOR_UDID
47
+
44
48
# Tap with timing controls
45
49
axe tap -x 100 -y 200 --pre-delay 1.0 --post-delay 0.5 --udid SIMULATOR_UDID
46
50
···
389
393
# Modifier keys
390
394
LeftCtrl=224, LeftShift=225, LeftAlt=226, LeftGUI=227
391
395
RightCtrl=228, RightShift=229, RightAlt=230, RightGUI=231
392
-
```
396
+
```