完善UI自动动画的选择元素、滚动元素、展开元素功能

This commit is contained in:
fofolee 2025-01-18 01:14:40 +08:00
parent 76f710323e
commit 053e9793c3
4 changed files with 430 additions and 171 deletions

View File

@ -63,6 +63,9 @@ public class AutomationManager
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool GetCursorPos(out Point lpPoint);
[DllImport("user32.dll")]
private static extern IntPtr WindowFromPoint(Point point);
[DllImport("user32.dll")]
static extern int GetWindowLong(IntPtr hwnd, int index);
@ -335,7 +338,14 @@ public class AutomationManager
{
// 通过窗口句柄查找
int handle = int.Parse(windowArg);
root = AutomationElement.FromHandle(new IntPtr(handle));
try
{
root = AutomationElement.FromHandle(new IntPtr(handle));
}
catch
{
throw new Exception("无法获取指定的窗口");
}
}
else
{
@ -439,95 +449,175 @@ public class AutomationManager
private static void ShowHelp()
{
string help = @"UI自动化工具
Console.WriteLine(@"UI自动化工具 v1.0
: automation.exe -type <> []
:
-window <> 使
: automation.exe <> []
:
1. inspect -
2. click -
-xpath <XPath路径>
-id <AutomationId>
-name <>
-condition ""name=xx;type=Button""
2. click -
: -xpath <XPath路径> [-window <>]
:
- Button ()
- MenuItem ()
- TreeItem ()
- ListItem ()
- TabItem ()
- RadioButton ()
- CheckBox ()
:
- : -xpath ""//Button[@Name='确定']""
- : -xpath ""//MenuBar/MenuItem[@Name='文件']/MenuItem[@Name='打开']""
- : -xpath ""//Tree/TreeItem[@Name='节点1']""
3. setvalue -
-xpath <XPath路径> -value <>
: -xpath <XPath路径> -value <> [-window <>]
4. getvalue -
-xpath <XPath路径>
: -xpath <XPath路径> [-window <>]
5. select -
-xpath <XPath路径> -item <>
5. select -
: -xpath <XPath路径> -item <> [-window <>]
:
- ComboBox ()
*
* ListItem
- TreeView ()
* TreeItem
* 使
- ListBox ()
* ListItem
- DataGrid ()
* DataItem ListItem
- Table ()
* DataItem ListItem
- Tab ()
* TabItem
- MenuBar/Menu ()
* MenuItem
- RadioButton ()
*
- CheckBox ()
*
:
- : -xpath ""//ComboBox"" -item ""选项1""
- : -xpath ""//Tree"" -item ""父节点/子节点""
- : -xpath ""//List"" -item ""列表项1""
- : -xpath ""//Tab"" -item ""标签页2""
- : -xpath ""//MenuBar"" -item ""文件""
- : -xpath ""//RadioButton[@Name='选项1']"" -item ""选项1""
6. expand - /
-xpath <XPath路径> -expand <true/false>
: -xpath <XPath路径> -expand <true/false> [-window <>]
:
- TreeItem () /
- ComboBox () /
- Menu () /
- GroupBox () /
- Expander () /
:
- : -xpath ""//Tree/TreeItem[@Name='父节点']"" -expand true
- : -xpath ""//Tree/TreeItem[@Name='父节点']"" -expand false
- : -xpath ""//ComboBox"" -expand true
- : -xpath ""//ComboBox"" -expand false
- : -xpath ""//Menu/MenuItem[@Name='文件']"" -expand true
- : -xpath ""//GroupBox[@Name='详细信息']"" -expand true
- : -xpath ""//Expander"" -expand true
7. scroll -
-xpath <XPath路径> -direction <vertical/horizontal> -amount <0-100>
: -xpath <XPath路径> -direction <vertical/horizontal> -amount <0-100> [-window <>]
:
- ScrollBar ()
- ListBox ()
- ComboBox ()
- DataGrid ()
- TreeView ()
- TextBox ()
- Document ()
:
- : -xpath ""//List"" -direction vertical -amount 100
- : -xpath ""//DataGrid"" -direction horizontal -amount 50
- : -xpath ""//Edit"" -direction vertical -amount 0
- : -xpath ""//Document"" -direction horizontal -amount 100
8. wait -
-xpath <XPath路径> -timeout <>
: -xpath <XPath路径> -timeout <> [-window <>]
9. focus -
-xpath <XPath路径>
: -xpath <XPath路径> [-window <>]
10. highlight -
-xpath <XPath路径> -duration <>
: -xpath <XPath路径> -duration <> [-window <>]
11. sendkeys -
-xpath <XPath路径> -keys <>
: -xpath <XPath路径> -keys <> [-window <>]
:
- ""abc""
- {} {ENTER}{TAB}
- ^c Ctrl+C
- :
{BACKSPACE}, {BS}, {BKSP} - 退
{BREAK} - Break键
{CAPSLOCK} - Caps Lock键
{DELETE}, {DEL} - Delete键
{DOWN} -
{END} - End键
{ENTER}, {RETURN} - Enter键
{ESC} - Esc键
{HELP} - Help键
{HOME} - Home键
{INSERT}, {INS} - Insert键
{LEFT} -
{NUMLOCK} - Num Lock键
{PGDN} - Page Down键
{PGUP} - Page Up键
{PRTSC} - Print Screen键
{RIGHT} -
{SCROLLLOCK} - Scroll Lock键
{TAB} - Tab键
{UP} -
{F1} - {F12} -
{ADD} -
{SUBTRACT} -
{MULTIPLY} -
{DIVIDE} -
{NUMPAD0} - {NUMPAD9} -
- {} ""{ENTER}""""{TAB}""
- ""^c"" Ctrl+C
:
- {BACKSPACE}, {BS}, {BKSP} - 退
- {BREAK} - Break键
- {CAPSLOCK} - Caps Lock键
- {DELETE}, {DEL} - Delete键
- {DOWN} -
- {END} - End键
- {ENTER}, {RETURN} - Enter键
- {ESC} - Esc键
- {HELP} - Help键
- {HOME} - Home键
- {INSERT}, {INS} - Insert键
- {LEFT} -
- {NUMLOCK} - Num Lock键
- {PGDN} - Page Down键
- {PGUP} - Page Up键
- {PRTSC} - Print Screen键
- {RIGHT} -
- {SCROLLLOCK} - Scroll Lock键
- {TAB} - Tab键
- {UP} -
- {F1} - {F12} -
- {ADD} -
- {SUBTRACT} -
- {MULTIPLY} -
- {DIVIDE} -
- {NUMPAD0} - {NUMPAD9} -
:
+ () - SHIFT
^ () - CTRL
% () - ALT
+ () - SHIFT
^ () - CTRL
% () - ALT
:
""Hello"" - Hello
""{ENTER}"" - Enter
""^c"" - Ctrl+C
""^{HOME}"" - Ctrl+Home
""%{F4}"" - Alt+F4
""+{TAB}"" - Shift+Tab";
- : -keys ""Hello""
- Enter键: -keys ""{ENTER}""
- Ctrl+C: -keys ""^c""
- Ctrl+Home: -keys ""^{HOME}""
- Alt+F4: -keys ""%{F4}""
- Shift+Tab: -keys ""+{TAB}""
Console.WriteLine(help);
:
-window <> 使
:
1. XPath定位
-xpath <XPath路径>
: -xpath ""//Button[@Name='确定']""
2. AutomationId定位
-id <AutomationId>
: -id ""btnOK""
3. Name定位
-name <>
: -name """"
4.
-condition ""name=xx;type=Button;class=xx;automation=xx""
: -condition ""name=;type=Button""");
}
private static string GetArgumentValue(string[] args, string key, bool checkNextArg = true)
@ -600,34 +690,105 @@ public class AutomationManager
throw new Exception("必须指定要选择的项目");
}
try
// 根据控件类型使用不同的查找策略
TreeScope searchScope;
Condition searchCondition;
int controlTypeId = element.Current.ControlType.Id;
if (controlTypeId == ControlType.ComboBox.Id)
{
// ComboBox需要先展开
var expandPattern = element.GetCurrentPattern(ExpandCollapsePattern.Pattern) as ExpandCollapsePattern;
if (expandPattern != null)
{
expandPattern.Expand();
System.Threading.Thread.Sleep(100);
}
searchScope = TreeScope.Descendants;
searchCondition = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.ListItem);
}
else if (controlTypeId == ControlType.Tree.Id)
{
// TreeView查找所有TreeItem
searchScope = TreeScope.Descendants;
searchCondition = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.TreeItem);
}
else if (controlTypeId == ControlType.List.Id || controlTypeId == ControlType.DataGrid.Id || controlTypeId == ControlType.Table.Id)
{
// ListBox, DataGrid, Table 查找直接子项
searchScope = TreeScope.Children;
searchCondition = new OrCondition(
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.ListItem),
new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.DataItem)
);
}
else if (controlTypeId == ControlType.Tab.Id)
{
// Tab查找直接子项
searchScope = TreeScope.Children;
searchCondition = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.TabItem);
}
else if (controlTypeId == ControlType.MenuBar.Id || controlTypeId == ControlType.Menu.Id)
{
// 菜单项查找
searchScope = TreeScope.Descendants;
searchCondition = new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.MenuItem);
}
else if (controlTypeId == ControlType.RadioButton.Id || controlTypeId == ControlType.CheckBox.Id)
{
// 单选框和复选框直接选择自身
if (element.Current.Name == item)
{
var selectionItemPattern = element.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern;
if (selectionItemPattern != null)
{
selectionItemPattern.Select();
Console.WriteLine("true");
return;
}
}
throw new Exception(string.Format("找不到指定的选项: {0}", item));
}
else
{
// 对于其他类型的控件尝试直接使用SelectionPattern
var selectionPattern = element.GetCurrentPattern(SelectionPattern.Pattern) as SelectionPattern;
if (selectionPattern != null)
{
var children = element.FindAll(TreeScope.Children, Condition.TrueCondition);
foreach (AutomationElement child in children)
{
if (child.Current.Name == item)
{
var selectionItemPattern = child.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern;
if (selectionItemPattern != null)
{
selectionItemPattern.Select();
Console.WriteLine("true");
return;
}
}
}
throw new Exception("找不到指定的项目: " + item);
searchScope = TreeScope.Children;
searchCondition = Condition.TrueCondition;
}
else
{
throw new Exception("不支持的控件类型");
}
}
throw new Exception("元素不支持选择操作");
}
catch (Exception ex)
// 查找所有子项
var children = element.FindAll(searchScope, searchCondition);
if (children.Count == 0)
{
throw new Exception("选择操作失败: " + ex.Message);
throw new Exception("未找到可选择的项目");
}
// 遍历查找匹配项并选择
foreach (AutomationElement child in children)
{
if (child.Current.Name == item)
{
// 尝试使用SelectionItemPattern选择
var selectionItemPattern = child.GetCurrentPattern(SelectionItemPattern.Pattern) as SelectionItemPattern;
if (selectionItemPattern != null)
{
selectionItemPattern.Select();
Console.WriteLine("true");
return;
}
}
}
throw new Exception(string.Format("找不到指定的项目: {0}", item));
}
private static void ExpandElement(AutomationElement element, bool expand)
@ -655,24 +816,82 @@ public class AutomationManager
private static void ScrollElement(AutomationElement element, string direction, double amount)
{
if (element == null)
{
throw new Exception("未找到目标元素");
}
try
{
var scrollPattern = element.GetCurrentPattern(ScrollPattern.Pattern) as ScrollPattern;
if (scrollPattern != null)
// 首先尝试使用 ScrollPattern
object scrollPattern;
if (element.TryGetCurrentPattern(ScrollPattern.Pattern, out scrollPattern))
{
var scroll = scrollPattern as ScrollPattern;
if (direction.ToLower() == "horizontal")
scrollPattern.SetScrollPercent(amount, ScrollPattern.NoScroll);
{
if (!scroll.Current.HorizontallyScrollable)
{
throw new Exception("元素不支持水平滚动");
}
scroll.SetScrollPercent(amount, ScrollPattern.NoScroll);
}
else
scrollPattern.SetScrollPercent(ScrollPattern.NoScroll, amount);
{
if (!scroll.Current.VerticallyScrollable)
{
throw new Exception("元素不支持垂直滚动");
}
scroll.SetScrollPercent(ScrollPattern.NoScroll, amount);
}
Console.WriteLine("true");
return;
}
// 尝试使用 ScrollItemPattern
object scrollItemPattern;
if (element.TryGetCurrentPattern(ScrollItemPattern.Pattern, out scrollItemPattern))
{
var scrollItem = scrollItemPattern as ScrollItemPattern;
scrollItem.ScrollIntoView();
Console.WriteLine("true");
return;
}
// 检查是否有滚动条子元素
var scrollBars = element.FindAll(TreeScope.Children, new PropertyCondition(
AutomationElement.ControlTypeProperty, ControlType.ScrollBar));
if (scrollBars.Count > 0)
{
foreach (AutomationElement scrollBar in scrollBars)
{
// 获取滚动条的方向
bool isHorizontal = scrollBar.Current.BoundingRectangle.Width > scrollBar.Current.BoundingRectangle.Height;
if ((direction.ToLower() == "horizontal" && isHorizontal) ||
(direction.ToLower() == "vertical" && !isHorizontal))
{
// 使用 RangeValuePattern 设置滚动条的值
var rangeValuePattern = scrollBar.GetCurrentPattern(RangeValuePattern.Pattern) as RangeValuePattern;
if (rangeValuePattern != null)
{
double maxValue = rangeValuePattern.Current.Maximum;
double minValue = rangeValuePattern.Current.Minimum;
double targetValue = minValue + ((maxValue - minValue) * amount / 100);
rangeValuePattern.SetValue(targetValue);
Console.WriteLine("true");
return;
}
}
}
}
throw new Exception("元素不支持滚动操作");
}
catch (Exception ex)
{
throw new Exception("滚动操作失败: " + ex.Message);
throw new Exception(string.Format("滚动操作失败: {0}", ex.Message));
}
}
@ -842,48 +1061,63 @@ public class AutomationManager
// 循环直到找到根元素
while (current != null && current != AutomationElement.RootElement)
{
var parent = walker.GetParent(current);
// 是否是最后一个元素
bool isLastElement = (parent == AutomationElement.RootElement || parent == null);
// 是否是句柄不为0的窗口
bool isValidWindow = (current.Current.ControlType.Id == UIA_ControlTypeIds.Window && current.Current.NativeWindowHandle != 0);
// 是否是任务栏
bool isTaskbar = (current.Current.ClassName == "Shell_TrayWnd" ||
current.Current.ClassName == "Shell_SecondaryTrayWnd");
// 如果是窗口/任务栏,或者是最后一个元素,获取其句柄
if (isValidWindow || isTaskbar || isLastElement)
// 获取父元素,如果获取失败,则停止遍历
AutomationElement parent;
try
{
windowHandle = new IntPtr(current.Current.NativeWindowHandle);
break; // 获取到句柄后就停止遍历
parent = walker.GetParent(current);
// 是否是最后一个元素
bool isLastElement = (parent == AutomationElement.RootElement || parent == null);
// 是否是句柄不为0的窗口
bool isValidWindow = (current.Current.ControlType.Id == UIA_ControlTypeIds.Window && current.Current.NativeWindowHandle != 0);
// 是否是任务栏
bool isTaskbar = (current.Current.ClassName == "Shell_TrayWnd" ||
current.Current.ClassName == "Shell_SecondaryTrayWnd");
// 如果是窗口/任务栏,或者是最后一个元素,获取其句柄
if (isValidWindow || isTaskbar || isLastElement)
{
windowHandle = new IntPtr(current.Current.NativeWindowHandle);
break; // 获取到句柄后就停止遍历
}
else
{
// 获取同级元素中的索引
int index = 1;
var siblings = parent.FindAll(TreeScope.Children, new PropertyCondition(
AutomationElement.ControlTypeProperty, current.Current.ControlType));
foreach (AutomationElement sibling in siblings)
{
if (sibling == current) break;
index++;
}
// 构建路径段
string type = current.Current.ControlType.ProgrammaticName.Replace("ControlType.", "");
string pathSegment = type;
// 如果有多个同类型元素,添加索引
if (siblings.Count > 1)
{
pathSegment = string.Format("{0}[{1}]", type, index);
}
path.Insert(0, pathSegment);
}
}
else
catch
{
// 获取同级元素中的索引
int index = 1;
var siblings = parent.FindAll(TreeScope.Children, new PropertyCondition(
AutomationElement.ControlTypeProperty, current.Current.ControlType));
foreach (AutomationElement sibling in siblings)
{
if (sibling == current) break;
index++;
}
// 构建路径段
string type = current.Current.ControlType.ProgrammaticName.Replace("ControlType.", "");
string pathSegment = type;
// 如果有多个同类型元素,添加索引
if (siblings.Count > 1)
{
pathSegment = string.Format("{0}[{1}]", type, index);
}
path.Insert(0, pathSegment);
break;
}
current = parent;
}
if (windowHandle == IntPtr.Zero)
{
Point currentMousePosition;
GetCursorPos(out currentMousePosition);
windowHandle = WindowFromPoint(currentMousePosition);
}
return new ElementHierarchyInfo
@ -1002,32 +1236,6 @@ public class AutomationManager
}
}
// 将 HandleKeyPress 方法移到类级别
private static void HandleKeyPress(object sender, KeyPressEventArgs e)
{
if (e.KeyChar == (char)27) // ESC键
{
mouseTimer.Stop();
overlayForm.Close();
previewForm.Close();
Environment.Exit(0);
}
else if (e.KeyChar == 'c' || e.KeyChar == 'C') // 添加复制功能
{
if (lastElement != null)
{
try
{
Clipboard.SetText(lastElement.Current.Name);
}
catch (Exception ex)
{
Console.Error.WriteLine(string.Format("Error copying name: {0}", ex.Message));
}
}
}
}
// 元素检查器
private static void InspectElement(string[] args)
{
@ -1243,7 +1451,8 @@ public class AutomationManager
"名称: {2}\r\n" +
"大小: {3}x{4}\r\n" +
"类型: {5}\r\n" +
"C复制名称ESC退出",
"C复制名称X复制名称并退出\r\n" +
"ESC退出",
cursorPos.X,
cursorPos.Y,
elementName,
@ -1452,4 +1661,39 @@ public class AutomationManager
return element;
}
private static void stopInspect()
{
mouseTimer.Stop();
overlayForm.Close();
previewForm.Close();
Environment.Exit(0);
}
private static void HandleKeyPress(object sender, KeyPressEventArgs e)
{
if (e.KeyChar == (char)27) // ESC键
{
stopInspect();
}
else if (e.KeyChar == 'c' || e.KeyChar == 'C' || e.KeyChar == 'x' || e.KeyChar == 'X') // 添加复制功能
{
if (lastElement != null)
{
try
{
Clipboard.SetText(lastElement.Current.Name);
if (e.KeyChar == 'x' || e.KeyChar == 'X')
{
stopInspect();
}
}
catch (Exception ex)
{
Console.Error.WriteLine(string.Format("Error copying name: {0}", ex.Message));
}
}
}
}
}

View File

@ -164,31 +164,43 @@ const runCsharpFeature = async (feature, args = [], options = {}) => {
const { alwaysBuildNewExe = window.utools.isDev(), killPrevious = true } =
options;
try {
if (killPrevious && currentChild) {
currentChild.kill();
}
const featureExePath = await getCsharpFeatureExe(
feature,
alwaysBuildNewExe
);
if (killPrevious && currentChild) {
quickcommand.kill(currentChild.pid, "SIGKILL");
}
console.log(featureExePath, args.join(" "));
currentChild = child_process.execFile(
featureExePath,
args,
{
encoding: null,
},
(err, stdout, stderr) => {
console.log({
err,
stdout: iconv.decode(stdout, "gbk"),
stderr: iconv.decode(stderr, "gbk"),
});
if (err || Buffer.byteLength(stderr) > 0)
reject(iconv.decode(stderr || stdout, "gbk"));
else reslove(iconv.decode(stdout, "gbk"));
currentChild = child_process.spawn(featureExePath, args, {
encoding: null,
windowsHide: true,
});
let stdoutData = Buffer.from([]);
let stderrData = Buffer.from([]);
currentChild.stdout.on("data", (data) => {
stdoutData = Buffer.concat([stdoutData, data]);
});
currentChild.stderr.on("data", (data) => {
stderrData = Buffer.concat([stderrData, data]);
});
currentChild.on("error", (err) => {
reject(err.toString());
});
currentChild.on("close", (code) => {
if (code !== 0 || stderrData.length > 0) {
reject(
iconv.decode(stderrData.length ? stderrData : stdoutData, "gbk")
);
} else {
reslove(iconv.decode(stdoutData, "gbk"));
}
);
});
} catch (error) {
return reject(error.toString());
}

View File

@ -46,7 +46,7 @@ async function runAutomation(
// 特定命令的参数处理
switch (type) {
case "inspect":
if (params) {
if (params.usePosition) {
args.push("-position");
if (params.x && params.y) {
args.push(`${params.x},${params.y}`);
@ -130,7 +130,10 @@ async function runAutomation(
module.exports = {
inspect: () => runAutomation("inspect"),
inspectPosition: (position) =>
runAutomation("inspect", null, null, null, position || {}),
runAutomation("inspect", null, null, null, {
...position,
usePosition: true,
}),
click: (...args) => runAutomation("click", ...args),
setvalue: (...args) => runAutomation("setvalue", ...args),
getvalue: (...args) => runAutomation("getvalue", ...args),

View File

@ -630,7 +630,7 @@ export const windowsCommands = {
// sendmessage
{
value: "quickcomposer.windows.sendmessage.inspectWindow",
label: "界面自动化(后台)",
label: "发送控制消息",
desc: "Windows界面自动化操作",
icon: "smart_button",
isAsync: true,