diff --git a/plugin/lib/quickcommand.js b/plugin/lib/quickcommand.js
index e9d6987..f033b2b 100644
--- a/plugin/lib/quickcommand.js
+++ b/plugin/lib/quickcommand.js
@@ -157,6 +157,21 @@ const quickcommand = {
writeClipboard: function (text) {
electron.clipboard.writeText(text.toString());
},
+
+ readClipboardImage: function () {
+ // 从剪贴板获取图片
+ const image = electron.clipboard.readImage();
+ if (!image.isEmpty()) {
+ return image.toDataURL();
+ }
+
+ // 尝试获取文本(可能是base64格式的图片)
+ const clipboardText = electron.clipboard.readText();
+ if (clipboardText && clipboardText.startsWith("data:image")) {
+ return clipboardText;
+ }
+ return null;
+ },
};
if (process.platform === "win32") {
diff --git a/plugin/lib/quickcomposer.js b/plugin/lib/quickcomposer.js
index ceac8d6..066fed2 100644
--- a/plugin/lib/quickcomposer.js
+++ b/plugin/lib/quickcomposer.js
@@ -1,5 +1,6 @@
const quickcomposer = {
textProcessing: require("./quickcomposer/textProcessing"),
+ simulate: require("./quickcomposer/simulate"),
};
module.exports = quickcomposer;
diff --git a/plugin/lib/quickcomposer/simulate/imageFinder.js b/plugin/lib/quickcomposer/simulate/imageFinder.js
new file mode 100644
index 0000000..8138109
--- /dev/null
+++ b/plugin/lib/quickcomposer/simulate/imageFinder.js
@@ -0,0 +1,238 @@
+const { nativeImage } = require("electron");
+const { captureScreen } = require("./screenCapture");
+
+// 将颜色值映射到8个区间
+function mapColorValue(val) {
+ if (val > 223) return 7; // [224 ~ 255]
+ if (val > 191) return 6; // [192 ~ 223]
+ if (val > 159) return 5; // [160 ~ 191]
+ if (val > 127) return 4; // [128 ~ 159]
+ if (val > 95) return 3; // [96 ~ 127]
+ if (val > 63) return 2; // [64 ~ 95]
+ if (val > 31) return 1; // [32 ~ 63]
+ return 0; // [0 ~ 31]
+}
+
+// 计算图像特征向量
+function calculateFeatureVector(
+ buffer,
+ width,
+ height,
+ startX = 0,
+ startY = 0,
+ w = width,
+ h = height
+) {
+ // 8^4 = 4096 维向量,表示RGBA各8个区间的组合
+ const vector = new Array(8 * 8 * 8 * 8).fill(0);
+
+ for (let y = startY; y < startY + h; y++) {
+ for (let x = startX; x < startX + w; x++) {
+ const idx = (y * width + x) * 4;
+ // 计算四个通道的量化值
+ const r = mapColorValue(buffer[idx]);
+ const g = mapColorValue(buffer[idx + 1]);
+ const b = mapColorValue(buffer[idx + 2]);
+ const a = mapColorValue(buffer[idx + 3]);
+
+ // 计算在向量中的位置
+ const vectorIdx = r * 512 + g * 64 + b * 8 + a;
+ vector[vectorIdx]++;
+ }
+ }
+
+ return vector;
+}
+
+// 计算余弦相似度
+function calculateCosineSimilarity(v1, v2) {
+ let dotProduct = 0;
+ let norm1 = 0;
+ let norm2 = 0;
+
+ for (let i = 0; i < v1.length; i++) {
+ dotProduct += v1[i] * v2[i];
+ norm1 += v1[i] * v1[i];
+ norm2 += v2[i] * v2[i];
+ }
+
+ return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
+}
+
+// 获取显示器缩放比例
+function getDisplayScale() {
+ if (process.platform === "darwin") {
+ // 在 macOS 上,通过比较实际分辨率和报告的分辨率来计算缩放比例
+ const primaryDisplay = utools.getPrimaryDisplay();
+ const { scaleFactor } = primaryDisplay;
+ return scaleFactor;
+ }
+ return 1;
+}
+
+// 在屏幕上查找图片
+async function findImage(targetImageData, options = {}) {
+ try {
+ // 获取屏幕截图
+ const screenDataUrl = await captureScreen();
+ if (!screenDataUrl) return null;
+
+ // 获取显示器缩放比例
+ const scale = getDisplayScale();
+
+ // 读取屏幕截图
+ const screenImage = nativeImage.createFromDataURL(screenDataUrl);
+ const screenBuffer = screenImage.toBitmap();
+ const { width: actualWidth, height: actualHeight } = screenImage.getSize();
+
+ // 计算缩放后的实际尺寸
+ const screenWidth = Math.round(actualWidth / scale);
+ const screenHeight = Math.round(actualHeight / scale);
+
+ // 从 base64 字符串创建目标图片
+ const targetImage = nativeImage.createFromDataURL(targetImageData);
+ const targetBuffer = targetImage.toBitmap();
+ const { width: targetWidth, height: targetHeight } = targetImage.getSize();
+
+ // 计算目标图片的特征向量
+ const targetVector = calculateFeatureVector(
+ targetBuffer,
+ targetWidth,
+ targetHeight
+ );
+
+ // 设置匹配阈值
+ const threshold = options.threshold || 0.9;
+
+ let bestMatch = null;
+ let bestSimilarity = 0;
+
+ // 使用滑动窗口搜索
+ const stepSize = Math.round(8 * scale); // 根据缩放比例调整步长
+ for (let y = 0; y <= actualHeight - targetHeight; y += stepSize) {
+ for (let x = 0; x <= actualWidth - targetWidth; x += stepSize) {
+ // 计算当前区域的特征向量
+ const regionVector = calculateFeatureVector(
+ screenBuffer,
+ actualWidth,
+ actualHeight,
+ x,
+ y,
+ targetWidth,
+ targetHeight
+ );
+
+ // 计算相似度
+ const similarity = calculateCosineSimilarity(
+ targetVector,
+ regionVector
+ );
+
+ // 更新最佳匹配
+ if (similarity > bestSimilarity) {
+ bestSimilarity = similarity;
+ bestMatch = { x: Math.round(x / scale), y: Math.round(y / scale) };
+
+ // 如果相似度已经很高,进行精确搜索
+ if (similarity >= threshold) {
+ // 在周围进行精确搜索,注意搜索范围也要考虑缩放
+ const searchRange = Math.round(4 * scale);
+ for (let dy = -searchRange; dy <= searchRange; dy++) {
+ for (let dx = -searchRange; dx <= searchRange; dx++) {
+ const newX = x + dx;
+ const newY = y + dy;
+
+ if (
+ newX < 0 ||
+ newY < 0 ||
+ newX > actualWidth - targetWidth ||
+ newY > actualHeight - targetHeight
+ ) {
+ continue;
+ }
+
+ const preciseVector = calculateFeatureVector(
+ screenBuffer,
+ actualWidth,
+ actualHeight,
+ newX,
+ newY,
+ targetWidth,
+ targetHeight
+ );
+
+ const preciseSimilarity = calculateCosineSimilarity(
+ targetVector,
+ preciseVector
+ );
+ if (preciseSimilarity > bestSimilarity) {
+ bestSimilarity = preciseSimilarity;
+ bestMatch = {
+ x: Math.round(newX / scale),
+ y: Math.round(newY / scale),
+ };
+ }
+ }
+ }
+ }
+ }
+
+ // 如果找到足够好的匹配,提前返回
+ if (bestSimilarity >= threshold) {
+ const position = {
+ x: bestMatch.x,
+ y: bestMatch.y,
+ width: Math.round(targetWidth / scale),
+ height: Math.round(targetHeight / scale),
+ confidence: bestSimilarity,
+ };
+
+ clickImage(position, options.mouseAction);
+
+ return position;
+ }
+ }
+ }
+
+ // 如果没有找到足够好的匹配,但有最佳匹配且相似度不太低,也返回
+ if (bestMatch && bestSimilarity > threshold * 0.8) {
+ const position = {
+ x: bestMatch.x,
+ y: bestMatch.y,
+ width: Math.round(targetWidth / scale),
+ height: Math.round(targetHeight / scale),
+ confidence: bestSimilarity,
+ };
+ clickImage(position, options.mouseAction);
+ return position;
+ }
+
+ return null;
+ } catch (error) {
+ console.error("查找图片失败:", error);
+ return null;
+ }
+}
+
+const clickImage = (position, mouseAction) => {
+ // 计算中心点
+ const centerX = position.x + position.width / 2;
+ const centerY = position.y + position.height / 2;
+
+ // 根据配置执行鼠标动作
+ switch (mouseAction) {
+ case "none":
+ break;
+ case "click":
+ window.utools.simulateMouseClick(centerX, centerY);
+ break;
+ case "dblclick":
+ window.utools.simulateMouseDoubleClick(centerX, centerY);
+ break;
+ case "rightclick":
+ window.utools.simulateMouseRightClick(centerX, centerY);
+ break;
+ }
+};
+
+module.exports = { findImage };
diff --git a/plugin/lib/quickcomposer/simulate/index.js b/plugin/lib/quickcomposer/simulate/index.js
new file mode 100644
index 0000000..e191b83
--- /dev/null
+++ b/plugin/lib/quickcomposer/simulate/index.js
@@ -0,0 +1,7 @@
+const { findImage } = require("./imageFinder");
+const { captureScreen } = require("./screenCapture");
+
+module.exports = {
+ findImage,
+ captureScreen,
+};
diff --git a/plugin/lib/quickcomposer/simulate/screenCapture.js b/plugin/lib/quickcomposer/simulate/screenCapture.js
new file mode 100644
index 0000000..3825eaa
--- /dev/null
+++ b/plugin/lib/quickcomposer/simulate/screenCapture.js
@@ -0,0 +1,189 @@
+const { execFile } = require("child_process");
+const { promisify } = require("util");
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+
+const execFileAsync = promisify(execFile);
+const readFileAsync = promisify(fs.readFile);
+const unlinkAsync = promisify(fs.unlink);
+
+// Windows C# 截图代码
+const csharpScript = Buffer.from(
+ `
+using System;
+using System.Runtime.InteropServices;
+using System.Drawing;
+using System.Drawing.Imaging;
+
+public class ScreenCapture {
+ [DllImport("user32.dll")]
+ static extern IntPtr GetDC(IntPtr hwnd);
+
+ [DllImport("user32.dll")]
+ static extern int ReleaseDC(IntPtr hwnd, IntPtr hdc);
+
+ [DllImport("gdi32.dll")]
+ static extern IntPtr CreateCompatibleDC(IntPtr hdc);
+
+ [DllImport("gdi32.dll")]
+ static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int width, int height);
+
+ [DllImport("gdi32.dll")]
+ static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj);
+
+ [DllImport("gdi32.dll")]
+ static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int width, int height,
+ IntPtr hdcSrc, int xSrc, int ySrc, int rop);
+
+ [DllImport("gdi32.dll")]
+ static extern bool DeleteDC(IntPtr hdc);
+
+ [DllImport("gdi32.dll")]
+ static extern bool DeleteObject(IntPtr hObject);
+
+ [DllImport("user32.dll")]
+ static extern bool GetCursorPos(ref Point point);
+
+ [DllImport("user32.dll")]
+ static extern bool GetWindowRect(IntPtr hwnd, ref Rectangle rect);
+
+ public static void CaptureScreen(string outputPath) {
+ IntPtr desktopDC = GetDC(IntPtr.Zero);
+ Rectangle bounds = System.Windows.Forms.Screen.PrimaryScreen.Bounds;
+
+ IntPtr memoryDC = CreateCompatibleDC(desktopDC);
+ IntPtr bitmap = CreateCompatibleBitmap(desktopDC, bounds.Width, bounds.Height);
+ IntPtr oldBitmap = SelectObject(memoryDC, bitmap);
+
+ try {
+ BitBlt(memoryDC, 0, 0, bounds.Width, bounds.Height, desktopDC, 0, 0, 0x00CC0020);
+
+ using (var bmp = Image.FromHbitmap(bitmap)) {
+ bmp.Save(outputPath, ImageFormat.Png);
+ }
+ }
+ finally {
+ SelectObject(memoryDC, oldBitmap);
+ DeleteObject(bitmap);
+ DeleteDC(memoryDC);
+ ReleaseDC(IntPtr.Zero, desktopDC);
+ }
+ }
+}`
+).toString("base64");
+
+// Windows 截图实现
+async function captureWindowsScreen() {
+ const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`);
+
+ try {
+ // 使用base64编码的C#代码执行
+ const command = `
+ $code = [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${csharpScript}'));
+ Add-Type -TypeDefinition $code;
+ [ScreenCapture]::CaptureScreen('${tmpFile.replace(/\\/g, "\\\\")}');
+ `;
+
+ await execFileAsync("powershell", [
+ "-NoProfile",
+ "-NonInteractive",
+ "-ExecutionPolicy",
+ "Bypass",
+ "-Command",
+ command,
+ ]);
+
+ // 读取截图
+ const imageBuffer = await readFileAsync(tmpFile);
+ return `data:image/png;base64,${imageBuffer.toString("base64")}`;
+ } catch (error) {
+ console.error("Windows截图失败:", error);
+ return null;
+ } finally {
+ // 清理临时文件
+ await unlinkAsync(tmpFile).catch(() => {});
+ }
+}
+
+// macOS 截图实现
+async function captureMacScreen() {
+ const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`);
+
+ try {
+ await execFileAsync("screencapture", ["-x", "-C", "-T", "0", tmpFile]);
+ const imageBuffer = await readFileAsync(tmpFile);
+ return `data:image/png;base64,${imageBuffer.toString("base64")}`;
+ } catch (error) {
+ console.error("macOS截图失败:", error);
+ return null;
+ } finally {
+ await unlinkAsync(tmpFile).catch(() => {});
+ }
+}
+
+// Linux 截图实现
+async function captureLinuxScreen() {
+ const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`);
+
+ try {
+ // 检查可用的截图工具
+ let tool = null;
+ try {
+ await execFileAsync("which", ["gnome-screenshot"]);
+ tool = "gnome-screenshot";
+ } catch {
+ try {
+ await execFileAsync("which", ["scrot"]);
+ tool = "scrot";
+ } catch {
+ try {
+ await execFileAsync("which", ["import"]);
+ tool = "import";
+ } catch {
+ console.error("未找到可用的Linux截图工具");
+ return null;
+ }
+ }
+ }
+
+ // 根据可用工具执行截图
+ switch (tool) {
+ case "gnome-screenshot":
+ await execFileAsync("gnome-screenshot", ["-f", tmpFile]);
+ break;
+ case "scrot":
+ await execFileAsync("scrot", [tmpFile]);
+ break;
+ case "import":
+ await execFileAsync("import", ["-window", "root", tmpFile]);
+ break;
+ }
+
+ const imageBuffer = await readFileAsync(tmpFile);
+ return `data:image/png;base64,${imageBuffer.toString("base64")}`;
+ } catch (error) {
+ console.error("Linux截图失败:", error);
+ return null;
+ } finally {
+ await unlinkAsync(tmpFile).catch(() => {});
+ }
+}
+
+// 统一的截图接口
+async function captureScreen() {
+ try {
+ if (process.platform === "darwin") {
+ return await captureMacScreen();
+ } else if (process.platform === "win32") {
+ return await captureWindowsScreen();
+ } else if (process.platform === "linux") {
+ return await captureLinuxScreen();
+ }
+ } catch (error) {
+ console.error("截图失败:", error);
+ }
+ return null;
+}
+
+module.exports = { captureScreen };
diff --git a/src/components/composer/CommandComposer.vue b/src/components/composer/CommandComposer.vue
index 0d09c22..aaaebd6 100644
--- a/src/components/composer/CommandComposer.vue
+++ b/src/components/composer/CommandComposer.vue
@@ -85,7 +85,6 @@ export default defineComponent({
...action,
id: this.nextId++,
argv: "",
- argvType: "string",
saveOutput: false,
outputVariable: null,
cmd: action.value || action.cmd,
diff --git a/src/components/composer/ComposerCard.vue b/src/components/composer/ComposerCard.vue
index 3eb3c8b..caa9d30 100644
--- a/src/components/composer/ComposerCard.vue
+++ b/src/components/composer/ComposerCard.vue
@@ -137,32 +137,12 @@ export default defineComponent({
},
argvLocal: {
get() {
- if (this.command.hasAxiosEditor) {
- // 如果是编辑现有配置
- if (
- this.command.argv &&
- !this.command.argv.includes("axios.") &&
- !this.command.argv.includes("fetch(")
- ) {
- try {
- return JSON.parse(this.command.argv);
- } catch (e) {
- return {};
- }
- }
- // 如果已经是格式化的代码,直接返回
- return this.command.argv || {};
- }
return this.command.argv;
},
set(value) {
const updatedCommand = {
...this.command,
- argv: this.command.hasAxiosEditor
- ? typeof value === "string"
- ? value
- : JSON.stringify(value)
- : value,
+ argv: value,
};
this.$emit("update:command", updatedCommand);
},
diff --git a/src/components/composer/ComposerFlow.vue b/src/components/composer/ComposerFlow.vue
index 8b26fec..b6f828b 100644
--- a/src/components/composer/ComposerFlow.vue
+++ b/src/components/composer/ComposerFlow.vue
@@ -360,8 +360,7 @@ export default defineComponent({
const tempFlow = [
command,
{
- value: "console.log",
- argv: command.outputVariable,
+ argv: `console.log(${command.outputVariable})`,
},
];
// 触发运行事件
diff --git a/src/components/composer/http/AxiosConfigEditor.vue b/src/components/composer/http/AxiosConfigEditor.vue
index a7cd30d..2952d67 100644
--- a/src/components/composer/http/AxiosConfigEditor.vue
+++ b/src/components/composer/http/AxiosConfigEditor.vue
@@ -466,3 +466,10 @@ export default defineComponent({
},
});
+
+
diff --git a/src/components/composer/simulate/ImageSearchEditor.vue b/src/components/composer/simulate/ImageSearchEditor.vue
new file mode 100644
index 0000000..33d32c1
--- /dev/null
+++ b/src/components/composer/simulate/ImageSearchEditor.vue
@@ -0,0 +1,305 @@
+
+
+
+
+
+
+
+
+
+ 移除图片
+
+
+
+
+
+
+ 点击上传或粘贴图片
支持从剪贴板读取
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 从剪贴板读取
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/components/composer/simulate/KeyComboEditor.vue b/src/components/composer/simulate/KeyComboEditor.vue
new file mode 100644
index 0000000..e69de29
diff --git a/src/js/composer/cardComponents.js b/src/js/composer/cardComponents.js
index da20447..e131233 100644
--- a/src/js/composer/cardComponents.js
+++ b/src/js/composer/cardComponents.js
@@ -4,6 +4,9 @@ import { defineAsyncComponent } from "vue";
export const KeyEditor = defineAsyncComponent(() =>
import("components/composer/ui/KeyEditor.vue")
);
+export const ImageSearchEditor = defineAsyncComponent(() =>
+ import("components/composer/simulate/ImageSearchEditor.vue")
+);
// Control Flow Components
export const ConditionalJudgment = defineAsyncComponent(() =>
diff --git a/src/js/composer/commands/simulateCommands.js b/src/js/composer/commands/simulateCommands.js
index e271b9a..f8b9c78 100644
--- a/src/js/composer/commands/simulateCommands.js
+++ b/src/js/composer/commands/simulateCommands.js
@@ -10,7 +10,7 @@ export const simulateCommands = {
component: "KeyEditor",
},
{
- value: "utools",
+ value: "utools.simulateMouseClick",
label: "鼠标点击",
allowEmptyArgv: true,
config: [
@@ -77,5 +77,12 @@ export const simulateCommands = {
config: [],
allowEmptyArgv: true,
},
+ {
+ value: "quickcomposer.simulate.findImage",
+ label: "屏幕找图",
+ component: "ImageSearchEditor",
+ config: [],
+ isAsync: true,
+ },
],
};
diff --git a/src/plugins/monaco/types/quickcommand.api.d.ts b/src/plugins/monaco/types/quickcommand.api.d.ts
index 7f98756..f706505 100644
--- a/src/plugins/monaco/types/quickcommand.api.d.ts
+++ b/src/plugins/monaco/types/quickcommand.api.d.ts
@@ -406,6 +406,11 @@ interface quickcommandApi {
*/
readClipboard(): text;
+ /**
+ * 读剪贴板图片
+ */
+ readClipboardImage(): text;
+
/**
* 写剪贴板
*