diff --git a/plugin/lib/quickcommand.js b/plugin/lib/quickcommand.js index e9d6987..f033b2b 100644 --- a/plugin/lib/quickcommand.js +++ b/plugin/lib/quickcommand.js @@ -157,6 +157,21 @@ const quickcommand = { writeClipboard: function (text) { electron.clipboard.writeText(text.toString()); }, + + readClipboardImage: function () { + // 从剪贴板获取图片 + const image = electron.clipboard.readImage(); + if (!image.isEmpty()) { + return image.toDataURL(); + } + + // 尝试获取文本(可能是base64格式的图片) + const clipboardText = electron.clipboard.readText(); + if (clipboardText && clipboardText.startsWith("data:image")) { + return clipboardText; + } + return null; + }, }; if (process.platform === "win32") { diff --git a/plugin/lib/quickcomposer.js b/plugin/lib/quickcomposer.js index ceac8d6..066fed2 100644 --- a/plugin/lib/quickcomposer.js +++ b/plugin/lib/quickcomposer.js @@ -1,5 +1,6 @@ const quickcomposer = { textProcessing: require("./quickcomposer/textProcessing"), + simulate: require("./quickcomposer/simulate"), }; module.exports = quickcomposer; diff --git a/plugin/lib/quickcomposer/simulate/imageFinder.js b/plugin/lib/quickcomposer/simulate/imageFinder.js new file mode 100644 index 0000000..8138109 --- /dev/null +++ b/plugin/lib/quickcomposer/simulate/imageFinder.js @@ -0,0 +1,238 @@ +const { nativeImage } = require("electron"); +const { captureScreen } = require("./screenCapture"); + +// 将颜色值映射到8个区间 +function mapColorValue(val) { + if (val > 223) return 7; // [224 ~ 255] + if (val > 191) return 6; // [192 ~ 223] + if (val > 159) return 5; // [160 ~ 191] + if (val > 127) return 4; // [128 ~ 159] + if (val > 95) return 3; // [96 ~ 127] + if (val > 63) return 2; // [64 ~ 95] + if (val > 31) return 1; // [32 ~ 63] + return 0; // [0 ~ 31] +} + +// 计算图像特征向量 +function calculateFeatureVector( + buffer, + width, + height, + startX = 0, + startY = 0, + w = width, + h = height +) { + // 8^4 = 4096 维向量,表示RGBA各8个区间的组合 + const vector = new Array(8 * 8 * 8 * 8).fill(0); + + for (let y = startY; y < startY + h; y++) { + for (let x = startX; x < startX + w; x++) { + const idx = (y * width + x) * 4; + // 计算四个通道的量化值 + const r = mapColorValue(buffer[idx]); + const g = mapColorValue(buffer[idx + 1]); + const b = mapColorValue(buffer[idx + 2]); + const a = mapColorValue(buffer[idx + 3]); + + // 计算在向量中的位置 + const vectorIdx = r * 512 + g * 64 + b * 8 + a; + vector[vectorIdx]++; + } + } + + return vector; +} + +// 计算余弦相似度 +function calculateCosineSimilarity(v1, v2) { + let dotProduct = 0; + let norm1 = 0; + let norm2 = 0; + + for (let i = 0; i < v1.length; i++) { + dotProduct += v1[i] * v2[i]; + norm1 += v1[i] * v1[i]; + norm2 += v2[i] * v2[i]; + } + + return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2)); +} + +// 获取显示器缩放比例 +function getDisplayScale() { + if (process.platform === "darwin") { + // 在 macOS 上,通过比较实际分辨率和报告的分辨率来计算缩放比例 + const primaryDisplay = utools.getPrimaryDisplay(); + const { scaleFactor } = primaryDisplay; + return scaleFactor; + } + return 1; +} + +// 在屏幕上查找图片 +async function findImage(targetImageData, options = {}) { + try { + // 获取屏幕截图 + const screenDataUrl = await captureScreen(); + if (!screenDataUrl) return null; + + // 获取显示器缩放比例 + const scale = getDisplayScale(); + + // 读取屏幕截图 + const screenImage = nativeImage.createFromDataURL(screenDataUrl); + const screenBuffer = screenImage.toBitmap(); + const { width: actualWidth, height: actualHeight } = screenImage.getSize(); + + // 计算缩放后的实际尺寸 + const screenWidth = Math.round(actualWidth / scale); + const screenHeight = Math.round(actualHeight / scale); + + // 从 base64 字符串创建目标图片 + const targetImage = nativeImage.createFromDataURL(targetImageData); + const targetBuffer = targetImage.toBitmap(); + const { width: targetWidth, height: targetHeight } = targetImage.getSize(); + + // 计算目标图片的特征向量 + const targetVector = calculateFeatureVector( + targetBuffer, + targetWidth, + targetHeight + ); + + // 设置匹配阈值 + const threshold = options.threshold || 0.9; + + let bestMatch = null; + let bestSimilarity = 0; + + // 使用滑动窗口搜索 + const stepSize = Math.round(8 * scale); // 根据缩放比例调整步长 + for (let y = 0; y <= actualHeight - targetHeight; y += stepSize) { + for (let x = 0; x <= actualWidth - targetWidth; x += stepSize) { + // 计算当前区域的特征向量 + const regionVector = calculateFeatureVector( + screenBuffer, + actualWidth, + actualHeight, + x, + y, + targetWidth, + targetHeight + ); + + // 计算相似度 + const similarity = calculateCosineSimilarity( + targetVector, + regionVector + ); + + // 更新最佳匹配 + if (similarity > bestSimilarity) { + bestSimilarity = similarity; + bestMatch = { x: Math.round(x / scale), y: Math.round(y / scale) }; + + // 如果相似度已经很高,进行精确搜索 + if (similarity >= threshold) { + // 在周围进行精确搜索,注意搜索范围也要考虑缩放 + const searchRange = Math.round(4 * scale); + for (let dy = -searchRange; dy <= searchRange; dy++) { + for (let dx = -searchRange; dx <= searchRange; dx++) { + const newX = x + dx; + const newY = y + dy; + + if ( + newX < 0 || + newY < 0 || + newX > actualWidth - targetWidth || + newY > actualHeight - targetHeight + ) { + continue; + } + + const preciseVector = calculateFeatureVector( + screenBuffer, + actualWidth, + actualHeight, + newX, + newY, + targetWidth, + targetHeight + ); + + const preciseSimilarity = calculateCosineSimilarity( + targetVector, + preciseVector + ); + if (preciseSimilarity > bestSimilarity) { + bestSimilarity = preciseSimilarity; + bestMatch = { + x: Math.round(newX / scale), + y: Math.round(newY / scale), + }; + } + } + } + } + } + + // 如果找到足够好的匹配,提前返回 + if (bestSimilarity >= threshold) { + const position = { + x: bestMatch.x, + y: bestMatch.y, + width: Math.round(targetWidth / scale), + height: Math.round(targetHeight / scale), + confidence: bestSimilarity, + }; + + clickImage(position, options.mouseAction); + + return position; + } + } + } + + // 如果没有找到足够好的匹配,但有最佳匹配且相似度不太低,也返回 + if (bestMatch && bestSimilarity > threshold * 0.8) { + const position = { + x: bestMatch.x, + y: bestMatch.y, + width: Math.round(targetWidth / scale), + height: Math.round(targetHeight / scale), + confidence: bestSimilarity, + }; + clickImage(position, options.mouseAction); + return position; + } + + return null; + } catch (error) { + console.error("查找图片失败:", error); + return null; + } +} + +const clickImage = (position, mouseAction) => { + // 计算中心点 + const centerX = position.x + position.width / 2; + const centerY = position.y + position.height / 2; + + // 根据配置执行鼠标动作 + switch (mouseAction) { + case "none": + break; + case "click": + window.utools.simulateMouseClick(centerX, centerY); + break; + case "dblclick": + window.utools.simulateMouseDoubleClick(centerX, centerY); + break; + case "rightclick": + window.utools.simulateMouseRightClick(centerX, centerY); + break; + } +}; + +module.exports = { findImage }; diff --git a/plugin/lib/quickcomposer/simulate/index.js b/plugin/lib/quickcomposer/simulate/index.js new file mode 100644 index 0000000..e191b83 --- /dev/null +++ b/plugin/lib/quickcomposer/simulate/index.js @@ -0,0 +1,7 @@ +const { findImage } = require("./imageFinder"); +const { captureScreen } = require("./screenCapture"); + +module.exports = { + findImage, + captureScreen, +}; diff --git a/plugin/lib/quickcomposer/simulate/screenCapture.js b/plugin/lib/quickcomposer/simulate/screenCapture.js new file mode 100644 index 0000000..3825eaa --- /dev/null +++ b/plugin/lib/quickcomposer/simulate/screenCapture.js @@ -0,0 +1,189 @@ +const { execFile } = require("child_process"); +const { promisify } = require("util"); +const fs = require("fs"); +const path = require("path"); +const os = require("os"); + +const execFileAsync = promisify(execFile); +const readFileAsync = promisify(fs.readFile); +const unlinkAsync = promisify(fs.unlink); + +// Windows C# 截图代码 +const csharpScript = Buffer.from( + ` +using System; +using System.Runtime.InteropServices; +using System.Drawing; +using System.Drawing.Imaging; + +public class ScreenCapture { + [DllImport("user32.dll")] + static extern IntPtr GetDC(IntPtr hwnd); + + [DllImport("user32.dll")] + static extern int ReleaseDC(IntPtr hwnd, IntPtr hdc); + + [DllImport("gdi32.dll")] + static extern IntPtr CreateCompatibleDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int width, int height); + + [DllImport("gdi32.dll")] + static extern IntPtr SelectObject(IntPtr hdc, IntPtr hgdiobj); + + [DllImport("gdi32.dll")] + static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int width, int height, + IntPtr hdcSrc, int xSrc, int ySrc, int rop); + + [DllImport("gdi32.dll")] + static extern bool DeleteDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + static extern bool DeleteObject(IntPtr hObject); + + [DllImport("user32.dll")] + static extern bool GetCursorPos(ref Point point); + + [DllImport("user32.dll")] + static extern bool GetWindowRect(IntPtr hwnd, ref Rectangle rect); + + public static void CaptureScreen(string outputPath) { + IntPtr desktopDC = GetDC(IntPtr.Zero); + Rectangle bounds = System.Windows.Forms.Screen.PrimaryScreen.Bounds; + + IntPtr memoryDC = CreateCompatibleDC(desktopDC); + IntPtr bitmap = CreateCompatibleBitmap(desktopDC, bounds.Width, bounds.Height); + IntPtr oldBitmap = SelectObject(memoryDC, bitmap); + + try { + BitBlt(memoryDC, 0, 0, bounds.Width, bounds.Height, desktopDC, 0, 0, 0x00CC0020); + + using (var bmp = Image.FromHbitmap(bitmap)) { + bmp.Save(outputPath, ImageFormat.Png); + } + } + finally { + SelectObject(memoryDC, oldBitmap); + DeleteObject(bitmap); + DeleteDC(memoryDC); + ReleaseDC(IntPtr.Zero, desktopDC); + } + } +}` +).toString("base64"); + +// Windows 截图实现 +async function captureWindowsScreen() { + const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`); + + try { + // 使用base64编码的C#代码执行 + const command = ` + $code = [System.Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${csharpScript}')); + Add-Type -TypeDefinition $code; + [ScreenCapture]::CaptureScreen('${tmpFile.replace(/\\/g, "\\\\")}'); + `; + + await execFileAsync("powershell", [ + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-Command", + command, + ]); + + // 读取截图 + const imageBuffer = await readFileAsync(tmpFile); + return `data:image/png;base64,${imageBuffer.toString("base64")}`; + } catch (error) { + console.error("Windows截图失败:", error); + return null; + } finally { + // 清理临时文件 + await unlinkAsync(tmpFile).catch(() => {}); + } +} + +// macOS 截图实现 +async function captureMacScreen() { + const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`); + + try { + await execFileAsync("screencapture", ["-x", "-C", "-T", "0", tmpFile]); + const imageBuffer = await readFileAsync(tmpFile); + return `data:image/png;base64,${imageBuffer.toString("base64")}`; + } catch (error) { + console.error("macOS截图失败:", error); + return null; + } finally { + await unlinkAsync(tmpFile).catch(() => {}); + } +} + +// Linux 截图实现 +async function captureLinuxScreen() { + const tmpFile = path.join(os.tmpdir(), `screen-${Date.now()}.png`); + + try { + // 检查可用的截图工具 + let tool = null; + try { + await execFileAsync("which", ["gnome-screenshot"]); + tool = "gnome-screenshot"; + } catch { + try { + await execFileAsync("which", ["scrot"]); + tool = "scrot"; + } catch { + try { + await execFileAsync("which", ["import"]); + tool = "import"; + } catch { + console.error("未找到可用的Linux截图工具"); + return null; + } + } + } + + // 根据可用工具执行截图 + switch (tool) { + case "gnome-screenshot": + await execFileAsync("gnome-screenshot", ["-f", tmpFile]); + break; + case "scrot": + await execFileAsync("scrot", [tmpFile]); + break; + case "import": + await execFileAsync("import", ["-window", "root", tmpFile]); + break; + } + + const imageBuffer = await readFileAsync(tmpFile); + return `data:image/png;base64,${imageBuffer.toString("base64")}`; + } catch (error) { + console.error("Linux截图失败:", error); + return null; + } finally { + await unlinkAsync(tmpFile).catch(() => {}); + } +} + +// 统一的截图接口 +async function captureScreen() { + try { + if (process.platform === "darwin") { + return await captureMacScreen(); + } else if (process.platform === "win32") { + return await captureWindowsScreen(); + } else if (process.platform === "linux") { + return await captureLinuxScreen(); + } + } catch (error) { + console.error("截图失败:", error); + } + return null; +} + +module.exports = { captureScreen }; diff --git a/src/components/composer/CommandComposer.vue b/src/components/composer/CommandComposer.vue index 0d09c22..aaaebd6 100644 --- a/src/components/composer/CommandComposer.vue +++ b/src/components/composer/CommandComposer.vue @@ -85,7 +85,6 @@ export default defineComponent({ ...action, id: this.nextId++, argv: "", - argvType: "string", saveOutput: false, outputVariable: null, cmd: action.value || action.cmd, diff --git a/src/components/composer/ComposerCard.vue b/src/components/composer/ComposerCard.vue index 3eb3c8b..caa9d30 100644 --- a/src/components/composer/ComposerCard.vue +++ b/src/components/composer/ComposerCard.vue @@ -137,32 +137,12 @@ export default defineComponent({ }, argvLocal: { get() { - if (this.command.hasAxiosEditor) { - // 如果是编辑现有配置 - if ( - this.command.argv && - !this.command.argv.includes("axios.") && - !this.command.argv.includes("fetch(") - ) { - try { - return JSON.parse(this.command.argv); - } catch (e) { - return {}; - } - } - // 如果已经是格式化的代码,直接返回 - return this.command.argv || {}; - } return this.command.argv; }, set(value) { const updatedCommand = { ...this.command, - argv: this.command.hasAxiosEditor - ? typeof value === "string" - ? value - : JSON.stringify(value) - : value, + argv: value, }; this.$emit("update:command", updatedCommand); }, diff --git a/src/components/composer/ComposerFlow.vue b/src/components/composer/ComposerFlow.vue index 8b26fec..b6f828b 100644 --- a/src/components/composer/ComposerFlow.vue +++ b/src/components/composer/ComposerFlow.vue @@ -360,8 +360,7 @@ export default defineComponent({ const tempFlow = [ command, { - value: "console.log", - argv: command.outputVariable, + argv: `console.log(${command.outputVariable})`, }, ]; // 触发运行事件 diff --git a/src/components/composer/http/AxiosConfigEditor.vue b/src/components/composer/http/AxiosConfigEditor.vue index a7cd30d..2952d67 100644 --- a/src/components/composer/http/AxiosConfigEditor.vue +++ b/src/components/composer/http/AxiosConfigEditor.vue @@ -466,3 +466,10 @@ export default defineComponent({ }, }); + + diff --git a/src/components/composer/simulate/ImageSearchEditor.vue b/src/components/composer/simulate/ImageSearchEditor.vue new file mode 100644 index 0000000..33d32c1 --- /dev/null +++ b/src/components/composer/simulate/ImageSearchEditor.vue @@ -0,0 +1,305 @@ + + + + + diff --git a/src/components/composer/simulate/KeyComboEditor.vue b/src/components/composer/simulate/KeyComboEditor.vue new file mode 100644 index 0000000..e69de29 diff --git a/src/js/composer/cardComponents.js b/src/js/composer/cardComponents.js index da20447..e131233 100644 --- a/src/js/composer/cardComponents.js +++ b/src/js/composer/cardComponents.js @@ -4,6 +4,9 @@ import { defineAsyncComponent } from "vue"; export const KeyEditor = defineAsyncComponent(() => import("components/composer/ui/KeyEditor.vue") ); +export const ImageSearchEditor = defineAsyncComponent(() => + import("components/composer/simulate/ImageSearchEditor.vue") +); // Control Flow Components export const ConditionalJudgment = defineAsyncComponent(() => diff --git a/src/js/composer/commands/simulateCommands.js b/src/js/composer/commands/simulateCommands.js index e271b9a..f8b9c78 100644 --- a/src/js/composer/commands/simulateCommands.js +++ b/src/js/composer/commands/simulateCommands.js @@ -10,7 +10,7 @@ export const simulateCommands = { component: "KeyEditor", }, { - value: "utools", + value: "utools.simulateMouseClick", label: "鼠标点击", allowEmptyArgv: true, config: [ @@ -77,5 +77,12 @@ export const simulateCommands = { config: [], allowEmptyArgv: true, }, + { + value: "quickcomposer.simulate.findImage", + label: "屏幕找图", + component: "ImageSearchEditor", + config: [], + isAsync: true, + }, ], }; diff --git a/src/plugins/monaco/types/quickcommand.api.d.ts b/src/plugins/monaco/types/quickcommand.api.d.ts index 7f98756..f706505 100644 --- a/src/plugins/monaco/types/quickcommand.api.d.ts +++ b/src/plugins/monaco/types/quickcommand.api.d.ts @@ -406,6 +406,11 @@ interface quickcommandApi { */ readClipboard(): text; + /** + * 读剪贴板图片 + */ + readClipboardImage(): text; + /** * 写剪贴板 *