diff --git a/TODO.md b/TODO.md index 72aabdc..58dc650 100644 --- a/TODO.md +++ b/TODO.md @@ -254,6 +254,43 @@ - [ ] **OCR 本轮未实现**:设置里只有字段与横幅说明("本轮未实现,仅保存配置"),不假装能识别。实现需要引入引擎与中文语言包,并明确截图可能含患者信息、默认不上传外部服务 - [x] 回归:三来源控件树(UIA 56 / MSAA 167 / Win32 7,含 ListView 句柄)、拾取下钻、列表读取三例 5 行×3 列、布局自检 93 个控件"异常区域 0 个"、`--read-saved` 仍 found/取值(实测读到 `系统`,随设置页面变化) +#### 第十轮·段一:界面小修 + +- [x] 先用 git 固化可回退基线(仓库此前没有任何提交):`b4c736d 基线提交` +- [x] **窗口列表**颜色标识:Z 序最前的一行浅黄底(屏幕上最上层)、真实前台窗口浅绿底、最小化灰字。 + 之前只标前台窗口所以"看不到颜色"——点「刷新窗口」时前台就是本工具自己,而本工具被排除在列表外 +- [x] 控件树搜索命中改为**加粗**(不再用底色):颜色只留给窗口列表 +- [x] 点监控列表行头三角 = **在控件树里定位**(按来源比较身份:UIA 用 类型/AutomationId/名称、MSAA 用 PathKey、Win32 用 HWND)→ 展开祖先、选中、属性网格随选中联动;**再点一次隐藏高亮框**;锚定窗口被关闭/最小化时定时器自动隐藏框,不留旧框 +- [x] **命令设置重新布局**:执行时机分组(启动/退出/定时(on)/启动单次(once))+ 左侧时机列表 + `[新增执行时机] [删除执行时机]` + 右侧命令表格 + `[新增] [删除]` + `[立即执行当前时机的命令] [启动定时命令] [停止定时命令]` + 输出框 +- [x] 设置默认给出四个常用时机分组(启动/退出/定时/单次),可直接用也可再加 + +#### 第十轮·段二:截图 + OCR + +- [x] **`ScreenshotService`**:整窗口 / 客户区 / 任意矩形;依次尝试 `PrintWindow(PW_RENDERFULLCONTENT)` → `PrintWindow(0)` → `BitBlt` → 屏幕拷贝,并记录每一步结果 +- [x] **有效性校验**:尺寸合理 + 采样颜色数 > 1(全黑/全白/单色判为无效)→ 无效就报失败原因,**不把空白图当成功**。实测记事本客户区 1268x775 成功保存(PrintWindowFull) +- [x] 最小化窗口**默认不截**;只有设置里显式授权才临时恢复,并在截完还原为最小化、尽量还原原前台窗口,同时提示会闪屏 +- [x] 截图保存 PNG(文件名含时间与进程/HWND),日志只记方式/尺寸/路径,**不记录图像内容** +- [x] **`OcrClient`**:对接本机 ocr-server(`/healthz`、`/api/v1/upload` multipart、`/api/v1/i2t`);i2t 是异步的,按 0.5 秒轮询到 completed/failed 或超时;失败分类明确(不可达/上传失败/识别失败/超时) +- [x] 新增跨来源 valueKey **`OCR文本`**(截取该控件矩形 → OCR);并且在**正常取值失败时自动降级为 OCR**(用户要求:"path 读不到就用截图 + OCR"),结果里的 valueKey 会变成 `OCR文本`、resolution 写明"该值来自截图识别,不是控件原生值" +- [x] OCR 设置:**总开关(默认关闭)**、服务地址、最小置信度、总超时、**识别时保存截图**、截图目录 +- [x] 主界面新增「窗口截图」「OCR 读取」按钮;无界面模式新增 `--ocr`(本次运行强制开启) +- [x] 实测:OCR 关闭时**完全没有**任何 OCR 调用;开启时截图并尝试上传,服务未启动则如实报 `OCR 失败:上传图片失败:ConnectFailure`,值标为不可用 +- [ ] 用真实 ocr-server(`D:\code\work\ocr-server`,`uv run` 或分发包启动,默认 127.0.0.1:8000)验证识别成功路径与中文识别效果 + +#### 段二附带修掉的问题 + +- [x] `ControlMonitorDefinition` 的 `ControlPath/PageFeatures/ContentKey` 从 `IsRequired=true` 改为可选:手写配置少写字段时, + 以前会直接反序列化失败并报"不是有效的定义文档",现在会走到 store 的中文校验提示("页面特征不再强制") +- [x] **提醒**:截图与 OCR 可能捕获到屏幕上的敏感内容(实测就截到过一行带 token 的地址)。代码只记录长度与路径、 + 不记录识别文本,但**截图文件会落在磁盘上**,请自行控制保存目录与保留期限 + +#### 下一段(段三,待做) + +- [ ] 条件评估(窗口条件 + 控件条件)`ConditionEvaluator`,同时供监听规则与悬浮条使用 +- [ ] `MonitorRule` + `monitor-rules.json` + 设置里的「监听」页:控件出现/消失/值变化/选中变化/焦点变化 + 单击/双击/右键 → 动作(执行命令 / 读取值 / OCR / 截图 / 显示悬浮条) +- [ ] 轮询(默认 1 秒)+ 具体窗口元素上的 UIA 事件(回调只投递后台任务)+ `WH_MOUSE_LL` 低级鼠标钩子(**只观察不拦截**) +- [ ] 条件悬浮条:满足条件才显示、位置 = 锚定窗口左上角 + 相对 XY(跟随窗口移动) + #### 本轮暴露的限制(已实测,非推测) - [ ] **WinForms `DataGridView` 的单元格值无法通过 UIA 读取**:`TablePattern` 能给出正确的行列数,但 `GetItem` 每次都抛 `E_UNEXPECTED`/`ElementNotAvailable`,随后该 HWND 的整个 UIA Provider 失效(连 `FromHandle` 都失败)。当前实现会如实报告 Partial + 明确提示,不会伪造数据。若要读取,需要评估 Win32 消息 / 跨进程读取方案(见下方待办) diff --git a/src/pacs_win_uia/Interop/NativeMethods.cs b/src/pacs_win_uia/Interop/NativeMethods.cs index 4b95bae..af5f69e 100644 --- a/src/pacs_win_uia/Interop/NativeMethods.cs +++ b/src/pacs_win_uia/Interop/NativeMethods.cs @@ -60,6 +60,44 @@ namespace pacs_win_uia.Interop [DllImport("user32.dll", SetLastError = true)] internal static extern IntPtr GetForegroundWindow(); + // ---------------------------------------------------------------- screenshots (segment 2) + internal const uint PW_RENDERFULLCONTENT = 2; + internal const uint SRCCOPY = 0x00CC0020; + internal const int SW_MINIMIZE = 6; + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool PrintWindow(IntPtr hWnd, IntPtr hdcBlt, uint nFlags); + + [DllImport("user32.dll", SetLastError = true)] + internal static extern IntPtr GetWindowDC(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + internal static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC); + + [DllImport("gdi32.dll", SetLastError = true)] + internal static extern IntPtr CreateCompatibleDC(IntPtr hdc); + + [DllImport("gdi32.dll", SetLastError = true)] + internal static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int width, int height); + + [DllImport("gdi32.dll", SetLastError = true)] + internal static extern IntPtr SelectObject(IntPtr hdc, IntPtr hObject); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool BitBlt( + IntPtr hdcDest, int xDest, int yDest, int width, int height, + IntPtr hdcSrc, int xSrc, int ySrc, uint rop); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool DeleteObject(IntPtr hObject); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static extern bool DeleteDC(IntPtr hdc); + [DllImport("user32.dll", SetLastError = true)] [return: MarshalAs(UnmanagedType.Bool)] internal static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam); diff --git a/src/pacs_win_uia/Models/AppSettings.cs b/src/pacs_win_uia/Models/AppSettings.cs index 54ab9cf..4546028 100644 --- a/src/pacs_win_uia/Models/AppSettings.cs +++ b/src/pacs_win_uia/Models/AppSettings.cs @@ -21,6 +21,7 @@ namespace pacs_win_uia.Models Commands = new CommandSettings(); Ocr = new OcrSettings(); Read = new ReadSettings(); + Screenshot = new ScreenshotSettings(); } [DataMember(Order = 1)] @@ -41,6 +42,9 @@ namespace pacs_win_uia.Models [DataMember(Order = 6)] public ReadSettings Read { get; set; } + [DataMember(Order = 7)] + public ScreenshotSettings Screenshot { get; set; } + public static AppSettings CreateDefault() { var settings = new AppSettings { Version = CurrentVersion }; @@ -80,27 +84,46 @@ namespace pacs_win_uia.Models Read = new ReadSettings(); } + if (Screenshot == null) + { + Screenshot = new ScreenshotSettings(); + } + + Screenshot.RestoreSettleMs = Clamp(Screenshot.RestoreSettleMs, 0, 5000, 350); + Ui.TreeMaxDepth = Clamp(Ui.TreeMaxDepth, 1, 32, 12); Ui.TreeMaxNodes = Clamp(Ui.TreeMaxNodes, 50, 50000, 3000); Ui.TreeTimeoutSeconds = Clamp(Ui.TreeTimeoutSeconds, 1, 600, 10); FloatingButton.OpacityPercent = Clamp(FloatingButton.OpacityPercent, 20, 100, 90); Read.DefaultTimeoutSeconds = Clamp(Read.DefaultTimeoutSeconds, 0, 3600, 10); + Ocr.ApplyDefaults(); - if (Commands.Continuous == null) + if (Commands.Triggers == null || Commands.Triggers.Count == 0) { - Commands.Continuous = new List(); + // A missing section means "nothing configured yet": give the operator the four + // familiar groups instead of an empty screen. + Commands.Triggers = CommandSettings.CreateDefaultTriggers(); } - if (Commands.Once == null) + foreach (CommandTrigger trigger in Commands.Triggers) { - Commands.Once = new List(); - } - - foreach (CommandDefinition command in Commands.Continuous) - { - if (command != null) + if (trigger == null) { - command.IntervalSeconds = Clamp(command.IntervalSeconds, 1, 86400, 60); + continue; + } + + trigger.IntervalSeconds = Clamp(trigger.IntervalSeconds, 1, 86400, 60); + if (trigger.Commands == null) + { + trigger.Commands = new List(); + } + + foreach (CommandDefinition command in trigger.Commands) + { + if (command != null) + { + command.TimeoutSeconds = Clamp(command.TimeoutSeconds, 1, 3600, 30); + } } } } @@ -179,24 +202,101 @@ namespace pacs_win_uia.Models public List Actions { get; set; } } + /// When a group of commands runs. + [DataContract] + public enum CommandTriggerKind + { + [EnumMember] + Start, + + [EnumMember] + Exit, + + /// Runs repeatedly on an interval while the tool is open. + [EnumMember] + Interval, + + [EnumMember] + Once + } + + /// + /// A named group of commands with one execution moment. Several groups may share a moment, which + /// is what the "新增执行时机" button in the settings dialog creates. + /// + [DataContract] + public sealed class CommandTrigger + { + public CommandTrigger() + { + Kind = CommandTriggerKind.Once; + IntervalSeconds = 60; + Enabled = true; + Commands = new List(); + } + + [DataMember(Order = 1)] + [Description("这组命令的名字,只用于界面显示。")] + public string Name { get; set; } + + [DataMember(Order = 2)] + [Description("执行时机:启动 / 退出 / 定时(on) / 单次(once)。")] + public CommandTriggerKind Kind { get; set; } + + [DataMember(Order = 3)] + [Description("定时(on)时机的间隔秒数。")] + public int IntervalSeconds { get; set; } + + [DataMember(Order = 4)] + [Description("整组是否启用。停用的时机不会执行任何命令。")] + public bool Enabled { get; set; } + + [DataMember(Order = 5)] + public List Commands { get; set; } + + public override string ToString() + { + return (string.IsNullOrEmpty(Name) ? "(未命名)" : Name) + " —— " + DescribeKind(Kind); + } + + public static string DescribeKind(CommandTriggerKind kind) + { + switch (kind) + { + case CommandTriggerKind.Start: + return "启动时一次"; + case CommandTriggerKind.Exit: + return "退出时一次"; + case CommandTriggerKind.Interval: + return "定时持续(on)"; + default: + return "启动时单次(once)"; + } + } + } + [DataContract] public sealed class CommandSettings { + public CommandSettings() + { + Triggers = new List(); + } + + /// Execution moments; the user can add more than one group per moment. [DataMember(Order = 1)] - [Description("工具启动时执行一次的命令行(通过 cmd.exe /c 运行,留空表示不执行)。")] - public string OnStart { get; set; } + public List Triggers { get; set; } - [DataMember(Order = 2)] - [Description("工具退出时执行一次的命令行(留空表示不执行)。")] - public string OnExit { get; set; } - - /// Commands that keep running on an interval while the tool is open. - [DataMember(Order = 3)] - public List Continuous { get; set; } - - /// Commands that run exactly once, when the tool starts. - [DataMember(Order = 4)] - public List Once { get; set; } + public static List CreateDefaultTriggers() + { + return new List + { + new CommandTrigger { Name = "启动", Kind = CommandTriggerKind.Start }, + new CommandTrigger { Name = "退出", Kind = CommandTriggerKind.Exit }, + new CommandTrigger { Name = "定时", Kind = CommandTriggerKind.Interval }, + new CommandTrigger { Name = "单次", Kind = CommandTriggerKind.Once } + }; + } } [DataContract] @@ -239,8 +339,8 @@ namespace pacs_win_uia.Models } /// - /// OCR is deliberately not implemented yet: the fields are stored so the shape of the - /// configuration is settled, and the UI says so plainly instead of pretending to work. + /// OCR runs against a local ocr-server (FastAPI + PaddleOCR). The master switch defaults to off: + /// nothing is sent anywhere until the operator turns it on, and only to the configured address. /// [DataContract] public sealed class OcrSettings @@ -248,24 +348,83 @@ namespace pacs_win_uia.Models public OcrSettings() { Language = "zh-CN"; - Engine = "(未实现)"; + BaseUrl = "http://127.0.0.1:8000"; + TimeoutSeconds = 30; + ApplyDefaults(); } [DataMember(Order = 1)] - [Description("是否启用 OCR。**本轮未实现**:勾选也只保存配置,不会执行识别。")] + [Description("OCR 总开关。关闭时本工具**完全不会**调用任何 OCR 服务(段二实现)。")] public bool Enabled { get; set; } [DataMember(Order = 2)] - [Description("识别引擎。本轮未实现,预留字段。")] - public string Engine { get; set; } + [Description("OCR 服务地址(默认本机 ocr-server)。识别内容只发到这个地址,不上传外部服务。")] + public string BaseUrl { get; set; } [DataMember(Order = 3)] - [Description("识别语言(预留)。")] + [Description("识别语言(服务端模型决定,仅记录用途)。")] public string Language { get; set; } [DataMember(Order = 4)] - [Description("截图保存目录(预留)。敏感数据不要上传到外部服务。")] - public string CaptureDirectory { get; set; } + [Description("识别时是否保存截图。保存的图片可能包含患者信息,请自行控制目录与保留期限。")] + public bool SaveScreenshots { get; set; } + + [DataMember(Order = 5)] + [Description("截图保存目录。留空表示 exe 目录下的 screenshots。")] + public string ScreenshotDirectory { get; set; } + + [DataMember(Order = 6)] + [Description("识别结果的最小置信度(0-1),低于该值的文本会被服务端过滤。")] + public double MinConfidence { get; set; } + + [DataMember(Order = 7)] + [Description("单次识别的总超时(秒),包含排队等待。")] + public int TimeoutSeconds { get; set; } + + public void ApplyDefaults() + { + if (string.IsNullOrWhiteSpace(BaseUrl)) + { + BaseUrl = "http://127.0.0.1:8000"; + } + + if (string.IsNullOrWhiteSpace(Language)) + { + Language = "zh-CN"; + } + + if (TimeoutSeconds <= 0) + { + TimeoutSeconds = 30; + } + + if (MinConfidence < 0 || MinConfidence > 1) + { + MinConfidence = 0; + } + } + } + + /// Screenshot behaviour. Restoring a minimized window is opt-in because it is visible. + [DataContract] + public sealed class ScreenshotSettings + { + public ScreenshotSettings() + { + RestoreSettleMs = 350; + } + + [DataMember(Order = 1)] + [Description("截图保存目录,留空表示 exe 目录下的 screenshots。")] + public string Directory { get; set; } + + [DataMember(Order = 2)] + [Description("是否允许为截图临时恢复最小化窗口。开启后窗口会闪一下、可能改变前台窗口,默认关闭。")] + public bool AllowRestoreMinimized { get; set; } + + [DataMember(Order = 3)] + [Description("临时恢复最小化窗口后等待界面绘制的时间(毫秒)。")] + public int RestoreSettleMs { get; set; } } [DataContract] diff --git a/src/pacs_win_uia/Models/ControlLocator.cs b/src/pacs_win_uia/Models/ControlLocator.cs index 55b9a23..60dd206 100644 --- a/src/pacs_win_uia/Models/ControlLocator.cs +++ b/src/pacs_win_uia/Models/ControlLocator.cs @@ -156,6 +156,12 @@ namespace pacs_win_uia.Models /// Cross-source key: does this control currently have keyboard focus. public const string Focused = "焦点状态"; + /// + /// Cross-source key: read the text by screenshotting this control and sending it to the local + /// OCR service. Only usable when OCR is switched on; the result is marked as OCR-derived. + /// + public const string OcrText = "OCR文本"; + public static readonly IList Uia = new List { "Name", "AutomationId", "ControlType", "ClassName", "FrameworkId", @@ -193,20 +199,33 @@ namespace pacs_win_uia.Models keys.AddRange(Win32); } + keys.Add(OcrText); return keys; } public static IList ForSource(ControlTreeSource source) { + IList keys; switch (source) { case ControlTreeSource.Msaa: - return Msaa; + keys = Msaa; + break; case ControlTreeSource.Win32: - return Win32; + keys = Win32; + break; default: - return Uia; + keys = Uia; + break; } + + var withOcr = new List(keys); + if (!withOcr.Contains(OcrText)) + { + withOcr.Add(OcrText); + } + + return withOcr; } } } diff --git a/src/pacs_win_uia/Models/ControlMonitorDefinition.cs b/src/pacs_win_uia/Models/ControlMonitorDefinition.cs index 3c50676..1b06368 100644 --- a/src/pacs_win_uia/Models/ControlMonitorDefinition.cs +++ b/src/pacs_win_uia/Models/ControlMonitorDefinition.cs @@ -20,13 +20,14 @@ namespace pacs_win_uia.Models public string Id { get; set; } /// Human-readable path from the top-level window down to the control. Display only. - [DataMember(Order = 2, IsRequired = true)] + [DataMember(Order = 2, IsRequired = false, EmitDefaultValue = false)] public string ControlPath { get; set; } - [DataMember(Order = 3, IsRequired = true)] + /// Optional: empty means "do not check the page" when re-resolving. + [DataMember(Order = 3, IsRequired = false, EmitDefaultValue = false)] public string PageFeatures { get; set; } - [DataMember(Order = 4, IsRequired = true)] + [DataMember(Order = 4, IsRequired = false, EmitDefaultValue = false)] public string ContentKey { get; set; } /// diff --git a/src/pacs_win_uia/Program.cs b/src/pacs_win_uia/Program.cs index 3966a71..2849d46 100644 --- a/src/pacs_win_uia/Program.cs +++ b/src/pacs_win_uia/Program.cs @@ -25,6 +25,7 @@ namespace pacs_win_uia "给 --output 但不是文件路径时输出到标准输出\r\n" + " --timeout <秒> 等待目标窗口出现的秒数,默认 10,0 表示不等待\r\n" + " --activate 读取前先尝试激活目标无障碍(实验性,会改变目标程序状态)\r\n" + + " --ocr 本次运行强制启用 OCR 降级(默认按 settings.json 里的开关)\r\n" + " --help 显示本帮助\r\n" + "\r\n" + "默认路径:\r\n" + @@ -71,6 +72,7 @@ namespace pacs_win_uia string outputPath = null; double timeoutSeconds = 10; bool activate = false; + bool forceOcr = false; for (int index = 0; index < args.Length; index++) { @@ -119,6 +121,9 @@ namespace pacs_win_uia case "--activate": activate = true; break; + case "--ocr": + forceOcr = true; + break; default: return UsageError("无法识别的参数:" + arg); } @@ -150,11 +155,32 @@ namespace pacs_win_uia Console.Error.WriteLine("输出文件:" + Path.GetFullPath(outputPath)); } + // OCR is off unless the settings file switches it on; --ocr forces it for this run. + AppSettings settings = new AppSettingsStore().Load(); + if (forceOcr) + { + settings.Ocr.Enabled = true; + } + + OcrReadContext ocr = OcrReadContext.Create(settings.Ocr); + if (ocr != null) + { + string healthError; + if (ocr.Client.CheckHealth(out healthError)) + { + AppLog.Info("OCR 已启用:" + ocr.Client.BaseUrl); + } + else + { + AppLog.Warn("OCR 已启用但服务不可达(取值会失败并如实报告):" + healthError); + } + } + HeadlessReport report; try { var reader = new HeadlessConfigReader(new ControlMonitorStore(fullConfigPath)); - report = reader.Read(fullConfigPath, TimeSpan.FromSeconds(timeoutSeconds), activate, CancellationToken.None); + report = reader.Read(fullConfigPath, TimeSpan.FromSeconds(timeoutSeconds), activate, ocr, CancellationToken.None); } catch (InvalidDataException ex) { diff --git a/src/pacs_win_uia/Services/CommandRunner.cs b/src/pacs_win_uia/Services/CommandRunner.cs index 9fd3a11..645141c 100644 --- a/src/pacs_win_uia/Services/CommandRunner.cs +++ b/src/pacs_win_uia/Services/CommandRunner.cs @@ -71,63 +71,79 @@ namespace pacs_win_uia.Services public void RunStartCommands(AppSettings settings) { - if (settings == null || settings.Commands == null) - { - return; - } - - RunSimple("启动", settings.Commands.OnStart); - foreach (CommandDefinition command in settings.Commands.Once ?? new List()) - { - if (command != null && command.Enabled && !string.IsNullOrWhiteSpace(command.CommandLine)) - { - RunCommand(command, "once"); - } - } - - StartContinuous(settings.Commands.Continuous); + RunByKind(settings, CommandTriggerKind.Start); + RunByKind(settings, CommandTriggerKind.Once); + StartContinuous(settings); } public void RunExitCommands(AppSettings settings) { StopContinuous(); - if (settings == null || settings.Commands == null) - { - return; - } - - RunSimple("退出", settings.Commands.OnExit); + RunByKind(settings, CommandTriggerKind.Exit); } - public void StartContinuous(IList commands) + /// Runs every enabled command in the groups that use the given moment. + public void RunByKind(AppSettings settings, CommandTriggerKind kind) { - StopContinuous(); - if (commands == null) + foreach (CommandTrigger trigger in Triggers(settings)) { - return; - } - - foreach (CommandDefinition command in commands) - { - if (command == null || !command.Enabled || string.IsNullOrWhiteSpace(command.CommandLine)) + if (trigger == null || !trigger.Enabled || trigger.Kind != kind) { continue; } - CommandDefinition captured = command; - int interval = Math.Max(1, command.IntervalSeconds) * 1000; - var timer = new System.Windows.Forms.Timer { Interval = Math.Min(interval, int.MaxValue - 1) }; - timer.Tick += delegate { RunCommandInBackground(captured, "on"); }; - lock (_sync) + foreach (CommandDefinition command in trigger.Commands ?? new List()) { - _continuousTimers.Add(timer); + if (command != null && command.Enabled && !string.IsNullOrWhiteSpace(command.CommandLine)) + { + RunCommand(command, CommandTrigger.DescribeKind(kind) + "/" + trigger.Name); + } + } + } + } + + public void StartContinuous(AppSettings settings) + { + StopContinuous(); + foreach (CommandTrigger trigger in Triggers(settings)) + { + if (trigger == null || !trigger.Enabled || trigger.Kind != CommandTriggerKind.Interval) + { + continue; } - timer.Start(); - AppLog.Info("持续命令已启动:" + captured + "(每 " + command.IntervalSeconds + " 秒)"); + int interval = Math.Max(1, trigger.IntervalSeconds) * 1000; + foreach (CommandDefinition command in trigger.Commands ?? new List()) + { + if (command == null || !command.Enabled || string.IsNullOrWhiteSpace(command.CommandLine)) + { + continue; + } + + CommandDefinition captured = command; + var timer = new System.Windows.Forms.Timer { Interval = Math.Min(interval, int.MaxValue - 1) }; + timer.Tick += delegate { RunCommandInBackground(captured, "定时/" + trigger.Name); }; + lock (_sync) + { + _continuousTimers.Add(timer); + } + + timer.Start(); + AppLog.Info("定时命令已启动:" + captured + "(每 " + trigger.IntervalSeconds + " 秒)"); + } } } + private static IEnumerable Triggers(AppSettings settings) + { + if (settings == null || settings.Commands == null || settings.Commands.Triggers == null) + { + return new List(); + } + + return settings.Commands.Triggers; + } + public void StopContinuous() { List timers; diff --git a/src/pacs_win_uia/Services/ControlValueReader.cs b/src/pacs_win_uia/Services/ControlValueReader.cs index d87ace9..099a823 100644 --- a/src/pacs_win_uia/Services/ControlValueReader.cs +++ b/src/pacs_win_uia/Services/ControlValueReader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Windows.Automation; +using pacs_win_uia.Diagnostics; using pacs_win_uia.Models; namespace pacs_win_uia.Services @@ -18,6 +19,21 @@ namespace pacs_win_uia.Services /// from "control not found". /// public static bool TryRead(ControlTreeNode node, string valueKey, ListReadOptions options, out string value, out string error) + { + return TryRead(node, valueKey, options, null, out value, out error); + } + + /// + /// Null when OCR is switched off; when present it is the fallback for controls whose text + /// cannot be read through UIA/MSAA/Win32. + /// + public static bool TryRead( + ControlTreeNode node, + string valueKey, + ListReadOptions options, + OcrReadContext ocr, + out string value, + out string error) { value = null; error = null; @@ -33,6 +49,11 @@ namespace pacs_win_uia.Services return false; } + if (string.Equals(valueKey, ControlValueKeys.OcrText, StringComparison.Ordinal)) + { + return TryReadByOcr(node, ocr, out value, out error); + } + switch (node.Source) { case ControlTreeSource.Msaa: @@ -44,6 +65,94 @@ namespace pacs_win_uia.Services } } + /// + /// Screenshots the control's rectangle and asks the local OCR service for the text. The value is + /// explicitly OCR-derived: it is not the control's own value and is never presented as one. + /// + private static bool TryReadByOcr(ControlTreeNode node, OcrReadContext ocr, out string value, out string error) + { + value = null; + error = null; + if (ocr == null || ocr.Settings == null || !ocr.Settings.Enabled) + { + error = "OCR 未启用:请在「设置 → OCR」里打开总开关(默认关闭,关闭时不会调用任何 OCR 服务)。"; + return false; + } + + NativeRectangleInfo bounds = TryGetBounds(node); + if (bounds == null || !bounds.IsValid) + { + error = "该控件没有可用的屏幕矩形,无法截图识别。"; + return false; + } + + ScreenshotResult shot = ocr.Screenshots.CaptureRectangle(bounds.ToRectangle(), new ScreenshotOptions()); + if (shot.Image == null) + { + error = "截图失败:" + (shot.FailureReason ?? "未知原因") + + (shot.Attempts.Count > 0 ? "(" + string.Join(";", shot.Attempts.ToArray()) + ")" : string.Empty); + return false; + } + + try + { + string path = null; + if (ocr.Settings.SaveScreenshots) + { + try + { + path = ScreenshotService.Save(shot.Image, ocr.Settings.ScreenshotDirectory, node.Source.ToString()); + AppLog.Info("OCR:截图已保存到 " + path); + } + catch (Exception ex) + { + AppLog.Warn("OCR:截图保存失败。", ex); + } + } + + OcrResult result = ocr.Client.Recognize(shot.Image, path); + if (!string.IsNullOrEmpty(result.Error)) + { + error = "OCR 失败:" + result.Error; + return false; + } + + if (string.IsNullOrEmpty(result.Text)) + { + error = "OCR 没有识别出文字(截图有效但服务返回了空文本)。"; + return false; + } + + value = result.Text; + return true; + } + finally + { + shot.Image.Dispose(); + } + } + + /// The tightest rectangle available for a node, whichever source it came from. + public static NativeRectangleInfo TryGetBounds(ControlTreeNode node) + { + if (node == null) + { + return null; + } + + if (node.Uia != null && node.Uia.BoundingRectangle != null && node.Uia.BoundingRectangle.IsValid) + { + return node.Uia.BoundingRectangle; + } + + if (node.Msaa != null && node.Msaa.Location != null && node.Msaa.Location.IsValid) + { + return node.Msaa.Location; + } + + return node.Window == null ? null : node.Window.WindowBounds; + } + private static bool TryReadUia(UiaElementInfo info, string key, ListReadOptions options, out string value, out string error) { value = null; diff --git a/src/pacs_win_uia/Services/HeadlessConfigReader.cs b/src/pacs_win_uia/Services/HeadlessConfigReader.cs index 13a127c..b4169fd 100644 --- a/src/pacs_win_uia/Services/HeadlessConfigReader.cs +++ b/src/pacs_win_uia/Services/HeadlessConfigReader.cs @@ -42,7 +42,13 @@ namespace pacs_win_uia.Services /// How long to keep waiting for a target window to appear. /// Ask targets to switch their accessibility support on first. - public HeadlessReport Read(string configPath, TimeSpan windowTimeout, bool activate, CancellationToken cancellationToken) + /// OCR fallback context, or null when OCR is off (the default). + public HeadlessReport Read( + string configPath, + TimeSpan windowTimeout, + bool activate, + OcrReadContext ocr, + CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); var report = new HeadlessReport @@ -79,7 +85,7 @@ namespace pacs_win_uia.Services continue; } - report.Items.Add(ReadItem(definition, windowTimeout, activation, cancellationToken)); + report.Items.Add(ReadItem(definition, windowTimeout, activation, ocr, cancellationToken)); } } finally @@ -120,6 +126,7 @@ namespace pacs_win_uia.Services ControlMonitorDefinition definition, TimeSpan windowTimeout, AccessibilityActivationService activation, + OcrReadContext ocr, CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); @@ -176,7 +183,7 @@ namespace pacs_win_uia.Services string value; string valueError; - if (ControlValueReader.TryRead(resolution.Node, definition.ValueKey, new ListReadOptions(), out value, out valueError)) + if (ControlValueReader.TryRead(resolution.Node, definition.ValueKey, new ListReadOptions(), ocr, out value, out valueError)) { item.Value = value; item.ValueAvailable = true; @@ -188,6 +195,26 @@ namespace pacs_win_uia.Services item.Error = "控件已定位到,但取值失败:" + valueError; steps.Add(item.Error); AppLog.Warn("条目 " + definition.ContentKey + ":" + item.Error); + + // The operator's rule: when the control's own value cannot be read, try a screenshot + // plus OCR — but only when OCR is switched on. + if (ocr != null && !string.Equals(definition.ValueKey, ControlValueKeys.OcrText, StringComparison.Ordinal)) + { + string ocrValue; + string ocrError; + if (ControlValueReader.TryRead(resolution.Node, ControlValueKeys.OcrText, new ListReadOptions(), ocr, out ocrValue, out ocrError)) + { + item.Value = ocrValue; + item.ValueAvailable = true; + item.ValueKey = ControlValueKeys.OcrText; + item.Error = null; + steps.Add("改用 OCR 降级取值成功(该值来自截图识别,不是控件原生值)。"); + } + else + { + steps.Add("OCR 降级也未取到值:" + ocrError); + } + } } stopwatch.Stop(); diff --git a/src/pacs_win_uia/Services/OcrClient.cs b/src/pacs_win_uia/Services/OcrClient.cs new file mode 100644 index 0000000..711fbba --- /dev/null +++ b/src/pacs_win_uia/Services/OcrClient.cs @@ -0,0 +1,402 @@ +using System; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; +using System.Net; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Json; +using System.Text; +using System.Threading; +using pacs_win_uia.Diagnostics; +using pacs_win_uia.Models; + +namespace pacs_win_uia.Services +{ + /// What one recognition attempt produced. + public sealed class OcrResult + { + public string Text { get; set; } + + public long ElapsedMs { get; set; } + + /// Path of the screenshot sent for recognition, when screenshots are being saved. + public string ScreenshotPath { get; set; } + + public string Error { get; set; } + } + + /// + /// Everything the value reader needs to fall back to OCR: settings, client and screenshot service. + /// Created only from configured settings, so nothing is ever sent anywhere without the operator + /// switching OCR on. + /// + public sealed class OcrReadContext + { + private OcrReadContext(OcrSettings settings) + { + Settings = settings; + Client = new OcrClient(settings); + Screenshots = new ScreenshotService(); + } + + public OcrSettings Settings { get; private set; } + + public OcrClient Client { get; private set; } + + public ScreenshotService Screenshots { get; private set; } + + /// Null when OCR is off, which is the default. + public static OcrReadContext Create(OcrSettings settings) + { + if (settings == null) + { + return null; + } + + settings.ApplyDefaults(); + return settings.Enabled ? new OcrReadContext(settings) : null; + } + } + + /// + /// Client for the local ocr-server (FastAPI + PaddleOCR): + /// GET /healthz -> liveness + /// POST /api/v1/upload (multipart) -> { data: { file_id } } + /// POST /api/v1/i2t (json) -> { data: { status, progress, result: { text }, error } } + /// + /// i2t is asynchronous: it may answer "queued"/"processing" first, so it is polled until the task + /// finishes or the timeout expires. The image only ever goes to the configured address, and the + /// recognized text is never written to the log (it can contain patient data). + /// + public sealed class OcrClient + { + private const string UploadPath = "/api/v1/upload"; + private const string I2TPath = "/api/v1/i2t"; + private const string HealthPath = "/healthz"; + + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(500); + + private readonly string _baseUrl; + private readonly double _minConfidence; + private readonly TimeSpan _timeout; + + public OcrClient(OcrSettings settings) + { + if (settings == null) + { + throw new ArgumentNullException("settings"); + } + + settings.ApplyDefaults(); + _baseUrl = settings.BaseUrl.TrimEnd('/'); + _minConfidence = settings.MinConfidence; + _timeout = TimeSpan.FromSeconds(Math.Max(1, settings.TimeoutSeconds)); + } + + public string BaseUrl + { + get { return _baseUrl; } + } + + public bool CheckHealth(out string error) + { + error = null; + try + { + var request = (HttpWebRequest)WebRequest.Create(_baseUrl + HealthPath); + request.Method = "GET"; + request.Timeout = 5000; + using (var response = (HttpWebResponse)request.GetResponse()) + { + if (response.StatusCode != HttpStatusCode.OK) + { + error = "服务返回 " + (int)response.StatusCode; + return false; + } + } + + return true; + } + catch (Exception ex) + { + error = Describe(ex, "无法连接 OCR 服务(" + _baseUrl + ")"); + return false; + } + } + + /// Recognizes an image; the caller keeps ownership of the bitmap. + public OcrResult Recognize(Bitmap image, string screenshotPath) + { + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + var result = new OcrResult { ScreenshotPath = screenshotPath }; + if (image == null) + { + result.Error = "没有图像。"; + return result; + } + + string fileId; + string error; + if (!TryUpload(image, out fileId, out error)) + { + result.Error = error; + return result; + } + + var deadline = DateTime.UtcNow + _timeout; + while (DateTime.UtcNow < deadline) + { + string status; + string text; + int progress; + int? queuePosition; + if (!TryI2T(fileId, out status, out text, out progress, out queuePosition, out error)) + { + result.Error = error; + stopwatch.Stop(); + result.ElapsedMs = stopwatch.ElapsedMilliseconds; + return result; + } + + if (string.Equals(status, "completed", StringComparison.OrdinalIgnoreCase)) + { + stopwatch.Stop(); + result.Text = text ?? string.Empty; + result.ElapsedMs = stopwatch.ElapsedMilliseconds; + AppLog.Info("OCR:识别完成,用时 " + result.ElapsedMs + " ms,取到 " + result.Text.Length + " 个字符。"); + return result; + } + + if (string.Equals(status, "failed", StringComparison.OrdinalIgnoreCase)) + { + result.Error = string.IsNullOrEmpty(error) ? "服务端识别失败。" : error; + stopwatch.Stop(); + result.ElapsedMs = stopwatch.ElapsedMilliseconds; + return result; + } + + AppLog.Debug("OCR:状态 " + status + "(" + progress + "%" + + (queuePosition.HasValue ? ",队列第 " + queuePosition + " 位" : string.Empty) + ")"); + Thread.Sleep(PollInterval); + } + + stopwatch.Stop(); + result.ElapsedMs = stopwatch.ElapsedMilliseconds; + result.Error = "等待识别结果超时(" + _timeout.TotalSeconds + " 秒)。"; + return result; + } + + private bool TryUpload(Bitmap image, out string fileId, out string error) + { + fileId = null; + error = null; + string boundary = "----pacsUiaOcr" + Guid.NewGuid().ToString("N"); + byte[] imageBytes; + using (var stream = new MemoryStream()) + { + image.Save(stream, ImageFormat.Png); + imageBytes = stream.ToArray(); + } + + try + { + var request = (HttpWebRequest)WebRequest.Create(_baseUrl + UploadPath); + request.Method = "POST"; + request.ContentType = "multipart/form-data; boundary=" + boundary; + request.Timeout = 30000; + request.ReadWriteTimeout = 30000; + + using (var body = new MemoryStream()) + { + WriteAscii(body, "--" + boundary + "\r\n"); + WriteAscii(body, "Content-Disposition: form-data; name=\"file\"; filename=\"capture.png\"\r\n"); + WriteAscii(body, "Content-Type: image/png\r\n\r\n"); + body.Write(imageBytes, 0, imageBytes.Length); + WriteAscii(body, "\r\n--" + boundary + "--\r\n"); + + byte[] payload = body.ToArray(); + request.ContentLength = payload.Length; + using (Stream requestStream = request.GetRequestStream()) + { + requestStream.Write(payload, 0, payload.Length); + } + } + + using (var response = (HttpWebResponse)request.GetResponse()) + using (Stream responseStream = response.GetResponseStream()) + { + Envelope envelope = ReadEnvelope(responseStream); + if (envelope == null || envelope.Data == null || string.IsNullOrEmpty(envelope.Data.FileId)) + { + error = "上传返回里没有 file_id(" + (envelope == null ? "响应无法解析" : envelope.Message) + ")。"; + return false; + } + + fileId = envelope.Data.FileId; + return true; + } + } + catch (Exception ex) + { + error = Describe(ex, "上传图片失败"); + return false; + } + } + + private bool TryI2T( + string fileId, + out string status, + out string text, + out int progress, + out int? queuePosition, + out string error) + { + status = null; + text = null; + progress = 0; + queuePosition = null; + error = null; + string payload = "{\"file_id\":\"" + fileId + "\",\"min_confidence\":" + _minConfidence.ToString("0.###", System.Globalization.CultureInfo.InvariantCulture) + "}"; + try + { + var request = (HttpWebRequest)WebRequest.Create(_baseUrl + I2TPath); + request.Method = "POST"; + request.ContentType = "application/json"; + request.Timeout = 30000; + request.ReadWriteTimeout = 30000; + byte[] data = Encoding.UTF8.GetBytes(payload); + request.ContentLength = data.Length; + using (Stream stream = request.GetRequestStream()) + { + stream.Write(data, 0, data.Length); + } + + using (var response = (HttpWebResponse)request.GetResponse()) + using (Stream responseStream = response.GetResponseStream()) + { + Envelope envelope = ReadEnvelope(responseStream); + if (envelope == null || envelope.Data == null) + { + error = "识别返回无法解析。"; + return false; + } + + status = envelope.Data.Status; + progress = envelope.Data.Progress; + queuePosition = envelope.Data.QueuePosition; + text = envelope.Data.Result == null ? null : envelope.Data.Result.Text; + if (!string.IsNullOrEmpty(envelope.Data.Error)) + { + error = envelope.Data.Error; + } + + return true; + } + } + catch (Exception ex) + { + error = Describe(ex, "请求识别结果失败"); + return false; + } + } + + private static void WriteAscii(Stream stream, string text) + { + byte[] bytes = Encoding.UTF8.GetBytes(text); + stream.Write(bytes, 0, bytes.Length); + } + + private static Envelope ReadEnvelope(Stream stream) + { + if (stream == null) + { + return null; + } + + var serializer = new DataContractJsonSerializer(typeof(Envelope)); + return (Envelope)serializer.ReadObject(stream); + } + + private static string Describe(Exception exception, string prefix) + { + if (exception is WebException) + { + var web = (WebException)exception; + string detail = web.Status.ToString(); + if (web.Response is HttpWebResponse) + { + var http = (HttpWebResponse)web.Response; + detail += " " + (int)http.StatusCode; + try + { + using (Stream stream = http.GetResponseStream()) + using (var reader = new StreamReader(stream ?? Stream.Null, Encoding.UTF8)) + { + string body = reader.ReadToEnd(); + if (!string.IsNullOrEmpty(body) && body.Length < 300) + { + detail += " " + body; + } + } + } + catch (Exception) + { + // The body is optional diagnostic detail. + } + } + + return prefix + ":" + detail; + } + + return prefix + ":" + exception.GetType().Name + ": " + exception.Message; + } + + // ---------------------------------------------------------------- wire contracts + + [DataContract] + private sealed class Envelope + { + [DataMember(Name = "code")] + public int Code { get; set; } + + [DataMember(Name = "message")] + public string Message { get; set; } + + [DataMember(Name = "data")] + public Payload Data { get; set; } + } + + /// Union of the upload and i2t payloads: both endpoints use the same envelope. + [DataContract] + private sealed class Payload + { + [DataMember(Name = "file_id")] + public string FileId { get; set; } + + [DataMember(Name = "status")] + public string Status { get; set; } + + [DataMember(Name = "progress")] + public int Progress { get; set; } + + [DataMember(Name = "queue_position")] + public int? QueuePosition { get; set; } + + [DataMember(Name = "elapsed_ms")] + public double ElapsedMs { get; set; } + + [DataMember(Name = "result")] + public Document Result { get; set; } + + [DataMember(Name = "error")] + public string Error { get; set; } + } + + [DataContract] + private sealed class Document + { + [DataMember(Name = "text")] + public string Text { get; set; } + } + } +} diff --git a/src/pacs_win_uia/Services/ScreenshotService.cs b/src/pacs_win_uia/Services/ScreenshotService.cs new file mode 100644 index 0000000..6a0773f --- /dev/null +++ b/src/pacs_win_uia/Services/ScreenshotService.cs @@ -0,0 +1,533 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; +using System.Runtime.InteropServices; +using pacs_win_uia.Diagnostics; +using pacs_win_uia.Interop; +using pacs_win_uia.Models; + +namespace pacs_win_uia.Services +{ + public enum ScreenshotTarget + { + /// Whole window including the frame. + Window, + + /// Client area only. + Client, + + /// An explicit screen rectangle. + Rectangle + } + + public enum ScreenshotMethod + { + None, + + /// PrintWindow with PW_RENDERFULLCONTENT (works for many GPU-composited windows on Win 8.1+). + PrintWindowFull, + + /// Plain PrintWindow. + PrintWindow, + + /// BitBlt from the window DC. + BitBlt, + + /// Screen copy: only valid while the window is actually visible and unobstructed. + ScreenCopy + } + + public sealed class ScreenshotOptions + { + public ScreenshotOptions() + { + Target = ScreenshotTarget.Window; + AllowRestoreMinimized = false; + RestoreSettleMs = 350; + AllowScreenCopyFallback = true; + } + + public ScreenshotTarget Target { get; set; } + + /// + /// Temporarily restoring a minimized window makes it flash on screen and changes the foreground. + /// It never happens unless the operator authorizes it in the settings. + /// + public bool AllowRestoreMinimized { get; set; } + + public int RestoreSettleMs { get; set; } + + /// Screen copy needs the window to be visible; it is a last resort. + public bool AllowScreenCopyFallback { get; set; } + } + + public sealed class ScreenshotResult + { + public ScreenshotResult() + { + Attempts = new List(); + } + + /// Caller owns and must dispose the bitmap. + public Bitmap Image { get; set; } + + public ScreenshotMethod Method { get; set; } + + public NativeRectangleInfo Bounds { get; set; } + + public string FilePath { get; set; } + + /// False when no method produced a usable image, or the image failed validation. + public bool IsComplete { get; set; } + + public string FailureReason { get; set; } + + /// What each method returned, for diagnosis. + public List Attempts { get; private set; } + + public bool WasMinimized { get; set; } + + public bool WasRestoredTemporarily { get; set; } + } + + /// + /// Captures windows and screen rectangles. Every method is tried in turn and the result is + /// validated: an API returning success is not proof of a usable image, so a blank or uniform + /// bitmap is reported as a failure rather than being handed on as a screenshot. + /// + public sealed class ScreenshotService + { + private const int MinUsefulSize = 4; + + public ScreenshotResult CaptureWindow(IntPtr handle, ScreenshotOptions options) + { + if (options == null) + { + options = new ScreenshotOptions(); + } + + var result = new ScreenshotResult(); + if (handle == IntPtr.Zero || !NativeMethods.IsWindow(handle)) + { + result.FailureReason = "HWND 无效。"; + return result; + } + + NativeWindowInfo window = new NativeWindowService().Capture(handle); + result.WasMinimized = window != null && window.IsMinimized; + + Rectangle bounds = options.Target == ScreenshotTarget.Client + ? (window == null ? Rectangle.Empty : window.ClientBounds.ToRectangle()) + : (window == null ? Rectangle.Empty : window.WindowBounds.ToRectangle()); + if (bounds.Width < MinUsefulSize || bounds.Height < MinUsefulSize) + { + result.FailureReason = "窗口矩形不可用(可能已最小化或尺寸为 0)。"; + return result; + } + + result.Bounds = NativeRectangleInfo.FromRectangle(bounds); + + IntPtr foregroundBefore = NativeMethods.GetForegroundWindow(); + bool restored = false; + if (result.WasMinimized) + { + if (!options.AllowRestoreMinimized) + { + result.FailureReason = "窗口已最小化:默认不做截图。若必须截取,请在设置里授权“临时恢复最小化窗口”," + + "并知悉那会让窗口闪一下并可能改变前台窗口。"; + result.Attempts.Add("最小化:未尝试(未授权临时恢复)"); + return result; + } + + restored = TryRestoreForCapture(handle, bounds, options, result, foregroundBefore); + if (!restored) + { + return result; + } + + result.WasRestoredTemporarily = true; + } + + try + { + TryCaptureWith(NativeMethods.PW_RENDERFULLCONTENT, ScreenshotMethod.PrintWindowFull, handle, bounds, options, result); + if (result.Image == null) + { + TryCaptureWith(0, ScreenshotMethod.PrintWindow, handle, bounds, options, result); + } + + if (result.Image == null) + { + TryBitBlt(handle, bounds, options, result); + } + + if (result.Image == null && options.AllowScreenCopyFallback && !result.WasMinimized) + { + TryScreenCopy(bounds, result); + } + } + finally + { + if (restored) + { + RestoreAfterCapture(handle, foregroundBefore, result); + } + } + + if (result.Image == null && string.IsNullOrEmpty(result.FailureReason)) + { + result.FailureReason = "所有截图方式都失败。"; + } + + return result; + } + + /// Screen copy of an explicit rectangle: used for a control's bounding box. + public ScreenshotResult CaptureRectangle(Rectangle bounds, ScreenshotOptions options) + { + var result = new ScreenshotResult + { + Bounds = NativeRectangleInfo.FromRectangle(bounds) + }; + if (bounds.Width < MinUsefulSize || bounds.Height < MinUsefulSize) + { + result.FailureReason = "矩形无效或过小。"; + return result; + } + + TryScreenCopy(bounds, result); + if (result.Image == null && string.IsNullOrEmpty(result.FailureReason)) + { + result.FailureReason = "屏幕拷贝失败(区域可能不在可见桌面上)。"; + } + + return result; + } + + private static void TryCaptureWith( + uint flags, + ScreenshotMethod method, + IntPtr handle, + Rectangle bounds, + ScreenshotOptions options, + ScreenshotResult result) + { + Bitmap bitmap = null; + IntPtr hdc = IntPtr.Zero; + IntPtr memoryDc = IntPtr.Zero; + IntPtr previous = IntPtr.Zero; + try + { + bitmap = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppArgb); + using (Graphics graphics = Graphics.FromImage(bitmap)) + { + hdc = graphics.GetHdc(); + bool ok = NativeMethods.PrintWindow(handle, hdc, flags); + graphics.ReleaseHdc(hdc); + hdc = IntPtr.Zero; + if (!ok) + { + result.Attempts.Add(method + ":PrintWindow 返回失败"); + bitmap.Dispose(); + return; + } + } + + string reason; + if (TryValidate(bitmap, out reason)) + { + result.Image = bitmap; + result.Method = method; + result.IsComplete = true; + result.Attempts.Add(method + ":成功"); + AppLog.Info("截图:" + method + " 成功(" + bounds.Width + "x" + bounds.Height + ")。"); + return; + } + + result.Attempts.Add(method + ":图像无效(" + reason + ")"); + bitmap.Dispose(); + } + catch (Exception ex) + { + result.Attempts.Add(method + ":异常 " + ex.GetType().Name + ": " + ex.Message); + if (bitmap != null) + { + bitmap.Dispose(); + } + } + finally + { + if (hdc != IntPtr.Zero) + { + NativeMethods.ReleaseDC(handle, hdc); + } + + if (previous != IntPtr.Zero && memoryDc != IntPtr.Zero) + { + NativeMethods.SelectObject(memoryDc, previous); + } + + if (memoryDc != IntPtr.Zero) + { + NativeMethods.DeleteDC(memoryDc); + } + } + } + + private static void TryBitBlt(IntPtr handle, Rectangle bounds, ScreenshotOptions options, ScreenshotResult result) + { + Bitmap bitmap = null; + IntPtr windowDc = IntPtr.Zero; + IntPtr memoryDc = IntPtr.Zero; + IntPtr bitmapHandle = IntPtr.Zero; + IntPtr previous = IntPtr.Zero; + try + { + windowDc = NativeMethods.GetWindowDC(handle); + if (windowDc == IntPtr.Zero) + { + result.Attempts.Add("BitBlt:GetWindowDC 失败"); + return; + } + + memoryDc = NativeMethods.CreateCompatibleDC(windowDc); + bitmapHandle = NativeMethods.CreateCompatibleBitmap(windowDc, bounds.Width, bounds.Height); + if (memoryDc == IntPtr.Zero || bitmapHandle == IntPtr.Zero) + { + result.Attempts.Add("BitBlt:创建兼容位图失败"); + return; + } + + previous = NativeMethods.SelectObject(memoryDc, bitmapHandle); + bool ok = NativeMethods.BitBlt(memoryDc, 0, 0, bounds.Width, bounds.Height, windowDc, 0, 0, NativeMethods.SRCCOPY); + if (!ok) + { + result.Attempts.Add("BitBlt:BitBlt 返回失败"); + return; + } + + bitmap = Image.FromHbitmap(bitmapHandle); + string reason; + if (TryValidate(bitmap, out reason)) + { + result.Image = bitmap; + result.Method = ScreenshotMethod.BitBlt; + result.IsComplete = true; + result.Attempts.Add("BitBlt:成功"); + return; + } + + result.Attempts.Add("BitBlt:图像无效(" + reason + ")"); + bitmap.Dispose(); + } + catch (Exception ex) + { + result.Attempts.Add("BitBlt:异常 " + ex.GetType().Name + ": " + ex.Message); + if (bitmap != null) + { + bitmap.Dispose(); + } + } + finally + { + if (previous != IntPtr.Zero && memoryDc != IntPtr.Zero) + { + NativeMethods.SelectObject(memoryDc, previous); + } + + if (bitmapHandle != IntPtr.Zero) + { + NativeMethods.DeleteObject(bitmapHandle); + } + + if (memoryDc != IntPtr.Zero) + { + NativeMethods.DeleteDC(memoryDc); + } + + if (windowDc != IntPtr.Zero) + { + NativeMethods.ReleaseDC(handle, windowDc); + } + } + } + + private static void TryScreenCopy(Rectangle bounds, ScreenshotResult result) + { + Bitmap bitmap = null; + try + { + bitmap = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppArgb); + using (Graphics graphics = Graphics.FromImage(bitmap)) + { + graphics.CopyFromScreen(bounds.Left, bounds.Top, 0, 0, new Size(bounds.Width, bounds.Height), CopyPixelOperation.SourceCopy); + } + + string reason; + if (TryValidate(bitmap, out reason)) + { + result.Image = bitmap; + result.Method = ScreenshotMethod.ScreenCopy; + result.IsComplete = true; + result.Attempts.Add("屏幕拷贝:成功(注意:窗口被遮挡时会截到遮挡物)"); + return; + } + + result.Attempts.Add("屏幕拷贝:图像无效(" + reason + ")"); + bitmap.Dispose(); + } + catch (Exception ex) + { + result.Attempts.Add("屏幕拷贝:异常 " + ex.GetType().Name + ": " + ex.Message); + if (bitmap != null) + { + bitmap.Dispose(); + } + } + } + + /// + /// "Did we actually get a picture": non-empty size and more than one colour. A successful API + /// call that produced a uniformly black/white image is reported as a failure. + /// + public static bool TryValidate(Bitmap bitmap, out string reason) + { + reason = null; + if (bitmap == null) + { + reason = "没有图像"; + return false; + } + + if (bitmap.Width < MinUsefulSize || bitmap.Height < MinUsefulSize) + { + reason = "尺寸过小"; + return false; + } + + try + { + Color first = bitmap.GetPixel(0, 0); + int stepX = Math.Max(1, bitmap.Width / 40); + int stepY = Math.Max(1, bitmap.Height / 40); + var seen = new HashSet(); + int samples = 0; + for (int y = 0; y < bitmap.Height; y += stepY) + { + for (int x = 0; x < bitmap.Width; x += stepX) + { + Color pixel = bitmap.GetPixel(x, y); + seen.Add(pixel.ToArgb()); + samples++; + if (seen.Count > 3) + { + return true; + } + } + } + + if (seen.Count <= 1) + { + reason = "整张图只有一种颜色(" + first.R + "," + first.G + "," + first.B + + "),通常是黑屏或空白"; + return false; + } + + if (samples >= 100 && seen.Count * 100 / Math.Max(1, samples) < 2) + { + reason = "颜色过于单一,可能是空白图"; + return false; + } + + return true; + } + catch (Exception ex) + { + reason = "校验失败:" + ex.GetType().Name; + return false; + } + } + + /// Saves the image and returns the path; also used for OCR evidence. + public static string Save(Bitmap bitmap, string directory, string tag) + { + if (bitmap == null) + { + throw new ArgumentNullException("bitmap"); + } + + string target = string.IsNullOrWhiteSpace(directory) + ? Path.Combine(ExecutableDirectory(), "screenshots") + : directory; + Directory.CreateDirectory(target); + string name = DateTime.Now.ToString("yyyyMMdd-HHmmss-fff") + "-" + Sanitize(tag) + ".png"; + string path = Path.Combine(target, name); + bitmap.Save(path, ImageFormat.Png); + return path; + } + + private static bool TryRestoreForCapture( + IntPtr handle, + Rectangle bounds, + ScreenshotOptions options, + ScreenshotResult result, + IntPtr foregroundBefore) + { + try + { + AppLog.Warn("截图:目标窗口处于最小化,按设置临时恢复以便截取(会闪屏、可能改变前台窗口)。"); + NativeMethods.ShowWindow(handle, NativeMethods.SW_SHOWNOACTIVATE); + System.Threading.Thread.Sleep(Math.Max(0, options.RestoreSettleMs)); + return true; + } + catch (Exception ex) + { + result.FailureReason = "临时恢复最小化窗口失败:" + ex.GetType().Name + ": " + ex.Message; + result.Attempts.Add("最小化:恢复失败"); + return false; + } + } + + private static void RestoreAfterCapture(IntPtr handle, IntPtr foregroundBefore, ScreenshotResult result) + { + try + { + NativeMethods.ShowWindow(handle, NativeMethods.SW_MINIMIZE); + if (foregroundBefore != IntPtr.Zero && NativeMethods.IsWindow(foregroundBefore)) + { + NativeMethods.SetForegroundWindow(foregroundBefore); + } + + result.Attempts.Add("已把目标窗口恢复为最小化,并尝试还原原前台窗口(不能保证完全无感)"); + } + catch (Exception ex) + { + result.Attempts.Add("恢复窗口状态失败:" + ex.GetType().Name); + AppLog.Warn("截图后恢复窗口状态失败。", ex); + } + } + + private static string ExecutableDirectory() + { + string directory = Path.GetDirectoryName(typeof(ScreenshotService).Assembly.Location); + return string.IsNullOrEmpty(directory) ? Directory.GetCurrentDirectory() : directory; + } + + private static string Sanitize(string tag) + { + if (string.IsNullOrWhiteSpace(tag)) + { + return "capture"; + } + + foreach (char invalid in Path.GetInvalidFileNameChars()) + { + tag = tag.Replace(invalid, '_'); + } + + return tag.Length > 40 ? tag.Substring(0, 40) : tag; + } + } +} diff --git a/src/pacs_win_uia/UI/ControlMonitorEditorControl.cs b/src/pacs_win_uia/UI/ControlMonitorEditorControl.cs index 1529f36..eb3b220 100644 --- a/src/pacs_win_uia/UI/ControlMonitorEditorControl.cs +++ b/src/pacs_win_uia/UI/ControlMonitorEditorControl.cs @@ -16,14 +16,19 @@ namespace pacs_win_uia.UI Models.ControlMonitorDefinition definition, Models.ControlTreeNode node, Models.NativeRectangleInfo bounds, + IntPtr anchorWindow, IList steps) { Definition = definition; Node = node; Bounds = bounds; + AnchorWindow = anchorWindow; Steps = steps; } + /// Top-level window the control lives in; used to hide the frame when it goes away. + public IntPtr AnchorWindow { get; private set; } + public Models.ControlMonitorDefinition Definition { get; private set; } public Models.ControlTreeNode Node { get; private set; } @@ -622,7 +627,7 @@ namespace pacs_win_uia.UI if (handler != null) { handler(this, new ControlLocatedEventArgs(definition, resolution.Node, - BoundsOf(resolution.Node), steps)); + BoundsOf(resolution.Node), resolution.WindowHandle, steps)); } Report(string.Format("已定位到「{0}」({1}),焦点:{2}", diff --git a/src/pacs_win_uia/UI/MainForm.cs b/src/pacs_win_uia/UI/MainForm.cs index 48fffb1..30c017d 100644 --- a/src/pacs_win_uia/UI/MainForm.cs +++ b/src/pacs_win_uia/UI/MainForm.cs @@ -66,9 +66,17 @@ namespace pacs_win_uia private readonly ControlTreeSearchPanel _treeSearchPanel; private readonly Label _listResultHint; private readonly Button _settingsButton; + private readonly Button _screenshotButton; + private readonly Button _ocrButton; + private readonly ScreenshotService _screenshotService; + private OcrReadContext _ocrContext; + private ControlTreeNode _currentNode; private readonly Services.AppSettingsStore _settingsStore; private readonly Services.CommandRunner _commandRunner; private readonly HighlightOverlayForm _locateHighlight; + private readonly System.Windows.Forms.Timer _locateWatchTimer; + private ControlMonitorDefinition _locatedDefinition; + private IntPtr _locatedAnchor = IntPtr.Zero; private AppSettings _settings; private FloatingToolbarForm _floatingToolbar; private readonly List _treeMatches = new List(); @@ -118,6 +126,7 @@ namespace pacs_win_uia _controlTreeService = new ControlTreeService(); _settingsStore = new Services.AppSettingsStore(); _settings = _settingsStore.Load(); + _ocrContext = OcrReadContext.Create(_settings.Ocr); _commandRunner = new Services.CommandRunner(); _commandRunner.Output += delegate(object sender, CommandOutputEventArgs e) { @@ -176,6 +185,9 @@ namespace pacs_win_uia Enabled = false }; _settingsButton = new Button { Text = "设置", AutoSize = true }; + _screenshotButton = new Button { Text = "窗口截图", AutoSize = true, Enabled = false }; + _ocrButton = new Button { Text = "OCR 读取", AutoSize = true, Enabled = false }; + _screenshotService = new ScreenshotService(); _pickerPreviewLabel = new Label { Text = "拾取器未启动", @@ -199,6 +211,8 @@ namespace pacs_win_uia toolbar.Controls.Add(new Label { Text = "控件树来源:", AutoSize = true, Padding = new Padding(10, 7, 0, 0) }); toolbar.Controls.Add(_treeSourceCombo); toolbar.Controls.Add(_activateButton); + toolbar.Controls.Add(_screenshotButton); + toolbar.Controls.Add(_ocrButton); toolbar.Controls.Add(_settingsButton); toolbar.Controls.Add(_pickerPreviewLabel); toolbar.Controls.Add(limitationLabel); @@ -262,6 +276,8 @@ namespace pacs_win_uia _monitorEditor.ControlLocated += MonitorEditorOnControlLocated; _monitorEditor.DefinitionSelected += MonitorEditorOnDefinitionSelected; _locateHighlight = new HighlightOverlayForm(); + _locateWatchTimer = new System.Windows.Forms.Timer { Interval = 1000 }; + _locateWatchTimer.Tick += LocateWatchTimerOnTick; _statusText = new TextBox { Dock = DockStyle.Bottom, @@ -352,6 +368,8 @@ namespace pacs_win_uia }; _activateButton.Click += delegate { ActivateTargetAccessibility(); }; _settingsButton.Click += delegate { ShowSettings(); }; + _screenshotButton.Click += delegate { CaptureCurrentWindow(); }; + _ocrButton.Click += delegate { ReadCurrentByOcr(); }; _windowSearchBox.TextChanged += delegate { ApplyWindowFilter(); }; _windowList.SelectedIndexChanged += WindowListOnSelectedIndexChanged; _controlTree.AfterSelect += ControlTreeOnAfterSelect; @@ -360,6 +378,83 @@ namespace pacs_win_uia Shown += MainFormOnShown; } + /// + /// Captures the selected window (client area) and saves it. The method actually used and the + /// validation outcome are reported, so a rejected/blank capture is never presented as success. + /// + private void CaptureCurrentWindow() + { + NativeWindowInfo window = GetSelectedWindow(); + if (window == null) + { + SetStatus("请先在窗口列表里选中一个窗口,再截图。", null); + return; + } + + var options = new ScreenshotOptions + { + Target = ScreenshotTarget.Client, + AllowRestoreMinimized = _settings != null && _settings.Screenshot.AllowRestoreMinimized + }; + + ScreenshotResult result = _screenshotService.CaptureWindow(window.Handle, options); + AppLog.Info("截图(" + window + "):方式=" + result.Method + ";完整=" + result.IsComplete + + ";尝试=" + string.Join(" / ", result.Attempts.ToArray())); + + if (result.Image == null) + { + SetStatus("截图失败:" + (result.FailureReason ?? "未知原因"), null); + return; + } + + try + { + string path = ScreenshotService.Save(result.Image, _settings == null ? null : _settings.Screenshot.Directory, + window.ProcessName + "-" + window.HwndHex); + SetStatus(string.Format("截图完成:方式 {0};{1}x{2};已保存 {3}{4}", + result.Method, result.Image.Width, result.Image.Height, path, + result.WasRestoredTemporarily ? "(曾临时恢复最小化窗口)" : string.Empty), null); + } + catch (Exception ex) + { + AppLog.Error("截图保存失败。", ex); + SetStatus("截图成功但保存失败。", ex); + } + finally + { + result.Image.Dispose(); + } + } + + /// Screenshots the selected control and asks the local OCR service for its text. + private void ReadCurrentByOcr() + { + if (_currentNode == null) + { + SetStatus("请先拾取控件或在控件树里选中一个节点,再点 OCR 读取。", null); + return; + } + + if (_ocrContext == null) + { + SetStatus("OCR 未启用:请在「设置 → OCR」里打开总开关(默认关闭)。", null); + return; + } + + string value; + string error; + if (!ControlValueReader.TryRead(_currentNode, ControlValueKeys.OcrText, new ListReadOptions(), _ocrContext, out value, out error)) + { + SetStatus("OCR 读取失败:" + error, null); + return; + } + + // The recognized text is shown on screen only; it may contain patient data, so the log + // records the length instead of the content. + AppLog.Info("OCR 读取成功:" + value.Length + " 个字符。"); + SetStatus("OCR 结果(" + value.Length + " 字):" + value, null); + } + /// Opens the settings dialog and applies whatever was saved. private void ShowSettings() { @@ -372,6 +467,7 @@ namespace pacs_win_uia } _settings = _settingsStore.Load(); + _ocrContext = OcrReadContext.Create(_settings.Ocr); ApplyFloatingToolbar(); _commandRunner.RunStartCommands(_settings); SetStatus("设置已保存并生效:" + _settingsStore.FilePath, null); @@ -676,9 +772,12 @@ namespace pacs_win_uia _windowList.Groups.Add(primaryGroup); _windowList.Groups.Add(otherGroup); - IntPtr foreground = _settings != null && _settings.Ui.HighlightForegroundWindow - ? NativeMethods.GetForegroundWindow() - : IntPtr.Zero; + bool markColours = _settings == null || _settings.Ui.HighlightForegroundWindow; + IntPtr foreground = markColours ? NativeMethods.GetForegroundWindow() : IntPtr.Zero; + // _windows is in EnumWindows (z-order) order, so the first entry is the window that is + // physically on top. When the operator clicks 刷新, the tool itself is foreground and + // excluded from the list, which is why marking only the foreground window showed nothing. + IntPtr topmost = markColours && _windows.Count > 0 ? _windows[0].Handle : IntPtr.Zero; int shown = 0; foreach (NativeWindowInfo window in _windows) { @@ -689,11 +788,15 @@ namespace pacs_win_uia ListViewItem item = CreateWindowItem(window); item.Group = IsPrimaryWindow(window) ? primaryGroup : otherGroup; - if (foreground != IntPtr.Zero && window.Handle == foreground) + if (window.Handle == topmost) { - // The window the user is actually looking at. item.BackColor = Color.FromArgb(255, 250, 205); - item.ToolTipText = "当前前台窗口(屏幕上最上方的那个)"; + item.ToolTipText = "Z 序最前的窗口(屏幕上最上层)"; + } + else if (foreground != IntPtr.Zero && window.Handle == foreground) + { + item.BackColor = Color.FromArgb(225, 245, 225); + item.ToolTipText = "当前前台窗口(键盘焦点所在)"; } else if (window.IsMinimized) { @@ -973,7 +1076,7 @@ namespace pacs_win_uia _monitorEditor.CommitEdits(); // Captured before hiding so focus can be handed back to the target application. - _locateHighlight.HideHighlight(); + HideLocateFrame(null); _foregroundBeforePick = NativeMethods.GetForegroundWindow(); AppLog.Info("拾取开始:隐藏主界面;原前台窗口=" + NativeWindowInfo.FormatHandle(_foregroundBeforePick)); Hide(); @@ -1630,8 +1733,11 @@ namespace pacs_win_uia element.EnsureRuntimeId(); } + _currentNode = node; _propertyGrid.SelectedObject = SelectedObjectFor(node); _propertyGrid.Refresh(); + _screenshotButton.Enabled = node != null; + _ocrButton.Enabled = node != null; } /// Object shown in the property grid for a node, per source. @@ -1744,7 +1850,8 @@ namespace pacs_win_uia if (selfMatches) { _treeMatches.Add(treeNode); - treeNode.BackColor = Color.LightYellow; + // Bold, not coloured: the operator asked for colours to stay in the window list. + treeNode.NodeFont = new Font(_controlTree.Font, FontStyle.Bold); } if (!node.ChildrenComplete) @@ -2266,19 +2373,141 @@ namespace pacs_win_uia : _windowList.SelectedItems[0].Tag as NativeWindowInfo; } - /// Shows where a monitor entry actually is, by drawing the same non-activating frame the picker uses. + /// + /// Shows where a monitor entry actually is: selects it in the control tree, links the property + /// grid, and draws the same non-activating frame the picker uses. Clicking the same row again + /// hides the frame. + /// private void MonitorEditorOnControlLocated(object sender, ControlLocatedEventArgs e) { _propertyGrid.SelectedObject = e.Definition; _propertyGrid.Refresh(); + + if (_locatedDefinition == e.Definition && _locateHighlight.Visible) + { + HideLocateFrame("已隐藏高亮框。"); + return; + } + if (e.Bounds == null || !e.Bounds.IsValid) { - SetStatus("已定位到控件,但该控件没有可用的屏幕矩形,无法高亮。", null); + HideLocateFrame("已定位到控件,但该控件没有可用的屏幕矩形,无法高亮。"); return; } _locateHighlight.ShowHighlight(e.Bounds.ToRectangle()); - SetStatus("已在屏幕上高亮该控件(" + e.Bounds + ")。", null); + _locatedDefinition = e.Definition; + _locatedAnchor = e.AnchorWindow; + _locateWatchTimer.Start(); + SelectTreeNodeFor(e.Node); + SetStatus("已在屏幕上高亮该控件(" + e.Bounds + ");再点一次行头可隐藏。", null); + } + + /// Hides the frame when the target window is gone or minimized, so nothing stale is left behind. + private void LocateWatchTimerOnTick(object sender, EventArgs e) + { + if (!_locateHighlight.Visible) + { + _locateWatchTimer.Stop(); + return; + } + + if (_locatedAnchor == IntPtr.Zero) + { + return; + } + + if (!NativeMethods.IsWindow(_locatedAnchor)) + { + HideLocateFrame("目标窗口已关闭,已隐藏高亮框。"); + return; + } + + if (NativeMethods.IsIconic(_locatedAnchor)) + { + HideLocateFrame("目标窗口已最小化,已隐藏高亮框。"); + } + } + + private void HideLocateFrame(string reason) + { + _locateHighlight.HideHighlight(); + _locateWatchTimer.Stop(); + _locatedDefinition = null; + _locatedAnchor = IntPtr.Zero; + if (!string.IsNullOrEmpty(reason)) + { + SetStatus(reason, null); + } + } + + /// Finds the tree node for a resolved control and selects it (expanding its ancestors). + private void SelectTreeNodeFor(ControlTreeNode target) + { + if (target == null) + { + return; + } + + if (_currentTreeRoot == null) + { + SetStatus("已在屏幕上高亮该控件;当前没有控件树,先选中它的窗口再试可以看到节点联动。", null); + return; + } + + TreeNode found = FindTreeNode(_controlTree.Nodes, target); + if (found == null) + { + AppLog.Info("定位到控件,但当前控件树里没有对应节点(来源或窗口不同)。"); + SetStatus("已在屏幕上高亮该控件;但当前控件树是其它窗口/来源的,没有对应节点。", null); + return; + } + + found.EnsureVisible(); + _controlTree.SelectedNode = found; + } + + private static TreeNode FindTreeNode(TreeNodeCollection nodes, ControlTreeNode target) + { + foreach (TreeNode node in nodes) + { + var model = node.Tag as ControlTreeNode; + if (model != null && SameControl(model, target)) + { + return node; + } + + TreeNode child = FindTreeNode(node.Nodes, target); + if (child != null) + { + return child; + } + } + + return null; + } + + /// Identity comparison per source: the resolved node is not the same object as the tree's. + private static bool SameControl(ControlTreeNode left, ControlTreeNode right) + { + if (left == null || right == null || left.Source != right.Source) + { + return false; + } + + switch (left.Source) + { + case ControlTreeSource.Msaa: + return left.Msaa != null && right.Msaa != null && + string.Equals(left.Msaa.PathKey, right.Msaa.PathKey, StringComparison.Ordinal); + case ControlTreeSource.Win32: + return left.Window != null && right.Window != null && left.Window.Handle == right.Window.Handle; + default: + return left.Uia != null && right.Uia != null && + string.Equals(left.Uia.ControlType, right.Uia.ControlType, StringComparison.Ordinal) && + string.Equals(left.Uia.AutomationId, right.Uia.AutomationId, StringComparison.Ordinal) && + string.Equals(left.Uia.Name, right.Uia.Name, StringComparison.Ordinal); + } } /// The property grid doubles as the editor for the selected monitor entry. @@ -2353,6 +2582,9 @@ namespace pacs_win_uia _shutdown.Cancel(); _picker.Dispose(); _activationService.Dispose(); + _locateWatchTimer.Stop(); + _locateWatchTimer.Tick -= LocateWatchTimerOnTick; + _locateWatchTimer.Dispose(); _locateHighlight.Dispose(); _autoInspectTimer.Tick -= AutoInspectTimerOnTick; _autoInspectTimer.Dispose(); diff --git a/src/pacs_win_uia/UI/SettingsForm.cs b/src/pacs_win_uia/UI/SettingsForm.cs index 29fa74d..eeb88f2 100644 --- a/src/pacs_win_uia/UI/SettingsForm.cs +++ b/src/pacs_win_uia/UI/SettingsForm.cs @@ -21,9 +21,15 @@ namespace pacs_win_uia.UI private readonly AppSettings _settings; private readonly AppSettingsStore _store; private readonly CommandRunner _runner; - private readonly BindingList _continuous; - private readonly BindingList _once; - private readonly TextBox _outputBox; + private readonly BindingList _triggers; + private ListBox _triggerList; + private DataGridView _commandGrid; + private BindingList _currentCommands; + private TextBox _triggerName; + private ComboBox _triggerKind; + private NumericUpDown _triggerInterval; + private CheckBox _triggerEnabled; + private TextBox _outputBox; private CheckBox _autoReadTree; private CheckBox _highlightForeground; @@ -35,11 +41,12 @@ namespace pacs_win_uia.UI private NumericUpDown _treeTimeout; private NumericUpDown _readTimeout; private CheckBox _skipDisabled; - private TextBox _onStart; - private TextBox _onExit; + private CheckBox _ocrEnabled; - private TextBox _ocrEngine; - private TextBox _ocrLanguage; + private TextBox _ocrBaseUrl; + private NumericUpDown _ocrConfidence; + private NumericUpDown _ocrTimeout; + private CheckBox _ocrSaveScreenshots; private TextBox _ocrDirectory; public SettingsForm(AppSettings settings, AppSettingsStore store, CommandRunner runner) @@ -49,8 +56,7 @@ namespace pacs_win_uia.UI _runner = runner; _settings.Normalize(); - _continuous = new BindingList(CopyCommands(_settings.Commands.Continuous)); - _once = new BindingList(CopyCommands(_settings.Commands.Once)); + _triggers = new BindingList(CopyTriggers(_settings.Commands.Triggers)); Text = "设置 - PACS Windows UI Automation Inspector"; StartPosition = FormStartPosition.CenterParent; @@ -107,6 +113,52 @@ namespace pacs_win_uia.UI get { return _settings; } } + private static IList CopyTriggers(IList source) + { + var copy = new List(); + if (source == null) + { + return copy; + } + + foreach (CommandTrigger trigger in source) + { + if (trigger == null) + { + continue; + } + + var item = new CommandTrigger + { + Name = trigger.Name, + Kind = trigger.Kind, + IntervalSeconds = trigger.IntervalSeconds, + Enabled = trigger.Enabled + }; + foreach (CommandDefinition command in trigger.Commands ?? new List()) + { + if (command == null) + { + continue; + } + + item.Commands.Add(new CommandDefinition + { + Name = command.Name, + CommandLine = command.CommandLine, + WorkingDirectory = command.WorkingDirectory, + IntervalSeconds = command.IntervalSeconds, + TimeoutSeconds = command.TimeoutSeconds, + Enabled = command.Enabled + }); + } + + copy.Add(item); + } + + return copy; + } + private static IList CopyCommands(IList source) { var copy = new List(); @@ -281,70 +333,293 @@ namespace pacs_win_uia.UI private TabPage BuildCommandPage() { var page = new TabPage("命令"); - var layout = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 1, RowCount = 6 }; - layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 30)); - layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 30)); - layout.RowStyles.Add(new RowStyle(SizeType.Percent, 40)); - layout.RowStyles.Add(new RowStyle(SizeType.Percent, 40)); - layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 34)); - layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 60)); + var layout = new TableLayoutPanel { Dock = DockStyle.Fill, ColumnCount = 1, RowCount = 4 }; + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 62)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 36)); + layout.RowStyles.Add(new RowStyle(SizeType.Absolute, 58)); + layout.RowStyles.Add(new RowStyle(SizeType.Percent, 38)); - _onStart = AddCommandLine(layout, "启动执行(工具启动时执行一次):", _settings.Commands.OnStart); - _onExit = AddCommandLine(layout, "结束执行(工具退出时执行一次):", _settings.Commands.OnExit); + // ---- left: execution moments; right: commands of the selected moment + var split = new SplitContainer { Dock = DockStyle.Fill, Orientation = Orientation.Vertical }; - layout.Controls.Add(new Label + var left = new Panel { Dock = DockStyle.Fill }; + var leftButtons = new FlowLayoutPanel { Dock = DockStyle.Bottom, Height = 34, WrapContents = false }; + var addTrigger = new Button { Text = "新增执行时机", AutoSize = true }; + addTrigger.Click += delegate { AddTrigger(); }; + var removeTrigger = new Button { Text = "删除执行时机", AutoSize = true }; + removeTrigger.Click += delegate { RemoveTrigger(); }; + leftButtons.Controls.Add(addTrigger); + leftButtons.Controls.Add(removeTrigger); + + var triggerDetails = new Panel { Dock = DockStyle.Bottom, Height = 92 }; + var detailLayout = new FlowLayoutPanel { Dock = DockStyle.Fill, WrapContents = false, FlowDirection = FlowDirection.LeftToRight }; + detailLayout.Controls.Add(new Label { Text = "名称", AutoSize = true, Padding = new Padding(2, 8, 0, 0) }); + _triggerName = new TextBox { Width = 110 }; + _triggerName.TextChanged += delegate { ApplyTriggerDetails(); }; + detailLayout.Controls.Add(_triggerName); + detailLayout.Controls.Add(new Label { Text = "时机", AutoSize = true, Padding = new Padding(8, 8, 0, 0) }); + _triggerKind = new ComboBox { DropDownStyle = ComboBoxStyle.DropDownList, Width = 120 }; + _triggerKind.Items.AddRange(new object[] { - Text = "on(持续执行,按间隔循环,可随时停止)", - AutoSize = true, - Font = new Font(SystemFonts.DefaultFont, FontStyle.Bold) + "启动时一次", "退出时一次", "定时持续(on)", "启动时单次(once)" }); - layout.Controls.Add(BuildCommandGrid(_continuous)); + _triggerKind.SelectedIndexChanged += delegate { ApplyTriggerDetails(); }; + detailLayout.Controls.Add(_triggerKind); + _triggerEnabled = new CheckBox { Text = "启用", AutoSize = true, Padding = new Padding(8, 8, 0, 0) }; + _triggerEnabled.CheckedChanged += delegate { ApplyTriggerDetails(); }; + detailLayout.Controls.Add(_triggerEnabled); + detailLayout.Controls.Add(new Label { Text = "间隔秒", AutoSize = true, Padding = new Padding(8, 8, 0, 0) }); + _triggerInterval = new NumericUpDown { Minimum = 1, Maximum = 86400, Width = 70 }; + _triggerInterval.ValueChanged += delegate { ApplyTriggerDetails(); }; + detailLayout.Controls.Add(_triggerInterval); + triggerDetails.Controls.Add(detailLayout); - layout.Controls.Add(new Label + _triggerList = new ListBox { Dock = DockStyle.Fill, IntegralHeight = false }; + _triggerList.SelectedIndexChanged += delegate { LoadSelectedTrigger(); }; + left.Controls.Add(_triggerList); + left.Controls.Add(triggerDetails); + left.Controls.Add(leftButtons); + split.Panel1.Controls.Add(left); + split.Panel1MinSize = 240; + + var right = new Panel { Dock = DockStyle.Fill }; + var rightButtons = new FlowLayoutPanel { Dock = DockStyle.Bottom, Height = 34, WrapContents = false }; + var addCommand = new Button { Text = "新增", AutoSize = true }; + addCommand.Click += delegate { AddCommand(); }; + var removeCommand = new Button { Text = "删除", AutoSize = true }; + removeCommand.Click += delegate { RemoveCommand(); }; + rightButtons.Controls.Add(addCommand); + rightButtons.Controls.Add(removeCommand); + _commandGrid = BuildCommandGrid(); + right.Controls.Add(_commandGrid); + right.Controls.Add(rightButtons); + split.Panel2.Controls.Add(right); + split.Panel2MinSize = 320; + layout.Controls.Add(split); + + var actions = new FlowLayoutPanel { Dock = DockStyle.Fill, WrapContents = false }; + var runNow = new Button { Text = "立即执行当前时机的命令", AutoSize = true }; + runNow.Click += delegate { RunSelectedTrigger(); }; + actions.Controls.Add(runNow); + var startTimers = new Button { Text = "启动定时命令", AutoSize = true }; + startTimers.Click += delegate { - Text = "once(只在启动时执行一次)", - AutoSize = true, - Font = new Font(SystemFonts.DefaultFont, FontStyle.Bold) - }); - layout.Controls.Add(BuildCommandGrid(_once)); - - var buttons = new FlowLayoutPanel { Dock = DockStyle.Fill, WrapContents = false }; - var runSelected = new Button { Text = "立即执行选中的 once 命令", AutoSize = true }; - runSelected.Click += delegate { RunSelectedOnce(); }; - buttons.Controls.Add(runSelected); - var stopContinuous = new Button { Text = "停止持续命令", AutoSize = true }; - stopContinuous.Click += delegate + if (_runner != null) + { + ApplyTriggerDetails(); + _runner.StartContinuous(_settings); + ShowOutput("已按当前配置启动定时命令(保存后下次启动同样生效)。"); + } + }; + actions.Controls.Add(startTimers); + var stopTimers = new Button { Text = "停止定时命令", AutoSize = true }; + stopTimers.Click += delegate { if (_runner != null) { _runner.StopContinuous(); - ShowOutput("已停止持续命令。"); + ShowOutput("已停止定时命令。"); } }; - buttons.Controls.Add(stopContinuous); - var startContinuous = new Button { Text = "启动持续命令", AutoSize = true }; - startContinuous.Click += delegate - { - if (_runner != null) - { - _runner.StartContinuous(_continuous); - ShowOutput("已按当前列表启动持续命令(保存后下次启动同样生效)。"); - } - }; - buttons.Controls.Add(startContinuous); - layout.Controls.Add(buttons); + actions.Controls.Add(stopTimers); + layout.Controls.Add(actions); layout.Controls.Add(new Label { - Text = "命令通过 cmd.exe /c 执行,以当前用户身份运行;输出会记入控制台日志。" - + "不要把患者信息写进命令行。", + Text = "命令通过 cmd.exe /c 以当前用户身份执行;输出进控制台日志。" + + "默认只读、不注入目标进程、不提权。不要把患者信息写进命令行。", AutoSize = true, ForeColor = Color.DarkSlateGray }); + + var output = new TextBox + { + Dock = DockStyle.Fill, + Multiline = true, + ReadOnly = true, + ScrollBars = ScrollBars.Vertical, + BackColor = SystemColors.Window + }; + layout.Controls.Add(output); + _outputBox = output; + page.Controls.Add(layout); + RefreshTriggerList(); return page; } + /// Rebuilds the moment list and selects the first entry. + private void RefreshTriggerList() + { + int previous = _triggerList.SelectedIndex; + _triggerList.Items.Clear(); + foreach (CommandTrigger trigger in _triggers) + { + _triggerList.Items.Add(trigger); + } + + if (_triggerList.Items.Count > 0) + { + _triggerList.SelectedIndex = previous >= 0 && previous < _triggerList.Items.Count ? previous : 0; + } + else + { + LoadSelectedTrigger(); + } + } + + private void AddTrigger() + { + var trigger = new CommandTrigger + { + Name = "时机" + (_triggers.Count + 1), + Kind = CommandTriggerKind.Once + }; + _triggers.Add(trigger); + RefreshTriggerList(); + _triggerList.SelectedIndex = _triggerList.Items.Count - 1; + } + + private void RemoveTrigger() + { + CommandTrigger trigger = _triggerList.SelectedItem as CommandTrigger; + if (trigger == null) + { + return; + } + + if (MessageBox.Show( + this, + "删除执行时机「" + trigger + "」及其下面的 " + trigger.Commands.Count + " 条命令?", + "设置", + MessageBoxButtons.YesNo, + MessageBoxIcon.Warning) != DialogResult.Yes) + { + return; + } + + _triggers.Remove(trigger); + RefreshTriggerList(); + } + + /// Shows the selected moment's commands and copies its fields into the editors. + private void LoadSelectedTrigger() + { + var trigger = _triggerList.SelectedItem as CommandTrigger; + _commandGrid.Enabled = trigger != null; + if (trigger == null) + { + _commandGrid.DataSource = null; + return; + } + + _suppressTriggerDetailEvents = true; + try + { + _triggerName.Text = trigger.Name; + _triggerKind.SelectedIndex = (int)trigger.Kind; + _triggerInterval.Value = Math.Max(_triggerInterval.Minimum, Math.Min(_triggerInterval.Maximum, trigger.IntervalSeconds)); + _triggerEnabled.Checked = trigger.Enabled; + } + finally + { + _suppressTriggerDetailEvents = false; + } + + // BindingList wraps the trigger's own list, so edits land in the model directly. + _currentCommands = new BindingList(trigger.Commands); + _commandGrid.DataSource = _currentCommands; + } + + private bool _suppressTriggerDetailEvents; + + private void ApplyTriggerDetails() + { + if (_suppressTriggerDetailEvents) + { + return; + } + + var trigger = _triggerList.SelectedItem as CommandTrigger; + if (trigger == null) + { + return; + } + + trigger.Name = _triggerName.Text; + trigger.Kind = (CommandTriggerKind)Math.Max(0, _triggerKind.SelectedIndex); + trigger.IntervalSeconds = (int)_triggerInterval.Value; + trigger.Enabled = _triggerEnabled.Checked; + + // The list shows ToString(), so refresh the row text without losing the selection. + int index = _triggerList.SelectedIndex; + _triggerList.Items[index] = trigger; + _triggerList.SelectedIndex = index; + } + + private void AddCommand() + { + var trigger = _triggerList.SelectedItem as CommandTrigger; + if (trigger == null) + { + return; + } + + var command = new CommandDefinition + { + Name = "命令" + (trigger.Commands.Count + 1), + CommandLine = "echo hello", + IntervalSeconds = trigger.IntervalSeconds + }; + trigger.Commands.Add(command); + _currentCommands.ResetBindings(); + } + + private void RemoveCommand() + { + if (_currentCommands == null || _commandGrid.CurrentRow == null) + { + return; + } + + int index = _commandGrid.CurrentRow.Index; + if (index >= 0 && index < _currentCommands.Count) + { + _currentCommands.RemoveAt(index); + } + } + + private void RunSelectedTrigger() + { + var trigger = _triggerList.SelectedItem as CommandTrigger; + if (_runner == null || trigger == null) + { + return; + } + + if (trigger.Commands.Count == 0) + { + ShowOutput("当前时机下没有命令。"); + return; + } + + foreach (CommandDefinition command in trigger.Commands) + { + if (command == null || string.IsNullOrWhiteSpace(command.CommandLine)) + { + continue; + } + + ShowOutput("正在执行:" + command.CommandLine); + CommandResult result = _runner.RunCommand(command, "手动/" + trigger.Name); + ShowOutput(result.TimedOut + ? "执行超时。" + : result.Error != null && !result.Started + ? "执行失败:" + result.Error + : "执行结束,退出码 " + result.ExitCode + "。"); + } + } + private TextBox AddCommandLine(TableLayoutPanel layout, string label, string value) { var panel = new Panel { Dock = DockStyle.Fill }; @@ -356,7 +631,7 @@ namespace pacs_win_uia.UI return (TextBox)panel.Controls[0]; } - private DataGridView BuildCommandGrid(BindingList list) + private DataGridView BuildCommandGrid() { var grid = new DataGridView { @@ -372,7 +647,6 @@ namespace pacs_win_uia.UI grid.Columns.Add(new DataGridViewTextBoxColumn { HeaderText = "工作目录", DataPropertyName = "WorkingDirectory", FillWeight = 20 }); grid.Columns.Add(new DataGridViewTextBoxColumn { HeaderText = "间隔秒", DataPropertyName = "IntervalSeconds", FillWeight = 8 }); grid.Columns.Add(new DataGridViewTextBoxColumn { HeaderText = "超时秒", DataPropertyName = "TimeoutSeconds", FillWeight = 8 }); - grid.DataSource = list; return grid; } @@ -386,30 +660,72 @@ namespace pacs_win_uia.UI WrapContents = false, Padding = new Padding(10) }; + + _ocrEnabled = new CheckBox + { + Text = "启用 OCR(关闭时本工具完全不会调用 OCR 服务)", + Checked = _settings.Ocr.Enabled, + AutoSize = true + }; + panel.Controls.Add(_ocrEnabled); panel.Controls.Add(new Label { - Text = "本轮未实现:下面只是配置占位,不会执行任何识别。", + Text = "识别时机的选择:控件用 path 读不到内容、或原文取值失败时,才会截取该控件的矩形去识别。" + + "识别结果会标注“来源=OCR、不是控件原生值”,不会被当成控件自己的值。", AutoSize = true, - Font = new Font(SystemFonts.DefaultFont, FontStyle.Bold), - ForeColor = Color.Firebrick + ForeColor = Color.DarkSlateGray, + Padding = new Padding(0, 6, 0, 6) }); - _ocrEnabled = new CheckBox { Text = "启用 OCR(未实现)", Checked = _settings.Ocr.Enabled, AutoSize = true, Enabled = false }; - panel.Controls.Add(_ocrEnabled); - panel.Controls.Add(new Label { Text = "识别引擎(预留):", AutoSize = true, Padding = new Padding(0, 8, 0, 0) }); - _ocrEngine = new TextBox { Width = 260, Text = _settings.Ocr.Engine ?? string.Empty, Enabled = false }; - panel.Controls.Add(_ocrEngine); - panel.Controls.Add(new Label { Text = "语言(预留):", AutoSize = true, Padding = new Padding(0, 8, 0, 0) }); - _ocrLanguage = new TextBox { Width = 160, Text = _settings.Ocr.Language ?? string.Empty, Enabled = false }; - panel.Controls.Add(_ocrLanguage); - panel.Controls.Add(new Label { Text = "截图目录(预留):", AutoSize = true, Padding = new Padding(0, 8, 0, 0) }); - _ocrDirectory = new TextBox { Width = 360, Text = _settings.Ocr.CaptureDirectory ?? string.Empty, Enabled = false }; + + panel.Controls.Add(new Label { Text = "OCR 服务地址:", AutoSize = true, Padding = new Padding(0, 6, 0, 0) }); + _ocrBaseUrl = new TextBox { Width = 320, Text = _settings.Ocr.BaseUrl ?? string.Empty }; + panel.Controls.Add(_ocrBaseUrl); + panel.Controls.Add(new Label + { + Text = "默认 http://127.0.0.1:8000(本机 ocr-server:/healthz、/api/v1/upload、/api/v1/i2t)。" + + "只发到这个地址,不会上传到外部服务。", + AutoSize = true, + ForeColor = Color.DarkSlateGray + }); + + panel.Controls.Add(new Label { Text = "最小置信度(0-1):", AutoSize = true, Padding = new Padding(0, 6, 0, 0) }); + _ocrConfidence = new NumericUpDown + { + Minimum = 0, + Maximum = 1, + DecimalPlaces = 2, + Increment = 0.05M, + Width = 80, + Value = (decimal)Math.Max(0, Math.Min(1, _settings.Ocr.MinConfidence)) + }; + panel.Controls.Add(_ocrConfidence); + panel.Controls.Add(new Label { Text = "单次识别总超时(秒,含排队):", AutoSize = true, Padding = new Padding(0, 6, 0, 0) }); + _ocrTimeout = new NumericUpDown + { + Minimum = 1, + Maximum = 3600, + Width = 80, + Value = Math.Max(1, Math.Min(3600, _settings.Ocr.TimeoutSeconds)) + }; + panel.Controls.Add(_ocrTimeout); + + _ocrSaveScreenshots = new CheckBox + { + Text = "识别时保存截图", + Checked = _settings.Ocr.SaveScreenshots, + AutoSize = true, + Padding = new Padding(0, 8, 0, 0) + }; + panel.Controls.Add(_ocrSaveScreenshots); + panel.Controls.Add(new Label { Text = "截图保存目录(留空 = exe 目录下的 screenshots):", AutoSize = true, Padding = new Padding(0, 6, 0, 0) }); + _ocrDirectory = new TextBox { Width = 360, Text = _settings.Ocr.ScreenshotDirectory ?? string.Empty }; panel.Controls.Add(_ocrDirectory); panel.Controls.Add(new Label { - Text = "实现时需引入识别引擎与中文语言包,并明确“截图可能包含患者信息”,默认不上传外部服务。", + Text = "截图可能包含患者信息:请自行控制保存目录与保留期限,默认不写入日志、不上传外部服务。", AutoSize = true, - ForeColor = Color.DarkSlateGray, - Padding = new Padding(0, 8, 0, 0) + ForeColor = Color.Firebrick, + Padding = new Padding(0, 6, 0, 0) }); page.Controls.Add(panel); return page; @@ -437,10 +753,15 @@ namespace pacs_win_uia.UI _settings.FloatingButton.Actions = actions; - _settings.Commands.OnStart = _onStart.Text; - _settings.Commands.OnExit = _onExit.Text; - _settings.Commands.Continuous = new List(_continuous); - _settings.Commands.Once = new List(_once); + // The grid wraps each trigger's own list, so the edits are already in _triggers. + _settings.Commands.Triggers = new List(_triggers); + + _settings.Ocr.Enabled = _ocrEnabled.Checked; + _settings.Ocr.BaseUrl = _ocrBaseUrl.Text; + _settings.Ocr.MinConfidence = (double)_ocrConfidence.Value; + _settings.Ocr.TimeoutSeconds = (int)_ocrTimeout.Value; + _settings.Ocr.SaveScreenshots = _ocrSaveScreenshots.Checked; + _settings.Ocr.ScreenshotDirectory = _ocrDirectory.Text; if (!_store.Save(_settings)) { @@ -458,35 +779,6 @@ namespace pacs_win_uia.UI Close(); } - private void RunSelectedOnce() - { - if (_runner == null) - { - ShowOutput("命令执行器不可用。"); - return; - } - - CommandDefinition command = null; - if (_once.Count > 0) - { - command = _once[0]; - } - - if (command == null) - { - ShowOutput("once 列表为空,先新增一条命令。"); - return; - } - - ShowOutput("正在执行:" + command.CommandLine); - CommandResult result = _runner.RunCommand(command, "手动"); - ShowOutput(result.TimedOut - ? "执行超时。" - : result.Error != null - ? "执行失败:" + result.Error - : "执行结束,退出码 " + result.ExitCode + "。"); - } - private void AppendOutput(CommandOutputEventArgs e) { if (IsDisposed) diff --git a/src/pacs_win_uia/pacs_win_uia.csproj b/src/pacs_win_uia/pacs_win_uia.csproj index ed6c117..ed8bcf9 100644 --- a/src/pacs_win_uia/pacs_win_uia.csproj +++ b/src/pacs_win_uia/pacs_win_uia.csproj @@ -75,6 +75,8 @@ + + diff --git a/src/pacs_win_uia/settings.json b/src/pacs_win_uia/settings.json index 6104c27..3eb2f98 100644 --- a/src/pacs_win_uia/settings.json +++ b/src/pacs_win_uia/settings.json @@ -22,12 +22,20 @@ }, "Ocr": { "Enabled": false, - "Engine": "(未实现)", + "BaseUrl": "http://127.0.0.1:8000", "Language": "zh-CN", - "CaptureDirectory": "" + "SaveScreenshots": false, + "ScreenshotDirectory": "", + "MinConfidence": 0, + "TimeoutSeconds": 30 }, "Read": { "DefaultTimeoutSeconds": 10, "SkipDisabled": true + }, + "Screenshot": { + "Directory": "", + "AllowRestoreMinimized": false, + "RestoreSettleMs": 350 } }