From d90ed03f121394203a3b1c27064a5dbfd37e59a2 Mon Sep 17 00:00:00 2001 From: "yineng.huang" Date: Tue, 13 Aug 2024 16:08:57 +0800 Subject: [PATCH] =?UTF-8?q?=E6=BB=91=E5=8A=A8=E8=A7=A3=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../constants/AutomationRequestCmd.java | 2 + .../automation/constants/UpperParamKey.java | 7 +- .../handler/AbstractAutomationHandler.java | 2 + .../handler/AndroidAutomationHandler.java | 6 ++ .../handler/AutomationMessageHandler.java | 2 + .../handler/IosAutomationHandler.java | 92 +++++++++++++++++++ .../upperComputer/driver/ios/NKAgent.java | 16 +++- .../mobile/ios/constants/UpperParamKey.java | 7 +- .../mobile/ios/entity/ElementHandleParam.java | 15 +++ .../mobile/ios/keywords/IOSNewTools.java | 44 +++++++++ .../ios/utils/AutomationHandleUtil.java | 32 +++++++ 11 files changed, 219 insertions(+), 6 deletions(-) diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/AutomationRequestCmd.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/AutomationRequestCmd.java index 9b9e299..9338a6c 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/AutomationRequestCmd.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/AutomationRequestCmd.java @@ -55,4 +55,6 @@ public interface AutomationRequestCmd { String SCREEN_SHOT = "screen_shot"; //屏幕截图 String GET_ELEMENT_VALUE_BY_PATH_OCR = "get_element_value_by_path_ocr"; //获取控件的值（OCR方式） + + String PRESS_AND_SWIPE = "press_and_swipe"; //滑动解锁 } diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/UpperParamKey.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/UpperParamKey.java index 47c9a6d..7ce6618 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/UpperParamKey.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/UpperParamKey.java @@ -77,7 +77,7 @@ public interface UpperParamKey { /** * 超时时间 */ - String WAIT_TIMEOUT = "waitTimeout"; + String WAIT_TIMEOUT = "wait_time_out"; /** * 识别类型 @@ -88,4 +88,9 @@ public interface UpperParamKey { * 租户id */ String TENANT_ID = "tenant_id"; + + /** + * 滑动解说的坐标点 + */ + String SWIPE_POINT = "swipe_point"; } diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AbstractAutomationHandler.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AbstractAutomationHandler.java index 12aead3..c94653d 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AbstractAutomationHandler.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AbstractAutomationHandler.java @@ -79,6 +79,8 @@ public abstract class AbstractAutomationHandler implements AutomationMessageHand screenShot(request); } else if (AutomationRequestCmd.GET_ELEMENT_VALUE_BY_PATH_OCR.equals(cmd)) { getElementValueByPathOcr(request); + } else if (AutomationRequestCmd.PRESS_AND_SWIPE.equals(cmd)) { + pressAndSwipe(request); } } diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AndroidAutomationHandler.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AndroidAutomationHandler.java index 481bdc0..28c0510 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AndroidAutomationHandler.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AndroidAutomationHandler.java @@ -194,4 +194,10 @@ public class AndroidAutomationHandler extends AbstractAutomationHandler{ public void getElementValueByPathOcr(CmdAutomationRequest request) { } + + + @Override + public void pressAndSwipe(CmdAutomationRequest request) { + + } } diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AutomationMessageHandler.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AutomationMessageHandler.java index a383750..b1dd8ef 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AutomationMessageHandler.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AutomationMessageHandler.java @@ -80,4 +80,6 @@ public interface AutomationMessageHandler { void getElementValueByPathOcr(CmdAutomationRequest request); ////获取控件的值（OCR方式） + void pressAndSwipe(CmdAutomationRequest request); //滑动解锁 + } diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/IosAutomationHandler.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/IosAutomationHandler.java index e7c9ed4..d10932c 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/IosAutomationHandler.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/IosAutomationHandler.java @@ -1617,6 +1617,98 @@ public class IosAutomationHandler extends AbstractAutomationHandler { } } + @Override + public void pressAndSwipe(CmdAutomationRequest request) { + CmdAutomationResponse response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(false); + boolean success = false; + Map data = request.getData(); + NKAgent nkAgent = getNkAgent(); + try { + if (null == nkAgent) { + response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开"); + return; + } + String nodeTree = (String) data.get(UpperParamKey.NODE_TREE); + List passPoint = (List) data.get(UpperParamKey.SWIPE_POINT); + Integer waitTimeOut = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT); + logger.debug("拿到的nodeTree:{}", nodeTree); + logger.debug("拿到的经过的点:{}", JSON.toJSONString(passPoint)); + SearchUiNodeData searchUiNodeData = new SearchUiNodeData(); + searchUiNodeData.setChain(nodeTree); + long startTime = System.currentTimeMillis(); + long endTime = startTime; + logger.debug("步骤：{}开始查找控件，超时时间：{}", request.getStepToken(), waitTimeOut); + boolean alreadyFind = false; + UiNodeData uiNodeData = null; + int findIndex = 1; + do { + logger.debug("步骤：{}，第{}次开始查找元素", request.getStepToken(), findIndex); + uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData); + endTime = System.currentTimeMillis(); + logger.debug("步骤：{}，第{}次查找元素的总时间（累加）：{}ms，结果：{}", request.getStepToken(), findIndex, endTime - startTime, JSON.toJSONString(uiNodeData)); + findIndex++; + if (null != uiNodeData && uiNodeData.getX() > 0 && uiNodeData.getY() > 0) { + alreadyFind = true; + break; + } + } while ((endTime - startTime) < waitTimeOut * 1000 && !alreadyFind); + if (null == uiNodeData) { + response = CmdAutomationResponse.builderSuccess(request, "未找到滑动的区域").withData(false); + } else { + if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) { + response = CmdAutomationResponse.builderSuccess(request, "未找到滑动的区域").withData(false); + } else { + int x = uiNodeData.getX(); + int y = uiNodeData.getY(); + int width = uiNodeData.getWidth(); + int height = uiNodeData.getHeight(); + List> pointList = new ArrayList<>(); + logger.info("手势区域，位置=>x:{},y:{},宽：{}，高：{}",x,y,width,height); + int perWidth = width /3; + int perHeight = height /3; + int pointY = y - perHeight/2; + int n = 1; + for (int i = 0; i < 3; i++) { + pointY = pointY + perHeight; //每一行的y坐标 + int pointX = x - perWidth/2; + for (int j = 0; j < 3; j++) { + pointX = pointX + perWidth; + logger.info("第{}个点的坐标x:{},y:{}",n,pointX,pointY); + Map point = new HashMap<>(); + point.put("x", pointX); + point.put("y", pointY); + pointList.add(n-1,point); + n++; + } + } + logger.info("token:{},滑动解锁的所有坐标集合:{}",request.getStepToken(),JSON.toJSONString(pointList)); + List passPointNum = new ArrayList<>(); + if (!CollectionUtils.isEmpty(passPoint)) { + for (int i=0;i point = pointList.get(index - 1); + passPointNum.add(point.get("x")); + passPointNum.add(point.get("y")); + } + logger.info("token:{},滑动解锁通过的坐标点:{}",request.getStepToken(),JSON.toJSONString(passPointNum)); + success = nkAgent.drawLine(JSON.toJSONString(passPointNum)); + } else { + logger.warn("token:{},没有滑动通过的坐标",request.getStepToken()); + } + logger.info("token:{},滑动解锁的结果:{}",request.getStepToken(),success); + } + } + if (success) { + response = CmdAutomationResponse.builderSuccess(request, "滑动解锁成功").withData(success); + } else { + response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(success); + } + } finally { + sendResultToEngine(response); + } + } + + private String getValueByOcr(UiNodeData uiNodeData) { String value = null; String imgBase64 = getOcrAreaByBodeToBase64(uiNodeData); diff --git a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/driver/ios/NKAgent.java b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/driver/ios/NKAgent.java index dd09611..c63cb56 100644 --- a/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/driver/ios/NKAgent.java +++ b/cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/driver/ios/NKAgent.java @@ -1,10 +1,7 @@ package net.northking.cctp.upperComputer.driver.ios; -import net.northking.cctp.upperComputer.driver.ios.command.data.AppBatteryData; -import net.northking.cctp.upperComputer.driver.ios.command.data.ScreenInfoData; -import net.northking.cctp.upperComputer.driver.ios.command.data.SearchUiNodeData; -import net.northking.cctp.upperComputer.driver.ios.command.data.UiNodeData; +import net.northking.cctp.upperComputer.driver.ios.command.data.*; import net.northking.cctp.upperComputer.driver.ios.command.listener.AppBatteryDataListener; import net.northking.cctp.upperComputer.driver.ios.command.listener.ScreenInfoDataListener; import net.northking.cctp.upperComputer.driver.ios.packet.*; @@ -214,6 +211,17 @@ public final class NKAgent { return bool.isSuccess(); } + + public boolean drawLine(String string) { + TextData data = new TextData(); + data.setText(string); + ICommandPacket packet = packetTransfer.syncSend(29, data); + if (packet == null) return false; + BooleanCommandData bool = new BooleanCommandData(); + packet.fillICommandData(bool); + return bool.isSuccess(); + } + public boolean getStatus() { return this.packetTransfer.isConnected(); } diff --git a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/constants/UpperParamKey.java b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/constants/UpperParamKey.java index 502ea53..d9725d7 100644 --- a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/constants/UpperParamKey.java +++ b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/constants/UpperParamKey.java @@ -76,7 +76,7 @@ public interface UpperParamKey { /** * 超时时间 */ - String WAIT_TIMEOUT = "waitTimeout"; + String WAIT_TIMEOUT = "wait_time_out"; /** * 识别类型 @@ -88,4 +88,9 @@ public interface UpperParamKey { */ String TENANT_ID = "tenant_id"; + /** + * 滑动解锁的坐标 + */ + String SWIPE_POINT = "swipe_point"; + } diff --git a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/entity/ElementHandleParam.java b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/entity/ElementHandleParam.java index aa82199..43e5331 100644 --- a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/entity/ElementHandleParam.java +++ b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/entity/ElementHandleParam.java @@ -42,6 +42,13 @@ public class ElementHandleParam { private String type; //应用操作类型 + private List passingPoints; //滑动经过的点 + + public ElementHandleParam withPassingPoints(List passingPoints) { + this.setPassingPoints(passingPoints); + return this; + } + public ElementHandleParam withType(String type) { this.setType(type); return this; @@ -266,4 +273,12 @@ public class ElementHandleParam { public void setInputTextValue(String inputTextValue) { this.inputTextValue = inputTextValue; } + + public void setPassingPoints(List passingPoints) { + this.passingPoints = passingPoints; + } + + public List getPassingPoints() { + return passingPoints; + } } diff --git a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/keywords/IOSNewTools.java b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/keywords/IOSNewTools.java index 571b126..f7889fa 100644 --- a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/keywords/IOSNewTools.java +++ b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/keywords/IOSNewTools.java @@ -937,4 +937,48 @@ public class IOSNewTools { return result; } + /** + *

滑动解锁（ios新）

+ * + * @param deviceDriver 设备连接驱动 + * @param targets 滑动解锁的区域 + * @param waitTime 默认超时时间 + * @param passingPoints 经过的点，填[1,3,5,6]数组的形式 + * @param preExecuteWait 执行前等待 + * @param sufExecuteWait 执行后等待 + * @return + */ + @Keyword(alias = "滑动解锁（ios新）", category = "0", attributes = "5",commonlyUse = true) + @Return(name = "result", comment = "是否滑动成功", type = DataType.BOOLEAN) + public Boolean pressAndSwipe( + IExecuteContext context, + @Argument(name = "__deviceDriver", scope = ParamScope.CONTEXT, comment = "设备连接", type = DataType.OBJECT) DeviceDriver deviceDriver, + @Argument(name = "targets", scope = ParamScope.STEP, comment = "控件位置", type = DataType.LIST, defaultDisplay = false) List targets, + @Argument(name = "passingPoints", scope = ParamScope.ARGS, comment = "经过的点", type = DataType.LIST, defaultDisplay = true) List passingPoints, + @Argument(name = "waitTime", scope = ParamScope.ARGS, comment = "等待时间(单位s)", type = DataType.INTEGER, defaultValue = "30") Integer waitTime, + @Argument(name = "preExecuteWait", scope = ParamScope.ARGS, comment = "执行前等待(单位:s)", type = DataType.INTEGER, required = false, defaultValue = "1.0", defaultDisplay = false) Float preExecuteWait, + @Argument(name = "sufExecuteWait", scope = ParamScope.ARGS, comment = "执行后等待(单位:s)", type = DataType.INTEGER, required = false, defaultValue = "1.0", defaultDisplay = false) Float sufExecuteWait + + ) { + CommonUtils.handlePreExecuteWait(preExecuteWait); //执行前等待 + ElementHandleParam param = ElementHandleParam.builder(deviceDriver) + .useContext(context) + .waitTimeout(waitTime) + .withTargets(targets) + .withPassingPoints(passingPoints); + Boolean result = false; + try { + result = AutomationHandleUtil.pressAndSwipe(param); + } catch (InterruptedException ie) { + logger.warn("用户取消查询元素"); + throw new ExecuteException("取消操作"); + } catch (Exception e) { + logger.error("出现了其他的问题。。。。", e); + throw new ExecuteException(e.getMessage()); + } + CommonUtils.handleSufExecuteWait(sufExecuteWait); //执行后等待 + return result; + } + + } diff --git a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/utils/AutomationHandleUtil.java b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/utils/AutomationHandleUtil.java index 269b73e..9a4dd67 100644 --- a/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/utils/AutomationHandleUtil.java +++ b/cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/utils/AutomationHandleUtil.java @@ -1080,4 +1080,36 @@ public class AutomationHandleUtil { } return result; } + + public static Boolean pressAndSwipe(ElementHandleParam param) throws Exception{ + boolean success = false; + String token = UUID.randomUUID().toString(); + String targetValue = null; + for (IStepTarget target : param.getTargets()) { + if (UsingType.SELECTOR.equalsIgnoreCase(target.getUsing())) { + targetValue = target.getValue(); + } + } + if (StringUtils.isBlank(targetValue)) { + throw new ExecuteException("无法定位到滑动解锁的区域"); + } + Map paramMap = new HashMap<>(); + JSONObject object = JSON.parseObject(targetValue, JSONObject.class); + String xpath = object.getString(UsingType.Key.SELECTOR_KEY); + logger.info("拿到的xpath:{}", xpath); + paramMap.put(UpperParamKey.NODE_TREE, xpath); + paramMap.put(UpperParamKey.SWIPE_POINT, param.getPassingPoints()); + paramMap.put(UpperParamKey.WAIT_TIMEOUT, param.getWaitTimeout()); + logger.info("参数：{}", JSON.toJSONString(paramMap)); + CmdAutomationRequest builder = CmdAutomationRequest.builder(AutomationRequestCmd.PRESS_AND_SWIPE, token, paramMap); + Object o = sendUpperMessageAndWaitReturn(param.getDeviceDriver(), builder, token, param.getWaitTimeout()); + logger.debug("滑动解锁的结果：{}",o); + if (null != o) { + success = (boolean) o; + } + if (!success) { + throw new ExecuteException("滑动解锁失败"); + } + return success; + } }