yineng.huang
/
hz
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

滑动解锁

master
yineng.huang 2024-08-13 16:08:57 +08:00
parent 5de13aaf46
commit d90ed03f12
11 changed files with 219 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/AutomationRequestCmd.java
View File

@ -55,4 +55,6 @@ public interface AutomationRequestCmd {
String SCREEN_SHOT = "screen_shot"; //屏幕截图
String GET_ELEMENT_VALUE_BY_PATH_OCR = "get_element_value_by_path_ocr"; //获取控件的值（OCR方式）
String PRESS_AND_SWIPE = "press_and_swipe"; //滑动解锁
}

7
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/constants/UpperParamKey.java
View File

@ -77,7 +77,7 @@ public interface UpperParamKey {
/**
* 超时时间
*/
String WAIT_TIMEOUT = "waitTimeout";
String WAIT_TIMEOUT = "wait_time_out";
/**
* 识别类型
@ -88,4 +88,9 @@ public interface UpperParamKey {
* 租户id
*/
String TENANT_ID = "tenant_id";
/**
* 滑动解说的坐标点
*/
String SWIPE_POINT = "swipe_point";
}

2
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AbstractAutomationHandler.java
View File

@ -79,6 +79,8 @@ public abstract class AbstractAutomationHandler implements AutomationMessageHand
screenShot(request);
} else if (AutomationRequestCmd.GET_ELEMENT_VALUE_BY_PATH_OCR.equals(cmd)) {
getElementValueByPathOcr(request);
} else if (AutomationRequestCmd.PRESS_AND_SWIPE.equals(cmd)) {
pressAndSwipe(request);
}
}

6
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AndroidAutomationHandler.java
View File

@ -194,4 +194,10 @@ public class AndroidAutomationHandler extends AbstractAutomationHandler{
public void getElementValueByPathOcr(CmdAutomationRequest request) {
}
@Override
public void pressAndSwipe(CmdAutomationRequest request) {
}
}

2
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/AutomationMessageHandler.java
View File

@ -80,4 +80,6 @@ public interface AutomationMessageHandler {
void getElementValueByPathOcr(CmdAutomationRequest request); ////获取控件的值（OCR方式）
void pressAndSwipe(CmdAutomationRequest request); //滑动解锁
}

92
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/automation/handler/IosAutomationHandler.java
View File

@ -1617,6 +1617,98 @@ public class IosAutomationHandler extends AbstractAutomationHandler {
}
}
@Override
public void pressAndSwipe(CmdAutomationRequest request) {
CmdAutomationResponse response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(false);
boolean success = false;
Map<String, Object> data = request.getData();
NKAgent nkAgent = getNkAgent();
try {
if (null == nkAgent) {
response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
return;
}
String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
List<Integer> passPoint = (List<Integer>) data.get(UpperParamKey.SWIPE_POINT);
Integer waitTimeOut = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
logger.debug("拿到的nodeTree:{}", nodeTree);
logger.debug("拿到的经过的点:{}", JSON.toJSONString(passPoint));
SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
searchUiNodeData.setChain(nodeTree);
long startTime = System.currentTimeMillis();
long endTime = startTime;
logger.debug("步骤：{}开始查找控件，超时时间：{}", request.getStepToken(), waitTimeOut);
boolean alreadyFind = false;
UiNodeData uiNodeData = null;
int findIndex = 1;
do {
logger.debug("步骤：{}，第{}次开始查找元素", request.getStepToken(), findIndex);
uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData);
endTime = System.currentTimeMillis();
logger.debug("步骤：{}，第{}次查找元素的总时间（累加）：{}ms，结果：{}", request.getStepToken(), findIndex, endTime - startTime, JSON.toJSONString(uiNodeData));
findIndex++;
if (null != uiNodeData && uiNodeData.getX() > 0 && uiNodeData.getY() > 0) {
alreadyFind = true;
break;
}
} while ((endTime - startTime) < waitTimeOut * 1000 && !alreadyFind);
if (null == uiNodeData) {
response = CmdAutomationResponse.builderSuccess(request, "未找到滑动的区域").withData(false);
} else {
if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
response = CmdAutomationResponse.builderSuccess(request, "未找到滑动的区域").withData(false);
} else {
int x = uiNodeData.getX();
int y = uiNodeData.getY();
int width = uiNodeData.getWidth();
int height = uiNodeData.getHeight();
List<Map<String, Integer>> pointList = new ArrayList<>();
logger.info("手势区域，位置=>x:{},y:{},宽：{}，高：{}",x,y,width,height);
int perWidth = width /3;
int perHeight = height /3;
int pointY = y - perHeight/2;
int n = 1;
for (int i = 0; i < 3; i++) {
pointY = pointY + perHeight; //每一行的y坐标
int pointX = x - perWidth/2;
for (int j = 0; j < 3; j++) {
pointX = pointX + perWidth;
logger.info("第{}个点的坐标x:{},y:{}",n,pointX,pointY);
Map<String, Integer> point = new HashMap<>();
point.put("x", pointX);
point.put("y", pointY);
pointList.add(n-1,point);
n++;
}
}
logger.info("token:{},滑动解锁的所有坐标集合:{}",request.getStepToken(),JSON.toJSONString(pointList));
List<Integer> passPointNum = new ArrayList<>();
if (!CollectionUtils.isEmpty(passPoint)) {
for (int i=0;i<passPoint.size();i++) {
Integer index = passPoint.get(i);
Map<String, Integer> point = pointList.get(index - 1);
passPointNum.add(point.get("x"));
passPointNum.add(point.get("y"));
}
logger.info("token:{},滑动解锁通过的坐标点:{}",request.getStepToken(),JSON.toJSONString(passPointNum));
success = nkAgent.drawLine(JSON.toJSONString(passPointNum));
} else {
logger.warn("token:{},没有滑动通过的坐标",request.getStepToken());
}
logger.info("token:{},滑动解锁的结果:{}",request.getStepToken(),success);
}
}
if (success) {
response = CmdAutomationResponse.builderSuccess(request, "滑动解锁成功").withData(success);
} else {
response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(success);
}
} finally {
sendResultToEngine(response);
}
}
private String getValueByOcr(UiNodeData uiNodeData) {
String value = null;
String imgBase64 = getOcrAreaByBodeToBase64(uiNodeData);

16
cctp-atu/atu-upper-computer/src/main/java/net/northking/cctp/upperComputer/driver/ios/NKAgent.java
View File

@ -1,10 +1,7 @@
package net.northking.cctp.upperComputer.driver.ios;
import net.northking.cctp.upperComputer.driver.ios.command.data.AppBatteryData;
import net.northking.cctp.upperComputer.driver.ios.command.data.ScreenInfoData;
import net.northking.cctp.upperComputer.driver.ios.command.data.SearchUiNodeData;
import net.northking.cctp.upperComputer.driver.ios.command.data.UiNodeData;
import net.northking.cctp.upperComputer.driver.ios.command.data.*;
import net.northking.cctp.upperComputer.driver.ios.command.listener.AppBatteryDataListener;
import net.northking.cctp.upperComputer.driver.ios.command.listener.ScreenInfoDataListener;
import net.northking.cctp.upperComputer.driver.ios.packet.*;
@ -214,6 +211,17 @@ public final class NKAgent {
return bool.isSuccess();
}
public boolean drawLine(String string) {
TextData data = new TextData();
data.setText(string);
ICommandPacket packet = packetTransfer.syncSend(29, data);
if (packet == null) return false;
BooleanCommandData bool = new BooleanCommandData();
packet.fillICommandData(bool);
return bool.isSuccess();
}
public boolean getStatus() {
return this.packetTransfer.isConnected();
}

7
cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/constants/UpperParamKey.java
View File

@ -76,7 +76,7 @@ public interface UpperParamKey {
/**
* 超时时间
*/
String WAIT_TIMEOUT = "waitTimeout";
String WAIT_TIMEOUT = "wait_time_out";
/**
* 识别类型
@ -88,4 +88,9 @@ public interface UpperParamKey {
*/
String TENANT_ID = "tenant_id";
/**
* 滑动解锁的坐标
*/
String SWIPE_POINT = "swipe_point";
}

15
cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/entity/ElementHandleParam.java
View File

@ -42,6 +42,13 @@ public class ElementHandleParam {
private String type; //应用操作类型
private List<Integer> passingPoints; //滑动经过的点
public ElementHandleParam withPassingPoints(List<Integer> passingPoints) {
this.setPassingPoints(passingPoints);
return this;
}
public ElementHandleParam withType(String type) {
this.setType(type);
return this;
@ -266,4 +273,12 @@ public class ElementHandleParam {
public void setInputTextValue(String inputTextValue) {
this.inputTextValue = inputTextValue;
}
public void setPassingPoints(List<Integer> passingPoints) {
this.passingPoints = passingPoints;
}
public List<Integer> getPassingPoints() {
return passingPoints;
}
}

44
cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/keywords/IOSNewTools.java
View File

@ -937,4 +937,48 @@ public class IOSNewTools {
return result;
}
/**
* <p>滑动解锁（ios新）</p>
*
* @param deviceDriver 设备连接驱动
* @param targets 滑动解锁的区域
* @param waitTime 默认超时时间
* @param passingPoints 经过的点，填[1,3,5,6]数组的形式
* @param preExecuteWait 执行前等待
* @param sufExecuteWait 执行后等待
* @return
*/
@Keyword(alias = "滑动解锁（ios新）", category = "0", attributes = "5",commonlyUse = true)
@Return(name = "result", comment = "是否滑动成功", type = DataType.BOOLEAN)
public Boolean pressAndSwipe(
IExecuteContext context,
@Argument(name = "__deviceDriver", scope = ParamScope.CONTEXT, comment = "设备连接", type = DataType.OBJECT) DeviceDriver deviceDriver,
@Argument(name = "targets", scope = ParamScope.STEP, comment = "控件位置", type = DataType.LIST, defaultDisplay = false) List<IStepTarget> targets,
@Argument(name = "passingPoints", scope = ParamScope.ARGS, comment = "经过的点", type = DataType.LIST, defaultDisplay = true) List<Integer> passingPoints,
@Argument(name = "waitTime", scope = ParamScope.ARGS, comment = "等待时间(单位s)", type = DataType.INTEGER, defaultValue = "30") Integer waitTime,
@Argument(name = "preExecuteWait", scope = ParamScope.ARGS, comment = "执行前等待(单位:s)", type = DataType.INTEGER, required = false, defaultValue = "1.0", defaultDisplay = false) Float preExecuteWait,
@Argument(name = "sufExecuteWait", scope = ParamScope.ARGS, comment = "执行后等待(单位:s)", type = DataType.INTEGER, required = false, defaultValue = "1.0", defaultDisplay = false) Float sufExecuteWait
) {
CommonUtils.handlePreExecuteWait(preExecuteWait); //执行前等待
ElementHandleParam param = ElementHandleParam.builder(deviceDriver)
.useContext(context)
.waitTimeout(waitTime)
.withTargets(targets)
.withPassingPoints(passingPoints);
Boolean result = false;
try {
result = AutomationHandleUtil.pressAndSwipe(param);
} catch (InterruptedException ie) {
logger.warn("用户取消查询元素");
throw new ExecuteException("取消操作");
} catch (Exception e) {
logger.error("出现了其他的问题。。。。", e);
throw new ExecuteException(e.getMessage());
}
CommonUtils.handleSufExecuteWait(sufExecuteWait); //执行后等待
return result;
}
}

32
cctp-test-element/cctp-test-element-library/cctp-test-element-mobile-ios/src/main/java/net/northking/cctp/element/mobile/ios/utils/AutomationHandleUtil.java
View File

@ -1080,4 +1080,36 @@ public class AutomationHandleUtil {
}
return result;
}
public static Boolean pressAndSwipe(ElementHandleParam param) throws Exception{
boolean success = false;
String token = UUID.randomUUID().toString();
String targetValue = null;
for (IStepTarget target : param.getTargets()) {
if (UsingType.SELECTOR.equalsIgnoreCase(target.getUsing())) {
targetValue = target.getValue();
}
}
if (StringUtils.isBlank(targetValue)) {
throw new ExecuteException("无法定位到滑动解锁的区域");
}
Map<String, Object> paramMap = new HashMap<>();
JSONObject object = JSON.parseObject(targetValue, JSONObject.class);
String xpath = object.getString(UsingType.Key.SELECTOR_KEY);
logger.info("拿到的xpath:{}", xpath);
paramMap.put(UpperParamKey.NODE_TREE, xpath);
paramMap.put(UpperParamKey.SWIPE_POINT, param.getPassingPoints());
paramMap.put(UpperParamKey.WAIT_TIMEOUT, param.getWaitTimeout());
logger.info("参数：{}", JSON.toJSONString(paramMap));
CmdAutomationRequest builder = CmdAutomationRequest.builder(AutomationRequestCmd.PRESS_AND_SWIPE, token, paramMap);
Object o = sendUpperMessageAndWaitReturn(param.getDeviceDriver(), builder, token, param.getWaitTimeout());
logger.debug("滑动解锁的结果：{}",o);
if (null != o) {
success = (boolean) o;
}
if (!success) {
throw new ExecuteException("滑动解锁失败");
}
return success;
}
}