package net.northking.cctp.upperComputer.automation.handler;

import cn.hutool.core.codec.Base64;
import cn.hutool.extra.spring.SpringUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import net.northking.cctp.upperComputer.automation.constants.Command;
import net.northking.cctp.upperComputer.automation.constants.UpperParamKey;
import net.northking.cctp.upperComputer.automation.entity.CmdAutomationRequest;
import net.northking.cctp.upperComputer.automation.entity.CmdAutomationResponse;
import net.northking.cctp.upperComputer.automation.entity.ScreenInfo;
import net.northking.cctp.upperComputer.constants.HandCommand;
import net.northking.cctp.upperComputer.deviceManager.IOSDeviceManager;
import net.northking.cctp.upperComputer.deviceManager.UpperComputerManager;
import net.northking.cctp.upperComputer.deviceManager.thread.IosDeviceInitThread;
import net.northking.cctp.upperComputer.driver.ios.NKAgent;
import net.northking.cctp.upperComputer.driver.ios.command.data.*;
import net.northking.cctp.upperComputer.driver.ios.packet.EmptyCommandData;
import net.northking.cctp.upperComputer.entity.Attachment;
import net.northking.cctp.upperComputer.entity.PhoneEntity;
import net.northking.cctp.upperComputer.enums.FileBusinessTypeEnum;
import net.northking.cctp.upperComputer.exception.ExecuteException;
import net.northking.cctp.upperComputer.service.IosDebuggerServiceImpl;
import net.northking.cctp.upperComputer.utils.HttpUtils;
import net.northking.cctp.upperComputer.utils.SpringUtils;
import net.northking.cctp.upperComputer.utils.deviceHepler.ios.IosDeviceHandleHelper;
import net.northking.cctp.upperComputer.utils.deviceHepler.ios.LinuxAndWindowsIosHandleHelper;
import net.northking.cctp.upperComputer.utils.deviceHepler.ios.MacIosHandleHelper;
import net.northking.cctp.upperComputer.utils.hzBank.HzBankOcrUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import sun.misc.BASE64Decoder;

import javax.imageio.ImageIO;
import javax.websocket.Session;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.*;

/**
 * @author : yineng.huang
 * @date : 2024/4/16 19:59
 */
public class IosAutomationHandler extends AbstractAutomationHandler {

    private final Logger logger = LoggerFactory.getLogger(IosAutomationHandler.class);

    private PhoneEntity phoneEntity;

    public IosAutomationHandler(PhoneEntity phoneEntity, Session session) {
        super();
        this.phoneEntity = phoneEntity;
        this.serial = phoneEntity.getUdid();
        this.engineSession = session;
        boolean isMac = UpperComputerManager.getInstance().getOperatingSystem().startsWith("Mac");
        if (isMac) {
            deviceHandleHelper = new MacIosHandleHelper();
        } else {
            deviceHandleHelper = new LinuxAndWindowsIosHandleHelper();
        }
    }

    // 密码输入（ocr）相关参数
    private Map<String, Map<String, JSONObject>> totalMap = new HashMap<>();
    private List<Map<String, JSONObject>> list;
    private int index = 1;
    private String lastKeyBoard;
    private String preFlag = "0";    //0-小写字母，1-数字，2-符号,3-大写字母

    // 元素查找间隔时间
    private Long findSleepTime = SpringUtils.getProperties("nk.mobile-computer.findSleepTime") != null ?
            Long.parseLong(SpringUtils.getProperties("nk.mobile-computer.findSleepTime")) : 2000L;

    @Override
    public void destroy() {

    }

    @Override
    public void openMobileNotificationBar(CmdAutomationRequest request) {

    }

    @Override
    public void closeMobileNotificationBar(CmdAutomationRequest request) {

    }

    @Override
    public void initApplication(CmdAutomationRequest request) {
        CmdAutomationResponse response = null;
        Map<String, String> data = new HashMap<>();
        try {
            Map<String, Object> initData = request.getData();
            boolean reInstall = false;
            if (null != initData.get("forceReInstall")) {
                reInstall = (boolean) initData.get("forceReInstall");
            }
            boolean coverInstall = false;
            if (null != initData.get("coverInstall")) {
                coverInstall = (boolean) initData.get("coverInstall");
            }
            logger.debug("覆盖安装：{}",coverInstall);
            String appId = (String) initData.get("appId");
            Map<String, Object> appDetail = null;
            try {
                appDetail = getAppDetail(appId);
            } catch (Exception e) {
                logger.error("查询app信息出错，原因：", e);
                response = CmdAutomationResponse.builderFailure(request, "查询app信息出错");
                return;
            }
            if (null == appDetail) {
                response = CmdAutomationResponse.builderFailure(request, "app应用已被删除，无法初始化");
                return;
            }
            String appUrl = (String) appDetail.get("appUrl");
            String appPackage = (String) appDetail.get("packageName");
            String appVersion = (String) appDetail.get("buildVersion");
            String appName = appUrl.substring(appUrl.lastIndexOf("_") + 1);
            boolean appInstalled = deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage);
            logger.debug("手机[{}]上app[{}]安装的结果：{}",serial,appPackage,appInstalled);
            boolean needInstall = !appInstalled;
            if (reInstall) {  //选择了卸载安装
                if (appInstalled) {
                    logger.info("开始从设备【{}】卸载 app[{}] ...", phoneEntity.getUdid(), appPackage);
                    deviceHandleHelper.removeApp(phoneEntity.getUdid(), appPackage);
                    logger.info("从设备【{}】上卸载App[{}] 完成", phoneEntity.getUdid(), appPackage);
                } else {
                    logger.info("设备【{}】上App[{}]未安装，无需卸载", phoneEntity.getUdid(), appPackage);
                }
                needInstall = true;
            }else if (coverInstall) { //覆盖安装
                if (appInstalled) {
                    String oldPackageCode = deviceHandleHelper.getOldPackageCode(phoneEntity.getUdid(), appPackage);
                    logger.debug("选择了覆盖安装，设备【{}】上app【{}】的版本是：{}", serial, appPackage, oldPackageCode);
                } else {
                    logger.info("设备【{}】上App[{}]未安装", phoneEntity.getUdid(), appPackage);
                }
                needInstall = true;
            }
            if(needInstall) {
                boolean installSuccess = installApp(appId, appName);
                if (!installSuccess) {
                    response = CmdAutomationResponse.builderFailure(request, "app安装失败").withData(data);
                    return;
                }
            }
            if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                response = CmdAutomationResponse.builderFailure(request, "app安装失败");
                return;
            }
            logger.info("activate app[{}]", appPackage);
            boolean success = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
            if (!success) {
                response = CmdAutomationResponse.builderFailure(request, "应用启动失败");
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "app初始化成功");
            }
        } catch (Exception e) {
            logger.error("设备【{}】应用初始化失败", phoneEntity.getUdid(), e);
            CmdAutomationResponse.builderFailure(request, "应用初始化失败");
        } finally {
            sendResultToEngine(response);
        }
        logger.info("自动化执行环境【device:{}】准备就绪", phoneEntity.getUdid());
    }


    @Override
    public void clickPoint(CmdAutomationRequest request) {

    }

    @Override
    public void searchNodeByNodeTree(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (null == getNkAgent()) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            String type = (String) data.get(UpperParamKey.NODE_TYPE);
            Integer searchFrequency = (Integer) data.get(UpperParamKey.TIMES);
            logger.debug("拿到的nodeTree:{}, type:{}", nodeTree, type);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            if (StringUtils.isNotBlank(type)) {
                searchUiNodeData.setType(type);
            }
            UiNodeData uiNodeData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            int times = 1;
            while (elapsedTime < waitTimeout * 1000) {
                uiNodeData = getNkAgent().uiNodeInfo(searchUiNodeData);
//                logger.info("拿到的uiNodeData：{}", uiNodeData);
                if (uiNodeData != null && !(uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0)) {
                    break;
                } else {
                    try {
                        if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                            Thread.sleep(findSleepTime);
                        }
                    } catch (InterruptedException e) {
                        logger.warn("查找控件失败", e);
                    }
                }
                if (searchFrequency != null && times >= searchFrequency) {
                    break;
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
                times++;
            }
            logger.info("元素查找找了{}次", times);
            if (null == uiNodeData) {
                UiNodeData uiNodeDataNotFound = getNkAgent().uiTreeNode();
                logger.error("元素查找未找到元素：{}，元素树为：{}", nodeTree, uiNodeDataNotFound);
                response = CmdAutomationResponse.builderFailure(request, "元素不存在");
            } else {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    UiNodeData uiNodeDataNotFound = getNkAgent().uiTreeNode();
                    logger.error("元素查找未找到元素：{}，元素树为：{}", nodeTree, uiNodeDataNotFound);
                    response = CmdAutomationResponse.builderFailure(request, "元素不存在");
                } else {
                    TapXYData tapXYData = new TapXYData();
                    //转换比例
                    int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                    int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                    int realityWidth = width;
                    int realityHeight = height;
                    if (width > height) {  //宽高反了的情况，调换
                        realityWidth = height;
                        realityHeight = width;
                    }
                    Double realityCutWidth = (double) uiNodeData.getWidth() / phoneEntity.getScreenWidth() * realityWidth;
                    Double realityCutHeight = (double) uiNodeData.getHeight() / phoneEntity.getScreenHeight() * realityHeight;
                    Double realityX = (double) uiNodeData.getX() / phoneEntity.getScreenWidth() * realityWidth;
                    Double realityY = (double) uiNodeData.getY() / phoneEntity.getScreenHeight() * realityHeight;
                    tapXYData.setX((int)(realityX + realityCutWidth / 2));
                    tapXYData.setY((int)(realityY + realityCutHeight / 2));
                    tapXYData.setXOriginal(uiNodeData.getX() + uiNodeData.getWidth() / 2);
                    tapXYData.setYOriginal(uiNodeData.getY() + uiNodeData.getHeight() / 2);
                    tapXYData.setHeightOriginal(uiNodeData.getHeight());
                    tapXYData.setWidthOriginal(uiNodeData.getWidth());
                    response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
                }
            }
        } catch (Exception e) {
            logger.error("根据xpath判断元素是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally{
            logger.info("根据nodetree查找元素的response：{}", response);
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByPoint(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            TapXYData tapXYData = null;
            tapXYData = queryPoint(request);
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "查找控件失败").withData(false);
            } else {
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = (double) tapXYData.getX() / phoneEntity.getScreenWidth() * realityWidth;
                Double realityY = (double) tapXYData.getY() / phoneEntity.getScreenHeight() * realityHeight;
                tapXYData.setX(realityX.intValue());
                tapXYData.setY(realityY.intValue());
                tapXYData.setXOriginal(tapXYData.getX() / phoneEntity.getScale());
                tapXYData.setYOriginal(tapXYData.getY() / phoneEntity.getScale());
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            logger.error("根据坐标是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            Integer searchFrequency = (Integer) data.get(UpperParamKey.TIMES);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                long startTime = System.currentTimeMillis(); //当前时间
                long elapsedTime = 0; // 初始化经过的时间
                int times = 1;
                while (elapsedTime < waitTimeout * 1000) {
                    tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryOcrArea(request);
                    if (tapXYData != null || (searchFrequency != null && times >= searchFrequency)) {
                        break;
                    }
                    long currentTime = System.currentTimeMillis();
                    elapsedTime = currentTime - startTime;
                    if (elapsedTime >= waitTimeout * 1000) {
                        break;
                    }
                    times++;
                }
                logger.info("ocr找了{}次", times);
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
            } else {
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = (double) tapXYData.getX() / phoneEntity.getScreenWidth() * realityWidth;
                Double realityY = (double) tapXYData.getY() / phoneEntity.getScreenHeight() * realityHeight;
                tapXYData.setX(realityX.intValue());
                tapXYData.setY(realityY.intValue());
                tapXYData.setXOriginal(tapXYData.getX() / phoneEntity.getScale());
                tapXYData.setYOriginal(tapXYData.getY() / phoneEntity.getScale());
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByImage(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            Integer searchFrequency = (Integer) data.get(UpperParamKey.TIMES);
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            int times = 1;
            while (elapsedTime < waitTimeout * 1000) {
                tapXYData = queryImageArea(request);
                if (tapXYData != null || (searchFrequency != null && times >= searchFrequency)) {
                    break;
                } else {
                    try {
                        if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                            Thread.sleep(findSleepTime);
                        }
                    } catch (InterruptedException e) {
                        logger.warn("查找控件失败", e);
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
                if (elapsedTime >= waitTimeout * 1000) {
                    break;
                }
                times++;
            }
            logger.info("image找了{}次", times);
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
            } else {
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = (double) tapXYData.getX() / phoneEntity.getScreenWidth() * realityWidth;
                Double realityY = (double) tapXYData.getY() / phoneEntity.getScreenHeight() * realityHeight;
                tapXYData.setX(realityX.intValue());
                tapXYData.setY(realityY.intValue());
                tapXYData.setXOriginal(tapXYData.getX() / phoneEntity.getScale());
                tapXYData.setYOriginal(tapXYData.getY() / phoneEntity.getScale());
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            logger.error("根据图像是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementValueByNode(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (null == getNkAgent()) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            String type = (String) data.get(UpperParamKey.NODE_TYPE);
            logger.debug("拿到的nodeTree:{}, type:{}", nodeTree, type);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            if (StringUtils.isNotBlank(type)) {
                searchUiNodeData.setType(type);
            }
            UiNodeData uiNodeData = null;
            String value = null;

            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000) {
                uiNodeData = getNkAgent().uiNodeInfo(searchUiNodeData);
                if (uiNodeData != null && !(uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0)) {
                    value = uiNodeData.getLabel();
                    if (StringUtils.isBlank(value)) {
                        value = uiNodeData.getValue();
                    }
                    if (StringUtils.isBlank(value)) {
                        value = uiNodeData.getName();
                    }
                    if (StringUtils.isNotBlank(value)) {
                        break;
                    } else {
                        try {
                            if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                                Thread.sleep(findSleepTime);
                            }
                        } catch (InterruptedException e) {
                            logger.warn("查找控件失败",e);
                        }
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }

            if (null != uiNodeData) {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    response = CmdAutomationResponse.builderFailure(request, "找不到控件");
                    return;
                }
                if (StringUtils.isNotBlank(value)) {
                    response = CmdAutomationResponse.builderSuccess(request, "已获取控件的值").withData(value);
                }
            } else {
                response = CmdAutomationResponse.builderFailure(request, "找不到控件");
            }
        } catch (Exception e) {
            logger.error("根据xpath获取控件的值失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementValueByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
        try {
            String value = null;
            Map<String, Object> requestData = request.getData();
            String targetBase64 = "";
            if (requestData.get(UpperParamKey.X) != null) {
                targetBase64 = getOcrAreaToBase64(request);
            } else {
                targetBase64 = getClickTextOcrAreaToBase64(request);
            }
            if (StringUtils.isBlank(targetBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                return;
            }
            // 是否使用杭银ocr
            Map<String, Object> dataOcr = request.getData();
            String usingType = (String) dataOcr.get(UpperParamKey.USING_TYPE);
            if (usingType.equals("hzocr")) {
                String ocrText = (String) dataOcr.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(targetBase64);
                    value = HzBankOcrUtils.getAllTextByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                // 是否使用新版ocr
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                if (useNewOcr) {
                    String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
                    HttpHeaders headers = new HttpHeaders();
                    headers.setContentType(MediaType.APPLICATION_JSON);
                    Map<String, Object> ocrParamMap = new HashMap<>();
                    ocrParamMap.put("img_base64", targetBase64);
                    ocrParamMap.put("targets", "");
                    HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
                    List ocrResultList = null;
                    try {
                        logger.info("传参：{}", JSON.toJSONString(ocrParamMap));
                        ocrResultList = HttpUtils.doPost(ocrAddress, ocrEntity, List.class);
                        logger.info("得到的ocr结果：{}", ocrResultList);
                    } catch (Exception e) {
                        logger.error("ocr失败", e);
                    }
//            List<String> dataList = new ArrayList<>();
                    StringJoiner dataList = new StringJoiner("");
                    if (!CollectionUtils.isEmpty(ocrResultList)) {
                        if (!CollectionUtils.isEmpty(ocrResultList)) {
                            for (Object o : ocrResultList) {
                                dataList.add(((Map) o).get("text") + "");
                            }
                        }
                    }
                    value =  dataList.toString();
                } else {
                    String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetText");
                    try {
                        HttpHeaders headers = new HttpHeaders();
                        headers.setContentType(MediaType.APPLICATION_JSON);
                        HttpEntity<String> ocrEntity = new HttpEntity<>("\"" + targetBase64 + "\"", headers);
                        JSONObject object = HttpUtils.doPost(new URI(ocrAddress), ocrEntity, JSONObject.class);
                        JSONObject data = object.getJSONObject("data");
                        if (data != null) {
                            value = data.getString("ocrText");
                        }
                    } catch (URISyntaxException e) {
                        logger.error("ocr地址有误！===>{}", ocrAddress, e);
                    } catch (Exception e) {
                        logger.error("请求ocr接口失败，接口地址：{}", ocrAddress, e);
                    }
                }
            }

            logger.info("识别的图片文字为：{}", value);
            if (StringUtils.isNotBlank(value)) {
                response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
            }
        } catch (Exception e) {
            logger.error("根据ocr获取控件的值失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }





    @Override
    public void getVerificationCodeByNode(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "识别验证码失败");
        Map<String, Object> data = request.getData();
        NKAgent nkAgent = getNkAgent();
        try {
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            String type = (String) data.get(UpperParamKey.NODE_TYPE);
            Integer codeType = (Integer) data.get(UpperParamKey.CODE_TYPE);
            Integer commandId = (Integer) data.get(UpperParamKey.COMMAND_ID);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            if (StringUtils.isNotBlank(type)) {
                searchUiNodeData.setType(type);
            }
            UiNodeData uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData);
            if (null == uiNodeData) {
                response = CmdAutomationResponse.builderFailure(request, "元素不存在");
            } else {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    response = CmdAutomationResponse.builderFailure(request, "元素不存在");
                } else {
                    int x = uiNodeData.getX() * phoneEntity.getScale();
                    int y = uiNodeData.getY() * phoneEntity.getScale();
                    int width = uiNodeData.getWidth() * phoneEntity.getScale();
                    int height = uiNodeData.getHeight() * phoneEntity.getScale();
                    File screenShotFile = deviceHandleHelper.getScreenShotFile(phoneEntity.getUdid(), x, y, width, height);
                    String targetBase64 = getFileBase64(screenShotFile);
                    if (StringUtils.isBlank(targetBase64)) {
                        response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                        return;
                    }
                    String value = getVerificationCodeByBase64(targetBase64, codeType);
                    if (StringUtils.isNotBlank(value)) {
                        response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
                    } else {
                        response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
                    }
                }
            }
        } catch (Exception e) {
            logger.error("识别验证码失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getVerificationCodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr获取验证码失败");
        try {
            String value = null;
            Map<String, Object> requestData = request.getData();
            String targetBase64 = "";
            if (requestData.get(UpperParamKey.X) != null) {
                targetBase64 = getOcrAreaToBase64(request);
            } else {
                targetBase64 = getClickTextOcrAreaToBase64(request);
            }
            if (StringUtils.isBlank(targetBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                return;
            }
            Map<String, Object> data = request.getData();
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            Integer codeType = (Integer) data.get(UpperParamKey.CODE_TYPE);
            if ("hzocr".equals(usingType)) {
                String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(targetBase64);
                    value = HzBankOcrUtils.getAllTextListByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                value = getVerificationCodeByBase64(targetBase64,codeType);
            }
            if (StringUtils.isNotBlank(value)) {
                response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
            }
        } catch (Exception e) {
            logger.error("根据ocr获取验证码失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private String getVerificationCodeByBase64(String base64, Integer codeType){
        String value = null;
        String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetVerificationCode");
        try {
            HttpHeaders headers = new HttpHeaders();
            headers.setContentType(MediaType.APPLICATION_JSON);
            Map<String, Object> ocrParamMap = new HashMap<>();
            ocrParamMap.put("img_base64", base64);
            ocrParamMap.put("type", codeType);
            HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
            value = HttpUtils.doPost(new URI(ocrAddress), ocrEntity, String.class);
        }catch (URISyntaxException e) {
            logger.error("ocr地址有误！===>{}",ocrAddress,e);
        } catch (Exception e) {
            logger.error("请求ocr接口失败，接口地址：{}",ocrAddress,e);
        }
        logger.info("识别到的验证码为：{}", value);
        return value == null ? value : value.replace("\"", "");
    }

    @Override
    public void longPress(CmdAutomationRequest request) {

    }

    @Override
    public void swipe(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "滑动失败");
        try {
            Map<String, Object> data = request.getData();
            String direction = (String) data.get(UpperParamKey.SWIPE_DIRECTION);
            Integer size = (Integer) data.get(UpperParamKey.SWIPE_SIZE);
            int width = phoneEntity.getScreenWidth();
            int height = phoneEntity.getScreenHeight();
            int x1, y1, x2, y2;
            if ("up".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height * 8 / 10;
                if (null == size || size <= 0) {
                    y2 = y1 - height / 2;
                } else {
                    y2 = y1 - size;
                }
            } else if ("down".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height * 2 / 10;
                if (null == size || size <= 0) {
                    y2 = y1 + height / 2;
                } else {
                    y2 = y1 + size;
                }
            } else if ("left".equalsIgnoreCase(direction)) {
                y1 = y2 = height / 2;
                x1 = width * 9 / 10;
                if (null == size || size <= 0) {
                    x2 = x1 - width / 2;
                } else {
                    x2 = x1 - size;
                }
            } else if ("right".equalsIgnoreCase(direction)) {
                y1 = y2 = height / 2;
                x1 = width * 1 / 10;
                if (null == size || size <= 0) {
                    x2 = x1 + width / 2;
                } else {
                    x2 = x1 + size;
                }
            } else {
                response = CmdAutomationResponse.builderFailure(request, "只支持上下左右滑动");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            DragXYData dragXYData = new DragXYData();
            dragXYData.setX(x1);
            dragXYData.setToX(x2);
            dragXYData.setY(y1);
            dragXYData.setToY(y2);
            dragXYData.setDuration(0);
            logger.debug("滑动dragXYData对象：{}", JSON.toJSONString(dragXYData));
            boolean result = nkAgent.handleTextUiNode(dragXYData, 24);
            if (result) {
                response = CmdAutomationResponse.builderSuccess(request, "滑动成功").withData(result);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "滑动失败");
            }
        } catch (Exception e) {
            logger.error("滑动失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void inputText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "输入失败").withData(false);
        Map<String, Object> data = request.getData();
        NKAgent nkAgent = getNkAgent();
        try {
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            Integer x = (Integer) data.get(UpperParamKey.X);
            Integer y = (Integer) data.get(UpperParamKey.Y);
            TapXYData tapXYData = new TapXYData();
            tapXYData.setX(x);
            tapXYData.setY(y);
            boolean clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (!clickSuccess) {
                throw new ExecuteException("未找到控件");
            }
            // 等待键盘弹出
            Thread.sleep(1000);
            boolean clear = (boolean) data.get(UpperParamKey.CLEAR);
            if (clear) {
                String xpath = (String) data.get(UpperParamKey.NODE_TREE);
                logger.debug("步骤【{}】输入前清空数据",request.getStepToken());
                boolean clearSuccess = nkAgent.handleDeleteTextUiNode(Command.CLEAR.getCode());
                logger.debug("步骤【{}】输入前清空数据结果：{}",request.getStepToken(),clearSuccess);
            }
            String inputText = (String) data.get(UpperParamKey.INPUT_TEXT);
            Integer commandId = (Integer) data.get(UpperParamKey.COMMAND_ID);
            logger.debug("步骤【{}】即将输入文本：{},commandId:{}",request.getStepToken(),inputText,commandId);
            TypeKeysUiNodeData typeKeysUiNodeData = new TypeKeysUiNodeData();
            typeKeysUiNodeData.setText(inputText);
            boolean success = nkAgent.handleTextUiNode(typeKeysUiNodeData, 18);
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "输入成功").withData(success);
            }else {
                response = CmdAutomationResponse.builderFailure(request, "输入失败，请检查输入时键盘是否弹出").withData(success);
            }
        } catch (Exception e) {
            logger.error("输入失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleNodeByNodeTree(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据node节点点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (null == getNkAgent()) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            String type = (String) data.get(UpperParamKey.NODE_TYPE);
            Integer commandId = (Integer) data.get(UpperParamKey.COMMAND_ID);
            logger.debug("拿到的nodeTree:{}, type:{}", nodeTree, type);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            if (StringUtils.isNotBlank(type)) {
                searchUiNodeData.setType(type);
            }
            boolean success = false;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000) {
                success = getNkAgent().clickUiNode(searchUiNodeData, 5);
                if (success) {
                    break;
                } else {
                    try {
                        if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                            Thread.sleep(findSleepTime);
                        }
                    } catch (InterruptedException e) {
                        logger.warn("查找控件失败",e);
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(success);
            }else {
                response = CmdAutomationResponse.builderFailure(request, "控件不存在").withData(success);
            }
        } catch (Exception e) {
            logger.error("根据node节点点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleNodeByPoint(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据坐标点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            TapXYData tapXYData = null;
            boolean clickSuccess = false;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000) {
                tapXYData = queryPoint(request);
                if (tapXYData != null) {
                    clickSuccess = getNkAgent().handleTextUiNode(tapXYData, 21);
                    if (clickSuccess) {
                        break;
                    } else {
                        try {
                            if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                                Thread.sleep(findSleepTime);
                            }
                        } catch (InterruptedException e) {
                            logger.warn("根据坐标点击控件失败",e);
                        }
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }

            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }

            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据坐标点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleNodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                long startTime = System.currentTimeMillis(); //当前时间
                long elapsedTime = 0; // 初始化经过的时间
                while (elapsedTime < waitTimeout * 1000) {
                    tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryOcrArea(request);;
                    if (tapXYData != null) {
                        break;
                    }
                    long currentTime = System.currentTimeMillis();
                    elapsedTime = currentTime - startTime;
                }

            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = getNkAgent().handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private TapXYData queryClickTextOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            Boolean success = ocrResultMap.getBoolean("success");
            if (success) {
                JSONArray pointData = ocrResultMap.getJSONArray("data");
                Integer index = data.get(UpperParamKey.INDEX) != null ? (Integer) data.get(UpperParamKey.INDEX) : 1;
                JSONObject pointMap = null;
                if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                    pointMap = pointData.getJSONObject(0);
                } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                    if (index > 0) {
                        if (index <= pointData.size()) {
                            pointMap = pointData.getJSONObject(index - 1);
                        } else {
                            logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                            throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", pointData.size()));
                        }
                    }
                }
                if (null != pointMap) {
                    result = new TapXYData();
                    Double scale = Double.valueOf(phoneEntity.getScale());
                    result.setX((int) (pointMap.getDouble("x").intValue() / scale));
                    result.setY((int) (pointMap.getDouble("y").intValue() / scale));
                }
            }
        }
        return result;
    }


    private boolean queryIfTextOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            boolean success = ocrResultMap.getBoolean("success");
            return success;
        } else {
            return false;
        }
    }

    private boolean queryIfTextOcrAreaNew(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultList) {
            return true;
        } else {
            return false;
        }
    }

    private TapXYData queryOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        Double realityX = 0.0;
        Double realityY = 0.0;
        if(data.get(UpperParamKey.X) != null) {
            Integer x = (Integer) data.get(UpperParamKey.X);
            Integer y = (Integer) data.get(UpperParamKey.Y);
            Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
            Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
            //转换比例
            int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
            int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
            int realityWidth = width;
            int realityHeight = height;
            if (width > height) {  //宽高反了的情况，调换
                realityWidth = height;
                realityHeight = width;
            }
            realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
            realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;

        }
        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (!CollectionUtils.isEmpty(ocrResultList)) {
            Object all = ocrResultList.get(0);
            JSONObject jsonObject = JSONObject.parseObject(JSONArray.toJSONString(all));
            //todo：如果传了;是多个文本一起找，暂时只考虑一个文本
            JSONArray pointData = jsonObject.getJSONArray(ocrText);   //所有坐标
            if (pointData == null || pointData.size() == 0) {
                return result;
            } else {
                JSONObject  pointMap = pointData.getJSONObject(0);
                logger.info("只找到一个，直接返回结果：{}", pointMap);
                result = new TapXYData();
                result.setX((realityX.intValue() + pointMap.getInteger("x") + pointMap.getInteger("w") / 2) / phoneEntity.getScale());
                result.setY((realityY.intValue() + pointMap.getInteger("y") + pointMap.getInteger("h") / 2) / phoneEntity.getScale());
                return result;
            }
        } else {
            return result;
        }
    }

    private TapXYData queryOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = "";
        Map<String, Object> requestData = request.getData();
        if (requestData.get(UpperParamKey.X) != null) {
            base64Str = getOcrAreaToBase64(request);
        } else {
            base64Str = getClickTextOcrAreaToBase64(request);
        }
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            Boolean success = ocrResultMap.getBoolean("success");
            if (success) {
                Integer x = (Integer) data.get(UpperParamKey.X);
                Integer y = (Integer) data.get(UpperParamKey.Y);
                Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
                Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight()* phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
                Double realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;
                JSONArray pointData = ocrResultMap.getJSONArray("data");
                Integer index = data.get(UpperParamKey.INDEX) != null ? (Integer) data.get(UpperParamKey.INDEX) : 1;
                JSONObject pointMap = null;
                if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                    pointMap = pointData.getJSONObject(0);
                } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                    if (index > 0) {
                        if (index <= pointData.size()) {
                            pointMap = pointData.getJSONObject(index - 1);
                        } else {
                            logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                        }
                    }
                }
                if (null != pointMap) {
                    int clickTrueX = new Double((realityX + pointMap.getDouble("x")) / phoneEntity.getScale()).intValue();
                    int clickTrueY = new Double((realityY + pointMap.getDouble("y")) / phoneEntity.getScale()).intValue();
                    result = new TapXYData();
                    result.setX(clickTrueX);
                    result.setY(clickTrueY);
                }
            }
        }
        return result;
    }

    private TapXYData queryClickTextOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str =  "";
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        Integer diffussion =  data.get(UpperParamKey.DIFFUSSION) != null ? (Integer) data.get(UpperParamKey.DIFFUSSION) : null;
        double realityX = 0.0;
        double realityY = 0.0;
        if(data.get(UpperParamKey.X) != null) {
            base64Str =  getOcrAreaToBase64(request);
            Integer x = (Integer) data.get(UpperParamKey.X);
            Integer y = (Integer) data.get(UpperParamKey.Y);
            Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
            Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
            //转换比例
            int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
            int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
            int realityWidth = width;
            int realityHeight = height;
            if (width > height) {  //宽高反了的情况，调换
                realityWidth = height;
                realityHeight = width;
            }
            realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
            realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;
        } else {
            base64Str = getClickTextOcrAreaToBase64(request);
        }
        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        ocrParamMap.put("diffussion", diffussion != null ? diffussion : 1);
        logger.debug("ocr查找文本入参ocrParamMap为：{}", JSON.toJSONString(ocrParamMap));
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (!CollectionUtils.isEmpty(ocrResultList)) {
            Object all = ocrResultList.get(0);
            JSONObject jsonObject = JSONObject.parseObject(JSONArray.toJSONString(all));
            //todo：如果传了;是多个文本一起找，暂时只考虑一个文本
            JSONArray pointData = jsonObject.getJSONArray(ocrText);   //所有坐标
            Integer index = data.get(UpperParamKey.INDEX) != null ? (Integer) data.get(UpperParamKey.INDEX) : 1;
            JSONObject pointMap = null;
            if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                pointMap = pointData.getJSONObject(0);
                logger.info("只找到一个，直接返回结果：{}", pointMap);
                result = new TapXYData();
                result.setX(((int) realityX + pointMap.getInteger("x") + pointMap.getInteger("w") / 2) / phoneEntity.getScale());
                result.setY(((int) realityY + pointMap.getInteger("y") + pointMap.getInteger("h") / 2) / phoneEntity.getScale());
            } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                if (index > 0) {
                    if (index <= pointData.size()) {
//                        result =  sortPointAndGetIndexPoint(pointData, index);
                        JSONObject dataJSONObject = pointData.getJSONObject(index - 1);
                        result = new TapXYData();
                        result.setX((dataJSONObject.getInteger("x") + dataJSONObject.getInteger("w") / 2) / phoneEntity.getScale());
                        result.setY((dataJSONObject.getInteger("y") + dataJSONObject.getInteger("h") / 2) / phoneEntity.getScale());
                    } else {
                        logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                        throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", pointData.size()));
                    }
                }
            }
        }
        return result;
    }

    private TapXYData queryClickTextHzOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        String textByHzOcr;
        List<TapXYData> allTextPointByData = new ArrayList<>();
        try {
            textByHzOcr = HzBankOcrUtils.getTextByHzOcr(base64Str);
            allTextPointByData = HzBankOcrUtils.getAllTextPointByData(textByHzOcr, ocrText);
        } catch (Exception e) {
            logger.error("查找杭银ocr失败，原因为： {}", e);
            throw e;
        }
        if (!CollectionUtils.isEmpty(allTextPointByData)) {
            if (allTextPointByData.size() == 1) {   //只找到一个，直接返回
                result = allTextPointByData.get(0);
                logger.info("只找到一个，直接返回结果：{}", result);
            } else { //找到多个，选择用户选择的第几个
                if (index > 0) {
                    if (index <= allTextPointByData.size()) {
                        result = allTextPointByData.get(index - 1);
                    } else {
                        logger.warn("实际找到{}个，选择第{}个", allTextPointByData.size(), index);
                        throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", allTextPointByData.size()));
                    }
                }
            }
        }
        return result;
    }


    private static TapXYData sortPointAndGetIndexPoint(JSONArray data, Integer index) {
        List<JSONObject> list = new ArrayList<>();
        List<TapXYData> allPoint = new ArrayList<>();
//        data.stream().forEach(item -> System.out.println(JSONObject.toJSONString(item)));
        //y排好
        data.stream().sorted(new Comparator() {
            @Override
            public int compare(Object o1, Object o2) {
                JSONObject one = JSONObject.parseObject(JSONObject.toJSONString(o1));
                JSONObject two = JSONObject.parseObject(JSONObject.toJSONString(o2));
                return one.getIntValue("y") - two.getIntValue("y");
            }
        }).forEach(item -> list.add(JSONObject.parseObject(JSONObject.toJSONString(item))));
        JSONObject[][] plane = new JSONObject[list.size()][list.size()];
        int yIndex = 0;
        int xIndex = 0;
        for (int j = 0; j < list.size()-1; j++) {
            plane[xIndex][yIndex] = list.get(j);
            yIndex++;
            JSONObject pre = list.get(j);
            JSONObject suf = list.get(j+1);
            if (suf.getIntValue("y") - pre.getIntValue("y") > 10) {
                yIndex = 0;
                xIndex++;
            }
        }
        plane[xIndex][yIndex] = list.get(list.size() - 1);  //最后一个
        //对每一行排序
        for (int k = 0; k <= xIndex; k++) {
            List<JSONObject> lineList = new ArrayList<>();
            for (int m = 0; m < list.size(); m++) {
                if (null != plane[k][m]) {
                    lineList.add(plane[k][m]);
                } else {
                    break;
                }
            }
            lineList.stream().sorted(new Comparator<JSONObject>() {
                @Override
                public int compare(JSONObject o1, JSONObject o2) {
                    return o1.getIntValue("x") - o2.getIntValue("x");
                }
            }).forEach(item -> {
                TapXYData pointMessage = new TapXYData();
                pointMessage.setX(item.getInteger("x") + (item.getInteger("w") / 2));
                pointMessage.setY(item.getInteger("y") + (item.getInteger("h") / 2));
                allPoint.add(pointMessage);
            });
        }
        return allPoint.get(index - 1);
    }


    @Override
    public void handleNodeByImage(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据图像点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000) {
                tapXYData = queryImageArea(request);
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
                if (tapXYData != null) {
                    break;
                } else {
                    try {
                        if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                            Thread.sleep(findSleepTime);
                        }
                    } catch (InterruptedException e) {
                        logger.warn("根据图像点击控件失败",e);
                    }
                }
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未找到元素").withData(false);
                return;
            }
            boolean clickSuccess = false;
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据图像点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void clickTextByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                long startTime = System.currentTimeMillis(); //当前时间
                long elapsedTime = 0; // 初始化经过的时间
                while (elapsedTime < waitTimeout * 1000) {
                    tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryClickTextOcrArea(request);
                    if (tapXYData != null) {
                        break;
                    }
                    long currentTime = System.currentTimeMillis();
                    elapsedTime = currentTime - startTime;
                }
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = getNkAgent().handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void clickTextByOcrNew(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                long startTime = System.currentTimeMillis(); //当前时间
                long elapsedTime = 0; // 初始化经过的时间
                while (elapsedTime < waitTimeout * 1000) {
                    tapXYData = queryClickTextOcrAreaNew(request);
                    if (tapXYData != null) {
                        break;
                    }
                    long currentTime = System.currentTimeMillis();
                    elapsedTime = currentTime - startTime;
                }

            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = getNkAgent().handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void ifText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr是否存在文本失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            Integer searchFrequency = (Integer) data.get(UpperParamKey.TIMES);

            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            int times = 1;
            while (elapsedTime < waitTimeout * 1000) {
                tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryClickTextOcrArea(request);
                if (tapXYData != null || (searchFrequency != null && times >= searchFrequency)) {
                    break;
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
                if (elapsedTime >= waitTimeout * 1000) {
                    break;
                }
                times++;
            }
            logger.info("ocr找了{}次", times);
            if (tapXYData == null) {
                // 执行成功，但找不到文本
                response = CmdAutomationResponse.builderSuccess(request, "未查找到文本");
            } else {
                // 转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = (double) tapXYData.getX() / phoneEntity.getScreenWidth() * realityWidth;
                Double realityY = (double) tapXYData.getY() / phoneEntity.getScreenHeight() * realityHeight;
                tapXYData.setX(realityX.intValue());
                tapXYData.setY(realityY.intValue());
                // 执行成功，找到文本
                response = CmdAutomationResponse.builderSuccess(request, "存在文本");
            }
            response.withData(tapXYData);
        } catch (Exception e) {
            // 执行失败
            logger.error("根据ocr是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleApp(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        String appPackage = (String) data.get(UpperParamKey.APP_PACKAGE);
        String type = (String) data.get(UpperParamKey.TYPE);
        String serial = phoneEntity.getUdid();
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "app处理失败").withData(false);
        IosDebuggerServiceImpl iosService = SpringUtils.getBean(IosDebuggerServiceImpl.class);
        try {
            if ("0".equalsIgnoreCase(type)) {
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderFailure(request, "app未安装");
                    return;
                }
                logger.info("关闭ios的app:{}", appPackage);
                boolean success = iosService.terminateApp(serial, appPackage);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app关闭失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app关闭成功").withData(true);
                }
            } else if ("1".equalsIgnoreCase(type)) {
                logger.info("启动ios的app:{}", appPackage);
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderFailure(request, "app未安装");
                    return;
                }
                boolean success = false;
                int count = 0;
                do {
                    success = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
                    if (!success) {
                        logger.info("app启动失败，尝试重试启动：第 {} 次", ++count);
                    }
                } while (!success && count < 5);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app启动失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app启动成功").withData(true);
                }
            } else {
                logger.info("重启ios的app:{}", appPackage);
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderFailure(request, "app未安装");
                    return;
                }
                boolean success = iosService.terminateApp(serial, appPackage);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app关闭失败").withData(false);
                }
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException ie) {
                    logger.error(ie.getMessage());
                }
                boolean successActive = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
                if (!successActive) {
                    response = CmdAutomationResponse.builderFailure(request, "app启动失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app启动成功").withData(true);
                }
            }
        } catch (Exception e) {
            logger.error("处理app失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void initScreenData() {
        this.screenInfo = new ScreenInfo();
        this.screenInfo.setWidth(this.phoneEntity.getScreenWidth() * this.phoneEntity.getScale());
        this.screenInfo.setHeight(this.phoneEntity.getScreenHeight() * this.phoneEntity.getScale());
        this.screenInfo.setScale(this.phoneEntity.getScale());
    }

    @Override
    public void pressHomeKey(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "点击home键失败").withData(false);
        try {
            NKAgent nkAgent = getNkAgent();
            EmptyCommandData emptyCommandData = new EmptyCommandData();
            boolean success = nkAgent.handleTextUiNode(emptyCommandData, HandCommand.PRESS_HOME.getCode());
            logger.debug("设备【{}】点击home成功?------->{}", phoneEntity.getUdid(), success);
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "点击home键成功").withData(true);
            }
        } catch (Exception e) {
            logger.error("点击home键失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }

    }

    @Override
    public void inputPasswordByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "密码输入（ocr）失败").withData(false);
        totalMap = new HashMap<>();
        index = 1;
        lastKeyBoard = "";
        preFlag = "0";
        try {
            Map<String, Object> data = request.getData();
            String password = (String) data.get(UpperParamKey.INPUT_TEXT);
            String ocrArea = getClickTextOcrAreaToBase64(request);
            if (null == ocrArea) {
                throw new ExecuteException("未找到键盘区域");
            }
            lastKeyBoard = ocrArea;
            getKeyBoardInfo(ocrArea);
            char[] passwordChars = password.toCharArray();
            for (char passwordChar : passwordChars) {
                inputPasswordChar(passwordChar, request);
            }
            response = CmdAutomationResponse.builderSuccess(request, "密码输入（ocr）成功").withData(true);
        } catch (Exception e) {
            logger.error("密码输入ocr失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementMoneyText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "识别金额失败");
        String value = null;
        try {
            Map<String, Object> data = request.getData();
            String imgBase64 = "";
            if(data.get(UpperParamKey.X) != null) {
                imgBase64 = getOcrAreaToBase64(request);
            } else {
                imgBase64 = getClickTextOcrAreaToBase64(request);
            }
            if (StringUtils.isBlank(imgBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");

            }
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            if ("hzocr".equals(usingType)) {
                String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(imgBase64);
                    value = HzBankOcrUtils.getAllTextByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetMoneyValue");
                try {
                    HttpHeaders headers = new HttpHeaders();
                    headers.setContentType(MediaType.APPLICATION_JSON);
                    Map<String, Object> ocrParamMap = new HashMap<>();
                    ocrParamMap.put("img_base64", imgBase64);
                    ocrParamMap.put("targets", "");
                    HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
                    List ocrResultList = null;
                    try {
                        logger.info("传参：{}", JSON.toJSONString(ocrParamMap));
                        ocrResultList = HttpUtils.doPost(ocrAddress, ocrEntity, List.class);
                        logger.info("得到的ocr结果：{}", ocrResultList);
                    } catch (Exception e) {
                        logger.error("ocr失败", e);
                    }
                    List<String> dataList = new ArrayList<>();
                    if (!CollectionUtils.isEmpty(ocrResultList)) {
                        if (!CollectionUtils.isEmpty(ocrResultList)) {
                            for (Object o : ocrResultList) {
                                dataList.add(((Map) o).get("text") + "");
                            }
                        }
                    }
                    value = JSON.toJSONString(dataList);
                } catch (Exception e) {
                    logger.error("请求ocr接口失败，接口地址：{}",ocrAddress,e);
                }
                logger.info("识别金额为：{}", value);
                response = CmdAutomationResponse.builderSuccess(request, "识别金额成功").withData(value);
            }
        } catch (Exception e) {
            logger.error("识别失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void screenShot(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "屏幕截图失败");
        logger.debug("开始上传截图，信息：{}", JSON.toJSONString(request));
        try {
            String serial = phoneEntity.getUdid();
            Map<String, Object> data = request.getData();
            Integer x = (Integer) data.get(UpperParamKey.X);
            Integer y = (Integer) data.get(UpperParamKey.Y);
            File file = deviceHandleHelper.getScreenShotFile(serial);
            File addSignatureToImage = addSignatureToImage(file, x, y);
            String path = null;
            try {
                //租户id
                String tenantId = (String) data.get(UpperParamKey.TENANT_ID);
                String serverAddr = SpringUtils.getProperties("nk.mobile-computer.serverAddr");
                String publicUploadAddr = SpringUtils.getProperties("nk.mobile-computer.publicUploadAddr");
                Attachment upload = HttpUtils.upload(serverAddr + publicUploadAddr, addSignatureToImage.getAbsolutePath(), tenantId, "", FileBusinessTypeEnum.MOBILE_TASK_SCREENSHOT.getCode());
                if (null != upload && org.apache.commons.lang3.StringUtils.isNotBlank(upload.getId())) {
                    logger.debug("文件上传成功，返回id：{}", upload.getId());
                    path = upload.getUrlPath();
                }
            } catch (Exception e) {
                logger.error("截图失败", e);
                response = CmdAutomationResponse.builderFailure(request, e.getMessage());
            } finally {
                if (file.exists()) {
                    boolean delete = file.delete();
                    if (!delete) {
                        logger.warn("临时文件【{}】删除失败", file.getAbsolutePath());
                    }
                }
                if (addSignatureToImage.exists()) {
                    boolean delete = addSignatureToImage.delete();
                    if (!delete) {
                        logger.warn("临时文件【{}】删除失败", addSignatureToImage.getAbsolutePath());
                    }
                }
            }
            if (StringUtils.isNotBlank(path)) {
                response = CmdAutomationResponse.builderSuccess(request, "屏幕截图成功").withData(path);
            }
        } catch (Exception e) {
            logger.error("屏幕截图失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }

    }



    @Override
    public void getElementValueByPathOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            String type = (String) data.get(UpperParamKey.NODE_TYPE);
            String ifMoney = (String) data.get(UpperParamKey.IF_MONEY);
            logger.debug("拿到的nodeTree:{}, type:{}", nodeTree, type);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            if (StringUtils.isNotBlank(type)) {
                searchUiNodeData.setType(type);
            }
            UiNodeData uiNodeData = null;
            String value = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000) {
                uiNodeData = getNkAgent().uiNodeInfo(searchUiNodeData);
//                logger.info("拿到的uiNodeData：{}", uiNodeData);
                if (uiNodeData != null && !(uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0)) {
                    value = getValueByOcr(uiNodeData, ifMoney);
                    break;
                } else {
                    try {
                        if (System.currentTimeMillis() - startTime + 2000 < waitTimeout * 1000) {
                            Thread.sleep(findSleepTime);
                        }
                    } catch (InterruptedException e) {
                        logger.warn("查找控件失败",e);
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            response = CmdAutomationResponse.builderSuccess(request, "获取控件的值（OCR方式）成功").withData(value);
        } catch (Exception e) {
            logger.error("根据xpath判断元素是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally{
            logger.info("根据nodetree查找元素失败的response：{}", response);
            sendResultToEngine(response);
        }
    }

    @Override
    public void getTextDirectionText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "获取指定文字指定方向的内容失败");
        String value = "";
        try {
            Map<String, Object> data = request.getData();
            String text = (String) data.get(UpperParamKey.INPUT_TEXT);
            Integer index = data.get(UpperParamKey.INDEX) != null ? (Integer) data.get(UpperParamKey.INDEX) : 1;
            String direction = (String) data.get(UpperParamKey.DIRECTION);
            Integer amount = data.get(UpperParamKey.IF_MONEY) != null ? Integer.parseInt(data.get(UpperParamKey.IF_MONEY).toString()) : 1;
            String imgBase64 = getClickTextOcrAreaToBase64(request);
            if (StringUtils.isBlank(imgBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");

            }
            String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetTextDirectionText");
            HttpHeaders headers = new HttpHeaders();
            headers.setContentType(MediaType.APPLICATION_JSON);
            Map<String, Object> ocrParamMap = new HashMap<>();
            ocrParamMap.put("image_base64", imgBase64);
            ocrParamMap.put("targets", text);
            ocrParamMap.put("direction", direction);
            ocrParamMap.put("n", index);
            ocrParamMap.put("amount", amount);
            HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
            JSONObject ocrResultMap = null;
            try {
                logger.info("传参：{}", JSON.toJSONString(ocrParamMap));
                ocrResultMap = HttpUtils.doPostForGetTextDirectionText(ocrAddress, ocrEntity, JSONObject.class);
                logger.info("得到的ocr结果：{}", ocrResultMap);
            } catch (Exception e) {
                logger.error("ocr失败", e);
                if (e != null && e.getMessage() != null && e.getMessage().contains("Target text not found in image")) {
                    throw new ExecuteException("未找到指定文本");
                }
            }
            if (null != ocrResultMap) {
                JSONObject target_boxes = ocrResultMap.getJSONObject("target_boxes");
                logger.info("target_boxes为：{}", JSON.toJSONString(target_boxes));
                if ("右".equals(direction) || "左".equals(direction)) {
                    if (null != target_boxes) {
                        String targetText = (String) target_boxes.get("text");
                        if (StringUtils.isNotBlank(targetText)) {
                            int targetIndex = targetText.indexOf(text);
                            if ("右".equals(direction) && targetIndex < (targetText.length() - text.length())) {
                                value = targetText.substring(targetIndex + text.length());
                            } else if ("左".equals(direction) && targetIndex > 0) {
                                value = targetText.substring(0, targetIndex);
                            }
                        }
                    }
                }

                if (StringUtils.isBlank(value)) {
                    try {
                        JSONObject neighbor_boxes = ocrResultMap.getJSONObject("neighbor_boxes");
                        logger.info("neighbor_boxes为：{}", JSON.toJSONString(neighbor_boxes));
                        if (null != neighbor_boxes) {
                            String neighborText = (String) neighbor_boxes.get("text");
                            value = neighborText;
                        }
                    } catch (Exception e) {
                        JSONArray neighbor_boxes = ocrResultMap.getJSONArray("neighbor_boxes");
                        if (!CollectionUtils.isEmpty(neighbor_boxes)) {
                            value = (String) ((Map<String, Object>) neighbor_boxes.get(0)).get("text");
                        }
                    }
                }
            }
            logger.info("获取指定文字指定方向的内容为：{}", value);
            response = CmdAutomationResponse.builderSuccess(request, "获取指定文字指定方向的内容成功").withData(value);
        } catch (Exception e) {
            logger.error("识别失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void pressAndSwipe(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(false);
        boolean success = false;
        Map<String, Object> data = request.getData();
        NKAgent nkAgent = getNkAgent();
        try {
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String[] pointXpath = new String[3];
            JSONObject xpathObject = JSON.parseObject((String) data.get(UpperParamKey.FIRST_POINT), JSONObject.class);
            pointXpath[0] = xpathObject.getString("selector");
            xpathObject = JSON.parseObject((String) data.get(UpperParamKey.SECOND_POINT), JSONObject.class);
            pointXpath[1] = xpathObject.getString("selector");
            xpathObject = JSON.parseObject((String) data.get(UpperParamKey.THIRD_POINT), JSONObject.class);
            pointXpath[2] = xpathObject.getString("selector");
            Integer num = (Integer) data.get(UpperParamKey.POINT_NUM);
            List<Integer> passPoint = (List<Integer>) data.get(UpperParamKey.SWIPE_POINT);
            Integer waitTimeOut = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            logger.debug("拿到的左上角的点:{}", pointXpath[0]);
            logger.debug("拿到的左上角的右边点:{}", pointXpath[1]);
            logger.debug("拿到的左上角的下边点:{}", pointXpath[2]);
            logger.debug("拿到的每行每列的点数:{}", num);
            logger.debug("拿到的经过的点:{}", JSON.toJSONString(passPoint));
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            long startTime = System.currentTimeMillis();
            long endTime = startTime;
            logger.debug("步骤：{}开始查找控件，超时时间：{}", request.getStepToken(), waitTimeOut);
            Map<String, Integer>[] point = new Map[3];
            UiNodeData uiNodeData = null;
            int index = 0;
            int findIndex = 1;
            do {
                searchUiNodeData.setChain(pointXpath[index]);
                logger.debug("步骤：{}，第{}次开始查找第{}个元素", request.getStepToken(), findIndex,index+1);
                uiNodeData = getNkAgent().uiNodeInfo(searchUiNodeData);
                endTime = System.currentTimeMillis();
                logger.debug("步骤：{}，查找元素的总时间（累加）：{}ms，结果：{}", request.getStepToken(), endTime - startTime, JSON.toJSONString(uiNodeData));
                findIndex++;
                if (null != uiNodeData && (uiNodeData.getX() > 0 || uiNodeData.getY() > 0)) {
                    logger.debug("步骤：{}，第{}次开始查找第{}个元素已经找到", request.getStepToken(), findIndex,index+1);
                    int x = uiNodeData.getX();
                    int y = uiNodeData.getY();
                    int width = uiNodeData.getWidth();
                    int height = uiNodeData.getHeight();
                    Map<String, Integer> perPoint = new HashMap<>();
                    perPoint.put("x", x + width / 2);
                    perPoint.put("y", y + height / 2);
                    point[index] = perPoint;
                    index++;
                    findIndex = 1;
                }
            } while ((endTime - startTime) < waitTimeOut * 1000 && index < 3);
            for (int i = 0; i < 3; i++) {
                if (point[i] == null) {
                    response = CmdAutomationResponse.builderSuccess(request, "定位解锁区域的点不全").withData(false);
                    return;
                }
            }
            //计算每个点的坐标
            int perWidth = point[1].get("x") - point[0].get("x");
            int perHeight = point[2].get("y") - point[0].get("y");
            List<Map<String, Integer>> pointList = new ArrayList<>();
            int n = 1;
            for (int i = 0; i < num; i++) {
                int pointY = point[0].get("y") + i*perHeight;   //每一行的y坐标
                for (int j = 0; j < num; j++) {
                    int pointX = point[0].get("x") + j*perWidth;
                    logger.info("第{}个点的坐标x:{},y:{}",n,pointX,pointY);
                    Map<String, Integer> perPoint = new HashMap<>();
                    perPoint.put("x", pointX);
                    perPoint.put("y", pointY);
                    pointList.add(n-1,perPoint);
                    n++;
                }
            }
            logger.info("token:{},滑动解锁的所有坐标集合:{}",request.getStepToken(),JSON.toJSONString(pointList));
            //开始滑动
            List<Integer> passPointNum = new ArrayList<>();
            if (!CollectionUtils.isEmpty(passPoint)) {
                for (int i=0;i<passPoint.size();i++) {
                    Integer passIndex = passPoint.get(i);
                    Map<String, Integer> pointMessage = pointList.get(passIndex - 1);
                    passPointNum.add(pointMessage.get("x"));
                    passPointNum.add(pointMessage.get("y"));
                }
                logger.info("token:{},滑动解锁通过的坐标点:{}",request.getStepToken(),JSON.toJSONString(passPointNum));
                success = getNkAgent().drawLine(JSON.toJSONString(passPointNum));
            } else {
                logger.warn("token:{},没有滑动通过的坐标",request.getStepToken());
            }
            logger.info("token:{},滑动解锁的结果:{}",request.getStepToken(),success);
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "滑动解锁成功").withData(success);
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "滑动解锁失败").withData(success);
            }
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void swipeForSwipeToFindTargetElement(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "滑动查找控件失败");
        try {
            Map<String, Object> data = request.getData();
            String direction = (String) data.get(UpperParamKey.SWIPE_DIRECTION);
            Double swipeSizePer = StringUtils.isNotBlank((String) data.get(UpperParamKey.SWIPE_SIZE_PER)) ?
                    Double.parseDouble((String) data.get(UpperParamKey.SWIPE_SIZE_PER)) : null;
            int width = phoneEntity.getScreenWidth();
            int height = phoneEntity.getScreenHeight();
            double sizeUpDouble = swipeSizePer != null ? height * swipeSizePer : height / 2.0;
            double sizeLeftDouble = swipeSizePer != null ? width * swipeSizePer : width / 2.0;
            int sizeUp = (int) sizeUpDouble;
            int sizeLeft = (int) sizeLeftDouble;
            int x1, y1, x2, y2;
            if ("up".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height / 2;
                y2 = y1 - sizeUp;
            } else if ("down".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height / 2;
                y2 = y1 + sizeUp;
            } else if ("left".equalsIgnoreCase(direction)) {
                x1 = width / 2;
                x2 = x1 - sizeLeft;
                y1 = y2 = height / 2;
            } else if ("right".equalsIgnoreCase(direction)) {
                x1 = width / 2;
                x2 = x1 + sizeLeft;
                y1 = y2 = height / 2;
            } else {
                response = CmdAutomationResponse.builderFailure(request, "只支持上下左右滑动");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            DragXYData dragXYData = new DragXYData();
            dragXYData.setX(x1);
            dragXYData.setToX(x2);
            dragXYData.setY(y1);
            dragXYData.setToY(y2);
            dragXYData.setDuration(0);
            logger.debug("滑动dragXYData对象：{}", JSON.toJSONString(dragXYData));
            boolean result = nkAgent.handleTextUiNode(dragXYData, 24);
            if (result) {
                response = CmdAutomationResponse.builderSuccess(request, "滑动查找控件成功").withData(result);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "滑动查找控件失败");
            }
        } catch (Exception e) {
            logger.error("滑动查找控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    public void elementSwipe(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "控件滑动失败");
        try {
            Map<String, Object> data = request.getData();
            String direction = (String) data.get(UpperParamKey.SWIPE_DIRECTION);
            Integer size = (Integer) data.get(UpperParamKey.SWIPE_SIZE);
            Integer x1 = (Integer) data.get(UpperParamKey.X);
            Integer y1 = (Integer) data.get(UpperParamKey.Y);
            Double swipeSizePer = StringUtils.isNotBlank((String) data.get(UpperParamKey.SWIPE_SIZE_PER)) ?
                    Double.parseDouble((String) data.get(UpperParamKey.SWIPE_SIZE_PER)) : null;
            int width = phoneEntity.getScreenWidth();
            int height = phoneEntity.getScreenHeight();
            int x2, y2;
            if (swipeSizePer != null) {
                double sizeUpDouble =height * swipeSizePer;
                double sizeLeftDouble =  width * swipeSizePer;
                int sizeUp = (int) sizeUpDouble;
                int sizeLeft = (int) sizeLeftDouble;
                if ("up".equalsIgnoreCase(direction)) {
                    x2 = x1;
                    y2 = y1 - sizeUp;
                } else if ("down".equalsIgnoreCase(direction)) {
                    x2 = x1;
                    y2 = y1 + sizeUp;
                } else if ("left".equalsIgnoreCase(direction)) {
                    x2 = x1 - sizeLeft;
                    y2 = y1;
                } else if ("right".equalsIgnoreCase(direction)) {
                    x2 = x1 + sizeLeft;
                    y2 = y1;
                } else {
                    response = CmdAutomationResponse.builderFailure(request, "只支持上下左右滑动");
                    return;
                }
            } else {
                if ("up".equalsIgnoreCase(direction)) {
                    x2 = x1;
                    y2 = y1 - size;
                } else if ("down".equalsIgnoreCase(direction)) {
                    x2 = x1;
                    y2 = y1 + size;
                } else if ("left".equalsIgnoreCase(direction)) {
                    x2 = x1 - size;
                    y2 = y1;
                } else if ("right".equalsIgnoreCase(direction)) {
                    x2 = x1 + size;
                    y2 = y1;
                } else {
                    response = CmdAutomationResponse.builderFailure(request, "只支持上下左右滑动");
                    return;
                }
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            DragXYData dragXYData = new DragXYData();
            dragXYData.setX(x1);
            dragXYData.setToX(x2);
            dragXYData.setY(y1);
            dragXYData.setToY(y2);
            dragXYData.setDuration(0);
            logger.debug("滑动dragXYData对象：{}", JSON.toJSONString(dragXYData));
            boolean result = nkAgent.handleTextUiNode(dragXYData, 24);
            if (result) {
                response = CmdAutomationResponse.builderSuccess(request, "控件滑动成功").withData(result);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "控件滑动失败");
            }
        } catch (Exception e) {
            logger.error("控件滑动失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private String getValueByOcr(UiNodeData uiNodeData, String ifMoney) {
        String value = null;
        String imgBase64 = getOcrAreaByBodeToBase64(uiNodeData);
        String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        if ("1".equals(ifMoney)) {
            ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetMoneyValue");
        }
        try {
            HttpHeaders headers = new HttpHeaders();
            headers.setContentType(MediaType.APPLICATION_JSON);
            Map<String, Object> ocrParamMap = new HashMap<>();
            ocrParamMap.put("img_base64", imgBase64);
            ocrParamMap.put("targets", "");
            HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
            List ocrResultList = null;
            try {
                logger.info("ocr地址：{},传参：{}", ocrAddress, JSON.toJSONString(ocrParamMap));
                ocrResultList = HttpUtils.doPost(ocrAddress, ocrEntity, List.class);
                logger.info("得到的ocr结果：{}", ocrResultList);
            } catch (Exception e) {
                logger.error("ocr失败", e);
            }
            List<String> dataList = new ArrayList<>();
            if (!CollectionUtils.isEmpty(ocrResultList)) {
                if (!CollectionUtils.isEmpty(ocrResultList)) {
                    for (Object o : ocrResultList) {
                        dataList.add(((Map) o).get("text") + "");
                    }
                }
            }
            value = JSON.toJSONString(dataList);
        } catch (Exception e) {
            logger.error("请求ocr接口失败，接口地址：{}",ocrAddress,e);
        }
        logger.info("识别文字为：{}", value);
        return value;
    }

    private String getOcrAreaByBodeToBase64(UiNodeData uiNodeData) {
        int x = uiNodeData.getX() * phoneEntity.getScale();
        int y = uiNodeData.getY() * phoneEntity.getScale();
        int width = uiNodeData.getWidth() * phoneEntity.getScale();
        int height = uiNodeData.getHeight() * phoneEntity.getScale();
        logger.debug("拿到的截图参数----->>>>>x:{},y:{},cutWidth:{},cutHeight:{}", x, y, width, height);
        File file = deviceHandleHelper.getScreenShotFile(phoneEntity.getUdid(), x, y, width, height);
        logger.debug("ocr截图：{}", file.getAbsolutePath());
        String base64Str = getFileBase64(file);
        return base64Str;
    }

    private void getKeyBoardInfo(String ocrArea) {
        Map<String, JSONObject> itemMap = new HashMap<>();
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        logger.debug("第{}次进来键盘base64：{}",index,ocrArea);
        paramMap.put("img_base64", ocrArea);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        logger.info("识别键盘地址为：{}", keyboardAddress);
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            boolean numKeyboard = true;
            logger.debug("ocr识别键盘结果：{}", result);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                logger.debug("第{}次进来识别键盘结果：{}",index,parseArray);
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                // orc返回数组顺序 x y w h
                String name = (String) parseArray.get(0);
                JSONObject item = new JSONObject();
                item.put("name", name);
                item.put("x", parseArray.get(1));
                item.put("y", parseArray.get(2));
                item.put("w", parseArray.get(3));
                item.put("h", parseArray.get(4));
                if (name.equals("shift")) {
                    logger.debug("第{}次进来是小写字母键盘........", index);
                    preFlag = "0";
                    numKeyboard = false;
                } else if (name.equals("highshift")) {
                    logger.debug("第{}次进来是大写字母键盘........", index);
                    preFlag = "3";
                    numKeyboard = false;
                } else if (name.equals("&")) {
                    logger.debug("第{}次进来是字符键盘...........", index);
                    preFlag = "2";
                    numKeyboard = false;
                }
                itemMap.put(name, item);
            }
            if (numKeyboard) {
                logger.debug("第{}次进来是数字界面...........", index);
                preFlag = "1";
            }
            logger.info("第{}次进来的界面标志为：{}", index, preFlag);
            totalMap.put(preFlag, itemMap);
            index++;
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
            throw new ExecuteException("键盘识别失败");
        }
    }

    private void inputPasswordChar(char passwordChar, CmdAutomationRequest request) {
        logger.info("密码字符：{}", passwordChar);
        boolean isNumber = Character.isDigit(passwordChar);
        if (isNumber) {   //数字
            if (!"1".equals(preFlag)) {  //当前不在数字页面
                logger.info("切换数字页面，密码字符：{}", passwordChar);
                exchangePage("num", request);
                preFlag = "1";
            }
            Map<String, JSONObject> numMap = totalMap.get(preFlag);
            if (null == numMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                numMap = totalMap.get(preFlag);
            }
            JSONObject item = numMap.get(String.valueOf(passwordChar));
            if (item == null) {
                throw new ExecuteException("未识别到数字：" + passwordChar);
            }
            click(item, request);
            return;
        }
        boolean letter = Character.isLetter(passwordChar);
        if (letter) {  //字母
            if (!"0".equals(preFlag) && !"3".equals(preFlag)) {  //既不是大写字母也不是小写字母
                exchangePage("letter", request);
                preFlag = determinePage();
            }
            Map<String, JSONObject> letterMap = totalMap.get(preFlag);
            if (null == letterMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                letterMap = totalMap.get(preFlag);
            }
            boolean lowerCase = Character.isLowerCase(passwordChar);
            if (!lowerCase) {   //大写
                if ("3".equals(preFlag)) {  //当前处于大写键盘界面
                    if (totalMap.get("3") == null) {
                        Map<String, Object> requestData = request.getData();
                        String ocrArea = getClickTextOcrAreaToBase64(request);
                        getKeyBoardInfo(ocrArea);
                    }
                    JSONObject item = letterMap.get(String.valueOf(passwordChar));
                    if (item == null) {
                        throw new ExecuteException("未识别到字母：" + passwordChar);
                    }
                    click(item, request);  //点字母
                } else {//当前处于小写键盘界面
                    JSONObject shiftItem = letterMap.get("shift");
                    click(shiftItem, request);  //点大写
                    if (totalMap.get("3") == null) {
                        Map<String, Object> requestData = request.getData();
                        String ocrArea = getClickTextOcrAreaToBase64(request);
                        getKeyBoardInfo(ocrArea);
                    }
                    JSONObject item = totalMap.get("3") != null ? totalMap.get("3").get(String.valueOf(passwordChar)) : null;
                    if (item == null) {
                        throw new ExecuteException("未识别到字母：" + passwordChar);
                    }
                    click(item, request);  //点字母
                    preFlag = "0";
                }
            } else {  //小写
                if ("0".equals(preFlag)) {
                    JSONObject item = totalMap.get("0") != null ? totalMap.get("0").get(String.valueOf(passwordChar)) : null;
                    if (item == null) {
                        throw new ExecuteException("未识别到字母：" + passwordChar);
                    }
                    click(item, request);  //点字母
                } else {
                    JSONObject shiftItem = totalMap.get("1") != null ? totalMap.get("1").get("highshift") : null;
                    click(shiftItem, request);  //点大写
                    preFlag = determinePage();
                    if ("3".equals(preFlag)){//还处于大写
                        click(shiftItem, request);  //点大写
                    }
                    JSONObject item = letterMap.get(String.valueOf(passwordChar));
                    if (item == null) {
                        throw new ExecuteException("未识别到字母：" + passwordChar);
                    }
                    click(item, request);  //点字母
                }
            }
        } else {  //特殊字符
            if (!"2".equals(preFlag)) {    //不是特殊字符，切换到特殊字符
                exchangePage("character", request);
                preFlag = "2";//当前页面是字符界面
            }
            Map<String, JSONObject> characterMap = totalMap.get(preFlag);
            if (null == characterMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                characterMap = totalMap.get(preFlag);
            }
            JSONObject item = characterMap.get(String.valueOf(passwordChar));
            if (item == null) {
                throw new ExecuteException("未识别到字符：" + passwordChar);
            }
            click(item, request);
        }
    }

    private String determinePage() {
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("img_base64", lastKeyBoard);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                String name = (String) parseArray.get(0);
                if (name.equals("shift")) {
                    logger.debug("第{}次进来是小写字母键盘........", index);
                    return "0";
                } else if (name.equals("highshift")) {
                    logger.debug("第{}次进来是大写字母键盘........", index);
                    return "3";
                }
            }
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
        }
        return "0";
    }

    private void exchangePage(String flag, CmdAutomationRequest request) {
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("img_base64", lastKeyBoard);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                String name = (String) parseArray.get(0);
                if (name.equals(flag)) {
                    logger.info("切换页面flag：{}", flag);
                    JSONObject item = new JSONObject();
                    item.put("name",parseArray.get(0));
                    item.put("x", parseArray.get(1));
                    item.put("y", parseArray.get(2));
                    item.put("w", parseArray.get(3));
                    item.put("h", parseArray.get(4));
                    click(item, request);
                    break;
                }
            }
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
        }
        try {
            Thread.sleep(1000);
        } catch (Exception exception) {
            logger.error(exception.getMessage());
        }
        Map<String, Object> requestData = request.getData();
        String ocrArea = getClickTextOcrAreaToBase64(request);
        if (null != ocrArea) {
            lastKeyBoard = ocrArea;
        }
    }

    private void click(JSONObject item, CmdAutomationRequest request) {
        if (null != item) {
            CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "密码输入（ocr）点击控件失败").withData(false);
            try {
                double realityX = 0.0;
                double realityY = 0.0;
                Map<String, Object> data = request.getData();
                int clickTrueX = new Double((realityX + (item.getDouble("x") + item.getDouble("w") / 2)) / phoneEntity.getScale()).intValue();
                int clickTrueY = new Double((realityY + (item.getDouble("y") + item.getDouble("h") / 2)) / phoneEntity.getScale()).intValue();
                TapXYData tapXYData = new TapXYData();
                tapXYData.setX(clickTrueX);
                tapXYData.setY(clickTrueY);
                NKAgent nkAgent = getNkAgent();
                logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
                nkAgent.handleTextUiNode(tapXYData, 21);
            } catch (Exception e) {
                logger.error("密码输入（ocr）点击失败，原因：", e);
                response = CmdAutomationResponse.builderFailure(request, e.getMessage());
                sendResultToEngine(response);
            }
        }
    }

    private NKAgent getNkAgent() {
        int times = 0;
        NKAgent nkAgent = null;
        while (nkAgent == null && times < 3) {
            IosDeviceInitThread iosInitThread = IOSDeviceManager.getInstance().getIosInitThread(phoneEntity.getUdid());
            if (null == iosInitThread || !iosInitThread.isAlive() || iosInitThread.isInterrupted()) {
                return nkAgent;
            }
            try {
                nkAgent = iosInitThread.getNkAgent();
                if (!nkAgent.getStatus()) {
                    logger.error("设备Agent还未准备好，稍后再试");
                    nkAgent = null;
                }
            } catch (Exception e) {
                logger.error("创建nkAgent失败，e", e);
                nkAgent = null;
            }
            if (nkAgent == null) {
                try {
                    // 如果失败是因为agent失效，等待重新创建
                    logger.debug("nkAgent重试创建{}次", times);
                    Thread.sleep(5000L);
                } catch (InterruptedException e) {
                    logger.error("启动app休眠打断。");
                }
            }
            times++;
        }
        return nkAgent;
    }




}
