package net.northking.cctp.upperComputer.automation.handler;

import cn.hutool.core.codec.Base64;
import cn.hutool.extra.spring.SpringUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import net.northking.cctp.upperComputer.automation.constants.Command;
import net.northking.cctp.upperComputer.automation.constants.UpperParamKey;
import net.northking.cctp.upperComputer.automation.entity.CmdAutomationRequest;
import net.northking.cctp.upperComputer.automation.entity.CmdAutomationResponse;
import net.northking.cctp.upperComputer.deviceManager.IOSDeviceManager;
import net.northking.cctp.upperComputer.deviceManager.UpperComputerManager;
import net.northking.cctp.upperComputer.deviceManager.thread.IosDeviceInitThread;
import net.northking.cctp.upperComputer.driver.ios.NKAgent;
import net.northking.cctp.upperComputer.driver.ios.command.data.*;
import net.northking.cctp.upperComputer.entity.PhoneEntity;
import net.northking.cctp.upperComputer.exception.ExecuteException;
import net.northking.cctp.upperComputer.service.IosDebuggerServiceImpl;
import net.northking.cctp.upperComputer.utils.HttpUtils;
import net.northking.cctp.upperComputer.utils.ScreenShotUtils;
import net.northking.cctp.upperComputer.utils.SpringUtils;
import net.northking.cctp.upperComputer.utils.hzBank.HzBankOcrUtils;
import net.northking.cctp.upperComputer.utils.ios.IosDeviceHandleHelper;
import net.northking.cctp.upperComputer.utils.ios.LinuxAndWindowsIosHandleHelper;
import net.northking.cctp.upperComputer.utils.ios.MacIosHandleHelper;
import net.northking.cctp.upperComputer.webSocket.function.IosFunction;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.util.CollectionUtils;
import sun.misc.BASE64Decoder;

import javax.imageio.ImageIO;
import javax.websocket.Session;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;

/**
 * @author : yineng.huang
 * @date : 2024/4/16 19:59
 */
public class IosAutomationHandler extends AbstractAutomationHandler {

    private final Logger logger = LoggerFactory.getLogger(IosAutomationHandler.class);

    private PhoneEntity phoneEntity;

    private IosDeviceHandleHelper deviceHandleHelper;

    public IosAutomationHandler(PhoneEntity phoneEntity, Session session) {
        this.phoneEntity = phoneEntity;
        this.serial = phoneEntity.getUdid();
        this.engineSession = session;
        boolean isMac = UpperComputerManager.getInstance().getOperatingSystem().startsWith("Mac");
        if (isMac) {
            deviceHandleHelper = new MacIosHandleHelper();
        } else {
            deviceHandleHelper = new LinuxAndWindowsIosHandleHelper();
        }
    }

    // 密码输入（ocr）相关参数
    private Map<String, Map<String, JSONObject>> totalMap = new HashMap<>();
    private List<Map<String, JSONObject>> list;
    private int index = 1;
    private String lastKeyBoard;
    private String preFlag = "0";    //0-小写字母，1-数字，2-符号,3-大写字母


    @Override
    public void destroy() {

    }

    @Override
    public void openMobileNotificationBar(CmdAutomationRequest request) {

    }

    @Override
    public void closeMobileNotificationBar(CmdAutomationRequest request) {

    }

    @Override
    public void initApplication(CmdAutomationRequest request) {
        CmdAutomationResponse response = null;
        try {
            Map<String, Object> initData = request.getData();
            boolean reInstall = false;
            if (null != initData.get("forceReInstall")) {
                reInstall = (boolean) initData.get("forceReInstall");
            }
            String appId = (String) initData.get("appId");
            Map<String, Object> appDetail = null;
            try {
                appDetail = getAppDetail(appId);
            } catch (Exception e) {
                logger.error("查询app信息出错，原因：", e);
                response = CmdAutomationResponse.builderFailure(request, "查询app信息出错");
                return;
            }
            if (null == appDetail) {
                response = CmdAutomationResponse.builderFailure(request, "app应用已被删除，无法初始化");
                return;
            }
            String appUrl = (String) appDetail.get("appUrl");
            String appPackage = (String) appDetail.get("packageName");
            String appVersion = (String) appDetail.get("buildVersion");
            String appName = appUrl.substring(appUrl.lastIndexOf("_") + 1);
            if (reInstall) {
                if (deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    logger.info("Begin to uninstall app[{}] from device[{}]...", appPackage, phoneEntity.getUdid());
                    deviceHandleHelper.removeApp(phoneEntity.getUdid(), appPackage);
                    logger.info("App[{}] from device[{}] uninstalled.", appPackage, phoneEntity.getUdid());
                } else {
                    logger.info("App[{}] on device[{}] is not installed.", appPackage, phoneEntity.getUdid());
                }
            }
            if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                installApp(appId, appName);
            } else {
                logger.info("device[{}] already install app[{}]...", phoneEntity.getUdid(), appPackage);
                String oldPackageCode = deviceHandleHelper.getOldPackageCode(phoneEntity.getUdid(), appPackage);
                if (!appVersion.equalsIgnoreCase(oldPackageCode)) {
                    logger.info("install app version[{}],app in device version[{}]  ===========> reInstall", appVersion, oldPackageCode);
                    deviceHandleHelper.removeApp(phoneEntity.getUdid(), appPackage);
                    logger.info("app version[{}] remove from device[{}] successfully", appVersion, phoneEntity.getUdid());
                    installApp(appId, appName);
                } else {
                    logger.info("App[{}] on device[{}] already exists ===========> terminate", appPackage, phoneEntity.getUdid());
                }
            }
            if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                response = CmdAutomationResponse.builderSuccess(request, "app安装失败");
                return;
            }
            logger.info("activate app[{}]", appPackage);
            boolean success = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
            if (!success) {
                response = CmdAutomationResponse.builderFailure(request, "应用启动失败");
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "app初始化成功");
            }
        } catch (Exception e) {
            logger.error("设备【{}】应用初始化失败", phoneEntity.getUdid(), e);
            CmdAutomationResponse.builderFailure(request, "应用初始化失败");
        } finally {
            sendResultToEngine(response);
        }
        logger.info("自动化执行环境【device:{}】准备就绪", phoneEntity.getUdid());
    }


    @Override
    public void clickPoint(CmdAutomationRequest request) {

    }

    @Override
    public void searchNodeByNodeTree(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            logger.debug("拿到的nodeTree:{}", nodeTree);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            UiNodeData uiNodeData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300) {
                uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData);
//                logger.info("拿到的uiNodeData：{}", uiNodeData);
                if (uiNodeData != null && !(uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0)) {
                    break;
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            if (null == uiNodeData) {
                response = CmdAutomationResponse.builderSuccess(request, "元素不存在");
            } else {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    response = CmdAutomationResponse.builderSuccess(request, "元素不存在");
                } else {
                    TapXYData tapXYData = new TapXYData();
                    tapXYData.setX(uiNodeData.getX());
                    tapXYData.setY(uiNodeData.getY());
                    response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
                }
            }
        } catch (Exception e) {
            logger.error("根据xpath判断元素是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally{
            logger.info("根据nodetree查找元素失败的response：{}", response);
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByPoint(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 - 300) {
                tapXYData = queryPoint(request);
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
                if (tapXYData != null) {
                    break;
                }
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderSuccess(request, "查找控件失败").withData(false);
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            logger.error("根据坐标是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                tapXYData = useNewOcr ? queryOcrAreaNew(request) : queryOcrArea(request);
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void searchNodeByImage(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300 && tapXYData == null) {
                tapXYData = queryImageArea(request);
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件");
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "查找控件完成").withData(tapXYData);
            }
        } catch (Exception e) {
            logger.error("根据图像是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementValueByNode(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "未查到控件");
        try {
            Map<String, Object> data = request.getData();
            NKAgent nkAgent = getNkAgent();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            logger.debug("拿到的nodeTree:{}", nodeTree);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            UiNodeData uiNodeData = null;
            String value = null;

            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300) {
                uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData);
                if (uiNodeData != null && !(uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0)) {
                    value = uiNodeData.getLabel();
                    if (StringUtils.isBlank(value)) {
                        value = uiNodeData.getValue();
                    }
                    if (StringUtils.isBlank(value)) {
                        value = uiNodeData.getName();
                    }
                    if (StringUtils.isNotBlank(value)) {
                        break;
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }

            if (null != uiNodeData) {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    response = CmdAutomationResponse.builderSuccess(request, "找不到控件");
                    return;
                }
                if (StringUtils.isNotBlank(value)) {
                    response = CmdAutomationResponse.builderSuccess(request, "已获取控件的值").withData(value);
                }
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "找不到控件");
            }
        } catch (Exception e) {
            logger.error("根据xpath获取控件的值失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementValueByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
        try {
            String value = null;
            String targetBase64 = getOcrAreaToBase64(request);
            if (StringUtils.isBlank(targetBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                return;
            }
            // 是否使用杭银ocr
            Map<String, Object> dataOcr = request.getData();
            String usingType = (String) dataOcr.get(UpperParamKey.USING_TYPE);
            if (usingType.equals("hzocr")) {
                String ocrText = (String) dataOcr.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(targetBase64);
                    value = HzBankOcrUtils.getAllTextByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                // 是否使用新版ocr
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                if (useNewOcr) {
                    String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
                    HttpHeaders headers = new HttpHeaders();
                    headers.setContentType(MediaType.APPLICATION_JSON);
                    Map<String, Object> ocrParamMap = new HashMap<>();
                    ocrParamMap.put("img_base64", targetBase64);
                    ocrParamMap.put("targets", "");
                    HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
                    List ocrResultList = null;
                    try {
                        logger.info("传参：{}", JSON.toJSONString(ocrParamMap));
                        ocrResultList = HttpUtils.doPost(ocrAddress, ocrEntity, List.class);
                        logger.info("得到的ocr结果：{}", ocrResultList);
                    } catch (Exception e) {
                        logger.error("ocr失败", e);
                    }
//            List<String> dataList = new ArrayList<>();
                    StringJoiner dataList = new StringJoiner("");
                    if (!CollectionUtils.isEmpty(ocrResultList)) {
                        if (!CollectionUtils.isEmpty(ocrResultList)) {
                            for (Object o : ocrResultList) {
                                dataList.add(((Map) o).get("text") + "");
                            }
                        }
                    }
                    value =  dataList.toString();
                } else {
                    String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetText");
                    try {
                        HttpHeaders headers = new HttpHeaders();
                        headers.setContentType(MediaType.APPLICATION_JSON);
                        HttpEntity<String> ocrEntity = new HttpEntity<>("\"" + targetBase64 + "\"", headers);
                        JSONObject object = HttpUtils.doPost(new URI(ocrAddress), ocrEntity, JSONObject.class);
                        JSONObject data = object.getJSONObject("data");
                        if (data != null) {
                            value = data.getString("ocrText");
                        }
                    } catch (URISyntaxException e) {
                        logger.error("ocr地址有误！===>{}", ocrAddress, e);
                    } catch (Exception e) {
                        logger.error("请求ocr接口失败，接口地址：{}", ocrAddress, e);
                    }
                }
            }

            logger.info("识别的图片文字为：{}", value);
            if (StringUtils.isNotBlank(value)) {
                response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
            }
        } catch (Exception e) {
            logger.error("根据ocr获取控件的值失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private String getOcrAreaToBase64(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        Integer x = (Integer) data.get(UpperParamKey.X);
        Integer y = (Integer) data.get(UpperParamKey.Y);
        Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
        Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
        Integer cutWidth = (Integer) data.get(UpperParamKey.WIDTH);
        Integer cutHeight = (Integer) data.get(UpperParamKey.HEIGHT);
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        logger.debug("拿到的截图参数----->>>>>x:{},y:{},screenWidth:{},screenHeight:{},cutWidth:{},cutHeight:{},查找的文本:{}", x, y, screenWidth, screenHeight, cutWidth, cutHeight, ocrText);
        File file = deviceHandleHelper.getScreenShotFile(serial, x, y, cutWidth, cutHeight, screenWidth, screenHeight);
        logger.debug("执行步骤token：{}，ocr截图：{}", request.getStepToken(), file.getAbsolutePath());
        String base64Str = getFileBase64(file);
        return base64Str;
    }

    private String getClickTextOcrAreaToBase64(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        logger.debug("拿到的截图参数----->>>>>查找的文本:{}", ocrText);
        File file = ScreenShotUtils.getIOSMobileScreenShot(serial);
        logger.debug("执行步骤token：{}，ocr截图：{}", request.getStepToken(), file.getAbsolutePath());
        return getFileBase64(file);
    }

    @Override
    public void getVerificationCodeByNode(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "识别验证码失败");
        Map<String, Object> data = request.getData();
        NKAgent nkAgent = getNkAgent();
        try {
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            logger.debug("拿到的nodeTree:{}", nodeTree);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            UiNodeData uiNodeData = nkAgent.uiNodeInfo(searchUiNodeData);
            if (null == uiNodeData) {
                response = CmdAutomationResponse.builderFailure(request, "元素不存在");
            } else {
                if (uiNodeData.getX() <= 0 && uiNodeData.getY() <= 0) {
                    response = CmdAutomationResponse.builderFailure(request, "元素不存在");
                } else {
                    File shotFile = deviceHandleHelper.getScreenShotFile(serial, uiNodeData.getX(), uiNodeData.getY(), uiNodeData.getWidth(), uiNodeData.getHeight(), phoneEntity.getScreenWidth(), phoneEntity.getScreenHeight());
                    String targetBase64 = getFileBase64(shotFile);
                    if (StringUtils.isBlank(targetBase64)) {
                        response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                        return;
                    }
                    String value = getVerificationCodeByBase64(targetBase64);
                    if (StringUtils.isNotBlank(value)) {
                        response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
                    } else {
                        response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
                    }
                }
            }
        } catch (Exception e) {
            logger.error("识别验证码失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getVerificationCodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr获取验证码失败");
        try {
            String value = null;
            String targetBase64 = getOcrAreaToBase64(request);
            if (StringUtils.isBlank(targetBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");
                return;
            }
            Map<String, Object> data = request.getData();
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            if ("hzocr".equals(usingType)) {
                String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(targetBase64);
                    value = HzBankOcrUtils.getAllTextListByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                value = getVerificationCodeByBase64(targetBase64);
            }
            if (StringUtils.isNotBlank(value)) {
                response = CmdAutomationResponse.builderSuccess(request, "根据ocr获取控件值成功").withData(value);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "根据ocr获取控件值失败");
            }
        } catch (Exception e) {
            logger.error("根据ocr获取验证码失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private String getVerificationCodeByBase64(String base64){
        String value = null;
        String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetVerificationCode");
        try {
            HttpHeaders headers = new HttpHeaders();
            headers.setContentType(MediaType.APPLICATION_JSON);
            Map<String, Object> ocrParamMap = new HashMap<>();
            ocrParamMap.put("img_base64", base64);
            HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
            value = HttpUtils.doPost(new URI(ocrAddress), ocrEntity, String.class);
        }catch (URISyntaxException e) {
            logger.error("ocr地址有误！===>{}",ocrAddress,e);
        } catch (Exception e) {
            logger.error("请求ocr接口失败，接口地址：{}",ocrAddress,e);
        }
        logger.info("识别到的验证码为：{}", value);
        return value == null ? value : value.replace("\"", "");
    }

    @Override
    public void longPress(CmdAutomationRequest request) {

    }

    @Override
    public void swipe(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "滑动失败");
        try {
            Map<String, Object> data = request.getData();
            String direction = (String) data.get(UpperParamKey.SWIPE_DIRECTION);
            Integer size = (Integer) data.get(UpperParamKey.SWIPE_SIZE);
            int width = phoneEntity.getScreenWidth() / phoneEntity.getScale();
            int height = phoneEntity.getScreenHeight() / phoneEntity.getScale();
            int x1, y1, x2, y2;
            if ("up".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height * 8 / 10;
                if (null == size || size <= 0) {
                    y2 = y1 - height / 2;
                } else {
                    y2 = y1 - size;
                }
            } else if ("down".equalsIgnoreCase(direction)) {
                x1 = x2 = width / 2;
                y1 = height * 2 / 10;
                if (null == size || size <= 0) {
                    y2 = y1 + height / 2;
                } else {
                    y2 = y1 + size;
                }
            } else if ("left".equalsIgnoreCase(direction)) {
                y1 = y2 = height / 2;
                x1 = width * 9 / 10;
                if (null == size || size <= 0) {
                    x2 = x1 - width / 2;
                } else {
                    x2 = x1 - size;
                }
            } else if ("right".equalsIgnoreCase(direction)) {
                y1 = y2 = height / 2;
                x1 = width * 1 / 10;
                if (null == size || size <= 0) {
                    x2 = x1 + width / 2;
                } else {
                    x2 = x1 + size;
                }
            } else {
                response = CmdAutomationResponse.builderFailure(request, "只支持上下左右滑动");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            DragXYData dragXYData = new DragXYData();
            dragXYData.setX(x1);
            dragXYData.setToX(x2);
            dragXYData.setY(y1);
            dragXYData.setToY(y2);
            dragXYData.setDuration(0);
            boolean result = nkAgent.handleTextUiNode(dragXYData, 24);
            if (result) {
                response = CmdAutomationResponse.builderSuccess(request, "滑动成功").withData(result);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "滑动失败");
            }
        } catch (Exception e) {
            logger.error("滑动失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void inputText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "输入失败").withData(false);
        Map<String, Object> data = request.getData();
        NKAgent nkAgent = getNkAgent();
        try {
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clear = (boolean) data.get(UpperParamKey.CLEAR);
            if (clear) {
                logger.debug("步骤【{}】输入前清空数据",request.getStepToken());
                boolean clearSuccess = nkAgent.handleDeleteTextUiNode(Command.CLEAR.getCode());
                logger.debug("步骤【{}】输入前清空数据结果：{}",request.getStepToken(),clearSuccess);
            }
            String inputText = (String) data.get(UpperParamKey.INPUT_TEXT);
            Integer commandId = (Integer) data.get(UpperParamKey.COMMAND_ID);
            logger.debug("步骤【{}】即将输入文本：{},commandId:{}",request.getStepToken(),inputText,commandId);
            TypeKeysUiNodeData typeKeysUiNodeData = new TypeKeysUiNodeData();
            typeKeysUiNodeData.setText(inputText);
            boolean success = nkAgent.handleTextUiNode(typeKeysUiNodeData, 18);
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "输入成功").withData(success);
            }else {
                response = CmdAutomationResponse.builderFailure(request, "输入失败").withData(success);
            }
        } catch (Exception e) {
            logger.error("输入失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleNodeByNodeTree(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据node节点点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            NKAgent nkAgent = getNkAgent();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            String nodeTree = (String) data.get(UpperParamKey.NODE_TREE);
            Integer commandId = (Integer) data.get(UpperParamKey.COMMAND_ID);
            logger.debug("拿到的nodeTree:{}", nodeTree);
            SearchUiNodeData searchUiNodeData = new SearchUiNodeData();
            searchUiNodeData.setChain(nodeTree);
            boolean success = false;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300) {
                success = nkAgent.clickUiNode(searchUiNodeData, 5);
                if (success) {
                    break;
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            if (success) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(success);
            }else {
                response = CmdAutomationResponse.builderFailure(request, "控件不存在").withData(success);
            }
        } catch (Exception e) {
            logger.error("根据node节点点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleNodeByPoint(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据坐标点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            TapXYData tapXYData = null;
            boolean clickSuccess = false;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300) {
                tapXYData = queryPoint(request);
                if (tapXYData != null) {
                    clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
                    if (clickSuccess) {
                        break;
                    }
                }
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }

            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }

            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据坐标点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private TapXYData queryPoint(CmdAutomationRequest request) {
        TapXYData tapXYData = null;
        Map<String, Object> data = request.getData();
        Integer x = (Integer) data.get(UpperParamKey.X);
        Integer y = (Integer) data.get(UpperParamKey.Y);
        Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
        Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
        boolean overTurn = false;
        if (screenWidth > screenHeight) {
            overTurn = true;
        }
        int width = phoneEntity.getScreenWidth()* phoneEntity.getScale();
        int height = phoneEntity.getScreenHeight()*phoneEntity.getScale();
        int realityWidth = width;
        int realityHeight = height;
        if (overTurn) {  //宽高反了的情况，调换
            if (width < height) {
                realityWidth = height;
                realityHeight = width;
            }
        } else {
            if (width > height) {
                realityWidth = height;
                realityHeight = width;
            }
        }
        Double realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
        Double realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;
        if (realityX <= 0 || realityX >= realityWidth || realityY <= 0 || realityY >= realityHeight) {
            logger.warn("步骤id【{}】坐标方式超出了屏幕范围", request.getStepToken());
        }
        int clickTrueX = new Double(realityX / phoneEntity.getScale()).intValue();
        int clickTrueY = new Double(realityY / phoneEntity.getScale()).intValue();
        tapXYData = new TapXYData();
        tapXYData.setX(clickTrueX);
        tapXYData.setY(clickTrueY);
        logger.debug("实际点击的位置------->>>>>>x:{},y:{}", clickTrueX, clickTrueY);
        return tapXYData;
    }

    @Override
    public void handleNodeByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryOcrArea(request);;
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private TapXYData queryClickTextOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            Boolean success = ocrResultMap.getBoolean("success");
            if (success) {
                JSONArray pointData = ocrResultMap.getJSONArray("data");
                Integer index = (Integer) data.get(UpperParamKey.INDEX);
                JSONObject pointMap = null;
                if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                    pointMap = pointData.getJSONObject(0);
                } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                    if (index > 0) {
                        if (index <= pointData.size()) {
                            pointMap = pointData.getJSONObject(index - 1);
                        } else {
                            logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                            throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", pointData.size()));
                        }
                    }
                }
                if (null != pointMap) {
                    result = new TapXYData();
                    Double scale = Double.valueOf(phoneEntity.getScale());
                    result.setX((int) (pointMap.getDouble("x").intValue() / scale));
                    result.setY((int) (pointMap.getDouble("y").intValue() / scale));
                }
            }
        }
        return result;
    }


    private boolean queryIfTextOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            boolean success = ocrResultMap.getBoolean("success");
            return success;
        } else {
            return false;
        }
    }

    private boolean queryIfTextOcrAreaNew(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultList) {
            return true;
        } else {
            return false;
        }
    }

    private TapXYData queryOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (!CollectionUtils.isEmpty(ocrResultList)) {
            Object all = ocrResultList.get(0);
            JSONObject jsonObject = JSONObject.parseObject(JSONArray.toJSONString(all));
            //todo：如果传了;是多个文本一起找，暂时只考虑一个文本
            JSONArray pointData = jsonObject.getJSONArray(ocrText);   //所有坐标
            if (pointData == null || pointData.size() == 0) {
                return result;
            } else {
                return new TapXYData();
            }
        } else {
            return result;
        }
    }

    private TapXYData queryOcrArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        ocrParamMap.put("base64Image", base64Str);
        ocrParamMap.put("CaseSensitive", true);
        ocrParamMap.put("ocrText", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        JSONObject ocrResultMap = null;
        String ocrFindArea = SpringUtils.getProperties("nk.http-request-path.ocrFindArea");
        try {
            ocrResultMap = HttpUtils.doPost(ocrFindArea, ocrEntity, JSONObject.class);
            logger.info("得到的ocr结果：{}", ocrResultMap);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (null != ocrResultMap) {
            Boolean success = ocrResultMap.getBoolean("success");
            if (success) {
                Integer x = (Integer) data.get(UpperParamKey.X);
                Integer y = (Integer) data.get(UpperParamKey.Y);
                Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
                Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight()* phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
                Double realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;
                JSONArray pointData = ocrResultMap.getJSONArray("data");
                Integer index = (Integer) data.get(UpperParamKey.INDEX);
                JSONObject pointMap = null;
                if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                    pointMap = pointData.getJSONObject(0);
                } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                    if (index > 0) {
                        if (index <= pointData.size()) {
                            pointMap = pointData.getJSONObject(index - 1);
                        } else {
                            logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                        }
                    }
                }
                if (null != pointMap) {
                    int clickTrueX = new Double((realityX + pointMap.getDouble("x")) / phoneEntity.getScale()).intValue();
                    int clickTrueY = new Double((realityY + pointMap.getDouble("y")) / phoneEntity.getScale()).intValue();
                    result = new TapXYData();
                    result.setX(clickTrueX);
                    result.setY(clickTrueY);
                }
            }
        }
        return result;
    }

    private TapXYData queryClickTextOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        Map<String, Object> ocrParamMap = new HashMap<>();
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        Integer x = (Integer) data.get(UpperParamKey.X);
        Integer y = (Integer) data.get(UpperParamKey.Y);
        Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
        Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
        //转换比例
        int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
        int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
        int realityWidth = width;
        int realityHeight = height;
        if (width > height) {  //宽高反了的情况，调换
            realityWidth = height;
            realityHeight = width;
        }
        Double realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
        Double realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;

        ocrParamMap.put("img_base64", base64Str);
        ocrParamMap.put("targets", ocrText);
        HttpEntity<Map<String, Object>> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
        List ocrResultList = null;
        String ocrGetAllTextNum = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
        try {
            ocrResultList = HttpUtils.doPost(ocrGetAllTextNum, ocrEntity, List.class);
            logger.info("得到的ocr结果：{}", ocrResultList);
        } catch (Exception e) {
            logger.error("ocr失败", e);
        }
        if (!CollectionUtils.isEmpty(ocrResultList)) {
            Object all = ocrResultList.get(0);
            JSONObject jsonObject = JSONObject.parseObject(JSONArray.toJSONString(all));
            //todo：如果传了;是多个文本一起找，暂时只考虑一个文本
            JSONArray pointData = jsonObject.getJSONArray(ocrText);   //所有坐标
            Integer index = (Integer) data.get(UpperParamKey.INDEX);
            JSONObject pointMap = null;
            if (pointData != null && pointData.size() == 1) {   //只找到一个，直接返回
                pointMap = pointData.getJSONObject(0);
                logger.info("只找到一个，直接返回结果：{}", pointMap);
                result = new TapXYData();
                result.setX((realityX.intValue() + pointMap.getInteger("x") + pointMap.getInteger("w") / 2) / phoneEntity.getScale());
                result.setY((realityY.intValue() + pointMap.getInteger("y") + pointMap.getInteger("h") / 2) / phoneEntity.getScale());
            } else if (pointData != null && pointData.size() > 1) {   //找到多个，选择用户选择的第几个
                if (index > 0) {
                    if (index <= pointData.size()) {
//                        result =  sortPointAndGetIndexPoint(pointData, index);
                        JSONObject dataJSONObject = pointData.getJSONObject(index - 1);
                        result = new TapXYData();
                        result.setX((dataJSONObject.getInteger("x") + dataJSONObject.getInteger("w") / 2) / phoneEntity.getScale());
                        result.setY((dataJSONObject.getInteger("y") + dataJSONObject.getInteger("h") / 2) / phoneEntity.getScale());
                    } else {
                        logger.warn("实际找到{}个，选择第{}个", pointData.size(), index);
                        throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", pointData.size()));
                    }
                }
            }
        }
        return result;
    }

    private TapXYData queryClickTextHzOcrAreaNew(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String base64Str = getClickTextOcrAreaToBase64(request);
        String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
        String textByHzOcr;
        List<TapXYData> allTextPointByData = new ArrayList<>();
        try {
            textByHzOcr = HzBankOcrUtils.getTextByHzOcr(base64Str);
            allTextPointByData = HzBankOcrUtils.getAllTextPointByData(textByHzOcr, ocrText);
        } catch (Exception e) {
            logger.error("查找杭银ocr失败，原因为： {}", e);
            throw e;
        }
        if (!CollectionUtils.isEmpty(allTextPointByData)) {
            if (allTextPointByData.size() == 1) {   //只找到一个，直接返回
                result = allTextPointByData.get(0);
                logger.info("只找到一个，直接返回结果：{}", result);
            } else { //找到多个，选择用户选择的第几个
                if (index > 0) {
                    if (index <= allTextPointByData.size()) {
                        result = allTextPointByData.get(index - 1);
                    } else {
                        logger.warn("实际找到{}个，选择第{}个", allTextPointByData.size(), index);
                        throw new ExecuteException(String.format("选择元素的位置大于找到元素的个数，找到元素%s个", allTextPointByData.size()));
                    }
                }
            }
        }
        return result;
    }


    private static TapXYData sortPointAndGetIndexPoint(JSONArray data, Integer index) {
        List<JSONObject> list = new ArrayList<>();
        List<TapXYData> allPoint = new ArrayList<>();
//        data.stream().forEach(item -> System.out.println(JSONObject.toJSONString(item)));
        //y排好
        data.stream().sorted(new Comparator() {
            @Override
            public int compare(Object o1, Object o2) {
                JSONObject one = JSONObject.parseObject(JSONObject.toJSONString(o1));
                JSONObject two = JSONObject.parseObject(JSONObject.toJSONString(o2));
                return one.getIntValue("y") - two.getIntValue("y");
            }
        }).forEach(item -> list.add(JSONObject.parseObject(JSONObject.toJSONString(item))));
        JSONObject[][] plane = new JSONObject[list.size()][list.size()];
        int yIndex = 0;
        int xIndex = 0;
        for (int j = 0; j < list.size()-1; j++) {
            plane[xIndex][yIndex] = list.get(j);
            yIndex++;
            JSONObject pre = list.get(j);
            JSONObject suf = list.get(j+1);
            if (suf.getIntValue("y") - pre.getIntValue("y") > 10) {
                yIndex = 0;
                xIndex++;
            }
        }
        plane[xIndex][yIndex] = list.get(list.size() - 1);  //最后一个
        //对每一行排序
        for (int k = 0; k <= xIndex; k++) {
            List<JSONObject> lineList = new ArrayList<>();
            for (int m = 0; m < list.size(); m++) {
                if (null != plane[k][m]) {
                    lineList.add(plane[k][m]);
                } else {
                    break;
                }
            }
            lineList.stream().sorted(new Comparator<JSONObject>() {
                @Override
                public int compare(JSONObject o1, JSONObject o2) {
                    return o1.getIntValue("x") - o2.getIntValue("x");
                }
            }).forEach(item -> {
                TapXYData pointMessage = new TapXYData();
                pointMessage.setX(item.getInteger("x") + (item.getInteger("w") / 2));
                pointMessage.setY(item.getInteger("y") + (item.getInteger("h") / 2));
                allPoint.add(pointMessage);
            });
        }
        return allPoint.get(index - 1);
    }

    private String getFileBase64(File file) {
        String encode = Base64.encode(file);
        return encode;
    }

    @Override
    public void handleNodeByImage(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据图像点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            Integer waitTimeout = (Integer) data.get(UpperParamKey.WAIT_TIMEOUT);
            TapXYData tapXYData = null;
            long startTime = System.currentTimeMillis(); //当前时间
            long elapsedTime = 0; // 初始化经过的时间
            while(elapsedTime < waitTimeout * 1000 -300 && tapXYData == null) {
                tapXYData = queryImageArea(request);
                long currentTime = System.currentTimeMillis();
                elapsedTime = currentTime - startTime;
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未找到元素").withData(false);
                return;
            }
            boolean clickSuccess = false;
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据图像点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void clickTextByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
                tapXYData = useNewOcr ? queryClickTextOcrAreaNew(request) : queryClickTextOcrArea(request);
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderFailure(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void clickTextByOcrNew(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr点击控件失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean clickSuccess = false;   //点击成功
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            TapXYData tapXYData = null;
            if ("hzocr".equals(usingType)) {
                tapXYData = queryClickTextHzOcrAreaNew(request);
            } else {
                tapXYData = queryClickTextOcrAreaNew(request);
            }
            if (null == tapXYData) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到控件").withData(false);
                return;
            }
            logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
            clickSuccess = nkAgent.handleTextUiNode(tapXYData, 21);
            if (clickSuccess) {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件成功").withData(clickSuccess);
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "点击控件失败").withData(false);
            }
        } catch (Exception e) {
            logger.error("根据ocr点击控件失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void ifText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "根据ocr是否存在文本失败").withData(false);
        try {
            Map<String, Object> data = request.getData();
            String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
            if (StringUtils.isBlank(ocrText)) {
                response = CmdAutomationResponse.builderFailure(request, "ocr查找的文本不能为空！");
                return;
            }
            NKAgent nkAgent = getNkAgent();
            if (null == nkAgent) {
                response = CmdAutomationResponse.builderFailure(request, "设备与上位机连接断开");
                return;
            }
            boolean useNewOcr = Boolean.parseBoolean(SpringUtil.getProperty("nk.http-request-path.useNewOcr"));
            boolean success = useNewOcr ? queryIfTextOcrAreaNew(request) : queryIfTextOcrArea(request);
            if (!success) {
                response = CmdAutomationResponse.builderFailure(request, "未查找到文本").withData(success);
                return;
            } else {
                response = CmdAutomationResponse.builderSuccess(request, "存在文本").withData(success);
            }
        } catch (Exception e) {
            logger.error("根据ocr是否存在失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void handleApp(CmdAutomationRequest request) {
        Map<String, Object> data = request.getData();
        String appPackage = (String) data.get(UpperParamKey.APP_PACKAGE);
        String type = (String) data.get(UpperParamKey.TYPE);
        String serial = phoneEntity.getUdid();
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "app处理失败").withData(false);
        IosDebuggerServiceImpl iosService = SpringUtils.getBean(IosDebuggerServiceImpl.class);
        try {
            if ("0".equalsIgnoreCase(type)) {
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderFailure(request, "app未安装");
                    return;
                }
                logger.info("关闭ios的app:{}", appPackage);
                boolean success = iosService.terminateApp(serial, appPackage);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app关闭失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app关闭成功").withData(true);
                }
            } else if ("1".equalsIgnoreCase(type)) {
                logger.info("启动ios的app:{}", appPackage);
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderSuccess(request, "app未安装");
                    return;
                }
                boolean success = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app启动失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app启动成功").withData(true);
                }
            } else {
                logger.info("重启ios的app:{}", appPackage);
                if (!deviceHandleHelper.isAppInstalled(phoneEntity.getUdid(), appPackage)) {
                    response = CmdAutomationResponse.builderSuccess(request, "app未安装");
                    return;
                }
                boolean success = iosService.terminateApp(serial, appPackage);
                if (!success) {
                    response = CmdAutomationResponse.builderFailure(request, "app关闭失败").withData(false);
                }
                boolean successActive = deviceHandleHelper.activateApp(phoneEntity.getUdid(), appPackage);
                if (!successActive) {
                    response = CmdAutomationResponse.builderFailure(request, "app启动失败").withData(false);
                } else {
                    response = CmdAutomationResponse.builderSuccess(request, "app启动成功").withData(true);
                }
            }
        } catch (Exception e) {
            logger.error("处理app失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void pressHomeKey(CmdAutomationRequest request) {
        String serial = request.getSerial();
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "点击home键失败").withData(false);
        try {
            IosFunction iosFunction = new IosFunction(phoneEntity.getPort8100());
            JSONObject jsonObject = iosFunction.pressHome();
            response = CmdAutomationResponse.builderSuccess(request, "点击home键成功").withData(true);
        } catch (Exception e) {
            logger.error("点击home键失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }

    }

    @Override
    public void inputPasswordByOcr(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "密码输入（ocr）失败").withData(false);
        totalMap = new HashMap<>();
        index = 1;
        lastKeyBoard = "";
        preFlag = "0";
        try {
            Map<String, Object> data = request.getData();
            String password = (String) data.get(UpperParamKey.INPUT_TEXT);
            String ocrArea = getOcrAreaToBase64(request);
            if (null == ocrArea) {
                throw new ExecuteException("未找到键盘区域");
            }
            lastKeyBoard = ocrArea;
            getKeyBoardInfo(ocrArea);
            char[] passwordChars = password.toCharArray();
            for (char passwordChar : passwordChars) {
                inputPasswordChar(passwordChar, request);
            }
            response = CmdAutomationResponse.builderSuccess(request, "密码输入（ocr）成功").withData(true);
        } catch (Exception e) {
            logger.error("密码输入ocr失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    @Override
    public void getElementMoneyText(CmdAutomationRequest request) {
        CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "识别金额失败");
        String value = null;
        try {
            String imgBase64 = getOcrAreaToBase64(request);
            if (StringUtils.isBlank(imgBase64)) {
                response = CmdAutomationResponse.builderFailure(request, "未获取到ocr区域");

            }
            Map<String, Object> data = request.getData();
            String usingType = (String) data.get(UpperParamKey.USING_TYPE);
            if ("hzocr".equals(usingType)) {
                String ocrText = (String) data.get(UpperParamKey.OCR_TEXT);
                String textByHzOcr;
                try {
                    textByHzOcr = HzBankOcrUtils.getTextByHzOcr(imgBase64);
                    value = HzBankOcrUtils.getAllTextByData(textByHzOcr);
                } catch (Exception e) {
                    logger.error("查找杭银ocr失败，原因为： {}", e);
                    throw e;
                }
            } else {
                String ocrAddress = SpringUtils.getProperties("nk.http-request-path.ocrGetAllTextNum");
                try {
                    HttpHeaders headers = new HttpHeaders();
                    headers.setContentType(MediaType.APPLICATION_JSON);
                    Map<String, Object> ocrParamMap = new HashMap<>();
                    ocrParamMap.put("img_base64", imgBase64);
                    ocrParamMap.put("targets", "");
                    HttpEntity<Map> ocrEntity = new HttpEntity<>(ocrParamMap, headers);
                    List ocrResultList = null;
                    try {
                        logger.info("传参：{}", JSON.toJSONString(ocrParamMap));
                        ocrResultList = HttpUtils.doPost(ocrAddress, ocrEntity, List.class);
                        logger.info("得到的ocr结果：{}", ocrResultList);
                    } catch (Exception e) {
                        logger.error("ocr失败", e);
                    }
                    StringJoiner dataList = new StringJoiner("");
                    if (!CollectionUtils.isEmpty(ocrResultList)) {
                        if (!CollectionUtils.isEmpty(ocrResultList)) {
                            for (Object o : ocrResultList) {
                                dataList.add(((Map) o).get("text") + "");
                            }
                        }
                    }
                    value =  dataList.toString();
                } catch (Exception e) {
                    logger.error("请求ocr接口失败，接口地址：{}",ocrAddress,e);
                }
                logger.info("识别金额为：{}", value);
                if (StringUtils.isNotBlank(value)) {
                    response = CmdAutomationResponse.builderSuccess(request, "识别金额成功").withData(value);
                } else {
                    response = CmdAutomationResponse.builderFailure(request, "识别金额失败");
                }
            }
        } catch (Exception e) {
            logger.error("识别失败，原因：", e);
            response = CmdAutomationResponse.builderFailure(request, e.getMessage());
        } finally {
            sendResultToEngine(response);
        }
    }

    private void getKeyBoardInfo(String ocrArea) {
        Map<String, JSONObject> itemMap = new HashMap<>();
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        logger.debug("第{}次进来键盘base64：{}",index,ocrArea);
        paramMap.put("img_base64", ocrArea);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        logger.info("识别键盘地址为：{}", keyboardAddress);
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            boolean numKeyboard = true;
            logger.debug("ocr识别键盘结果：{}", result);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                logger.debug("第{}次进来识别键盘结果：{}",index,parseArray);
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                // orc返回数组顺序 x y w h
                String name = (String) parseArray.get(0);
                JSONObject item = new JSONObject();
                item.put("name", name);
                item.put("x", parseArray.get(1));
                item.put("y", parseArray.get(2));
                item.put("w", parseArray.get(3));
                item.put("h", parseArray.get(4));
                if (name.equals("shift")) {
                    logger.debug("第{}次进来是小写字母键盘........", index);
                    preFlag = "0";
                    numKeyboard = false;
                } else if (name.equals("highshift")) {
                    logger.debug("第{}次进来是大写字母键盘........", index);
                    preFlag = "3";
                    numKeyboard = false;
                } else if (name.equals("@")) {
                    logger.debug("第{}次进来是字符键盘...........", index);
                    preFlag = "2";
                    numKeyboard = false;
                }
                itemMap.put(name, item);
            }
            if (numKeyboard) {
                logger.debug("第{}次进来是数字界面...........", index);
                preFlag = "1";
            }
            logger.info("第{}次进来的界面标志为：{}", index, preFlag);
            totalMap.put(preFlag, itemMap);
            index++;
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
            throw new ExecuteException("键盘识别失败");
        }
    }

    private void inputPasswordChar(char passwordChar, CmdAutomationRequest request) {
        logger.info("密码字符：{}", passwordChar);
        boolean isNumber = Character.isDigit(passwordChar);
        if (isNumber) {   //数字
            if (!"1".equals(preFlag)) {  //当前不在数字页面
                logger.info("切换数字页面，密码字符：{}", passwordChar);
                exchangePage("num", request);
                preFlag = "1";
            }
            Map<String, JSONObject> numMap = totalMap.get(preFlag);
            if (null == numMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                numMap = totalMap.get(preFlag);
            }
            JSONObject item = numMap.get(String.valueOf(passwordChar));
            click(item, request);
            return;
        }
        boolean letter = Character.isLetter(passwordChar);
        if (letter) {  //字母
            if (!"0".equals(preFlag) && !"3".equals(preFlag)) {  //既不是大写字母也不是小写字母
                exchangePage("letter", request);
                preFlag = determinePage();
            }
            Map<String, JSONObject> letterMap = totalMap.get(preFlag);
            if (null == letterMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                letterMap = totalMap.get(preFlag);
            }
            boolean lowerCase = Character.isLowerCase(passwordChar);
            if (!lowerCase) {   //大写
                if ("3".equals(preFlag)) {  //当前处于大写键盘界面
                    if (totalMap.get("3") == null) {
                        getKeyBoardInfo(getOcrAreaToBase64(request));
                    }
                    JSONObject item = letterMap.get(String.valueOf(passwordChar));
                    click(item, request);  //点字母
                } else {//当前处于小写键盘界面
                    JSONObject shiftItem = letterMap.get("shift");
                    click(shiftItem, request);  //点大写
                    if (totalMap.get("3") == null) {
                        getKeyBoardInfo(getOcrAreaToBase64(request));
                    }
                    JSONObject item = totalMap.get("3").get(String.valueOf(passwordChar));
                    click(item, request);  //点字母
                    preFlag = "0";
                }
            } else {  //小写
                if ("0".equals(preFlag)) {
                    JSONObject item = totalMap.get("0").get(String.valueOf(passwordChar));
                    click(item, request);  //点字母
                } else {
                    JSONObject shiftItem = totalMap.get("1").get("highshift");
                    click(shiftItem, request);  //点大写
                    preFlag = determinePage();
                    if ("3".equals(preFlag)){//还处于大写
                        click(shiftItem, request);  //点大写
                    }
                    JSONObject item = letterMap.get(String.valueOf(passwordChar));
                    click(item, request);  //点字母
                }
            }
        } else {  //特殊字符
            if (!"2".equals(preFlag)) {    //不是特殊字符，切换到特殊字符
                exchangePage("character", request);
                preFlag = "2";//当前页面是字符界面
            }
            Map<String, JSONObject> characterMap = totalMap.get(preFlag);
            if (null == characterMap){  //还没有对应的键盘页面
                getKeyBoardInfo(lastKeyBoard);
                characterMap = totalMap.get(preFlag);
            }
            JSONObject item = characterMap.get(String.valueOf(passwordChar));
            click(item, request);
        }
    }

    private String determinePage() {
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("img_base64", lastKeyBoard);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                String name = (String) parseArray.get(0);
                if (name.equals("shift")) {
                    logger.debug("第{}次进来是小写字母键盘........", index);
                    return "0";
                } else if (name.equals("highshift")) {
                    logger.debug("第{}次进来是大写字母键盘........", index);
                    return "3";
                }
            }
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
        }
        return "0";
    }

    private void exchangePage(String flag, CmdAutomationRequest request) {
        HttpHeaders httpHeaders = new HttpHeaders();
        httpHeaders.add("Content-Type", "application/json");
        Map<String, Object> paramMap = new HashMap<>();
        paramMap.put("img_base64", lastKeyBoard);
        HttpEntity<Map> entity = new HttpEntity<>(paramMap, httpHeaders);
        String keyboardAddress = SpringUtils.getProperties("nk.http-request-path.getSafeKeyBoardNum");
        List result = null;
        try {
            result = HttpUtils.doPost(keyboardAddress, entity, List.class);
            for (Object o : JSONObject.parseArray(JSONArray.toJSONString(result.get(0)))) {
                JSONArray parseArray = JSONObject.parseArray(JSON.toJSONString(o));
                if (parseArray.size() < 5) {
                    throw new ExecuteException("键盘识别失败，ocr返回结果错误");
                }
                String name = (String) parseArray.get(0);
                if (name.equals(flag)) {
                    logger.info("切换页面flag：{}", flag);
                    JSONObject item = new JSONObject();
                    item.put("name",parseArray.get(0));
                    item.put("x", parseArray.get(1));
                    item.put("y", parseArray.get(2));
                    item.put("w", parseArray.get(3));
                    item.put("h", parseArray.get(4));
                    click(item, request);
                    break;
                }
            }
        } catch (Exception e) {
            logger.error("识别键盘失败", e);
        }
        String ocrArea = getOcrAreaToBase64(request);
        if (null != ocrArea) {
            lastKeyBoard = ocrArea;
        }
    }

    private void click(JSONObject item, CmdAutomationRequest request) {
        if (null != item) {
            CmdAutomationResponse response = CmdAutomationResponse.builderFailure(request, "密码输入（ocr）点击控件失败").withData(false);
            try {
                Map<String, Object> data = request.getData();
                Integer x = (Integer) data.get(UpperParamKey.X);
                Integer y = (Integer) data.get(UpperParamKey.Y);
                Integer screenWidth = (Integer) data.get(UpperParamKey.SCREEN_WIDTH);
                Integer screenHeight = (Integer) data.get(UpperParamKey.SCREEN_HEIGHT);
                //转换比例
                int width = phoneEntity.getScreenWidth() * phoneEntity.getScale();
                int height = phoneEntity.getScreenHeight() * phoneEntity.getScale();
                int realityWidth = width;
                int realityHeight = height;
                if (width > height) {  //宽高反了的情况，调换
                    realityWidth = height;
                    realityHeight = width;
                }
                Double realityX = x.doubleValue() / screenWidth.doubleValue() * realityWidth;
                Double realityY = y.doubleValue() / screenHeight.doubleValue() * realityHeight;

                int clickTrueX = new Double((realityX + (item.getDouble("x") + item.getDouble("w") / 2)) / phoneEntity.getScale()).intValue();
                int clickTrueY = new Double((realityY + (item.getDouble("y") + item.getDouble("h") / 2)) / phoneEntity.getScale()).intValue();
                TapXYData tapXYData = new TapXYData();
                tapXYData.setX(clickTrueX);
                tapXYData.setY(clickTrueY);
                NKAgent nkAgent = getNkAgent();
                logger.debug("实际点击的位置------->>>>>>x:{},y:{}", tapXYData.getX(), tapXYData.getY());
                nkAgent.handleTextUiNode(tapXYData, 21);
            } catch (Exception e) {
                logger.error("密码输入（ocr）点击失败，原因：", e);
                response = CmdAutomationResponse.builderFailure(request, e.getMessage());
                sendResultToEngine(response);
            }
        }
    }

    private TapXYData queryImageArea(CmdAutomationRequest request) {
        TapXYData result = null;
        Map<String, Object> data = request.getData();
        String picName = (String) data.get(UpperParamKey.IMG_URL);
        String serverAddress = SpringUtils.getProperties("nk.mobile-computer.serverAddr");  //图片下载地址
        String picDownloadAddress = SpringUtils.getProperties("nk.mobile-computer.publicDownloadFile");  //图片下载地址
        logger.info("图片下载地址{}", picDownloadAddress);
        String picDownloadUrl = serverAddress + picDownloadAddress.replace("{appUrl}", picName);
        HttpUtils.downloadFileToLocal(picDownloadUrl, System.getProperty("user.dir") + "/tempPic", picName);
        File file = new File(System.getProperty("user.dir") + "/tempPic/" + picName);
        String targetBase64 = Base64.encode(file);
        File sourceFile = ScreenShotUtils.getIOSMobileScreenShot(serial);
        String sourceBase64 = Base64.encode(sourceFile);
        if (StringUtils.isNotBlank(targetBase64) && StringUtils.isNotBlank(sourceBase64)) {
            Map<String, BigDecimal> resultPoint = null;
            String url = SpringUtils.getProperties("nk.http-request-path.imgFindArea");
            HttpHeaders headers = new HttpHeaders();
            headers.setContentType(MediaType.APPLICATION_JSON);
            Map<String, Object> param = new HashMap<>();
            param.put("sourceImage", sourceBase64);
            param.put("targetImage", targetBase64);
            param.put("matchingThreshold", 0.8);
            HttpEntity<Map> httpEntity = new HttpEntity<>(param, headers);
            try {
                URI uri = new URI(url);
                Map body = HttpUtils.doPost(uri, httpEntity, Map.class);
                Map<String, Object> bodyMap = JSONObject.parseObject(JSON.toJSONString(body)).getInnerMap();
                if (Boolean.parseBoolean(String.valueOf(bodyMap.get("success")))) {
                    resultPoint = (Map<String, BigDecimal>) bodyMap.get("data");
                }
            } catch (URISyntaxException e) {
                logger.error("找图像地址有误,:{}", e);
            } catch (Exception e) {
                logger.error("请求图像解析服务失败,参数:{},错误:{}", JSON.toJSONString(param), e);
            }
            if (null != resultPoint) {
                logger.info("步骤【{}】找到区域位置为:{}", request.getStepToken(), JSON.toJSONString(resultPoint));
                try {
                    byte[] bytes = new BASE64Decoder().decodeBuffer(targetBase64);
                    BufferedImage read = ImageIO.read(new ByteArrayInputStream(bytes));
                    Integer width = read.getWidth();
                    Integer height = read.getHeight();
                    int clickTrueX = new Double((resultPoint.get("x").doubleValue() + width.doubleValue() / 2) / phoneEntity.getScale()).intValue();
                    int clickTrueY = new Double((resultPoint.get("y").doubleValue() + height.doubleValue() / 2) / phoneEntity.getScale()).intValue();
                    result = new TapXYData();
                    result.setX(clickTrueX);
                    result.setY(clickTrueY);
                } catch (IOException e) {
                    logger.error("转换base64图片失败：{}", e);
                }
            } else {
                logger.warn("步骤【{}】未找到对应的图像", request.getStepToken());
            }
        } else {
            logger.warn("步骤【{}】下载的图像或者截图为空...............", request.getStepToken());
        }
        return result;
    }

    private NKAgent getNkAgent() {
        IosDeviceInitThread iosInitThread = IOSDeviceManager.getInstance().getIosInitThread(phoneEntity.getUdid());
        if (null == iosInitThread || iosInitThread.isInterrupted() || !iosInitThread.isAlive()) {
            return null;
        }
        return iosInitThread.getNkAgent();
    }

    private Map<String, Object> getAppDetail(String appId) throws URISyntaxException {
        Map<String, Object> data = null;
        String path = SpringUtils.getProperties("nk.mobile-computer.publicFindAppInfo");
        String server = SpringUtils.getProperties("nk.mobile-computer.serverAddr");
        JSONObject result = HttpUtils.doGet(new URI(server + path + "?appId=" + appId), JSONObject.class);
        logger.debug("查询app的结果：{}", JSONObject.toJSONString(result));
        if (result.getBoolean("success")) {
            data = result.getJSONObject("data");
        }
        return data;
    }

    /**
     * 安装app
     *
     * @param appId
     * @param appName
     */
    private boolean installApp(String appId, String appName) {
        String appPath = "";
        logger.info("Begin to download app[{}] to upperComputer...", appName);
        try {
            appPath = deviceHandleHelper.downloadAppToLocal(appId, appName);
        } catch (Exception e) {
            logger.error("下载app[{}]出错，app名字：{}", appId, appName, e);
            return false;
        }
        if (StringUtils.isBlank(appPath)) {
            return false;
        }
        logger.info("download app[{}] finish,start install ............", appName);
        boolean installSuccess = false;
        try {
            installSuccess = deviceHandleHelper.installApp(phoneEntity.getUdid(), appPath);
        } catch (Exception e) {
            logger.error("安装app[{}]出错，app名字：{}", appId, appName, e);
            return false;
        }
        logger.info("App[{}] on device[{}] installed finish,success:{}", appName, phoneEntity.getUdid(), installSuccess);
        return installSuccess;
    }
}
