ESP32实现iOS模拟点击

ESP32实现iOS模拟点击

相关项目

https://github.com/T-vK/ESP32-BLE-Mouse

https://github.com/sobrinho/ESP32-BLE-Abs-Mouse

实现过程

ESP32-BLE-Mouse是模拟鼠标，iOS设备上可以使用(部分设备需要开启辅助触控，设置-辅助功能-触控-辅助触控)。但不能根据屏幕坐标精准点击。原因是系统在计算鼠标移动时会计算加速度之类的，尝试多次小范围移动，依然无法精确点击。

iOS设备并不支持ESP32-BLE-Abs-Mouse，ESP32-BLE-Abs-Mouse是模拟触控笔，部分安卓设备可以使用，可以精确的点击指定坐标。

ESP32源码：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
#include <WiFi.h>
#include <WebServer.h>
#include <BleMouse.h>
#include <math.h>

BleMouse bleMouse;
#define LED_BUILTIN 2

const char *ssid = "xxxx";
const char *password = "xxxx";

// 静态地址、网关、子网掩码
IPAddress local_IP(192, 168, 31, 112);
IPAddress gateway(192, 168, 31, 1);
IPAddress subnet(255, 255, 255, 0);

/* 创建一个web服务器对象，使用80端口，HTTP网络服务器标准端口号即为80 */
WebServer esp8266_server(80);

/* 处理访问网站根目录“/”的访问请求 */
void handleRoot()
{
  esp8266_server.send(200, "text/plain", "OK"); // NodeMCU将调用此函数。
}

void handle_click()
{
  esp8266_server.send(200, "text/plain", "OK");
  if (bleMouse.isConnected())
  {
    bleMouse.click(MOUSE_LEFT);
  }
}
void handle_move3()
{

  int x = esp8266_server.arg("x").toInt();
  int y = esp8266_server.arg("y").toInt();

  if (bleMouse.isConnected() && x > -1 && y > -1)
  {
    int a = -1000;
    int b = 1000;
    signed char xStep = 0;
    signed char yStep = 0;
    delay(10);
    while (a != 0 || b != 0)
    {
      xStep = (a > 127) ? 127 : (a < -128) ? -128
                                           : (signed char)a;
      yStep = (b > 127) ? 127 : (b < -128) ? -128
                                           : (signed char)b;

      // Move the mouse by the current step
      bleMouse.move(xStep, yStep);

      // Update x and y to reflect the remaining distance to move
      a -= xStep;
      b -= yStep;

      // Small delay to ensure smooth movement
      delay(70);
    }

    if (bleMouse.isConnected())
    {
      while (x != 0 || y != 0)
      {
        // Determine step size within the range of -128 to 127
        xStep = (x > 1) ? 1 : x;
        yStep = (y > 1) ? 1 : y;

        // Move the mouse by the current step
        bleMouse.move(xStep, yStep);

        // Update x and y to reflect the remaining distance to move
        x -= xStep;
        y -= yStep;
        delay(1);
      }
    }
    esp8266_server.send(200, "text/plain", "OK");
  }
}

#define SCREEN_WIDTH 580
#define SCREEN_HEIGHT 1158
#define MOUSE_DPI 500
#define MAX_MOVE_STEP 127
void handle_move()
{

  int x = esp8266_server.arg("x").toInt();
  int y = esp8266_server.arg("y").toInt();

  int targetX = esp8266_server.arg("x").toInt();
  int targetY = esp8266_server.arg("y").toInt();

  if (bleMouse.isConnected() && x > -1 && y > -1)
  {
    int a = -1000;
    int b = 1000;
    signed char xStep = 0;
    signed char yStep = 0;
    signed char max_move_step = 16;
    //    bleMouse.move(0, 0, -1);
    delay(10);
    while (a != 0 || b != 0)
    {
      xStep = (a > 127) ? 127 : (a < -128) ? -128: (signed char)a;
      yStep = (b > 127) ? 127 : (b < -128) ? -128: (signed char)b;

      // Move the mouse by the current step
      bleMouse.move(xStep, yStep);

      // Update x and y to reflect the remaining distance to move
      a -= xStep;
      b -= yStep;

      // Small delay to ensure smooth movement
      delay(70);
    }

    //    bleMouse.move(0, 0);
    
    delay(100);
    while (1)
    {
      if (y == 0)
      {
        break;
      }
      yStep = (y > max_move_step) ? max_move_step : (signed char)y;

      bleMouse.move(0, yStep * -1);
      y -= yStep;
      delay(10);
    }
    delay(100);
    while (1)
    {
      if (x == 0)
      {
        break;
      }
      xStep = (x > max_move_step) ? max_move_step : (signed char)x;
      bleMouse.move(xStep, 0);
      x -= xStep;
      delay(10);
    }
  }
  esp8266_server.send(200, "text/plain", "OK");
}

void handle_movex()
{

  int x = esp8266_server.arg("x").toInt();
  int y = esp8266_server.arg("y").toInt();

  int targetX = esp8266_server.arg("x").toInt();
  int targetY = esp8266_server.arg("y").toInt();

  if (bleMouse.isConnected() && x > -1 && y > -1)
  {
    int a = -1000;
    int b = 1000;
    signed char xStep = 0;
    signed char yStep = 0;
    signed char max_move_step = 16;
    //    bleMouse.move(0, 0, -1);
    delay(10);
    while (a != 0 || b != 0)
    {
      xStep = (a > 127) ? 127 : (a < -128) ? -128: (signed char)a;
      yStep = (b > 127) ? 127 : (b < -128) ? -128: (signed char)b;

      // Move the mouse by the current step
      bleMouse.move(xStep, yStep);

      // Update x and y to reflect the remaining distance to move
      a -= xStep;
      b -= yStep;

      // Small delay to ensure smooth movement
      delay(70);
    }

    //    bleMouse.move(0, 0);
    delay(100);
    while (1)
    {
      if (x == 0 && y == 0)
      {
        break;
      }
      //      xStep = (x > 127) ? 127 :(signed char)x;
      //      yStep = (y > 127) ? 127 :(signed char)y;

      xStep = (x > max_move_step) ? max_move_step : (signed char)x;
      yStep = (y > max_move_step) ? max_move_step : (signed char)y;

      bleMouse.move(xStep, yStep * -1);
      x -= xStep;
      y -= yStep;
      delay(10);
    }
  }
  esp8266_server.send(200, "text/plain", "OK");
}

/* 设置处理404情况的函数'handleNotFound' */
void handleNotFound()
{
  esp8266_server.send(404, "text/plain", "404: Not found");
}

void setup()
{
  Serial.begin(115200);
  pinMode(LED_BUILTIN, OUTPUT);
  bleMouse.begin();

  WiFi.config(local_IP, gateway, subnet); // 设置静态IP
  WiFi.mode(WIFI_STA);                    // 设置Wifi工作模式为STA,默认为AP+STA模式
  WiFi.begin(ssid, password);             // 通过wifi名和密码连接到Wifi
  int i = 0;                              // 检查WiFi是否连接成功
  while (WiFi.status() != WL_CONNECTED)   // WiFi.status()函数的返回值是由NodeMCU的WiFi连接状态所决定的。
  {                                       // 如果WiFi连接成功则返回值为WL_CONNECTED
    delay(1000);                          // 此处通过While循环让NodeMCU每隔一秒钟检查一次WiFi.status()函数返回值
  }

  /* 3. 开启http网络服务器功能 */
  esp8266_server.begin(); // 启动http网络服务器
  esp8266_server.on("/", handleRoot);
  esp8266_server.on("/click", handle_click);
  esp8266_server.on("/move", handle_move);
  // esp8266_server.on("/move2", handle_move2);
  esp8266_server.onNotFound(handleNotFound); // 设置无法响应时的处理函数
}

void loop()
{
  esp8266_server.handleClient();
  if (bleMouse.isConnected())
  {
    digitalWrite(LED_BUILTIN, HIGH);
  }
  else
  {
    digitalWrite(LED_BUILTIN, LOW);
  }
}

为了尽可能精确的点击，我尝试先移动Y方向，再移动X方向，并采集鼠标移动距离与实际移动距离的数据。

采集数据需要创建一个PWA应用，这样可以几乎采集全屏的点击事件（状态栏等位置无法采集点击事件）。

每个不同的设备采集一份数据，就可以通过计算得到点击屏幕坐标映射后的鼠标坐标，勉强实现“精准”点击。

为了实现自动化脚本，需要借助爱思助手的投屏功能，PC端采集并检测屏幕数据，再用python脚本发送点击事件，即可实现iOS自动化脚本。