AWTK 嵌入式Linux平台实现多点触控缩放旋转以及触点丢点问题解决

前言

最近涉及海图的功能交互，多点触摸又开始找麻烦。

在PC/Web平台awtk是通过底层的sdl2库来实现多点触摸，但是在嵌入式Linux平台，可能是考虑到性能原因，awtk并没有采用sdl库来做事件处理，而是自己实现一个awtk-linux-fb来做适配，多点触摸的相关逻辑必须自己适配。

去年11月的时候，匆忙赶工，自己适配了一份tslib的代码，思路是循环线程内读取触点数据后，直接调用awtk自带的一个multi_gesture.inc文件去计算距离和旋转角度，在应用层注册EVT_MULTI_GESTURE事件。

static ret_t tslib_dispatch_one_event(run_info_t* info) {
  #ifdef MT_TOUCH_ENABLE
  int ret = -1;
  if (info->ts != NULL) {
    ret = ts_read_mt(info->ts, info->samp_mt, info->max_slots, info->read_samples);
  }

  uint8_t event_number = 0, down_number = 0;
  ...

  int max_slots = info->max_slots;
  // note: down, up only trigger once(ET), move will be keep trigger(LT)
  for (int i = 0; i < max_slots; i++) {
    // printf(YELLOW "sample %d - %ld.%06ld -" RESET " (slot %d) %6d %6d %6d\n",
    // 0,
    // info->samp_mt[0][i].tv.tv_sec,
    // info->samp_mt[0][i].tv.tv_usec,
    // info->samp_mt[0][i].slot,
    // info->samp_mt[0][i].x,
    // info->samp_mt[0][i].y,
    // info->samp_mt[0][i].pressure);

    if (!(info->samp_mt[0][i].valid & TSLIB_MT_VALID)){
      //for boards that no clear pressure when slot is invaild, need to do it manually
      if(!s_points[i].type == EVT_MULTI_TOUCH_DOWN){
        info->samp_mt[0][i].pressure = 0;
      }
      continue;
    }
    event_number++;
  }
  
  for (int i = 0; i < CT_MAX_TOUCH; i++){
      s_points[i].touch_id = 0;
      s_points[i].finger_id = i;
      if(info->samp_mt[0][i].pressure > 0) {
        down_number++;
        s_points[i].type = (s_points[i].x == info->samp_mt[0][i].x && s_points[i].y == info->samp_mt[0][i].y) ? EVT_MULTI_TOUCH_DOWN : EVT_MULTI_TOUCH_MOVE;
        s_points[i].x = info->samp_mt[0][i].x;
        s_points[i].y = info->samp_mt[0][i].y;
      }
      else{
        s_points[i].type = EVT_MULTI_TOUCH_UP;
        s_points[i].x = 0;
        s_points[i].y = 0;
      }   
  }
    
  if(event_number == 1 && (info->last_down_number == 0 || info->last_down_number == 1)){
    //单点触摸
   ...
    tslib_dispatch(info);
  }
  else{
    //多点触摸
    main_loop_t *loop = (main_loop_t*)info->dispatch_ctx;
    //multi_gesture.inc提供的函数
    multi_gesture_post_event_from_points(loop, touch_points, down_number, s_points);
  }
...

后面实测multi_gesture.inc完全不堪用，机器上双指缩放判定经常失准，双指拉大，会出现一会放大一会缩小的情况，multi_gesture.inc的业务代码我没有细看，但我估计逻辑肯定是有问题的，至少在现在的机器上无法使用。

适配层设计

只能寻找新的触摸方案了，这时候老板指出来我对触摸屏的理解有误，我一开始的想法是手指一触摸屏幕，立刻会在应用层生成一份即时的完整的触点数组数据对象，通过这个对象来获取各触点坐标，状态（按下，移动，弹起），按下手指的数量。后面才知道，触摸屏是逐个去解析按下的各个手指的数据然后处理的，并不是一次就能全部读取到，也就是说，我想像的这种数据对象的组装只能到应用层去做而不是适配层去做，之前的思路相当于把解析和业务逻辑都在适配层做，结果业务一复杂就变得不敷使用了。

照着这种思路，应该是适配层只需要把所有手指的数据一个个上报上去，业务层实现实际逻辑，这样适配层的代码逻辑会更加简单，那么AWTK有没有代表单个手指的数据？

查阅AWTK代码, 发现有一个touch_event正好符合要求，很幸运也很讽刺，awtk官方在去年12月4日的更新里面已经支持在awtk-linux-fb对touch_event事件上报，还在awtk-web整了一个多点触摸的演示demo，里面的finger_status_t已经把我设想的数据对象的功能给完成了，我累死累活自己适配了一版触摸，结果人家一个月后自己就整好了一个更好的方案，我还一直没有注意到，惭愧。。。

不过项目紧急，有现成的轮子直接拿来用总是好事，重改适配层，这次只需要把ts_read_mt读到的事件封装成touch_event事件直接丢给队列即可：

#define CT_MAX_TOUCH 10
static multi_touch_point_event_t s_points[CT_MAX_TOUCH];

ret_t my_main_loop_post_touch_event(main_loop_t* l, event_type_t event_type, touch_event_t* event) {
  event_queue_req_t r;
  touch_event_t evt;
  main_loop_simple_t* loop = (main_loop_simple_t*)l;
  
  memset(&r, 0x00, sizeof(r));
  memset(&evt, 0x00, sizeof(multi_gesture_event_t));
  return_value_if_fail(loop != NULL, RET_BAD_PARAMS);

  memcpy(&evt, event, sizeof(touch_event_t));

  evt.e.type = event_type;
  evt.e.time = time_now_ms();
  evt.e.size = sizeof(touch_event_t);
  r.touch_event = evt;

  return main_loop_queue_event(l, &r);
}


static ret_t tslib_dispatch_one_event(run_info_t* info) {
	....
  // note: down, up only trigger once(ET), move will be keep trigger(LT)
  for (int i = 0; i < max_slots; i++) {
    if (!(info->samp_mt[0][i].valid & TSLIB_MT_VALID)){
      continue;
    }

    if(info->samp_mt[0][i].pressure > 0){
      down_number++;
      if(s_points[i].type == 0 || s_points[i].type == EVT_TOUCH_UP){
        s_points[i].type = EVT_TOUCH_DOWN;
      }
      else{
        s_points[i].type = EVT_TOUCH_MOVE;
      }
    } else{
      s_points[i].type = EVT_TOUCH_UP;
    }
    
    s_points[i].finger_id = info->samp_mt[0][i].slot;
    s_points[i].x = info->samp_mt[0][i].x;
    s_points[i].y = info->samp_mt[0][i].y;
    event_number++;

    main_loop_t *loop = (main_loop_t*)info->dispatch_ctx;
    touch_event_t touch_event;
    touch_event_init(&touch_event, s_points[i].type, NULL, 0, info->samp_mt[0][i].slot, info->samp_mt[0][i].x / (double)info->max_x, info->samp_mt[0][i].y / (double)info->max_y, info->samp_mt[0][i].pressure);
    my_main_loop_post_touch_event(loop, s_points[i].type, &touch_event);
  
  ...

触点丢失事件

乐极生悲，接下来的一个坑卡了一周都无法解决, 发现应用程序一旦负担比较高，比如渲染复杂图形或者事件处理极为频繁的时候，touch_up的事件经常会被打断。最后问了微信群，才知道是awtk的活动队列处理不过来导致丢事件，把main_loop_simple.h的MAIN_LOOP_QUEUE_SIZE宏从默认的20调高到100，重新编译awtk-linux-fb和应用程序后，问题解决。

对于之前的那个测试demo, 它管理了一个触点对象数组s_fingers_status，一旦检测到手指按下就会生成一个finger_status_t类型的触点对象并加入到s_fingers_status 中，finger_status_t内部也维护了一个point数组，在手指移动的时候就会将手指移动的即时坐标数据加入到数组中，用于on_paint事件的画线测试，然后在手指弹起时，该触点对象会被从s_fingers_status中移除并销毁，如果touch_up事件不上报的话这个事件就会一直留在s_fingers_status里面，导致无法正常获取正确的按下手指数量，出问题的手指对象也会一直停留在move的状态，无法恢复。

on_touch_up: 3 size=0
on_touch_down : 1 size=1 460 103
on_touch_down : 3 size=2 591 214
on_touch_move: 1 size=2 459 105
on_touch_move: 3 size=2 591 214
on_touch_move: 1 size=2 459 112
on_touch_move: 3 size=2 591 219
on_touch_move: 1 size=2 458 121
on_touch_move: 3 size=2 590 225
on_touch_move: 1 size=2 458 139
on_touch_move: 3 size=2 589 237
on_touch_move: 1 size=2 459 162
on_touch_move: 3 size=2 587 251
on_touch_down : 2 size=3 295 112
on_touch_move: 2 size=3 295 113
on_touch_move: 2 size=3 296 118
on_touch_move: 2 size=3 298 125
on_touch_move: 2 size=3 302 143
on_touch_move: 1 size=3 461 193
on_touch_move: 2 size=3 308 166
on_touch_move: 3 size=3 585 272
on_touch_move: 1 size=3 464 228
on_touch_move: 2 size=3 318 200
on_touch_move: 3 size=3 582 296
on_touch_down : 0 size=4 137 279
on_touch_move: 0 size=4 137 280
on_touch_move: 0 size=4 144 294
on_touch_move: 0 size=4 154 312
on_touch_move: 0 size=4 173 348
on_touch_move: 0 size=4 195 394
on_touch_move: 1 size=4 468 268
on_touch_move: 1 size=4 493 388
on_touch_move: 2 size=4 378 391
on_touch_move: 1 size=4 506 420
on_touch_up: 2 size=3
on_touch_move: 1 size=3 518 447
on_touch_move: 1 size=3 525 466
on_touch_move: 1 size=3 525 481
on_touch_up: 1 size=2

之前没找到问题根源的时候就隐约觉得问题跟机器的性能有关，因为测试demo在公司的开发板上两指到五指都正常，但是到了自己住处, 用百问网的imx6ull开发板一测试很快就出现丢点问题，后面又在适配层尝试限制touch_move事件的频率，改为30ms上报一次touch_move事件，发现有一定的降低丢点概率的效果（手指一多还是会丢点），但是仍旧未往事件队列本身的处理能力去设想，更没有想到在awtk库里面可以调这个事件队列大小，GUI开发还有很多基础知识需要完善。

最终代码

tslib_thread.c

/**
 * File:   tslib_thread.c
 * Author: AWTK Develop Team
 * Brief:  thread to handle touch screen events
 *
 * Copyright (c) 2018 - 2025 Guangzhou ZHIYUAN Electronics Co.,Ltd.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * License file for more details.
 *
 */

/**
 * History:
 * ================================================================
 * 2018-09-07 Li XianJing <xianjimli@hotmail.com> created
 *
 */

#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include "tslib.h"
#include "tkc/mem.h"
#include "tkc/utils.h"
#include "tkc/thread.h"
#include "base/keys.h"

#include "tslib_thread.h"
#include <linux/input.h>
#include "multi_gesture.inc"


typedef struct _run_info_t {
  int32_t max_x;
  int32_t max_y;
  struct tsdev* ts;
  void* dispatch_ctx;
  char* filename;
  input_dispatch_t dispatch;

  event_queue_req_t req;
  struct ts_sample_mt **samp_mt;
  struct input_absinfo slot;

  int32_t user_slots;
  int32_t max_slots;
  int read_samples;
  int last_down_number;
} run_info_t;

#define RESET   "\033[0m"
#define RED     "\033[31m"
#define GREEN   "\033[32m"
#define BLUE    "\033[34m"
#define YELLOW  "\033[33m"

#define CT_MAX_TOUCH 10
static multi_gesture_touch_points_t* touch_points = NULL;
static multi_touch_point_event_t s_points[CT_MAX_TOUCH];

static ret_t tslib_dispatch(run_info_t* info) {
  ret_t ret = RET_FAIL;
  char message[MAX_PATH + 1] = {0};
  // tk_snprintf(message, sizeof(message) - 1, "ts[%s]", info->filename);

  ret = info->dispatch(info->dispatch_ctx, &(info->req), message);
  info->req.event.type = EVT_NONE;

  return ret;
}




ret_t my_main_loop_post_touch_event(main_loop_t* l, event_type_t event_type, touch_event_t* event) {
  event_queue_req_t r;
  touch_event_t evt;
  main_loop_simple_t* loop = (main_loop_simple_t*)l;
  
  memset(&r, 0x00, sizeof(r));
  memset(&evt, 0x00, sizeof(multi_gesture_event_t));
  return_value_if_fail(loop != NULL, RET_BAD_PARAMS);

  memcpy(&evt, event, sizeof(touch_event_t));

  evt.e.type = event_type;
  evt.e.time = time_now_ms();
  evt.e.size = sizeof(touch_event_t);
  r.touch_event = evt;

  return main_loop_queue_event(l, &r);
}



static ret_t tslib_dispatch_one_event(run_info_t* info) {
  int ret = -1;
  if (info->ts != NULL) {
    ret = ts_read_mt(info->ts, info->samp_mt, info->max_slots, info->read_samples);
  }

  uint8_t event_number = 0, down_number = 0;
  event_queue_req_t* req = &(info->req);

  if (ret == 0) {
    log_warn("%s:%d: get tslib data failed, filename=%s\n", __func__, __LINE__, info->filename);
    sleep(1);
    return RET_OK;
  } else if (ret < 0) {
    sleep(2);
    if (access(info->filename, R_OK) == 0) {
      if (info->ts != NULL) {
        ts_close(info->ts);
      }
      info->ts = ts_open(info->filename, 0);
      return_value_if_fail(info->ts != NULL, RET_OK);
      ts_config(info->ts);

      if (info->ts == NULL) {
        log_debug("%s:%d: open tslib failed, filename=%s\n", __func__, __LINE__, info->filename);
        perror("print tslib: ");
      } else {
        log_debug("%s:%d: open tslib successful, filename=%s\n", __func__, __LINE__,
                  info->filename);
      }
    }
    return RET_OK;
  }

  int max_slots = info->max_slots;
  // note: down, up only trigger once(ET), move will be keep trigger(LT)
  for (int i = 0; i < max_slots; i++) {
    if (!(info->samp_mt[0][i].valid & TSLIB_MT_VALID)){
      continue;
    }

    // printf(YELLOW "BSP sample %d - %ld.%06ld -" RESET " (slot %d) %6d %6d %6d\n",
    // 0,
    // info->samp_mt[0][i].tv.tv_sec,
    // info->samp_mt[0][i].tv.tv_usec,
    // info->samp_mt[0][i].slot,
    // info->samp_mt[0][i].x,
    // info->samp_mt[0][i].y,
    // info->samp_mt[0][i].pressure);

    if(info->samp_mt[0][i].pressure > 0){
      down_number++;
      if(s_points[i].type == 0 || s_points[i].type == EVT_TOUCH_UP){
        s_points[i].type = EVT_TOUCH_DOWN;
      }
      else{
        s_points[i].type = EVT_TOUCH_MOVE;
      }
    } else{
      s_points[i].type = EVT_TOUCH_UP;
    }
    
    s_points[i].finger_id = info->samp_mt[0][i].slot;
    s_points[i].x = info->samp_mt[0][i].x;
    s_points[i].y = info->samp_mt[0][i].y;
    event_number++;

    main_loop_t *loop = (main_loop_t*)info->dispatch_ctx;
    touch_event_t touch_event;
    touch_event_init(&touch_event, s_points[i].type, NULL, 0, info->samp_mt[0][i].slot, info->samp_mt[0][i].x / (double)info->max_x, info->samp_mt[0][i].y / (double)info->max_y, info->samp_mt[0][i].pressure);
    my_main_loop_post_touch_event(loop, s_points[i].type, &touch_event);
  
  
    // print debug
    // char *msg = "down";
    // if(s_points[i].type == EVT_TOUCH_MOVE){
    //   msg = "move";
    // }
    // else if(s_points[i].type == EVT_TOUCH_UP){
    //   msg = "up";
    // }
    // printf("slot %d %s at (%d %d) press %d\r\n", 
    //                             s_points[i].finger_id, 
    //                             msg,
    //                             s_points[i].x, 
    //                             s_points[i].y, 
    //                             info->samp_mt[0][i].pressure);   
                                                 
  }

  // printf("down number: %d\r\n", down_number);
 
  if(event_number == 1  && (info->last_down_number == 0 || info->last_down_number == 1)){
    struct ts_sample_mt e = info->samp_mt[0][0];
    req->event.type = EVT_NONE;
    req->pointer_event.x = e.x;
    req->pointer_event.y = e.y;

    // log_debug("%s%d: e.pressure=%d x=%d y=%d ret=%d\n", __func__, __LINE__, e.pressure, e.x, e.y,
    //           ret);

    if (e.pressure > 0) {
      if (req->pointer_event.pressed) {
        req->event.type = EVT_POINTER_MOVE;
      } else {
        req->event.type = EVT_POINTER_DOWN;
        req->pointer_event.pressed = TRUE;
      }
    } else {
      if (req->pointer_event.pressed) {
        req->event.type = EVT_POINTER_UP;
      }
      req->pointer_event.pressed = FALSE;
    }

    info->last_down_number = down_number;
    return tslib_dispatch(info);
  }
  
  info->last_down_number = down_number;
  return 0;
}

void* tslib_run(void* ctx) {
  run_info_t info = *(run_info_t*)ctx;
  if (info.ts == NULL) {
    log_debug("%s:%d: open tslib failed, filename=%s\n", __func__, __LINE__, info.filename);
  } else {
    log_debug("%s:%d: open tslib successful, filename=%s\n", __func__, __LINE__, info.filename);
  }

  TKMEM_FREE(ctx);
  while (tslib_dispatch_one_event(&info) == RET_OK) {
  };
  ts_close(info.ts);
  TKMEM_FREE(info.filename);

  return NULL;
}

static run_info_t* info_dup(run_info_t* info) {
  run_info_t* new_info = TKMEM_ZALLOC(run_info_t);

  *new_info = *info;

  return new_info;
}



tk_thread_t* tslib_thread_run(const char* filename, input_dispatch_t dispatch, void* ctx,
                              int32_t max_x, int32_t max_y) {
  run_info_t info;
  tk_thread_t* thread = NULL;
  return_value_if_fail(filename != NULL && dispatch != NULL, NULL);

  memset(&info, 0x00, sizeof(info));

  info.max_x = max_x;
  info.max_y = max_y;
  info.dispatch_ctx = ctx;
  info.dispatch = dispatch;
  info.ts = ts_open(filename, 0);
  info.filename = tk_strdup(filename);

  if (info.ts != NULL) {
    ts_config(info.ts);
  }
///
  struct tsdev *ts = info.ts;
  info.read_samples = 1;
	info.max_slots = CT_MAX_TOUCH;
  printf("max_x %d max_y %d TOUCH MAX SLOT=%d\r\n", info.max_x, info.max_y, info.max_slots);
  info.samp_mt = malloc(info.read_samples * sizeof(struct ts_sample_mt *));
  if (!info.samp_mt) {
    printf("create samp_mt failed\r\n");
    ts_close(ts);
    return NULL;
  }
  for (int i = 0; i < info.read_samples; i++) {
    info.samp_mt[i] = calloc(info.max_slots, sizeof(struct ts_sample_mt));
    if (!info.samp_mt[i]) {
      printf("create samp_mt[%d] failed\r\n", i);
      for (i--; i >= 0; i--)
        free(info.samp_mt[i]);
      free(info.samp_mt);
      ts_close(ts);
      return NULL;
    }
  }

  /* 创建不可识别手指类型的多点触控句柄 */
  touch_points = multi_gesture_touch_points_create(15);
  memset(s_points, 0x0, sizeof(multi_touch_point_event_t) * CT_MAX_TOUCH);
/

  thread = tk_thread_create(tslib_run, info_dup(&info));
  if (thread != NULL) {
    tk_thread_start(thread);
  } else {
    multi_gesture_gesture_touch_points_destroy(touch_points);
    if (info.samp_mt) {
        for (int i = 0; i < info.read_samples; i++) {
            free(info.samp_mt[i]);
        }
        free(info.samp_mt);
    }
    
    ts_close(info.ts);
    TKMEM_FREE(info.filename);
  }

  return thread;
}

练习demo地址

https://gitee.com/tracker647/awtk-practice/tree/master/MapImageTouchZoomTest

为了记录做业务学到的东西，我额外写了个demo来演示手指缩放和旋转的操作，采用vgcanvas矢量画布来实现旋转和缩放的效果，原本我想直接从网上下载个地图图片然后用vgcanvas_draw_image绘制，来表示地图的旋转和缩放，但是不知道是不是imx6ull本身图形性能太拉，一在on_paint函数调用vgcanvas_draw_image事件整个应用fps就会下降至1，根本无法正常操作图片，最后只好放弃，改成画一个红色矩形来演示效果: