Add GUI.wait_key, GUI.get_key to python bindings.

examples/keyboard.py

@@ -0,0 +1,32 @@
+import taichi as ti
+import sys
+
+x, y = 0.5, 0.5
+delta = 0.01
+
+gui = ti.GUI("Keyboard", res=(400, 400))
+
+while True:
+  while gui.has_key_event():
+    e = gui.get_key_event()
+    if e.type == ti.GUI.RELEASE:
+      continue
+    if e.key == ti.GUI.ESCAPE:
+      sys.exit()
+    elif e.key == ti.GUI.RMB:
+      x, y = e.pos[0], e.pos[1]
+
+  if gui.is_pressed(ti.GUI.LEFT, 'a'):
+    x -= delta
+  if gui.is_pressed(ti.GUI.RIGHT, 'd'):
+    x += delta
+  if gui.is_pressed(ti.GUI.UP, 'w'):
+    y += delta
+  if gui.is_pressed(ti.GUI.DOWN, 's'):
+    y -= delta
+  if gui.is_pressed(ti.GUI.LMB):
+    pos = gui.get_cursor_pos()
+    x, y = pos[0], pos[1]
+
+  gui.circle((x, y), 0xffffff, 8)
+  gui.show()

python/taichi/misc/gui.py

@@ -2,6 +2,30 @@
 import numpy as np
 
 class GUI:
+
+  class Event:
+    pass
+
+  SHIFT = 'Shift'
+  ALT = 'Alt'
+  CTRL = 'Control'
+  ESCAPE = 'Escape'
+  RETURN = 'Return'
+  TAB = 'Tab'
+  BACKSPACE = 'BackSpace'
+  SPACE = ' '
+  UP = 'Up'
+  DOWN = 'Down'
+  LEFT = 'Left'
+  RIGHT = 'Right'
+  CAPSLOCK = 'Caps_Lock'
+  MOTION = 'Motion'
+  LMB = 'LMB'
+  MMB = 'MMB'
+  RMB = 'RMB'
+  RELEASE = False
+  PRESS = True
+
   def __init__(self, name, res=512, background_color=0x0):
     import taichi as ti
     self.name = name
@@ -11,6 +35,7 @@ def __init__(self, name, res=512, background_color=0x0):
     self.core = ti.core.GUI(name, ti.veci(*res))
     self.canvas = self.core.get_canvas()
     self.background_color = background_color
+    self.key_pressed = set()
     self.clear()
 
   def clear(self, color=None):
@@ -89,6 +114,45 @@ def show(self, file=None):
       self.core.screenshot(file)
     self.clear(self.background_color)
 
+  def has_key_event(self):
+    return self.core.has_key_event()
+
+  def get_key_event(self):
+    self.core.wait_key_event()
+    e = GUI.Event()
+    e.key = self.core.get_key_event_head_key()
+    e.type = self.core.get_key_event_head_type()
+    e.pos = self.core.get_key_event_head_pos()
+    e.modifier = []
+    for mod in ['Shift', 'Alt', 'Control']:
+      if self.is_pressed(mod):
+        e.modifier.append(mod)
+    if e.type == GUI.PRESS:
+      self.key_pressed.add(e.key)
+    else:
+      self.key_pressed.discard(e.key)
+    self.core.pop_key_event_head()
+    return e
+
+  def is_pressed(self, *keys):
+    for key in keys:
+      if key in ['Shift', 'Alt', 'Control']:
+        if key + '_L' in self.key_pressed or key + '_R' in self.key_pressed:
+          return True
+      elif key in self.key_pressed:
+        return True
+    else:
+      return False
+
+  def get_cursor_pos(self):
+    return self.core.get_cursor_pos()
+
+  def wait_key():
+    while True:
+      key, type = self.get_key_event()
+      if type == GUI.PRESS:
+        return key
+
 def rgb_to_hex(c):
   to255 = lambda x: min(255, max(0, int(x * 255)))
-  return 65536 * to255(c[0]) + 256 * to255(c[1]) + to255(c[2])
+  return 65536 * to255(c[0]) + 256 * to255(c[1]) + to255(c[2])

taichi/gui/win32.cpp

@@ -11,6 +11,55 @@
 
 std::map<HWND, taichi::GUI *> gui_from_hwnd;
 
+static std::string lookup_keysym(WPARAM wParam, LPARAM lParam)
+{
+  int key = wParam;
+  switch (key) {
+  /*** http://kbdedit.com/manual/low_level_vk_list.html ***/
+  case VK_LEFT:
+    return "Left";
+  case VK_RIGHT:
+    return "Right";
+  case VK_UP:
+    return "Up";
+  case VK_DOWN:
+    return "Down";
+  case VK_TAB:
+    return "Tab";
+  case VK_RETURN:
+    return "Return";
+  case VK_BACK:
+    return "BackSpace";
+  case VK_ESCAPE:
+    return "Escape";
+  case VK_SHIFT:
+  case VK_LSHIFT:
+    return "Shift_L";
+  case VK_RSHIFT:
+    return "Shift_R";
+  case VK_CONTROL:
+  case VK_LCONTROL:
+    return "Control_L";
+  case VK_RCONTROL:
+    return "Control_R";
+  case VK_MENU:
+  case VK_LMENU:
+    return "Alt_L";
+  case VK_RMENU:
+    return "Alt_R";
+  case VK_CAPITAL:
+    return "Caps_Lock";
+  /*** TODO: win32 keyboard WIP, add more cases, match XKeysymToString() ***/
+  default:
+    if (VK_F1 <= key && key <= VK_F12)
+      return fmt::format("F{}", key - VK_F1);
+    else if (isascii(key))
+      return std::string(1, key);
+    else
+      return fmt::format("Vk{}", key);
+  }
+}
+
 LRESULT CALLBACK WindowProc(HWND hwnd,
                             UINT uMsg,
                             WPARAM wParam,
@@ -27,10 +76,30 @@ LRESULT CALLBACK WindowProc(HWND hwnd,
     case WM_LBUTTONDOWN:
       gui->mouse_event(
           GUI::MouseEvent{GUI::MouseEvent::Type::press, gui->cursor_pos});
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::press, "LMB", gui->cursor_pos});
       break;
     case WM_LBUTTONUP:
       gui->mouse_event(
           GUI::MouseEvent{GUI::MouseEvent::Type::release, gui->cursor_pos});
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::release, "LMB", gui->cursor_pos});
+      break;
+    case WM_MBUTTONDOWN:
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::press, "MMB", gui->cursor_pos});
+      break;
+    case WM_MBUTTONUP:
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::release, "MMB", gui->cursor_pos});
+      break;
+    case WM_RBUTTONDOWN:
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::press, "RMB", gui->cursor_pos});
+      break;
+    case WM_RBUTTONUP:
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::release, "RMB", gui->cursor_pos});
       break;
     case WM_MOUSEMOVE:
       x = GET_X_LPARAM(lParam);
@@ -41,6 +110,17 @@ LRESULT CALLBACK WindowProc(HWND hwnd,
       break;
     case WM_PAINT:
       break;
+    case WM_KEYDOWN:
+      gui->key_pressed = true;
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::press,
+          lookup_keysym(wParam, lParam), gui->cursor_pos});
+      break;
+    case WM_KEYUP:
+      gui->key_events.push_back(
+          GUI::KeyEvent{GUI::KeyEvent::Type::release,
+          lookup_keysym(wParam, lParam), gui->cursor_pos});
+      break;
     case WM_CLOSE:
       exit(0);
       break;
@@ -124,4 +204,4 @@ GUI::~GUI() {
 
 TC_NAMESPACE_END
 
-#endif
+#endif

taichi/gui/x11.cpp

@@ -45,6 +45,25 @@ class CXImage {
   }
 };
 
+static std::string lookup_keysym(XEvent *ev)
+{
+  int key = XLookupKeysym(&ev->xkey, 0);
+  if (isascii(key))
+    return std::string(1, key);
+  else
+    return XKeysymToString(key);
+}
+
+static std::string lookup_button(XEvent *ev)
+{
+  switch (ev->xbutton.button) {
+  case 1: return "LMB";
+  case 2: return "MMB";
+  case 3: return "RMB";
+  default: return fmt::format("Button{}", ev->xbutton.button);
+  }
+}
+
 void GUI::process_event() {
   while (XPending((Display *)display)) {
     XEvent ev;
@@ -55,15 +74,29 @@ void GUI::process_event() {
       case MotionNotify:
         set_mouse_pos(ev.xbutton.x, height - ev.xbutton.y - 1);
         mouse_event(MouseEvent{MouseEvent::Type::move, cursor_pos});
+        key_events.push_back(
+            KeyEvent{KeyEvent::Type::move, "Motion", cursor_pos});
         break;
       case ButtonPress:
+        set_mouse_pos(ev.xbutton.x, height - ev.xbutton.y - 1);
         mouse_event(MouseEvent{MouseEvent::Type::press, cursor_pos});
+        key_events.push_back(
+            KeyEvent{KeyEvent::Type::press, lookup_button(&ev), cursor_pos});
         break;
       case ButtonRelease:
+        set_mouse_pos(ev.xbutton.x, height - ev.xbutton.y - 1);
         mouse_event(MouseEvent{MouseEvent::Type::release, cursor_pos});
+        key_events.push_back(
+            KeyEvent{KeyEvent::Type::release, lookup_button(&ev), cursor_pos});
         break;
       case KeyPress:
         key_pressed = true;
+        key_events.push_back(
+            KeyEvent{KeyEvent::Type::press, lookup_keysym(&ev), cursor_pos});
+        break;
+      case KeyRelease:
+        key_events.push_back(
+            KeyEvent{KeyEvent::Type::release, lookup_keysym(&ev), cursor_pos});
         break;
     }
   }
@@ -99,4 +132,4 @@ GUI::~GUI() {
 
 TC_NAMESPACE_END
 
-#endif
+#endif

taichi/python/export_visual.cpp

@@ -25,6 +25,13 @@ void export_visual(py::module &m) {
                          img.get_data_size());
            })
       .def("screenshot", &GUI::screenshot)
+      .def("has_key_event", &GUI::has_key_event)
+      .def("wait_key_event", &GUI::wait_key_event)
+      .def("get_key_event_head_key", &GUI::get_key_event_head_key)
+      .def("get_key_event_head_type", &GUI::get_key_event_head_type)
+      .def("get_key_event_head_pos", &GUI::get_key_event_head_pos)
+      .def("pop_key_event_head", &GUI::pop_key_event_head)
+      .def("get_cursor_pos", &GUI::get_cursor_pos)
       .def("update", &GUI::update);
   py::class_<Canvas>(m, "Canvas")
       .def("clear", static_cast<void (Canvas::*)(int)>(&Canvas::clear))

taichi/visual/gui.h

@@ -297,6 +297,10 @@ class Canvas {
     return Vector2(transform_matrix * Vector3(x, 1.0_f));
   }
 
+  TC_FORCE_INLINE Vector2 untransform(Vector2 x) const {
+    return Vector2(inversed(transform_matrix) * Vector3(x, 1.0_f));
+  }
+
   std::vector<Circle> circles;
   std::vector<Line> lines;
 
@@ -495,6 +499,15 @@ class GUI : public GUIBase {
     bool button_status[3];
   };
 
+  struct KeyEvent {
+    enum class Type { move, press, release };
+    Type type;
+    std::string key;
+    Vector2i pos;
+  };
+
+  std::vector<KeyEvent> key_events;
+
   struct Rect {
     Vector2i pos;
     Vector2i size;
@@ -785,6 +798,36 @@ class GUI : public GUIBase {
       set_title(fmt::format("{} ({:.02f} FPS)", window_name, real_fps));
   }
 
+  bool has_key_event() {
+    return !!key_events.size();
+  }
+
+  void wait_key_event() {
+    while (!key_events.size()) {
+      update();
+    }
+  }
+
+  std::string get_key_event_head_key() {
+    return key_events[0].key;
+  }
+
+  bool get_key_event_head_type() {
+    return key_events[0].type == KeyEvent::Type::press;
+  }
+
+  Vector2 get_key_event_head_pos() {
+    return canvas->untransform(Vector2(key_events[0].pos));
+  }
+
+  Vector2 get_cursor_pos() {
+    return canvas->untransform(Vector2(cursor_pos));
+  }
+
+  void pop_key_event_head() {
+    key_events.erase(key_events.begin());
+  }
+
   void wait_key() {
     while (true) {
       key_pressed = false;