Move IME candidate window following text cursor (#258)

* egui_web: enable IME support on web.

* Move candidate window following text cursor.

* Preclude too frequent agent movement.

* IME candidate window move on native app.

egui/src/data/output.rs

@@ -23,6 +23,9 @@ pub struct Output {
 
     /// Events that may be useful to e.g. a screen reader.
     pub events: Vec<OutputEvent>,
+
+    /// Position of text widgts' cursor
+    pub text_cursor: Option<crate::Pos2>,
 }
 
 impl Output {

egui/src/widgets/text_edit.rs

@@ -382,6 +382,7 @@ impl<'t> TextEdit<'t> {
             ui.output().cursor_icon = CursorIcon::Text;
         }
 
+        let mut text_cursor = None;
         if ui.memory().has_focus(id) && enabled {
             let mut cursorp = state
                 .cursorp
@@ -495,6 +496,7 @@ impl<'t> TextEdit<'t> {
                     };
                 }
             }
+            text_cursor = Some(cursorp);
             state.cursorp = Some(cursorp);
 
             state
@@ -503,6 +505,15 @@ impl<'t> TextEdit<'t> {
         }
 
         if ui.memory().has_focus(id) {
+            {
+                let mut output = ui.ctx().output();
+                output.text_cursor = text_cursor.map(|c| {
+                    galley
+                        .pos_from_cursor(&c.primary)
+                        .translate(response.rect.min.to_vec2())
+                        .left_top()
+                });
+            }
             if let Some(cursorp) = state.cursorp {
                 paint_cursor_selection(ui, response.rect.min, &galley, &cursorp);
                 paint_cursor_end(ui, response.rect.min, &galley, &cursorp.primary);

egui_glium/src/backend.rs

@@ -212,7 +212,7 @@ pub fn run(mut app: Box<dyn epi::App>) -> ! {
 
         set_cursor_icon(&display, egui_output.cursor_icon);
         current_cursor_icon = egui_output.cursor_icon;
-        handle_output(egui_output, clipboard.as_mut());
+        handle_output(egui_output, clipboard.as_mut(), &display);
 
         // TODO: handle app_output
         // eprintln!("Warmed up in {} ms", warm_up_start.elapsed().as_millis())
@@ -289,7 +289,7 @@ pub fn run(mut app: Box<dyn epi::App>) -> ! {
                 set_cursor_icon(&display, egui_output.cursor_icon);
                 current_cursor_icon = egui_output.cursor_icon;
             }
-            handle_output(egui_output, clipboard.as_mut());
+            handle_output(egui_output, clipboard.as_mut(), &display);
 
             #[cfg(feature = "persistence")]
             if let Some(storage) = &mut storage {

egui_glium/src/lib.rs

@@ -303,7 +303,11 @@ fn set_cursor_icon(display: &glium::backend::glutin::Display, cursor_icon: egui:
     }
 }
 
-pub fn handle_output(output: egui::Output, clipboard: Option<&mut ClipboardContext>) {
+pub fn handle_output(
+    output: egui::Output,
+    clipboard: Option<&mut ClipboardContext>,
+    display: &glium::Display,
+) {
     if let Some(open) = output.open_url {
         if let Err(err) = webbrowser::open(&open.url) {
             eprintln!("Failed to open url: {}", err);
@@ -317,6 +321,13 @@ pub fn handle_output(output: egui::Output, clipboard: Option<&mut ClipboardConte
             }
         }
     }
+
+    if let Some(egui::Pos2 { x, y }) = output.text_cursor {
+        display
+            .gl_window()
+            .window()
+            .set_ime_position(glium::glutin::dpi::LogicalPosition { x, y })
+    }
 }
 
 pub fn init_clipboard() -> Option<ClipboardContext> {

egui_web/src/backend.rs

@@ -138,6 +138,7 @@ pub struct AppRunner {
     screen_reader: crate::screen_reader::ScreenReader,
     #[cfg(feature = "http")]
     http: Arc<http::WebHttp>,
+    pub(crate) text_cursor: Option<egui::Pos2>,
 }
 
 impl AppRunner {
@@ -156,6 +157,7 @@ impl AppRunner {
             screen_reader: Default::default(),
             #[cfg(feature = "http")]
             http: Arc::new(http::WebHttp {}),
+            text_cursor: None,
         })
     }
 
@@ -222,7 +224,7 @@ impl AppRunner {
         if self.web_backend.ctx.memory().options.screen_reader {
             self.screen_reader.speak(&egui_output.events_description());
         }
-        handle_output(&egui_output);
+        handle_output(&egui_output, self);
 
         {
             let epi::backend::AppOutput {

egui_web/src/lib.rs

@@ -28,7 +28,7 @@ use std::rc::Rc;
 use std::sync::Arc;
 use wasm_bindgen::prelude::*;
 
-static AGENT_ID: &str = "text_agent";
+static AGENT_ID: &str = "egui_text_agent";
 
 // ----------------------------------------------------------------------------
 // Helpers to hide some of the verbosity of web_sys
@@ -234,13 +234,14 @@ impl epi::Storage for LocalStorage {
 
 // ----------------------------------------------------------------------------
 
-pub fn handle_output(output: &egui::Output) {
+pub fn handle_output(output: &egui::Output, runner: &mut AppRunner) {
     let egui::Output {
         cursor_icon,
         open_url,
         copied_text,
         needs_repaint: _, // handled elsewhere
         events: _,        // we ignore these (TODO: accessibility screen reader)
+        text_cursor: cursor,
     } = output;
 
     set_cursor_icon(*cursor_icon);
@@ -255,6 +256,11 @@ pub fn handle_output(output: &egui::Output) {
 
     #[cfg(not(web_sys_unstable_apis))]
     let _ = copied_text;
+
+    if &runner.text_cursor != cursor {
+        move_text_cursor(cursor, runner.canvas_id());
+        runner.text_cursor = *cursor;
+    }
 }
 
 pub fn set_cursor_icon(cursor: egui::CursorIcon) -> Option<()> {
@@ -464,7 +470,7 @@ fn paint_and_schedule(runner_ref: AppRunnerRef) -> Result<(), JsValue> {
     request_animation_frame(runner_ref)
 }
 
-fn text_agent_hidden() -> bool {
+fn text_agent() -> web_sys::HtmlInputElement {
     use wasm_bindgen::JsCast;
     web_sys::window()
         .unwrap()
@@ -472,9 +478,8 @@ fn text_agent_hidden() -> bool {
         .unwrap()
         .get_element_by_id(AGENT_ID)
         .unwrap()
-        .dyn_into::<web_sys::HtmlInputElement>()
+        .dyn_into()
         .unwrap()
-        .hidden()
 }
 
 fn install_document_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
@@ -508,7 +513,7 @@ fn install_document_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
                 && !modifiers.command
                 && !should_ignore_key(&key)
                 // When text agent is shown, it sends text event instead.
-                && text_agent_hidden()
+                && text_agent().hidden()
             {
                 runner_lock.input.raw.events.push(egui::Event::Text(key));
             }
@@ -703,7 +708,6 @@ fn install_text_agent(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
         let input_clone = input.clone();
         let runner_ref = runner_ref.clone();
         let on_compositionend = Closure::wrap(Box::new(move |event: web_sys::CompositionEvent| {
-            // let event_type = event.type_();
             match event.type_().as_ref() {
                 "compositionstart" => {
                     is_composing.set(true);
@@ -995,3 +999,35 @@ fn manipulate_agent(canvas_id: &str, latest_cursor: Option<egui::Pos2>) -> Optio
     }
     Some(())
 }
+
+const MOBILE_DEVICE: [&str; 6] = ["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"];
+/// If context is running under mobile device?
+fn is_mobile() -> Option<bool> {
+    let user_agent = web_sys::window()?.navigator().user_agent().ok()?;
+    let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name));
+    Some(is_mobile)
+}
+
+// Move angnt to text cursor's position, on desktop/laptop, candidate window moves following text elemt(agent),
+// so it appears that the IME candidate window moves with text cursor.
+// On mobile devices, there is no need to do that.
+fn move_text_cursor(cursor: &Option<egui::Pos2>, canvas_id: &str) -> Option<()> {
+    let style = text_agent().style();
+    // Note: movint agent on mobile devices will lead to unpreditable scroll.
+    if is_mobile() == Some(false) {
+        cursor.as_ref().and_then(|&egui::Pos2 { x, y }| {
+            let canvas = canvas_element(canvas_id)?;
+            let y = y + (canvas.scroll_top() + canvas.offset_top()) as f32;
+            let x = x + (canvas.scroll_left() + canvas.offset_left()) as f32;
+            // Canvas is translated 50% horizontally in html.
+            let x = x - canvas.offset_width() as f32 / 2.0;
+            style.set_property("position", "absolute").ok()?;
+            style.set_property("top", &(y.to_string() + "px")).ok()?;
+            style.set_property("left", &(x.to_string() + "px")).ok()
+        })
+    } else {
+        style.set_property("position", "absolute").ok()?;
+        style.set_property("top", "0px").ok()?;
+        style.set_property("left", "0px").ok()
+    }
+}