egui_web: improve text input on mobile and for IME

egui/src/data/output.rs

@@ -29,7 +29,11 @@ pub struct Output {
     /// Events that may be useful to e.g. a screen reader.
     pub events: Vec<OutputEvent>,
 
-    /// Position of text edit cursor (used for IME).
+    /// Is there a mutable `TextEdit` under the cursor?
+    /// Use by `egui_web` to show/hide mobile keyboard and IME agent.
+    pub mutable_text_under_cursor: bool,
+
+    /// Screen-space position of text edit cursor (used for IME).
     pub text_cursor_pos: Option<crate::Pos2>,
 }
 
@@ -65,6 +69,7 @@ impl Output {
             copied_text,
             needs_repaint,
             mut events,
+            mutable_text_under_cursor,
             text_cursor_pos,
         } = newer;
 
@@ -77,6 +82,7 @@ impl Output {
         }
         self.needs_repaint = needs_repaint; // if the last frame doesn't need a repaint, then we don't need to repaint
         self.events.append(&mut events);
+        self.mutable_text_under_cursor = mutable_text_under_cursor;
         self.text_cursor_pos = text_cursor_pos.or(self.text_cursor_pos);
     }
 

egui/src/widgets/text_edit.rs

@@ -611,6 +611,10 @@ impl<'t> TextEdit<'t> {
 
         if interactive {
             if let Some(pointer_pos) = ui.input().pointer.interact_pos() {
+                if response.hovered() && text.is_mutable() {
+                    ui.output().mutable_text_under_cursor = true;
+                }
+
                 // TODO: triple-click to select whole paragraph
                 // TODO: drag selected text to either move or clone (ctrl on windows, alt on mac)
                 let singleline_offset = vec2(state.singleline_offset, 0.0);
@@ -910,7 +914,9 @@ impl<'t> TextEdit<'t> {
                     &cursorp.primary,
                 );
 
-                if interactive {
+                if interactive && text.is_mutable() {
+                    // egui_web uses `text_cursor_pos` when showing IME,
+                    // so only set it when text is editable!
                     ui.ctx().output().text_cursor_pos = Some(
                         galley
                             .pos_from_cursor(&cursorp.primary)

egui_web/CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to the `egui_web` integration will be noted in this file.
 ### Fixed 🐛
 * Fix multiline paste.
 * Fix painting with non-opaque backgrounds.
+* Improve text input on mobile and for IME.
 
 
 ## 0.14.1 - 2021-08-28

egui_web/src/backend.rs

@@ -137,7 +137,8 @@ pub struct AppRunner {
     prefer_dark_mode: Option<bool>,
     last_save_time: f64,
     screen_reader: crate::screen_reader::ScreenReader,
-    pub(crate) last_text_cursor_pos: Option<egui::Pos2>,
+    pub(crate) text_cursor_pos: Option<egui::Pos2>,
+    pub(crate) mutable_text_under_cursor: bool,
 }
 
 impl AppRunner {
@@ -163,7 +164,8 @@ impl AppRunner {
             prefer_dark_mode,
             last_save_time: now_sec(),
             screen_reader: Default::default(),
-            last_text_cursor_pos: None,
+            text_cursor_pos: None,
+            mutable_text_under_cursor: false,
         };
 
         {

egui_web/src/lib.rs

@@ -300,6 +300,7 @@ pub fn handle_output(output: &egui::Output, runner: &mut AppRunner) {
         copied_text,
         needs_repaint: _, // handled elsewhere
         events: _,        // we ignore these (TODO: accessibility screen reader)
+        mutable_text_under_cursor,
         text_cursor_pos,
     } = output;
 
@@ -316,9 +317,11 @@ pub fn handle_output(output: &egui::Output, runner: &mut AppRunner) {
     #[cfg(not(web_sys_unstable_apis))]
     let _ = copied_text;
 
-    if &runner.last_text_cursor_pos != text_cursor_pos {
+    runner.mutable_text_under_cursor = *mutable_text_under_cursor;
+
+    if &runner.text_cursor_pos != text_cursor_pos {
         move_text_cursor(text_cursor_pos, runner.canvas_id());
-        runner.last_text_cursor_pos = *text_cursor_pos;
+        runner.text_cursor_pos = *text_cursor_pos;
     }
 }
 
@@ -903,7 +906,8 @@ fn install_canvas_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
                         modifiers,
                     });
                 runner_lock.needs_repaint.set_true();
-                manipulate_agent(runner_lock.canvas_id(), runner_lock.input.latest_touch_pos);
+
+                update_text_agent(&runner_lock);
             }
             event.stop_propagation();
             event.prevent_default();
@@ -987,6 +991,7 @@ fn install_canvas_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
         let runner_ref = runner_ref.clone();
         let closure = Closure::wrap(Box::new(move |event: web_sys::TouchEvent| {
             let mut runner_lock = runner_ref.0.lock();
+
             if let Some(pos) = runner_lock.input.latest_touch_pos {
                 let modifiers = runner_lock.input.raw.modifiers;
                 // First release mouse to click:
@@ -1007,10 +1012,10 @@ fn install_canvas_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
                 runner_lock.needs_repaint.set_true();
                 event.stop_propagation();
                 event.prevent_default();
-
-                // Finally, focus or blur on agent to toggle keyboard
-                manipulate_agent(runner_lock.canvas_id(), runner_lock.input.latest_touch_pos);
             }
+
+            // Finally, focus or blur text agent to toggle mobile keyboard:
+            update_text_agent(&runner_lock);
         }) as Box<dyn FnMut(_)>);
         canvas.add_event_listener_with_callback(event_name, closure.as_ref().unchecked_ref())?;
         closure.forget();
@@ -1169,40 +1174,52 @@ fn install_canvas_events(runner_ref: &AppRunnerRef) -> Result<(), JsValue> {
     Ok(())
 }
 
-fn manipulate_agent(canvas_id: &str, latest_cursor: Option<egui::Pos2>) -> Option<()> {
+/// Focus or blur text agent to toggle mobile keyboard.
+fn update_text_agent(runner: &AppRunner) -> Option<()> {
     use wasm_bindgen::JsCast;
     use web_sys::HtmlInputElement;
     let window = web_sys::window()?;
     let document = window.document()?;
     let input: HtmlInputElement = document.get_element_by_id(AGENT_ID)?.dyn_into().unwrap();
-    let cutsor_txt = document.body()?.style().get_property_value("cursor").ok()?;
+    let canvas_style = canvas_element(runner.canvas_id())?.style();
-    let style = canvas_element(canvas_id)?.style();
-    if cutsor_txt == cursor_web_name(egui::CursorIcon::Text) {
-        input.set_hidden(false);
-        input.focus().ok()?;
-        // Panning canvas so that text edit is shown at 30%
-        // Only on touch screens, when keyboard popups
-        if let Some(p) = latest_cursor {
-            let inner_height = window.inner_height().ok()?.as_f64()? as f32;
-            let current_rel = p.y / inner_height;
 
-            if current_rel > 0.5 {
+    if runner.mutable_text_under_cursor {
-                // probably below the keyboard
+        let is_already_editing = input.hidden();
+        if is_already_editing {
+            input.set_hidden(false);
+            input.focus().ok()?;
 
-                let target_rel = 0.3;
+            // Move up canvas so that text edit is shown at ~30% of screen height.
+            // Only on touch screens, when keyboard popups.
+            if let Some(latest_touch_pos) = runner.input.latest_touch_pos {
+                let window_height = window.inner_height().ok()?.as_f64()? as f32;
+                let current_rel = latest_touch_pos.y / window_height;
 
-                let delta = target_rel - current_rel;
+                // estimated amount of screen covered by keyboard
-                let new_pos_percent = (delta * 100.0).round().to_string() + "%";
+                let keyboard_fraction = 0.5;
 
-                style.set_property("position", "absolute").ok()?;
+                if current_rel > keyboard_fraction {
-                style.set_property("top", &new_pos_percent).ok()?;
+                    // below the keyboard
+
+                    let target_rel = 0.3;
+
+                    // Note: `delta` is negative, since we are moving the canvas UP
+                    let delta = target_rel - current_rel;
+
+                    let delta = delta.max(-keyboard_fraction); // Don't move it crazy much
+
+                    let new_pos_percent = (delta * 100.0).round().to_string() + "%";
+
+                    canvas_style.set_property("position", "absolute").ok()?;
+                    canvas_style.set_property("top", &new_pos_percent).ok()?;
+                }
             }
         }
     } else {
         input.blur().ok()?;
         input.set_hidden(true);
-        style.set_property("position", "absolute").ok()?;
+        canvas_style.set_property("position", "absolute").ok()?;
-        style.set_property("top", "0%").ok()?; // move back to normal position
+        canvas_style.set_property("top", "0%").ok()?; // move back to normal position
     }
     Some(())
 }