Merge pull request #9740 from keymanapp/feat/web/generalized-multitap

feat(web): generalized multitap 🐵

common/web/gesture-recognizer/src/engine/headless/gestureSource.ts

@@ -272,20 +272,16 @@ export class GestureSourceSubview<HoveredItemType, StateToken = any> extends Ges
     preserveBaseItem: boolean,
     stateTokenOverride?: StateToken
   ) {
-    let mayUpdate = true;
     let start = 0;
     let length = source.path.coords.length;
     if(source instanceof GestureSourceSubview) {
       start = source._baseStartIndex;
-      const expectedLength = start + length;
-      // Check against the full remaining length of the original source; does
-      // the subview provided to us include its source's most recent point?
-      const sampleCountSinceStart = source.baseSource.path.coords.length;
-      if(expectedLength != sampleCountSinceStart) {
-        mayUpdate = false;
-      }
     }
 
+    // While it'd be nice to validate that a previous subview, if used, has all 'current'
+    // entries, this gets tricky; race conditions are possible in which an extra input event
+    // occurs before subviews can be spun up when starting a model-matcher in some scenarios.
+
     super(source.rawIdentifier, configStack, source.isFromTouch);
 
     const baseSource = this._baseSource = source instanceof GestureSourceSubview ? source._baseSource : source;
@@ -299,18 +295,23 @@ export class GestureSourceSubview<HoveredItemType, StateToken = any> extends Ges
       const translation = this.recognizerTranslation;
       // Provide a coordinate-system translation for source subviews.
       // The base version still needs to use the original coord system, though.
-      const transformedSample = {...sample, targetX: sample.targetX - translation.x, targetY: sample.targetY - translation.y};
+      const transformedSample = {
+        ...sample,
+        targetX: sample.targetX - translation.x,
+        targetY: sample.targetY - translation.y
+      };
+
+      if(this.stateToken) {
+        transformedSample.stateToken = this.stateToken;
+      }
 
       // If the subview is operating from the perspective of a different state token than its base source,
       // its samples' item fields will need correction.
       //
       // This can arise during multitap-like scenarios.
-      if(this.stateToken != baseSource.stateToken) {
+      if(this.stateToken != baseSource.stateToken || this.stateToken != source.stateToken) {
         transformedSample.item = this.currentRecognizerConfig.itemIdentifier(
-          {
-            ...sample,
-            stateToken: this.stateToken
-          },
+          transformedSample,
           null
         );
       }
@@ -353,24 +354,22 @@ export class GestureSourceSubview<HoveredItemType, StateToken = any> extends Ges
       this._baseItem = lastSample?.item;
     }
 
-    if(mayUpdate) {
-      // Ensure that this 'subview' is updated whenever the "source of truth" is.
-      const completeHook    = ()       => this.path.terminate(false);
-      const invalidatedHook = ()       => this.path.terminate(true);
-      const stepHook        = (sample: InputSample<HoveredItemType, StateToken>) => {
-        super.update(translateSample(sample));
-      };
-      baseSource.path.on('complete',    completeHook);
-      baseSource.path.on('invalidated', invalidatedHook);
-      baseSource.path.on('step',        stepHook);
-
-      // But make sure we can "disconnect" it later once the gesture being matched
-      // with the subview has fully matched; it's good to have a snapshot left over.
-      this.subviewDisconnector = () => {
-        baseSource.path.off('complete',    completeHook);
-        baseSource.path.off('invalidated', invalidatedHook);
-        baseSource.path.off('step',        stepHook);
-      }
+    // Ensure that this 'subview' is updated whenever the "source of truth" is.
+    const completeHook    = ()       => this.path.terminate(false);
+    const invalidatedHook = ()       => this.path.terminate(true);
+    const stepHook        = (sample: InputSample<HoveredItemType, StateToken>) => {
+      super.update(translateSample(sample));
+    };
+    baseSource.path.on('complete',    completeHook);
+    baseSource.path.on('invalidated', invalidatedHook);
+    baseSource.path.on('step',        stepHook);
+
+    // But make sure we can "disconnect" it later once the gesture being matched
+    // with the subview has fully matched; it's good to have a snapshot left over.
+    this.subviewDisconnector = () => {
+      baseSource.path.off('complete',    completeHook);
+      baseSource.path.off('invalidated', invalidatedHook);
+      baseSource.path.off('step',        stepHook);
     }
   }
 

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts

@@ -388,13 +388,12 @@ export class GestureMatcher<Type, StateToken = any> implements PredecessorMatch<
       // should reflect this.
       simpleSource.baseItem = baseItem ?? simpleSource.baseItem;
       simpleSource.stateToken = baseStateToken;
+      simpleSource.currentSample.stateToken = baseStateToken;
 
       // May be missing during unit tests.
       if(simpleSource.currentRecognizerConfig) {
-        simpleSource.currentSample.item = simpleSource.currentRecognizerConfig.itemIdentifier({
-            ...simpleSource.currentSample,
-            stateToken: baseStateToken
-          },
+        simpleSource.currentSample.item = simpleSource.currentRecognizerConfig.itemIdentifier(
+          simpleSource.currentSample,
           null
         );
       }
@@ -422,12 +421,6 @@ export class GestureMatcher<Type, StateToken = any> implements PredecessorMatch<
       }
     }
 
-    // Now that we've done the initial-state check, we can check for instantly-matching path models.
-    const result = contactModel.update();
-    if(result.type == 'reject' && this.model.id == 'modipress-multitap-end') {
-      console.log('temp');
-    }
-
     contactModel.promise.then((resolution) => {
       this.finalize(resolution.type == 'resolve', resolution.cause);
     });

common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts

@@ -5,6 +5,8 @@ import { ManagedPromise, timedPromise } from "@keymanapp/web-utils";
 import { GestureSource, GestureSourceSubview } from "../../gestureSource.js";
 import { GestureMatcher, MatchResult, PredecessorMatch } from "./gestureMatcher.js";
 import { GestureModel } from "../specs/gestureModel.js";
+import { GestureSequence } from "./index.js";
+import { ItemIdentifier } from "../../../configuration/gestureRecognizerConfiguration.js";
 
 interface GestureSourceTracker<Type, StateToken> {
   /**

common/web/gesture-recognizer/src/test/auto/headless/gestureSource.spec.ts

@@ -185,30 +185,6 @@ describe("GestureSource", function() {
       assert.deepEqual(subview.currentSample, worldSample);
     });
 
-    it('is not updated if constructed from old, not-up-to-date subview', () => {
-      let source = new GestureSource<string>(0, null, true);
-      let updatingSubview = source.constructSubview(false, true);
-      let oldSubview = source.constructSubview(false, true);
-      oldSubview.disconnect();
-
-      source.update(helloSample);
-
-      let subview = oldSubview.constructSubview(false, true);
-
-      assert.equal(subview.path.coords.length, 0);
-
-      source.update(worldSample);
-
-      // Only the 'updating' one should update in this scenario.
-      //
-      // It's an error if the subview based on the non-updated subview updates,
-      // as there would be a gap in the path!
-      assert.equal(updatingSubview.path.coords.length, 2);
-      assert.equal(subview.path.coords.length, 0);
-      assert.deepEqual(updatingSubview.currentSample, worldSample);
-      assert.isNotOk(subview.currentSample);
-    });
-
     it("propagate path termination (complete)",  () => {
       let source = new GestureSource<string>(0, null, true);
       let subview = source.constructSubview(true, true);

common/web/input-processor/src/text/inputProcessor.ts

@@ -102,6 +102,18 @@ export default class InputProcessor {
         this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard;
       }
 
+      // Support for multitap context reversion; multitap keys should act as if they were
+      // the first thing typed since `preInput`, the state before the original base key.
+      if(keyEvent.baseTranscriptionToken) {
+        const transcription = this.contextCache.get(keyEvent.baseTranscriptionToken);
+        if(transcription) {
+          // Restores full context, including deadkeys in their exact pre-keystroke state.
+          outputTarget.restoreTo(transcription.preInput);
+        } else {
+          console.warn('The base context for the multitap could not be found');
+        }
+      }
+
       return this._processKeyEvent(keyEvent, outputTarget);
     } finally {
       if(kbdMismatch) {

common/web/keyboard-processor/src/keyboards/activeLayout.ts

@@ -311,6 +311,27 @@ export class ActiveKeyBase {
       hasMultitaps: false
     }
 
+    // The default-layer shift key on mobile platforms should have a default multitap under
+    // select conditions.
+    //
+    // Note:  whether or not any other key has multitaps doesn't matter here.  Just THIS one.
+    if(key.id == 'K_SHIFT' && displayLayer == 'default' && layout.formFactor != 'desktop') {
+      /* Extra requirements:
+       *
+       * 1. The SHIFT key must not specify longpress keys or have already-specified multitaps.
+       *
+       *    Note:  touch layouts specified on desktop layouts often do specify longpress keys;
+       *    utilized modifiers aside from 'shift' become longpress keys under K_SHIFT)
+       *
+       * 2. There exists a specified 'caps' layer.  Otherwise, there's no destination for
+       *    the default multitap.
+       *
+       */
+      if(!key.sk && !key.multitap && !!layout.layer.find((entry) => entry.id == 'caps')) {
+        key.multitap = [Layouts.dfltShiftMultitap];
+      }
+    }
+
     // Add class functions to the existing layout object, allowing it to act as an ActiveLayout.
     let dummy = new ActiveKeyBase();
     let proto = Object.getPrototypeOf(dummy);
@@ -804,9 +825,11 @@ export class ActiveLayout implements LayoutFormFactor{
    * @param formFactor
    */
   static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout {
+    /* c8 ignore start */
     if(layout == null) {
       throw new Error("Cannot build an ActiveLayout for a null specification.");
     }
+    /* c8 ignore end */
 
     const analysisMetadata: AnalysisMetadata = {
       hasFlicks: false,

common/web/keyboard-processor/src/keyboards/defaultLayouts.ts

@@ -526,6 +526,15 @@ export class Layouts {
     return KLS;
   }
 
+  static dfltShiftMultitap: LayoutSubKey = {
+    // Needs to be something special and unique.  Typing restricts us from
+    // using a reserved key-id prefix, though.
+    id: "T_*_MT_SHIFT_TO_CAPS",
+    text: '*ShiftLock*',
+    sp: 1,
+    nextlayer: 'caps'
+  }
+
   // Defines the default visual layout for a keyboard.
   /* c8 ignore start */
   static dfltLayout: LayoutSpec = {

common/web/keyboard-processor/src/keyboards/keyboard.ts

@@ -1,6 +1,6 @@
 import Codes from "../text/codes.js";
 import { Layouts, type LayoutFormFactor } from "./defaultLayouts.js";
-import { ActiveKey, ActiveKeyBase, ActiveLayout } from "./activeLayout.js";
+import { ActiveKey, ActiveLayout, ActiveSubKey } from "./activeLayout.js";
 import KeyEvent from "../text/keyEvent.js";
 import type OutputTarget from "../text/outputTarget.js";
 
@@ -468,7 +468,7 @@ export default class Keyboard {
     return keyEvent;
   }
 
-  constructKeyEvent(key: ActiveKeyBase, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent {
+  constructKeyEvent(key: ActiveKey | ActiveSubKey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent {
     // Make a deep copy of our preconstructed key event, filling it out from there.
     const Lkc = key.baseKeyEvent;
     Lkc.device = device;

common/web/keyboard-processor/src/text/keyEvent.ts

@@ -75,6 +75,7 @@ export default class KeyEvent implements KeyEventSpec {
   kLayer?: string;   // The key's layer property
   kbdLayer?: string; // The virtual keyboard's active layer
   kNextLayer?: string;
+  baseTranscriptionToken?: number;
 
   /**
    * Marks the active keyboard at the time that this KeyEvent was generated by the user.

common/web/lm-worker/src/main/correction/context-tracker.ts

@@ -83,7 +83,7 @@ export class TrackedContextState {
       this.tokens = source.tokens.map(function(token) {
         let copy = new TrackedContextToken();
         copy.raw = token.raw;
-        copy.replacements = [].concat(token.replacements)
+        copy.replacements = [].concat(token.replacements);
         copy.activeReplacementId = token.activeReplacementId;
         copy.transformDistributions = [].concat(token.transformDistributions);
 
@@ -98,7 +98,7 @@ export class TrackedContextState {
       const lexicalModel = this.model = obj.model;
       this.taggedContext = obj.taggedContext;
 
-      if(lexicalModel && lexicalModel.traverseFromRoot) {
+      if(lexicalModel?.traverseFromRoot) {
         // We need to construct a separate search space from other ContextStates.
         //
         // In case we are unable to perfectly track context (say, due to multitaps)

common/web/types/src/keyman-touch-layout/keyman-touch-layout-file.ts

@@ -41,8 +41,20 @@ export interface TouchLayoutRow {
 
 type Key_Type = 'T'|'K'|'U'|'t'|'k'|'u';
 type Key_Id = string;
+
 export type TouchLayoutKeyId = `${Key_Type}_${Key_Id}`; // pattern = /^[TKUtku]_[a-zA-Z0-9_]+$/
 
+/**
+ * Denotes private-use identifiers that should be considered 'reserved'.
+ */
+export const PRIVATE_USE_IDS = [
+  /**
+   * A private-use identifier used by KeymanWeb for the default multitap-into-caps-layer key
+   * for keyboards with a caps layer while not defining multitaps on shift.
+   */
+  'T_*_MT_SHIFT_TO_CAPS'
+] as const;
+
 /* A map of key field names with values matching the `typeof` the corresponding property
  * exists in common/web/keyboard-processor, keyboards/activeLayout.ts.
  *

web/src/engine/events/build.sh

@@ -16,7 +16,8 @@ cd "$THIS_SCRIPT_PATH"
 # ################################ Main script ################################
 
 builder_describe "Builds specialized event-related modules utilized by Keyman Engine for Web." \
-  "@/common/web/utils" \
+  "@/common/web/utils build" \
+  "@/common/web/keyboard-processor build" \
   "clean" \
   "configure" \
   "build" \

web/src/engine/events/src/index.ts

@@ -1,3 +1,4 @@
 export { DomEventTracker } from './domEventTracker.js';
 export { EmitterListenerSpy } from './emitterListenerSpy.js';
+export * from './keyEventSource.interface.js';
 export * from './legacyEventEmitter.js';

web/src/engine/main/src/keyEventSource.interface.ts → web/src/engine/events/src/keyEventSource.interface.ts

@@ -11,4 +11,4 @@ interface EventMap {
   'keyevent': KeyEventHandler;
 }
 
-export default interface KeyEventSourceInterface extends EventEmitter<EventMap> { }
+export interface KeyEventSourceInterface<Map extends EventMap> extends EventEmitter<Map> { }

web/src/engine/events/tsconfig.json

@@ -11,6 +11,7 @@
   "include": [ "src/**/*.ts" ],
 
   "references": [
-    { "path": "../../../../common/web/utils" }
+    { "path": "../../../../common/web/utils" },
+    { "path": "../../../../common/web/keyboard-processor" }
   ]
 }

web/src/engine/main/src/hardKeyboard.ts

@@ -1,6 +1,6 @@
 import EventEmitter from "eventemitter3";
 import { type KeyEvent, type RuleBehavior } from "@keymanapp/keyboard-processor";
-import KeyEventSourceInterface from './keyEventSource.interface.js';
+import { KeyEventSourceInterface } from 'keyman/engine/events';
 
 interface EventMap {
   /**
@@ -9,7 +9,7 @@ interface EventMap {
   'keyevent': (event: KeyEvent, callback?: (result: RuleBehavior, error?: Error) => void) => void
 }
 
-export default class HardKeyboard extends EventEmitter<EventMap> implements KeyEventSourceInterface { }
+export default class HardKeyboard extends EventEmitter<EventMap> implements KeyEventSourceInterface<EventMap> { }
 
 // Intended design:
 // - KeyEventKeyboard:  website-integrated handler for hardware-keystroke input; interprets DOM events.

web/src/engine/main/src/keymanEngine.ts

@@ -7,7 +7,7 @@ import { KeyboardRequisitioner, ModelCache, ModelSpec, toUnprefixedKeyboardId as
 import { EngineConfiguration, InitOptionSpec } from "./engineConfiguration.js";
 import KeyboardInterface from "./keyboardInterface.js";
 import { ContextManagerBase } from "./contextManagerBase.js";
-import { KeyEventHandler } from './keyEventSource.interface.js';
+import { KeyEventHandler } from 'keyman/engine/events';
 import HardKeyboardBase from "./hardKeyboard.js";
 import { LegacyAPIEvents } from "./legacyAPIEvents.js";
 import { EventNames, EventListener, LegacyEventEmitter } from "keyman/engine/events";