diff --git a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts index 13749d09f00..80ef2bac32b 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestureSource.ts @@ -272,20 +272,16 @@ export class GestureSourceSubview extends Ges preserveBaseItem: boolean, stateTokenOverride?: StateToken ) { - let mayUpdate = true; let start = 0; let length = source.path.coords.length; if(source instanceof GestureSourceSubview) { start = source._baseStartIndex; - const expectedLength = start + length; - // Check against the full remaining length of the original source; does - // the subview provided to us include its source's most recent point? - const sampleCountSinceStart = source.baseSource.path.coords.length; - if(expectedLength != sampleCountSinceStart) { - mayUpdate = false; - } } + // While it'd be nice to validate that a previous subview, if used, has all 'current' + // entries, this gets tricky; race conditions are possible in which an extra input event + // occurs before subviews can be spun up when starting a model-matcher in some scenarios. + super(source.rawIdentifier, configStack, source.isFromTouch); const baseSource = this._baseSource = source instanceof GestureSourceSubview ? source._baseSource : source; @@ -299,18 +295,23 @@ export class GestureSourceSubview extends Ges const translation = this.recognizerTranslation; // Provide a coordinate-system translation for source subviews. // The base version still needs to use the original coord system, though. - const transformedSample = {...sample, targetX: sample.targetX - translation.x, targetY: sample.targetY - translation.y}; + const transformedSample = { + ...sample, + targetX: sample.targetX - translation.x, + targetY: sample.targetY - translation.y + }; + + if(this.stateToken) { + transformedSample.stateToken = this.stateToken; + } // If the subview is operating from the perspective of a different state token than its base source, // its samples' item fields will need correction. // // This can arise during multitap-like scenarios. - if(this.stateToken != baseSource.stateToken) { + if(this.stateToken != baseSource.stateToken || this.stateToken != source.stateToken) { transformedSample.item = this.currentRecognizerConfig.itemIdentifier( - { - ...sample, - stateToken: this.stateToken - }, + transformedSample, null ); } @@ -353,24 +354,22 @@ export class GestureSourceSubview extends Ges this._baseItem = lastSample?.item; } - if(mayUpdate) { - // Ensure that this 'subview' is updated whenever the "source of truth" is. - const completeHook = () => this.path.terminate(false); - const invalidatedHook = () => this.path.terminate(true); - const stepHook = (sample: InputSample) => { - super.update(translateSample(sample)); - }; - baseSource.path.on('complete', completeHook); - baseSource.path.on('invalidated', invalidatedHook); - baseSource.path.on('step', stepHook); - - // But make sure we can "disconnect" it later once the gesture being matched - // with the subview has fully matched; it's good to have a snapshot left over. - this.subviewDisconnector = () => { - baseSource.path.off('complete', completeHook); - baseSource.path.off('invalidated', invalidatedHook); - baseSource.path.off('step', stepHook); - } + // Ensure that this 'subview' is updated whenever the "source of truth" is. + const completeHook = () => this.path.terminate(false); + const invalidatedHook = () => this.path.terminate(true); + const stepHook = (sample: InputSample) => { + super.update(translateSample(sample)); + }; + baseSource.path.on('complete', completeHook); + baseSource.path.on('invalidated', invalidatedHook); + baseSource.path.on('step', stepHook); + + // But make sure we can "disconnect" it later once the gesture being matched + // with the subview has fully matched; it's good to have a snapshot left over. + this.subviewDisconnector = () => { + baseSource.path.off('complete', completeHook); + baseSource.path.off('invalidated', invalidatedHook); + baseSource.path.off('step', stepHook); } } diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index 43fb9761029..d18c9bc976a 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -388,13 +388,12 @@ export class GestureMatcher implements PredecessorMatch< // should reflect this. simpleSource.baseItem = baseItem ?? simpleSource.baseItem; simpleSource.stateToken = baseStateToken; + simpleSource.currentSample.stateToken = baseStateToken; // May be missing during unit tests. if(simpleSource.currentRecognizerConfig) { - simpleSource.currentSample.item = simpleSource.currentRecognizerConfig.itemIdentifier({ - ...simpleSource.currentSample, - stateToken: baseStateToken - }, + simpleSource.currentSample.item = simpleSource.currentRecognizerConfig.itemIdentifier( + simpleSource.currentSample, null ); } @@ -422,12 +421,6 @@ export class GestureMatcher implements PredecessorMatch< } } - // Now that we've done the initial-state check, we can check for instantly-matching path models. - const result = contactModel.update(); - if(result.type == 'reject' && this.model.id == 'modipress-multitap-end') { - console.log('temp'); - } - contactModel.promise.then((resolution) => { this.finalize(resolution.type == 'resolve', resolution.cause); }); diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts index 4d1398fdb24..01a52874921 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts @@ -5,6 +5,8 @@ import { ManagedPromise, timedPromise } from "@keymanapp/web-utils"; import { GestureSource, GestureSourceSubview } from "../../gestureSource.js"; import { GestureMatcher, MatchResult, PredecessorMatch } from "./gestureMatcher.js"; import { GestureModel } from "../specs/gestureModel.js"; +import { GestureSequence } from "./index.js"; +import { ItemIdentifier } from "../../../configuration/gestureRecognizerConfiguration.js"; interface GestureSourceTracker { /** diff --git a/common/web/gesture-recognizer/src/test/auto/headless/gestureSource.spec.ts b/common/web/gesture-recognizer/src/test/auto/headless/gestureSource.spec.ts index 0386c8030e4..ac34dd4f4e6 100644 --- a/common/web/gesture-recognizer/src/test/auto/headless/gestureSource.spec.ts +++ b/common/web/gesture-recognizer/src/test/auto/headless/gestureSource.spec.ts @@ -185,30 +185,6 @@ describe("GestureSource", function() { assert.deepEqual(subview.currentSample, worldSample); }); - it('is not updated if constructed from old, not-up-to-date subview', () => { - let source = new GestureSource(0, null, true); - let updatingSubview = source.constructSubview(false, true); - let oldSubview = source.constructSubview(false, true); - oldSubview.disconnect(); - - source.update(helloSample); - - let subview = oldSubview.constructSubview(false, true); - - assert.equal(subview.path.coords.length, 0); - - source.update(worldSample); - - // Only the 'updating' one should update in this scenario. - // - // It's an error if the subview based on the non-updated subview updates, - // as there would be a gap in the path! - assert.equal(updatingSubview.path.coords.length, 2); - assert.equal(subview.path.coords.length, 0); - assert.deepEqual(updatingSubview.currentSample, worldSample); - assert.isNotOk(subview.currentSample); - }); - it("propagate path termination (complete)", () => { let source = new GestureSource(0, null, true); let subview = source.constructSubview(true, true); diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index 8dd45bc7741..76d8b288f42 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -102,6 +102,18 @@ export default class InputProcessor { this.keyboardInterface.activeKeyboard = keyEvent.srcKeyboard; } + // Support for multitap context reversion; multitap keys should act as if they were + // the first thing typed since `preInput`, the state before the original base key. + if(keyEvent.baseTranscriptionToken) { + const transcription = this.contextCache.get(keyEvent.baseTranscriptionToken); + if(transcription) { + // Restores full context, including deadkeys in their exact pre-keystroke state. + outputTarget.restoreTo(transcription.preInput); + } else { + console.warn('The base context for the multitap could not be found'); + } + } + return this._processKeyEvent(keyEvent, outputTarget); } finally { if(kbdMismatch) { diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 35f42646834..928263fb183 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -311,6 +311,27 @@ export class ActiveKeyBase { hasMultitaps: false } + // The default-layer shift key on mobile platforms should have a default multitap under + // select conditions. + // + // Note: whether or not any other key has multitaps doesn't matter here. Just THIS one. + if(key.id == 'K_SHIFT' && displayLayer == 'default' && layout.formFactor != 'desktop') { + /* Extra requirements: + * + * 1. The SHIFT key must not specify longpress keys or have already-specified multitaps. + * + * Note: touch layouts specified on desktop layouts often do specify longpress keys; + * utilized modifiers aside from 'shift' become longpress keys under K_SHIFT) + * + * 2. There exists a specified 'caps' layer. Otherwise, there's no destination for + * the default multitap. + * + */ + if(!key.sk && !key.multitap && !!layout.layer.find((entry) => entry.id == 'caps')) { + key.multitap = [Layouts.dfltShiftMultitap]; + } + } + // Add class functions to the existing layout object, allowing it to act as an ActiveLayout. let dummy = new ActiveKeyBase(); let proto = Object.getPrototypeOf(dummy); @@ -804,9 +825,11 @@ export class ActiveLayout implements LayoutFormFactor{ * @param formFactor */ static polyfill(layout: LayoutFormFactor, keyboard: Keyboard, formFactor: DeviceSpec.FormFactor): ActiveLayout { + /* c8 ignore start */ if(layout == null) { throw new Error("Cannot build an ActiveLayout for a null specification."); } + /* c8 ignore end */ const analysisMetadata: AnalysisMetadata = { hasFlicks: false, diff --git a/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts b/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts index 8a5aff7787b..e60a1824e43 100644 --- a/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts +++ b/common/web/keyboard-processor/src/keyboards/defaultLayouts.ts @@ -526,6 +526,15 @@ export class Layouts { return KLS; } + static dfltShiftMultitap: LayoutSubKey = { + // Needs to be something special and unique. Typing restricts us from + // using a reserved key-id prefix, though. + id: "T_*_MT_SHIFT_TO_CAPS", + text: '*ShiftLock*', + sp: 1, + nextlayer: 'caps' + } + // Defines the default visual layout for a keyboard. /* c8 ignore start */ static dfltLayout: LayoutSpec = { diff --git a/common/web/keyboard-processor/src/keyboards/keyboard.ts b/common/web/keyboard-processor/src/keyboards/keyboard.ts index 9738792cda1..6a2f60bd724 100644 --- a/common/web/keyboard-processor/src/keyboards/keyboard.ts +++ b/common/web/keyboard-processor/src/keyboards/keyboard.ts @@ -1,6 +1,6 @@ import Codes from "../text/codes.js"; import { Layouts, type LayoutFormFactor } from "./defaultLayouts.js"; -import { ActiveKey, ActiveKeyBase, ActiveLayout } from "./activeLayout.js"; +import { ActiveKey, ActiveLayout, ActiveSubKey } from "./activeLayout.js"; import KeyEvent from "../text/keyEvent.js"; import type OutputTarget from "../text/outputTarget.js"; @@ -468,7 +468,7 @@ export default class Keyboard { return keyEvent; } - constructKeyEvent(key: ActiveKeyBase, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent { + constructKeyEvent(key: ActiveKey | ActiveSubKey, device: DeviceSpec, stateKeys: StateKeyMap): KeyEvent { // Make a deep copy of our preconstructed key event, filling it out from there. const Lkc = key.baseKeyEvent; Lkc.device = device; diff --git a/common/web/keyboard-processor/src/text/keyEvent.ts b/common/web/keyboard-processor/src/text/keyEvent.ts index eb15bc0d1c8..4e7e25b1cc8 100644 --- a/common/web/keyboard-processor/src/text/keyEvent.ts +++ b/common/web/keyboard-processor/src/text/keyEvent.ts @@ -75,6 +75,7 @@ export default class KeyEvent implements KeyEventSpec { kLayer?: string; // The key's layer property kbdLayer?: string; // The virtual keyboard's active layer kNextLayer?: string; + baseTranscriptionToken?: number; /** * Marks the active keyboard at the time that this KeyEvent was generated by the user. diff --git a/common/web/lm-worker/src/main/correction/context-tracker.ts b/common/web/lm-worker/src/main/correction/context-tracker.ts index ec67030cc80..ce235562e2f 100644 --- a/common/web/lm-worker/src/main/correction/context-tracker.ts +++ b/common/web/lm-worker/src/main/correction/context-tracker.ts @@ -83,7 +83,7 @@ export class TrackedContextState { this.tokens = source.tokens.map(function(token) { let copy = new TrackedContextToken(); copy.raw = token.raw; - copy.replacements = [].concat(token.replacements) + copy.replacements = [].concat(token.replacements); copy.activeReplacementId = token.activeReplacementId; copy.transformDistributions = [].concat(token.transformDistributions); @@ -98,7 +98,7 @@ export class TrackedContextState { const lexicalModel = this.model = obj.model; this.taggedContext = obj.taggedContext; - if(lexicalModel && lexicalModel.traverseFromRoot) { + if(lexicalModel?.traverseFromRoot) { // We need to construct a separate search space from other ContextStates. // // In case we are unable to perfectly track context (say, due to multitaps) diff --git a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file.ts b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file.ts index bc7fa032263..599a0360569 100644 --- a/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file.ts +++ b/common/web/types/src/keyman-touch-layout/keyman-touch-layout-file.ts @@ -41,8 +41,20 @@ export interface TouchLayoutRow { type Key_Type = 'T'|'K'|'U'|'t'|'k'|'u'; type Key_Id = string; + export type TouchLayoutKeyId = `${Key_Type}_${Key_Id}`; // pattern = /^[TKUtku]_[a-zA-Z0-9_]+$/ +/** + * Denotes private-use identifiers that should be considered 'reserved'. + */ +export const PRIVATE_USE_IDS = [ + /** + * A private-use identifier used by KeymanWeb for the default multitap-into-caps-layer key + * for keyboards with a caps layer while not defining multitaps on shift. + */ + 'T_*_MT_SHIFT_TO_CAPS' +] as const; + /* A map of key field names with values matching the `typeof` the corresponding property * exists in common/web/keyboard-processor, keyboards/activeLayout.ts. * diff --git a/web/src/engine/events/build.sh b/web/src/engine/events/build.sh index c93c5358317..974a2c3332d 100755 --- a/web/src/engine/events/build.sh +++ b/web/src/engine/events/build.sh @@ -16,7 +16,8 @@ cd "$THIS_SCRIPT_PATH" # ################################ Main script ################################ builder_describe "Builds specialized event-related modules utilized by Keyman Engine for Web." \ - "@/common/web/utils" \ + "@/common/web/utils build" \ + "@/common/web/keyboard-processor build" \ "clean" \ "configure" \ "build" \ diff --git a/web/src/engine/events/src/index.ts b/web/src/engine/events/src/index.ts index ee5674b66c8..2d95307bc9f 100644 --- a/web/src/engine/events/src/index.ts +++ b/web/src/engine/events/src/index.ts @@ -1,3 +1,4 @@ export { DomEventTracker } from './domEventTracker.js'; export { EmitterListenerSpy } from './emitterListenerSpy.js'; +export * from './keyEventSource.interface.js'; export * from './legacyEventEmitter.js'; \ No newline at end of file diff --git a/web/src/engine/main/src/keyEventSource.interface.ts b/web/src/engine/events/src/keyEventSource.interface.ts similarity index 82% rename from web/src/engine/main/src/keyEventSource.interface.ts rename to web/src/engine/events/src/keyEventSource.interface.ts index e2698345af9..31a1eb41296 100644 --- a/web/src/engine/main/src/keyEventSource.interface.ts +++ b/web/src/engine/events/src/keyEventSource.interface.ts @@ -11,4 +11,4 @@ interface EventMap { 'keyevent': KeyEventHandler; } -export default interface KeyEventSourceInterface extends EventEmitter { } \ No newline at end of file +export interface KeyEventSourceInterface extends EventEmitter { } \ No newline at end of file diff --git a/web/src/engine/events/tsconfig.json b/web/src/engine/events/tsconfig.json index 9dbb4840b6d..bb25ac9c99b 100644 --- a/web/src/engine/events/tsconfig.json +++ b/web/src/engine/events/tsconfig.json @@ -11,6 +11,7 @@ "include": [ "src/**/*.ts" ], "references": [ - { "path": "../../../../common/web/utils" } + { "path": "../../../../common/web/utils" }, + { "path": "../../../../common/web/keyboard-processor" } ] } diff --git a/web/src/engine/main/src/hardKeyboard.ts b/web/src/engine/main/src/hardKeyboard.ts index 71504a0187f..05778fc5881 100644 --- a/web/src/engine/main/src/hardKeyboard.ts +++ b/web/src/engine/main/src/hardKeyboard.ts @@ -1,6 +1,6 @@ import EventEmitter from "eventemitter3"; import { type KeyEvent, type RuleBehavior } from "@keymanapp/keyboard-processor"; -import KeyEventSourceInterface from './keyEventSource.interface.js'; +import { KeyEventSourceInterface } from 'keyman/engine/events'; interface EventMap { /** @@ -9,7 +9,7 @@ interface EventMap { 'keyevent': (event: KeyEvent, callback?: (result: RuleBehavior, error?: Error) => void) => void } -export default class HardKeyboard extends EventEmitter implements KeyEventSourceInterface { } +export default class HardKeyboard extends EventEmitter implements KeyEventSourceInterface { } // Intended design: // - KeyEventKeyboard: website-integrated handler for hardware-keystroke input; interprets DOM events. diff --git a/web/src/engine/main/src/keymanEngine.ts b/web/src/engine/main/src/keymanEngine.ts index b39680dab9d..2b3fd71c874 100644 --- a/web/src/engine/main/src/keymanEngine.ts +++ b/web/src/engine/main/src/keymanEngine.ts @@ -7,7 +7,7 @@ import { KeyboardRequisitioner, ModelCache, ModelSpec, toUnprefixedKeyboardId as import { EngineConfiguration, InitOptionSpec } from "./engineConfiguration.js"; import KeyboardInterface from "./keyboardInterface.js"; import { ContextManagerBase } from "./contextManagerBase.js"; -import { KeyEventHandler } from './keyEventSource.interface.js'; +import { KeyEventHandler } from 'keyman/engine/events'; import HardKeyboardBase from "./hardKeyboard.js"; import { LegacyAPIEvents } from "./legacyAPIEvents.js"; import { EventNames, EventListener, LegacyEventEmitter } from "keyman/engine/events"; diff --git a/web/src/engine/osk/src/input/gestures/browser/multitap.ts b/web/src/engine/osk/src/input/gestures/browser/multitap.ts new file mode 100644 index 00000000000..26beec8d479 --- /dev/null +++ b/web/src/engine/osk/src/input/gestures/browser/multitap.ts @@ -0,0 +1,170 @@ +import { type KeyElement } from '../../../keyElement.js'; +import VisualKeyboard from '../../../visualKeyboard.js'; + +import { DeviceSpec, KeyEvent, ActiveSubKey, ActiveKey, KeyDistribution, ActiveKeyBase } from '@keymanapp/keyboard-processor'; +import { GestureSequence } from '@keymanapp/gesture-recognizer'; +import { GestureHandler } from '../gestureHandler.js'; +import { distributionFromDistanceMaps } from '@keymanapp/input-processor'; + +/** + * Represents a potential multitap gesture's implementation within KeymanWeb. + * Once a simple-tap gesture occurs on a key with specified multitap subkeys, + * this class is designed to take over further processing of said gesture. + * This includes providing: + * * UI feedback regarding the state of the ongoing multitap, as appropriate + * * Proper selection of the appropriate multitap key for subsequent taps. + */ +export default class Multitap implements GestureHandler { + public readonly baseKey: KeyElement; + public readonly baseContextToken: number; + public readonly hasModalVisualization = false; + + private readonly originalLayer: string; + + private readonly multitaps: ActiveSubKey[]; + private tapIndex = 0; + + private sequence: GestureSequence; + + constructor( + source: GestureSequence, + vkbd: VisualKeyboard, + e: KeyElement, + contextToken: number + ) { + this.baseKey = e; + this.baseContextToken = contextToken; + this.multitaps = [e.key.spec].concat(e.key.spec.multitap); + + this.originalLayer = vkbd.layerId; + + // // For multitaps, keeping the key highlighted makes sense. I think. + // this.baseKey.key.highlight(true); + + source.on('complete', () => { + if(source.stageReports.length > 1) { + } + // this.currentSelection?.key.highlight(false); + this.clear(); + }); + + source.on('stage', (tap) => { + switch(tap.matchedId) { + case 'modipress-multitap-end': + case 'modipress-end': + case 'multitap-end': + case 'simple-tap': + return; + // Once a multitap starts, it's better to emit keys on keydown; that way, + // if a user holds long, they get what they see if they decide to stop, + // but also have time to decide if they want to continue to what's next. + case 'multitap-start': + case 'modipress-multitap-start': + break; + default: + throw new Error(`Unsupported gesture state encountered during multitap: ${tap.matchedId}`); + } + + // For rota-style behavior + this.tapIndex = (this.tapIndex + 1) % this.multitaps.length; + const selection = this.multitaps[this.tapIndex]; + + const keyEvent = vkbd.keyEventFromSpec(selection); + keyEvent.baseTranscriptionToken = this.baseContextToken; + + const coord = tap.sources[0].currentSample; + const baseDistances = vkbd.getSimpleTapCorrectionDistances(coord, this.baseKey.key.spec as ActiveKey); + if(coord.stateToken != vkbd.layerId) { + const matchKey = vkbd.layerGroup.findNearestKey({...coord, stateToken: vkbd.layerId}); + + // Replace the key at the current location for the current layer key + // with the multitap base key. + const p = baseDistances.get(matchKey.key.spec); + if(p == null) { + console.warn("Could not find current layer's key") + } + baseDistances.delete(matchKey.key.spec); + baseDistances.set(coord.item.key.spec, p); + } + keyEvent.keyDistribution = this.currentStageKeyDistribution(baseDistances); + + // When _some_ multitap keys support layer-swapping but others don't, + // landing on a non-swap key should preserve the original layer... even + // if no such 'nextLayer' is specified by default. + keyEvent.kNextLayer ||= this.originalLayer; + + vkbd.raiseKeyEvent(keyEvent, null); + }); + + /* In theory, setting up a specialized recognizer config limited to the base key's surface area + * would be pretty ideal - it'd provide automatic cancellation if anywhere else were touched. + * + * However, because multitap keys can swap layers, and because an invisible layer doesn't provide + * the expected bounding-box that it would were it visible, it's anything but straightforward to + * do for certain supported cases. It's simpler to handle this problem by leveraging the + * key-finding operation specified on the gesture model and ensuring the base key remains in place. + */ + } + + currentStageKeyDistribution(baseDistances: Map): KeyDistribution { + /* Concept: use the base distance map - what if the tap was meant for elsewhere? + * That said, given the base key's probability... modify that by a 'tap distance' metric, + * where the probability of all taps in the multitap rota sum up to the base key's original + * probability. + */ + + const baseDistribution = distributionFromDistanceMaps(baseDistances); + const keyIndex = baseDistribution.findIndex((entry) => entry.keySpec == this.baseKey.key.spec); + + if(keyIndex == -1) { // also covers undefined, but does not include 0. + console.warn("Could not find base key's probability for multitap correction"); + + // Decently recoverable; just use the simple-tap distances instead. + return baseDistribution; + } + + const baseProb = baseDistribution.splice(keyIndex, 1)[0].p; + + let totalWeight = 0; + let multitapEntries: {keySpec: ActiveKeyBase, p: number}[] = []; + for(let i = 0; i < this.multitaps.length; i++) { + const key = this.multitaps[i]; + // 'standard distance', no real modular effects needed. + const distStd = Math.abs(i - this.tapIndex) % this.multitaps.length; + // 'wrapped distance', when the modular effects are definitely needed. + const distWrap = (i + this.multitaps.length - this.tapIndex) % this.multitaps.length; + const modularLinDist = distStd < distWrap ? distStd : distWrap; + + // Simple approach for now - we'll ignore timing considerations and + // just use raw modular distance. + // Actual tap: 1 (base weight) + // "one off": 1/4 as likely + // "two off": 1/9 as likely + // etc. + const keyWeight = 1.0 / ((1 + modularLinDist) * (1 + modularLinDist)); + totalWeight += keyWeight; + multitapEntries.push({ + keySpec: key, + p: keyWeight + }); + } + + // Converts from the weights to the final probability values specified by the + // top comment within this method. + const scalar = baseProb / totalWeight; + multitapEntries.forEach((entry) => { + entry.p = scalar * entry.p; + }); + + return baseDistribution.concat(multitapEntries).sort((a, b) => b.p - a.p); + } + + cancel() { + this.clear(); + this.sequence.cancel(); + } + + clear() { + // TODO: for hint stuff. + } +} \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/browser/pendingMultiTap.ts b/web/src/engine/osk/src/input/gestures/browser/pendingMultiTap.ts deleted file mode 100644 index 4f26862e8d8..00000000000 --- a/web/src/engine/osk/src/input/gestures/browser/pendingMultiTap.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { Codes, KeyEvent } from '@keymanapp/keyboard-processor'; -import { type KeyElement } from '../../../keyElement.js'; -import VisualKeyboard from '../../../visualKeyboard.js'; -// import PendingGesture from '../pendingGesture.interface.js'; - -export enum PendingMultiTapState { Waiting, Realized, Cancelled }; -/** - * Implements the multi-tap gesture, which is a series of taps on a single key - * (based on key id substring in the case of the shift key), within a - * specified timeout period. - */ -export default class PendingMultiTap { - public readonly vkbd: VisualKeyboard; - public readonly baseKey: KeyElement; - public readonly count: number; - private timerId; - private _touches = 1; // we start the multitap with a single touch - private _state: PendingMultiTapState = PendingMultiTapState.Waiting; - private _timeout: Promise; - private cancelDelayFactor = 125; // 125msec * count - private _destinationLayerId; - - public get timeout() { - return this._timeout; - } - public get realized() { - return this._state == PendingMultiTapState.Realized; - } - public get cancelled() { - return this._state == PendingMultiTapState.Cancelled; - } - - /** - * Construct a record of a potential multitap gesture - * @param vkbd - * @param baseKey key which is being tapped - * @param count number of taps required to finalize this gesture - */ - constructor(vkbd: VisualKeyboard, baseKey: KeyElement, count: number) { - this.vkbd = vkbd; - this.count = count; - this.baseKey = baseKey; - - this._destinationLayerId = 'caps'; - let multitap = baseKey?.key?.spec?.['multitap']; - if(multitap?.length && multitap[0]?.['nextlayer']) { - this._destinationLayerId = multitap[0]['nextlayer']; - } - - const _this = this; - this._timeout = new Promise(function(resolve) { - // If multiple taps do not occur within the timeout window, - // then we will abandon the gesture - _this.timerId = window.setTimeout(() => { - _this.cancel(); - resolve(); - }, _this.cancelDelayFactor * _this.count); - }); - } - - public static isValidTarget(vkbd: VisualKeyboard, baseKey: KeyElement) { - // Could use String.includes, but Chrome for Android must be version 41+. - // We support down to version 37. - return ( - baseKey['keyId'].indexOf('K_SHIFT') >= 0 && - vkbd.layerGroup.layers['caps'] && - !baseKey['subKeys'] && - vkbd.touchCount == 1 - ); - } - - private cleanup(): void { - if(this.timerId) { - window.clearTimeout(this.timerId); - } - this.timerId = null; - } - - /** - * Cancel a pending multitap gesture - */ - public cancel(): void { - this._state = PendingMultiTapState.Cancelled; - this.cleanup(); - } - - /** - * Increments the touch counter for the gesture, and - * if the touch count is reached, realize the gesture - * @returns new state of the gesture - */ - public incrementTouch(newKey: KeyElement): PendingMultiTapState { - // TODO: support for any key - if(this._state == PendingMultiTapState.Waiting) { - if(!newKey?.['keyId']?.includes('K_SHIFT')) { - this.cancel(); - } - else if(++this._touches == this.count) { - this.realize(); - } - } - return this._state; - } - - /** - * Realize the gesture. In Keyman 15, this supports only - * the Caps double-tap gesture on the Shift key. - */ - public realize(): void { - if(this._state != PendingMultiTapState.Waiting) { - return; - } - this._state = PendingMultiTapState.Realized; - this.cleanup(); - - // In Keyman 15, only the K_SHIFT key supports multi-tap, so we can hack - // in the switch to the caps layer. - // - // TODO: generalize this with double-tap key properties in touch layout - // description. - let e = KeyEvent.constructNullKeyEvent(this.vkbd.device); - e.kNextLayer = this._destinationLayerId; - e.Lstates = Codes.stateBitmasks.CAPS; - e.LmodifierChange = true; - this.vkbd.raiseKeyEvent(e, null); - } -} \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts b/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts index 536cc1eb8b6..3eb3a646445 100644 --- a/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts +++ b/web/src/engine/osk/src/input/gestures/browser/subkeyPopup.ts @@ -36,7 +36,7 @@ export default class SubkeyPopup implements GestureHandler { private readonly gestureParams: GestureParams; constructor( - source: GestureSequence, + source: GestureSequence, configChanger: ConfigChangeClosure, vkbd: VisualKeyboard, e: KeyElement, diff --git a/web/src/engine/osk/src/input/gestures/gestureHandler.ts b/web/src/engine/osk/src/input/gestures/gestureHandler.ts index da9b1c2cb0e..280ed03a153 100644 --- a/web/src/engine/osk/src/input/gestures/gestureHandler.ts +++ b/web/src/engine/osk/src/input/gestures/gestureHandler.ts @@ -1,4 +1,4 @@ -import { KeyDistribution } from "@keymanapp/keyboard-processor"; +import { ActiveKeyBase, KeyDistribution } from "@keymanapp/keyboard-processor"; export interface GestureHandler { /** @@ -12,5 +12,17 @@ export interface GestureHandler { */ readonly hasModalVisualization: boolean; - currentStageKeyDistribution(): KeyDistribution; + /** + * Implementations of this method should return an appropriate statistic distribution + * for the likelihood of most-relevant keys that may have been intended for the + * most recent keystroke generated. Alternatively, returning `null` or `undefined` + * will use the default simple-tap distribution. + * + * This method will be provided a map of the "corrective distance" used for + * simple-tap corrections, allowing gestures to utilize the values as a basis + * for their own calculations as appropriate. + * + * @param baseDistMap The distance map used for simple-tap corrections + */ + currentStageKeyDistribution(baseDistMap: Map): KeyDistribution | null; } \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/heldRepeater.ts b/web/src/engine/osk/src/input/gestures/heldRepeater.ts index 57c2b160b9b..a462758fac5 100644 --- a/web/src/engine/osk/src/input/gestures/heldRepeater.ts +++ b/web/src/engine/osk/src/input/gestures/heldRepeater.ts @@ -8,13 +8,13 @@ export class HeldRepeater implements GestureHandler { static readonly INITIAL_DELAY = 500; static readonly REPEAT_DELAY = 100; - readonly source: GestureSequence; + readonly source: GestureSequence; readonly hasModalVisualization = false; readonly repeatClosure: () => void; timerHandle: number; - constructor(source: GestureSequence, closureToRepeat: () => void) { + constructor(source: GestureSequence, closureToRepeat: () => void) { this.source = source; this.repeatClosure = closureToRepeat; diff --git a/web/src/engine/osk/src/input/gestures/specsForLayout.ts b/web/src/engine/osk/src/input/gestures/specsForLayout.ts index 64a21de48e2..7e1e1c0f1f8 100644 --- a/web/src/engine/osk/src/input/gestures/specsForLayout.ts +++ b/web/src/engine/osk/src/input/gestures/specsForLayout.ts @@ -41,10 +41,23 @@ export interface GestureParams { noiseTolerance: number, /** - * The duration that the base key must be held before the subkey menu will be displayed - * should the up-flick shortcut not be utilized. + * The duration (in ms) that the base key must be held before the subkey menu will be + * displayed should the up-flick shortcut not be utilized. */ waitLength: number + }, + multitap: { + /** + * The duration (in ms) permitted between taps. Taps with a greater time interval + * between them will be considered separate. + */ + waitLength: number; + + /** + * The duration (in ms) permitted for a tap to be held before it will no longer + * be considered part of a multitap. + */ + holdLength: number; } } @@ -54,6 +67,10 @@ export const DEFAULT_GESTURE_PARAMS: GestureParams = { flickDist: 5, waitLength: 500, noiseTolerance: 10 + }, + multitap: { + waitLength: 500, + holdLength: 500 } } @@ -65,7 +82,7 @@ export const DEFAULT_GESTURE_PARAMS: GestureParams = { * immediate effect during gesture processing. * @returns */ -export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GestureParams): GestureModelDefs { +export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GestureParams): GestureModelDefs { const layout = layerGroup.spec; // To be used among the `allowsInitialState` contact-model specifications as needed. @@ -76,16 +93,12 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa return ['K_LOPT', 'K_ROPT', 'K_BKSP'].indexOf(keySpec.baseKeyID) != -1; case 'longpress': return !!keySpec.sk; - case 'multitap': + case 'multitap-start': + case 'modipress-multitap-start': if(layout.hasMultitaps) { return !!keySpec.multitap; - } else if(layout.formFactor != 'desktop') { - // maintain our special caps-shifting? - // if(keySpec.baseKeyID == 'K_SHIFT') { - - // } else { + } else { return false; - // } } case 'flick': // This is a gesture-start check; there won't yet be any directional info available. @@ -95,6 +108,7 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa } }; + const _initialTapModel: GestureModel = deepCopy(layout.hasFlicks ? initialTapModel(params) : initialTapModelWithReset(params)); const simpleTapModel: GestureModel = deepCopy(layout.hasFlicks ? SimpleTapModel : SimpleTapModelWithReset); const longpressModel: GestureModel = deepCopy(layout.hasFlicks ? basicLongpressModel(params) : longpressModelWithShortcut(params)); @@ -124,63 +138,25 @@ export function gestureSetForLayout(layerGroup: OSKLayerGroup, params: GesturePa return model; } - - function withLayerChangeItemFix(model: GestureModel, contactIndices: number | number[]) { - // Creates deep copies of the model specifications that are safe to customize to the - // keyboard layout. - model = deepCopy(model); - - if(typeof contactIndices == 'number') { - contactIndices = [contactIndices]; - } - - model.contacts.forEach((contact, index) => { - if((contactIndices as number[]).indexOf(index) != -1) { - const baseInitialStateCheck = contact.model.allowsInitialState ?? (() => true); - - contact.model = { - ...contact.model, - // And now for the true purpose of the method. - allowsInitialState: (sample, ancestorSample, baseKey) => { - // By default, the state token is set to whatever the current layer is for a source. - // - // So, if the first tap of a key swaps layers, the second tap will be on the wrong layer and - // thus have a different state token. This is the perfect place to detect and correct that. - if(ancestorSample.stateToken != sample.stateToken) { - sample.stateToken = ancestorSample.stateToken; - - // Specialized item lookup is required here for proper 'correction' - we want the key - // corresponding to our original layer, not the new layer here. Now that we've identified - // the original OSK layer (state) for the gesture, we can find the best matching key - // from said layer instead of the current layer. - // - // Matters significantly for multitaps if and when they include layer-switching specs. - sample.item = layerGroup.findNearestKey(sample); - } - - return baseInitialStateCheck(sample, ancestorSample, baseKey); - } - }; - } - }); - - return model; - } // #endregion const gestureModels = [ withKeySpecFiltering(longpressModel, 0), - withLayerChangeItemFix(withKeySpecFiltering(MultitapModel, 0), 0), + withKeySpecFiltering(multitapStartModel(params), 0), + multitapEndModel(params), + _initialTapModel, simpleTapModel, withKeySpecFiltering(SpecialKeyStartModel, 0), SpecialKeyEndModel, SubkeySelectModel, withKeySpecFiltering(ModipressStartModel, 0), - ModipressEndModel + ModipressEndModel, + withKeySpecFiltering(modipressMultitapStartModel(params), 0), + modipressMultitapEndModel(params) ]; const defaultSet = [ - longpressModel.id, SimpleTapModel.id, ModipressStartModel.id, SpecialKeyStartModel.id + longpressModel.id, _initialTapModel.id, ModipressStartModel.id, SpecialKeyStartModel.id ]; if(layout.hasFlicks) { @@ -522,35 +498,101 @@ export function longpressModelWithRoaming(params: GestureParams): GestureModel { + return { + id: 'multitap-start', + resolutionPriority: 2, + contacts: [ + { + model: { + ...InstantContactResolutionModel, + itemPriority: 1, + pathInheritance: 'reject', + allowsInitialState(incomingSample, comparisonSample, baseItem) { + return incomingSample.item == baseItem; + }, + }, + } + ], + sustainTimer: { + duration: params.multitap.waitLength, + expectedResult: false, + baseItem: 'base' + }, + resolutionAction: { + type: 'chain', + next: 'multitap-end', + item: 'current' + } + } +} -export const MultitapModel: GestureModel = { - id: 'multitap', - resolutionPriority: 2, - contacts: [ - { - model: { - ...SimpleTapContactModel, - itemPriority: 1, - pathInheritance: 'reject', - allowsInitialState(incomingSample, comparisonSample, baseItem) { - return incomingSample.item == baseItem; +export function multitapEndModel(params: GestureParams): GestureModel { + return { + id: 'multitap-end', + resolutionPriority: 2, + contacts: [ + { + model: { + ...SimpleTapContactModel, + itemPriority: 1, + timer: { + duration: params.multitap.holdLength, + expectedResult: false + } }, - }, - endOnResolve: true - }, { - model: InstantContactResolutionModel, - resetOnResolve: true + endOnResolve: true + }, { + model: InstantContactResolutionModel, + resetOnResolve: true + } + ], + rejectionActions: { + timer: { + type: 'replace', + replace: 'simple-tap' + } + }, + resolutionAction: { + type: 'chain', + next: 'multitap-start', + item: 'none' + } + } +} + +export function initialTapModel(params: GestureParams): GestureModel { + return { + id: 'initial-tap', + resolutionPriority: 1, + contacts: [ + { + model: { + ...SimpleTapContactModel, + pathInheritance: 'chop', + itemPriority: 1, + timer: { + duration: params.multitap.holdLength, + expectedResult: false + } + }, + endOnResolve: true + }, { + model: InstantContactResolutionModel, + resetOnResolve: true + } + ], + rejectionActions: { + timer: { + type: 'replace', + replace: 'simple-tap' + } + }, + resolutionAction: { + type: 'chain', + next: 'multitap-start', + item: 'current' } - ], - sustainTimer: { - duration: 500, - expectedResult: false, - baseItem: 'base' - }, - resolutionAction: { - type: 'chain', - next: 'multitap', - item: 'current' } } @@ -571,15 +613,29 @@ export const SimpleTapModel: GestureModel = { } ], resolutionAction: { - type: 'chain', - next: 'multitap', + type: 'complete', item: 'current' } } +export function initialTapModelWithReset(params: GestureParams): GestureModel { + const base = initialTapModel(params); + return { + ...base, + rejectionActions: { + ...base.rejectionActions, + item: { + type: 'replace', + replace: 'initial-tap' + } + } + } +} + export const SimpleTapModelWithReset: GestureModel = { ...SimpleTapModel, rejectionActions: { + ...SimpleTapModel.rejectionActions, item: { type: 'replace', replace: 'simple-tap' @@ -655,13 +711,97 @@ export const ModipressEndModel: GestureModel = { { model: { ...ModipressContactEndModel, - itemChangeAction: 'reject' + itemChangeAction: 'reject', + pathInheritance: 'full' } } ], resolutionAction: { - type: 'complete', + type: 'chain', + // Because SHIFT -> CAPS multitap is a thing. Shift gets handled as a modipress first. + // Modipresses resolve before multitaps... unless there's a model designed to handle & disambiguate both. + next: 'modipress-multitap-start', + // Key was already emitted from the 'modipress-start' stage. item: 'none' } } + +export function modipressMultitapStartModel(params: GestureParams): GestureModel { + return { + id: 'modipress-multitap-start', + resolutionPriority: 6, + contacts: [ + { + model: { + ...ModipressContactStartModel, + pathInheritance: 'reject', + allowsInitialState(incomingSample, comparisonSample, baseItem) { + if(incomingSample.item != baseItem) { + return false; + } + // TODO: needs better abstraction, probably. + + // But, to get started... we can just use a simple hardcoded approach. + const modifierKeyIds = ['K_SHIFT', 'K_ALT', 'K_CTRL']; + for(const modKeyId of modifierKeyIds) { + if(baseItem.key.spec.id == modKeyId) { + return true; + } + } + + return false; + }, + itemChangeAction: 'reject', + itemPriority: 1 + } + } + ], + sustainTimer: { + duration: params.multitap.waitLength, + expectedResult: false, + baseItem: 'base' + }, + resolutionAction: { + type: 'chain', + next: 'modipress-multitap-end', + selectionMode: 'modipress', + item: 'current' // return the modifier key ID so that we know to shift to it! + } + } +} + +export function modipressMultitapEndModel(params: GestureParams): GestureModel { + return { + id: 'modipress-multitap-end', + resolutionPriority: 5, + contacts: [ + { + model: { + ...ModipressContactEndModel, + itemChangeAction: 'reject', + pathInheritance: 'full', + timer: { + // will need something similar for base modipress. + duration: params.multitap.holdLength, + expectedResult: false + } + } + } + ], + resolutionAction: { + type: 'chain', + // Because SHIFT -> CAPS multitap is a thing. Shift gets handled as a modipress first. + // TODO: maybe be selective about it: if the tap occurs within a set amount of time? + next: 'modipress-multitap-start', + // Key was already emitted from the 'modipress-start' stage. + item: 'none' + }, + rejectionActions: { + timer: { + type: 'replace', + replace: 'modipress-end' + } + } + } +} // #endregion \ No newline at end of file diff --git a/web/src/engine/osk/src/views/oskView.ts b/web/src/engine/osk/src/views/oskView.ts index 356e3beae5d..4697f9f8802 100644 --- a/web/src/engine/osk/src/views/oskView.ts +++ b/web/src/engine/osk/src/views/oskView.ts @@ -21,7 +21,7 @@ import { type SystemStoreMutationHandler } from '@keymanapp/keyboard-processor'; import { createUnselectableElement, getAbsoluteX, getAbsoluteY, StylesheetManager } from 'keyman/engine/dom-utils'; -import { EventListener, EventNames, LegacyEventEmitter } from 'keyman/engine/events'; +import { EventListener, EventNames, KeyEventHandler, KeyEventSourceInterface, LegacyEventEmitter } from 'keyman/engine/events'; import Configuration from '../config/viewConfiguration.js'; import Activator, { StaticActivator } from './activator.js'; @@ -66,7 +66,7 @@ export interface EventMap { * Note: the following code block was originally used to integrate with the keyboard & input * processors, but it requires entanglement with components external to this OSK module. */ - 'keyevent': (event: KeyEvent) => void, + 'keyevent': KeyEventHandler, /** * Indicates that the globe key has either been pressed (`on` == `true`) @@ -115,7 +115,9 @@ export interface EventMap { pointerinteraction: (promise: Promise) => void; } -export default abstract class OSKView extends EventEmitter implements MinimalCodesInterface { +export default abstract class OSKView + extends EventEmitter + implements MinimalCodesInterface, KeyEventSourceInterface { _Box: HTMLDivElement; readonly legacyEvents = new LegacyEventEmitter(); @@ -811,7 +813,7 @@ export default abstract class OSKView extends EventEmitter implements isEmbedded: this.config.isEmbedded }); - vkbd.on('keyevent', (keyEvent) => this.emit('keyevent', keyEvent)); + vkbd.on('keyevent', (keyEvent, callback) => this.emit('keyevent', keyEvent, callback)); vkbd.on('globekey', (keyElement, on) => this.emit('globekey', keyElement, on)); vkbd.on('hiderequested', (keyElement) => { this.doHide(true); diff --git a/web/src/engine/osk/src/visualKeyboard.ts b/web/src/engine/osk/src/visualKeyboard.ts index 8b77432218c..739629139b3 100644 --- a/web/src/engine/osk/src/visualKeyboard.ts +++ b/web/src/engine/osk/src/visualKeyboard.ts @@ -12,7 +12,10 @@ import { KeyEvent, Layouts, StateKeyMap, - LayoutKey + LayoutKey, + ActiveSubKey, + timedPromise, + ActiveKeyBase } from '@keymanapp/keyboard-processor'; import { buildCorrectiveLayout, distributionFromDistanceMaps, keyTouchDistances } from '@keymanapp/input-processor'; @@ -28,6 +31,8 @@ import { import { createStyleSheet, getAbsoluteX, getAbsoluteY, StylesheetManager } from 'keyman/engine/dom-utils'; +import { KeyEventHandler, KeyEventResultCallback } from 'keyman/engine/events'; + import GlobeHint from './globehint.interface.js'; import KeyboardView from './components/keyboardView.interface.js'; import { type KeyElement, getKeyFrom } from './keyElement.js'; @@ -37,7 +42,6 @@ import OSKLayer from './keyboard-layout/oskLayer.js'; import OSKLayerGroup from './keyboard-layout/oskLayerGroup.js'; import { LengthStyle, ParsedLengthStyle } from './lengthStyle.js'; import { defaultFontSize, getFontSizeStyle } from './fontSizeUtils.js'; -import PendingMultiTap, { PendingMultiTapState } from './input/gestures/browser/pendingMultiTap.js'; import InternalKeyTip from './input/gestures/browser/keytip.js'; import CommonConfiguration from './config/commonConfiguration.js'; @@ -46,6 +50,7 @@ import { DEFAULT_GESTURE_PARAMS, GestureParams, gestureSetForLayout } from './in import { getViewportScale } from './screenUtils.js'; import { HeldRepeater } from './input/gestures/heldRepeater.js'; import SubkeyPopup from './input/gestures/browser/subkeyPopup.js'; +import Multitap from './input/gestures/browser/multitap.js'; import { GestureHandler } from './input/gestures/gestureHandler.js'; export interface VisualKeyboardConfiguration extends CommonConfiguration { @@ -97,7 +102,7 @@ interface EventMap { * Note: the following code block was originally used to integrate with the keyboard & input * processors, but it requires entanglement with components external to this OSK module. */ - 'keyevent': (event: KeyEvent) => void, + 'keyevent': KeyEventHandler, 'hiderequested': (keyElement: KeyElement) => void, @@ -200,9 +205,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke activeGestures: GestureHandler[] = []; - // Multi-tap gesture management - pendingMultiTap: PendingMultiTap; - // The keyboard object corresponding to this VisualKeyboard. public readonly layoutKeyboard: Keyboard; public readonly layoutKeyboardProperties: KeyboardProperties; @@ -413,6 +415,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke const endHighlighting = () => { if(trackingEntry.key) { this.highlightKey(trackingEntry.key, false); + trackingEntry.key = null; } } @@ -455,13 +458,30 @@ export default class VisualKeyboard extends EventEmitter implements Ke // Disable roaming-touch highlighting (and current highlighting) for all // touchpoints included in a gesture, even newly-included ones as they occur. for(let id of gestureStage.allSourceIds) { - const trackingEntry = sourceTrackingMap[id]; - if(trackingEntry.key) { - this.highlightKey(trackingEntry.key, false); - trackingEntry.key = null; + const clearRoaming = (trackingEntry: typeof sourceTrackingMap['']) => { + if(trackingEntry.key) { + this.highlightKey(trackingEntry.key, false); + trackingEntry.key = null; + } + + trackingEntry.source.path.off('step', trackingEntry.roamingHighlightHandler); } - trackingEntry.source.path.off('step', trackingEntry.roamingHighlightHandler); + const trackingEntry = sourceTrackingMap[id]; + + if(trackingEntry) { + clearRoaming(trackingEntry); + } else { + // May arise during multitaps, as the 'wait' stage instantly accepts new incoming + // sources before they are reported fully to the `inputstart` event. + const _id = id; + timedPromise(0).then(() => { + const tracker = sourceTrackingMap[_id]; + if(tracker) { + clearRoaming(tracker); + } + }); + } } @@ -479,36 +499,41 @@ export default class VisualKeyboard extends EventEmitter implements Ke coord = coordSource.currentSample; } - if(gestureKey) { - let correctionKeyDistribution: KeyDistribution; - - if(gestureStage.matchedId == 'multitap') { - // TODO: examine sequence, determine rota-style index to apply; select THAT item instead. - } + let keyResult: { + contextToken?: number + } = null; - if(gestureStage.matchedId == 'subkey-select') { - if(!handler) { - throw new Error("Invalid state - reference to subkey menu is missing"); - } - // TODO: examine subkey menu, determine proper set of fat-finger alternates. - correctionKeyDistribution = handler.currentStageKeyDistribution(); + // Multitaps do special key-mapping stuff internally and produce + raise their + // key events directly. + if(gestureKey && !(handler instanceof Multitap)) { + let correctionKeyDistribution: KeyDistribution; + const baseDistanceMap = this.getSimpleTapCorrectionDistances(coord, gestureKey.key.spec as ActiveKey); + + if(handler) { + // Certain gestures (especially flicks) like to consider the base layout as part + // of their corrective-distribution calculations. + // + // May be `null` for gestures that don't need custom correction handling, + // such as modipresses or initial/simple-tap keystrokes. + correctionKeyDistribution = handler.currentStageKeyDistribution(baseDistanceMap); } if(!correctionKeyDistribution) { - correctionKeyDistribution = this.getSimpleTapCorrectionProbabilities(coord, gestureKey.key.spec as ActiveKey); + correctionKeyDistribution = distributionFromDistanceMaps(baseDistanceMap); } // Once the best coord to use for fat-finger calculations has been determined: - this.modelKeyClick(gestureStage.item, coord, correctionKeyDistribution); + keyResult = this.modelKeyClick(gestureStage.item, coord); } // Outside of passing keys along... the handling of later stages is delegated // to gesture-specific handling classes. - if(gestureSequence.stageReports.length > 1) { + if(gestureSequence.stageReports.length > 1 && gestureStage.matchedId != 'modipress-end') { return; } // So, if this is the first stage, this is where we need to perform that delegation. + const baseItem = gestureSequence.stageReports[0].item; // -- Scratch-space as gestures start becoming integrated -- // Reordering may follow at some point. @@ -533,6 +558,9 @@ export default class VisualKeyboard extends EventEmitter implements Ke gestureSequence.stageReports[0].sources[0].baseItem, DEFAULT_GESTURE_PARAMS ); + } else if(baseItem.key.spec.multitap && (gestureStage.matchedId == 'initial-tap' || gestureStage.matchedId == 'multitap' || gestureStage.matchedId == 'modipress-start')) { + // Likewise - mere construction is enough. + handler = new Multitap(gestureSequence, this, baseItem, keyResult.contextToken); } if(handler) { @@ -743,7 +771,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke * @param keySpec The spec of the key directly triggered by the input event. May be for a subkey. * @returns */ - getSimpleTapCorrectionProbabilities(input: InputSample, keySpec?: ActiveKey): KeyDistribution { + getSimpleTapCorrectionDistances(input: InputSample, keySpec?: ActiveKey): Map { // TODO: It'd be nice to optimize by keeping these off when unused, but the wiring // necessary would get in the way of modularization at the moment. // let keyman = com.keyman.singleton; @@ -764,114 +792,9 @@ export default class VisualKeyboard extends EventEmitter implements Ke let kbdAspectRatio = width / height; const correctiveLayout = buildCorrectiveLayout(this.kbdLayout.getLayer(this.layerId), kbdAspectRatio); - const rawSqDistances = keyTouchDistances(touchKbdPos, correctiveLayout); - return distributionFromDistanceMaps(rawSqDistances); + return keyTouchDistances(touchKbdPos, correctiveLayout); } - //#region Input handling start - - // /** - // * The main OSK touch start event handler - // * - // * @param {Event} e touch start event object - // * - // */ - // touch(input: InputEventCoordinate) { - // // Identify the key touched - // var t = input.target, key = this.keyTarget(t); - - // // Special function keys need immediate action - // if (is not special key) { - // if (!this.keyPending) { - // this.initGestures(key, input); - // } - // } - // } - - // /** - // * OSK touch release event handler - // * - // * @param {Event} e touch release event object - // * - // **/ - // release(input: InputEventCoordinate): void { - // // Prevent incorrect multi-touch behaviour if native or device popup visible - // var t = this.currentTarget; - - // // Clear repeated backspace if active, preventing 'sticky' behavior. - // this.cancelDelete(); - - // // Multi-Tap - // if (this.pendingMultiTap && this.pendingMultiTap.realized) { - // // Ignore pending key if we've just handled a multitap - // this.pendingMultiTap = null; - - // this.highlightKey(this.keyPending, false); - // this.keyPending = null; - // this.touchPending = null; - - // return; - // } - - // if (this.pendingMultiTap && this.pendingMultiTap.cancelled) { - // this.pendingMultiTap = null; - // } - // } - - // moveCancel(input: InputEventCoordinate): void { - // // Update all gesture tracking. The function returns true if further input processing - // // should be blocked. (Keeps the subkey array operating when the input coordinate has - // // moved outside the OSK's boundaries.) - // if (this.updateGestures(null, this.keyPending, input)) { - // return; - // } - - // this.cancelDelete(); - // } - - // /** - // * OSK touch move event handler - // * - // * @param {Event} e touch move event object - // * - // **/ - // moveOver(input: InputEventCoordinate): void { - // // Shouldn't be possible, but just in case. - // if (this.touchCount == 0) { - // this.cancelDelete(); - // return; - // } - - // // Get touch position - // const x = input.x - window.pageXOffset; - // const y = input.y - window.pageYOffset; - - // // Move target key and highlighting - // this.touchPending = input; - // // Operates on viewport-based coordinates, not page-based. - // var t1 = document.elementFromPoint(x, y); - // const key0 = this.keyPending; - // let key1 = this.keyTarget(t1); // Not only gets base keys, but also gets popup keys! - - // // Cancels if it's a multitouch attempt. - - // // Do not attempt to support reselection of target key for overlapped keystrokes. - // // Perform _after_ ensuring possible sticky keys have been cancelled. - // if (input.activeInputCount > 1) { - // return; - // } - - // // Gesture-updates should probably be a separate call from other touch-move aspects. - - // // Update all gesture tracking. The function returns true if further input processing - // // should be blocked. - // if (this.updateGestures(key1, key0, input)) { - // return; - // } - // } - - //#endregion - /** * Get the current key target from the touch point element within the key * @@ -930,7 +853,7 @@ export default class VisualKeyboard extends EventEmitter implements Ke keyEvent.keyDistribution = keyDistribution; } - this.raiseKeyEvent(keyEvent, e); + return this.raiseKeyEvent(keyEvent, e); } initKeyEvent(e: KeyElement) { @@ -948,6 +871,11 @@ export default class VisualKeyboard extends EventEmitter implements Ke return null; } + // Return the event object. + return this.keyEventFromSpec(keySpec); + } + + keyEventFromSpec(keySpec: ActiveKey | ActiveSubKey) { //let core = com.keyman.singleton.core; // only singleton-based ref currently needed here. // Start: mirrors _GetKeyEventProperties @@ -993,6 +921,8 @@ export default class VisualKeyboard extends EventEmitter implements Ke return; } + this.gestureEngine.stateToken = layerId; + // So... through KMW 14, we actually never tracked the capsKey, numKey, and scrollKey // properly for keyboard-defined layouts - only _default_, desktop-style layouts. // @@ -1460,79 +1390,6 @@ export default class VisualKeyboard extends EventEmitter implements Ke } } - // /** - // * Initializes all supported gestures given a base key and the triggering touch coordinates. - // * @param key The gesture's base key - // * @param touch The starting touch coordinates for the gesture - // * @returns - // */ - // initGestures(key: KeyElement, input: InputEventCoordinate) { - - // if (this.pendingMultiTap) { - // switch (this.pendingMultiTap.incrementTouch(key)) { - // case PendingMultiTapState.Cancelled: - // this.pendingMultiTap = null; - // break; - // case PendingMultiTapState.Realized: - // // Don't initialize any other gestures if the - // // multi tap is realized; we cleanup on touch - // // release because we need to cancel the base - // // key action - // return; - // } - // } - - // if (!this.pendingMultiTap && PendingMultiTap.isValidTarget(this, key)) { - // // We are only going to support double-tap on Shift - // // in Keyman 15, so we pass in the constant count = 2 - // this.pendingMultiTap = new PendingMultiTap(this, key, 2); - // this.pendingMultiTap.timeout.then(() => { - // this.pendingMultiTap = null; - // }); - // } - // } - - // /** - // * Updates all currently-pending and activated gestures. - // * - // * @param currentKey The key currently underneath the most recent touch coordinate - // * @param previousKey The previously-selected key - // * @param input The current mouse or touch coordinate for the gesture - // * @returns true if should fully capture input, false if input should 'fall through'. - // */ - // updateGestures(currentKey: KeyElement, previousKey: KeyElement, input: InputEventCoordinate): boolean { - // let key0 = previousKey; - // let key1 = currentKey; - - // if(!currentKey && this.pendingMultiTap) { - // this.pendingMultiTap.cancel(); - // this.pendingMultiTap = null; - // } - - // // Clear previous key highlighting, allow subkey controller to highlight as appropriate. - // if (this.subkeyGesture) { - // if (key0) { - // key0.key.highlight(false); - // } - // this.subkeyGesture.updateTouch(input); - - // this.keyPending = null; - // this.touchPending = null; - - // return true; - // } - - // this.currentTarget = null; - - // // If there is an active popup menu (which can occur from the previous block), - // // a subkey popup exists; do not allow base key output. - // if (this.subkeyGesture || this.pendingSubkey) { - // return true; - // } - - // return false; - // } - optionKey(e: KeyElement, keyName: string, keyDown: boolean) { if (keyName.indexOf('K_LOPT') >= 0) { this.emit('globekey', e, keyDown); @@ -1616,9 +1473,19 @@ export default class VisualKeyboard extends EventEmitter implements Ke // Exclude menu and OSK hide keys from normal click processing if(keyEvent.kName == 'K_LOPT' || keyEvent.kName == 'K_ROPT') { this.optionKey(e, keyEvent.kName, true); - return true; + return {}; + } + + let callbackData: { + contextToken?: number + } = { }; + + const keyEventCallback: KeyEventResultCallback = (result, error) => { + callbackData.contextToken = result?.transcription?.token; } - this.emit('keyevent', keyEvent); + this.emit('keyevent', keyEvent, keyEventCallback); + + return callbackData; } } diff --git a/web/src/test/auto/integrated/cases/events.js b/web/src/test/auto/integrated/cases/events.js index 5d7b3d20412..83c41904613 100644 --- a/web/src/test/auto/integrated/cases/events.js +++ b/web/src/test/auto/integrated/cases/events.js @@ -27,7 +27,7 @@ describe('Event Management', function() { teardownKMW(); }); - it('Keystroke-based onChange event generation', function() { + it('Keystroke-based onChange event generation', async function() { var simple_A = {"type":"key","key":"a","code":"KeyA","keyCode":65,"modifierSet":0,"location":0}; var event = new KMWRecorder.PhysicalInputEventSpec(simple_A); @@ -40,7 +40,7 @@ describe('Event Management', function() { keyman.setActiveElement(ele); let eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(event); + await eventDriver.simulateEvent(event); let focusEvent = new FocusEvent('blur', {relatedTarget: ele}); ele.dispatchEvent(focusEvent); @@ -71,14 +71,14 @@ describe('Event Management', function() { await eventDriver.simulateEvent(event); let focusEvent = new FocusEvent('blur', {relatedTarget: ele}); - ele.dispatchEvent(focusEvent); + await ele.dispatchEvent(focusEvent); // Asserts that the handler is called. As the handler clears itself, it will only // remain set if it hasn't been called. assert.isNull(ele.onchange, '`onchange` handler was not called'); }); - it('Keystroke-based onInput event generation', function() { + it('Keystroke-based onInput event generation', async function() { var simple_A = {"type":"key","key":"a","code":"KeyA","keyCode":65,"modifierSet":0,"location":0}; var event = new KMWRecorder.PhysicalInputEventSpec(simple_A); @@ -93,9 +93,9 @@ describe('Event Management', function() { }); let eventDriver = new KMWRecorder.BrowserDriver(ele); - eventDriver.simulateEvent(event); - eventDriver.simulateEvent(event); - eventDriver.simulateEvent(event); + await eventDriver.simulateEvent(event); + await eventDriver.simulateEvent(event); + await eventDriver.simulateEvent(event); assert.equal(counterObj.i, fin, "Event handler not called the expected number of times"); });