diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index a88688f7ab7..6acd33fbdb3 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -381,15 +381,11 @@ export class UnicodeSetItem extends VarsItem { }; export class SetVarItem extends VarsItem { - constructor(id: string, value: string[], sections: DependencySections, rawItems: string[]) { + constructor(id: string, value: string[], sections: DependencySections) { super(id, value.join(' '), sections); this.items = sections.elem.allocElementString(sections, value); - this.rawItems = rawItems; } - // element string array - items: ElementString; - // like items, but with unprocessed marker strings - rawItems: string[]; + items: ElementString; // element string array valid() : boolean { return !!this.items; } @@ -405,10 +401,12 @@ export class StringVarItem extends VarsItem { // 'tran' export class TranTransform { - from: StrsItem; - to: StrsItem; - mapFrom: StrsItem; // var name - mapTo: StrsItem; // var name + from: StrsItem; // "from" computed regex string + to: StrsItem; // "to" (replacement) computed regex string + mapFrom: StrsItem; // var name for map + mapTo: StrsItem; // var name for map + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _to?: string; // Not part of binary file: for use in the XML serializer. If present, sets the to= attribute for XML. } export class TranGroup { @@ -420,6 +418,9 @@ export class TranGroup { export class TranReorder { elements: ElementString; before: ElementString; + _before?: string; // Not part of binary file: for use in the XML serializer. If present, sets the before= attribute for XML. + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _order?: string; // Not part of binary file: for use in the XML serializer. If present, sets the order= attribute for XML. }; export class Tran extends Section { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 022328ff942..69a039aa00e 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -114,6 +114,13 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { textNodeName: '_', suppressEmptyNode: true, }, + keyboard3: { + attributeNamePrefix: '$', + ignoreAttributes: false, + format: true, + textNodeName: '_', + suppressEmptyNode: true, + }, }; /** wrapper for XML parsing support */ diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index e341f3ac876..d86d7e2bf54 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -15,8 +15,9 @@ import { SubstitutionUse, Substitutions } from './substitution-tracker.js'; /** reserved name for the special gap key. space is not allowed in key ids. */ const reserved_gap = "gap (reserved)"; - export class KeysCompiler extends SectionCompiler { + /** keys that are of a reserved type */ + public static RESERVED_KEY = Symbol('Reserved Key'); static validateSubstitutions( keyboard: LDMLKeyboard.LKKeyboard, st: Substitutions @@ -222,6 +223,19 @@ export class KeysCompiler extends SectionCompiler { /** count of reserved keys, for tests */ public static readonly reserved_count = KeysCompiler.reserved_keys.length; + /** mark as reserved */ + private static asReserved(k : KeysKeys) : KeysKeys { + const o = k as any; + o[KeysCompiler.RESERVED_KEY] = true; + return k; + } + + /** true if a reserved key */ + public static isReserved(k : KeysKeys) : boolean { + const o = k as any; + return !!o[KeysCompiler.RESERVED_KEY]; + } + /** load up all reserved keys */ getReservedKeys(sections: KMXPlus.DependencySections) : Map { const r = new Map(); @@ -231,7 +245,7 @@ export class KeysCompiler extends SectionCompiler { const no_list = sections.list.allocList([], {}, sections); // now add the reserved key(s). - r.set(reserved_gap, { + r.set(reserved_gap, KeysCompiler.asReserved({ flags: constants.keys_key_flags_gap | constants.keys_key_flags_extend, id: sections.strs.allocString(reserved_gap), flicks: '', @@ -241,7 +255,7 @@ export class KeysCompiler extends SectionCompiler { switch: no_string, to: no_string, width: 10.0, // 10 * .1 - }); + })); if (r.size !== KeysCompiler.reserved_count) { throw Error(`Internal Error: KeysCompiler.reserved_count=${KeysCompiler.reserved_count} != ${r.size} actual reserved keys.`); @@ -357,7 +371,7 @@ export class KeysCompiler extends SectionCompiler { flags |= constants.keys_key_flags_extend; } const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 - sect.keys.push({ + sect.keys.push(SectionCompiler.copySymbols({ flags, flicks: flickId, id, @@ -367,7 +381,7 @@ export class KeysCompiler extends SectionCompiler { switch: keySwitch, // 'switch' is a reserved word to, width, - }); + }, key)); } } diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts index e33eda1086b..dd83a9deda7 100644 --- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts @@ -55,4 +55,20 @@ export abstract class SectionCompiler { ]); return defaults; } + + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols such as ImportStatus + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any) : T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index e8d7b9eb370..1a91e8d3824 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -138,6 +138,9 @@ export abstract class TransformCompiler result.substituteMarkerString(v, false)); - result.sets.push(new SetVarItem(id, cookedItems, sections, rawItems)); + result.sets.push(new SetVarItem(id, cookedItems, sections)); } addUnicodeSet(result: Vars, e: LDMLKeyboard.LKUSet, sections: DependencySections): void { const { id } = e; diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts new file mode 100644 index 00000000000..ee02abadcc5 --- /dev/null +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -0,0 +1,273 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * This module contains routines for serializing from a KMXPlus file into XML. + */ + +import { KMXPlus } from "@keymanapp/common-types"; +import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { constants } from "@keymanapp/ldml-keyboard-constants"; +import { KeysCompiler } from "../compiler/keys.js"; + +/** + * Serialize a KMXPlusFile back to XML. + * This is implemented for the LDML editor to be able to mutate LDML (XML) content by: + * 1. reading the original XML + * 2. compiling to KMXPlusFile + * 3. modifying the KMXPlusFile + * 4. serializing back to XML + * + * There are limitations: + * - TODO-LDML-EDITOR: Transforms would not be serialized properly, due to + * regex munging around markers and such. + * + * To work around this, fields with underscores such as _from and _to are added to + * the KMXPlusFile classes. These provide hints as to what the output XML should be, + * and are populated by the tran compiler. + * + * - TODO-LDML-EDITOR: Comments, whitespace, etc. are not preserved by this + * approach. Updates to the XML parsing will support this, see #10622. + * + * @param kmx input KMXPlusFile + * @returns XML String + */ +export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { + const writer = new KeymanXMLWriter("keyboard3"); + const { kmxplus } = kmx; + const { + // sect, + bksp, + disp, + // elem, + keys, + layr, + // list, + loca, + meta, + // strs, + tran, + // uset, + vars, + } = kmxplus; + const data = { + keyboard3: { + ...getRootAttributes(), + ...getLocales(), + version: getVersion(), + info: getInfo(), + ...getDisplays(), + ...getKeys(), + ...getFlicks(), + ...getLayers(), + ...getVariables(), + ...getTransforms(), + } + }; + + return writer.write(data); + + function getRootAttributes() { + return { + '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`, + '$locale': kmx.kmxplus.loca.locales[0].value, + '$conformsTo': constants.cldr_version_latest, + }; + } + + function getLocales() { + if (loca?.locales?.length < 2) { + return {}; // no additional locales + } else { + return { + locales: + loca.locales.map(({ value }) => ({ '$id': value })), + } + } + } + + function getInfo() { + return { + '$author': meta.author.value, + '$name': meta.name.value, + '$layout': meta.layout.value, + '$indicator': meta.indicator.value, + }; + } + + function getVersion() { + return { '$number': kmx.kmxplus.meta.version.value }; + } + + function getDisplays() { + const displays = { + display: disp?.disps.map(disp => getDisplay(disp)) || [], + ...getDisplaySettings(), + }; + if (displays?.display?.length || displays?.displayOptions) { + return { displays } + } else { + return {}; + } + } + + function stringToAttr(attr: string, s?: KMXPlus.StrsItem, override?: string) { + if (override) return asAttr(attr, override); + if (!s || !s?.value?.length) return {}; + return Object.fromEntries([[`\$${attr}`, s.value]]); + } + + function asAttr(attr: string, s?: any) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s]]); + } + + function numberToAttr(attr: string, s?: number) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s.toString()]]); + } + + function getDisplay(disp: KMXPlus.DispItem) { + return { + ...stringToAttr('output', disp?.to), + ...stringToAttr('keyId', disp?.id), + ...stringToAttr('display', disp?.display), + }; + } + + function getDisplaySettings() { + if (!disp?.baseCharacter?.value) return {}; + return { + displayOptions: { + '$baseCharacter': disp?.baseCharacter?.value, + } + }; + } + + function getKeys() { + if (!keys?.keys?.length) { + return {}; + } + return { + keys: { + key: keys.keys + // skip reserved keys (gap) + .filter((key: KMXPlus.KeysKeys) => + !KeysCompiler.isReserved(key) && + !LDMLKeyboard.ImportStatus.isImpliedImport(key)) + .map((key: KMXPlus.KeysKeys) => ({ + ...stringToAttr('id', key.id), + ...stringToAttr('output', key.to), + ...asAttr('longPressKeyIds', key?.longPress?.join(' ') || undefined), + })), + }, + }; + } + + function getFlicks() { + // skip the null flicks + if (keys?.flicks?.length < 2) { + return {}; + } + return { + flicks: { + // keys.key.. + } + }; + } + + function getLayers() { + if (!layr?.lists?.length) { + return {}; + } + return { + layers: layr.lists.map(({ hardware, minDeviceWidth, layers }) => ({ + ...stringToAttr('formId', hardware), + ...numberToAttr('minDeviceWidth', minDeviceWidth), + layer: layers.map(({ id, mod, rows }) => ({ + ...stringToAttr('id', id), + ...asAttr('modifiers', modToString(mod)), + row: rows.map(({ keys }) => ({ + ...asAttr('keys', keys.map(({ value }) => value).join(' ')), + })), + })), + })), + }; + } + + function getVariables() { + if (!vars?.strings.length && !vars?.sets.length && !vars?.usets.length) { + return {}; + } + function varToObj(v: KMXPlus.VarsItem): any { + const { id, value } = v; + return { + ...stringToAttr('id', id), + ...stringToAttr('value', value), + }; + } + function varsToArray(vars: KMXPlus.VarsItem[]): any[] { + return vars.map(varToObj); + } + const { strings, sets, usets } = vars; + + return { + variables: { + string: varsToArray(strings), + set: varsToArray(sets), + uset: varsToArray(usets), + }, + }; + } + + function getTransforms() { + return { + transforms: [ + ...getTransformType("simple", tran), + ...getTransformType("backspace", bksp), + ], + }; + } + + /** NB: Bksp is a child class of Tran */ + function getTransformType(type: string, t: KMXPlus.Tran) { + if (!t?.groups?.length) { + return []; + } + const { groups } = t; + return [{ + ...asAttr('type', type), + transformGroup: groups.map((group) => { + if (group.type === constants.tran_group_type_transform) { + return { + transform: group.transforms.map(({from, to, _from, _to}) => ({ + ...stringToAttr('from', from, _from), + ...stringToAttr('to', to, _to), + })), + }; + } else if(group.type === constants.tran_group_type_reorder) { + return { + reorder: group.reorders.map(({before, elements, _before, _from, _order}) => ({ + ...asAttr('before', _before || before.toString()), + ...asAttr('from', _from || elements.toString()), + ...asAttr('order', _order), + })), + }; + } else { + throw Error(`Invalid tran.group.type ${group.type}`); + } + }), + }]; + } +} + + +/** convert a keys_mod value to a space-separated string list */ +function modToString(mod: number) { + // first try exact match + const matches: string[] = []; + for (const [name, value] of constants.keys_mod_map.entries()) { + if (mod === value) return name; // exact match + if (mod & value) matches.push(name); + } + return matches.sort().join(' '); +} diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index 1f87e55a22a..c1bafeedb21 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -3,7 +3,10 @@ import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; import { KMXBuilder } from '@keymanapp/developer-utils'; import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; +import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; +import { kmxToXml } from '../src/util/serialize.js'; +import { writeFileSync } from 'node:fs'; /** Overall compiler tests */ describe('compiler-tests', function() { @@ -35,8 +38,34 @@ describe('compiler-tests', function() { let expected = await hextobin(binaryFilename, undefined, {silent:true}); assert.deepEqual(code, expected); + + // now output it again as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + + }); + + it('should-serialize-kmx', async function() { + this.timeout(4000); + // Let's build basic.xml + // It should match basic.kmx (built from basic.txt) + + const inputFilename = makePathToFixture('basic.xml'); + + // Compile the keyboard + const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + assert.isNotNull(kmx); + + // now output it as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + + compareXml(outputFilename, inputFilename); }); + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; const k = new LdmlKeyboardCompiler(); diff --git a/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml new file mode 100644 index 00000000000..aa570fd6f0b --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts new file mode 100644 index 00000000000..f245da51a91 --- /dev/null +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -0,0 +1,25 @@ +import {assert} from 'chai'; +import {readFileSync} from 'node:fs'; +import { KeymanXMLReader } from "@keymanapp/developer-utils"; + +/** + * + * @param actual path to actual XML + * @param expect path to expected XML + * @param mutator optional function that will be applied to the parsed object + */ +export function compareXml(actual : string, expect: string, mutator?: (input: any) => any) { + if (!mutator) { + // no-op + mutator = (x: any) => x; + } + const reader = new KeymanXMLReader('keyboard3'); + + const actualStr = readFileSync(actual, 'utf-8'); + const expectStr = readFileSync(expect, 'utf-8'); + + const actualParsed = mutator(reader.parse(actualStr)); + const expectParsed = mutator(reader.parse(expectStr)); + + assert.deepEqual(actualParsed, expectParsed); +} diff --git a/developer/src/kmc-ldml/test/tsconfig.json b/developer/src/kmc-ldml/test/tsconfig.json index 354f5235f1b..5f6d6e93b69 100644 --- a/developer/src/kmc-ldml/test/tsconfig.json +++ b/developer/src/kmc-ldml/test/tsconfig.json @@ -10,7 +10,7 @@ }, "include": [ "**/*.tests.ts", - "./helpers/index.ts" + "./helpers/*.ts", ], "references": [ { "path": "../../../../common/web/keyman-version" },