From a1571830a2f4e76382b3a29a7fb9cf1768161587 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 10 Jan 2025 18:11:01 -0600 Subject: [PATCH 1/6] feat(developer): kmx+ to xml - initial steps Fixes: #12874 --- .../src/common/web/utils/src/xml-utils.ts | 7 + developer/src/kmc-ldml/src/util/serialize.ts | 124 ++++++++++++++++++ .../src/kmc-ldml/test/compiler-e2e.tests.ts | 8 ++ 3 files changed, 139 insertions(+) create mode 100644 developer/src/kmc-ldml/src/util/serialize.ts diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 022328ff942..69a039aa00e 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -114,6 +114,13 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { textNodeName: '_', suppressEmptyNode: true, }, + keyboard3: { + attributeNamePrefix: '$', + ignoreAttributes: false, + format: true, + textNodeName: '_', + suppressEmptyNode: true, + }, }; /** wrapper for XML parsing support */ diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts new file mode 100644 index 00000000000..dbcf39ae682 --- /dev/null +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -0,0 +1,124 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + */ + +import { KMXPlus } from "@keymanapp/common-types"; +import { KeymanXMLWriter } from "@keymanapp/developer-utils"; +import { constants } from "@keymanapp/ldml-keyboard-constants"; + +export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { + const writer = new KeymanXMLWriter("keyboard3"); + const { kmxplus } = kmx; + const { + // sect, + // bksp, + disp, + // elem, + // keys, + // layr, + // list, + loca, + meta, + // strs, + // tran, + // uset, + // vars, + } = kmxplus; + const data = { + keyboard3: { + ...getRootAttributes(), + ...getLocales(), + version: getVersion(), + info: getInfo(), + ...getDisplays(), + ...getKeys(), + ...getLayers(), + ...getVariables(), + ...getTransforms(), + } + }; + + return writer.write(data); + + function getRootAttributes() { + return { + '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`, + '$locale': kmx.kmxplus.loca.locales[0].value, + '$conffiormsTo': constants.cldr_version_latest, + }; + } + + function getLocales() { + if (loca?.locales?.length < 2) { + return {}; // no additional locales + } else { + return { + locales: + loca.locales.map(({ value }) => ({ '$id': value })), + } + } + } + + function getInfo() { + return { + '$author': meta.author.value, + '$name': meta.name.value, + '$layout': meta.layout.value, + '$indicator': meta.indicator.value, + }; + } + + function getVersion() { + return { '$value': kmx.kmxplus.meta.version.value }; + } + + function getDisplays() { + const displays = { + display: disp?.disps.map(disp => getDisplay(disp)) || [], + ...getDisplaySettings(), + }; + if (displays?.display?.length || displays?.displayOptions) { + return { displays } + } else { + return {}; + } + } + + function stringToAttr(attr: string, s?: KMXPlus.StrsItem) { + if (!s || !s?.value?.length) return {}; + return Object.fromEntries([[`\$${attr}`, s.value]]); + } + + function getDisplay(disp: KMXPlus.DispItem) { + return { + ...stringToAttr('output', disp?.to), + ...stringToAttr('keyId', disp?.id), + ...stringToAttr('display', disp?.display), + }; + } + + function getDisplaySettings() { + if (!disp?.baseCharacter?.value) return {}; + return { + displayOptions: { + '$baseCharacter': disp?.baseCharacter?.value, + } + }; + } + + function getKeys() { + return {}; + } + + function getLayers() { + return {}; + } + + function getVariables() { + return {}; + } + + function getTransforms() { + return {}; + } +} diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index 1f87e55a22a..f2bf37fa5be 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -4,6 +4,8 @@ import hextobin from '@keymanapp/hextobin'; import { KMXBuilder } from '@keymanapp/developer-utils'; import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; +import { kmxToXml } from '../src/util/serialize.js'; +import { writeFileSync } from 'node:fs'; /** Overall compiler tests */ describe('compiler-tests', function() { @@ -35,6 +37,12 @@ describe('compiler-tests', function() { let expected = await hextobin(binaryFilename, undefined, {silent:true}); assert.deepEqual(code, expected); + + // now output it again as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + }); it('should handle non existent files', async () => { From e553e0f89d927ac7d7a18ade15f39ffa5fb013e1 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 13 Jan 2025 16:00:09 -0600 Subject: [PATCH 2/6] feat(developer): serialize keybag, preserve import status - utility to copy symbols over Fixes: #12874 --- developer/src/kmc-ldml/src/compiler/keys.ts | 24 ++++++++++--- .../kmc-ldml/src/compiler/section-compiler.ts | 16 +++++++++ developer/src/kmc-ldml/src/util/serialize.ts | 36 ++++++++++++++++--- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index e341f3ac876..d86d7e2bf54 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -15,8 +15,9 @@ import { SubstitutionUse, Substitutions } from './substitution-tracker.js'; /** reserved name for the special gap key. space is not allowed in key ids. */ const reserved_gap = "gap (reserved)"; - export class KeysCompiler extends SectionCompiler { + /** keys that are of a reserved type */ + public static RESERVED_KEY = Symbol('Reserved Key'); static validateSubstitutions( keyboard: LDMLKeyboard.LKKeyboard, st: Substitutions @@ -222,6 +223,19 @@ export class KeysCompiler extends SectionCompiler { /** count of reserved keys, for tests */ public static readonly reserved_count = KeysCompiler.reserved_keys.length; + /** mark as reserved */ + private static asReserved(k : KeysKeys) : KeysKeys { + const o = k as any; + o[KeysCompiler.RESERVED_KEY] = true; + return k; + } + + /** true if a reserved key */ + public static isReserved(k : KeysKeys) : boolean { + const o = k as any; + return !!o[KeysCompiler.RESERVED_KEY]; + } + /** load up all reserved keys */ getReservedKeys(sections: KMXPlus.DependencySections) : Map { const r = new Map(); @@ -231,7 +245,7 @@ export class KeysCompiler extends SectionCompiler { const no_list = sections.list.allocList([], {}, sections); // now add the reserved key(s). - r.set(reserved_gap, { + r.set(reserved_gap, KeysCompiler.asReserved({ flags: constants.keys_key_flags_gap | constants.keys_key_flags_extend, id: sections.strs.allocString(reserved_gap), flicks: '', @@ -241,7 +255,7 @@ export class KeysCompiler extends SectionCompiler { switch: no_string, to: no_string, width: 10.0, // 10 * .1 - }); + })); if (r.size !== KeysCompiler.reserved_count) { throw Error(`Internal Error: KeysCompiler.reserved_count=${KeysCompiler.reserved_count} != ${r.size} actual reserved keys.`); @@ -357,7 +371,7 @@ export class KeysCompiler extends SectionCompiler { flags |= constants.keys_key_flags_extend; } const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 - sect.keys.push({ + sect.keys.push(SectionCompiler.copySymbols({ flags, flicks: flickId, id, @@ -367,7 +381,7 @@ export class KeysCompiler extends SectionCompiler { switch: keySwitch, // 'switch' is a reserved word to, width, - }); + }, key)); } } diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts index e33eda1086b..dd83a9deda7 100644 --- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts @@ -55,4 +55,20 @@ export abstract class SectionCompiler { ]); return defaults; } + + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols such as ImportStatus + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any) : T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } } diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts index dbcf39ae682..186e334015b 100644 --- a/developer/src/kmc-ldml/src/util/serialize.ts +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -3,8 +3,9 @@ */ import { KMXPlus } from "@keymanapp/common-types"; -import { KeymanXMLWriter } from "@keymanapp/developer-utils"; +import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils"; import { constants } from "@keymanapp/ldml-keyboard-constants"; +import { KeysCompiler } from "../compiler/keys.js"; export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { const writer = new KeymanXMLWriter("keyboard3"); @@ -14,7 +15,7 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { // bksp, disp, // elem, - // keys, + keys, // layr, // list, loca, @@ -32,6 +33,7 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { info: getInfo(), ...getDisplays(), ...getKeys(), + ...getFlicks(), ...getLayers(), ...getVariables(), ...getTransforms(), @@ -44,7 +46,7 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { return { '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`, '$locale': kmx.kmxplus.loca.locales[0].value, - '$conffiormsTo': constants.cldr_version_latest, + '$conformsTo': constants.cldr_version_latest, }; } @@ -107,7 +109,33 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { } function getKeys() { - return {}; + if (!keys?.keys?.length) { + return {}; + } + return { + keys: { + key: keys.keys + // skip reserved keys (gap) + .filter((key: KMXPlus.KeysKeys) => + !KeysCompiler.isReserved(key) && + !LDMLKeyboard.ImportStatus.isImpliedImport(key)) + .map((key: KMXPlus.KeysKeys) => ({ + ...stringToAttr('id', key.id), + ...stringToAttr('output', key.to), + })), + }, + }; + } + + function getFlicks() { + if (!keys?.flicks?.length) { + return {}; + } + return { + flicks: { + // keys.keu + } + }; } function getLayers() { From 31d72e5eea54e9babb37e8d929dbd981d421282c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 15 Jan 2025 20:18:16 -0500 Subject: [PATCH 3/6] feat(developer): more steps for XML serializing Fixes: #12874 --- developer/src/kmc-ldml/src/util/serialize.ts | 336 ++++++++++++------- 1 file changed, 215 insertions(+), 121 deletions(-) diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts index 186e334015b..11b9d306afe 100644 --- a/developer/src/kmc-ldml/src/util/serialize.ts +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -8,145 +8,239 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KeysCompiler } from "../compiler/keys.js"; export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { - const writer = new KeymanXMLWriter("keyboard3"); - const { kmxplus } = kmx; - const { - // sect, - // bksp, - disp, - // elem, - keys, - // layr, - // list, - loca, - meta, - // strs, - // tran, - // uset, - // vars, - } = kmxplus; - const data = { - keyboard3: { - ...getRootAttributes(), - ...getLocales(), - version: getVersion(), - info: getInfo(), - ...getDisplays(), - ...getKeys(), - ...getFlicks(), - ...getLayers(), - ...getVariables(), - ...getTransforms(), - } - }; - - return writer.write(data); - - function getRootAttributes() { - return { - '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`, - '$locale': kmx.kmxplus.loca.locales[0].value, - '$conformsTo': constants.cldr_version_latest, - }; + const writer = new KeymanXMLWriter("keyboard3"); + const { kmxplus } = kmx; + const { + // sect, + bksp, + disp, + // elem, + keys, + layr, + // list, + loca, + meta, + // strs, + tran, + // uset, + vars, + } = kmxplus; + const data = { + keyboard3: { + ...getRootAttributes(), + ...getLocales(), + version: getVersion(), + info: getInfo(), + ...getDisplays(), + ...getKeys(), + ...getFlicks(), + ...getLayers(), + ...getVariables(), + ...getTransforms(), } + }; - function getLocales() { - if (loca?.locales?.length < 2) { - return {}; // no additional locales - } else { - return { - locales: - loca.locales.map(({ value }) => ({ '$id': value })), - } - } - } + return writer.write(data); - function getInfo() { - return { - '$author': meta.author.value, - '$name': meta.name.value, - '$layout': meta.layout.value, - '$indicator': meta.indicator.value, - }; + function getRootAttributes() { + return { + '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`, + '$locale': kmx.kmxplus.loca.locales[0].value, + '$conformsTo': constants.cldr_version_latest, + }; + } + + function getLocales() { + if (loca?.locales?.length < 2) { + return {}; // no additional locales + } else { + return { + locales: + loca.locales.map(({ value }) => ({ '$id': value })), + } } + } + + function getInfo() { + return { + '$author': meta.author.value, + '$name': meta.name.value, + '$layout': meta.layout.value, + '$indicator': meta.indicator.value, + }; + } - function getVersion() { - return { '$value': kmx.kmxplus.meta.version.value }; - } + function getVersion() { + return { '$value': kmx.kmxplus.meta.version.value }; + } - function getDisplays() { - const displays = { - display: disp?.disps.map(disp => getDisplay(disp)) || [], - ...getDisplaySettings(), - }; - if (displays?.display?.length || displays?.displayOptions) { - return { displays } - } else { - return {}; - } + function getDisplays() { + const displays = { + display: disp?.disps.map(disp => getDisplay(disp)) || [], + ...getDisplaySettings(), + }; + if (displays?.display?.length || displays?.displayOptions) { + return { displays } + } else { + return {}; } + } + + function stringToAttr(attr: string, s?: KMXPlus.StrsItem) { + if (!s || !s?.value?.length) return {}; + return Object.fromEntries([[`\$${attr}`, s.value]]); + } + + function asAttr(attr: string, s?: any) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s]]); + } + + function numberToAttr(attr: string, s?: number) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s.toString()]]); + } + + function getDisplay(disp: KMXPlus.DispItem) { + return { + ...stringToAttr('output', disp?.to), + ...stringToAttr('keyId', disp?.id), + ...stringToAttr('display', disp?.display), + }; + } + + function getDisplaySettings() { + if (!disp?.baseCharacter?.value) return {}; + return { + displayOptions: { + '$baseCharacter': disp?.baseCharacter?.value, + } + }; + } - function stringToAttr(attr: string, s?: KMXPlus.StrsItem) { - if (!s || !s?.value?.length) return {}; - return Object.fromEntries([[`\$${attr}`, s.value]]); + function getKeys() { + if (!keys?.keys?.length) { + return {}; } + return { + keys: { + key: keys.keys + // skip reserved keys (gap) + .filter((key: KMXPlus.KeysKeys) => + !KeysCompiler.isReserved(key) && + !LDMLKeyboard.ImportStatus.isImpliedImport(key)) + .map((key: KMXPlus.KeysKeys) => ({ + ...stringToAttr('id', key.id), + ...stringToAttr('output', key.to), + })), + }, + }; + } - function getDisplay(disp: KMXPlus.DispItem) { - return { - ...stringToAttr('output', disp?.to), - ...stringToAttr('keyId', disp?.id), - ...stringToAttr('display', disp?.display), - }; + function getFlicks() { + // skip the null flicks + if (keys?.flicks?.length < 2) { + return {}; } + return { + flicks: { + // keys.key.. + } + }; + } - function getDisplaySettings() { - if (!disp?.baseCharacter?.value) return {}; - return { - displayOptions: { - '$baseCharacter': disp?.baseCharacter?.value, - } - }; + function getLayers() { + if (!layr?.lists?.length) { + return {}; } + return { + layers: layr.lists.map(({ hardware, minDeviceWidth, layers }) => ({ + ...stringToAttr('formId', hardware), + ...numberToAttr('minDeviceWidth', minDeviceWidth), + layer: layers.map(({ id, mod, rows }) => ({ + ...stringToAttr('id', id), + ...asAttr('modifiers', modToString(mod)), + row: rows.map(({ keys }) => ({ + ...asAttr('keys', keys.map(({ value }) => value).join(' ')), + })), + })), + })), + }; + } - function getKeys() { - if (!keys?.keys?.length) { - return {}; - } - return { - keys: { - key: keys.keys - // skip reserved keys (gap) - .filter((key: KMXPlus.KeysKeys) => - !KeysCompiler.isReserved(key) && - !LDMLKeyboard.ImportStatus.isImpliedImport(key)) - .map((key: KMXPlus.KeysKeys) => ({ - ...stringToAttr('id', key.id), - ...stringToAttr('output', key.to), - })), - }, - }; + function getVariables() { + if (!vars?.strings.length && !vars?.sets.length && !vars?.usets.length) { + return {}; } - - function getFlicks() { - if (!keys?.flicks?.length) { - return {}; - } - return { - flicks: { - // keys.keu - } - }; + function varToObj(v: KMXPlus.VarsItem): any { + const { id, value } = v; + return { + ...stringToAttr('id', id), + ...stringToAttr('value', value), + }; } - - function getLayers() { - return {}; + function varsToArray(vars: KMXPlus.VarsItem[]): any[] { + return vars.map(varToObj); } + const { strings, sets, usets } = vars; + + return { + variables: { + string: varsToArray(strings), + set: varsToArray(sets), + uset: varsToArray(usets), + }, + }; + } + + function getTransforms() { + return { + transforms: [ + ...getTransformType("simple", tran), + ...getTransformType("backspace", bksp), + ], + }; + } - function getVariables() { - return {}; + /** NB: Bksp is a child class of Tran */ + function getTransformType(type: string, t: KMXPlus.Tran) { + if (!t?.groups?.length) { + return []; } + const { groups } = t; + return [{ + ...asAttr('type', type), + transformGroup: groups.map((group) => { + if (group.type === constants.tran_group_type_transform) { + return { + transform: group.transforms.map(({from, to}) => ({ + ...stringToAttr('from', from), + ...stringToAttr('to', to), + })), + }; + } else if(group.type === constants.tran_group_type_reorder) { + return { + transform: group.reorders.map(({before, elements}) => ({ + ...asAttr('before', before.toString()), + ...asAttr('from', elements.toString()), + })), + }; + } else { + throw Error(`Invalid tran.group.type ${group.type}`); + } + }), + }]; + } +} - function getTransforms() { - return {}; - } + +/** convert a keys_mod value to a space-separated string list */ +function modToString(mod: number) { + // first try exact match + const matches: string[] = []; + for (const [name, value] of constants.keys_mod_map.entries()) { + if (mod === value) return name; // exact match + if (mod & value) matches.push(name); + } + return matches.sort().join(' '); } From 1703567445e13d64b0c03901ae96d539e6c68081 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 21 Jan 2025 09:05:53 -0600 Subject: [PATCH 4/6] =?UTF-8?q?chore(developer):=20update=20serializer=20?= =?UTF-8?q?=F0=9F=97=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Note: does not handle transforms yet Fixes: #12784 --- developer/src/kmc-ldml/src/util/serialize.ts | 10 ++++- .../src/kmc-ldml/test/compiler-e2e.tests.ts | 25 +++++++++++ .../test/fixtures/basic-serialized.xml | 42 +++++++++++++++++++ .../src/kmc-ldml/test/helpers/compareXml.ts | 25 +++++++++++ developer/src/kmc-ldml/test/tsconfig.json | 2 +- 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 developer/src/kmc-ldml/test/fixtures/basic-serialized.xml create mode 100644 developer/src/kmc-ldml/test/helpers/compareXml.ts diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts index 11b9d306afe..d66c0905131 100644 --- a/developer/src/kmc-ldml/src/util/serialize.ts +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -7,6 +7,13 @@ import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils"; import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KeysCompiler } from "../compiler/keys.js"; +/** + * Serialize a KMXPlusFile back to XML + * TODO-EPIC-LDML: Does not handle transforms properly (marker strings). + * TODO-EPIC-LDML: Does not retain the original XML formatting. + * @param kmx input KMXPlusFile + * @returns XML String + */ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { const writer = new KeymanXMLWriter("keyboard3"); const { kmxplus } = kmx; @@ -71,7 +78,7 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { } function getVersion() { - return { '$value': kmx.kmxplus.meta.version.value }; + return { '$number': kmx.kmxplus.meta.version.value }; } function getDisplays() { @@ -132,6 +139,7 @@ export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { .map((key: KMXPlus.KeysKeys) => ({ ...stringToAttr('id', key.id), ...stringToAttr('output', key.to), + ...asAttr('longPressKeyIds', key?.longPress?.join(' ') || undefined), })), }, }; diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index f2bf37fa5be..40e78c1f3e6 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -3,6 +3,7 @@ import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; import { KMXBuilder } from '@keymanapp/developer-utils'; import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; +import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import { kmxToXml } from '../src/util/serialize.js'; import { writeFileSync } from 'node:fs'; @@ -45,6 +46,30 @@ describe('compiler-tests', function() { }); + it('should-serialize-kmx', async function() { + this.timeout(4000); + // Let's build basic.xml + // It should match basic.kmx (built from basic.txt) + + const inputFilename = makePathToFixture('basic.xml'); + + // Compile the keyboard + const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + assert.isNotNull(kmx); + + // now output it as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + + compareXml(outputFilename, inputFilename, (data) => { + // TODO-LDML-EDITOR: serializer doesn't handle transforms + delete data.keyboard3.transforms; + return data; + }); + }); + + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; const k = new LdmlKeyboardCompiler(); diff --git a/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml new file mode 100644 index 00000000000..a4e4fa2e3c5 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts new file mode 100644 index 00000000000..f245da51a91 --- /dev/null +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -0,0 +1,25 @@ +import {assert} from 'chai'; +import {readFileSync} from 'node:fs'; +import { KeymanXMLReader } from "@keymanapp/developer-utils"; + +/** + * + * @param actual path to actual XML + * @param expect path to expected XML + * @param mutator optional function that will be applied to the parsed object + */ +export function compareXml(actual : string, expect: string, mutator?: (input: any) => any) { + if (!mutator) { + // no-op + mutator = (x: any) => x; + } + const reader = new KeymanXMLReader('keyboard3'); + + const actualStr = readFileSync(actual, 'utf-8'); + const expectStr = readFileSync(expect, 'utf-8'); + + const actualParsed = mutator(reader.parse(actualStr)); + const expectParsed = mutator(reader.parse(expectStr)); + + assert.deepEqual(actualParsed, expectParsed); +} diff --git a/developer/src/kmc-ldml/test/tsconfig.json b/developer/src/kmc-ldml/test/tsconfig.json index 354f5235f1b..5f6d6e93b69 100644 --- a/developer/src/kmc-ldml/test/tsconfig.json +++ b/developer/src/kmc-ldml/test/tsconfig.json @@ -10,7 +10,7 @@ }, "include": [ "**/*.tests.ts", - "./helpers/index.ts" + "./helpers/*.ts", ], "references": [ { "path": "../../../../common/web/keyman-version" }, From 48e57d454e53e4461ac7af70b103b995ede30c18 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 21 Jan 2025 09:31:43 -0600 Subject: [PATCH 5/6] =?UTF-8?q?chore(developer):=20update=20serializer=20?= =?UTF-8?q?=F0=9F=97=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - workaround transform issue by adding bypass (_) fields in the KMXPlus structs Fixes: #12784 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 5 +++++ developer/src/kmc-ldml/src/compiler/tran.ts | 7 +++++++ developer/src/kmc-ldml/src/util/serialize.ts | 16 +++++++++------- .../src/kmc-ldml/test/compiler-e2e.tests.ts | 6 +----- .../kmc-ldml/test/fixtures/basic-serialized.xml | 8 ++++---- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index a88688f7ab7..20877f3aa02 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -409,6 +409,8 @@ export class TranTransform { to: StrsItem; mapFrom: StrsItem; // var name mapTo: StrsItem; // var name + _from?: string; // for serialization + _to?: string; // for serialization } export class TranGroup { @@ -420,6 +422,9 @@ export class TranGroup { export class TranReorder { elements: ElementString; before: ElementString; + _before?: string; // for serializing + _from?: string; // for serializing + _order?: string; // for serializing }; export class Tran extends Section { diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index e8d7b9eb370..1a91e8d3824 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -138,6 +138,9 @@ export abstract class TransformCompiler { if (group.type === constants.tran_group_type_transform) { return { - transform: group.transforms.map(({from, to}) => ({ - ...stringToAttr('from', from), - ...stringToAttr('to', to), + transform: group.transforms.map(({from, to, _from, _to}) => ({ + ...stringToAttr('from', from, _from), + ...stringToAttr('to', to, _to), })), }; } else if(group.type === constants.tran_group_type_reorder) { return { - transform: group.reorders.map(({before, elements}) => ({ - ...asAttr('before', before.toString()), - ...asAttr('from', elements.toString()), + reorder: group.reorders.map(({before, elements, _before, _from, _order}) => ({ + ...asAttr('before', _before || before.toString()), + ...asAttr('from', _from || elements.toString()), + ...asAttr('order', _order), })), }; } else { diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index 40e78c1f3e6..c1bafeedb21 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -62,11 +62,7 @@ describe('compiler-tests', function() { const asXml = kmxToXml(kmx); writeFileSync(outputFilename, asXml, 'utf-8'); - compareXml(outputFilename, inputFilename, (data) => { - // TODO-LDML-EDITOR: serializer doesn't handle transforms - delete data.keyboard3.transforms; - return data; - }); + compareXml(outputFilename, inputFilename); }); diff --git a/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml index a4e4fa2e3c5..aa570fd6f0b 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml +++ b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml @@ -24,14 +24,14 @@ - - + + - + - + From d1c22716b3dae95eec3ec94efd5843bf6f2b5659 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 23 Jan 2025 12:40:18 -0600 Subject: [PATCH 6/6] =?UTF-8?q?chore(developer):=20clarify=20comments=20ar?= =?UTF-8?q?ound=20serializer=20and=20KMX+=20=F0=9F=97=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clarify the purpose of the serializer and the serializer-required fields in KMXPlusFile - remove an unused SetVarItems.rawItems property added in #11059 Fixes: #12784 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 26 ++++++++----------- developer/src/kmc-ldml/src/compiler/vars.ts | 2 +- developer/src/kmc-ldml/src/util/serialize.ts | 23 +++++++++++++--- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 20877f3aa02..6acd33fbdb3 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -381,15 +381,11 @@ export class UnicodeSetItem extends VarsItem { }; export class SetVarItem extends VarsItem { - constructor(id: string, value: string[], sections: DependencySections, rawItems: string[]) { + constructor(id: string, value: string[], sections: DependencySections) { super(id, value.join(' '), sections); this.items = sections.elem.allocElementString(sections, value); - this.rawItems = rawItems; } - // element string array - items: ElementString; - // like items, but with unprocessed marker strings - rawItems: string[]; + items: ElementString; // element string array valid() : boolean { return !!this.items; } @@ -405,12 +401,12 @@ export class StringVarItem extends VarsItem { // 'tran' export class TranTransform { - from: StrsItem; - to: StrsItem; - mapFrom: StrsItem; // var name - mapTo: StrsItem; // var name - _from?: string; // for serialization - _to?: string; // for serialization + from: StrsItem; // "from" computed regex string + to: StrsItem; // "to" (replacement) computed regex string + mapFrom: StrsItem; // var name for map + mapTo: StrsItem; // var name for map + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _to?: string; // Not part of binary file: for use in the XML serializer. If present, sets the to= attribute for XML. } export class TranGroup { @@ -422,9 +418,9 @@ export class TranGroup { export class TranReorder { elements: ElementString; before: ElementString; - _before?: string; // for serializing - _from?: string; // for serializing - _order?: string; // for serializing + _before?: string; // Not part of binary file: for use in the XML serializer. If present, sets the before= attribute for XML. + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _order?: string; // Not part of binary file: for use in the XML serializer. If present, sets the order= attribute for XML. }; export class Tran extends Section { diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts index 0e9740fe443..6db1bd53ea2 100644 --- a/developer/src/kmc-ldml/src/compiler/vars.ts +++ b/developer/src/kmc-ldml/src/compiler/vars.ts @@ -285,7 +285,7 @@ export class VarsCompiler extends SectionCompiler { // this is not 'forMatch', all variables are to be assumed as string literals, not regex // content. const cookedItems: string[] = rawItems.map(v => result.substituteMarkerString(v, false)); - result.sets.push(new SetVarItem(id, cookedItems, sections, rawItems)); + result.sets.push(new SetVarItem(id, cookedItems, sections)); } addUnicodeSet(result: Vars, e: LDMLKeyboard.LKUSet, sections: DependencySections): void { const { id } = e; diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts index 203c1a190da..ee02abadcc5 100644 --- a/developer/src/kmc-ldml/src/util/serialize.ts +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -1,5 +1,7 @@ /* * Keyman is copyright (C) SIL Global. MIT License. + * + * This module contains routines for serializing from a KMXPlus file into XML. */ import { KMXPlus } from "@keymanapp/common-types"; @@ -8,9 +10,24 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KeysCompiler } from "../compiler/keys.js"; /** - * Serialize a KMXPlusFile back to XML - * TODO-EPIC-LDML: Does not handle transforms properly (marker strings). - * TODO-EPIC-LDML: Does not retain the original XML formatting. + * Serialize a KMXPlusFile back to XML. + * This is implemented for the LDML editor to be able to mutate LDML (XML) content by: + * 1. reading the original XML + * 2. compiling to KMXPlusFile + * 3. modifying the KMXPlusFile + * 4. serializing back to XML + * + * There are limitations: + * - TODO-LDML-EDITOR: Transforms would not be serialized properly, due to + * regex munging around markers and such. + * + * To work around this, fields with underscores such as _from and _to are added to + * the KMXPlusFile classes. These provide hints as to what the output XML should be, + * and are populated by the tran compiler. + * + * - TODO-LDML-EDITOR: Comments, whitespace, etc. are not preserved by this + * approach. Updates to the XML parsing will support this, see #10622. + * * @param kmx input KMXPlusFile * @returns XML String */