✨ Add support for languages whose word break is whitespace

README.md

@@ -10,25 +10,24 @@ It complements the text with tokens that exists in a current file.
 
 #### Commands
 
-| Name          | Default Shortcut Key |
-| ------------- | -------------------- |
-| Auto Complete | `Ctrl + Space`       |
-
-#### Support languages
-
-- English
-- Japanese
+| Name                      | Default Shortcut Key | Support Languages                        |
+| ------------------------- | -------------------- | ---------------------------------------- |
+| Auto Complete             | `Ctrl + Space`       | Languages whose word break is whitespace |
+| Auto Complete as Japanese |                      | Japanese                                 |
 
 I would like to add any other languages if anyone needs them.😉
 
 #### Demo
 
-![Basic demo](https://raw.githubusercontent.com/tadashi-aikawa/obsidian-various-complements-plugin/main/demo/demo.gif)
+##### Auto Complete
+
+
 
+##### Auto Complete as Japanese
 
-## Settings
+![Basic demo](https://raw.githubusercontent.com/tadashi-aikawa/obsidian-various-complements-plugin/main/demo/demo2.gif)
 
-Add settings if needed in the future.
+![Basic demo japanese](https://raw.githubusercontent.com/tadashi-aikawa/obsidian-various-complements-plugin/main/demo/demo.gif)
 
 
 ## For Developers

main.ts

@@ -1,18 +1,9 @@
 import { MarkdownView, Plugin } from "obsidian";
 import { Editor } from "codemirror";
-var CodeMirror: any = window.CodeMirror;
 import "./show-hint";
-import TinySegmenter from "./tiny-segmenter";
-// @ts-ignore
-const segmenter = new TinySegmenter();
+import { createTokenizer, TokenizeStrategy } from "./tokenizer";
 
-function pickTokens(cmEditor: Editor): string[] {
-  return cmEditor
-    .getValue()
-    .split(`\n`)
-    .flatMap<string>((x) => segmenter.segment(x))
-    .map((x) => x.replace(/[\[\]()<>"'`]/, ""));
-}
+var CodeMirror: any = window.CodeMirror;
 
 /**
  * This function uses case-sensitive logic if a second argument has an upper case. Otherwise, uses case-insensitive logic.
@@ -40,7 +31,7 @@ function selectSuggestedTokens(tokens: string[], word: string) {
 }
 
 export default class VariousComponentsPlugin extends Plugin {
-  private execAutoComplete() {
+  private execAutoComplete(strategy: TokenizeStrategy) {
     const currentView = this.app.workspace.getActiveViewOfType(MarkdownView);
     if (!currentView) {
       // Do nothing if the command is triggered outside a MarkdownView
@@ -52,28 +43,23 @@ export default class VariousComponentsPlugin extends Plugin {
     CodeMirror.showHint(
       cmEditor,
       () => {
-        const cursor = cmEditor.getCursor();
-        const token = cmEditor.getTokenAt(cursor);
-        if (!token.string) {
+        const tokenized = createTokenizer(cmEditor, strategy).tokenize();
+        if (!tokenized) {
           return;
         }
 
-        const words = segmenter.segment(token.string);
-        const word = words.pop();
-        const restWordsLength = words.reduce(
-          (t: number, x: string) => t + x.length,
-          0
+        const suggestedTokens = selectSuggestedTokens(
+          tokenized.tokens,
+          tokenized.currentToken
         );
-
-        const tokens = pickTokens(cmEditor);
-        const suggestedTokens = selectSuggestedTokens(tokens, word);
         if (suggestedTokens.length === 0) {
           return;
         }
 
+        const cursor = cmEditor.getCursor();
         return {
           list: suggestedTokens,
-          from: CodeMirror.Pos(cursor.line, token.start + restWordsLength),
+          from: CodeMirror.Pos(cursor.line, tokenized.currentTokenStart),
           to: CodeMirror.Pos(cursor.line, cursor.ch),
         };
       },
@@ -93,7 +79,18 @@ export default class VariousComponentsPlugin extends Plugin {
           return !!this.app.workspace.getActiveViewOfType(MarkdownView);
         }
 
-        this.execAutoComplete();
+        this.execAutoComplete("default");
+      },
+    });
+    this.addCommand({
+      id: "auto-complete-as-japanese",
+      name: "Auto Complete as Japanese",
+      checkCallback: (checking: boolean) => {
+        if (checking) {
+          return !!this.app.workspace.getActiveViewOfType(MarkdownView);
+        }
+
+        this.execAutoComplete("japanese");
       },
     });
   }

tokenizer.ts

@@ -0,0 +1,106 @@
+import TinySegmenter from "./tiny-segmenter";
+import CodeMirror from "codemirror";
+// @ts-ignore
+const segmenter = new TinySegmenter();
+
+export type TokenizeStrategy = "default" | "japanese";
+
+function pickTokens(cmEditor: CodeMirror.Editor): string[] {
+  const maxLineIndex = cmEditor.getDoc().lineCount();
+  return [...Array(maxLineIndex).keys()]
+    .flatMap((x) =>
+      cmEditor
+        .getLineTokens(x)
+        .flatMap((x) =>
+          x.type?.includes("hmd-codeblock") ? x.string.split(" ") : [x.string]
+        )
+    )
+    .map((x) => x.replace(/[\[\]()<>"'.,|; `]/g, ""))
+    .filter((x) => x !== "");
+}
+
+function pickTokensAsJapanese(cmEditor: CodeMirror.Editor): string[] {
+  return cmEditor
+    .getValue()
+    .split(`\n`)
+    .flatMap<string>((x) => segmenter.segment(x))
+    .map((x) => x.replace(/[\[\]()<>"'.,|; `]/, ""));
+}
+
+interface TokenizedResult {
+  currentToken: string;
+  currentTokenStart: number;
+  tokens: string[];
+}
+
+interface Tokenizer {
+  /**
+   * Return undefined if current token is empty.
+   */
+  tokenize(): TokenizedResult | undefined;
+}
+
+class DefaultTokenizer implements Tokenizer {
+  private readonly cmEditor: CodeMirror.Editor;
+
+  constructor(cmEditor: CodeMirror.Editor) {
+    this.cmEditor = cmEditor;
+  }
+
+  tokenize(): TokenizedResult | undefined {
+    const cursor = this.cmEditor.getCursor();
+    const token = this.cmEditor.getTokenAt(cursor);
+    if (!token.string) {
+      return undefined;
+    }
+
+    console.log(pickTokens(this.cmEditor));
+    return {
+      currentToken: token.string,
+      currentTokenStart: token.start,
+      tokens: pickTokens(this.cmEditor),
+    };
+  }
+}
+
+class JapaneseTokenizer implements Tokenizer {
+  private readonly cmEditor: CodeMirror.Editor;
+
+  constructor(cmEditor: CodeMirror.Editor) {
+    this.cmEditor = cmEditor;
+  }
+
+  tokenize(): TokenizedResult | undefined {
+    const cursor = this.cmEditor.getCursor();
+    const token = this.cmEditor.getTokenAt(cursor);
+    if (!token.string) {
+      return undefined;
+    }
+
+    const words = segmenter.segment(token.string);
+    const currentToken = words.pop();
+    const currentTokenStart =
+      token.start + words.reduce((t: number, x: string) => t + x.length, 0);
+    const tokens = pickTokensAsJapanese(this.cmEditor);
+
+    return {
+      currentToken,
+      currentTokenStart,
+      tokens,
+    };
+  }
+}
+
+export function createTokenizer(
+  cmEditor: CodeMirror.Editor,
+  strategy: TokenizeStrategy
+): Tokenizer {
+  switch (strategy) {
+    case "default":
+      return new DefaultTokenizer(cmEditor);
+    case "japanese":
+      return new JapaneseTokenizer(cmEditor);
+    default:
+      throw new Error(`Unexpected strategy name: ${strategy}`);
+  }
+}