Highlight Markdown extension from Obsidian Flavored Markdown This patch also works as foundation / reference code for other internal Markdown extensions.

Changes

16 changed files (+502/-3)

• content_parser/obsidian_markdown.ts
  • @@ -8,6 +8,9 @@ import { fromMarkdown } from "../deps/esm.sh/mdast-util-from-markdown/mod.ts";
    import { gfmFromMarkdown } from "../deps/esm.sh/mdast-util-gfm/mod.ts"; import { gfm } from "../deps/esm.sh/micromark-extension-gfm/mod.ts"; + import { ofmHighlightFromMarkdown } from "./obsidian_markdown/mdast_util_ofm_highlight.ts"; + import { ofmHighlight } from "./obsidian_markdown/micromark_extension_ofm_highlight.ts"; + import type { ContentParser, ContentParseResult,
  • @@ -59,8 +62,8 @@ }
    function parseMarkdown(markdown: string | Uint8Array) { return fromMarkdown(markdown, { - extensions: [gfm()], - mdastExtensions: [gfmFromMarkdown()], + extensions: [gfm(), ofmHighlight()], + mdastExtensions: [gfmFromMarkdown(), ofmHighlightFromMarkdown()], }); }
• content_parser/obsidian_markdown/mdast_util_ofm_highlight.test.ts (new)
  • @@ -0,0 +1,37 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import { assertObjectMatch } from "../../deps/deno.land/std/assert/mod.ts"; + + import { fromMarkdown } from "../../deps/esm.sh/mdast-util-from-markdown/mod.ts"; + + import { ofmHighlight } from "./micromark_extension_ofm_highlight.ts"; + import { ofmHighlightFromMarkdown } from "./mdast_util_ofm_highlight.ts"; + + Deno.test("Should parse highlights markdown into Mdast node", () => { + const mdast = fromMarkdown("==Hello, World!==", { + extensions: [ofmHighlight()], + mdastExtensions: [ofmHighlightFromMarkdown()], + }); + + assertObjectMatch(mdast, { + type: "root", + children: [ + { + type: "paragraph", + children: [ + { + type: "ofmHighlight", + children: [ + { + type: "text", + value: "Hello, World!", + }, + ], + }, + ], + }, + ], + }); + });
• content_parser/obsidian_markdown/mdast_util_ofm_highlight.ts (new)
  • @@ -0,0 +1,28 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import type { Extension } from "../../deps/esm.sh/mdast-util-from-markdown/mod.ts"; + + export function ofmHighlightFromMarkdown(): Extension { + return { + enter: { + ofmHighlight(token) { + this.enter({ + // @ts-expect-error: unist-related libraries heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + type: "ofmHighlight", + children: [], + data: { + hName: "mark", + }, + }, token); + }, + }, + exit: { + ofmHighlight(token) { + this.exit(token); + }, + }, + }; + }
• content_parser/obsidian_markdown/micromark_extension_ofm_highlight.test.ts (new)
  • @@ -0,0 +1,73 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import { assertEquals } from "../../deps/deno.land/std/assert/mod.ts"; + + import { micromark } from "../../deps/esm.sh/micromark/mod.ts"; + + import { + ofmHighlight, + ofmHighlightHtml, + } from "./micromark_extension_ofm_highlight.ts"; + + function f(markdown: string): string { + return micromark(markdown, { + extensions: [ofmHighlight()], + htmlExtensions: [ofmHighlightHtml()], + }); + } + + Deno.test("Should parse basic highlights", () => { + assertEquals( + f("==Hello, World!=="), + "<p><mark>Hello, World!</mark></p>", + ); + }); + + Deno.test("Should not treat as highlight when no subsequent equal signs", () => { + assertEquals( + f("=Hello, World!="), + "<p>=Hello, World!=</p>", + ); + }); + + Deno.test("Should not open when the opening sequence is escaped", () => { + assertEquals( + f("\\==foo=="), + "<p>==foo==</p>", + ); + }); + + Deno.test("Should not treat setext heading symbols as highlight", () => { + assertEquals( + f("====="), + "<p>=====</p>", + ); + }); + + Deno.test("Should not conflict with setext heading", () => { + assertEquals( + f("Foo bar\n======"), + "<h1>Foo bar</h1>", + ); + }); + + Deno.test("Should not parse when space exists right after the opening sequence", () => { + assertEquals( + f("== Not highlighted =="), + "<p>== Not highlighted ==</p>", + ); + + assertEquals( + f("== Me too=="), + "<p>== Me too==</p>", + ); + }); + + Deno.test("Should not nest", () => { + assertEquals( + f("==Foo==Bar==Baz=="), + "<p><mark>Foo</mark>Bar<mark>Baz</mark></p>", + ); + });
• content_parser/obsidian_markdown/micromark_extension_ofm_highlight.ts (new)
  • @@ -0,0 +1,223 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + import type { + CompileContext, + Construct, + Event, + Extension, + HtmlExtension, + Resolver, + State, + Token, + Tokenizer, + } from "../../deps/esm.sh/micromark-util-types/types.ts"; + import { + codes, + constants, + types, + } from "../../deps/esm.sh/micromark-util-symbol/mod.ts"; + import { classifyCharacter } from "../../deps/esm.sh/micromark-util-classify-character/mod.ts"; + import { splice } from "../../deps/esm.sh/micromark-util-chunked/mod.ts"; + import { resolveAll } from "../../deps/esm.sh/micromark-util-resolve-all/mod.ts"; + + const enum TokenTypeMap { + sequenceTemporary = "ofmHighlightSequenceTemporary", + sequence = "ofmHighlightSequence", + highlight = "ofmHighlight", + highlightText = "ofmHighlightText", + } + + export function ofmHighlightHtml(): HtmlExtension { + return { + enter: { + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + [TokenTypeMap.highlight](this: CompileContext) { + this.tag("<mark>"); + }, + }, + exit: { + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + [TokenTypeMap.highlight](this: CompileContext) { + this.tag("</mark>"); + }, + }, + }; + } + + export function ofmHighlight(): Extension { + return { + text: { + [codes.equalsTo]: construct, + }, + attentionMarkers: { null: [codes.equalsTo] }, + }; + } + + const tokenizer: Tokenizer = function (effects, ok, nok) { + const { previous, events } = this; + let count = 0; + + const start: State = function (code) { + // This is from strikethrough extension, but I have no idea what it does. + if ( + previous === codes.equalsTo && + events[events.length - 1][1].type !== types.characterEscape + ) { + return nok(code); + } + + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + effects.enter(TokenTypeMap.sequenceTemporary); + return more(code); + }; + + const more: State = function (code) { + // The first or subsequent equal signs. No other characters appeared yet. + if (code === codes.equalsTo) { + effects.consume(code); + count += 1; + return more; + } + + // Reject single equals, e.g. "=foo=" + if (count < 2) { + return nok(code); + } + + // Since the character we are looking right now is no longer an equal sign, + // we can exit temporary state and start consuming next phrasing-something-idk. + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + const token = effects.exit(TokenTypeMap.sequenceTemporary); + + const before = classifyCharacter(previous); + const after = classifyCharacter(code); + + token._open = !after || + (after === constants.attentionSideAfter && !!before); + token._close = !before || + (before === constants.attentionSideAfter && !!after); + + return ok(code); + }; + + return start; + }; + + // Mutate events + const resolver: Resolver = function (events, context) { + for (let i = 0; i < events.length; i++) { + // If the event is not an end of temporary sequence we set on the tokenizer, skip. + if ( + events[i][0] !== "enter" || + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + events[i][1].type !== TokenTypeMap.sequenceTemporary || + !events[i][1]._close + ) { + continue; + } + + // Walk back to find an open sequence + for (let j = i; j > 0; j--) { + // If the event is not a start of temporary sequence we set on the tokenizer, skip. + if ( + events[j][0] !== "exit" || + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + events[j][1].type !== TokenTypeMap.sequenceTemporary || + !events[j][1]._open + ) { + continue; + } + + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + events[i][1].type = TokenTypeMap.sequence; + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + events[j][1].type = TokenTypeMap.sequence; + + // The whole highlight section, including sequences. + const highlight: Token = { + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + type: TokenTypeMap.highlight, + start: { ...events[j][1].start }, + end: { ...events[i][1].end }, + }; + + // Text between highlight sequences. + const text: Token = { + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + type: TokenTypeMap.highlightText, + // text starts at the end of opening sequence, and + start: { ...events[j][1].end }, + // ends with at the start of closing sequence. + end: { ...events[i][1].start }, + }; + + // Initialize with opening events + const nextEvents: Event[] = [ + ["enter", highlight, context], + ["enter", events[j][1], context], + ["exit", events[j][1], context], + ["enter", text, context], + ]; + + const insideSpan = context.parser.constructs.insideSpan.null; + if (insideSpan) { + // Resolve text? idk + splice( + nextEvents, + nextEvents.length, + 0, + resolveAll(insideSpan, events.slice(j + 1, i), context), + ); + } + + // Append closing events to `nextEvents` + splice( + nextEvents, + nextEvents.length, + 0, + [ + ["exit", text, context], + ["enter", events[i][1], context], + ["exit", events[i][1], context], + ["exit", highlight, context], + ], + ); + + // Replace part of original events with `nextEvents` + splice(events, j - 1, i - j + 3, nextEvents); + + i = j + nextEvents.length - 2; + break; + } + } + + for (const event of events) { + // If there are temporary sequences left, alter them to `data` type. + // (effectively making them invisible?, idk) + // @ts-expect-error: micromark heavily relies on ambiend module declarations, + // which Deno does not support. APIs also don't accept type parameters. + if (event[1].type === TokenTypeMap.sequenceTemporary) { + event[1].type = types.data; + } + } + + return events; + }; + + const construct: Construct = { + name: "ofm-highlight", + tokenize: tokenizer, + resolveAll: resolver, + };
• deno.lock
  • @@ -284,7 +284,9 @@ "https://esm.sh/v135/micromark-factory-space@2.0.0/denonext/micromark-factory-space.mjs": "1ac7c90dec53f7f634767c5470c2dcf204f4df99ec318a27832786153d5c8110",
    "https://esm.sh/v135/micromark-factory-title@2.0.0/denonext/micromark-factory-title.mjs": "1b202816c09c57894c8e254962ffbd8ad559439b0eb90ff0eca743f0dbfff470", "https://esm.sh/v135/micromark-factory-whitespace@2.0.0/denonext/micromark-factory-whitespace.mjs": "f85efacaec053453a9445b4147d2fb7ce02c1d083f0b0da9a730a112ff934d9e", "https://esm.sh/v135/micromark-util-character@2.0.1/denonext/micromark-util-character.mjs": "18b451d148e1ccc3a9b18e5c4061d44a0485e8ec65ad805d20b2950a51c7213b", + "https://esm.sh/v135/micromark-util-chunked@2.0.0": "6137fb50257da8a18125ac21ac885bd56f79e6ead241b0b004b0416c7906eaa2", "https://esm.sh/v135/micromark-util-chunked@2.0.0/denonext/micromark-util-chunked.mjs": "531cf323ba53649fdc30cd39ebba54253dfd847a4b23f806058ecc6cf67bca69", + "https://esm.sh/v135/micromark-util-classify-character@2.0.0": "c92c93be5f8370abbd73783460c6e183f2b31fe74085175389e1c86d96aeef9b", "https://esm.sh/v135/micromark-util-classify-character@2.0.0/denonext/micromark-util-classify-character.mjs": "7e78c1341df2227c29cf5125c17f29bec3887bf8a3178476e129740271bdbd96", "https://esm.sh/v135/micromark-util-combine-extensions@2.0.0/denonext/micromark-util-combine-extensions.mjs": "68268c6cb6119bc8b0865156e81a360a3b94cd73bddbb9a2a33eda627e51573a", "https://esm.sh/v135/micromark-util-decode-numeric-character-reference@2.0.1/denonext/micromark-util-decode-numeric-character-reference.mjs": "284addb1c2303e02ca074d6a5e529c8c9c3cdb58b6815669e2ec65a2f717cf28",
  • @@ -292,9 +294,15 @@ "https://esm.sh/v135/micromark-util-decode-string@2.0.0/denonext/micromark-util-decode-string.mjs": "c9abaf0b645a2ca8dd0a8d988a7a25173f4bc5e46154d2875890507c0d2e5a51",
    "https://esm.sh/v135/micromark-util-encode@2.0.0/denonext/micromark-util-encode.mjs": "6077703d774b2fd968ef53977add5d5d1e39ca0db74c5ee4359c540a5febcf48", "https://esm.sh/v135/micromark-util-html-tag-name@2.0.0/denonext/micromark-util-html-tag-name.mjs": "a32f6a4aa82498405a88103fd5b0b2e27a8c7f27dc506862f993bf1a1f1716b6", "https://esm.sh/v135/micromark-util-normalize-identifier@2.0.0/denonext/micromark-util-normalize-identifier.mjs": "f5b933ea50544d63e505ef11b7f257b97c5056e06417bac15ec02d3f00174c0e", + "https://esm.sh/v135/micromark-util-resolve-all@2.0.0": "7497db04242d229406c3ddd67af2f3cabd479e11404f5d0828153105fdcad173", "https://esm.sh/v135/micromark-util-resolve-all@2.0.0/denonext/micromark-util-resolve-all.mjs": "c11d87d63d808a26231323012295490931159830a66c00854693ec9279fa09fd", "https://esm.sh/v135/micromark-util-sanitize-uri@2.0.0/denonext/micromark-util-sanitize-uri.mjs": "cde22cced5a18a41dcdacbad2cc9e6fc930450118bff7dd764e60eea4aa6c5b5", "https://esm.sh/v135/micromark-util-subtokenize@2.0.0/denonext/micromark-util-subtokenize.mjs": "670326123b1f9d91218cee035248437b97e5b6828f1d1eb01d40bc46f89bdc64", + "https://esm.sh/v135/micromark-util-symbol@2.0.0": "662e9dab6f1c75152381b6c40497348af31dece3ae38d81056b0dacea81555f0", + "https://esm.sh/v135/micromark-util-symbol@2.0.0/denonext/micromark-util-symbol.mjs": "58c67ebaeb978385a5aeb119074fc4a918ecfbfd1628ba82f5df90d7b2de2985", + "https://esm.sh/v135/micromark-util-types@2.0.0": "84a2a99a9f9aa159f1935ffe17a43847a0c36d163389117d3100ebddb764c8c3", + "https://esm.sh/v135/micromark-util-types@2.0.0/denonext/micromark-util-types.mjs": "e9ed9735f978ab0109a89d83c2c230281cbf881341a60c027344b987c4acb3cd", + "https://esm.sh/v135/micromark@4.0.0": "caad3cf8d257301f26f48d2cc53267370f2f7108e98e1e75e93f411a9ec766eb", "https://esm.sh/v135/micromark@4.0.0/denonext/micromark.mjs": "dc73ed793c5bbc49ad7201a326fc724d08bc94372a291c184167b22c4edc320d", "https://esm.sh/v135/nth-check@2.1.1/denonext/nth-check.mjs": "638b4f5a22236cd05c7d1d43e5c6ea719695c4a8bc7beccdf8d97a434bea96dc", "https://esm.sh/v135/preact-render-to-string@6.4.1": "a9ca466e5daf03595e041d96829c74f8e493f928e35a64b56365d952fda04ec3",
• deps/esm.sh/micromark-util-chunked/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark-util-chunked@2.0.0";
• deps/esm.sh/micromark-util-classify-character/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark-util-classify-character@2.0.0";
• deps/esm.sh/micromark-util-resolve-all/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark-util-resolve-all@2.0.0";
• deps/esm.sh/micromark-util-symbol/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark-util-symbol@2.0.0";
• deps/esm.sh/micromark-util-types/types.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark-util-types@2.0.0";
• deps/esm.sh/micromark/mod.ts (new)
  • @@ -0,0 +1,5 @@
    + // SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> + // + // SPDX-License-Identifier: Apache-2.0 + + export * from "https://esm.sh/v135/micromark@4.0.0";
• docs/en/Internals/Tests/Obsidian Flavored Markdown/Highlight extension.md (new)
  • @@ -0,0 +1,80 @@
    + Highlight extension in Obsidian is so buggy... it changes between editing view and reading view. + So Macana does not strictly following the implementation: rather follow the same rule to GFM's strikethrough extension. + + Many of these cases are **intentionally** excluded from unit tests, as there is no "correct" behavior. + + ## Basics + + ```markdown + ==Hello, World!== + ``` + + ==Hello, World!== + + ## Escapes + + ```markdown + \==foo== + ==bar\==baz + =====\========= + ``` + + \==foo== + ==bar\==baz + =====\========= + + ## Edge cases + + ### Avoid conflict with header usage (CommonMark) + + ```markdown + ===== + ``` + + ===== + + ### Whitespaces + + Whitespace right after the start `==` prevents the section from being highlighted. + + ```markdown + - == Not highlighted == + - == Me too== + - ==I'm not highlighted in reading view == + ``` + + - == Not highlighted == + - == Me too== + - ==I'm not highlighted in reading view == + + ### No closing tags + + It seems closing tag is optional. Ends at block end. + + ```markdown + ==This is highlighted, + and this line too. + + but this is not. + ``` + + ==This is highlighted, + and this line too. + + but this is not. + + ### More than two symbols + + ```markdown + ==========This is highlighted and 1+1=2 this too, ========but this isn't. + ``` + + ==========This is highlighted and 1+1=2 this too, ========but this isn't. + + ### Nested markers + + ```markdown + ==Foo==Bar==Baz== + ``` + + ==Foo==Bar==Baz==
• docs/en/Internals/Tests/Obsidian Flavored Markdown/What's this?.md (new)
  • @@ -0,0 +1,8 @@
    + This folder is to test how Obsidian handles their Markdown extensions. + + Code correctness is testable via unit tests. + However, as neither of format spec nor test cases exist, we need to *see* how it works by actually writing down and displaying it inside Obsidian. + + Most of the unit tests are picked up from this folder. + + - [[Highlight extension]]
• docs/en/Roadmap.md
  • @@ -27,7 +27,7 @@ - [ ] Audio file
    - [ ] PDF file - [ ] List from another file - [ ] Search results - - [ ] ==Highlight== + - [x] ==Highlight== - [ ] Callouts - [ ] Comments %% You can check this item once I'm no longer visible %% - [ ] Strip Raw HTML (only `<title>` is troublesome, but align behavior to Obsidian's)
• docs/en/Samples/Obsidian Flavored Markdown.md (new)
  • @@ -0,0 +1,9 @@
    + [Obsidian docs](https://help.obsidian.md/Editing+and+formatting/Basic+formatting+syntax) + + ## Highlight extension + + ```markdown + ==Hello, World!== + ``` + + ==Hello, World!==