Changes
19 changed files (+596/-24)
-
-
@@ -3,8 +3,10 @@ //// SPDX-License-Identifier: Apache-2.0 import type { AssetToken, DocumentContent, DocumentMetadata, DocumentToken, FileReader, } from "../types.ts";
-
@@ -18,6 +20,11 @@export interface ParseParameters { fileReader: FileReader; documentMetadata: DocumentMetadata; getAssetToken(path: readonly string[]): AssetToken | Promise<AssetToken>; getDocumentToken( path: readonly string[], ): DocumentToken | Promise<DocumentToken>; } export interface ContentParser {
-
-
-
@@ -6,7 +6,15 @@ import { assertObjectMatch } from "../deps/deno.land/std/assert/mod.ts";import { MemoryFsReader } from "../filesystem_reader/memory_fs.ts"; import { ObsidianMarkdownParser } from "./obsidian_markdown.ts"; import type { FileReader } from "../types.ts"; import type { AssetToken, DocumentToken, FileReader } from "../types.ts"; function getAssetToken(path: readonly string[]): AssetToken { return `mxa_${path.join(".")}`; } function getDocumentToken(path: readonly string[]): DocumentToken { return `mxt_${path.join(".")}`; } Deno.test("Should parse CommonMark syntax", async () => { const fs = new MemoryFsReader([
-
@@ -33,6 +41,8 @@ title: "Test",name: "Test", }, fileReader, getAssetToken, getDocumentToken, }); assertObjectMatch(
-
@@ -87,6 +97,8 @@ title: "Test",name: "Test", }, fileReader, getAssetToken, getDocumentToken, }); assertObjectMatch(content, {
-
-
-
@@ -11,6 +11,7 @@import { ofmHighlightFromMarkdown } from "./obsidian_markdown/mdast_util_ofm_highlight.ts"; import { ofmHighlight } from "./obsidian_markdown/micromark_extension_ofm_highlight.ts"; import { ofmImageSize } from "./obsidian_markdown/mdast_util_ofm_image_size.ts"; import { macanaMarkAssets } from "./obsidian_markdown/mdast_util_macana_mark_assets.ts"; import type { ContentParser,
-
@@ -18,6 +19,8 @@ ContentParseResult,ParseParameters, } from "./interface.ts"; import type { DocumentContent } from "../types.ts"; export { macanaReplaceAssetTokens } from "./obsidian_markdown/mdast_util_macana_replace_asset_tokens.ts"; function getFrontMatterValue( frontmatter: Record<string, unknown>,
-
@@ -61,13 +64,21 @@ */frontmatter?: boolean; } function parseMarkdown(markdown: string | Uint8Array) { async function parseMarkdown( markdown: string | Uint8Array, { getAssetToken }: Pick< ParseParameters, "getAssetToken" | "getDocumentToken" >, ): Promise<Mdast.Root> { const mdast = fromMarkdown(markdown, { extensions: [gfm(), ofmHighlight()], mdastExtensions: [gfmFromMarkdown(), ofmHighlightFromMarkdown()], }); ofmImageSize(mdast); await macanaMarkAssets(mdast, getAssetToken); return mdast; }
-
@@ -80,14 +91,18 @@ this.#frontmatter = frontmatter;} async parse( { fileReader, documentMetadata }: ParseParameters, { fileReader, documentMetadata, getDocumentToken, getAssetToken }: ParseParameters, ): Promise<ContentParseResult<ObsidianMarkdownDocument>> { const bytes = await fileReader.read(); if (!this.#frontmatter) { return { kind: "obsidian_markdown", content: parseMarkdown(bytes), content: await parseMarkdown(bytes, { getDocumentToken, getAssetToken, }), }; }
-
@@ -108,7 +123,10 @@ language: lang || documentMetadata.language,}, documentContent: { kind: "obsidian_markdown", content: parseMarkdown(frontmatter.body), content: await parseMarkdown(frontmatter.body, { getAssetToken, getDocumentToken, }), }, }; }
-
-
-
@@ -0,0 +1,72 @@// SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> // // SPDX-License-Identifier: Apache-2.0 import { assert, assertObjectMatch, } from "../../deps/deno.land/std/assert/mod.ts"; import type * as Mdast from "../../deps/esm.sh/mdast/types.ts"; import { fromMarkdown } from "../../deps/esm.sh/mdast-util-from-markdown/mod.ts"; import { macanaMarkAssets } from "./mdast_util_macana_mark_assets.ts"; const getAssetToken = (path: readonly string[]) => `mxa_${path.join("/")}` as const; Deno.test("Should set Asset Token on image's data", async () => { const mdast = fromMarkdown(""); await macanaMarkAssets(mdast, getAssetToken); assertObjectMatch(mdast, { type: "root", children: [ { type: "paragraph", children: [ { type: "image", alt: "Foo", data: { macanaAssetToken: "mxa_./foo.png", }, }, ], }, ], }); }); Deno.test("Should support Vaule root absolute path for image", async () => { const mdast = fromMarkdown(""); await macanaMarkAssets(mdast, getAssetToken); assertObjectMatch(mdast, { type: "root", children: [ { type: "paragraph", children: [ { type: "image", alt: "Foo", data: { macanaAssetToken: "mxa_/foo.png", }, }, ], }, ], }); }); Deno.test("Should skip full image URL", async () => { const mdast = fromMarkdown(""); await macanaMarkAssets(mdast, getAssetToken); assert(!((mdast.children[0] as Mdast.Paragraph).children[0].data)); });
-
-
-
@@ -0,0 +1,75 @@// SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> // // SPDX-License-Identifier: Apache-2.0 import type * as Mdast from "../../deps/esm.sh/mdast/types.ts"; import { SKIP, visit } from "../../deps/esm.sh/unist-util-visit/mod.ts"; import { definitions } from "../../deps/esm.sh/mdast-util-definitions/mod.ts"; import type { ParseParameters } from "../interface.ts"; const SEPARATOR = "/"; const URL_REGEXP_PATTERN = /^[a-z0-9]+:\/\//i; /** * Searches asset references mark those node by setting `macanaAssetToken` with Asset Token. * This function modifies the Mdast tree in place. */ export async function macanaMarkAssets( tree: Mdast.Nodes, getAssetToken: ParseParameters["getAssetToken"], ): Promise<void> { const promises: Promise<unknown>[] = []; const defs = definitions(tree); visit( tree, (node) => node.type === "image" || node.type === "imageReference", (node) => { switch (node.type) { case "image": { // Full URL if (URL_REGEXP_PATTERN.test(node.url)) { return SKIP; } const path = node.url.split(SEPARATOR); promises.push( Promise.resolve(getAssetToken(path)).then((token) => { node.data ??= {}; // @ts-expect-error: incorrect library type definition. node.data.macanaAssetToken = token; }), ); return SKIP; } case "imageReference": { const def = defs(node.identifier); if (!def) { return; } // Full URL if (URL_REGEXP_PATTERN.test(def.url)) { return SKIP; } const path = def.url.split(SEPARATOR); promises.push( Promise.resolve(getAssetToken(path)).then((token) => { def.data ??= {}; // @ts-expect-error: incorrect library type definition. def.data.macanaAssetToken = token; }), ); return SKIP; } } }, ); await Promise.all(promises); }
-
-
-
@@ -0,0 +1,42 @@// SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> // // SPDX-License-Identifier: Apache-2.0 import { assertObjectMatch } from "../../deps/deno.land/std/assert/mod.ts"; import { fromMarkdown } from "../../deps/esm.sh/mdast-util-from-markdown/mod.ts"; import { macanaMarkAssets } from "./mdast_util_macana_mark_assets.ts"; import { macanaReplaceAssetTokens } from "./mdast_util_macana_replace_asset_tokens.ts"; Deno.test("Should replace Asset Token on images", async () => { const mdast = fromMarkdown(""); await macanaMarkAssets(mdast, () => { return "mxa_0"; }); await macanaReplaceAssetTokens(mdast, (token) => { if (token !== "mxa_0") { throw new Error("Unexpected token"); } return "../FOO.PNG"; }); assertObjectMatch(mdast, { type: "root", children: [ { type: "paragraph", children: [ { type: "image", alt: "Foo", url: "../FOO.PNG", }, ], }, ], }); });
-
-
-
@@ -0,0 +1,72 @@// SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> // // SPDX-License-Identifier: Apache-2.0 import type * as Mdast from "../../deps/esm.sh/mdast/types.ts"; import { visit } from "../../deps/esm.sh/unist-util-visit/mod.ts"; import type { AssetToken } from "../../types.ts"; function extractToken(node: Mdast.Node): AssetToken { if ( !node.data || !("macanaAssetToken" in node.data) || typeof node.data.macanaAssetToken !== "string" || !node.data.macanaAssetToken.startsWith("mxa_") ) { throw new Error(`Asset Token not found on the node: ${node.type}`); } return node.data.macanaAssetToken as AssetToken; } function replace(node: Mdast.Nodes, replacedPath: string): void { switch (node.type) { case "image": { node.url = replacedPath; return; } case "definition": { node.url = replacedPath; return; } } } /** * Modifies the given Mdast tree by searching nodes having `macanaAssetToken` * property then replacing node properties. * This function modifies Mdast tree in place. * * @param tree - Mdast tree to modify. * @param replacer - A function that takes Asset Token and returns path *string* for the asset. */ export async function macanaReplaceAssetTokens( tree: Mdast.Nodes, replacer: (token: AssetToken) => string | Promise<string>, ): Promise<void> { const promises: Promise<unknown>[] = []; visit( tree, (node) => node.data && "macanaAssetToken" in node.data && typeof node.data.macanaAssetToken === "string", (node) => { const token = extractToken(node); const replaced = replacer(token); if (replaced instanceof Promise) { promises.push(replaced.then((str) => { replace(node, str); })); return; } replace(node, replaced); }, ); await Promise.all(promises); }
-
-
-
@@ -8,9 +8,18 @@ unreachable,} from "../deps/deno.land/std/assert/mod.ts"; import { MemoryFsReader } from "../filesystem_reader/memory_fs.ts"; import type { AssetToken, DocumentToken } from "../types.ts"; import type { ContentParser } from "./interface.ts"; import { oneof } from "./oneof.ts"; function getAssetToken(path: readonly string[]): AssetToken { return `mxa_${path.join(".")}`; } function getDocumentToken(path: readonly string[]): DocumentToken { return `mxt_${path.join(".")}`; } function literal(x: string): ContentParser { return {
-
@@ -55,6 +64,8 @@ name: item.name,title: item.name, }, fileReader: item, getDocumentToken, getAssetToken, }); switch (count++) {
-
-
-
@@ -248,6 +248,8 @@ "https://esm.sh/v135/hast-util-whitespace@3.0.0/denonext/hast-util-whitespace.mjs": "b2988a87d03b42636bca6ebee778326993a23953ba6638973be17ce03100a357","https://esm.sh/v135/inline-style-parser@0.2.2/denonext/inline-style-parser.mjs": "6b516634a2a716b57fa335bdaf9910c568a01330bfb7686ca534228aff59149e", "https://esm.sh/v135/longest-streak@3.1.0/denonext/longest-streak.mjs": "97b1d8c42d407e285971fea218d89a0404270c3b841a26ec62b83f62450ad573", "https://esm.sh/v135/markdown-table@3.0.3/denonext/markdown-table.mjs": "1dd9cb2d2d95fc440cc10489ac77bf979c944e76070997d962e05f627f45df0e", "https://esm.sh/v135/mdast-util-definitions@6.0.0": "b335a17ac86505677a62667dd26fecddb8e94a8629c3193220655ccfd6c50081", "https://esm.sh/v135/mdast-util-definitions@6.0.0/denonext/mdast-util-definitions.mjs": "09d288c69212c2b751b5cc3d187cc0f59ae41db94e7a0fefe2561b3b6c83475e", "https://esm.sh/v135/mdast-util-find-and-replace@3.0.1/denonext/mdast-util-find-and-replace.mjs": "dd75ab7e2d5060ab738fe5da72eeada37c295d5e506b6f8cae61d56dbd875446", "https://esm.sh/v135/mdast-util-from-markdown@2.0.0": "6faa1f66f444b738c9cf8a0e97084ff0ab08008e0236379c1f25c250dd8457f1", "https://esm.sh/v135/mdast-util-from-markdown@2.0.0/denonext/mdast-util-from-markdown.bundle.mjs": "df1b3422a8b65ba9b34a1ff13c48f4b4838e31abfe2b959f21d1d72aa9b8c576",
-
-
-
@@ -0,0 +1,5 @@// SPDX-FileCopyrightText: 2024 Shota FUJI <pockawoooh@gmail.com> // // SPDX-License-Identifier: Apache-2.0 export * from "https://esm.sh/v135/mdast-util-definitions@6.0.0";
-
-
-
@@ -28,4 +28,20 @@ Tree structured data contains *document metadata* and *document directories*.## Document content A parsed content of note or canvas. A parsed content of note or canvas. ## Document token A string starts with `mxt_` that is used for referencing a document. There are places where *Content Parser* needs a reference to *a document* while parsing another *document*, but unable to hold an object reference for the *document*. In such case, *Content Parser* exchanges target *document*'s file path for a *document token*. Then, *Page Builder* can exchange the *document token* for an actual *document* object reference later. ## Asset token A string starts with `mxa_` that is used for referencing an asset. Because of Macana's file I/O design and multi-phase build process, it's not preferable to pass a reference to an asset file around. To avoid keeping references for asset files, *Content Parser* exchanges target asset's file path for an *asset token*. Then, *Page Builder* can exchange the *asset token* for an actual asset file reference later.
-
-
-
@@ -5,6 +5,7 @@## Links - [Source code](https://github.com/pocka/macana) - [日本語版簡易ドã‚ュメント](../ja/Overview.md) ## Goals
-
-
-
@@ -13,7 +13,7 @@ - [ ] Syntax Highlighting- [ ] Obsidian Extensions - [ ] Internal Link path resolution - [ ] Absolute path in vault - [ ] Relative path to file - [x] Relative path to file - [ ] Shortest path when possible - [ ] Wikilink - [ ] Label
-
-
-
@@ -103,6 +103,53 @@ }return converted; }, openFile: (path) => { const resolvedPath = this.#resolve(path); const fileInfo = Deno.statSync(resolvedPath); if (!fileInfo.isFile) { throw new Error(`DenoFsReader: ${resolvedPath} is not a file`); } if (!path.length) { throw new Error(`DenoFsReader: path cannot be empty`); } let parent: DirectoryReader | RootDirectoryReader = root; for (let i = 0, l = path.length - 1; i < l; i++) { const dir: DirectoryReader = { type: "directory", name: path[i], path: parent.type === "root" ? [path[i]] : [...parent.path, path[i]], parent, read: async () => { const converted: Array<FileReader | DirectoryReader> = []; for await (const entry of Deno.readDir(this.#resolve(path))) { converted.push(this.#fromDirEntry(entry, dir)); } return converted; }, }; parent = dir; } const file = path[path.length - 1]; return { type: "file", name: file, path, parent, read: async () => { return Deno.readFile(resolvedPath); }, }; }, }; return root;
-
-
-
@@ -14,6 +14,7 @@ /*** Directly read file contents at given path. * Throws if path does not exist or found directory. * You should traverse from `getRootDirectory()` for most cases. * @deprecated Use `RootDirectoryReader.openFile` then `FileReader.read` instead. */ readFile(path: readonly string[]): Promise<Uint8Array>; }
-
-
-
@@ -173,6 +173,15 @@ getRootDirectory(): Promise<RootDirectoryReader> {const root: RootDirectoryReader = { type: "root", read: () => Promise.resolve(this.#mapToReaders(this.#tree, root)), openFile: async (path) => { const file = await this.#getAtRecur(path, root); if (file.type === "directory") { throw new Error(`MemoryFsReader: ${path.join(SEP)} is directory`); } return file; }, }; return Promise.resolve(root);
-
-
-
@@ -8,7 +8,10 @@ import { extname } from "../../deps/deno.land/std/path/mod.ts";import { h, renderSSR } from "../../deps/deno.land/x/nano_jsx/mod.ts"; import type { BuildParameters, PageBuilder } from "../interface.ts"; import type { ObsidianMarkdownDocument } from "../../content_parser/obsidian_markdown.ts"; import { macanaReplaceAssetTokens, type ObsidianMarkdownDocument, } from "../../content_parser/obsidian_markdown.ts"; import type { JSONCanvasDocument } from "../../content_parser/json_canvas.ts"; import type { Document, DocumentDirectory, DocumentTree } from "../../types.ts";
-
@@ -26,6 +29,14 @@ }function isJSONCanvas(x: Document): x is Document<JSONCanvasDocument> { return x.content.kind === "json_canvas"; } function toRelativePath( path: readonly string[], from: readonly string[], ): string { return Array.from({ length: from.length }, () => "../").join("") + path.join("/"); } export interface Assets {
-
@@ -168,6 +179,25 @@ const { fileSystemWriter } = buildParameters;if ("file" in item) { if (isObsidianMarkdown(item) || isJSONCanvas(item)) { const assetWrites: Promise<unknown>[] = []; if (item.content.kind === "obsidian_markdown") { await macanaReplaceAssetTokens( item.content.content, async (token) => { const file = tree.exchangeToken(token); assetWrites.push(fileSystemWriter.write( file.path, await file.read(), )); // Add trailing slash (empty string) return toRelativePath(file.path, [...item.path, ""]); }, ); } const html = "<!DOCTYPE html>" + renderSSR( () => ( // Adds 1 to depth due to `<name>/index.html` conversion.
-
@@ -185,11 +215,14 @@ );const enc = new TextEncoder(); await fileSystemWriter.write([ ...pathPrefix, item.metadata.name, "index.html", ], enc.encode(html)); await Promise.all([ ...assetWrites, fileSystemWriter.write([ ...pathPrefix, item.metadata.name, "index.html", ], enc.encode(html)), ]); return; }
-
-
-
@@ -6,13 +6,18 @@ import { extname } from "../deps/deno.land/std/path/mod.ts";import type { BuildParameters, TreeBuilder } from "./interface.ts"; import type { AssetToken, DirectoryReader, Document, DocumentDirectory, DocumentMetadata, DocumentToken, DocumentTree, FileReader, RootDirectoryReader, } from "../types.ts"; const INTERNAL_PATH_SEPARATOR = "/"; export type TreeBuildStrategyFunctionReturns = { skip: true;
-
@@ -151,6 +156,48 @@ return { metadata };}; } function isAssetToken(token: unknown): token is AssetToken { return typeof token === "string" && token.startsWith("mxa_"); } function isDocumentToken(token: unknown): token is DocumentToken { return typeof token === "string" && token.startsWith("mxd_"); } function resolveFsrPath( path: readonly string[], base: readonly string[], ): readonly string[] { let buf: string[] = base.slice(0, -1); for (const fragment of path) { switch (fragment) { case ".": break; case "..": buf = buf.slice(0, -1); break; default: buf.push(fragment); break; } } return buf; } interface InternalBuildParameters { contentParser: BuildParameters["contentParser"]; root: RootDirectoryReader; parentPath?: readonly string[]; assetTokensToFiles: Map<AssetToken, FileReader>; documentTokenToPaths: Map<DocumentToken, string>; pathToDocuments: Map<string, Document>; } export interface DefaultTreeBuilderConfig { /** * Default language tag (BCP 47).
-
@@ -200,10 +247,22 @@ { fileSystemReader, contentParser }: BuildParameters,): Promise<DocumentTree> { const root = await fileSystemReader.getRootDirectory(); const assetTokensToFiles = new Map<AssetToken, FileReader>(); const documentTokenToPaths = new Map<DocumentToken, string>(); const pathToDocuments = new Map<string, Document>(); const children = await root.read(); const entries = await Promise.all( children.map((child) => this.#build(child, { contentParser })), children.map((child) => this.#build(child, { contentParser, root, assetTokensToFiles, documentTokenToPaths, pathToDocuments, }) ), ); const nodes = entries.filter((entry): entry is NonNullable<typeof entry> =>
-
@@ -222,16 +281,51 @@ type: "tree",nodes, defaultDocument, defaultLanguage: this.#defaultLanguage, exchangeToken: ((token) => { if (isAssetToken(token)) { const found = assetTokensToFiles.get(token); if (!found) { throw new Error( `DefaultTreeBuilder: No asset file correspond to Asset Token ${token}`, ); } return found; } if (isDocumentToken(token)) { const path = documentTokenToPaths.get(token); if (!path) { throw new Error( `DefaultTreeBuilder: No document path registered for the Document Token ${token}`, ); } const doc = pathToDocuments.get(path); if (!doc) { throw new Error( `DefaultTreeBuilder: No document at the path ${path}, referenced by token ${token}`, ); } return doc; } throw new Error(`DefaultTreeBuilder: Invalid token type: ${token}`); }) as DocumentTree["exchangeToken"], }; } async #build( node: FileReader | DirectoryReader, { contentParser }: Omit< BuildParameters, "fileSystemReader" >, parentPath: readonly string[] = [], { contentParser, root, assetTokensToFiles, documentTokenToPaths, pathToDocuments, parentPath = [], }: InternalBuildParameters, ): Promise<DocumentDirectory | Document | null> { let metadata: DocumentMetadata = { name: node.name,
-
@@ -252,9 +346,31 @@ if (node.type === "file") {const result = await contentParser.parse({ fileReader: node, documentMetadata: metadata, async getAssetToken(path) { const id = crypto.randomUUID(); const token: AssetToken = `mxa_${id}`; assetTokensToFiles.set( token, await root.openFile(resolveFsrPath(path, node.path)), ); return token; }, getDocumentToken(path) { const id = crypto.randomUUID(); const token: DocumentToken = `mxt_${id}`; documentTokenToPaths.set( token, resolveFsrPath(path, node.path).join(INTERNAL_PATH_SEPARATOR), ); return token; }, }); return { const document: Document = { type: "document", metadata: "documentMetadata" in result ? result.documentMetadata
-
@@ -263,16 +379,27 @@ file: node,content: "documentContent" in result ? result.documentContent : result, path: [...parentPath, metadata.name], }; pathToDocuments.set(node.path.join(INTERNAL_PATH_SEPARATOR), document); return document; } const children = await node.read(); const entries = await Promise.all( children.map((child) => this.#build(child, { contentParser }, [ ...parentPath, metadata.name, ]) this.#build(child, { contentParser, root, assetTokensToFiles, documentTokenToPaths, pathToDocuments, parentPath: [ ...parentPath, metadata.name, ], }) ), );
-
-
-
@@ -20,10 +20,20 @@read(): Promise<ReadonlyArray<FileReader | DirectoryReader>>; } export type DocumentToken = `mxt_${string}`; export type AssetToken = `mxa_${string}`; export interface RootDirectoryReader { readonly type: "root"; read(): Promise<ReadonlyArray<FileReader | DirectoryReader>>; /** * Returns a file at the path. * This function may throw an error if the file not found. */ openFile(path: readonly string[]): Promise<FileReader> | FileReader; } export interface DocumentMetadata {
-
@@ -98,4 +108,16 @@ /*** Representive, facade document. */ readonly defaultDocument: Document; /** * Get a document in exchange for the token. * Throws an error if the token is invalid or target document is missing. */ exchangeToken(token: DocumentToken): Document; /** * Get an asset file in exchange for the token. * Throws an error if the token is invalid or target file is missing. */ exchangeToken(token: AssetToken): FileReader; }
-