diff --git a/LICENSE b/LICENSE index b0e20f5..5a1ad2d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,21 @@ -Copyright (c) All rights reserved. +MIT License -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +Copyright (c) 2022 Izzuddin Natsir -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index dc27f0f..2248a51 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,170 @@ -# osr-package-template +# Deepmark -Package basics \ No newline at end of file +Translate markdown files correctly with `mdast` and DeepL. + +## Getting Started + +1. Install `deepmark`: + +```bash +# NPM +npm install -D deepmark + +# PNPM +pnpm add -D deepmark + +# Yarn +yarn add -D deepmark +``` + +2. Create a `deepmark.config.mjs` on your project root: + +```js +// deepmark.config.mjs + +/** @type {import("deepmark").UserConfig} */ +export default { + sourceLanguage: 'en', + outputLanguages: ['zh', 'ja'], + directories: [ + ['i18n/$langcode$', 'i18n/$langcode$'], + ['docs', 'i18n/$langcode$/docs'], + ['blog', 'i18n/$langcode$/blog'] + ] +}; +``` + +3. Set `DEEPL_AUTH_KEY` environment variable containing DeepL developer auth key: + +```bash +# If you're on Linux +export DEEPL_AUTH_KEY=your_auth_key +``` + +You can also use something like `dotenv` package. For CI and remote environment such as Github Actions and Gitpod, look into their setting page to set environment variables. + +4. Set NPM scripts in your `package.json`: + +```json +{ + "scripts": { + "translate": "deepmark translate" + } +} +``` + +```bash +# NPM +npm run translate + +# PNPM +pnpm run translate + +# Yarn +yarn translate +``` + +Notes: + +> `deepmark` also supports translating JSON and YAML files. + +## Limitations + +1. It is not possible to add `mdast` plugins to the workflow. +2. Only support `.md` and `.mdx`. Other extended verisons of markdown such as Svelte extended `mdsvex` are not supported. + +## Documentation + +#### Translation modes: + +```bash +deepmark translate --mode hybrid|offline|online +``` + +- `hybrid` (default): Look for translation from local translation memory first before using DeepL API. +- `offline`: Translate using the local translation memory only, passthrough if not found. This mode will not update the translation memory. +- `online`: Translate using DeepL API only. Will overwrite existing translation memory. + +#### Configuration: + +```ts +interface UserConfig { + /** + * Source's language code. Based on DeepL supported languages. + */ + sourceLanguage: SourceLanguageCode; + /** + * Output's languages code. Based on DeepL supported languages. + */ + outputLanguages: TargetLanguageCode[]; + /** + * Sources and ouputs directories pairs. $langcode$ variable + * is provided to dynamically define directory. + * + * e.g. [ ["docs", "i18n/$langcode$/docs"], ["blog", "i18n/$langcode$/blog"] ] + */ + directories: [string, string][]; + /** + * Override current working directory, defaults to `process.cwd()`. + */ + cwd?: string; + /** + * By default, all .md, .mdx, .json, and .yaml|.yml files inside + * source directories will be included. + * + * Define glob patterns to filter what files to include or exclude. + * But, the end result is still restricted by file types (.md, .mdx, .json). + */ + files?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Frontmatter fields. + */ + frontmatterFields?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`. + */ + markdownNodes?: { + default?: boolean; + include?: MdNodeType[]; + exclude?: MdNodeType[]; + }; + /** + * HTML elements to include and exlcude, down to the level of attributes + * and children. Include all HTML elements text content + * and some global attributes such as title and placeholder. + */ + htmlElements?: { + default?: boolean; + include?: Partial<{ [Tag in HtmlTag]: { children: boolean; attributes: string[] } }>; + exclude?: HtmlTag[]; + }; + /** + * JSX components to include and exclude, down to the level of attributes + * and children. Include all JSX components text children + * and exclude all attributes by default. + * + * Support array, object, and jsx attribute value. For object and array value, + * you can specify the access path starting with the attribute name + * e.g. `items.description` to translate `items={[{description: "..."}]}. + */ + jsxComponents?: { + default?: boolean; + include?: { [Name: string]: { children: boolean; attributes: string[] } }; + exclude?: string[]; + }; + /** + * JSON or YAML file properties to include and exclude. + * Exclude all properties by default. + */ + jsonOrYamlProperties?: { + include?: string[]; + exclude?: string[]; + }; +} +``` diff --git a/build.js b/build.js new file mode 100644 index 0000000..bd7d002 --- /dev/null +++ b/build.js @@ -0,0 +1,44 @@ +import { transform } from 'esbuild'; +import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises'; +import { join, dirname, relative } from 'node:path'; + +const SRC = 'src'; +const DIST = 'dist'; + +async function getFiles(dir) { + const entries = await readdir(dir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === '__test__' || entry.name === 'types') continue; + files.push(...(await getFiles(full))); + } else if (entry.name.endsWith('.ts')) { + files.push(full); + } + } + return files; +} + +async function main() { + const files = await getFiles(SRC); + + for (const file of files) { + const input = await readFile(file, 'utf-8'); + const { code } = await transform(input, { + format: 'esm', + loader: 'ts', + target: 'es2021' + }); + + const rel = relative(SRC, file).replace(/\.ts$/, '.js'); + const outPath = join(DIST, rel); + + await mkdir(dirname(outPath), { recursive: true }); + await writeFile(outPath, code); + } + + console.log(`Built ${files.length} files to ${DIST}/`); +} + +main(); diff --git a/dist/ast/estree.d.ts b/dist/ast/estree.d.ts new file mode 100644 index 0000000..fe314c2 --- /dev/null +++ b/dist/ast/estree.d.ts @@ -0,0 +1,115 @@ +import type { BaseNode as EsBaseNode, Identifier as EsIdentifier, Program as EsProgram, SwitchCase as EsSwitchCase, CatchClause as EsCatchClause, VariableDeclarator as EsVariableDeclarator, ExpressionStatement as EsExpressionStatement, BlockStatement as EsBlockStatement, EmptyStatement as EsEmptyStatement, DebuggerStatement as EsDebuggerStatement, WithStatement as EsWithStatement, ReturnStatement as EsReturnStatement, LabeledStatement as EsLabeledStatement, BreakStatement as EsBreakStatement, ContinueStatement as EsContinueStatement, IfStatement as EsIfStatement, SwitchStatement as EsSwitchStatement, ThrowStatement as EsThrowStatement, TryStatement as EsTryStatement, WhileStatement as EsWhileStatement, DoWhileStatement as EsDoWhileStatement, ForStatement as EsForStatement, ForInStatement as EsForInStatement, ForOfStatement as EsForOfStatement, ClassDeclaration as EsClassDeclaration, FunctionDeclaration as EsFunctionDeclaration, VariableDeclaration as EsVariableDeclaration, ModuleDeclaration as EsModuleDeclaration, ImportDeclaration as EsImportDeclaration, ExportDefaultDeclaration as EsExportDefaultDeclaration, ExportNamedDeclaration as EsExportNamedDeclaration, ExportAllDeclaration as EsExportAllDeclaration, ThisExpression as EsThisExpression, ArrayExpression as EsArrayExpression, ObjectExpression as EsObjectExpression, FunctionExpression as EsFunctionExpression, ArrowFunctionExpression as EsArrowFunctionExpression, YieldExpression as EsYieldExpression, UnaryExpression as EsUnaryExpression, UpdateExpression as EsUpdateExpression, BinaryExpression as EsBinaryExpression, AssignmentExpression as EsAssignmentExpression, LogicalExpression as EsLogicalExpression, MemberExpression as EsMemberExpression, ConditionalExpression as EsConditionalExpression, CallExpression as EsCallExpression, NewExpression as EsNewExpression, SequenceExpression as EsSequenceExpression, TaggedTemplateExpression as EsTaggedTemplateExpression, ClassExpression as EsClassExpression, AwaitExpression as EsAwaitExpression, ImportExpression as EsImportExpression, ChainExpression as EsChainExpression, SimpleLiteral as EsSimpleLiteral, RegExpLiteral as EsRegExpLiteral, BigIntLiteral as EsBigIntLiteral, TemplateLiteral as EsTemplateLiteral, PrivateIdentifier as EsPrivateIdentifier, Property as EsProperty, MetaProperty as EsMetaProperty, PropertyDefinition as EsPropertyDefinition, AssignmentProperty as EsAssignmentProperty, Super as EsSuper, TemplateElement as EsTemplateElement, SpreadElement as EsSpreadElement, ObjectPattern as EsObjectPattern, ArrayPattern as EsArrayPattern, RestElement as EsRestElement, AssignmentPattern as EsAssignmentPattern, Class as EsClass, ClassBody as EsClassBody, StaticBlock as EsStaticBlock, MethodDefinition as EsMethodDefinition, ModuleSpecifier as EsModuleSpecifier, ImportSpecifier as EsImportSpecifier, ImportNamespaceSpecifier as EsImportNamespaceSpecifier, ImportDefaultSpecifier as EsImportDefaultSpecifier, ExportSpecifier as EsExportSpecifier } from 'estree'; +import type { JSXAttribute as EsJsxAttribute, JSXClosingElement as EsJsxClosingElement, JSXClosingFragment as EsJsxClosingFragment, JSXElement as EsJsxElement, JSXEmptyExpression as EsJsxEmptyExpression, JSXExpressionContainer as EsJsxExpressionContainer, JSXFragment as EsJsxFragment, JSXIdentifier as EsJsxIdentifier, JSXMemberExpression as EsJsxMemberExpression, JSXNamespacedName as EsJsxNamespacedName, JSXOpeningElement as EsJsxOpeningElement, JSXOpeningFragment as EsJsxOpeningFragment, JSXSpreadAttribute as EsJsxSpreadAttribute, JSXSpreadChild as EsJsxSpreadChild, JSXText as EsJsxText } from 'estree-jsx'; +export declare function esNodeIs(node: EsNode, type: T): node is EsNodeMap[T]; +export declare function resolveEstreePropertyPath(node: EsProperty, parents: EsNode[], attributeName: string): string | undefined; +/** + * ============================================================ + */ +export type { EsBaseNode, EsIdentifier, EsProgram, EsSwitchCase, EsCatchClause, EsVariableDeclarator, EsExpressionStatement, EsBlockStatement, EsEmptyStatement, EsDebuggerStatement, EsWithStatement, EsReturnStatement, EsLabeledStatement, EsBreakStatement, EsContinueStatement, EsIfStatement, EsSwitchStatement, EsThrowStatement, EsTryStatement, EsWhileStatement, EsDoWhileStatement, EsForStatement, EsForInStatement, EsForOfStatement, EsClassDeclaration, EsFunctionDeclaration, EsVariableDeclaration, EsModuleDeclaration, EsImportDeclaration, EsExportDefaultDeclaration, EsExportNamedDeclaration, EsExportAllDeclaration, EsThisExpression, EsArrayExpression, EsObjectExpression, EsFunctionExpression, EsArrowFunctionExpression, EsYieldExpression, EsUnaryExpression, EsUpdateExpression, EsBinaryExpression, EsAssignmentExpression, EsLogicalExpression, EsMemberExpression, EsConditionalExpression, EsCallExpression, EsNewExpression, EsSequenceExpression, EsTaggedTemplateExpression, EsClassExpression, EsAwaitExpression, EsImportExpression, EsChainExpression, EsSimpleLiteral, EsRegExpLiteral, EsBigIntLiteral, EsTemplateLiteral, EsPrivateIdentifier, EsProperty, EsMetaProperty, EsPropertyDefinition, EsAssignmentProperty, EsSuper, EsTemplateElement, EsSpreadElement, EsObjectPattern, EsArrayPattern, EsRestElement, EsAssignmentPattern, EsClass, EsClassBody, EsStaticBlock, EsMethodDefinition, EsModuleSpecifier, EsImportSpecifier, EsImportNamespaceSpecifier, EsImportDefaultSpecifier, EsExportSpecifier }; +export type { EsJsxAttribute, EsJsxClosingElement, EsJsxClosingFragment, EsJsxElement, EsJsxEmptyExpression, EsJsxExpressionContainer, EsJsxFragment, EsJsxIdentifier, EsJsxMemberExpression, EsJsxNamespacedName, EsJsxOpeningElement, EsJsxOpeningFragment, EsJsxSpreadAttribute, EsJsxSpreadChild, EsJsxText }; +export type EsNode = EsNodeMap[keyof EsNodeMap]; +export type EsNodeMap = EsExpressionMap & EsLiteralMap & EsFunctionMap & EsPatternMap & EsStatementMap & EsJsxMap & { + CatchClause: EsCatchClause; + Class: EsClass; + ClassBody: EsClassBody; + MethodDefinition: EsMethodDefinition; + ModuleDeclaration: EsModuleDeclaration; + ModuleSpecifier: EsModuleSpecifier; + PrivateIdentifier: EsPrivateIdentifier; + Program: EsProgram; + Property: EsProperty; + PropertyDefinition: EsPropertyDefinition; + SpreadElement: EsSpreadElement; + Super: EsSuper; + SwitchCase: EsSwitchCase; + TemplateElement: EsTemplateElement; + VariableDeclarator: EsVariableDeclarator; +}; +export type EsExpressionMap = EsLiteralMap & { + ArrayExpression: EsArrayExpression; + ArrowFunctionExpression: EsArrowFunctionExpression; + AssignmentExpression: EsAssignmentExpression; + AwaitExpression: EsAwaitExpression; + BinaryExpression: EsBinaryExpression; + CallExpression: EsCallExpression; + ChainExpression: EsChainExpression; + ClassExpression: EsClassExpression; + ConditionalExpression: EsConditionalExpression; + FunctionExpression: EsFunctionExpression; + Identifier: EsIdentifier; + ImportExpression: EsImportExpression; + LogicalExpression: EsLogicalExpression; + MemberExpression: EsMemberExpression; + MetaProperty: EsMetaProperty; + NewExpression: EsNewExpression; + ObjectExpression: EsObjectExpression; + SequenceExpression: EsSequenceExpression; + TaggedTemplateExpression: EsTaggedTemplateExpression; + TemplateLiteral: EsTemplateLiteral; + ThisExpression: EsThisExpression; + UnaryExpression: EsUnaryExpression; + UpdateExpression: EsUpdateExpression; + YieldExpression: EsYieldExpression; +}; +export interface EsLiteralMap { + Literal: EsSimpleLiteral | EsRegExpLiteral | EsBigIntLiteral; + SimpleLiteral: EsSimpleLiteral; + RegExpLiteral: EsRegExpLiteral; + BigIntLiteral: EsBigIntLiteral; +} +export interface EsFunctionMap { + FunctionDeclaration: EsFunctionDeclaration; + FunctionExpression: EsFunctionExpression; + ArrowFunctionExpression: EsArrowFunctionExpression; +} +export interface EsPatternMap { + Identifier: EsIdentifier; + ObjectPattern: EsObjectPattern; + ArrayPattern: EsArrayPattern; + RestElement: EsRestElement; + AssignmentPattern: EsAssignmentPattern; + MemberExpression: EsMemberExpression; +} +export type EsStatementMap = EsDeclarationMap & { + ExpressionStatement: EsExpressionStatement; + BlockStatement: EsBlockStatement; + StaticBlock: EsStaticBlock; + EmptyStatement: EsEmptyStatement; + DebuggerStatement: EsDebuggerStatement; + WithStatement: EsWithStatement; + ReturnStatement: EsReturnStatement; + LabeledStatement: EsLabeledStatement; + BreakStatement: EsBreakStatement; + ContinueStatement: EsContinueStatement; + IfStatement: EsIfStatement; + SwitchStatement: EsSwitchStatement; + ThrowStatement: EsThrowStatement; + TryStatement: EsTryStatement; + WhileStatement: EsWhileStatement; + DoWhileStatement: EsDoWhileStatement; + ForStatement: EsForStatement; + ForInStatement: EsForInStatement; + ForOfStatement: EsForOfStatement; +}; +export interface EsDeclarationMap { + FunctionDeclaration: EsFunctionDeclaration; + VariableDeclaration: EsVariableDeclaration; + ClassDeclaration: EsClassDeclaration; +} +export interface EsJsxMap { + JSXAttribute: EsJsxAttribute; + JSXClosingElement: EsJsxClosingElement; + JSXClosingFragment: EsJsxClosingFragment; + JSXElement: EsJsxElement; + JSXEmptyExpression: EsJsxEmptyExpression; + JSXExpressionContainer: EsJsxExpressionContainer; + JSXFragment: EsJsxFragment; + JSXIdentifier: EsJsxIdentifier; + JSXMemberExpression: EsJsxMemberExpression; + JSXNamespacedName: EsJsxNamespacedName; + JSXOpeningElement: EsJsxOpeningElement; + JSXOpeningFragment: EsJsxOpeningFragment; + JSXSpreadAttribute: EsJsxSpreadAttribute; + JSXSpreadChild: EsJsxSpreadChild; + JSXText: EsJsxText; +} diff --git a/dist/ast/estree.js b/dist/ast/estree.js new file mode 100644 index 0000000..a6bf1b5 --- /dev/null +++ b/dist/ast/estree.js @@ -0,0 +1,28 @@ +function esNodeIs(node, type) { + return node ? node.type === type : false; +} +function resolveEstreePropertyPath(node, parents, attributeName) { + if (!esNodeIs(parents[2], "ArrayExpression") && !esNodeIs(parents[2], "ObjectExpression")) + return; + if (!esNodeIs(node.key, "Identifier")) + return; + const names = [node.key.name]; + for (let i = parents.length - 1; i > 1; i--) { + const parent = parents[i]; + if (esNodeIs(parent, "ArrayExpression") || esNodeIs(parent, "ObjectExpression")) + continue; + if (esNodeIs(parent, "Property")) { + if (!esNodeIs(parent.key, "Identifier")) + return; + names.push(parent.key.name); + continue; + } + return; + } + names.push(attributeName); + return names.reverse().join("."); +} +export { + esNodeIs, + resolveEstreePropertyPath +}; diff --git a/dist/ast/eswalk.d.ts b/dist/ast/eswalk.d.ts new file mode 100644 index 0000000..470b3da --- /dev/null +++ b/dist/ast/eswalk.d.ts @@ -0,0 +1,18 @@ +import { EsNode, EsNodeMap, EsProgram } from './estree.js'; +export declare const DEFAULT_ESWALKERS: EsWalkers; +export declare function eswalk(ast: EsProgram, visitors: EsVisitors, walkers?: EsWalkers): void; +export interface EsProcessor { + (node: EsNode | null, parents: EsNode[]): void; +} +export interface EsVisitor { + (node: EsNodeMap[NodeType], parents: EsNode[]): boolean | void; +} +export type EsVisitors = { + [NodeType in keyof EsNodeMap]?: EsVisitor; +}; +export interface EsWalker { + (node: EsNodeMap[NodeType], parents: EsNode[], process: EsProcessor): void; +} +export type EsWalkers = { + [NodeType in keyof Partial]: EsWalker; +}; diff --git a/dist/ast/eswalk.js b/dist/ast/eswalk.js new file mode 100644 index 0000000..908b6b3 --- /dev/null +++ b/dist/ast/eswalk.js @@ -0,0 +1,77 @@ +import { isRegExp } from "node:util/types"; +import { esNodeIs } from "./estree.js"; +const DEFAULT_ESWALKERS = { + Program(node, parents, process) { + parents.push(node); + for (const statement of node.body) { + process(statement, parents); + } + parents.pop(); + }, + ExpressionStatement(node, parents, process) { + parents.push(node); + process(node.expression, parents); + parents.pop(); + }, + ArrayExpression(node, parents, process) { + parents.push(node); + for (const element of node.elements) { + process(element, parents); + } + parents.pop(); + }, + ObjectExpression(node, parents, process) { + parents.push(node); + for (const property of node.properties) { + process(property, parents); + } + parents.pop(); + }, + Property(node, parents, process) { + parents.push(node); + process(node.key, parents); + process(node.value, parents); + parents.pop(); + }, + JSXElement(node, parents, process) { + parents.push(node); + for (const child of node.children) { + process(child, parents); + } + for (const attribute of node.openingElement.attributes) { + process(attribute, parents); + } + parents.pop(); + }, + JSXAttribute(node, parents, process) { + parents.push(node); + if (node.value) { + process(node.value, parents); + } + parents.pop(); + } +}; +function eswalk(ast, visitors, walkers = DEFAULT_ESWALKERS) { + const process = (node, parents) => { + if (!node) + return; + let type = node.type; + if (esNodeIs(node, "Literal")) { + type = typeof node.value === "bigint" ? "BigIntLiteral" : isRegExp(node.value) ? "RegExpLiteral" : "SimpleLiteral"; + } + const visit = visitors[type]; + const walk = walkers[type]; + let keepWalking = true; + if (visit !== void 0) { + const signal = visit(node, parents); + keepWalking = signal === false ? false : true; + } + if (keepWalking && walk) + walk(node, parents, process); + }; + process(ast, []); +} +export { + DEFAULT_ESWALKERS, + eswalk +}; diff --git a/dist/ast/mdast.d.ts b/dist/ast/mdast.d.ts new file mode 100644 index 0000000..5c8f32c --- /dev/null +++ b/dist/ast/mdast.d.ts @@ -0,0 +1,24 @@ +import type { Root as MdRoot, Blockquote as MdBlockquote, Break as MdBreak, Code as MdCode, Definition as MdDefinition, Delete as MdDelete, Emphasis as MdEmphasis, Footnote as MdFootnote, FootnoteDefinition as MdFootnoteDefinition, FootnoteReference as MdFootnoteReference, HTML as MdHTML, Heading as MdHeading, Image as MdImage, ImageReference as MdImageReference, InlineCode as MdInlineCode, Link as MdLink, LinkReference as MdLinkReference, List as MdList, ListItem as MdListItem, Paragraph as MdParagraph, Strong as MdStrong, Table as MdTable, TableCell as MdTableCell, TableRow as MdTableRow, Text as MdText, ThematicBreak as MdThematicBreak, YAML as MdYaml } from 'mdast'; +import type { MdxFlowExpression, MdxJsxAttribute, MdxJsxAttributeValueExpression, MdxJsxExpressionAttribute, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, MdxjsEsm } from 'mdast-util-mdx'; +import type { UnNode } from './unist.js'; +declare module 'mdast' { + interface PhrasingContentMap extends StaticPhrasingContentMap { + mdxJsxFlowElement: MdxJsxFlowElement; + mdxJsxTextElement: MdxJsxTextElement; + mdxFlowExpression: MdxFlowExpression; + mdxTextExpression: MdxTextExpression; + } +} +export declare function mdNodeIs(node: UnNode | undefined, type: T): node is T extends MdRoot['type'] ? MdRoot : T extends MdBlockquote['type'] ? MdBlockquote : T extends MdBreak['type'] ? MdBreak : T extends MdCode['type'] ? MdCode : T extends MdDefinition['type'] ? MdDefinition : T extends MdDelete['type'] ? MdDelete : T extends MdEmphasis['type'] ? MdEmphasis : T extends MdFootnote['type'] ? MdFootnote : T extends MdFootnoteDefinition['type'] ? MdFootnoteDefinition : T extends MdFootnoteReference['type'] ? MdFootnoteReference : T extends MdHTML['type'] ? MdHTML : T extends MdHeading['type'] ? MdHeading : T extends MdImage['type'] ? MdImage : T extends MdImageReference['type'] ? MdImageReference : T extends MdInlineCode['type'] ? MdInlineCode : T extends MdLink['type'] ? MdLink : T extends MdLinkReference['type'] ? MdLinkReference : T extends MdList['type'] ? MdList : T extends MdListItem['type'] ? MdListItem : T extends MdParagraph['type'] ? MdParagraph : T extends MdStrong['type'] ? MdStrong : T extends MdTable['type'] ? MdTable : T extends MdTableCell['type'] ? MdTableCell : T extends MdTableRow['type'] ? MdTableRow : T extends MdText['type'] ? MdText : T extends MdThematicBreak['type'] ? MdThematicBreak : T extends MdYaml ? MdYaml : T extends MdxFlowExpression['type'] ? MdxFlowExpression : T extends MdxJsxAttribute['type'] ? MdxJsxAttribute : T extends MdxJsxAttributeValueExpression['type'] ? MdxJsxAttributeValueExpression : T extends MdxJsxExpressionAttribute['type'] ? MdxJsxExpressionAttribute : T extends MdxJsxFlowElement['type'] ? MdxJsxFlowElement : T extends MdxJsxTextElement['type'] ? MdxJsxTextElement : T extends MdxTextExpression['type'] ? MdxTextExpression : MdxjsEsm; +export declare function mdNodeIsJsxElement(node: UnNode): node is MdxJsxFlowElement | MdxJsxTextElement; +/** + * Get MDX flavored `mdast`. + */ +export declare function getMdast(markdown: string): MdRoot; +export declare function getMarkdown(mdast: MdRoot): string; +/** + * ============================================================ + */ +export type MdNodeType = MdRoot['type'] | MdBlockquote['type'] | MdBreak['type'] | MdCode['type'] | MdDefinition['type'] | MdDelete['type'] | MdEmphasis['type'] | MdFootnote['type'] | MdFootnoteDefinition['type'] | MdFootnoteReference['type'] | MdHTML['type'] | MdHeading['type'] | MdImage['type'] | MdImageReference['type'] | MdInlineCode['type'] | MdLink['type'] | MdLinkReference['type'] | MdList['type'] | MdListItem['type'] | MdParagraph['type'] | MdStrong['type'] | MdTable['type'] | MdTableCell['type'] | MdTableRow['type'] | MdText['type'] | MdThematicBreak['type'] | MdYaml['type'] | MdxFlowExpression['type'] | MdxJsxAttribute['type'] | MdxJsxAttributeValueExpression['type'] | MdxJsxExpressionAttribute['type'] | MdxJsxFlowElement['type'] | MdxJsxTextElement['type'] | MdxTextExpression['type'] | MdxjsEsm['type']; +export type { MdRoot, MdBlockquote, MdBreak, MdCode, MdDefinition, MdDelete, MdEmphasis, MdFootnote, MdFootnoteDefinition, MdFootnoteReference, MdHTML, MdHeading, MdImage, MdImageReference, MdInlineCode, MdLink, MdLinkReference, MdList, MdListItem, MdParagraph, MdStrong, MdTable, MdTableCell, MdTableRow, MdText, MdThematicBreak, MdYaml }; +export type { MdxFlowExpression, MdxJsxAttribute, MdxJsxAttributeValueExpression, MdxJsxExpressionAttribute, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, MdxjsEsm }; diff --git a/dist/ast/mdast.js b/dist/ast/mdast.js new file mode 100644 index 0000000..b8f15e3 --- /dev/null +++ b/dist/ast/mdast.js @@ -0,0 +1,39 @@ +import { fromMarkdown } from "mdast-util-from-markdown"; +import { frontmatterFromMarkdown, frontmatterToMarkdown } from "mdast-util-frontmatter"; +import { htmlCommentFromMarkdown, htmlCommentToMarkdown } from "../vendor/mdast-util-html-comment.js"; +import { mdxFromMarkdown, mdxToMarkdown } from "mdast-util-mdx"; +import { toMarkdown } from "mdast-util-to-markdown"; +import { frontmatter } from "micromark-extension-frontmatter"; +import { htmlComment } from "../vendor/micromark-extension-html-comment.js"; +import { mdxjs } from "micromark-extension-mdxjs"; +function mdNodeIs(node, type) { + return node ? node.type === type : false; +} +function mdNodeIsJsxElement(node) { + return mdNodeIs(node, "mdxJsxFlowElement") || mdNodeIs(node, "mdxJsxTextElement"); +} +function getMdast(markdown) { + return fromMarkdown(markdown, { + extensions: [frontmatter("yaml"), mdxjs(), htmlComment()], + mdastExtensions: [frontmatterFromMarkdown("yaml"), mdxFromMarkdown(), htmlCommentFromMarkdown()] + }); +} +function getMarkdown(mdast) { + return toMarkdown(mdast, { + extensions: [frontmatterToMarkdown("yaml"), mdxToMarkdown(), htmlCommentToMarkdown()], + join: [ + (__, _, parent) => { + if (mdNodeIsJsxElement(parent)) { + return 0; + } + return 1; + } + ] + }); +} +export { + getMarkdown, + getMdast, + mdNodeIs, + mdNodeIsJsxElement +}; diff --git a/dist/ast/unist.d.ts b/dist/ast/unist.d.ts new file mode 100644 index 0000000..568370b --- /dev/null +++ b/dist/ast/unist.d.ts @@ -0,0 +1,13 @@ +import type { Position as UnPosition } from 'unist'; +export declare function unNodeIsParent(node: UnNode): node is UnParent; +/** + * ============================================================ + */ +export interface UnNode { + type: string; + position?: UnPosition; + data?: unknown; +} +export interface UnParent extends UnNode { + children: (UnNode | UnParent)[]; +} diff --git a/dist/ast/unist.js b/dist/ast/unist.js new file mode 100644 index 0000000..a7d081a --- /dev/null +++ b/dist/ast/unist.js @@ -0,0 +1,6 @@ +function unNodeIsParent(node) { + return "children" in node; +} +export { + unNodeIsParent +}; diff --git a/dist/ast/unwalk.d.ts b/dist/ast/unwalk.d.ts new file mode 100644 index 0000000..4b22db9 --- /dev/null +++ b/dist/ast/unwalk.d.ts @@ -0,0 +1,5 @@ +import { type UnNode, type UnParent } from './unist.js'; +export declare function unwalk(node: UnNode, visit: UnVisitor, filter?: (node: UnNode, parent: UnParent | undefined) => boolean): void; +export interface UnVisitor { + (node: UnNode | UnParent, parent: UnParent | undefined, index: number | undefined): boolean | void; +} diff --git a/dist/ast/unwalk.js b/dist/ast/unwalk.js new file mode 100644 index 0000000..230a39b --- /dev/null +++ b/dist/ast/unwalk.js @@ -0,0 +1,27 @@ +import { unNodeIsParent } from "./unist.js"; +const NEXT = true; +const STOP = false; +function unwalk(node, visit, filter) { + let next = true; + function step(node2, parent, index) { + if (filter && !filter(node2, parent)) + return; + if (unNodeIsParent(node2)) { + for (let i = 0; i < node2.children.length; i++) { + if (!next) + break; + const child = node2.children[i]; + step(child, node2, i); + } + node2.children = node2.children.filter((child) => child); + } + if (!next) + return; + const signal = visit(node2, parent, index); + next = signal === void 0 || NEXT ? NEXT : STOP; + } + step(node, void 0, void 0); +} +export { + unwalk +}; diff --git a/dist/config.d.ts b/dist/config.d.ts new file mode 100644 index 0000000..47779ee --- /dev/null +++ b/dist/config.d.ts @@ -0,0 +1,215 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +import type { MdNodeType } from './ast/mdast.js'; +export interface ConfigBase { + /** + * Source's language code. Based on DeepL supported languages. + */ + sourceLanguage: SourceLanguageCode; + /** + * Output's languages code. Based on DeepL supported languages. + */ + outputLanguages: TargetLanguageCode[]; + /** + * Sources and ouputs directories pairs. $langcode$ variable + * is provided to dynamically define directory. + * + * e.g. [ ["docs", "i18n/$langcode$/docs"], ["blog", "i18n/$langcode$/blog"] ] + */ + directories: [string, string][]; +} +export interface Config extends ConfigBase { + /** + * Override current working directory, defaults to `process.cwd()`. + */ + cwd: string; + /** + * By default, all .md, .mdx, .json, and .yaml|.yml files inside + * source directories will be included. + * + * Define glob patterns to filter what files to include or exclude. + * But, the end result is still restricted by file types (.md, .mdx, .json). + */ + files: { + include?: string[]; + exclude: string[]; + }; + /** + * Frontmatter fields. + */ + frontmatterFields: { + include: string[]; + exclude: string[]; + }; + /** + * Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`. + */ + markdownNodes: { + default: boolean; + include: MdNodeType[]; + exclude: MdNodeType[]; + }; + /** + * HTML elements to include and exlcude, down to the level of attributes + * and children. Include all HTML elements text content + * and some global attributes such as title and placeholder. + */ + htmlElements: { + include: Partial<{ + [Tag in HtmlTag]: { + children: boolean; + attributes: string[]; + }; + }>; + exclude: HtmlTag[]; + }; + /** + * JSX components to include and exclude, down to the level of attributes + * and children. Include all JSX components text children + * and exclude all attributes by default. + * + * Support array, object, and jsx attribute value. For object and array value, + * you can specify the access path starting with the attribute name + * e.g. `items.description` to translate `items={[{description: "..."}]}. + */ + jsxComponents: { + default: boolean; + include: { + [Name: string]: { + children: boolean; + attributes: string[]; + }; + }; + exclude: string[]; + }; + /** + * JSON or YAML file properties to include and exclude. + * Exclude all properties by default. + */ + jsonOrYamlProperties: { + include: (string | number | symbol)[]; + exclude: (string | number | symbol)[]; + }; +} +export interface UserConfig extends ConfigBase { + /** + * Override current working directory, defaults to `process.cwd()`. + */ + cwd?: string; + /** + * By default, all .md, .mdx, .json, and .yaml|.yml files inside + * source directories will be included. + * + * Define glob patterns to filter what files to include or exclude. + * But, the end result is still restricted by file types (.md, .mdx, .json). + */ + files?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Frontmatter fields. + */ + frontmatterFields?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`. + */ + markdownNodes?: { + default?: boolean; + include?: MdNodeType[]; + exclude?: MdNodeType[]; + }; + /** + * HTML elements to include and exlcude, down to the level of attributes + * and children. Include all HTML elements text content + * and some global attributes such as title and placeholder. + */ + htmlElements?: { + default?: boolean; + include?: Partial<{ + [Tag in HtmlTag]: { + children: boolean; + attributes: string[]; + }; + }>; + exclude?: HtmlTag[]; + }; + /** + * JSX components to include and exclude, down to the level of attributes + * and children. Include all JSX components text children + * and exclude all attributes by default. + * + * Support array, object, and jsx attribute value. For object and array value, + * you can specify the access path starting with the attribute name + * e.g. `items.description` to translate `items={[{description: "..."}]}. + */ + jsxComponents?: { + default?: boolean; + include?: { + [Name: string]: { + children: boolean; + attributes: string[]; + }; + }; + exclude?: string[]; + }; + /** + * JSON or YAML file properties to include and exclude. + * Exclude all properties by default. + */ + jsonOrYamlProperties?: { + include?: string[]; + exclude?: string[]; + }; +} +export type HtmlElementsConfig = { + [Tag in HtmlTag]: { + children: boolean; + attributes: string[]; + }; +}; +export declare const HTML_ELEMENTS_CONFIG: HtmlElementsConfig; +export declare const HTML_TAGS: HtmlTag[]; +export declare function isHtmlTag(name: string): name is HtmlTag; +export declare function resolveConfig({ sourceLanguage, outputLanguages, directories, cwd, files, markdownNodes, frontmatterFields, htmlElements, jsxComponents, jsonOrYamlProperties }: UserConfig): Config; +export declare function isFrontmatterFieldIncluded({ field, config }: { + field: string; + config: Config; +}): boolean; +export declare function isMarkdownNodeIncluded({ type, config }: { + type: MdNodeType; + config: Config; +}): boolean; +export declare function isHtmlElementIncluded({ tag, config }: { + tag: HtmlTag; + config: Config; +}): boolean; +export declare function isHtmlElementAttributeIncluded({ tag, attribute, config }: { + tag: HtmlTag; + attribute: string; + config: Config; +}): boolean; +export declare function isHtmlElementChildrenIncluded({ tag, config }: { + tag: HtmlTag; + config: Config; +}): boolean; +export declare function isJsxComponentIncluded({ name, config }: { + name: string; + config: Config; +}): boolean; +export declare function isJsxComponentAttributeIncluded({ name, attribute, config }: { + name: string; + attribute: string; + config: Config; +}): boolean; +export declare function isJsxComponentChildrenIncluded({ name, config }: { + name: string; + config: Config; +}): boolean; +export declare function isJsonOrYamlPropertyIncluded({ property, config }: { + config: Config; + property: string | number | symbol; +}): boolean; +export type HtmlTag = 'a' | 'abbr' | 'address' | 'article' | 'aside' | 'audio' | 'b' | 'bdi' | 'bdo' | 'blockquote' | 'body' | 'button' | 'canvas' | 'caption' | 'cite' | 'col' | 'colgroup' | 'data' | 'datalist' | 'dd' | 'del' | 'details' | 'dfn' | 'dialog' | 'div' | 'dl' | 'dt' | 'em' | 'fieldset' | 'figcaption' | 'figure' | 'footer' | 'form' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'header' | 'html' | 'i' | 'input' | 'ins' | 'label' | 'legend' | 'li' | 'main' | 'mark' | 'meter' | 'nav' | 'ol' | 'optgroup' | 'output' | 'p' | 'progress' | 'q' | 'rp' | 's' | 'samp' | 'section' | 'select' | 'small' | 'span' | 'strong' | 'sub' | 'summary' | 'sup' | 'table' | 'tbody' | 'td' | 'template' | 'text-area' | 'tfoot' | 'th' | 'thead' | 'time' | 'title' | 'tr' | 'track' | 'u' | 'ul' | 'area' | 'base' | 'br' | 'code' | 'embed' | 'head' | 'hr' | 'iframe' | 'img' | 'kbd' | 'link' | 'meta' | 'noscript' | 'object' | 'param' | 'picture' | 'pre' | 'rt' | 'ruby' | 'script' | 'source' | 'style' | 'svg' | 'var' | 'video' | 'qbr'; diff --git a/dist/config.js b/dist/config.js new file mode 100644 index 0000000..799e02b --- /dev/null +++ b/dist/config.js @@ -0,0 +1,244 @@ +import { isBoolean } from "./utils.js"; +const HTML_ELEMENTS_CONFIG = getHtmlElementsConfig(); +function getHtmlElementsConfig() { + const includeChildren = [ + "a", + "abbr", + "address", + "article", + "aside", + "audio", + "b", + "bdi", + "bdo", + "blockquote", + "body", + "button", + "canvas", + "caption", + "cite", + "col", + "colgroup", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "header", + "html", + "i", + "input", + "ins", + "label", + "legend", + "li", + "main", + "mark", + "meter", + "nav", + "ol", + "optgroup", + "output", + "p", + "progress", + "q", + "rp", + "s", + "samp", + "section", + "select", + "small", + "span", + "strong", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "template", + "text-area", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "u", + "ul" + ]; + const excludeChildren = [ + "area", + "base", + "br", + "code", + "embed", + "head", + "hr", + "iframe", + "img", + "kbd", + "link", + "meta", + "noscript", + "object", + "param", + "picture", + "pre", + "rt", + "ruby", + "script", + "source", + "style", + "svg", + "var", + "video", + "qbr" + ]; + const config = {}; + for (const tag of includeChildren) { + config[tag] = { + children: true, + attributes: ["title"] + }; + } + for (const tag of excludeChildren) { + config[tag] = { + children: false, + attributes: ["title"] + }; + } + return config; +} +const HTML_TAGS = Object.keys(HTML_ELEMENTS_CONFIG); +function isHtmlTag(name) { + return HTML_TAGS.includes(name); +} +function resolveConfig({ + sourceLanguage, + outputLanguages, + directories, + cwd, + files, + markdownNodes, + frontmatterFields, + htmlElements, + jsxComponents, + jsonOrYamlProperties +}) { + return { + sourceLanguage, + outputLanguages, + directories, + cwd: cwd ?? "", + files: files ? { + include: files.include, + exclude: files.exclude ?? [] + } : { exclude: [] }, + markdownNodes: markdownNodes ? { + default: isBoolean(markdownNodes.default) ? markdownNodes.default : true, + include: markdownNodes.include ?? [], + exclude: markdownNodes.exclude ?? ["code"] + } : { default: true, include: [], exclude: ["code"] }, + frontmatterFields: frontmatterFields ? { + include: frontmatterFields.include ?? [], + exclude: frontmatterFields.exclude ?? [] + } : { include: [], exclude: [] }, + htmlElements: htmlElements ? { + include: htmlElements.include ? isBoolean(htmlElements.default) && htmlElements.default || htmlElements.default === void 0 ? { ...HTML_ELEMENTS_CONFIG, ...htmlElements.include } : htmlElements.include : isBoolean(htmlElements.default) && !htmlElements.default ? {} : HTML_ELEMENTS_CONFIG, + exclude: htmlElements.exclude ?? [] + } : { include: HTML_ELEMENTS_CONFIG, exclude: [] }, + jsxComponents: jsxComponents ? { + default: isBoolean(jsxComponents.default) ? jsxComponents.default : true, + include: jsxComponents.include ?? {}, + exclude: jsxComponents.exclude ?? [] + } : { default: true, include: {}, exclude: [] }, + jsonOrYamlProperties: jsonOrYamlProperties ? { include: jsonOrYamlProperties.include ?? [], exclude: jsonOrYamlProperties.exclude ?? [] } : { include: [], exclude: [] } + }; +} +function isFrontmatterFieldIncluded({ + field, + config +}) { + return !config.frontmatterFields.exclude.includes(field) && config.frontmatterFields.include.includes(field); +} +function isMarkdownNodeIncluded({ + type, + config +}) { + return !config.markdownNodes.exclude.includes(type) && (config.markdownNodes.default || config.markdownNodes.include.includes(type)); +} +function isHtmlElementIncluded({ tag, config }) { + return !config.htmlElements.exclude.includes(tag) && Object.keys(config.htmlElements.include).includes(tag); +} +function isHtmlElementAttributeIncluded({ + tag, + attribute, + config +}) { + return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag].attributes.includes(attribute); +} +function isHtmlElementChildrenIncluded({ + tag, + config +}) { + return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag].children; +} +function isJsxComponentIncluded({ + name, + config +}) { + return !config.jsxComponents.exclude.includes(name) && (config.jsxComponents.default || Object.keys(config.jsxComponents.include).includes(name)); +} +function isJsxComponentAttributeIncluded({ + name, + attribute, + config +}) { + return !config.jsxComponents.exclude.includes(name) && Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.include[name].attributes.includes(attribute); +} +function isJsxComponentChildrenIncluded({ + name, + config +}) { + return !config.jsxComponents.exclude.includes(name) && (Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.include[name].children || !Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.default); +} +function isJsonOrYamlPropertyIncluded({ + property, + config +}) { + return !config.jsonOrYamlProperties.exclude.includes(property) && config.jsonOrYamlProperties.include.includes(property); +} +export { + HTML_ELEMENTS_CONFIG, + HTML_TAGS, + isFrontmatterFieldIncluded, + isHtmlElementAttributeIncluded, + isHtmlElementChildrenIncluded, + isHtmlElementIncluded, + isHtmlTag, + isJsonOrYamlPropertyIncluded, + isJsxComponentAttributeIncluded, + isJsxComponentChildrenIncluded, + isJsxComponentIncluded, + isMarkdownNodeIncluded, + resolveConfig +}; diff --git a/dist/extract.d.ts b/dist/extract.d.ts new file mode 100644 index 0000000..7dd3d23 --- /dev/null +++ b/dist/extract.d.ts @@ -0,0 +1,11 @@ +import type { UnNode } from './ast/unist.js'; +import { type Config } from './config.js'; +export declare function extractMdastStrings({ mdast, config }: { + mdast: UnNode; + config: Config; +}): string[]; +export declare function extractJsonOrYamlStrings({ source, type, config }: { + source: string; + type?: 'json' | 'yaml'; + config: Config; +}): string[]; diff --git a/dist/extract.js b/dist/extract.js new file mode 100644 index 0000000..a13fbc8 --- /dev/null +++ b/dist/extract.js @@ -0,0 +1,212 @@ +import { parse as parseYaml } from "yaml"; +import { + esNodeIs, + resolveEstreePropertyPath +} from "./ast/estree.js"; +import { eswalk } from "./ast/eswalk.js"; +import { mdNodeIs, mdNodeIsJsxElement } from "./ast/mdast.js"; +import { unwalk } from "./ast/unwalk.js"; +import { + isHtmlTag, + isFrontmatterFieldIncluded, + isHtmlElementIncluded, + isHtmlElementAttributeIncluded, + isJsonOrYamlPropertyIncluded, + isJsxComponentIncluded, + isJsxComponentAttributeIncluded, + isMarkdownNodeIncluded, + isHtmlElementChildrenIncluded, + isJsxComponentChildrenIncluded +} from "./config.js"; +import { isArray, isEmptyArray, isEmptyString, isObject, isString } from "./utils.js"; +function extractMdastStrings({ + mdast, + config +}) { + const strings = []; + unwalk( + mdast, + (node, __, _) => { + if (mdNodeIs(node, "text")) { + pushTidyString({ array: strings, string: node.value }); + return; + } + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, "mdxJsxAttribute")) + continue; + if (!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config })) + continue; + if (isString(attribute.value)) { + strings.push(attribute.value.trim()); + } else if (attribute.value?.data?.estree) { + const estree = attribute.value.data.estree; + eswalk(estree, { + SimpleLiteral(esnode, _2) { + if (isString(esnode.value)) + pushTidyString({ array: strings, string: esnode.value }); + } + }); + } + } + } else { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, "mdxJsxAttribute")) + continue; + const componentName = node.name; + const isAttributeIncluded = isJsxComponentAttributeIncluded({ + name: componentName, + attribute: attribute.name, + config + }); + if (isString(attribute.value)) { + if (!isAttributeIncluded) + continue; + strings.push(attribute.value.trim()); + } else if (attribute.value?.data?.estree) { + if (!config.jsxComponents.include[componentName] || !config.jsxComponents.include[componentName].attributes.some( + (attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + )) + continue; + const estree = attribute.value.data.estree; + eswalk(estree, { + SimpleLiteral(esnode, _2) { + if (isString(esnode.value)) + pushTidyString({ array: strings, string: esnode.value }); + if (esnode.value === "aye") + console.log("passed"); + }, + JSXElement(esnode, _2) { + const name = esnode.openingElement.name.name; + if (isHtmlTag(name)) { + if (!isHtmlElementIncluded({ tag: name, config }) || !isHtmlElementChildrenIncluded({ tag: name, config })) + return false; + } else if (!isJsxComponentIncluded({ name, config }) || !isJsxComponentChildrenIncluded({ name, config })) + return false; + }, + JSXAttribute(esnode, parents) { + const name = typeof esnode.name.name === "string" ? esnode.name.name : esnode.name.name.name; + const parentName = parents[parents.length - 1].openingElement.name.name; + if (isHtmlTag(parentName)) { + if (!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config })) + return false; + } else if (!config.jsxComponents.include[name] || !config.jsxComponents.include[name].attributes.some( + (attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + )) { + return false; + } + }, + JSXText(esnode, _2) { + pushTidyString({ array: strings, string: esnode.value }); + }, + Property(esnode, parents) { + if (!esNodeIs(esnode, "Identifier")) + return false; + const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name); + if (!propertyPath || !isJsxComponentAttributeIncluded({ + name: componentName, + attribute: propertyPath, + config + })) + return false; + } + }); + } + } + } + } + if (mdNodeIs(node, "yaml")) { + if (isEmptyArray(config.frontmatterFields.include)) + return; + if (isEmptyString(node.value)) + return; + const object = parseYaml(node.value); + for (const field in object) { + if (!isFrontmatterFieldIncluded({ field, config })) + continue; + const value = object[field]; + if (isString(value)) { + strings.push(value); + continue; + } + if (isArray(value)) { + for (const item of value) { + if (!isString(item)) + continue; + strings.push(item); + } + } + } + return; + } + }, + (node, parent) => { + if (!isMarkdownNodeIncluded({ type: node.type, config })) + return false; + if (parent && mdNodeIsJsxElement(parent) && parent.name) { + if (isHtmlTag(parent.name)) { + if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) + return false; + } else { + if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) + return false; + } + return true; + } + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + if (!isHtmlElementIncluded({ tag: node.name, config })) + return false; + } else { + if (!isJsxComponentIncluded({ name: node.name, config })) + return false; + } + return true; + } + return true; + } + ); + return strings; +} +function extractJsonOrYamlStrings({ + source, + type = "json", + config +}) { + const strings = []; + if (isEmptyArray(config.jsonOrYamlProperties.include)) + return strings; + const parsed = type === "json" ? JSON.parse(source) : parseYaml(source); + process(parsed); + function process(value, property) { + if (typeof value === "string") { + if (property && isJsonOrYamlPropertyIncluded({ property, config })) + strings.push(value); + return; + } + if (isArray(value)) { + for (const item of value) { + process(item); + } + return; + } + if (isObject(value)) { + for (const property2 in value) { + const item = value[property2]; + process(item, property2); + } + return; + } + } + return strings; +} +function pushTidyString({ array, string }) { + if (!/^\s*$/.test(string)) { + array.push(string.replace(/(^\n|\r|\t|\v)+\s*/, "").replace(/\s+$/, " ")); + } +} +export { + extractJsonOrYamlStrings, + extractMdastStrings +}; diff --git a/dist/format.d.ts b/dist/format.d.ts new file mode 100644 index 0000000..7344911 --- /dev/null +++ b/dist/format.d.ts @@ -0,0 +1 @@ +export declare function format(markdown: string): Promise; diff --git a/dist/format.js b/dist/format.js new file mode 100644 index 0000000..5731289 --- /dev/null +++ b/dist/format.js @@ -0,0 +1,33 @@ +import prettier from "prettier"; +import { getMarkdown, getMdast, mdNodeIs } from "./ast/mdast.js"; +import { unwalk } from "./ast/unwalk.js"; +async function format(markdown) { + const mdast = getMdast( + prettier.format(markdown, { + parser: "mdx", + printWidth: Infinity, + proseWrap: "never", + useTabs: true + }) + ); + unwalk( + mdast, + (node, parent, index) => { + if (mdNodeIs(node, "mdxFlowExpression") && expressionIsEmpty(node.value)) { + parent.children[index] = void 0; + } + }, + (node, parent) => { + delete node.position; + return mdNodeIs(parent, "root"); + } + ); + return getMarkdown(mdast); +} +function expressionIsEmpty(text) { + const regex = /^('|")\s*('|")$/; + return regex.test(text); +} +export { + format +}; diff --git a/dist/index.d.ts b/dist/index.d.ts new file mode 100644 index 0000000..1dc1d03 --- /dev/null +++ b/dist/index.d.ts @@ -0,0 +1,21 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +/** + * Translate markdown/MDX content from one language to another using DeepL. + * + * Requires `DEEPL_AUTH_KEY` environment variable to be set. + * + * @param content - Markdown or MDX string to translate + * @param sourceLang - Source language code (e.g. 'en', 'de', 'fr') + * @param targetLang - Target language code (e.g. 'de', 'en-US', 'fr') + * @returns Translated markdown string + * + * @example + * ```ts + * import { translate } from 'deepmark'; + * + * const result = await translate('# Hello World', 'en', 'de'); + * console.log(result); // '# Hallo Welt' + * ``` + */ +export declare function translate(content: string, sourceLang: SourceLanguageCode, targetLang: TargetLanguageCode): Promise; +export type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; diff --git a/dist/index.js b/dist/index.js new file mode 100644 index 0000000..98251bc --- /dev/null +++ b/dist/index.js @@ -0,0 +1,24 @@ +import { getMarkdown, getMdast } from "./ast/mdast.js"; +import { resolveConfig } from "./config.js"; +import { extractMdastStrings } from "./extract.js"; +import { format } from "./format.js"; +import { replaceMdastStrings } from "./replace.js"; +import { translateStrings } from "./translate.js"; +async function translate(content, sourceLang, targetLang) { + const config = resolveConfig({ + sourceLanguage: sourceLang, + outputLanguages: [targetLang], + directories: [["", ""]] + }); + const formatted = await format(content); + const mdast = getMdast(formatted); + const strings = extractMdastStrings({ mdast, config }); + if (strings.length === 0) + return content; + const translated = await translateStrings(strings, sourceLang, targetLang); + const result = replaceMdastStrings({ mdast, strings: translated, config }); + return getMarkdown(result); +} +export { + translate +}; diff --git a/dist/lint.d.ts b/dist/lint.d.ts new file mode 100644 index 0000000..cb0ff5c --- /dev/null +++ b/dist/lint.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/dist/lint.js b/dist/lint.js new file mode 100644 index 0000000..e69de29 diff --git a/dist/replace.d.ts b/dist/replace.d.ts new file mode 100644 index 0000000..f664e40 --- /dev/null +++ b/dist/replace.d.ts @@ -0,0 +1,13 @@ +import type { MdRoot } from './ast/mdast.js'; +import { type Config } from './config.js'; +export declare function replaceMdastStrings({ mdast, config, strings }: { + mdast: MdRoot; + strings: string[]; + config: Config; +}): MdRoot; +export declare function replaceJsonOrYamlStrings({ source, type, strings, config }: { + source: string; + type?: 'json' | 'yaml'; + strings: string[]; + config: Config; +}): string; diff --git a/dist/replace.js b/dist/replace.js new file mode 100644 index 0000000..c0f4f90 --- /dev/null +++ b/dist/replace.js @@ -0,0 +1,222 @@ +import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; +import { + esNodeIs, + resolveEstreePropertyPath +} from "./ast/estree.js"; +import { eswalk } from "./ast/eswalk.js"; +import { mdNodeIs, mdNodeIsJsxElement } from "./ast/mdast.js"; +import { unwalk } from "./ast/unwalk.js"; +import { + isHtmlTag, + isFrontmatterFieldIncluded, + isHtmlElementIncluded, + isHtmlElementAttributeIncluded, + isJsonOrYamlPropertyIncluded, + isJsxComponentIncluded, + isJsxComponentAttributeIncluded, + isMarkdownNodeIncluded, + isHtmlElementChildrenIncluded, + isJsxComponentChildrenIncluded +} from "./config.js"; +import { isArray, isEmptyArray, isEmptyString, isObject, isString } from "./utils.js"; +function replaceMdastStrings({ + mdast, + config, + strings +}) { + strings = strings.reverse(); + unwalk( + mdast, + (node, __, _) => { + if (mdNodeIs(node, "text")) { + node.value = strings.pop(); + return; + } + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, "mdxJsxAttribute")) + continue; + if (!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config })) + continue; + if (isString(attribute.value)) { + attribute.value = strings.pop(); + } else if (attribute.value?.data?.estree) { + const estree = attribute.value.data.estree; + eswalk(estree, { + SimpleLiteral(esnode, _2) { + if (isString(esnode.value)) + esnode.value = strings.pop(); + } + }); + } + } + } else { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, "mdxJsxAttribute")) + continue; + const componentName = node.name; + const isAttributeIncluded = isJsxComponentAttributeIncluded({ + name: componentName, + attribute: attribute.name, + config + }); + if (isString(attribute.value)) { + if (!isAttributeIncluded) + continue; + attribute.value = strings.pop(); + } else if (attribute.value?.data?.estree) { + if (!config.jsxComponents.include[componentName] || !config.jsxComponents.include[componentName].attributes.some( + (attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + )) + continue; + const estree = attribute.value.data.estree; + eswalk(estree, { + SimpleLiteral(esnode, _2) { + if (isString(esnode.value)) + esnode.value = strings.pop(); + }, + JSXElement(esnode, _2) { + const name = esnode.openingElement.name.name; + if (isHtmlTag(name)) { + if (!isHtmlElementIncluded({ tag: name, config }) || !isHtmlElementChildrenIncluded({ tag: name, config })) + return false; + } else if (!isJsxComponentIncluded({ name, config }) || !isJsxComponentChildrenIncluded({ name, config })) + return false; + }, + JSXAttribute(esnode, parents) { + const name = typeof esnode.name.name === "string" ? esnode.name.name : esnode.name.name.name; + const parentName = parents[parents.length - 1].openingElement.name.name; + if (isHtmlTag(parentName)) { + if (!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config })) + return false; + } else if (!config.jsxComponents.include[name] || !config.jsxComponents.include[name].attributes.some( + (attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + )) { + return false; + } + }, + JSXText(esnode, _2) { + esnode.value = strings.pop(); + }, + Property(esnode, parents) { + if (!esNodeIs(esnode, "Identifier")) + return false; + const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name); + if (!propertyPath || !isJsxComponentAttributeIncluded({ + name: componentName, + attribute: propertyPath, + config + })) + return false; + } + }); + } + } + } + } + if (mdNodeIs(node, "yaml")) { + if (isEmptyArray(config.frontmatterFields.include)) + return; + if (isEmptyString(node.value)) + return; + const object = parseYaml(node.value); + for (const field in object) { + if (!isFrontmatterFieldIncluded({ field, config })) + continue; + const value = object[field]; + if (isString(value)) { + object[field] = strings.pop(); + continue; + } + if (isArray(value)) { + for (const [index, item] of value.entries()) { + if (!isString(item)) + continue; + value[index] = strings.pop(); + } + } + } + return; + } + }, + (node, parent) => { + if (!isMarkdownNodeIncluded({ type: node.type, config })) + return false; + if (parent && mdNodeIsJsxElement(parent) && parent.name) { + if (isHtmlTag(parent.name)) { + if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) + return false; + } else { + if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) + return false; + } + return true; + } + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + if (!isHtmlElementIncluded({ tag: node.name, config })) + return false; + } else { + if (!isJsxComponentIncluded({ name: node.name, config })) + return false; + } + return true; + } + return true; + } + ); + return mdast; +} +function replaceJsonOrYamlStrings({ + source, + type = "json", + strings, + config +}) { + if (isEmptyArray(config.jsonOrYamlProperties.include)) + return source; + strings = strings.reverse(); + const parsed = type === "json" ? JSON.parse(source) : parseYaml(source); + process({ value: parsed }); + function process({ + value, + parent, + property, + index + }) { + if (isArray(value)) { + for (const [index2, item] of value.entries()) { + process({ value: item, parent: value, property, index: index2 }); + } + return; + } + if (isObject(value)) { + for (const property2 in value) { + const item = value[property2]; + process({ value: item, parent: value, property: property2 }); + } + return; + } + if (typeof value === "string") { + if (property && isJsonOrYamlPropertyIncluded({ property, config })) { + if (isArray(parent) && index) { + parent[index] = strings.pop(); + return; + } + if (isObject(parent)) { + parent[property] = strings.pop(); + return; + } + } + return; + } + } + if (type === "json") + return JSON.stringify(parsed); + return stringifyYaml(parsed); +} +export { + replaceJsonOrYamlStrings, + replaceMdastStrings +}; diff --git a/dist/translate.d.ts b/dist/translate.d.ts new file mode 100644 index 0000000..71d7149 --- /dev/null +++ b/dist/translate.d.ts @@ -0,0 +1,7 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +/** + * Translate an array of strings from sourceLang to targetLang using DeepL. + * Batches requests in groups of 10. + * Requires DEEPL_AUTH_KEY environment variable. + */ +export declare function translateStrings(strings: string[], sourceLang: SourceLanguageCode, targetLang: TargetLanguageCode): Promise; diff --git a/dist/translate.js b/dist/translate.js new file mode 100644 index 0000000..ddbdb1f --- /dev/null +++ b/dist/translate.js @@ -0,0 +1,35 @@ +import { Translator } from "deepl-node"; +async function translateStrings(strings, sourceLang, targetLang) { + if (strings.length === 0) + return []; + const DEEPL_AUTH_KEY = process.env.DEEPL_AUTH_KEY; + if (!DEEPL_AUTH_KEY) + throw new Error("DEEPL_AUTH_KEY environment variable must be set"); + const deepl = new Translator(DEEPL_AUTH_KEY); + const translations = new Array(strings.length).fill(""); + const queue = []; + for (const [index, string] of strings.entries()) { + queue.push([index, string]); + if (index === strings.length - 1 || queue.length === 10) { + const indexes = queue.map(([i]) => i); + const batch = queue.map(([, s]) => s); + const results = await deepl.translateText( + batch, + sourceLang, + targetLang, + { + tagHandling: "html", + splitSentences: "nonewlines" + } + ); + for (let j = 0; j < indexes.length; j++) { + translations[indexes[j]] = results[j].text; + } + queue.length = 0; + } + } + return translations; +} +export { + translateStrings +}; diff --git a/dist/utils.d.ts b/dist/utils.d.ts new file mode 100644 index 0000000..5df7aa2 --- /dev/null +++ b/dist/utils.d.ts @@ -0,0 +1,7 @@ +export declare function isArray(value: unknown): value is any[]; +export declare function isBoolean(value: unknown): value is boolean; +export declare function isEmptyArray(array: any[]): boolean; +export declare function isEmptyObject(object: Object): boolean; +export declare function isEmptyString(string: string): boolean; +export declare function isObject(value: unknown): value is Record; +export declare function isString(value: unknown): value is string; diff --git a/dist/utils.js b/dist/utils.js new file mode 100644 index 0000000..9b4035e --- /dev/null +++ b/dist/utils.js @@ -0,0 +1,30 @@ +function isArray(value) { + return Array.isArray(value); +} +function isBoolean(value) { + return typeof value === "boolean"; +} +function isEmptyArray(array) { + return array.length === 0; +} +function isEmptyObject(object) { + return Object.keys(object).length === 0; +} +function isEmptyString(string) { + return string.length === 0; +} +function isObject(value) { + return isArray(value) ? false : typeof value == "object" ? true : false; +} +function isString(value) { + return typeof value === "string"; +} +export { + isArray, + isBoolean, + isEmptyArray, + isEmptyObject, + isEmptyString, + isObject, + isString +}; diff --git a/dist/vendor/mdast-util-html-comment.d.ts b/dist/vendor/mdast-util-html-comment.d.ts new file mode 100644 index 0000000..aedb0bc --- /dev/null +++ b/dist/vendor/mdast-util-html-comment.d.ts @@ -0,0 +1,4 @@ +import type { Extension } from 'mdast-util-from-markdown'; +import type { Options } from 'mdast-util-to-markdown'; +export declare function htmlCommentFromMarkdown(): Extension; +export declare function htmlCommentToMarkdown(): Options; diff --git a/dist/vendor/mdast-util-html-comment.js b/dist/vendor/mdast-util-html-comment.js new file mode 100644 index 0000000..12cadc8 --- /dev/null +++ b/dist/vendor/mdast-util-html-comment.js @@ -0,0 +1,37 @@ +function htmlCommentFromMarkdown() { + return { + canContainEols: ["htmlComment"], + enter: { + htmlComment() { + this.buffer(); + } + }, + exit: { + htmlComment(token) { + const string = this.resume(); + this.enter( + { + // @ts-ignore + type: "htmlComment", + value: string.slice(0, -3) + }, + token + ); + this.exit(token); + } + } + }; +} +function htmlCommentToMarkdown() { + return { + handlers: { + htmlComment(node) { + return ``; + } + } + }; +} +export { + htmlCommentFromMarkdown, + htmlCommentToMarkdown +}; diff --git a/dist/vendor/micromark-extension-html-comment.d.ts b/dist/vendor/micromark-extension-html-comment.d.ts new file mode 100644 index 0000000..77fb0cb --- /dev/null +++ b/dist/vendor/micromark-extension-html-comment.d.ts @@ -0,0 +1,3 @@ +import type { Extension, HtmlExtension } from 'micromark-util-types'; +export declare function htmlComment(): Extension; +export declare function htmlCommentToHtml(): HtmlExtension; diff --git a/dist/vendor/micromark-extension-html-comment.js b/dist/vendor/micromark-extension-html-comment.js new file mode 100644 index 0000000..3b5628b --- /dev/null +++ b/dist/vendor/micromark-extension-html-comment.js @@ -0,0 +1,109 @@ +import { factorySpace } from "micromark-factory-space"; +import { markdownLineEnding } from "micromark-util-character"; +import { codes } from "micromark-util-symbol/codes.js"; +import { types } from "micromark-util-symbol/types.js"; +function htmlComment() { + return { + flow: { + [codes.lessThan]: { tokenize, concrete: true } + }, + text: { + [codes.lessThan]: { tokenize } + } + }; +} +function htmlCommentToHtml() { + return { + enter: { + htmlComment() { + this.buffer(); + } + }, + exit: { + htmlComment() { + this.resume(); + } + } + }; +} +const tokenize = (effects, ok, nok) => { + let value = ""; + return start; + function start(code) { + effects.enter("htmlComment"); + effects.enter("htmlCommentMarker"); + effects.consume(code); + value += "<"; + return open; + } + function open(code) { + if (value === "<" && code === codes.exclamationMark) { + effects.consume(code); + value += "!"; + return open; + } + if (code === codes.dash) { + if (value === "= 6 && value.slice(-2) === "--") { + effects.consume(code); + effects.exit(types.data); + effects.exit("htmlCommentString"); + effects.exit("htmlComment"); + value += ">"; + return ok; + } + return nok(code); + } +}; +export { + htmlComment, + htmlCommentToHtml +}; diff --git a/package.json b/package.json index e67de54..4ba9512 100644 --- a/package.json +++ b/package.json @@ -1,45 +1,52 @@ { - "name": "@plastichub/template", - "description": "", - "version": "0.3.1", - "main": "main.js", - "typings": "index.d.ts", - "publishConfig": { - "access": "public" - }, - "bin": { - "osr-bin": "main.js" - }, - "dependencies": { - "@types/node": "^14.17.5", - "@types/yargs": "^17.0.2", - "chalk": "^2.4.1", - "convert-units": "^2.3.4", - "env-var": "^7.0.1", - "typescript": "^4.3.5", - "yargs": "^14.2.3", - "yargs-parser": "^15.0.3" - }, - "scripts": { - "test": "tsc; mocha --full-trace mocha \"spec/**/*.spec.js\"", - "test-with-coverage": "istanbul cover node_modules/.bin/_mocha -- 'spec/**/*.spec.js'", - "lint": "tslint --project=./tsconfig.json", - "build": "tsc -p .", - "dev": "tsc -p . --declaration -w", - "typings": "tsc --declaration", - "docs": "npx typedoc src/index.ts", - "dev-test-watch": "mocha-typescript-watch" - }, - "homepage": "https://git.osr-plastic.org/plastichub/lib-content", - "repository": { - "type": "git", - "url": "https://git.osr-plastic.org/plastichub/lib-content.git" - }, - "engines": { - "node": ">= 14.0.0" - }, - "license": "BSD-3-Clause", - "keywords": [ - "typescript" - ] -} + "name": "deepmark", + "description": "Translate markdown files correctly with `mdast` and DeepL.", + "version": "0.2.0", + "license": "MIT", + "author": "Izzuddin Natsir", + "type": "module", + "files": [ + "dist/*" + ], + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc && node build.js", + "test": "vitest run --reporter verbose", + "test:watch": "vitest watch --reporter verbose" + }, + "dependencies": { + "acorn": "^8.8.2", + "acorn-jsx": "^5.3.2", + "astring": "^1.8.4", + "deepl-node": "^1.8.0", + "mdast-util-from-markdown": "^1.3.0", + "mdast-util-frontmatter": "^1.0.1", + "mdast-util-mdx": "^2.0.1", + "mdast-util-to-markdown": "^1.5.0", + "micromark-extension-frontmatter": "^1.0.0", + "micromark-extension-mdxjs": "^1.0.0", + "micromark-factory-space": "^1.0.0", + "micromark-util-character": "^1.1.0", + "micromark-util-symbol": "^1.0.1", + "micromark-util-types": "^1.0.2", + "prettier": "^2.8.3", + "yaml": "^2.2.1" + }, + "devDependencies": { + "@types/estree": "^1.0.0", + "@types/mdast": "^3.0.10", + "@types/node": "^18.11.19", + "@types/prettier": "^2.7.2", + "@types/unist": "^2.0.6", + "esbuild": "^0.17.5", + "typescript": "^5.0.0", + "vitest": "^0.28.4" + } +} \ No newline at end of file diff --git a/src/__test__/__samples__/config/base.mjs b/src/__test__/__samples__/config/base.mjs new file mode 100644 index 0000000..95b5b49 --- /dev/null +++ b/src/__test__/__samples__/config/base.mjs @@ -0,0 +1,11 @@ +/** @type {import("../../../config").UserConfig} */ +export default { + sourceLanguage: 'en', + outputLanguages: ['zh', 'ja'], + directories: [ + ['i18n/$langcode$', 'i18n/$langcode$'], + ['docs', 'i18n/$langcode$/docs'], + ['blog', 'i18n/$langcode$/blog'] + ], + cwd: '../../example' +}; diff --git a/src/__test__/__samples__/config/hybrid.mjs b/src/__test__/__samples__/config/hybrid.mjs new file mode 100644 index 0000000..44ff554 --- /dev/null +++ b/src/__test__/__samples__/config/hybrid.mjs @@ -0,0 +1,12 @@ +import base from './base.mjs'; + +/** @type {import("../../../config").UserConfig} */ +export default { + ...base, + files: { + include: ['docs/intro.md', 'docs/tutorial-basics/markdown-features.mdx', 'i18n/en/code.json'] + }, + jsonOrYamlProperties: { + include: ['message', 'description'] + } +}; diff --git a/src/__test__/__samples__/config/offline.mjs b/src/__test__/__samples__/config/offline.mjs new file mode 100644 index 0000000..d824a89 --- /dev/null +++ b/src/__test__/__samples__/config/offline.mjs @@ -0,0 +1,9 @@ +import base from './base.mjs'; + +/** @type {import("../../../config").UserConfig} */ +export default { + ...base, + jsonOrYamlProperties: { + include: ['message', 'description'] + } +}; diff --git a/src/__test__/__samples__/config/online.mjs b/src/__test__/__samples__/config/online.mjs new file mode 100644 index 0000000..145890d --- /dev/null +++ b/src/__test__/__samples__/config/online.mjs @@ -0,0 +1,12 @@ +import base from './base.mjs'; + +/** @type {import("../../../config").UserConfig} */ +export default { + ...base, + files: { + include: ['docs/tutorial-basics/markdown-features.mdx', 'i18n/en/code.json'] + }, + jsonOrYamlProperties: { + include: ['message', 'description'] + } +}; diff --git a/src/__test__/__samples__/frontmatter/empty.md b/src/__test__/__samples__/frontmatter/empty.md new file mode 100644 index 0000000..a845151 --- /dev/null +++ b/src/__test__/__samples__/frontmatter/empty.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/src/__test__/__samples__/frontmatter/index.md b/src/__test__/__samples__/frontmatter/index.md new file mode 100644 index 0000000..096a558 --- /dev/null +++ b/src/__test__/__samples__/frontmatter/index.md @@ -0,0 +1,6 @@ +--- +author: Izzuddin Natsir +title: A Short Title +tags: [tagone, tagtwo] +description: A short description. +--- diff --git a/src/__test__/__samples__/json/navbar.json b/src/__test__/__samples__/json/navbar.json new file mode 100644 index 0000000..a430cde --- /dev/null +++ b/src/__test__/__samples__/json/navbar.json @@ -0,0 +1,18 @@ +{ + "title": { + "message": "My Site", + "description": "The title in the navbar" + }, + "item.label.Tutorial": { + "message": "Tutorial", + "description": "Navbar item with label Tutorial" + }, + "item.label.Blog": { + "message": "Blog", + "description": "Navbar item with label Blog" + }, + "item.label.GitHub": { + "message": "GitHub", + "description": "Navbar item with label GitHub" + } +} diff --git a/src/__test__/__samples__/jsx/code-and-pre.mdx b/src/__test__/__samples__/jsx/code-and-pre.mdx new file mode 100644 index 0000000..06a7859 --- /dev/null +++ b/src/__test__/__samples__/jsx/code-and-pre.mdx @@ -0,0 +1,5 @@ +

+ This is a text. + function + 

preformatted
+

diff --git a/src/__test__/__samples__/jsx/jsx-in-prop.mdx b/src/__test__/__samples__/jsx/jsx-in-prop.mdx new file mode 100644 index 0000000..88902fc --- /dev/null +++ b/src/__test__/__samples__/jsx/jsx-in-prop.mdx @@ -0,0 +1,14 @@ +This is a text inside a jsx prop.}> + This is a text inside a custom component. + + + + This is the text of jsx item one. This the nested text of jsx item one. + , +
+ This is the text of jsx item two. This the nested text of jsx item two. +
+ ]} +> diff --git a/src/__test__/__samples__/jsx/nested.mdx b/src/__test__/__samples__/jsx/nested.mdx new file mode 100644 index 0000000..1d26b02 --- /dev/null +++ b/src/__test__/__samples__/jsx/nested.mdx @@ -0,0 +1,4 @@ +

+ This is a paragraph.This is a span. + This is a text inside a custom component. +

diff --git a/src/__test__/__samples__/yaml/authors.yml b/src/__test__/__samples__/yaml/authors.yml new file mode 100644 index 0000000..bcb2991 --- /dev/null +++ b/src/__test__/__samples__/yaml/authors.yml @@ -0,0 +1,17 @@ +endi: + name: Endilie Yacop Sucipto + title: Maintainer of Docusaurus + url: https://github.com/endiliey + image_url: https://github.com/endiliey.png + +yangshun: + name: Yangshun Tay + title: Front End Engineer @ Facebook + url: https://github.com/yangshun + image_url: https://github.com/yangshun.png + +slorber: + name: Sébastien Lorber + title: Docusaurus maintainer + url: https://sebastienlorber.com + image_url: https://github.com/slorber.png diff --git a/src/__test__/__snapshots__/config.test.ts.snap b/src/__test__/__snapshots__/config.test.ts.snap new file mode 100644 index 0000000..59932c5 --- /dev/null +++ b/src/__test__/__snapshots__/config.test.ts.snap @@ -0,0 +1,153 @@ +// Vitest Snapshot v1 + +exports[`resolve paths > resolve source and output directories absolute paths 1`] = ` +[ + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$", + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs", + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog", +] +`; + +exports[`resolve paths > resolve source and output directories absolute paths 2`] = ` +[ + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$", + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs", + "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog", +] +`; + +exports[`resolve paths > resolve source file absolute paths 1`] = ` +{ + "json": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/code.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/en/code.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docusaurus-plugin-content-blog/options.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/en/docusaurus-plugin-content-blog/options.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docusaurus-plugin-content-docs/current.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/en/docusaurus-plugin-content-docs/current.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docusaurus-theme-classic/footer.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/en/docusaurus-theme-classic/footer.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docusaurus-theme-classic/navbar.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/en/docusaurus-theme-classic/navbar.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/_category_.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/_category_.json", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-extras/_category_.json", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-extras/_category_.json", + }, + ], + "md": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/intro.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/intro.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/congratulations.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/congratulations.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/create-a-blog-post.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/create-a-blog-post.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/create-a-document.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/create-a-document.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/create-a-page.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/create-a-page.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/deploy-your-site.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/deploy-your-site.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-basics/markdown-features.mdx", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-basics/markdown-features.mdx", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-extras/manage-docs-versions.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-extras/manage-docs-versions.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-extras/translate-your-site.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-extras/translate-your-site.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2019-05-28-first-blog-post.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2019-05-28-first-blog-post.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2019-05-29-long-blog-post.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2019-05-29-long-blog-post.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2021-08-01-mdx-blog-post.mdx", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2021-08-01-mdx-blog-post.mdx", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2021-08-26-welcome/index.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2021-08-26-welcome/index.md", + }, + ], + "others": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-extras/img/docsVersionDropdown.png", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-extras/img/docsVersionDropdown.png", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/docs/tutorial-extras/img/localeDropdown.png", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/docs/tutorial-extras/img/localeDropdown.png", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg", + }, + ], + "yaml": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/authors.yml", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/authors.yml", + }, + ], +} +`; + +exports[`resolve paths with files include and exclude patterns > resolve source file absolute paths 1`] = ` +{ + "json": [], + "md": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2019-05-28-first-blog-post.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2019-05-28-first-blog-post.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2019-05-29-long-blog-post.md", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2019-05-29-long-blog-post.md", + }, + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/2021-08-01-mdx-blog-post.mdx", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/2021-08-01-mdx-blog-post.mdx", + }, + ], + "others": [], + "yaml": [ + { + "outputFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/i18n/$langcode$/blog/authors.yml", + "sourceFilePath": "/home/izznatsir/Codes/github/izznatsir/deepmark/example/blog/authors.yml", + }, + ], +} +`; diff --git a/src/__test__/__snapshots__/extract.test.ts.snap b/src/__test__/__snapshots__/extract.test.ts.snap new file mode 100644 index 0000000..c0e10d3 --- /dev/null +++ b/src/__test__/__snapshots__/extract.test.ts.snap @@ -0,0 +1,52 @@ +// Vitest Snapshot v1 + +exports[`extract frontmatter field string values > filter frontmatter fields based on configuration 1`] = ` +[ + "A Short Title", + "tagone", + "tagtwo", +] +`; + +exports[`extract jsx children and attribute string values > ignore some HTML elements by default 1`] = ` +[ + "This is a text. ", +] +`; + +exports[`extract jsx children and attribute string values > recursively extract strings from html elements and jsx components inside attributes 1`] = ` +[ + "This is a text inside a custom component.", + "This is a text inside a jsx prop.", + "This is the text of jsx item one. ", + "This the nested text of jsx item one.", + "A short title inside title attribute inside HTML element inside an attribute.", + "This is the text of jsx item two. ", + "This the nested text of jsx item two.", +] +`; + +exports[`extract jsx children and attribute string values > recursively extract strings from nested jsx components 1`] = ` +[ + "This is a paragraph.", + "This is a span.", + "This is a text inside a custom component.", +] +`; + +exports[`extract strings from JSON based on configuration > filter properties based on the config 1`] = ` +[ + "The title in the navbar", + "Navbar item with label Tutorial", + "Navbar item with label Blog", + "Navbar item with label GitHub", +] +`; + +exports[`extract strings from yaml based on configuration > filter properties based on the config 1`] = ` +[ + "Maintainer of Docusaurus", + "Front End Engineer @ Facebook", + "Docusaurus maintainer", +] +`; diff --git a/src/__test__/config.test.ts b/src/__test__/config.test.ts new file mode 100644 index 0000000..db806ab --- /dev/null +++ b/src/__test__/config.test.ts @@ -0,0 +1,62 @@ +import { describe } from 'vitest'; +import type { Config, UserConfig } from '../config'; +import { + isFrontmatterFieldIncluded, + isHtmlElementIncluded, + isHtmlElementAttributeIncluded, + isJsonOrYamlPropertyIncluded, + isJsxComponentIncluded, + isJsxComponentAttributeIncluded, + isMarkdownNodeIncluded, + resolveConfig +} from '../config'; + +const baseConfig: UserConfig = { + sourceLanguage: 'en', + outputLanguages: ['zh'], + directories: [['', '']] +}; + +describe('default configurations', (test) => { + const config: Config = resolveConfig(baseConfig); + + test('frontmatter fields', ({ expect }) => { + expect(isFrontmatterFieldIncluded({ field: 'title', config })).toBe(false); + expect(isFrontmatterFieldIncluded({ field: 'description', config })).toBe(false); + }); + + test('markdown nodes', ({ expect }) => { + expect(isMarkdownNodeIncluded({ type: 'code', config })).toBe(false); + expect(isMarkdownNodeIncluded({ type: 'blockquote', config })).toBe(true); + expect(isMarkdownNodeIncluded({ type: 'heading', config })).toBe(true); + }); + + test('html elements', ({ expect }) => { + expect(isHtmlElementIncluded({ tag: 'a', config })).toBe(true); + expect(isHtmlElementIncluded({ tag: 'div', config })).toBe(true); + }); + + test('html element attributes', ({ expect }) => { + expect(isHtmlElementAttributeIncluded({ tag: 'div', attribute: 'title', config })).toBe(true); + expect(isHtmlElementAttributeIncluded({ tag: 'div', attribute: 'id', config })).toBe(false); + }); + + test('jsx components', ({ expect }) => { + expect(isJsxComponentIncluded({ name: 'Card', config })).toBe(true); + expect(isJsxComponentIncluded({ name: 'Warning', config })).toBe(true); + }); + + test('jsx component attributes', ({ expect }) => { + expect(isJsxComponentAttributeIncluded({ name: 'Card', attribute: 'icon', config })).toBe( + false + ); + expect(isJsxComponentAttributeIncluded({ name: 'Warning', attribute: 'title', config })).toBe( + false + ); + }); + + test('json or yaml properties', ({ expect }) => { + expect(isJsonOrYamlPropertyIncluded({ property: 'author', config })).toBe(false); + expect(isJsonOrYamlPropertyIncluded({ property: 'title', config })).toBe(false); + }); +}); diff --git a/src/__test__/extract.test.ts b/src/__test__/extract.test.ts new file mode 100644 index 0000000..1d1882a --- /dev/null +++ b/src/__test__/extract.test.ts @@ -0,0 +1,152 @@ +import { readFile } from 'node:fs/promises'; +import np from 'node:path'; +import { describe } from 'vitest'; +import { getMdast } from '../ast/mdast'; +import { Config, resolveConfig, UserConfig } from '../config'; +import { extractJsonOrYamlStrings, extractMdastStrings } from '../extract'; +import { format } from '../format'; + +const baseConfig: UserConfig = { + sourceLanguage: 'en', + outputLanguages: ['zh'], + directories: [['', '']], + cwd: 'src/__test__/__samples__' +}; + +async function extract( + path: string, + config: Config, + from: 'markdown' | 'json' | 'yaml' = 'markdown' +) { + const resolvedPath = np.resolve(config.cwd, path); + const source = await readFile(resolvedPath, { encoding: 'utf-8' }); + + if (from === 'markdown') + return extractMdastStrings({ mdast: getMdast(await format(source)), config }); + + return extractJsonOrYamlStrings({ source, type: from, config }); +} + +describe('extract frontmatter field string values', (test) => { + test('ignore empty frontmatter', async ({ expect }) => { + const strings = await extract('frontmatter/empty.md', resolveConfig(baseConfig)); + + expect(strings.length).toBe(0); + }); + + test('filter frontmatter fields based on configuration', async ({ expect }) => { + const strings = await extract( + 'frontmatter/index.md', + resolveConfig({ + ...baseConfig, + frontmatterFields: { + include: ['title', 'tags', 'description'], + exclude: ['description'] + } + }) + ); + + expect(strings).toMatchSnapshot(); + }); +}); + +describe('extract jsx children and attribute string values', (test) => { + test('recursively extract strings from nested jsx components', async ({ expect }) => { + const strings = await extract( + 'jsx/nested.mdx', + resolveConfig({ + ...baseConfig, + jsxComponents: { + include: { + Block: { children: true, attributes: [] } + } + } + }) + ); + + expect(strings).toMatchSnapshot(); + }); + + test('recursively extract strings from html elements and jsx components inside attributes', async ({ + expect + }) => { + const strings = await extract( + 'jsx/jsx-in-prop.mdx', + resolveConfig({ + ...baseConfig, + jsxComponents: { + include: { + Card: { + children: true, + attributes: ['header'] + }, + List: { + children: false, + attributes: ['items'] + } + } + } + }) + ); + + expect(strings).toMatchSnapshot(); + }); + + test('ignore some HTML elements by default', async ({ expect }) => { + const strings = await extract('jsx/code-and-pre.mdx', resolveConfig(baseConfig)); + + expect(strings).toMatchSnapshot(); + }); +}); + +describe('extract strings from JSON based on configuration', (test) => { + test('do not extract any string if no property name is included in the config', async ({ + expect + }) => { + const strings = await extract('json/navbar.json', resolveConfig(baseConfig), 'json'); + + expect(strings.length).toBe(0); + }); + + test('filter properties based on the config', async ({ expect }) => { + const strings = await extract( + 'json/navbar.json', + resolveConfig({ + ...baseConfig, + jsonOrYamlProperties: { + include: ['message', 'description'], + exclude: ['message'] + } + }), + 'json' + ); + + expect(strings).toMatchSnapshot(); + }); +}); + +describe('extract strings from yaml based on configuration', (test) => { + test('do not extract any string if no property name is included in the config', async ({ + expect + }) => { + const strings = await extract('yaml/authors.yml', resolveConfig(baseConfig), 'yaml'); + + expect(strings.length).toBe(0); + }); + + test('filter properties based on the config', async ({ expect }) => { + const strings = await extract( + 'yaml/authors.yml', + resolveConfig({ + ...baseConfig, + jsonOrYamlProperties: { + include: ['name', 'title'], + exclude: ['name'] + } + }), + 'yaml' + ); + + expect(strings).toMatchSnapshot(); + }); +}); diff --git a/src/ast/estree.ts b/src/ast/estree.ts new file mode 100644 index 0000000..5acf1b4 --- /dev/null +++ b/src/ast/estree.ts @@ -0,0 +1,352 @@ +import type { + BaseNode as EsBaseNode, + Identifier as EsIdentifier, + Program as EsProgram, + SwitchCase as EsSwitchCase, + CatchClause as EsCatchClause, + VariableDeclarator as EsVariableDeclarator, + ExpressionStatement as EsExpressionStatement, + BlockStatement as EsBlockStatement, + EmptyStatement as EsEmptyStatement, + DebuggerStatement as EsDebuggerStatement, + WithStatement as EsWithStatement, + ReturnStatement as EsReturnStatement, + LabeledStatement as EsLabeledStatement, + BreakStatement as EsBreakStatement, + ContinueStatement as EsContinueStatement, + IfStatement as EsIfStatement, + SwitchStatement as EsSwitchStatement, + ThrowStatement as EsThrowStatement, + TryStatement as EsTryStatement, + WhileStatement as EsWhileStatement, + DoWhileStatement as EsDoWhileStatement, + ForStatement as EsForStatement, + ForInStatement as EsForInStatement, + ForOfStatement as EsForOfStatement, + ClassDeclaration as EsClassDeclaration, + FunctionDeclaration as EsFunctionDeclaration, + VariableDeclaration as EsVariableDeclaration, + ModuleDeclaration as EsModuleDeclaration, + ImportDeclaration as EsImportDeclaration, + ExportDefaultDeclaration as EsExportDefaultDeclaration, + ExportNamedDeclaration as EsExportNamedDeclaration, + ExportAllDeclaration as EsExportAllDeclaration, + ThisExpression as EsThisExpression, + ArrayExpression as EsArrayExpression, + ObjectExpression as EsObjectExpression, + FunctionExpression as EsFunctionExpression, + ArrowFunctionExpression as EsArrowFunctionExpression, + YieldExpression as EsYieldExpression, + UnaryExpression as EsUnaryExpression, + UpdateExpression as EsUpdateExpression, + BinaryExpression as EsBinaryExpression, + AssignmentExpression as EsAssignmentExpression, + LogicalExpression as EsLogicalExpression, + MemberExpression as EsMemberExpression, + ConditionalExpression as EsConditionalExpression, + CallExpression as EsCallExpression, + NewExpression as EsNewExpression, + SequenceExpression as EsSequenceExpression, + TaggedTemplateExpression as EsTaggedTemplateExpression, + ClassExpression as EsClassExpression, + AwaitExpression as EsAwaitExpression, + ImportExpression as EsImportExpression, + ChainExpression as EsChainExpression, + SimpleLiteral as EsSimpleLiteral, + RegExpLiteral as EsRegExpLiteral, + BigIntLiteral as EsBigIntLiteral, + TemplateLiteral as EsTemplateLiteral, + PrivateIdentifier as EsPrivateIdentifier, + Property as EsProperty, + MetaProperty as EsMetaProperty, + PropertyDefinition as EsPropertyDefinition, + AssignmentProperty as EsAssignmentProperty, + Super as EsSuper, + TemplateElement as EsTemplateElement, + SpreadElement as EsSpreadElement, + ObjectPattern as EsObjectPattern, + ArrayPattern as EsArrayPattern, + RestElement as EsRestElement, + AssignmentPattern as EsAssignmentPattern, + Class as EsClass, + ClassBody as EsClassBody, + StaticBlock as EsStaticBlock, + MethodDefinition as EsMethodDefinition, + ModuleSpecifier as EsModuleSpecifier, + ImportSpecifier as EsImportSpecifier, + ImportNamespaceSpecifier as EsImportNamespaceSpecifier, + ImportDefaultSpecifier as EsImportDefaultSpecifier, + ExportSpecifier as EsExportSpecifier +} from 'estree'; + +import type { + JSXAttribute as EsJsxAttribute, + JSXClosingElement as EsJsxClosingElement, + JSXClosingFragment as EsJsxClosingFragment, + JSXElement as EsJsxElement, + JSXEmptyExpression as EsJsxEmptyExpression, + JSXExpressionContainer as EsJsxExpressionContainer, + JSXFragment as EsJsxFragment, + JSXIdentifier as EsJsxIdentifier, + JSXMemberExpression as EsJsxMemberExpression, + JSXNamespacedName as EsJsxNamespacedName, + JSXOpeningElement as EsJsxOpeningElement, + JSXOpeningFragment as EsJsxOpeningFragment, + JSXSpreadAttribute as EsJsxSpreadAttribute, + JSXSpreadChild as EsJsxSpreadChild, + JSXText as EsJsxText +} from 'estree-jsx'; + +export function esNodeIs(node: EsNode, type: T): node is EsNodeMap[T] { + return node ? node.type === type : false; +} + +export function resolveEstreePropertyPath( + node: EsProperty, + parents: EsNode[], + attributeName: string +): string | undefined { + if (!esNodeIs(parents[2], 'ArrayExpression') && !esNodeIs(parents[2], 'ObjectExpression')) return; + if (!esNodeIs(node.key, 'Identifier')) return; + + const names = [node.key.name]; + + for (let i = parents.length - 1; i > 1; i--) { + const parent = parents[i]; + if (esNodeIs(parent, 'ArrayExpression') || esNodeIs(parent, 'ObjectExpression')) continue; + if (esNodeIs(parent, 'Property')) { + if (!esNodeIs(parent.key, 'Identifier')) return; + names.push(parent.key.name); + continue; + } + + return; + } + + names.push(attributeName); + + return names.reverse().join('.'); +} + +/** + * ============================================================ + */ + +export type { + EsBaseNode, + EsIdentifier, + EsProgram, + EsSwitchCase, + EsCatchClause, + EsVariableDeclarator, + EsExpressionStatement, + EsBlockStatement, + EsEmptyStatement, + EsDebuggerStatement, + EsWithStatement, + EsReturnStatement, + EsLabeledStatement, + EsBreakStatement, + EsContinueStatement, + EsIfStatement, + EsSwitchStatement, + EsThrowStatement, + EsTryStatement, + EsWhileStatement, + EsDoWhileStatement, + EsForStatement, + EsForInStatement, + EsForOfStatement, + EsClassDeclaration, + EsFunctionDeclaration, + EsVariableDeclaration, + EsModuleDeclaration, + EsImportDeclaration, + EsExportDefaultDeclaration, + EsExportNamedDeclaration, + EsExportAllDeclaration, + EsThisExpression, + EsArrayExpression, + EsObjectExpression, + EsFunctionExpression, + EsArrowFunctionExpression, + EsYieldExpression, + EsUnaryExpression, + EsUpdateExpression, + EsBinaryExpression, + EsAssignmentExpression, + EsLogicalExpression, + EsMemberExpression, + EsConditionalExpression, + EsCallExpression, + EsNewExpression, + EsSequenceExpression, + EsTaggedTemplateExpression, + EsClassExpression, + EsAwaitExpression, + EsImportExpression, + EsChainExpression, + EsSimpleLiteral, + EsRegExpLiteral, + EsBigIntLiteral, + EsTemplateLiteral, + EsPrivateIdentifier, + EsProperty, + EsMetaProperty, + EsPropertyDefinition, + EsAssignmentProperty, + EsSuper, + EsTemplateElement, + EsSpreadElement, + EsObjectPattern, + EsArrayPattern, + EsRestElement, + EsAssignmentPattern, + EsClass, + EsClassBody, + EsStaticBlock, + EsMethodDefinition, + EsModuleSpecifier, + EsImportSpecifier, + EsImportNamespaceSpecifier, + EsImportDefaultSpecifier, + EsExportSpecifier +}; + +export type { + EsJsxAttribute, + EsJsxClosingElement, + EsJsxClosingFragment, + EsJsxElement, + EsJsxEmptyExpression, + EsJsxExpressionContainer, + EsJsxFragment, + EsJsxIdentifier, + EsJsxMemberExpression, + EsJsxNamespacedName, + EsJsxOpeningElement, + EsJsxOpeningFragment, + EsJsxSpreadAttribute, + EsJsxSpreadChild, + EsJsxText +}; + +export type EsNode = EsNodeMap[keyof EsNodeMap]; + +export type EsNodeMap = EsExpressionMap & + EsLiteralMap & + EsFunctionMap & + EsPatternMap & + EsStatementMap & + EsJsxMap & { + CatchClause: EsCatchClause; + Class: EsClass; + ClassBody: EsClassBody; + MethodDefinition: EsMethodDefinition; + ModuleDeclaration: EsModuleDeclaration; + ModuleSpecifier: EsModuleSpecifier; + PrivateIdentifier: EsPrivateIdentifier; + Program: EsProgram; + Property: EsProperty; + PropertyDefinition: EsPropertyDefinition; + SpreadElement: EsSpreadElement; + Super: EsSuper; + SwitchCase: EsSwitchCase; + TemplateElement: EsTemplateElement; + VariableDeclarator: EsVariableDeclarator; + }; + +export type EsExpressionMap = EsLiteralMap & { + ArrayExpression: EsArrayExpression; + ArrowFunctionExpression: EsArrowFunctionExpression; + AssignmentExpression: EsAssignmentExpression; + AwaitExpression: EsAwaitExpression; + BinaryExpression: EsBinaryExpression; + CallExpression: EsCallExpression; + ChainExpression: EsChainExpression; + ClassExpression: EsClassExpression; + ConditionalExpression: EsConditionalExpression; + FunctionExpression: EsFunctionExpression; + Identifier: EsIdentifier; + ImportExpression: EsImportExpression; + LogicalExpression: EsLogicalExpression; + MemberExpression: EsMemberExpression; + MetaProperty: EsMetaProperty; + NewExpression: EsNewExpression; + ObjectExpression: EsObjectExpression; + SequenceExpression: EsSequenceExpression; + TaggedTemplateExpression: EsTaggedTemplateExpression; + TemplateLiteral: EsTemplateLiteral; + ThisExpression: EsThisExpression; + UnaryExpression: EsUnaryExpression; + UpdateExpression: EsUpdateExpression; + YieldExpression: EsYieldExpression; +}; + +export interface EsLiteralMap { + Literal: EsSimpleLiteral | EsRegExpLiteral | EsBigIntLiteral; + SimpleLiteral: EsSimpleLiteral; + RegExpLiteral: EsRegExpLiteral; + BigIntLiteral: EsBigIntLiteral; +} + +export interface EsFunctionMap { + FunctionDeclaration: EsFunctionDeclaration; + FunctionExpression: EsFunctionExpression; + ArrowFunctionExpression: EsArrowFunctionExpression; +} + +export interface EsPatternMap { + Identifier: EsIdentifier; + ObjectPattern: EsObjectPattern; + ArrayPattern: EsArrayPattern; + RestElement: EsRestElement; + AssignmentPattern: EsAssignmentPattern; + MemberExpression: EsMemberExpression; +} + +export type EsStatementMap = EsDeclarationMap & { + ExpressionStatement: EsExpressionStatement; + BlockStatement: EsBlockStatement; + StaticBlock: EsStaticBlock; + EmptyStatement: EsEmptyStatement; + DebuggerStatement: EsDebuggerStatement; + WithStatement: EsWithStatement; + ReturnStatement: EsReturnStatement; + LabeledStatement: EsLabeledStatement; + BreakStatement: EsBreakStatement; + ContinueStatement: EsContinueStatement; + IfStatement: EsIfStatement; + SwitchStatement: EsSwitchStatement; + ThrowStatement: EsThrowStatement; + TryStatement: EsTryStatement; + WhileStatement: EsWhileStatement; + DoWhileStatement: EsDoWhileStatement; + ForStatement: EsForStatement; + ForInStatement: EsForInStatement; + ForOfStatement: EsForOfStatement; +}; + +export interface EsDeclarationMap { + FunctionDeclaration: EsFunctionDeclaration; + VariableDeclaration: EsVariableDeclaration; + ClassDeclaration: EsClassDeclaration; +} + +export interface EsJsxMap { + JSXAttribute: EsJsxAttribute; + JSXClosingElement: EsJsxClosingElement; + JSXClosingFragment: EsJsxClosingFragment; + JSXElement: EsJsxElement; + JSXEmptyExpression: EsJsxEmptyExpression; + JSXExpressionContainer: EsJsxExpressionContainer; + JSXFragment: EsJsxFragment; + JSXIdentifier: EsJsxIdentifier; + JSXMemberExpression: EsJsxMemberExpression; + JSXNamespacedName: EsJsxNamespacedName; + JSXOpeningElement: EsJsxOpeningElement; + JSXOpeningFragment: EsJsxOpeningFragment; + JSXSpreadAttribute: EsJsxSpreadAttribute; + JSXSpreadChild: EsJsxSpreadChild; + JSXText: EsJsxText; +} diff --git a/src/ast/eswalk.ts b/src/ast/eswalk.ts new file mode 100644 index 0000000..62c3cff --- /dev/null +++ b/src/ast/eswalk.ts @@ -0,0 +1,109 @@ +import { isRegExp } from 'node:util/types'; +import { EsNode, esNodeIs, EsNodeMap, EsProgram } from './estree.js'; + +export const DEFAULT_ESWALKERS: EsWalkers = { + Program(node, parents, process) { + parents.push(node); + for (const statement of node.body) { + process(statement, parents); + } + parents.pop(); + }, + ExpressionStatement(node, parents, process) { + parents.push(node); + process(node.expression, parents); + parents.pop(); + }, + ArrayExpression(node, parents, process) { + parents.push(node); + for (const element of node.elements) { + process(element, parents); + } + parents.pop(); + }, + ObjectExpression(node, parents, process) { + parents.push(node); + for (const property of node.properties) { + process(property, parents); + } + parents.pop(); + }, + Property(node, parents, process) { + parents.push(node); + process(node.key, parents); + process(node.value, parents); + parents.pop(); + }, + JSXElement(node, parents, process) { + parents.push(node); + for (const child of node.children) { + process(child, parents); + } + for (const attribute of node.openingElement.attributes) { + process(attribute, parents); + } + parents.pop(); + }, + JSXAttribute(node, parents, process) { + parents.push(node); + if (node.value) { + process(node.value, parents); + } + parents.pop(); + } +}; + +export function eswalk( + ast: EsProgram, + visitors: EsVisitors, + walkers: EsWalkers = DEFAULT_ESWALKERS +) { + const process: EsProcessor = (node, parents) => { + if (!node) return; + + let type = node.type as keyof EsNodeMap; + + if (esNodeIs(node, 'Literal')) { + type = + typeof node.value === 'bigint' + ? 'BigIntLiteral' + : isRegExp(node.value) + ? 'RegExpLiteral' + : 'SimpleLiteral'; + } + + const visit = visitors[type] as EsVisitor; + const walk = walkers[type] as EsWalker; + + let keepWalking = true; + + if (visit !== undefined) { + const signal = visit(node, parents); + keepWalking = signal === false ? false : true; + } + + if (keepWalking && walk) walk(node, parents, process); + }; + + process(ast, []); +} + +export interface EsProcessor { + (node: EsNode | null, parents: EsNode[]): void; +} + +export interface EsVisitor { + (node: EsNodeMap[NodeType], parents: EsNode[]): boolean | void; +} + +export type EsVisitors = { + [NodeType in keyof EsNodeMap]?: EsVisitor; +}; + +export interface EsWalker { + (node: EsNodeMap[NodeType], parents: EsNode[], process: EsProcessor): void; +} + +export type EsWalkers = { + [NodeType in keyof Partial]: EsWalker; +}; diff --git a/src/ast/mdast.ts b/src/ast/mdast.ts new file mode 100644 index 0000000..4918ebe --- /dev/null +++ b/src/ast/mdast.ts @@ -0,0 +1,246 @@ +import type { + Root as MdRoot, + Blockquote as MdBlockquote, + Break as MdBreak, + Code as MdCode, + Definition as MdDefinition, + Delete as MdDelete, + Emphasis as MdEmphasis, + Footnote as MdFootnote, + FootnoteDefinition as MdFootnoteDefinition, + FootnoteReference as MdFootnoteReference, + HTML as MdHTML, + Heading as MdHeading, + Image as MdImage, + ImageReference as MdImageReference, + InlineCode as MdInlineCode, + Link as MdLink, + LinkReference as MdLinkReference, + List as MdList, + ListItem as MdListItem, + Paragraph as MdParagraph, + Strong as MdStrong, + Table as MdTable, + TableCell as MdTableCell, + TableRow as MdTableRow, + Text as MdText, + ThematicBreak as MdThematicBreak, + YAML as MdYaml +} from 'mdast'; + +import type { + MdxFlowExpression, + MdxJsxAttribute, + MdxJsxAttributeValueExpression, + MdxJsxExpressionAttribute, + MdxJsxFlowElement, + MdxJsxTextElement, + MdxTextExpression, + MdxjsEsm +} from 'mdast-util-mdx'; + +import type { UnNode } from './unist.js'; + +import { fromMarkdown } from 'mdast-util-from-markdown'; +import { frontmatterFromMarkdown, frontmatterToMarkdown } from 'mdast-util-frontmatter'; +import { htmlCommentFromMarkdown, htmlCommentToMarkdown } from '../vendor/mdast-util-html-comment.js'; +import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx'; +import { toMarkdown } from 'mdast-util-to-markdown'; +import { frontmatter } from 'micromark-extension-frontmatter'; +import { htmlComment } from '../vendor/micromark-extension-html-comment.js'; +import { mdxjs } from 'micromark-extension-mdxjs'; + +declare module 'mdast' { + export interface PhrasingContentMap extends StaticPhrasingContentMap { + mdxJsxFlowElement: MdxJsxFlowElement; + mdxJsxTextElement: MdxJsxTextElement; + mdxFlowExpression: MdxFlowExpression; + mdxTextExpression: MdxTextExpression; + } +} + +export function mdNodeIs( + node: UnNode | undefined, + type: T +): node is T extends MdRoot['type'] +? MdRoot +: T extends MdBlockquote['type'] +? MdBlockquote +: T extends MdBreak['type'] +? MdBreak +: T extends MdCode['type'] +? MdCode +: T extends MdDefinition['type'] +? MdDefinition +: T extends MdDelete['type'] +? MdDelete +: T extends MdEmphasis['type'] +? MdEmphasis +: T extends MdFootnote['type'] +? MdFootnote +: T extends MdFootnoteDefinition['type'] +? MdFootnoteDefinition +: T extends MdFootnoteReference['type'] +? MdFootnoteReference +: T extends MdHTML['type'] +? MdHTML +: T extends MdHeading['type'] +? MdHeading +: T extends MdImage['type'] +? MdImage +: T extends MdImageReference['type'] +? MdImageReference +: T extends MdInlineCode['type'] +? MdInlineCode +: T extends MdLink['type'] +? MdLink +: T extends MdLinkReference['type'] +? MdLinkReference +: T extends MdList['type'] +? MdList +: T extends MdListItem['type'] +? MdListItem +: T extends MdParagraph['type'] +? MdParagraph +: T extends MdStrong['type'] +? MdStrong +: T extends MdTable['type'] +? MdTable +: T extends MdTableCell['type'] +? MdTableCell +: T extends MdTableRow['type'] +? MdTableRow +: T extends MdText['type'] +? MdText +: T extends MdThematicBreak['type'] +? MdThematicBreak +: T extends MdYaml +? MdYaml +: T extends MdxFlowExpression['type'] +? MdxFlowExpression +: T extends MdxJsxAttribute['type'] +? MdxJsxAttribute +: T extends MdxJsxAttributeValueExpression['type'] +? MdxJsxAttributeValueExpression +: T extends MdxJsxExpressionAttribute['type'] +? MdxJsxExpressionAttribute +: T extends MdxJsxFlowElement['type'] +? MdxJsxFlowElement +: T extends MdxJsxTextElement['type'] +? MdxJsxTextElement +: T extends MdxTextExpression['type'] +? MdxTextExpression +: MdxjsEsm { + return node ? node.type === type : false; +} + +export function mdNodeIsJsxElement(node: UnNode): node is MdxJsxFlowElement | MdxJsxTextElement { + return mdNodeIs(node, 'mdxJsxFlowElement') || mdNodeIs(node, 'mdxJsxTextElement'); +} + +/** + * Get MDX flavored `mdast`. + */ +export function getMdast(markdown: string): MdRoot { + return fromMarkdown(markdown, { + extensions: [frontmatter('yaml'), mdxjs(), htmlComment()], + mdastExtensions: [frontmatterFromMarkdown('yaml'), mdxFromMarkdown(), htmlCommentFromMarkdown()] + }); +} + +export function getMarkdown(mdast: MdRoot): string { + return toMarkdown(mdast, { + extensions: [frontmatterToMarkdown('yaml'), mdxToMarkdown(), htmlCommentToMarkdown()], + join: [ + (__, _, parent) => { + if (mdNodeIsJsxElement(parent)) { + return 0; + } + + return 1; + } + ] + }); +} + +/** + * ============================================================ + */ + +export type MdNodeType = + | MdRoot['type'] + | MdBlockquote['type'] + | MdBreak['type'] + | MdCode['type'] + | MdDefinition['type'] + | MdDelete['type'] + | MdEmphasis['type'] + | MdFootnote['type'] + | MdFootnoteDefinition['type'] + | MdFootnoteReference['type'] + | MdHTML['type'] + | MdHeading['type'] + | MdImage['type'] + | MdImageReference['type'] + | MdInlineCode['type'] + | MdLink['type'] + | MdLinkReference['type'] + | MdList['type'] + | MdListItem['type'] + | MdParagraph['type'] + | MdStrong['type'] + | MdTable['type'] + | MdTableCell['type'] + | MdTableRow['type'] + | MdText['type'] + | MdThematicBreak['type'] + | MdYaml['type'] + | MdxFlowExpression['type'] + | MdxJsxAttribute['type'] + | MdxJsxAttributeValueExpression['type'] + | MdxJsxExpressionAttribute['type'] + | MdxJsxFlowElement['type'] + | MdxJsxTextElement['type'] + | MdxTextExpression['type'] + | MdxjsEsm['type']; + +export type { + MdRoot, + MdBlockquote, + MdBreak, + MdCode, + MdDefinition, + MdDelete, + MdEmphasis, + MdFootnote, + MdFootnoteDefinition, + MdFootnoteReference, + MdHTML, + MdHeading, + MdImage, + MdImageReference, + MdInlineCode, + MdLink, + MdLinkReference, + MdList, + MdListItem, + MdParagraph, + MdStrong, + MdTable, + MdTableCell, + MdTableRow, + MdText, + MdThematicBreak, + MdYaml +}; + +export type { + MdxFlowExpression, + MdxJsxAttribute, + MdxJsxAttributeValueExpression, + MdxJsxExpressionAttribute, + MdxJsxFlowElement, + MdxJsxTextElement, + MdxTextExpression, + MdxjsEsm +}; diff --git a/src/ast/unist.ts b/src/ast/unist.ts new file mode 100644 index 0000000..b578e00 --- /dev/null +++ b/src/ast/unist.ts @@ -0,0 +1,19 @@ +import type { Position as UnPosition } from 'unist'; + +export function unNodeIsParent(node: UnNode): node is UnParent { + return 'children' in node; +} + +/** + * ============================================================ + */ + +export interface UnNode { + type: string; + position?: UnPosition; + data?: unknown; +} + +export interface UnParent extends UnNode { + children: (UnNode | UnParent)[]; +} diff --git a/src/ast/unwalk.ts b/src/ast/unwalk.ts new file mode 100644 index 0000000..57cfbf2 --- /dev/null +++ b/src/ast/unwalk.ts @@ -0,0 +1,40 @@ +import { type UnNode, type UnParent, unNodeIsParent } from './unist.js'; + +const NEXT = true; +const STOP = false; + +export function unwalk( + node: UnNode, + visit: UnVisitor, + filter?: (node: UnNode, parent: UnParent | undefined) => boolean +) { + let next = true; + + function step(node: UnNode, parent: UnParent | undefined, index: number | undefined) { + if (filter && !filter(node, parent)) return; + + if (unNodeIsParent(node)) { + for (let i = 0; i < node.children.length; i++) { + if (!next) break; + + const child = node.children[i]; + step(child, node, i); + } + + node.children = node.children.filter((child) => child); + } + + if (!next) return; + + const signal = visit(node, parent, index); + next = signal === undefined || NEXT ? NEXT : STOP; + } + + step(node, undefined, undefined); +} + +export interface UnVisitor { + (node: UnNode | UnParent, parent: UnParent | undefined, index: number | undefined): + | boolean + | void; +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..6c10bbe --- /dev/null +++ b/src/config.ts @@ -0,0 +1,586 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +import type { MdNodeType } from './ast/mdast.js'; +import { isBoolean } from './utils.js'; + +export interface ConfigBase { + /** + * Source's language code. Based on DeepL supported languages. + */ + sourceLanguage: SourceLanguageCode; + /** + * Output's languages code. Based on DeepL supported languages. + */ + outputLanguages: TargetLanguageCode[]; + /** + * Sources and ouputs directories pairs. $langcode$ variable + * is provided to dynamically define directory. + * + * e.g. [ ["docs", "i18n/$langcode$/docs"], ["blog", "i18n/$langcode$/blog"] ] + */ + directories: [string, string][]; +} + +export interface Config extends ConfigBase { + /** + * Override current working directory, defaults to `process.cwd()`. + */ + cwd: string; + /** + * By default, all .md, .mdx, .json, and .yaml|.yml files inside + * source directories will be included. + * + * Define glob patterns to filter what files to include or exclude. + * But, the end result is still restricted by file types (.md, .mdx, .json). + */ + files: { + include?: string[]; + exclude: string[]; + }; + /** + * Frontmatter fields. + */ + frontmatterFields: { + include: string[]; + exclude: string[]; + }; + /** + * Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`. + */ + markdownNodes: { + default: boolean; + include: MdNodeType[]; + exclude: MdNodeType[]; + }; + /** + * HTML elements to include and exlcude, down to the level of attributes + * and children. Include all HTML elements text content + * and some global attributes such as title and placeholder. + */ + htmlElements: { + include: Partial<{ [Tag in HtmlTag]: { children: boolean; attributes: string[] } }>; + exclude: HtmlTag[]; + }; + /** + * JSX components to include and exclude, down to the level of attributes + * and children. Include all JSX components text children + * and exclude all attributes by default. + * + * Support array, object, and jsx attribute value. For object and array value, + * you can specify the access path starting with the attribute name + * e.g. `items.description` to translate `items={[{description: "..."}]}. + */ + jsxComponents: { + default: boolean; + include: { [Name: string]: { children: boolean; attributes: string[] } }; + exclude: string[]; + }; + /** + * JSON or YAML file properties to include and exclude. + * Exclude all properties by default. + */ + jsonOrYamlProperties: { + include: (string | number | symbol)[]; + exclude: (string | number | symbol)[]; + }; +} + +export interface UserConfig extends ConfigBase { + /** + * Override current working directory, defaults to `process.cwd()`. + */ + cwd?: string; + /** + * By default, all .md, .mdx, .json, and .yaml|.yml files inside + * source directories will be included. + * + * Define glob patterns to filter what files to include or exclude. + * But, the end result is still restricted by file types (.md, .mdx, .json). + */ + files?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Frontmatter fields. + */ + frontmatterFields?: { + include?: string[]; + exclude?: string[]; + }; + /** + * Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`. + */ + markdownNodes?: { + default?: boolean; + include?: MdNodeType[]; + exclude?: MdNodeType[]; + }; + /** + * HTML elements to include and exlcude, down to the level of attributes + * and children. Include all HTML elements text content + * and some global attributes such as title and placeholder. + */ + htmlElements?: { + default?: boolean; + include?: Partial<{ [Tag in HtmlTag]: { children: boolean; attributes: string[] } }>; + exclude?: HtmlTag[]; + }; + /** + * JSX components to include and exclude, down to the level of attributes + * and children. Include all JSX components text children + * and exclude all attributes by default. + * + * Support array, object, and jsx attribute value. For object and array value, + * you can specify the access path starting with the attribute name + * e.g. `items.description` to translate `items={[{description: "..."}]}. + */ + jsxComponents?: { + default?: boolean; + include?: { [Name: string]: { children: boolean; attributes: string[] } }; + exclude?: string[]; + }; + /** + * JSON or YAML file properties to include and exclude. + * Exclude all properties by default. + */ + jsonOrYamlProperties?: { + include?: string[]; + exclude?: string[]; + }; +} + + + +export type HtmlElementsConfig = { [Tag in HtmlTag]: { children: boolean; attributes: string[] } }; + +export const HTML_ELEMENTS_CONFIG: HtmlElementsConfig = getHtmlElementsConfig(); + +function getHtmlElementsConfig(): HtmlElementsConfig { + const includeChildren: HtmlTag[] = [ + 'a', + 'abbr', + 'address', + 'article', + 'aside', + 'audio', + 'b', + 'bdi', + 'bdo', + 'blockquote', + 'body', + 'button', + 'canvas', + 'caption', + 'cite', + 'col', + 'colgroup', + 'data', + 'datalist', + 'dd', + 'del', + 'details', + 'dfn', + 'dialog', + 'div', + 'dl', + 'dt', + 'em', + 'fieldset', + 'figcaption', + 'figure', + 'footer', + 'form', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'header', + 'html', + 'i', + 'input', + 'ins', + 'label', + 'legend', + 'li', + 'main', + 'mark', + 'meter', + 'nav', + 'ol', + 'optgroup', + 'output', + 'p', + 'progress', + 'q', + 'rp', + 's', + 'samp', + 'section', + 'select', + 'small', + 'span', + 'strong', + 'sub', + 'summary', + 'sup', + 'table', + 'tbody', + 'td', + 'template', + 'text-area', + 'tfoot', + 'th', + 'thead', + 'time', + 'title', + 'tr', + 'track', + 'u', + 'ul' + ]; + + const excludeChildren: HtmlTag[] = [ + 'area', + 'base', + 'br', + 'code', + 'embed', + 'head', + 'hr', + 'iframe', + 'img', + 'kbd', + 'link', + 'meta', + 'noscript', + 'object', + 'param', + 'picture', + 'pre', + 'rt', + 'ruby', + 'script', + 'source', + 'style', + 'svg', + 'var', + 'video', + 'qbr' + ]; + + const config: Partial = {}; + + for (const tag of includeChildren) { + config[tag] = { + children: true, + attributes: ['title'] + }; + } + + for (const tag of excludeChildren) { + config[tag] = { + children: false, + attributes: ['title'] + }; + } + + return config as HtmlElementsConfig; +} + +export const HTML_TAGS = Object.keys(HTML_ELEMENTS_CONFIG) as HtmlTag[]; + +export function isHtmlTag(name: string): name is HtmlTag { + return HTML_TAGS.includes(name as HtmlTag); +} + +export function resolveConfig({ + sourceLanguage, + outputLanguages, + directories, + cwd, + files, + markdownNodes, + frontmatterFields, + htmlElements, + jsxComponents, + jsonOrYamlProperties +}: UserConfig): Config { + return { + sourceLanguage, + outputLanguages, + directories, + cwd: cwd ?? '', + files: files + ? { + include: files.include, + exclude: files.exclude ?? [] + } + : { exclude: [] }, + markdownNodes: markdownNodes + ? { + default: isBoolean(markdownNodes.default) ? markdownNodes.default : true, + include: markdownNodes.include ?? [], + exclude: markdownNodes.exclude ?? ['code'] + } + : { default: true, include: [], exclude: ['code'] }, + frontmatterFields: frontmatterFields + ? { + include: frontmatterFields.include ?? [], + exclude: frontmatterFields.exclude ?? [] + } + : { include: [], exclude: [] }, + + htmlElements: htmlElements + ? { + include: htmlElements.include + ? (isBoolean(htmlElements.default) && htmlElements.default) || + htmlElements.default === undefined + ? { ...HTML_ELEMENTS_CONFIG, ...htmlElements.include } + : htmlElements.include + : isBoolean(htmlElements.default) && !htmlElements.default + ? {} + : HTML_ELEMENTS_CONFIG, + exclude: htmlElements.exclude ?? [] + } + : { include: HTML_ELEMENTS_CONFIG, exclude: [] }, + jsxComponents: jsxComponents + ? { + default: isBoolean(jsxComponents.default) ? jsxComponents.default : true, + include: jsxComponents.include ?? {}, + exclude: jsxComponents.exclude ?? [] + } + : { default: true, include: {}, exclude: [] }, + jsonOrYamlProperties: jsonOrYamlProperties + ? { include: jsonOrYamlProperties.include ?? [], exclude: jsonOrYamlProperties.exclude ?? [] } + : { include: [], exclude: [] } + }; +} + + + +export function isFrontmatterFieldIncluded({ + field, + config +}: { + field: string; + config: Config; +}): boolean { + return ( + !config.frontmatterFields.exclude.includes(field) && + config.frontmatterFields.include.includes(field) + ); +} + +export function isMarkdownNodeIncluded({ + type, + config +}: { + type: MdNodeType; + config: Config; +}): boolean { + return ( + !config.markdownNodes.exclude.includes(type) && + (config.markdownNodes.default || config.markdownNodes.include.includes(type)) + ); +} + +export function isHtmlElementIncluded({ tag, config }: { tag: HtmlTag; config: Config }): boolean { + return ( + !config.htmlElements.exclude.includes(tag) && + Object.keys(config.htmlElements.include).includes(tag) + ); +} + +export function isHtmlElementAttributeIncluded({ + tag, + attribute, + config +}: { + tag: HtmlTag; + attribute: string; + config: Config; +}): boolean { + return ( + isHtmlElementIncluded({ tag, config }) && + config.htmlElements.include[tag]!.attributes.includes(attribute) + ); +} + +export function isHtmlElementChildrenIncluded({ + tag, + config +}: { + tag: HtmlTag; + config: Config; +}): boolean { + return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag]!.children; +} + +export function isJsxComponentIncluded({ + name, + config +}: { + name: string; + config: Config; +}): boolean { + return ( + !config.jsxComponents.exclude.includes(name) && + (config.jsxComponents.default || Object.keys(config.jsxComponents.include).includes(name)) + ); +} + +export function isJsxComponentAttributeIncluded({ + name, + attribute, + config +}: { + name: string; + attribute: string; + config: Config; +}): boolean { + return ( + !config.jsxComponents.exclude.includes(name) && + Object.keys(config.jsxComponents.include).includes(name) && + config.jsxComponents.include[name].attributes.includes(attribute) + ); +} + +export function isJsxComponentChildrenIncluded({ + name, + config +}: { + name: string; + config: Config; +}): boolean { + return ( + !config.jsxComponents.exclude.includes(name) && + ((Object.keys(config.jsxComponents.include).includes(name) && + config.jsxComponents.include[name].children) || + (!Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.default)) + ); +} + +export function isJsonOrYamlPropertyIncluded({ + property, + config +}: { + config: Config; + property: string | number | symbol; +}): boolean { + return ( + !config.jsonOrYamlProperties.exclude.includes(property) && + config.jsonOrYamlProperties.include.includes(property) + ); +} + +export type HtmlTag = + | 'a' + | 'abbr' + | 'address' + | 'article' + | 'aside' + | 'audio' + | 'b' + | 'bdi' + | 'bdo' + | 'blockquote' + | 'body' + | 'button' + | 'canvas' + | 'caption' + | 'cite' + | 'col' + | 'colgroup' + | 'data' + | 'datalist' + | 'dd' + | 'del' + | 'details' + | 'dfn' + | 'dialog' + | 'div' + | 'dl' + | 'dt' + | 'em' + | 'fieldset' + | 'figcaption' + | 'figure' + | 'footer' + | 'form' + | 'h1' + | 'h2' + | 'h3' + | 'h4' + | 'h5' + | 'h6' + | 'header' + | 'html' + | 'i' + | 'input' + | 'ins' + | 'label' + | 'legend' + | 'li' + | 'main' + | 'mark' + | 'meter' + | 'nav' + | 'ol' + | 'optgroup' + | 'output' + | 'p' + | 'progress' + | 'q' + | 'rp' + | 's' + | 'samp' + | 'section' + | 'select' + | 'small' + | 'span' + | 'strong' + | 'sub' + | 'summary' + | 'sup' + | 'table' + | 'tbody' + | 'td' + | 'template' + | 'text-area' + | 'tfoot' + | 'th' + | 'thead' + | 'time' + | 'title' + | 'tr' + | 'track' + | 'u' + | 'ul' + | 'area' + | 'base' + | 'br' + | 'code' + | 'embed' + | 'head' + | 'hr' + | 'iframe' + | 'img' + | 'kbd' + | 'link' + | 'meta' + | 'noscript' + | 'object' + | 'param' + | 'picture' + | 'pre' + | 'rt' + | 'ruby' + | 'script' + | 'source' + | 'style' + | 'svg' + | 'var' + | 'video' + | 'qbr'; diff --git a/src/extract.ts b/src/extract.ts new file mode 100644 index 0000000..d856b60 --- /dev/null +++ b/src/extract.ts @@ -0,0 +1,267 @@ +import { parse as parseYaml } from 'yaml'; +import { + EsJsxElement, + EsJsxIdentifier, + esNodeIs, + resolveEstreePropertyPath +} from './ast/estree.js'; +import { eswalk } from './ast/eswalk.js'; +import { mdNodeIs, mdNodeIsJsxElement, MdNodeType } from './ast/mdast.js'; +import type { UnNode } from './ast/unist.js'; +import { unwalk } from './ast/unwalk.js'; +import { + type Config, + isHtmlTag, + isFrontmatterFieldIncluded, + isHtmlElementIncluded, + isHtmlElementAttributeIncluded, + isJsonOrYamlPropertyIncluded, + isJsxComponentIncluded, + isJsxComponentAttributeIncluded, + isMarkdownNodeIncluded, + isHtmlElementChildrenIncluded, + isJsxComponentChildrenIncluded +} from './config.js'; +import { isArray, isEmptyArray, isEmptyString, isObject, isString } from './utils.js'; + +export function extractMdastStrings({ + mdast, + config +}: { + mdast: UnNode; + config: Config; +}): string[] { + const strings: string[] = []; + + unwalk( + mdast, + (node, __, _) => { + if (mdNodeIs(node, 'text')) { + pushTidyString({ array: strings, string: node.value }); + return; + } + + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue; + + if ( + !isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config }) + ) + continue; + + if (isString(attribute.value)) { + strings.push(attribute.value.trim()); + } else if (attribute.value?.data?.estree) { + const estree = attribute.value.data.estree; + + eswalk(estree, { + SimpleLiteral(esnode, _) { + if (isString(esnode.value)) + pushTidyString({ array: strings, string: esnode.value }); + } + }); + } + } + } else { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue; + + const componentName: string = node.name; + + const isAttributeIncluded = isJsxComponentAttributeIncluded({ + name: componentName, + attribute: attribute.name, + config + }); + + if (isString(attribute.value)) { + if (!isAttributeIncluded) continue; + + strings.push(attribute.value.trim()); + } else if (attribute.value?.data?.estree) { + if ( + !config.jsxComponents.include[componentName] || + !config.jsxComponents.include[componentName].attributes.some( + (attrName) => + attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + ) + ) + continue; + + const estree = attribute.value.data.estree; + + eswalk(estree, { + SimpleLiteral(esnode, _) { + if (isString(esnode.value)) + pushTidyString({ array: strings, string: esnode.value }); + + if (esnode.value === 'aye') console.log('passed'); + }, + JSXElement(esnode, _) { + const name = (esnode.openingElement.name as EsJsxIdentifier).name; + + if (isHtmlTag(name)) { + if ( + !isHtmlElementIncluded({ tag: name, config }) || + !isHtmlElementChildrenIncluded({ tag: name, config }) + ) + return false; + } else if ( + !isJsxComponentIncluded({ name: name, config }) || + !isJsxComponentChildrenIncluded({ name: name, config }) + ) + return false; + }, + JSXAttribute(esnode, parents) { + const name = + typeof esnode.name.name === 'string' ? esnode.name.name : esnode.name.name.name; + const parentName = ( + (parents[parents.length - 1] as EsJsxElement).openingElement + .name as EsJsxIdentifier + ).name; + + if (isHtmlTag(parentName)) { + if ( + !isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config }) + ) + return false; + } else if ( + !config.jsxComponents.include[name] || + !config.jsxComponents.include[name].attributes.some( + (attrName) => + attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + ) + ) { + return false; + } + }, + JSXText(esnode, _) { + pushTidyString({ array: strings, string: esnode.value }); + }, + Property(esnode, parents) { + if (!esNodeIs(esnode, 'Identifier')) return false; + + const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name); + + if ( + !propertyPath || + !isJsxComponentAttributeIncluded({ + name: componentName, + attribute: propertyPath, + config + }) + ) + return false; + } + }); + } + } + } + } + + if (mdNodeIs(node, 'yaml')) { + if (isEmptyArray(config.frontmatterFields.include)) return; + if (isEmptyString(node.value)) return; + + const object: Record = parseYaml(node.value); + + for (const field in object) { + if (!isFrontmatterFieldIncluded({ field, config })) continue; + + const value = object[field]; + + if (isString(value)) { + strings.push(value); + continue; + } + + if (isArray(value)) { + for (const item of value) { + if (!isString(item)) continue; + strings.push(item); + } + } + } + + return; + } + }, + (node, parent) => { + if (!isMarkdownNodeIncluded({ type: node.type as MdNodeType, config })) return false; + + if (parent && mdNodeIsJsxElement(parent) && parent.name) { + if (isHtmlTag(parent.name)) { + if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false; + } else { + if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false; + } + + return true; + } + + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + if (!isHtmlElementIncluded({ tag: node.name, config })) return false; + } else { + if (!isJsxComponentIncluded({ name: node.name, config })) return false; + } + + return true; + } + + return true; + } + ); + + return strings; +} + +export function extractJsonOrYamlStrings({ + source, + type = 'json', + config +}: { + source: string; + type?: 'json' | 'yaml'; + config: Config; +}): string[] { + const strings: string[] = []; + + if (isEmptyArray(config.jsonOrYamlProperties.include)) return strings; + + const parsed = type === 'json' ? JSON.parse(source) : parseYaml(source); + + process(parsed); + + function process(value: unknown, property?: string) { + if (typeof value === 'string') { + if (property && isJsonOrYamlPropertyIncluded({ property, config })) strings.push(value); + return; + } + + if (isArray(value)) { + for (const item of value) { + process(item); + } + return; + } + + if (isObject(value)) { + for (const property in value) { + const item = (value as Record)[property]; + process(item, property); + } + return; + } + } + + return strings; +} + +function pushTidyString({ array, string }: { array: string[]; string: string }) { + if (!/^\s*$/.test(string)) { + array.push(string.replace(/(^\n|\r|\t|\v)+\s*/, '').replace(/\s+$/, ' ')); + } +} diff --git a/src/format.ts b/src/format.ts new file mode 100644 index 0000000..3d28fe2 --- /dev/null +++ b/src/format.ts @@ -0,0 +1,42 @@ +import prettier from 'prettier'; +import { getMarkdown, getMdast, mdNodeIs } from './ast/mdast.js'; +import { unwalk } from './ast/unwalk.js'; + +export async function format(markdown: string) { + /** + * `printWidth` is set to Infinity and `proseWrap` is set to never + * to avoid unnecessary linebreaks that break translation result + */ + const mdast = getMdast( + prettier.format(markdown, { + parser: 'mdx', + printWidth: Infinity, + proseWrap: 'never', + useTabs: true + }) + ); + + /** + * remove empty surface flow expression nodes that sometimes + * are produced by prettier + */ + unwalk( + mdast, + (node, parent, index) => { + if (mdNodeIs(node, 'mdxFlowExpression') && expressionIsEmpty(node.value)) { + (parent!.children[index!] as unknown) = undefined; + } + }, + (node, parent) => { + delete node.position; + return mdNodeIs(parent, 'root'); + } + ); + + return getMarkdown(mdast); +} + +function expressionIsEmpty(text: string): boolean { + const regex = /^('|")\s*('|")$/; + return regex.test(text); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..24c12dd --- /dev/null +++ b/src/index.ts @@ -0,0 +1,54 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +import { getMarkdown, getMdast } from './ast/mdast.js'; +import { resolveConfig } from './config.js'; +import { extractMdastStrings } from './extract.js'; +import { format } from './format.js'; +import { replaceMdastStrings } from './replace.js'; +import { translateStrings } from './translate.js'; + +/** + * Translate markdown/MDX content from one language to another using DeepL. + * + * Requires `DEEPL_AUTH_KEY` environment variable to be set. + * + * @param content - Markdown or MDX string to translate + * @param sourceLang - Source language code (e.g. 'en', 'de', 'fr') + * @param targetLang - Target language code (e.g. 'de', 'en-US', 'fr') + * @returns Translated markdown string + * + * @example + * ```ts + * import { translate } from 'deepmark'; + * + * const result = await translate('# Hello World', 'en', 'de'); + * console.log(result); // '# Hallo Welt' + * ``` + */ +export async function translate( + content: string, + sourceLang: SourceLanguageCode, + targetLang: TargetLanguageCode +): Promise { + // Build a default config suitable for general markdown translation + const config = resolveConfig({ + sourceLanguage: sourceLang, + outputLanguages: [targetLang], + directories: [['', '']] + }); + + // Format, parse, extract translatable strings + const formatted = await format(content); + const mdast = getMdast(formatted); + const strings = extractMdastStrings({ mdast, config }); + + if (strings.length === 0) return content; + + // Translate via DeepL + const translated = await translateStrings(strings, sourceLang, targetLang); + + // Replace strings in the AST and serialize back to markdown + const result = replaceMdastStrings({ mdast, strings: translated, config }); + return getMarkdown(result); +} + +export type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; diff --git a/src/lint.ts b/src/lint.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/replace.ts b/src/replace.ts new file mode 100644 index 0000000..76a12b3 --- /dev/null +++ b/src/replace.ts @@ -0,0 +1,296 @@ +import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; +import { + EsJsxElement, + EsJsxIdentifier, + esNodeIs, + resolveEstreePropertyPath +} from './ast/estree.js'; +import { eswalk } from './ast/eswalk.js'; +import type { MdNodeType, MdRoot } from './ast/mdast.js'; +import { mdNodeIs, mdNodeIsJsxElement } from './ast/mdast.js'; +import { unwalk } from './ast/unwalk.js'; +import { + type Config, + isHtmlTag, + isFrontmatterFieldIncluded, + isHtmlElementIncluded, + isHtmlElementAttributeIncluded, + isJsonOrYamlPropertyIncluded, + isJsxComponentIncluded, + isJsxComponentAttributeIncluded, + isMarkdownNodeIncluded, + isHtmlElementChildrenIncluded, + isJsxComponentChildrenIncluded +} from './config.js'; +import { isArray, isEmptyArray, isEmptyString, isObject, isString } from './utils.js'; + +export function replaceMdastStrings({ + mdast, + config, + strings +}: { + mdast: MdRoot; + strings: string[]; + config: Config; +}): MdRoot { + strings = strings.reverse(); + + unwalk( + mdast, + (node, __, _) => { + if (mdNodeIs(node, 'text')) { + node.value = strings.pop()!; + return; + } + + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue; + + if ( + !isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config }) + ) + continue; + + if (isString(attribute.value)) { + attribute.value = strings.pop(); + } else if (attribute.value?.data?.estree) { + const estree = attribute.value.data.estree; + + eswalk(estree, { + SimpleLiteral(esnode, _) { + if (isString(esnode.value)) esnode.value = strings.pop()!; + } + }); + } + } + } else { + for (const attribute of node.attributes) { + if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue; + + const componentName: string = node.name; + + const isAttributeIncluded = isJsxComponentAttributeIncluded({ + name: componentName, + attribute: attribute.name, + config + }); + + if (isString(attribute.value)) { + if (!isAttributeIncluded) continue; + + attribute.value = strings.pop(); + } else if (attribute.value?.data?.estree) { + if ( + !config.jsxComponents.include[componentName] || + !config.jsxComponents.include[componentName].attributes.some( + (attrName) => + attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + ) + ) + continue; + + const estree = attribute.value.data.estree; + + eswalk(estree, { + SimpleLiteral(esnode, _) { + if (isString(esnode.value)) esnode.value = strings.pop()!; + }, + JSXElement(esnode, _) { + const name = (esnode.openingElement.name as EsJsxIdentifier).name; + + if (isHtmlTag(name)) { + if ( + !isHtmlElementIncluded({ tag: name, config }) || + !isHtmlElementChildrenIncluded({ tag: name, config }) + ) + return false; + } else if ( + !isJsxComponentIncluded({ name: name, config }) || + !isJsxComponentChildrenIncluded({ name: name, config }) + ) + return false; + }, + JSXAttribute(esnode, parents) { + const name = + typeof esnode.name.name === 'string' ? esnode.name.name : esnode.name.name.name; + const parentName = ( + (parents[parents.length - 1] as EsJsxElement).openingElement + .name as EsJsxIdentifier + ).name; + + if (isHtmlTag(parentName)) { + if ( + !isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config }) + ) + return false; + } else if ( + !config.jsxComponents.include[name] || + !config.jsxComponents.include[name].attributes.some( + (attrName) => + attrName === attribute.name || attrName.startsWith(`${attribute.name}.`) + ) + ) { + return false; + } + }, + JSXText(esnode, _) { + esnode.value = strings.pop()!; + }, + Property(esnode, parents) { + if (!esNodeIs(esnode, 'Identifier')) return false; + + const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name); + + if ( + !propertyPath || + !isJsxComponentAttributeIncluded({ + name: componentName, + attribute: propertyPath, + config + }) + ) + return false; + } + }); + } + } + } + } + + if (mdNodeIs(node, 'yaml')) { + if (isEmptyArray(config.frontmatterFields.include)) return; + if (isEmptyString(node.value)) return; + + const object: Record = parseYaml(node.value); + + for (const field in object) { + if (!isFrontmatterFieldIncluded({ field, config })) continue; + + const value = object[field]; + + if (isString(value)) { + object[field] = strings.pop(); + continue; + } + + if (isArray(value)) { + for (const [index, item] of value.entries()) { + if (!isString(item)) continue; + value[index] = strings.pop(); + } + } + } + + return; + } + }, + (node, parent) => { + if (!isMarkdownNodeIncluded({ type: node.type as MdNodeType, config })) return false; + + if (parent && mdNodeIsJsxElement(parent) && parent.name) { + if (isHtmlTag(parent.name)) { + if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false; + } else { + if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false; + } + + return true; + } + + if (mdNodeIsJsxElement(node) && node.name) { + if (isHtmlTag(node.name)) { + if (!isHtmlElementIncluded({ tag: node.name, config })) return false; + } else { + if (!isJsxComponentIncluded({ name: node.name, config })) return false; + } + + return true; + } + + return true; + } + ); + + return mdast; +} + +export function replaceJsonOrYamlStrings({ + source, + type = 'json', + strings, + config +}: { + source: string; + type?: 'json' | 'yaml'; + strings: string[]; + config: Config; +}): string { + if (isEmptyArray(config.jsonOrYamlProperties.include)) return source; + + strings = strings.reverse(); + const parsed = type === 'json' ? JSON.parse(source) : parseYaml(source); + + process({ value: parsed }); + + function process(args: { value: unknown; parent?: never; property?: never; index?: never }): void; + function process(args: { + value: unknown; + parent: unknown[]; + property?: string | number | symbol; + index: number; + }): void; + function process(args: { + value: unknown; + parent: Record; + property: string | number | symbol; + index?: never; + }): void; + function process({ + value, + parent, + property, + index + }: { + value: unknown; + parent?: unknown[] | Record; + property?: string | number | symbol; + index?: number; + }) { + if (isArray(value)) { + for (const [index, item] of value.entries()) { + process({ value: item, parent: value, property, index }); + } + return; + } + + if (isObject(value)) { + for (const property in value) { + const item = (value as Record)[property]; + process({ value: item, parent: value, property }); + } + return; + } + + if (typeof value === 'string') { + if (property && isJsonOrYamlPropertyIncluded({ property, config })) { + if (isArray(parent) && index) { + parent[index] = strings.pop(); + + return; + } + + if (isObject(parent)) { + parent[property] = strings.pop(); + + return; + } + } + return; + } + } + + if (type === 'json') return JSON.stringify(parsed); + return stringifyYaml(parsed); +} diff --git a/src/translate.ts b/src/translate.ts new file mode 100644 index 0000000..4e0e02b --- /dev/null +++ b/src/translate.ts @@ -0,0 +1,49 @@ +import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node'; +import { Translator } from 'deepl-node'; + +/** + * Translate an array of strings from sourceLang to targetLang using DeepL. + * Batches requests in groups of 10. + * Requires DEEPL_AUTH_KEY environment variable. + */ +export async function translateStrings( + strings: string[], + sourceLang: SourceLanguageCode, + targetLang: TargetLanguageCode +): Promise { + if (strings.length === 0) return []; + + const DEEPL_AUTH_KEY = process.env.DEEPL_AUTH_KEY; + if (!DEEPL_AUTH_KEY) throw new Error('DEEPL_AUTH_KEY environment variable must be set'); + + const deepl = new Translator(DEEPL_AUTH_KEY); + const translations: string[] = new Array(strings.length).fill(''); + const queue: [index: number, string: string][] = []; + + for (const [index, string] of strings.entries()) { + queue.push([index, string]); + + if (index === strings.length - 1 || queue.length === 10) { + const indexes = queue.map(([i]) => i); + const batch = queue.map(([, s]) => s); + + const results = await deepl.translateText( + batch, + sourceLang, + targetLang, + { + tagHandling: 'html', + splitSentences: 'nonewlines' + } + ); + + for (let j = 0; j < indexes.length; j++) { + translations[indexes[j]] = results[j].text; + } + + queue.length = 0; + } + } + + return translations; +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..48908a7 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,27 @@ +export function isArray(value: unknown): value is any[] { + return Array.isArray(value); +} + +export function isBoolean(value: unknown): value is boolean { + return typeof value === 'boolean'; +} + +export function isEmptyArray(array: any[]): boolean { + return array.length === 0; +} + +export function isEmptyObject(object: Object): boolean { + return Object.keys(object).length === 0; +} + +export function isEmptyString(string: string): boolean { + return string.length === 0; +} + +export function isObject(value: unknown): value is Record { + return isArray(value) ? false : typeof value == 'object' ? true : false; +} + +export function isString(value: unknown): value is string { + return typeof value === 'string'; +} diff --git a/src/vendor/mdast-util-html-comment.ts b/src/vendor/mdast-util-html-comment.ts new file mode 100644 index 0000000..dbe45c5 --- /dev/null +++ b/src/vendor/mdast-util-html-comment.ts @@ -0,0 +1,40 @@ +// @ts-nocheck — vendored from mdast-util-html-comment, uses custom handler types +import type { Extension } from 'mdast-util-from-markdown'; +import type { Options } from 'mdast-util-to-markdown'; + +export function htmlCommentFromMarkdown(): Extension { + return { + canContainEols: ['htmlComment'], + enter: { + htmlComment() { + this.buffer(); + } + }, + exit: { + htmlComment(token) { + const string = this.resume(); + + this.enter( + { + // @ts-ignore + type: 'htmlComment', + value: string.slice(0, -3) + }, + token + ); + + this.exit(token); + } + } + }; +} + +export function htmlCommentToMarkdown(): Options { + return { + handlers: { + htmlComment(node) { + return ``; + } + } + }; +} diff --git a/src/vendor/micromark-extension-html-comment.ts b/src/vendor/micromark-extension-html-comment.ts new file mode 100644 index 0000000..d0b2566 --- /dev/null +++ b/src/vendor/micromark-extension-html-comment.ts @@ -0,0 +1,128 @@ +// @ts-nocheck — vendored from micromark-extension-html-comment, uses custom token types +import { factorySpace } from 'micromark-factory-space'; +import { markdownLineEnding } from 'micromark-util-character'; +import { codes } from 'micromark-util-symbol/codes.js'; +import { types } from 'micromark-util-symbol/types.js'; +import type { Code, Extension, HtmlExtension, State, Tokenizer } from 'micromark-util-types'; + +export function htmlComment(): Extension { + return { + flow: { + [codes.lessThan]: { tokenize, concrete: true } + }, + text: { + [codes.lessThan]: { tokenize } + } + }; +} + +export function htmlCommentToHtml(): HtmlExtension { + return { + enter: { + htmlComment() { + this.buffer(); + } + }, + exit: { + htmlComment() { + this.resume(); + } + } + }; +} + +const tokenize: Tokenizer = (effects, ok, nok) => { + let value: string = ''; + + return start; + + function start(code: Code): State | void { + effects.enter('htmlComment'); + effects.enter('htmlCommentMarker'); + effects.consume(code); + value += '<'; + + return open; + } + + function open(code: Code): State | void { + if (value === '<' && code === codes.exclamationMark) { + effects.consume(code); + value += '!'; + return open; + } + + if (code === codes.dash) { + if (value === '= 6 && value.slice(-2) === '--') { + effects.consume(code); + effects.exit(types.data); + effects.exit('htmlCommentString'); + effects.exit('htmlComment'); + value += '>'; + + return ok; + } + + return nok(code); + } +}; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..0052f00 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "forceConsistentCasingInFileNames": true, + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": true, + "emitDeclarationOnly": true, + "lib": [ + "ESNext" + ], + "noEmit": false, + "outDir": "./dist", + "rootDir": "src", + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "target": "ESNext" + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/__test__/**" + ] +} \ No newline at end of file diff --git a/vite.config.js b/vite.config.js new file mode 100644 index 0000000..50a8759 --- /dev/null +++ b/vite.config.js @@ -0,0 +1,13 @@ +import np from 'path'; + +const CWD = process.cwd(); + +/** @type { import('vite').UserConfig } */ +export default { + resolve: { + alias: { + $types: np.resolve(CWD, './src/types/index.js'), + $utils: np.resolve(CWD, './src/utils/index.js') + } + } +};