deepl-mark

This commit is contained in:
lovebird 2026-04-05 14:09:39 +02:00
parent b3a64b8851
commit ef493ae64a
89 changed files with 9192 additions and 46 deletions

5
packages/deepl-mark/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
/node_modules
/coverage
*.log
.DS_Store
.env

View File

@ -0,0 +1,4 @@
./docs
./scripts
./tests
./incoming

View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Izzuddin Natsir
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1,119 @@
# @polymech/deepl-mark
Translate markdown and MDX content using [DeepL](https://www.deepl.com/), powered by `mdast`.
Correctly handles headings, paragraphs, lists, tables (GFM), links, JSX components, frontmatter, and inline formatting — preserving structure while translating only the text.
## Install
```bash
npm install @polymech/deepl-mark
```
## Usage
```ts
import { translate } from '@polymech/deepl-mark';
const markdown = '# Hello World\n\nThis is a paragraph.';
const result = await translate(markdown, 'en', 'de');
console.log(result);
// # Hallo Welt
//
// Dies ist ein Absatz.
```
### Authentication
Provide your DeepL API key via **options** or **environment variable**:
```ts
// Option 1: pass directly
await translate(md, 'en', 'de', { apiKey: 'your-deepl-key' });
// Option 2: environment variable
// Set DEEPL_AUTH_KEY=your-deepl-key
await translate(md, 'en', 'de');
```
### Options
The optional 4th argument accepts a `TranslateOptions` object:
```ts
await translate(content, 'en', 'de', {
// DeepL API key (falls back to DEEPL_AUTH_KEY env var)
apiKey: '...',
// DeepL translation options (tagHandling, splitSentences, formality, glossaryId, etc.)
deeplOptions: {
formality: 'more',
glossaryId: '...',
},
// Frontmatter fields to include/exclude
frontmatterFields: {
include: ['title', 'description'],
exclude: ['slug'],
},
// Markdown node types to include/exclude (defaults: exclude 'code')
markdownNodes: {
exclude: ['code'],
},
// HTML elements to include/exclude
htmlElements: {
exclude: ['pre', 'code'],
},
// JSX components to include/exclude (with attribute-level control)
jsxComponents: {
include: {
Card: { children: true, attributes: ['header'] },
},
},
});
```
#### DeepL defaults
The following DeepL options are applied by default and can be overridden via `deeplOptions`:
| Option | Default |
|------------------|----------------|
| `tagHandling` | `'html'` |
| `splitSentences` | `'nonewlines'` |
### Supported content
- **Markdown** (`.md`) — headings, paragraphs, lists, blockquotes, tables (GFM), links, images
- **MDX** (`.mdx`) — JSX components and expressions
- **Frontmatter** — YAML frontmatter fields
- **HTML** — inline HTML elements and attributes
## API
### `translate(content, sourceLang, targetLang, options?)`
| Parameter | Type | Description |
|--------------|------------------------|-----------------------------------------------|
| `content` | `string` | Markdown or MDX string to translate |
| `sourceLang` | `SourceLanguageCode` | Source language (e.g. `'en'`, `'de'`, `'fr'`) |
| `targetLang` | `TargetLanguageCode` | Target language (e.g. `'de'`, `'en-US'`) |
| `options` | `TranslateOptions` | Optional config (see above) |
Returns `Promise<string>` — the translated markdown.
## Scripts
```bash
npm test # run all tests
npm run test:tables # run table translation e2e test
npm run build # build for distribution
```
## License
MIT

View File

@ -0,0 +1,44 @@
import { transform } from 'esbuild';
import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
import { join, dirname, relative } from 'node:path';
const SRC = 'src';
const DIST = 'dist';
async function getFiles(dir) {
const entries = await readdir(dir, { withFileTypes: true });
const files = [];
for (const entry of entries) {
const full = join(dir, entry.name);
if (entry.isDirectory()) {
if (entry.name === '__test__' || entry.name === 'types') continue;
files.push(...(await getFiles(full)));
} else if (entry.name.endsWith('.ts')) {
files.push(full);
}
}
return files;
}
async function main() {
const files = await getFiles(SRC);
for (const file of files) {
const input = await readFile(file, 'utf-8');
const { code } = await transform(input, {
format: 'esm',
loader: 'ts',
target: 'es2021'
});
const rel = relative(SRC, file).replace(/\.ts$/, '.js');
const outPath = join(DIST, rel);
await mkdir(dirname(outPath), { recursive: true });
await writeFile(outPath, code);
}
console.log(`Built ${files.length} files to ${DIST}/`);
}
main();

115
packages/deepl-mark/dist/ast/estree.d.ts vendored Normal file
View File

@ -0,0 +1,115 @@
import type { BaseNode as EsBaseNode, Identifier as EsIdentifier, Program as EsProgram, SwitchCase as EsSwitchCase, CatchClause as EsCatchClause, VariableDeclarator as EsVariableDeclarator, ExpressionStatement as EsExpressionStatement, BlockStatement as EsBlockStatement, EmptyStatement as EsEmptyStatement, DebuggerStatement as EsDebuggerStatement, WithStatement as EsWithStatement, ReturnStatement as EsReturnStatement, LabeledStatement as EsLabeledStatement, BreakStatement as EsBreakStatement, ContinueStatement as EsContinueStatement, IfStatement as EsIfStatement, SwitchStatement as EsSwitchStatement, ThrowStatement as EsThrowStatement, TryStatement as EsTryStatement, WhileStatement as EsWhileStatement, DoWhileStatement as EsDoWhileStatement, ForStatement as EsForStatement, ForInStatement as EsForInStatement, ForOfStatement as EsForOfStatement, ClassDeclaration as EsClassDeclaration, FunctionDeclaration as EsFunctionDeclaration, VariableDeclaration as EsVariableDeclaration, ModuleDeclaration as EsModuleDeclaration, ImportDeclaration as EsImportDeclaration, ExportDefaultDeclaration as EsExportDefaultDeclaration, ExportNamedDeclaration as EsExportNamedDeclaration, ExportAllDeclaration as EsExportAllDeclaration, ThisExpression as EsThisExpression, ArrayExpression as EsArrayExpression, ObjectExpression as EsObjectExpression, FunctionExpression as EsFunctionExpression, ArrowFunctionExpression as EsArrowFunctionExpression, YieldExpression as EsYieldExpression, UnaryExpression as EsUnaryExpression, UpdateExpression as EsUpdateExpression, BinaryExpression as EsBinaryExpression, AssignmentExpression as EsAssignmentExpression, LogicalExpression as EsLogicalExpression, MemberExpression as EsMemberExpression, ConditionalExpression as EsConditionalExpression, CallExpression as EsCallExpression, NewExpression as EsNewExpression, SequenceExpression as EsSequenceExpression, TaggedTemplateExpression as EsTaggedTemplateExpression, ClassExpression as EsClassExpression, AwaitExpression as EsAwaitExpression, ImportExpression as EsImportExpression, ChainExpression as EsChainExpression, SimpleLiteral as EsSimpleLiteral, RegExpLiteral as EsRegExpLiteral, BigIntLiteral as EsBigIntLiteral, TemplateLiteral as EsTemplateLiteral, PrivateIdentifier as EsPrivateIdentifier, Property as EsProperty, MetaProperty as EsMetaProperty, PropertyDefinition as EsPropertyDefinition, AssignmentProperty as EsAssignmentProperty, Super as EsSuper, TemplateElement as EsTemplateElement, SpreadElement as EsSpreadElement, ObjectPattern as EsObjectPattern, ArrayPattern as EsArrayPattern, RestElement as EsRestElement, AssignmentPattern as EsAssignmentPattern, Class as EsClass, ClassBody as EsClassBody, StaticBlock as EsStaticBlock, MethodDefinition as EsMethodDefinition, ModuleSpecifier as EsModuleSpecifier, ImportSpecifier as EsImportSpecifier, ImportNamespaceSpecifier as EsImportNamespaceSpecifier, ImportDefaultSpecifier as EsImportDefaultSpecifier, ExportSpecifier as EsExportSpecifier } from 'estree';
import type { JSXAttribute as EsJsxAttribute, JSXClosingElement as EsJsxClosingElement, JSXClosingFragment as EsJsxClosingFragment, JSXElement as EsJsxElement, JSXEmptyExpression as EsJsxEmptyExpression, JSXExpressionContainer as EsJsxExpressionContainer, JSXFragment as EsJsxFragment, JSXIdentifier as EsJsxIdentifier, JSXMemberExpression as EsJsxMemberExpression, JSXNamespacedName as EsJsxNamespacedName, JSXOpeningElement as EsJsxOpeningElement, JSXOpeningFragment as EsJsxOpeningFragment, JSXSpreadAttribute as EsJsxSpreadAttribute, JSXSpreadChild as EsJsxSpreadChild, JSXText as EsJsxText } from 'estree-jsx';
export declare function esNodeIs<T extends keyof EsNodeMap>(node: EsNode, type: T): node is EsNodeMap[T];
export declare function resolveEstreePropertyPath(node: EsProperty, parents: EsNode[], attributeName: string): string | undefined;
/**
* ============================================================
*/
export type { EsBaseNode, EsIdentifier, EsProgram, EsSwitchCase, EsCatchClause, EsVariableDeclarator, EsExpressionStatement, EsBlockStatement, EsEmptyStatement, EsDebuggerStatement, EsWithStatement, EsReturnStatement, EsLabeledStatement, EsBreakStatement, EsContinueStatement, EsIfStatement, EsSwitchStatement, EsThrowStatement, EsTryStatement, EsWhileStatement, EsDoWhileStatement, EsForStatement, EsForInStatement, EsForOfStatement, EsClassDeclaration, EsFunctionDeclaration, EsVariableDeclaration, EsModuleDeclaration, EsImportDeclaration, EsExportDefaultDeclaration, EsExportNamedDeclaration, EsExportAllDeclaration, EsThisExpression, EsArrayExpression, EsObjectExpression, EsFunctionExpression, EsArrowFunctionExpression, EsYieldExpression, EsUnaryExpression, EsUpdateExpression, EsBinaryExpression, EsAssignmentExpression, EsLogicalExpression, EsMemberExpression, EsConditionalExpression, EsCallExpression, EsNewExpression, EsSequenceExpression, EsTaggedTemplateExpression, EsClassExpression, EsAwaitExpression, EsImportExpression, EsChainExpression, EsSimpleLiteral, EsRegExpLiteral, EsBigIntLiteral, EsTemplateLiteral, EsPrivateIdentifier, EsProperty, EsMetaProperty, EsPropertyDefinition, EsAssignmentProperty, EsSuper, EsTemplateElement, EsSpreadElement, EsObjectPattern, EsArrayPattern, EsRestElement, EsAssignmentPattern, EsClass, EsClassBody, EsStaticBlock, EsMethodDefinition, EsModuleSpecifier, EsImportSpecifier, EsImportNamespaceSpecifier, EsImportDefaultSpecifier, EsExportSpecifier };
export type { EsJsxAttribute, EsJsxClosingElement, EsJsxClosingFragment, EsJsxElement, EsJsxEmptyExpression, EsJsxExpressionContainer, EsJsxFragment, EsJsxIdentifier, EsJsxMemberExpression, EsJsxNamespacedName, EsJsxOpeningElement, EsJsxOpeningFragment, EsJsxSpreadAttribute, EsJsxSpreadChild, EsJsxText };
export type EsNode = EsNodeMap[keyof EsNodeMap];
export type EsNodeMap = EsExpressionMap & EsLiteralMap & EsFunctionMap & EsPatternMap & EsStatementMap & EsJsxMap & {
CatchClause: EsCatchClause;
Class: EsClass;
ClassBody: EsClassBody;
MethodDefinition: EsMethodDefinition;
ModuleDeclaration: EsModuleDeclaration;
ModuleSpecifier: EsModuleSpecifier;
PrivateIdentifier: EsPrivateIdentifier;
Program: EsProgram;
Property: EsProperty;
PropertyDefinition: EsPropertyDefinition;
SpreadElement: EsSpreadElement;
Super: EsSuper;
SwitchCase: EsSwitchCase;
TemplateElement: EsTemplateElement;
VariableDeclarator: EsVariableDeclarator;
};
export type EsExpressionMap = EsLiteralMap & {
ArrayExpression: EsArrayExpression;
ArrowFunctionExpression: EsArrowFunctionExpression;
AssignmentExpression: EsAssignmentExpression;
AwaitExpression: EsAwaitExpression;
BinaryExpression: EsBinaryExpression;
CallExpression: EsCallExpression;
ChainExpression: EsChainExpression;
ClassExpression: EsClassExpression;
ConditionalExpression: EsConditionalExpression;
FunctionExpression: EsFunctionExpression;
Identifier: EsIdentifier;
ImportExpression: EsImportExpression;
LogicalExpression: EsLogicalExpression;
MemberExpression: EsMemberExpression;
MetaProperty: EsMetaProperty;
NewExpression: EsNewExpression;
ObjectExpression: EsObjectExpression;
SequenceExpression: EsSequenceExpression;
TaggedTemplateExpression: EsTaggedTemplateExpression;
TemplateLiteral: EsTemplateLiteral;
ThisExpression: EsThisExpression;
UnaryExpression: EsUnaryExpression;
UpdateExpression: EsUpdateExpression;
YieldExpression: EsYieldExpression;
};
export interface EsLiteralMap {
Literal: EsSimpleLiteral | EsRegExpLiteral | EsBigIntLiteral;
SimpleLiteral: EsSimpleLiteral;
RegExpLiteral: EsRegExpLiteral;
BigIntLiteral: EsBigIntLiteral;
}
export interface EsFunctionMap {
FunctionDeclaration: EsFunctionDeclaration;
FunctionExpression: EsFunctionExpression;
ArrowFunctionExpression: EsArrowFunctionExpression;
}
export interface EsPatternMap {
Identifier: EsIdentifier;
ObjectPattern: EsObjectPattern;
ArrayPattern: EsArrayPattern;
RestElement: EsRestElement;
AssignmentPattern: EsAssignmentPattern;
MemberExpression: EsMemberExpression;
}
export type EsStatementMap = EsDeclarationMap & {
ExpressionStatement: EsExpressionStatement;
BlockStatement: EsBlockStatement;
StaticBlock: EsStaticBlock;
EmptyStatement: EsEmptyStatement;
DebuggerStatement: EsDebuggerStatement;
WithStatement: EsWithStatement;
ReturnStatement: EsReturnStatement;
LabeledStatement: EsLabeledStatement;
BreakStatement: EsBreakStatement;
ContinueStatement: EsContinueStatement;
IfStatement: EsIfStatement;
SwitchStatement: EsSwitchStatement;
ThrowStatement: EsThrowStatement;
TryStatement: EsTryStatement;
WhileStatement: EsWhileStatement;
DoWhileStatement: EsDoWhileStatement;
ForStatement: EsForStatement;
ForInStatement: EsForInStatement;
ForOfStatement: EsForOfStatement;
};
export interface EsDeclarationMap {
FunctionDeclaration: EsFunctionDeclaration;
VariableDeclaration: EsVariableDeclaration;
ClassDeclaration: EsClassDeclaration;
}
export interface EsJsxMap {
JSXAttribute: EsJsxAttribute;
JSXClosingElement: EsJsxClosingElement;
JSXClosingFragment: EsJsxClosingFragment;
JSXElement: EsJsxElement;
JSXEmptyExpression: EsJsxEmptyExpression;
JSXExpressionContainer: EsJsxExpressionContainer;
JSXFragment: EsJsxFragment;
JSXIdentifier: EsJsxIdentifier;
JSXMemberExpression: EsJsxMemberExpression;
JSXNamespacedName: EsJsxNamespacedName;
JSXOpeningElement: EsJsxOpeningElement;
JSXOpeningFragment: EsJsxOpeningFragment;
JSXSpreadAttribute: EsJsxSpreadAttribute;
JSXSpreadChild: EsJsxSpreadChild;
JSXText: EsJsxText;
}

24
packages/deepl-mark/dist/ast/estree.js vendored Normal file
View File

@ -0,0 +1,24 @@
function esNodeIs(node, type) {
return node ? node.type === type : false;
}
function resolveEstreePropertyPath(node, parents, attributeName) {
if (!esNodeIs(parents[2], "ArrayExpression") && !esNodeIs(parents[2], "ObjectExpression")) return;
if (!esNodeIs(node.key, "Identifier")) return;
const names = [node.key.name];
for (let i = parents.length - 1; i > 1; i--) {
const parent = parents[i];
if (esNodeIs(parent, "ArrayExpression") || esNodeIs(parent, "ObjectExpression")) continue;
if (esNodeIs(parent, "Property")) {
if (!esNodeIs(parent.key, "Identifier")) return;
names.push(parent.key.name);
continue;
}
return;
}
names.push(attributeName);
return names.reverse().join(".");
}
export {
esNodeIs,
resolveEstreePropertyPath
};

View File

@ -0,0 +1,18 @@
import { EsNode, EsNodeMap, EsProgram } from './estree.js';
export declare const DEFAULT_ESWALKERS: EsWalkers;
export declare function eswalk(ast: EsProgram, visitors: EsVisitors, walkers?: EsWalkers): void;
export interface EsProcessor {
(node: EsNode | null, parents: EsNode[]): void;
}
export interface EsVisitor<NodeType extends keyof EsNodeMap> {
(node: EsNodeMap[NodeType], parents: EsNode[]): boolean | void;
}
export type EsVisitors = {
[NodeType in keyof EsNodeMap]?: EsVisitor<NodeType>;
};
export interface EsWalker<NodeType extends keyof EsNodeMap> {
(node: EsNodeMap[NodeType], parents: EsNode[], process: EsProcessor): void;
}
export type EsWalkers = {
[NodeType in keyof Partial<EsNodeMap>]: EsWalker<NodeType>;
};

75
packages/deepl-mark/dist/ast/eswalk.js vendored Normal file
View File

@ -0,0 +1,75 @@
import { isRegExp } from "node:util/types";
import { esNodeIs } from "./estree.js";
const DEFAULT_ESWALKERS = {
Program(node, parents, process) {
parents.push(node);
for (const statement of node.body) {
process(statement, parents);
}
parents.pop();
},
ExpressionStatement(node, parents, process) {
parents.push(node);
process(node.expression, parents);
parents.pop();
},
ArrayExpression(node, parents, process) {
parents.push(node);
for (const element of node.elements) {
process(element, parents);
}
parents.pop();
},
ObjectExpression(node, parents, process) {
parents.push(node);
for (const property of node.properties) {
process(property, parents);
}
parents.pop();
},
Property(node, parents, process) {
parents.push(node);
process(node.key, parents);
process(node.value, parents);
parents.pop();
},
JSXElement(node, parents, process) {
parents.push(node);
for (const child of node.children) {
process(child, parents);
}
for (const attribute of node.openingElement.attributes) {
process(attribute, parents);
}
parents.pop();
},
JSXAttribute(node, parents, process) {
parents.push(node);
if (node.value) {
process(node.value, parents);
}
parents.pop();
}
};
function eswalk(ast, visitors, walkers = DEFAULT_ESWALKERS) {
const process = (node, parents) => {
if (!node) return;
let type = node.type;
if (esNodeIs(node, "Literal")) {
type = typeof node.value === "bigint" ? "BigIntLiteral" : isRegExp(node.value) ? "RegExpLiteral" : "SimpleLiteral";
}
const visit = visitors[type];
const walk = walkers[type];
let keepWalking = true;
if (visit !== void 0) {
const signal = visit(node, parents);
keepWalking = signal === false ? false : true;
}
if (keepWalking && walk) walk(node, parents, process);
};
process(ast, []);
}
export {
DEFAULT_ESWALKERS,
eswalk
};

24
packages/deepl-mark/dist/ast/mdast.d.ts vendored Normal file
View File

@ -0,0 +1,24 @@
import type { Root as MdRoot, Blockquote as MdBlockquote, Break as MdBreak, Code as MdCode, Definition as MdDefinition, Delete as MdDelete, Emphasis as MdEmphasis, Footnote as MdFootnote, FootnoteDefinition as MdFootnoteDefinition, FootnoteReference as MdFootnoteReference, HTML as MdHTML, Heading as MdHeading, Image as MdImage, ImageReference as MdImageReference, InlineCode as MdInlineCode, Link as MdLink, LinkReference as MdLinkReference, List as MdList, ListItem as MdListItem, Paragraph as MdParagraph, Strong as MdStrong, Table as MdTable, TableCell as MdTableCell, TableRow as MdTableRow, Text as MdText, ThematicBreak as MdThematicBreak, YAML as MdYaml } from 'mdast';
import type { MdxFlowExpression, MdxJsxAttribute, MdxJsxAttributeValueExpression, MdxJsxExpressionAttribute, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, MdxjsEsm } from 'mdast-util-mdx';
import type { UnNode } from './unist.js';
declare module 'mdast' {
interface PhrasingContentMap extends StaticPhrasingContentMap {
mdxJsxFlowElement: MdxJsxFlowElement;
mdxJsxTextElement: MdxJsxTextElement;
mdxFlowExpression: MdxFlowExpression;
mdxTextExpression: MdxTextExpression;
}
}
export declare function mdNodeIs<T extends MdNodeType>(node: UnNode | undefined, type: T): node is T extends MdRoot['type'] ? MdRoot : T extends MdBlockquote['type'] ? MdBlockquote : T extends MdBreak['type'] ? MdBreak : T extends MdCode['type'] ? MdCode : T extends MdDefinition['type'] ? MdDefinition : T extends MdDelete['type'] ? MdDelete : T extends MdEmphasis['type'] ? MdEmphasis : T extends MdFootnote['type'] ? MdFootnote : T extends MdFootnoteDefinition['type'] ? MdFootnoteDefinition : T extends MdFootnoteReference['type'] ? MdFootnoteReference : T extends MdHTML['type'] ? MdHTML : T extends MdHeading['type'] ? MdHeading : T extends MdImage['type'] ? MdImage : T extends MdImageReference['type'] ? MdImageReference : T extends MdInlineCode['type'] ? MdInlineCode : T extends MdLink['type'] ? MdLink : T extends MdLinkReference['type'] ? MdLinkReference : T extends MdList['type'] ? MdList : T extends MdListItem['type'] ? MdListItem : T extends MdParagraph['type'] ? MdParagraph : T extends MdStrong['type'] ? MdStrong : T extends MdTable['type'] ? MdTable : T extends MdTableCell['type'] ? MdTableCell : T extends MdTableRow['type'] ? MdTableRow : T extends MdText['type'] ? MdText : T extends MdThematicBreak['type'] ? MdThematicBreak : T extends MdYaml ? MdYaml : T extends MdxFlowExpression['type'] ? MdxFlowExpression : T extends MdxJsxAttribute['type'] ? MdxJsxAttribute : T extends MdxJsxAttributeValueExpression['type'] ? MdxJsxAttributeValueExpression : T extends MdxJsxExpressionAttribute['type'] ? MdxJsxExpressionAttribute : T extends MdxJsxFlowElement['type'] ? MdxJsxFlowElement : T extends MdxJsxTextElement['type'] ? MdxJsxTextElement : T extends MdxTextExpression['type'] ? MdxTextExpression : MdxjsEsm;
export declare function mdNodeIsJsxElement(node: UnNode): node is MdxJsxFlowElement | MdxJsxTextElement;
/**
* Get MDX flavored `mdast`.
*/
export declare function getMdast(markdown: string): MdRoot;
export declare function getMarkdown(mdast: MdRoot): string;
/**
* ============================================================
*/
export type MdNodeType = MdRoot['type'] | MdBlockquote['type'] | MdBreak['type'] | MdCode['type'] | MdDefinition['type'] | MdDelete['type'] | MdEmphasis['type'] | MdFootnote['type'] | MdFootnoteDefinition['type'] | MdFootnoteReference['type'] | MdHTML['type'] | MdHeading['type'] | MdImage['type'] | MdImageReference['type'] | MdInlineCode['type'] | MdLink['type'] | MdLinkReference['type'] | MdList['type'] | MdListItem['type'] | MdParagraph['type'] | MdStrong['type'] | MdTable['type'] | MdTableCell['type'] | MdTableRow['type'] | MdText['type'] | MdThematicBreak['type'] | MdYaml['type'] | MdxFlowExpression['type'] | MdxJsxAttribute['type'] | MdxJsxAttributeValueExpression['type'] | MdxJsxExpressionAttribute['type'] | MdxJsxFlowElement['type'] | MdxJsxTextElement['type'] | MdxTextExpression['type'] | MdxjsEsm['type'];
export type { MdRoot, MdBlockquote, MdBreak, MdCode, MdDefinition, MdDelete, MdEmphasis, MdFootnote, MdFootnoteDefinition, MdFootnoteReference, MdHTML, MdHeading, MdImage, MdImageReference, MdInlineCode, MdLink, MdLinkReference, MdList, MdListItem, MdParagraph, MdStrong, MdTable, MdTableCell, MdTableRow, MdText, MdThematicBreak, MdYaml };
export type { MdxFlowExpression, MdxJsxAttribute, MdxJsxAttributeValueExpression, MdxJsxExpressionAttribute, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, MdxjsEsm };

48
packages/deepl-mark/dist/ast/mdast.js vendored Normal file
View File

@ -0,0 +1,48 @@
import { fromMarkdown } from "mdast-util-from-markdown";
import { frontmatterFromMarkdown, frontmatterToMarkdown } from "mdast-util-frontmatter";
import { gfmTableFromMarkdown, gfmTableToMarkdown } from "mdast-util-gfm-table";
import { htmlCommentFromMarkdown, htmlCommentToMarkdown } from "../vendor/mdast-util-html-comment.js";
import { mdxFromMarkdown, mdxToMarkdown } from "mdast-util-mdx";
import { toMarkdown } from "mdast-util-to-markdown";
import { frontmatter } from "micromark-extension-frontmatter";
import { gfmTable } from "micromark-extension-gfm-table";
import { htmlComment } from "../vendor/micromark-extension-html-comment.js";
import { mdxjs } from "micromark-extension-mdxjs";
function mdNodeIs(node, type) {
return node ? node.type === type : false;
}
function mdNodeIsJsxElement(node) {
return mdNodeIs(node, "mdxJsxFlowElement") || mdNodeIs(node, "mdxJsxTextElement");
}
function getMdast(markdown) {
return fromMarkdown(markdown, {
extensions: [frontmatter("yaml"), mdxjs(), gfmTable, htmlComment()],
mdastExtensions: [frontmatterFromMarkdown("yaml"), mdxFromMarkdown(), gfmTableFromMarkdown, htmlCommentFromMarkdown()]
});
}
function getMarkdown(mdast) {
return toMarkdown(mdast, {
extensions: [frontmatterToMarkdown("yaml"), mdxToMarkdown(), gfmTableToMarkdown(), htmlCommentToMarkdown()],
listItemIndent: "one",
join: [
(__, _, parent) => {
if (mdNodeIsJsxElement(parent)) {
return 0;
}
if (mdNodeIs(parent, "list")) {
return 0;
}
if (mdNodeIs(parent, "listItem")) {
return 0;
}
return 1;
}
]
});
}
export {
getMarkdown,
getMdast,
mdNodeIs,
mdNodeIsJsxElement
};

13
packages/deepl-mark/dist/ast/unist.d.ts vendored Normal file
View File

@ -0,0 +1,13 @@
import type { Position as UnPosition } from 'unist';
export declare function unNodeIsParent(node: UnNode): node is UnParent;
/**
* ============================================================
*/
export interface UnNode {
type: string;
position?: UnPosition;
data?: unknown;
}
export interface UnParent extends UnNode {
children: (UnNode | UnParent)[];
}

6
packages/deepl-mark/dist/ast/unist.js vendored Normal file
View File

@ -0,0 +1,6 @@
function unNodeIsParent(node) {
return "children" in node;
}
export {
unNodeIsParent
};

View File

@ -0,0 +1,5 @@
import { type UnNode, type UnParent } from './unist.js';
export declare function unwalk(node: UnNode, visit: UnVisitor, filter?: (node: UnNode, parent: UnParent | undefined) => boolean): void;
export interface UnVisitor {
(node: UnNode | UnParent, parent: UnParent | undefined, index: number | undefined): boolean | void;
}

24
packages/deepl-mark/dist/ast/unwalk.js vendored Normal file
View File

@ -0,0 +1,24 @@
import { unNodeIsParent } from "./unist.js";
const NEXT = true;
const STOP = false;
function unwalk(node, visit, filter) {
let next = true;
function step(node2, parent, index) {
if (filter && !filter(node2, parent)) return;
if (unNodeIsParent(node2)) {
for (let i = 0; i < node2.children.length; i++) {
if (!next) break;
const child = node2.children[i];
step(child, node2, i);
}
node2.children = node2.children.filter((child) => child);
}
if (!next) return;
const signal = visit(node2, parent, index);
next = signal === void 0 || NEXT ? NEXT : STOP;
}
step(node, void 0, void 0);
}
export {
unwalk
};

215
packages/deepl-mark/dist/config.d.ts vendored Normal file
View File

@ -0,0 +1,215 @@
import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node';
import type { MdNodeType } from './ast/mdast.js';
export interface ConfigBase {
/**
* Source's language code. Based on DeepL supported languages.
*/
sourceLanguage: SourceLanguageCode;
/**
* Output's languages code. Based on DeepL supported languages.
*/
outputLanguages: TargetLanguageCode[];
/**
* Sources and ouputs directories pairs. $langcode$ variable
* is provided to dynamically define directory.
*
* e.g. [ ["docs", "i18n/$langcode$/docs"], ["blog", "i18n/$langcode$/blog"] ]
*/
directories: [string, string][];
}
export interface Config extends ConfigBase {
/**
* Override current working directory, defaults to `process.cwd()`.
*/
cwd: string;
/**
* By default, all .md, .mdx, .json, and .yaml|.yml files inside
* source directories will be included.
*
* Define glob patterns to filter what files to include or exclude.
* But, the end result is still restricted by file types (.md, .mdx, .json).
*/
files: {
include?: string[];
exclude: string[];
};
/**
* Frontmatter fields.
*/
frontmatterFields: {
include: string[];
exclude: string[];
};
/**
* Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`.
*/
markdownNodes: {
default: boolean;
include: MdNodeType[];
exclude: MdNodeType[];
};
/**
* HTML elements to include and exlcude, down to the level of attributes
* and children. Include all HTML elements text content
* and some global attributes such as title and placeholder.
*/
htmlElements: {
include: Partial<{
[Tag in HtmlTag]: {
children: boolean;
attributes: string[];
};
}>;
exclude: HtmlTag[];
};
/**
* JSX components to include and exclude, down to the level of attributes
* and children. Include all JSX components text children
* and exclude all attributes by default.
*
* Support array, object, and jsx attribute value. For object and array value,
* you can specify the access path starting with the attribute name
* e.g. `items.description` to translate `items={[{description: "..."}]}.
*/
jsxComponents: {
default: boolean;
include: {
[Name: string]: {
children: boolean;
attributes: string[];
};
};
exclude: string[];
};
/**
* JSON or YAML file properties to include and exclude.
* Exclude all properties by default.
*/
jsonOrYamlProperties: {
include: (string | number | symbol)[];
exclude: (string | number | symbol)[];
};
}
export interface UserConfig extends ConfigBase {
/**
* Override current working directory, defaults to `process.cwd()`.
*/
cwd?: string;
/**
* By default, all .md, .mdx, .json, and .yaml|.yml files inside
* source directories will be included.
*
* Define glob patterns to filter what files to include or exclude.
* But, the end result is still restricted by file types (.md, .mdx, .json).
*/
files?: {
include?: string[];
exclude?: string[];
};
/**
* Frontmatter fields.
*/
frontmatterFields?: {
include?: string[];
exclude?: string[];
};
/**
* Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`.
*/
markdownNodes?: {
default?: boolean;
include?: MdNodeType[];
exclude?: MdNodeType[];
};
/**
* HTML elements to include and exlcude, down to the level of attributes
* and children. Include all HTML elements text content
* and some global attributes such as title and placeholder.
*/
htmlElements?: {
default?: boolean;
include?: Partial<{
[Tag in HtmlTag]: {
children: boolean;
attributes: string[];
};
}>;
exclude?: HtmlTag[];
};
/**
* JSX components to include and exclude, down to the level of attributes
* and children. Include all JSX components text children
* and exclude all attributes by default.
*
* Support array, object, and jsx attribute value. For object and array value,
* you can specify the access path starting with the attribute name
* e.g. `items.description` to translate `items={[{description: "..."}]}.
*/
jsxComponents?: {
default?: boolean;
include?: {
[Name: string]: {
children: boolean;
attributes: string[];
};
};
exclude?: string[];
};
/**
* JSON or YAML file properties to include and exclude.
* Exclude all properties by default.
*/
jsonOrYamlProperties?: {
include?: string[];
exclude?: string[];
};
}
export type HtmlElementsConfig = {
[Tag in HtmlTag]: {
children: boolean;
attributes: string[];
};
};
export declare const HTML_ELEMENTS_CONFIG: HtmlElementsConfig;
export declare const HTML_TAGS: HtmlTag[];
export declare function isHtmlTag(name: string): name is HtmlTag;
export declare function resolveConfig({ sourceLanguage, outputLanguages, directories, cwd, files, markdownNodes, frontmatterFields, htmlElements, jsxComponents, jsonOrYamlProperties }: UserConfig): Config;
export declare function isFrontmatterFieldIncluded({ field, config }: {
field: string;
config: Config;
}): boolean;
export declare function isMarkdownNodeIncluded({ type, config }: {
type: MdNodeType;
config: Config;
}): boolean;
export declare function isHtmlElementIncluded({ tag, config }: {
tag: HtmlTag;
config: Config;
}): boolean;
export declare function isHtmlElementAttributeIncluded({ tag, attribute, config }: {
tag: HtmlTag;
attribute: string;
config: Config;
}): boolean;
export declare function isHtmlElementChildrenIncluded({ tag, config }: {
tag: HtmlTag;
config: Config;
}): boolean;
export declare function isJsxComponentIncluded({ name, config }: {
name: string;
config: Config;
}): boolean;
export declare function isJsxComponentAttributeIncluded({ name, attribute, config }: {
name: string;
attribute: string;
config: Config;
}): boolean;
export declare function isJsxComponentChildrenIncluded({ name, config }: {
name: string;
config: Config;
}): boolean;
export declare function isJsonOrYamlPropertyIncluded({ property, config }: {
config: Config;
property: string | number | symbol;
}): boolean;
export type HtmlTag = 'a' | 'abbr' | 'address' | 'article' | 'aside' | 'audio' | 'b' | 'bdi' | 'bdo' | 'blockquote' | 'body' | 'button' | 'canvas' | 'caption' | 'cite' | 'col' | 'colgroup' | 'data' | 'datalist' | 'dd' | 'del' | 'details' | 'dfn' | 'dialog' | 'div' | 'dl' | 'dt' | 'em' | 'fieldset' | 'figcaption' | 'figure' | 'footer' | 'form' | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'header' | 'html' | 'i' | 'input' | 'ins' | 'label' | 'legend' | 'li' | 'main' | 'mark' | 'meter' | 'nav' | 'ol' | 'optgroup' | 'output' | 'p' | 'progress' | 'q' | 'rp' | 's' | 'samp' | 'section' | 'select' | 'small' | 'span' | 'strong' | 'sub' | 'summary' | 'sup' | 'table' | 'tbody' | 'td' | 'template' | 'text-area' | 'tfoot' | 'th' | 'thead' | 'time' | 'title' | 'tr' | 'track' | 'u' | 'ul' | 'area' | 'base' | 'br' | 'code' | 'embed' | 'head' | 'hr' | 'iframe' | 'img' | 'kbd' | 'link' | 'meta' | 'noscript' | 'object' | 'param' | 'picture' | 'pre' | 'rt' | 'ruby' | 'script' | 'source' | 'style' | 'svg' | 'var' | 'video' | 'qbr';

244
packages/deepl-mark/dist/config.js vendored Normal file
View File

@ -0,0 +1,244 @@
import { isBoolean } from "./utils.js";
const HTML_ELEMENTS_CONFIG = getHtmlElementsConfig();
function getHtmlElementsConfig() {
const includeChildren = [
"a",
"abbr",
"address",
"article",
"aside",
"audio",
"b",
"bdi",
"bdo",
"blockquote",
"body",
"button",
"canvas",
"caption",
"cite",
"col",
"colgroup",
"data",
"datalist",
"dd",
"del",
"details",
"dfn",
"dialog",
"div",
"dl",
"dt",
"em",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"html",
"i",
"input",
"ins",
"label",
"legend",
"li",
"main",
"mark",
"meter",
"nav",
"ol",
"optgroup",
"output",
"p",
"progress",
"q",
"rp",
"s",
"samp",
"section",
"select",
"small",
"span",
"strong",
"sub",
"summary",
"sup",
"table",
"tbody",
"td",
"template",
"text-area",
"tfoot",
"th",
"thead",
"time",
"title",
"tr",
"track",
"u",
"ul"
];
const excludeChildren = [
"area",
"base",
"br",
"code",
"embed",
"head",
"hr",
"iframe",
"img",
"kbd",
"link",
"meta",
"noscript",
"object",
"param",
"picture",
"pre",
"rt",
"ruby",
"script",
"source",
"style",
"svg",
"var",
"video",
"qbr"
];
const config = {};
for (const tag of includeChildren) {
config[tag] = {
children: true,
attributes: ["title"]
};
}
for (const tag of excludeChildren) {
config[tag] = {
children: false,
attributes: ["title"]
};
}
return config;
}
const HTML_TAGS = Object.keys(HTML_ELEMENTS_CONFIG);
function isHtmlTag(name) {
return HTML_TAGS.includes(name);
}
function resolveConfig({
sourceLanguage,
outputLanguages,
directories,
cwd,
files,
markdownNodes,
frontmatterFields,
htmlElements,
jsxComponents,
jsonOrYamlProperties
}) {
return {
sourceLanguage,
outputLanguages,
directories,
cwd: cwd ?? "",
files: files ? {
include: files.include,
exclude: files.exclude ?? []
} : { exclude: [] },
markdownNodes: markdownNodes ? {
default: isBoolean(markdownNodes.default) ? markdownNodes.default : true,
include: markdownNodes.include ?? [],
exclude: markdownNodes.exclude ?? ["code"]
} : { default: true, include: [], exclude: ["code"] },
frontmatterFields: frontmatterFields ? {
include: frontmatterFields.include ?? [],
exclude: frontmatterFields.exclude ?? []
} : { include: [], exclude: [] },
htmlElements: htmlElements ? {
include: htmlElements.include ? isBoolean(htmlElements.default) && htmlElements.default || htmlElements.default === void 0 ? { ...HTML_ELEMENTS_CONFIG, ...htmlElements.include } : htmlElements.include : isBoolean(htmlElements.default) && !htmlElements.default ? {} : HTML_ELEMENTS_CONFIG,
exclude: htmlElements.exclude ?? []
} : { include: HTML_ELEMENTS_CONFIG, exclude: [] },
jsxComponents: jsxComponents ? {
default: isBoolean(jsxComponents.default) ? jsxComponents.default : true,
include: jsxComponents.include ?? {},
exclude: jsxComponents.exclude ?? []
} : { default: true, include: {}, exclude: [] },
jsonOrYamlProperties: jsonOrYamlProperties ? { include: jsonOrYamlProperties.include ?? [], exclude: jsonOrYamlProperties.exclude ?? [] } : { include: [], exclude: [] }
};
}
function isFrontmatterFieldIncluded({
field,
config
}) {
return !config.frontmatterFields.exclude.includes(field) && config.frontmatterFields.include.includes(field);
}
function isMarkdownNodeIncluded({
type,
config
}) {
return !config.markdownNodes.exclude.includes(type) && (config.markdownNodes.default || config.markdownNodes.include.includes(type));
}
function isHtmlElementIncluded({ tag, config }) {
return !config.htmlElements.exclude.includes(tag) && Object.keys(config.htmlElements.include).includes(tag);
}
function isHtmlElementAttributeIncluded({
tag,
attribute,
config
}) {
return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag].attributes.includes(attribute);
}
function isHtmlElementChildrenIncluded({
tag,
config
}) {
return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag].children;
}
function isJsxComponentIncluded({
name,
config
}) {
return !config.jsxComponents.exclude.includes(name) && (config.jsxComponents.default || Object.keys(config.jsxComponents.include).includes(name));
}
function isJsxComponentAttributeIncluded({
name,
attribute,
config
}) {
return !config.jsxComponents.exclude.includes(name) && Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.include[name].attributes.includes(attribute);
}
function isJsxComponentChildrenIncluded({
name,
config
}) {
return !config.jsxComponents.exclude.includes(name) && (Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.include[name].children || !Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.default);
}
function isJsonOrYamlPropertyIncluded({
property,
config
}) {
return !config.jsonOrYamlProperties.exclude.includes(property) && config.jsonOrYamlProperties.include.includes(property);
}
export {
HTML_ELEMENTS_CONFIG,
HTML_TAGS,
isFrontmatterFieldIncluded,
isHtmlElementAttributeIncluded,
isHtmlElementChildrenIncluded,
isHtmlElementIncluded,
isHtmlTag,
isJsonOrYamlPropertyIncluded,
isJsxComponentAttributeIncluded,
isJsxComponentChildrenIncluded,
isJsxComponentIncluded,
isMarkdownNodeIncluded,
resolveConfig
};

11
packages/deepl-mark/dist/extract.d.ts vendored Normal file
View File

@ -0,0 +1,11 @@
import type { UnNode } from './ast/unist.js';
import { type Config } from './config.js';
export declare function extractMdastStrings({ mdast, config }: {
mdast: UnNode;
config: Config;
}): string[];
export declare function extractJsonOrYamlStrings({ source, type, config }: {
source: string;
type?: 'json' | 'yaml';
config: Config;
}): string[];

196
packages/deepl-mark/dist/extract.js vendored Normal file
View File

@ -0,0 +1,196 @@
import { parse as parseYaml } from "yaml";
import {
esNodeIs,
resolveEstreePropertyPath
} from "./ast/estree.js";
import { eswalk } from "./ast/eswalk.js";
import { mdNodeIs, mdNodeIsJsxElement } from "./ast/mdast.js";
import { unwalk } from "./ast/unwalk.js";
import {
isHtmlTag,
isFrontmatterFieldIncluded,
isHtmlElementIncluded,
isHtmlElementAttributeIncluded,
isJsonOrYamlPropertyIncluded,
isJsxComponentIncluded,
isJsxComponentAttributeIncluded,
isMarkdownNodeIncluded,
isHtmlElementChildrenIncluded,
isJsxComponentChildrenIncluded
} from "./config.js";
import { isArray, isEmptyArray, isEmptyString, isObject, isString } from "./utils.js";
function extractMdastStrings({
mdast,
config
}) {
const strings = [];
unwalk(
mdast,
(node, __, _) => {
if (mdNodeIs(node, "text")) {
pushTidyString({ array: strings, string: node.value });
return;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, "mdxJsxAttribute")) continue;
if (!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config }))
continue;
if (isString(attribute.value)) {
strings.push(attribute.value.trim());
} else if (attribute.value?.data?.estree) {
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _2) {
if (isString(esnode.value))
pushTidyString({ array: strings, string: esnode.value });
}
});
}
}
} else {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, "mdxJsxAttribute")) continue;
const componentName = node.name;
const isAttributeIncluded = isJsxComponentAttributeIncluded({
name: componentName,
attribute: attribute.name,
config
});
if (isString(attribute.value)) {
if (!isAttributeIncluded) continue;
strings.push(attribute.value.trim());
} else if (attribute.value?.data?.estree) {
if (!config.jsxComponents.include[componentName] || !config.jsxComponents.include[componentName].attributes.some(
(attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
))
continue;
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _2) {
if (isString(esnode.value))
pushTidyString({ array: strings, string: esnode.value });
if (esnode.value === "aye") console.log("passed");
},
JSXElement(esnode, _2) {
const name = esnode.openingElement.name.name;
if (isHtmlTag(name)) {
if (!isHtmlElementIncluded({ tag: name, config }) || !isHtmlElementChildrenIncluded({ tag: name, config }))
return false;
} else if (!isJsxComponentIncluded({ name, config }) || !isJsxComponentChildrenIncluded({ name, config }))
return false;
},
JSXAttribute(esnode, parents) {
const name = typeof esnode.name.name === "string" ? esnode.name.name : esnode.name.name.name;
const parentName = parents[parents.length - 1].openingElement.name.name;
if (isHtmlTag(parentName)) {
if (!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config }))
return false;
} else if (!config.jsxComponents.include[name] || !config.jsxComponents.include[name].attributes.some(
(attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)) {
return false;
}
},
JSXText(esnode, _2) {
pushTidyString({ array: strings, string: esnode.value });
},
Property(esnode, parents) {
if (!esNodeIs(esnode, "Identifier")) return false;
const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name);
if (!propertyPath || !isJsxComponentAttributeIncluded({
name: componentName,
attribute: propertyPath,
config
}))
return false;
}
});
}
}
}
}
if (mdNodeIs(node, "yaml")) {
if (isEmptyArray(config.frontmatterFields.include)) return;
if (isEmptyString(node.value)) return;
const object = parseYaml(node.value);
for (const field in object) {
if (!isFrontmatterFieldIncluded({ field, config })) continue;
const value = object[field];
if (isString(value)) {
strings.push(value);
continue;
}
if (isArray(value)) {
for (const item of value) {
if (!isString(item)) continue;
strings.push(item);
}
}
}
return;
}
},
(node, parent) => {
if (!isMarkdownNodeIncluded({ type: node.type, config })) return false;
if (parent && mdNodeIsJsxElement(parent) && parent.name) {
if (isHtmlTag(parent.name)) {
if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false;
} else {
if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false;
}
return true;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
if (!isHtmlElementIncluded({ tag: node.name, config })) return false;
} else {
if (!isJsxComponentIncluded({ name: node.name, config })) return false;
}
return true;
}
return true;
}
);
return strings;
}
function extractJsonOrYamlStrings({
source,
type = "json",
config
}) {
const strings = [];
if (isEmptyArray(config.jsonOrYamlProperties.include)) return strings;
const parsed = type === "json" ? JSON.parse(source) : parseYaml(source);
process(parsed);
function process(value, property) {
if (typeof value === "string") {
if (property && isJsonOrYamlPropertyIncluded({ property, config })) strings.push(value);
return;
}
if (isArray(value)) {
for (const item of value) {
process(item);
}
return;
}
if (isObject(value)) {
for (const property2 in value) {
const item = value[property2];
process(item, property2);
}
return;
}
}
return strings;
}
function pushTidyString({ array, string }) {
if (!/^\s*$/.test(string)) {
array.push(string.replace(/(^\n|\r|\t|\v)+\s*/, "").replace(/\s+$/, " "));
}
}
export {
extractJsonOrYamlStrings,
extractMdastStrings
};

1
packages/deepl-mark/dist/format.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export declare function format(markdown: string): Promise<string>;

33
packages/deepl-mark/dist/format.js vendored Normal file
View File

@ -0,0 +1,33 @@
import prettier from "prettier";
import { getMarkdown, getMdast, mdNodeIs } from "./ast/mdast.js";
import { unwalk } from "./ast/unwalk.js";
async function format(markdown) {
const mdast = getMdast(
await prettier.format(markdown, {
parser: "mdx",
printWidth: Infinity,
proseWrap: "never",
useTabs: true
})
);
unwalk(
mdast,
(node, parent, index) => {
if (mdNodeIs(node, "mdxFlowExpression") && expressionIsEmpty(node.value)) {
parent.children[index] = void 0;
}
},
(node, parent) => {
delete node.position;
return mdNodeIs(parent, "root");
}
);
return getMarkdown(mdast);
}
function expressionIsEmpty(text) {
const regex = /^('|")\s*('|")$/;
return regex.test(text);
}
export {
format
};

32
packages/deepl-mark/dist/index.d.ts vendored Normal file
View File

@ -0,0 +1,32 @@
import type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';
import type { UserConfig } from './config.js';
/**
* Options to control which parts of the markdown are translated.
*/
export type TranslateOptions = Omit<UserConfig, 'sourceLanguage' | 'outputLanguages' | 'directories'> & {
/** DeepL API key. Falls back to `DEEPL_AUTH_KEY` env var if not provided. */
apiKey?: string;
/** DeepL translation options (tagHandling, splitSentences, formality, glossaryId, etc.) */
deeplOptions?: TranslateTextOptions;
};
/**
* Translate markdown/MDX content from one language to another using DeepL.
*
* Requires `DEEPL_AUTH_KEY` environment variable to be set.
*
* @param content - Markdown or MDX string to translate
* @param sourceLang - Source language code (e.g. 'en', 'de', 'fr')
* @param targetLang - Target language code (e.g. 'de', 'en-US', 'fr')
* @param options - Optional config to control extraction (frontmatter, jsx, html, etc.)
* @returns Translated markdown string
*
* @example
* ```ts
* import { translate } from 'deepmark';
*
* const result = await translate('# Hello World', 'en', 'de');
* console.log(result); // '# Hallo Welt'
* ```
*/
export declare function translate(content: string, sourceLang: SourceLanguageCode, targetLang: TargetLanguageCode, options?: TranslateOptions): Promise<string>;
export type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';

25
packages/deepl-mark/dist/index.js vendored Normal file
View File

@ -0,0 +1,25 @@
import { getMarkdown, getMdast } from "./ast/mdast.js";
import { resolveConfig } from "./config.js";
import { extractMdastStrings } from "./extract.js";
import { format } from "./format.js";
import { replaceMdastStrings } from "./replace.js";
import { translateStrings } from "./translate.js";
async function translate(content, sourceLang, targetLang, options) {
const { apiKey, deeplOptions, ...configOptions } = options ?? {};
const config = resolveConfig({
sourceLanguage: sourceLang,
outputLanguages: [targetLang],
directories: [["", ""]],
...configOptions
});
const formatted = await format(content);
const mdast = getMdast(formatted);
const strings = extractMdastStrings({ mdast, config });
if (strings.length === 0) return content;
const translated = await translateStrings(strings, sourceLang, targetLang, apiKey, deeplOptions);
const result = replaceMdastStrings({ mdast, strings: translated, config });
return getMarkdown(result);
}
export {
translate
};

1
packages/deepl-mark/dist/lint.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export {};

0
packages/deepl-mark/dist/lint.js vendored Normal file
View File

13
packages/deepl-mark/dist/replace.d.ts vendored Normal file
View File

@ -0,0 +1,13 @@
import type { MdRoot } from './ast/mdast.js';
import { type Config } from './config.js';
export declare function replaceMdastStrings({ mdast, config, strings }: {
mdast: MdRoot;
strings: string[];
config: Config;
}): MdRoot;
export declare function replaceJsonOrYamlStrings({ source, type, strings, config }: {
source: string;
type?: 'json' | 'yaml';
strings: string[];
config: Config;
}): string;

205
packages/deepl-mark/dist/replace.js vendored Normal file
View File

@ -0,0 +1,205 @@
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
import {
esNodeIs,
resolveEstreePropertyPath
} from "./ast/estree.js";
import { eswalk } from "./ast/eswalk.js";
import { mdNodeIs, mdNodeIsJsxElement } from "./ast/mdast.js";
import { unwalk } from "./ast/unwalk.js";
import {
isHtmlTag,
isFrontmatterFieldIncluded,
isHtmlElementIncluded,
isHtmlElementAttributeIncluded,
isJsonOrYamlPropertyIncluded,
isJsxComponentIncluded,
isJsxComponentAttributeIncluded,
isMarkdownNodeIncluded,
isHtmlElementChildrenIncluded,
isJsxComponentChildrenIncluded
} from "./config.js";
import { isArray, isEmptyArray, isEmptyString, isObject, isString } from "./utils.js";
function replaceMdastStrings({
mdast,
config,
strings
}) {
strings = strings.reverse();
unwalk(
mdast,
(node, __, _) => {
if (mdNodeIs(node, "text")) {
node.value = strings.pop();
return;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, "mdxJsxAttribute")) continue;
if (!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config }))
continue;
if (isString(attribute.value)) {
attribute.value = strings.pop();
} else if (attribute.value?.data?.estree) {
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _2) {
if (isString(esnode.value)) esnode.value = strings.pop();
}
});
}
}
} else {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, "mdxJsxAttribute")) continue;
const componentName = node.name;
const isAttributeIncluded = isJsxComponentAttributeIncluded({
name: componentName,
attribute: attribute.name,
config
});
if (isString(attribute.value)) {
if (!isAttributeIncluded) continue;
attribute.value = strings.pop();
} else if (attribute.value?.data?.estree) {
if (!config.jsxComponents.include[componentName] || !config.jsxComponents.include[componentName].attributes.some(
(attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
))
continue;
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _2) {
if (isString(esnode.value)) esnode.value = strings.pop();
},
JSXElement(esnode, _2) {
const name = esnode.openingElement.name.name;
if (isHtmlTag(name)) {
if (!isHtmlElementIncluded({ tag: name, config }) || !isHtmlElementChildrenIncluded({ tag: name, config }))
return false;
} else if (!isJsxComponentIncluded({ name, config }) || !isJsxComponentChildrenIncluded({ name, config }))
return false;
},
JSXAttribute(esnode, parents) {
const name = typeof esnode.name.name === "string" ? esnode.name.name : esnode.name.name.name;
const parentName = parents[parents.length - 1].openingElement.name.name;
if (isHtmlTag(parentName)) {
if (!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config }))
return false;
} else if (!config.jsxComponents.include[name] || !config.jsxComponents.include[name].attributes.some(
(attrName) => attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)) {
return false;
}
},
JSXText(esnode, _2) {
esnode.value = strings.pop();
},
Property(esnode, parents) {
if (!esNodeIs(esnode, "Identifier")) return false;
const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name);
if (!propertyPath || !isJsxComponentAttributeIncluded({
name: componentName,
attribute: propertyPath,
config
}))
return false;
}
});
}
}
}
}
if (mdNodeIs(node, "yaml")) {
if (isEmptyArray(config.frontmatterFields.include)) return;
if (isEmptyString(node.value)) return;
const object = parseYaml(node.value);
for (const field in object) {
if (!isFrontmatterFieldIncluded({ field, config })) continue;
const value = object[field];
if (isString(value)) {
object[field] = strings.pop();
continue;
}
if (isArray(value)) {
for (const [index, item] of value.entries()) {
if (!isString(item)) continue;
value[index] = strings.pop();
}
}
}
return;
}
},
(node, parent) => {
if (!isMarkdownNodeIncluded({ type: node.type, config })) return false;
if (parent && mdNodeIsJsxElement(parent) && parent.name) {
if (isHtmlTag(parent.name)) {
if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false;
} else {
if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false;
}
return true;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
if (!isHtmlElementIncluded({ tag: node.name, config })) return false;
} else {
if (!isJsxComponentIncluded({ name: node.name, config })) return false;
}
return true;
}
return true;
}
);
return mdast;
}
function replaceJsonOrYamlStrings({
source,
type = "json",
strings,
config
}) {
if (isEmptyArray(config.jsonOrYamlProperties.include)) return source;
strings = strings.reverse();
const parsed = type === "json" ? JSON.parse(source) : parseYaml(source);
process({ value: parsed });
function process({
value,
parent,
property,
index
}) {
if (isArray(value)) {
for (const [index2, item] of value.entries()) {
process({ value: item, parent: value, property, index: index2 });
}
return;
}
if (isObject(value)) {
for (const property2 in value) {
const item = value[property2];
process({ value: item, parent: value, property: property2 });
}
return;
}
if (typeof value === "string") {
if (property && isJsonOrYamlPropertyIncluded({ property, config })) {
if (isArray(parent) && index) {
parent[index] = strings.pop();
return;
}
if (isObject(parent)) {
parent[property] = strings.pop();
return;
}
}
return;
}
}
if (type === "json") return JSON.stringify(parsed);
return stringifyYaml(parsed);
}
export {
replaceJsonOrYamlStrings,
replaceMdastStrings
};

View File

@ -0,0 +1,6 @@
import type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';
/**
* Translate an array of strings from sourceLang to targetLang using DeepL.
* Batches requests and retries on rate-limit (429) or server (5xx) errors.
*/
export declare function translateStrings(strings: string[], sourceLang: SourceLanguageCode, targetLang: TargetLanguageCode, apiKey?: string, deeplOptions?: TranslateTextOptions, batchSize?: number): Promise<string[]>;

41
packages/deepl-mark/dist/translate.js vendored Normal file
View File

@ -0,0 +1,41 @@
import { Translator } from "deepl-node";
const DEFAULT_BATCH_SIZE = 50;
const MAX_RETRIES = 3;
async function translateStrings(strings, sourceLang, targetLang, apiKey, deeplOptions, batchSize = DEFAULT_BATCH_SIZE) {
if (strings.length === 0) return [];
const key = apiKey ?? process.env.DEEPL_AUTH_KEY;
if (!key) throw new Error("DeepL API key must be provided via options.apiKey or DEEPL_AUTH_KEY environment variable");
const deepl = new Translator(key);
const translations = new Array(strings.length).fill("");
const textOptions = {
tagHandling: "html",
splitSentences: "nonewlines",
...deeplOptions
};
for (let i = 0; i < strings.length; i += batchSize) {
const batch = strings.slice(i, i + batchSize);
const results = await retry(
() => deepl.translateText(batch, sourceLang, targetLang, textOptions)
);
for (let j = 0; j < batch.length; j++) {
translations[i + j] = results[j].text;
}
}
return translations;
}
async function retry(fn, retries = MAX_RETRIES) {
for (let attempt = 0; ; attempt++) {
try {
return await fn();
} catch (err) {
const status = err?.statusCode ?? err?.status;
const retryable = status === 429 || status === 456 || status >= 500 && status < 600;
if (!retryable || attempt >= retries) throw err;
const delay = Math.min(1e3 * 2 ** attempt, 1e4);
await new Promise((r) => setTimeout(r, delay));
}
}
}
export {
translateStrings
};

7
packages/deepl-mark/dist/utils.d.ts vendored Normal file
View File

@ -0,0 +1,7 @@
export declare function isArray(value: unknown): value is any[];
export declare function isBoolean(value: unknown): value is boolean;
export declare function isEmptyArray(array: any[]): boolean;
export declare function isEmptyObject(object: Object): boolean;
export declare function isEmptyString(string: string): boolean;
export declare function isObject(value: unknown): value is Record<string | number | symbol, unknown>;
export declare function isString(value: unknown): value is string;

30
packages/deepl-mark/dist/utils.js vendored Normal file
View File

@ -0,0 +1,30 @@
function isArray(value) {
return Array.isArray(value);
}
function isBoolean(value) {
return typeof value === "boolean";
}
function isEmptyArray(array) {
return array.length === 0;
}
function isEmptyObject(object) {
return Object.keys(object).length === 0;
}
function isEmptyString(string) {
return string.length === 0;
}
function isObject(value) {
return isArray(value) ? false : typeof value == "object" ? true : false;
}
function isString(value) {
return typeof value === "string";
}
export {
isArray,
isBoolean,
isEmptyArray,
isEmptyObject,
isEmptyString,
isObject,
isString
};

View File

@ -0,0 +1,4 @@
import type { Extension } from 'mdast-util-from-markdown';
import type { Options } from 'mdast-util-to-markdown';
export declare function htmlCommentFromMarkdown(): Extension;
export declare function htmlCommentToMarkdown(): Options;

View File

@ -0,0 +1,37 @@
function htmlCommentFromMarkdown() {
return {
canContainEols: ["htmlComment"],
enter: {
htmlComment() {
this.buffer();
}
},
exit: {
htmlComment(token) {
const string = this.resume();
this.enter(
{
// @ts-ignore
type: "htmlComment",
value: string.slice(0, -3)
},
token
);
this.exit(token);
}
}
};
}
function htmlCommentToMarkdown() {
return {
handlers: {
htmlComment(node) {
return `<!--${node.value}-->`;
}
}
};
}
export {
htmlCommentFromMarkdown,
htmlCommentToMarkdown
};

View File

@ -0,0 +1,3 @@
import type { Extension, HtmlExtension } from 'micromark-util-types';
export declare function htmlComment(): Extension;
export declare function htmlCommentToHtml(): HtmlExtension;

View File

@ -0,0 +1,107 @@
import { factorySpace } from "micromark-factory-space";
import { markdownLineEnding } from "micromark-util-character";
import { codes } from "micromark-util-symbol/codes.js";
import { types } from "micromark-util-symbol/types.js";
function htmlComment() {
return {
flow: {
[codes.lessThan]: { tokenize, concrete: true }
},
text: {
[codes.lessThan]: { tokenize }
}
};
}
function htmlCommentToHtml() {
return {
enter: {
htmlComment() {
this.buffer();
}
},
exit: {
htmlComment() {
this.resume();
}
}
};
}
const tokenize = (effects, ok, nok) => {
let value = "";
return start;
function start(code) {
effects.enter("htmlComment");
effects.enter("htmlCommentMarker");
effects.consume(code);
value += "<";
return open;
}
function open(code) {
if (value === "<" && code === codes.exclamationMark) {
effects.consume(code);
value += "!";
return open;
}
if (code === codes.dash) {
if (value === "<!") {
effects.consume(code);
value += "-";
return open;
}
if (value === "<!-") {
effects.consume(code);
effects.exit("htmlCommentMarker");
value += "-";
return inside;
}
}
return nok(code);
}
function inside(code) {
if (code === codes.eof) return nok(code);
if (markdownLineEnding(code)) {
effects.exit(types.data);
return atLineEnding(code);
}
if (code === codes.greaterThan) {
return close(code);
}
if (value === "<!--") {
effects.enter("htmlCommentString");
effects.enter(types.data);
}
effects.consume(code);
if (code === codes.dash) {
value += "-";
} else {
value += "*";
}
return inside;
}
function atLineEnding(code) {
effects.enter(types.lineEnding);
effects.consume(code);
effects.exit(types.lineEnding);
return factorySpace(effects, afterLinePrefix, types.linePrefix);
}
function afterLinePrefix(code) {
if (markdownLineEnding(code)) return atLineEnding(code);
effects.enter(types.data);
return inside(code);
}
function close(code) {
if (value.length >= 6 && value.slice(-2) === "--") {
effects.consume(code);
effects.exit(types.data);
effects.exit("htmlCommentString");
effects.exit("htmlComment");
value += ">";
return ok;
}
return nok(code);
}
};
export {
htmlComment,
htmlCommentToHtml
};

View File

@ -0,0 +1,38 @@
import prettier from 'prettier';
import { fromMarkdown } from 'mdast-util-from-markdown';
import { toMarkdown } from 'mdast-util-to-markdown';
const md = `The panel includes the following settings:
* ON/OFF Toggle: A main switch.
* Min Heating Time (1-60s): Minimum duration.
* Mode: Selects the sequential heating algorithm:
* 0 - All: Cycles through all devices with time-based control.
* 1 - SP: Cycles through devices in groups.
* 2 - SP Any: Heats any devices that need heating.
* Post-Heatup Mode: Mode to switch to after initial heatup phase.
* Current Status: Display field showing the current state.`;
const prettified = await prettier.format(md, {
parser: 'mdx',
printWidth: Infinity,
proseWrap: 'never',
useTabs: true
});
const tree = fromMarkdown(prettified);
// FIX: also handle listItem parent in the join rule
const result = toMarkdown(tree, {
listItemIndent: 'one',
join: [
(__, _, parent) => {
if (parent?.type === 'list') return 0;
if (parent?.type === 'listItem') return 0;
return 1;
}
]
});
console.log('=== FIXED output ===');
console.log(result);

View File

@ -0,0 +1,51 @@
import prettier from 'prettier';
import { fromMarkdown } from 'mdast-util-from-markdown';
import { toMarkdown } from 'mdast-util-to-markdown';
const md = `The panel includes the following settings:
* ON/OFF Toggle: A main switch.
* Min Heating Time (1-60s): Minimum duration.
* Mode: Selects the sequential heating algorithm:
* 0 - All: Cycles through all devices with time-based control.
* 1 - SP: Cycles through devices in groups.
* 2 - SP Any: Heats any devices that need heating.
* Post-Heatup Mode: Mode to switch to after initial heatup phase.
* Current Status: Display field showing the current state.`;
console.log('=== Input ===');
console.log(md);
const prettified = await prettier.format(md, {
parser: 'mdx',
printWidth: Infinity,
proseWrap: 'never',
useTabs: true
});
console.log('\n=== After Prettier ===');
console.log(JSON.stringify(prettified));
console.log(prettified);
const tree = fromMarkdown(prettified);
const list = tree.children[1]; // after the paragraph
console.log('=== AST Analysis of list ===');
console.log('list spread:', list.spread);
for (const [i, item] of list.children.entries()) {
console.log(`item[${i}] spread:`, item.spread, 'children:', item.children.length, item.children.map(c => c.type));
}
// Now serialize with the join rule:
const result = toMarkdown(tree, {
listItemIndent: 'one',
join: [
(__, _, parent) => {
if (parent?.type === 'list') return 0;
return 1;
}
]
});
console.log('\n=== Serialized output ===');
console.log(result);

View File

@ -0,0 +1,50 @@
import { fromMarkdown } from 'mdast-util-from-markdown';
import { toMarkdown } from 'mdast-util-to-markdown';
const md = `* Mode: Selects the algorithm:
* 0 - All: Cycles through all devices.
* 1 - SP: Cycles through devices in groups.
* Post-Mode: blah`;
const tree = fromMarkdown(md);
const list = tree.children[0];
console.log('=== AST Analysis ===');
console.log('list spread:', list.spread);
for (const [i, item] of list.children.entries()) {
console.log(`item[${i}] spread:`, item.spread, 'children count:', item.children.length, 'children types:', item.children.map(c => c.type));
}
console.log('\n=== Default toMarkdown ===');
console.log(JSON.stringify(toMarkdown(tree)));
console.log('\n=== With join rule ===');
const result = toMarkdown(tree, {
listItemIndent: 'one',
join: [
(__, _, parent) => {
if (parent?.type === 'list') return 0;
if (parent?.type === 'listItem') return 0;
return 1;
}
]
});
console.log(JSON.stringify(result));
console.log('\n=== With join rule + forced spread=false ===');
function clearSpread(node) {
if (node.spread !== undefined) node.spread = false;
if (node.children) node.children.forEach(clearSpread);
}
clearSpread(tree);
const result2 = toMarkdown(tree, {
listItemIndent: 'one',
join: [
(__, _, parent) => {
if (parent?.type === 'list') return 0;
if (parent?.type === 'listItem') return 0;
return 1;
}
]
});
console.log(JSON.stringify(result2));

4335
packages/deepl-mark/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,56 @@
{
"name": "@polymech/deepl-mark",
"description": "Translate markdown files correctly with `mdast` and DeepL.",
"version": "0.3.0",
"license": "MIT",
"author": "Izzuddin Natsir | Polymech",
"type": "module",
"files": [
"dist/*"
],
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"scripts": {
"build": "tsc && node build.js",
"dev": "tsc -w & node --watch build.js",
"test": "vitest run --reporter verbose",
"test:watch": "vitest watch --reporter verbose",
"test:tables": "vitest run src/__test__/e2e.test.ts --reporter verbose"
},
"dependencies": {
"acorn": "^8.8.2",
"acorn-jsx": "^5.3.2",
"astring": "^1.8.4",
"deepl-node": "^1.24.0",
"mdast-util-from-markdown": "^1.3.0",
"mdast-util-frontmatter": "^1.0.1",
"mdast-util-gfm-table": "^1.0.7",
"mdast-util-mdx": "^2.0.1",
"mdast-util-to-markdown": "^1.5.0",
"micromark-extension-frontmatter": "^1.0.0",
"micromark-extension-gfm-table": "^1.0.7",
"micromark-extension-mdxjs": "^1.0.0",
"micromark-factory-space": "^1.0.0",
"micromark-util-character": "^1.1.0",
"micromark-util-symbol": "^1.0.1",
"micromark-util-types": "^1.0.2",
"prettier": "^2.8.3",
"yaml": "^2.2.1"
},
"devDependencies": {
"@types/node": "^25.3.3",
"@types/estree": "^1.0.0",
"@types/mdast": "^3.0.10",
"@types/prettier": "^2.7.2",
"@types/unist": "^2.0.6",
"esbuild": "^0.25.0",
"typescript": "^5.9.3",
"vitest": "^3.0.0"
}
}

View File

@ -0,0 +1,11 @@
/** @type {import("../../../config").UserConfig} */
export default {
sourceLanguage: 'en',
outputLanguages: ['zh', 'ja'],
directories: [
['i18n/$langcode$', 'i18n/$langcode$'],
['docs', 'i18n/$langcode$/docs'],
['blog', 'i18n/$langcode$/blog']
],
cwd: '../../example'
};

View File

@ -0,0 +1,12 @@
import base from './base.mjs';
/** @type {import("../../../config").UserConfig} */
export default {
...base,
files: {
include: ['docs/intro.md', 'docs/tutorial-basics/markdown-features.mdx', 'i18n/en/code.json']
},
jsonOrYamlProperties: {
include: ['message', 'description']
}
};

View File

@ -0,0 +1,9 @@
import base from './base.mjs';
/** @type {import("../../../config").UserConfig} */
export default {
...base,
jsonOrYamlProperties: {
include: ['message', 'description']
}
};

View File

@ -0,0 +1,12 @@
import base from './base.mjs';
/** @type {import("../../../config").UserConfig} */
export default {
...base,
files: {
include: ['docs/tutorial-basics/markdown-features.mdx', 'i18n/en/code.json']
},
jsonOrYamlProperties: {
include: ['message', 'description']
}
};

View File

@ -0,0 +1,2 @@
---
---

View File

@ -0,0 +1,6 @@
---
author: Izzuddin Natsir
title: A Short Title
tags: [tagone, tagtwo]
description: A short description.
---

View File

@ -0,0 +1,18 @@
{
"title": {
"message": "My Site",
"description": "The title in the navbar"
},
"item.label.Tutorial": {
"message": "Tutorial",
"description": "Navbar item with label Tutorial"
},
"item.label.Blog": {
"message": "Blog",
"description": "Navbar item with label Blog"
},
"item.label.GitHub": {
"message": "GitHub",
"description": "Navbar item with label GitHub"
}
}

View File

@ -0,0 +1,5 @@
<p>
This is a text.
<code>function</code>
<pre>preformatted</pre>
</p>

View File

@ -0,0 +1,14 @@
<Card header={<h1>This is a text inside a jsx prop.</h1>}>
This is a text inside a custom component.
</Card>
<List
items={[
<div title="A short title inside title attribute inside HTML element inside an attribute.">
This is the text of jsx item one. <span>This the nested text of jsx item one.</span>
</div>,
<div>
This is the text of jsx item two. <span>This the nested text of jsx item two.</span>
</div>
]}
></List>

View File

@ -0,0 +1,4 @@
<p>
This is a paragraph.<span>This is a span.</span>
<Block>This is a text inside a custom component.</Block>
</p>

View File

@ -0,0 +1,17 @@
endi:
name: Endilie Yacop Sucipto
title: Maintainer of Docusaurus
url: https://github.com/endiliey
image_url: https://github.com/endiliey.png
yangshun:
name: Yangshun Tay
title: Front End Engineer @ Facebook
url: https://github.com/yangshun
image_url: https://github.com/yangshun.png
slorber:
name: Sébastien Lorber
title: Docusaurus maintainer
url: https://sebastienlorber.com
image_url: https://github.com/slorber.png

View File

@ -0,0 +1,52 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`extract frontmatter field string values > filter frontmatter fields based on configuration 1`] = `
[
"A Short Title",
"tagone",
"tagtwo",
]
`;
exports[`extract jsx children and attribute string values > ignore some HTML elements by default 1`] = `
[
"This is a text. ",
]
`;
exports[`extract jsx children and attribute string values > recursively extract strings from html elements and jsx components inside attributes 1`] = `
[
"This is a text inside a custom component.",
"This is a text inside a jsx prop.",
"This is the text of jsx item one. ",
"This the nested text of jsx item one.",
"A short title inside title attribute inside HTML element inside an attribute.",
"This is the text of jsx item two. ",
"This the nested text of jsx item two.",
]
`;
exports[`extract jsx children and attribute string values > recursively extract strings from nested jsx components 1`] = `
[
"This is a paragraph.",
"This is a span.",
"This is a text inside a custom component.",
]
`;
exports[`extract strings from JSON based on configuration > filter properties based on the config 1`] = `
[
"The title in the navbar",
"Navbar item with label Tutorial",
"Navbar item with label Blog",
"Navbar item with label GitHub",
]
`;
exports[`extract strings from yaml based on configuration > filter properties based on the config 1`] = `
[
"Maintainer of Docusaurus",
"Front End Engineer @ Facebook",
"Docusaurus maintainer",
]
`;

View File

@ -0,0 +1,62 @@
import { describe } from 'vitest';
import type { Config, UserConfig } from '../config';
import {
isFrontmatterFieldIncluded,
isHtmlElementIncluded,
isHtmlElementAttributeIncluded,
isJsonOrYamlPropertyIncluded,
isJsxComponentIncluded,
isJsxComponentAttributeIncluded,
isMarkdownNodeIncluded,
resolveConfig
} from '../config';
const baseConfig: UserConfig = {
sourceLanguage: 'en',
outputLanguages: ['zh'],
directories: [['', '']]
};
describe('default configurations', (test) => {
const config: Config = resolveConfig(baseConfig);
test('frontmatter fields', ({ expect }) => {
expect(isFrontmatterFieldIncluded({ field: 'title', config })).toBe(false);
expect(isFrontmatterFieldIncluded({ field: 'description', config })).toBe(false);
});
test('markdown nodes', ({ expect }) => {
expect(isMarkdownNodeIncluded({ type: 'code', config })).toBe(false);
expect(isMarkdownNodeIncluded({ type: 'blockquote', config })).toBe(true);
expect(isMarkdownNodeIncluded({ type: 'heading', config })).toBe(true);
});
test('html elements', ({ expect }) => {
expect(isHtmlElementIncluded({ tag: 'a', config })).toBe(true);
expect(isHtmlElementIncluded({ tag: 'div', config })).toBe(true);
});
test('html element attributes', ({ expect }) => {
expect(isHtmlElementAttributeIncluded({ tag: 'div', attribute: 'title', config })).toBe(true);
expect(isHtmlElementAttributeIncluded({ tag: 'div', attribute: 'id', config })).toBe(false);
});
test('jsx components', ({ expect }) => {
expect(isJsxComponentIncluded({ name: 'Card', config })).toBe(true);
expect(isJsxComponentIncluded({ name: 'Warning', config })).toBe(true);
});
test('jsx component attributes', ({ expect }) => {
expect(isJsxComponentAttributeIncluded({ name: 'Card', attribute: 'icon', config })).toBe(
false
);
expect(isJsxComponentAttributeIncluded({ name: 'Warning', attribute: 'title', config })).toBe(
false
);
});
test('json or yaml properties', ({ expect }) => {
expect(isJsonOrYamlPropertyIncluded({ property: 'author', config })).toBe(false);
expect(isJsonOrYamlPropertyIncluded({ property: 'title', config })).toBe(false);
});
});

View File

@ -0,0 +1,23 @@
import { readFile, writeFile } from 'node:fs/promises';
import np from 'node:path';
import { describe, test, expect } from 'vitest';
import { translate } from '../index';
const samplesDir = np.resolve('src/__test__');
describe('e2e', () => {
test(
'MarkdownTables',
async () => {
const source = await readFile(np.join(samplesDir, 'table.md'), 'utf-8');
const result = await translate(source, 'en', 'de');
const outPath = np.join(samplesDir, 'table.de.md');
await writeFile(outPath, result, 'utf-8');
const written = await readFile(outPath, 'utf-8');
expect(written).toBe(result);
},
{ timeout: 30_000 }
);
});

View File

@ -0,0 +1,152 @@
import { readFile } from 'node:fs/promises';
import np from 'node:path';
import { describe } from 'vitest';
import { getMdast } from '../ast/mdast';
import { Config, resolveConfig, UserConfig } from '../config';
import { extractJsonOrYamlStrings, extractMdastStrings } from '../extract';
import { format } from '../format';
const baseConfig: UserConfig = {
sourceLanguage: 'en',
outputLanguages: ['zh'],
directories: [['', '']],
cwd: 'src/__test__/__samples__'
};
async function extract(
path: string,
config: Config,
from: 'markdown' | 'json' | 'yaml' = 'markdown'
) {
const resolvedPath = np.resolve(config.cwd, path);
const source = await readFile(resolvedPath, { encoding: 'utf-8' });
if (from === 'markdown')
return extractMdastStrings({ mdast: getMdast(await format(source)), config });
return extractJsonOrYamlStrings({ source, type: from, config });
}
describe('extract frontmatter field string values', (test) => {
test('ignore empty frontmatter', async ({ expect }) => {
const strings = await extract('frontmatter/empty.md', resolveConfig(baseConfig));
expect(strings.length).toBe(0);
});
test('filter frontmatter fields based on configuration', async ({ expect }) => {
const strings = await extract(
'frontmatter/index.md',
resolveConfig({
...baseConfig,
frontmatterFields: {
include: ['title', 'tags', 'description'],
exclude: ['description']
}
})
);
expect(strings).toMatchSnapshot();
});
});
describe('extract jsx children and attribute string values', (test) => {
test('recursively extract strings from nested jsx components', async ({ expect }) => {
const strings = await extract(
'jsx/nested.mdx',
resolveConfig({
...baseConfig,
jsxComponents: {
include: {
Block: { children: true, attributes: [] }
}
}
})
);
expect(strings).toMatchSnapshot();
});
test('recursively extract strings from html elements and jsx components inside attributes', async ({
expect
}) => {
const strings = await extract(
'jsx/jsx-in-prop.mdx',
resolveConfig({
...baseConfig,
jsxComponents: {
include: {
Card: {
children: true,
attributes: ['header']
},
List: {
children: false,
attributes: ['items']
}
}
}
})
);
expect(strings).toMatchSnapshot();
});
test('ignore some HTML elements by default', async ({ expect }) => {
const strings = await extract('jsx/code-and-pre.mdx', resolveConfig(baseConfig));
expect(strings).toMatchSnapshot();
});
});
describe('extract strings from JSON based on configuration', (test) => {
test('do not extract any string if no property name is included in the config', async ({
expect
}) => {
const strings = await extract('json/navbar.json', resolveConfig(baseConfig), 'json');
expect(strings.length).toBe(0);
});
test('filter properties based on the config', async ({ expect }) => {
const strings = await extract(
'json/navbar.json',
resolveConfig({
...baseConfig,
jsonOrYamlProperties: {
include: ['message', 'description'],
exclude: ['message']
}
}),
'json'
);
expect(strings).toMatchSnapshot();
});
});
describe('extract strings from yaml based on configuration', (test) => {
test('do not extract any string if no property name is included in the config', async ({
expect
}) => {
const strings = await extract('yaml/authors.yml', resolveConfig(baseConfig), 'yaml');
expect(strings.length).toBe(0);
});
test('filter properties based on the config', async ({ expect }) => {
const strings = await extract(
'yaml/authors.yml',
resolveConfig({
...baseConfig,
jsonOrYamlProperties: {
include: ['name', 'title'],
exclude: ['name']
}
}),
'yaml'
);
expect(strings).toMatchSnapshot();
});
});

View File

@ -0,0 +1,14 @@
| Name | Kassandra - EDC |
| --------------------- | -------------------------------------------------------- |
| Druck | 20T - Hydraulischer Wagenheber mit pneumatischem Antrieb |
| Presse - Plattengröße | 1150mm \&amp; 1150mm x 2300mm verzahnt |
| Presse - Platten | 2-3 |
| Optionen | 2 aktive Kühlplatten \| 3 Heizplatten |
| Blattgröße | 60cm / 5-60mm dick |
| Elektrizität | 380V |
| Strom | 22kW |
| Gewicht | 710 kg |
| Größe | 1350 × 1350 × 1400 mm |
| Status | Ausgereift |
| Version | V1.0 (Revision A) |
| Lizenz | [CERN OHL v2](https://ohwr.org/cern_ohl_s_v2.txt) |

View File

@ -0,0 +1,15 @@
| Name | Cassandra EDC |
|------|------------------|
| Pressure | 20T Hydraulic Jack with Pneumatic Drive |
| Press Plate Size | 1150mm & 1150mm x 2300mm interlocked |
| Press Plates | 23 |
| Options | 2 Active Cooling Plates \| 3 Heating Plates |
| Sheet Size | 60cm / 560mm thick |
| Electricity | 380V |
| Power | 22kW |
| Weight | 710 Kg |
| Size | 1350 × 1350 × 1400 mm |
| Status | Mature |
| Version | V1.0 (Revision A) |
| License | [CERN OHL v2](https://ohwr.org/cern_ohl_s_v2.txt) |

View File

@ -0,0 +1,352 @@
import type {
BaseNode as EsBaseNode,
Identifier as EsIdentifier,
Program as EsProgram,
SwitchCase as EsSwitchCase,
CatchClause as EsCatchClause,
VariableDeclarator as EsVariableDeclarator,
ExpressionStatement as EsExpressionStatement,
BlockStatement as EsBlockStatement,
EmptyStatement as EsEmptyStatement,
DebuggerStatement as EsDebuggerStatement,
WithStatement as EsWithStatement,
ReturnStatement as EsReturnStatement,
LabeledStatement as EsLabeledStatement,
BreakStatement as EsBreakStatement,
ContinueStatement as EsContinueStatement,
IfStatement as EsIfStatement,
SwitchStatement as EsSwitchStatement,
ThrowStatement as EsThrowStatement,
TryStatement as EsTryStatement,
WhileStatement as EsWhileStatement,
DoWhileStatement as EsDoWhileStatement,
ForStatement as EsForStatement,
ForInStatement as EsForInStatement,
ForOfStatement as EsForOfStatement,
ClassDeclaration as EsClassDeclaration,
FunctionDeclaration as EsFunctionDeclaration,
VariableDeclaration as EsVariableDeclaration,
ModuleDeclaration as EsModuleDeclaration,
ImportDeclaration as EsImportDeclaration,
ExportDefaultDeclaration as EsExportDefaultDeclaration,
ExportNamedDeclaration as EsExportNamedDeclaration,
ExportAllDeclaration as EsExportAllDeclaration,
ThisExpression as EsThisExpression,
ArrayExpression as EsArrayExpression,
ObjectExpression as EsObjectExpression,
FunctionExpression as EsFunctionExpression,
ArrowFunctionExpression as EsArrowFunctionExpression,
YieldExpression as EsYieldExpression,
UnaryExpression as EsUnaryExpression,
UpdateExpression as EsUpdateExpression,
BinaryExpression as EsBinaryExpression,
AssignmentExpression as EsAssignmentExpression,
LogicalExpression as EsLogicalExpression,
MemberExpression as EsMemberExpression,
ConditionalExpression as EsConditionalExpression,
CallExpression as EsCallExpression,
NewExpression as EsNewExpression,
SequenceExpression as EsSequenceExpression,
TaggedTemplateExpression as EsTaggedTemplateExpression,
ClassExpression as EsClassExpression,
AwaitExpression as EsAwaitExpression,
ImportExpression as EsImportExpression,
ChainExpression as EsChainExpression,
SimpleLiteral as EsSimpleLiteral,
RegExpLiteral as EsRegExpLiteral,
BigIntLiteral as EsBigIntLiteral,
TemplateLiteral as EsTemplateLiteral,
PrivateIdentifier as EsPrivateIdentifier,
Property as EsProperty,
MetaProperty as EsMetaProperty,
PropertyDefinition as EsPropertyDefinition,
AssignmentProperty as EsAssignmentProperty,
Super as EsSuper,
TemplateElement as EsTemplateElement,
SpreadElement as EsSpreadElement,
ObjectPattern as EsObjectPattern,
ArrayPattern as EsArrayPattern,
RestElement as EsRestElement,
AssignmentPattern as EsAssignmentPattern,
Class as EsClass,
ClassBody as EsClassBody,
StaticBlock as EsStaticBlock,
MethodDefinition as EsMethodDefinition,
ModuleSpecifier as EsModuleSpecifier,
ImportSpecifier as EsImportSpecifier,
ImportNamespaceSpecifier as EsImportNamespaceSpecifier,
ImportDefaultSpecifier as EsImportDefaultSpecifier,
ExportSpecifier as EsExportSpecifier
} from 'estree';
import type {
JSXAttribute as EsJsxAttribute,
JSXClosingElement as EsJsxClosingElement,
JSXClosingFragment as EsJsxClosingFragment,
JSXElement as EsJsxElement,
JSXEmptyExpression as EsJsxEmptyExpression,
JSXExpressionContainer as EsJsxExpressionContainer,
JSXFragment as EsJsxFragment,
JSXIdentifier as EsJsxIdentifier,
JSXMemberExpression as EsJsxMemberExpression,
JSXNamespacedName as EsJsxNamespacedName,
JSXOpeningElement as EsJsxOpeningElement,
JSXOpeningFragment as EsJsxOpeningFragment,
JSXSpreadAttribute as EsJsxSpreadAttribute,
JSXSpreadChild as EsJsxSpreadChild,
JSXText as EsJsxText
} from 'estree-jsx';
export function esNodeIs<T extends keyof EsNodeMap>(node: EsNode, type: T): node is EsNodeMap[T] {
return node ? node.type === type : false;
}
export function resolveEstreePropertyPath(
node: EsProperty,
parents: EsNode[],
attributeName: string
): string | undefined {
if (!esNodeIs(parents[2], 'ArrayExpression') && !esNodeIs(parents[2], 'ObjectExpression')) return;
if (!esNodeIs(node.key, 'Identifier')) return;
const names = [node.key.name];
for (let i = parents.length - 1; i > 1; i--) {
const parent = parents[i];
if (esNodeIs(parent, 'ArrayExpression') || esNodeIs(parent, 'ObjectExpression')) continue;
if (esNodeIs(parent, 'Property')) {
if (!esNodeIs(parent.key, 'Identifier')) return;
names.push(parent.key.name);
continue;
}
return;
}
names.push(attributeName);
return names.reverse().join('.');
}
/**
* ============================================================
*/
export type {
EsBaseNode,
EsIdentifier,
EsProgram,
EsSwitchCase,
EsCatchClause,
EsVariableDeclarator,
EsExpressionStatement,
EsBlockStatement,
EsEmptyStatement,
EsDebuggerStatement,
EsWithStatement,
EsReturnStatement,
EsLabeledStatement,
EsBreakStatement,
EsContinueStatement,
EsIfStatement,
EsSwitchStatement,
EsThrowStatement,
EsTryStatement,
EsWhileStatement,
EsDoWhileStatement,
EsForStatement,
EsForInStatement,
EsForOfStatement,
EsClassDeclaration,
EsFunctionDeclaration,
EsVariableDeclaration,
EsModuleDeclaration,
EsImportDeclaration,
EsExportDefaultDeclaration,
EsExportNamedDeclaration,
EsExportAllDeclaration,
EsThisExpression,
EsArrayExpression,
EsObjectExpression,
EsFunctionExpression,
EsArrowFunctionExpression,
EsYieldExpression,
EsUnaryExpression,
EsUpdateExpression,
EsBinaryExpression,
EsAssignmentExpression,
EsLogicalExpression,
EsMemberExpression,
EsConditionalExpression,
EsCallExpression,
EsNewExpression,
EsSequenceExpression,
EsTaggedTemplateExpression,
EsClassExpression,
EsAwaitExpression,
EsImportExpression,
EsChainExpression,
EsSimpleLiteral,
EsRegExpLiteral,
EsBigIntLiteral,
EsTemplateLiteral,
EsPrivateIdentifier,
EsProperty,
EsMetaProperty,
EsPropertyDefinition,
EsAssignmentProperty,
EsSuper,
EsTemplateElement,
EsSpreadElement,
EsObjectPattern,
EsArrayPattern,
EsRestElement,
EsAssignmentPattern,
EsClass,
EsClassBody,
EsStaticBlock,
EsMethodDefinition,
EsModuleSpecifier,
EsImportSpecifier,
EsImportNamespaceSpecifier,
EsImportDefaultSpecifier,
EsExportSpecifier
};
export type {
EsJsxAttribute,
EsJsxClosingElement,
EsJsxClosingFragment,
EsJsxElement,
EsJsxEmptyExpression,
EsJsxExpressionContainer,
EsJsxFragment,
EsJsxIdentifier,
EsJsxMemberExpression,
EsJsxNamespacedName,
EsJsxOpeningElement,
EsJsxOpeningFragment,
EsJsxSpreadAttribute,
EsJsxSpreadChild,
EsJsxText
};
export type EsNode = EsNodeMap[keyof EsNodeMap];
export type EsNodeMap = EsExpressionMap &
EsLiteralMap &
EsFunctionMap &
EsPatternMap &
EsStatementMap &
EsJsxMap & {
CatchClause: EsCatchClause;
Class: EsClass;
ClassBody: EsClassBody;
MethodDefinition: EsMethodDefinition;
ModuleDeclaration: EsModuleDeclaration;
ModuleSpecifier: EsModuleSpecifier;
PrivateIdentifier: EsPrivateIdentifier;
Program: EsProgram;
Property: EsProperty;
PropertyDefinition: EsPropertyDefinition;
SpreadElement: EsSpreadElement;
Super: EsSuper;
SwitchCase: EsSwitchCase;
TemplateElement: EsTemplateElement;
VariableDeclarator: EsVariableDeclarator;
};
export type EsExpressionMap = EsLiteralMap & {
ArrayExpression: EsArrayExpression;
ArrowFunctionExpression: EsArrowFunctionExpression;
AssignmentExpression: EsAssignmentExpression;
AwaitExpression: EsAwaitExpression;
BinaryExpression: EsBinaryExpression;
CallExpression: EsCallExpression;
ChainExpression: EsChainExpression;
ClassExpression: EsClassExpression;
ConditionalExpression: EsConditionalExpression;
FunctionExpression: EsFunctionExpression;
Identifier: EsIdentifier;
ImportExpression: EsImportExpression;
LogicalExpression: EsLogicalExpression;
MemberExpression: EsMemberExpression;
MetaProperty: EsMetaProperty;
NewExpression: EsNewExpression;
ObjectExpression: EsObjectExpression;
SequenceExpression: EsSequenceExpression;
TaggedTemplateExpression: EsTaggedTemplateExpression;
TemplateLiteral: EsTemplateLiteral;
ThisExpression: EsThisExpression;
UnaryExpression: EsUnaryExpression;
UpdateExpression: EsUpdateExpression;
YieldExpression: EsYieldExpression;
};
export interface EsLiteralMap {
Literal: EsSimpleLiteral | EsRegExpLiteral | EsBigIntLiteral;
SimpleLiteral: EsSimpleLiteral;
RegExpLiteral: EsRegExpLiteral;
BigIntLiteral: EsBigIntLiteral;
}
export interface EsFunctionMap {
FunctionDeclaration: EsFunctionDeclaration;
FunctionExpression: EsFunctionExpression;
ArrowFunctionExpression: EsArrowFunctionExpression;
}
export interface EsPatternMap {
Identifier: EsIdentifier;
ObjectPattern: EsObjectPattern;
ArrayPattern: EsArrayPattern;
RestElement: EsRestElement;
AssignmentPattern: EsAssignmentPattern;
MemberExpression: EsMemberExpression;
}
export type EsStatementMap = EsDeclarationMap & {
ExpressionStatement: EsExpressionStatement;
BlockStatement: EsBlockStatement;
StaticBlock: EsStaticBlock;
EmptyStatement: EsEmptyStatement;
DebuggerStatement: EsDebuggerStatement;
WithStatement: EsWithStatement;
ReturnStatement: EsReturnStatement;
LabeledStatement: EsLabeledStatement;
BreakStatement: EsBreakStatement;
ContinueStatement: EsContinueStatement;
IfStatement: EsIfStatement;
SwitchStatement: EsSwitchStatement;
ThrowStatement: EsThrowStatement;
TryStatement: EsTryStatement;
WhileStatement: EsWhileStatement;
DoWhileStatement: EsDoWhileStatement;
ForStatement: EsForStatement;
ForInStatement: EsForInStatement;
ForOfStatement: EsForOfStatement;
};
export interface EsDeclarationMap {
FunctionDeclaration: EsFunctionDeclaration;
VariableDeclaration: EsVariableDeclaration;
ClassDeclaration: EsClassDeclaration;
}
export interface EsJsxMap {
JSXAttribute: EsJsxAttribute;
JSXClosingElement: EsJsxClosingElement;
JSXClosingFragment: EsJsxClosingFragment;
JSXElement: EsJsxElement;
JSXEmptyExpression: EsJsxEmptyExpression;
JSXExpressionContainer: EsJsxExpressionContainer;
JSXFragment: EsJsxFragment;
JSXIdentifier: EsJsxIdentifier;
JSXMemberExpression: EsJsxMemberExpression;
JSXNamespacedName: EsJsxNamespacedName;
JSXOpeningElement: EsJsxOpeningElement;
JSXOpeningFragment: EsJsxOpeningFragment;
JSXSpreadAttribute: EsJsxSpreadAttribute;
JSXSpreadChild: EsJsxSpreadChild;
JSXText: EsJsxText;
}

View File

@ -0,0 +1,109 @@
import { isRegExp } from 'node:util/types';
import { EsNode, esNodeIs, EsNodeMap, EsProgram } from './estree.js';
export const DEFAULT_ESWALKERS: EsWalkers = {
Program(node, parents, process) {
parents.push(node);
for (const statement of node.body) {
process(statement, parents);
}
parents.pop();
},
ExpressionStatement(node, parents, process) {
parents.push(node);
process(node.expression, parents);
parents.pop();
},
ArrayExpression(node, parents, process) {
parents.push(node);
for (const element of node.elements) {
process(element, parents);
}
parents.pop();
},
ObjectExpression(node, parents, process) {
parents.push(node);
for (const property of node.properties) {
process(property, parents);
}
parents.pop();
},
Property(node, parents, process) {
parents.push(node);
process(node.key, parents);
process(node.value, parents);
parents.pop();
},
JSXElement(node, parents, process) {
parents.push(node);
for (const child of node.children) {
process(child, parents);
}
for (const attribute of node.openingElement.attributes) {
process(attribute, parents);
}
parents.pop();
},
JSXAttribute(node, parents, process) {
parents.push(node);
if (node.value) {
process(node.value, parents);
}
parents.pop();
}
};
export function eswalk(
ast: EsProgram,
visitors: EsVisitors,
walkers: EsWalkers = DEFAULT_ESWALKERS
) {
const process: EsProcessor = (node, parents) => {
if (!node) return;
let type = node.type as keyof EsNodeMap;
if (esNodeIs(node, 'Literal')) {
type =
typeof node.value === 'bigint'
? 'BigIntLiteral'
: isRegExp(node.value)
? 'RegExpLiteral'
: 'SimpleLiteral';
}
const visit = visitors[type] as EsVisitor<typeof type>;
const walk = walkers[type] as EsWalker<typeof type>;
let keepWalking = true;
if (visit !== undefined) {
const signal = visit(node, parents);
keepWalking = signal === false ? false : true;
}
if (keepWalking && walk) walk(node, parents, process);
};
process(ast, []);
}
export interface EsProcessor {
(node: EsNode | null, parents: EsNode[]): void;
}
export interface EsVisitor<NodeType extends keyof EsNodeMap> {
(node: EsNodeMap[NodeType], parents: EsNode[]): boolean | void;
}
export type EsVisitors = {
[NodeType in keyof EsNodeMap]?: EsVisitor<NodeType>;
};
export interface EsWalker<NodeType extends keyof EsNodeMap> {
(node: EsNodeMap[NodeType], parents: EsNode[], process: EsProcessor): void;
}
export type EsWalkers = {
[NodeType in keyof Partial<EsNodeMap>]: EsWalker<NodeType>;
};

View File

@ -0,0 +1,257 @@
import type {
Root as MdRoot,
Blockquote as MdBlockquote,
Break as MdBreak,
Code as MdCode,
Definition as MdDefinition,
Delete as MdDelete,
Emphasis as MdEmphasis,
Footnote as MdFootnote,
FootnoteDefinition as MdFootnoteDefinition,
FootnoteReference as MdFootnoteReference,
HTML as MdHTML,
Heading as MdHeading,
Image as MdImage,
ImageReference as MdImageReference,
InlineCode as MdInlineCode,
Link as MdLink,
LinkReference as MdLinkReference,
List as MdList,
ListItem as MdListItem,
Paragraph as MdParagraph,
Strong as MdStrong,
Table as MdTable,
TableCell as MdTableCell,
TableRow as MdTableRow,
Text as MdText,
ThematicBreak as MdThematicBreak,
YAML as MdYaml
} from 'mdast';
import type {
MdxFlowExpression,
MdxJsxAttribute,
MdxJsxAttributeValueExpression,
MdxJsxExpressionAttribute,
MdxJsxFlowElement,
MdxJsxTextElement,
MdxTextExpression,
MdxjsEsm
} from 'mdast-util-mdx';
import type { UnNode } from './unist.js';
import { fromMarkdown } from 'mdast-util-from-markdown';
import { frontmatterFromMarkdown, frontmatterToMarkdown } from 'mdast-util-frontmatter';
import { gfmTableFromMarkdown, gfmTableToMarkdown } from 'mdast-util-gfm-table';
import { htmlCommentFromMarkdown, htmlCommentToMarkdown } from '../vendor/mdast-util-html-comment.js';
import { mdxFromMarkdown, mdxToMarkdown } from 'mdast-util-mdx';
import { toMarkdown } from 'mdast-util-to-markdown';
import { frontmatter } from 'micromark-extension-frontmatter';
import { gfmTable } from 'micromark-extension-gfm-table';
import { htmlComment } from '../vendor/micromark-extension-html-comment.js';
import { mdxjs } from 'micromark-extension-mdxjs';
declare module 'mdast' {
export interface PhrasingContentMap extends StaticPhrasingContentMap {
mdxJsxFlowElement: MdxJsxFlowElement;
mdxJsxTextElement: MdxJsxTextElement;
mdxFlowExpression: MdxFlowExpression;
mdxTextExpression: MdxTextExpression;
}
}
export function mdNodeIs<T extends MdNodeType>(
node: UnNode | undefined,
type: T
): node is T extends MdRoot['type']
? MdRoot
: T extends MdBlockquote['type']
? MdBlockquote
: T extends MdBreak['type']
? MdBreak
: T extends MdCode['type']
? MdCode
: T extends MdDefinition['type']
? MdDefinition
: T extends MdDelete['type']
? MdDelete
: T extends MdEmphasis['type']
? MdEmphasis
: T extends MdFootnote['type']
? MdFootnote
: T extends MdFootnoteDefinition['type']
? MdFootnoteDefinition
: T extends MdFootnoteReference['type']
? MdFootnoteReference
: T extends MdHTML['type']
? MdHTML
: T extends MdHeading['type']
? MdHeading
: T extends MdImage['type']
? MdImage
: T extends MdImageReference['type']
? MdImageReference
: T extends MdInlineCode['type']
? MdInlineCode
: T extends MdLink['type']
? MdLink
: T extends MdLinkReference['type']
? MdLinkReference
: T extends MdList['type']
? MdList
: T extends MdListItem['type']
? MdListItem
: T extends MdParagraph['type']
? MdParagraph
: T extends MdStrong['type']
? MdStrong
: T extends MdTable['type']
? MdTable
: T extends MdTableCell['type']
? MdTableCell
: T extends MdTableRow['type']
? MdTableRow
: T extends MdText['type']
? MdText
: T extends MdThematicBreak['type']
? MdThematicBreak
: T extends MdYaml
? MdYaml
: T extends MdxFlowExpression['type']
? MdxFlowExpression
: T extends MdxJsxAttribute['type']
? MdxJsxAttribute
: T extends MdxJsxAttributeValueExpression['type']
? MdxJsxAttributeValueExpression
: T extends MdxJsxExpressionAttribute['type']
? MdxJsxExpressionAttribute
: T extends MdxJsxFlowElement['type']
? MdxJsxFlowElement
: T extends MdxJsxTextElement['type']
? MdxJsxTextElement
: T extends MdxTextExpression['type']
? MdxTextExpression
: MdxjsEsm {
return node ? node.type === type : false;
}
export function mdNodeIsJsxElement(node: UnNode): node is MdxJsxFlowElement | MdxJsxTextElement {
return mdNodeIs(node, 'mdxJsxFlowElement') || mdNodeIs(node, 'mdxJsxTextElement');
}
/**
* Get MDX flavored `mdast`.
*/
export function getMdast(markdown: string): MdRoot {
return fromMarkdown(markdown, {
extensions: [frontmatter('yaml'), mdxjs(), gfmTable, htmlComment()],
mdastExtensions: [frontmatterFromMarkdown('yaml'), mdxFromMarkdown(), gfmTableFromMarkdown, htmlCommentFromMarkdown()]
});
}
export function getMarkdown(mdast: MdRoot): string {
return toMarkdown(mdast, {
extensions: [frontmatterToMarkdown('yaml'), mdxToMarkdown(), gfmTableToMarkdown(), htmlCommentToMarkdown()],
listItemIndent: 'one',
join: [
(__, _, parent) => {
if (mdNodeIsJsxElement(parent)) {
return 0;
}
// Keep list items tight (no blank lines between them)
if (mdNodeIs(parent, 'list')) {
return 0;
}
// Keep content within a list item tight (e.g. paragraph + nested sub-list)
if (mdNodeIs(parent, 'listItem')) {
return 0;
}
return 1;
}
]
});
}
/**
* ============================================================
*/
export type MdNodeType =
| MdRoot['type']
| MdBlockquote['type']
| MdBreak['type']
| MdCode['type']
| MdDefinition['type']
| MdDelete['type']
| MdEmphasis['type']
| MdFootnote['type']
| MdFootnoteDefinition['type']
| MdFootnoteReference['type']
| MdHTML['type']
| MdHeading['type']
| MdImage['type']
| MdImageReference['type']
| MdInlineCode['type']
| MdLink['type']
| MdLinkReference['type']
| MdList['type']
| MdListItem['type']
| MdParagraph['type']
| MdStrong['type']
| MdTable['type']
| MdTableCell['type']
| MdTableRow['type']
| MdText['type']
| MdThematicBreak['type']
| MdYaml['type']
| MdxFlowExpression['type']
| MdxJsxAttribute['type']
| MdxJsxAttributeValueExpression['type']
| MdxJsxExpressionAttribute['type']
| MdxJsxFlowElement['type']
| MdxJsxTextElement['type']
| MdxTextExpression['type']
| MdxjsEsm['type'];
export type {
MdRoot,
MdBlockquote,
MdBreak,
MdCode,
MdDefinition,
MdDelete,
MdEmphasis,
MdFootnote,
MdFootnoteDefinition,
MdFootnoteReference,
MdHTML,
MdHeading,
MdImage,
MdImageReference,
MdInlineCode,
MdLink,
MdLinkReference,
MdList,
MdListItem,
MdParagraph,
MdStrong,
MdTable,
MdTableCell,
MdTableRow,
MdText,
MdThematicBreak,
MdYaml
};
export type {
MdxFlowExpression,
MdxJsxAttribute,
MdxJsxAttributeValueExpression,
MdxJsxExpressionAttribute,
MdxJsxFlowElement,
MdxJsxTextElement,
MdxTextExpression,
MdxjsEsm
};

View File

@ -0,0 +1,19 @@
import type { Position as UnPosition } from 'unist';
export function unNodeIsParent(node: UnNode): node is UnParent {
return 'children' in node;
}
/**
* ============================================================
*/
export interface UnNode {
type: string;
position?: UnPosition;
data?: unknown;
}
export interface UnParent extends UnNode {
children: (UnNode | UnParent)[];
}

View File

@ -0,0 +1,40 @@
import { type UnNode, type UnParent, unNodeIsParent } from './unist.js';
const NEXT = true;
const STOP = false;
export function unwalk(
node: UnNode,
visit: UnVisitor,
filter?: (node: UnNode, parent: UnParent | undefined) => boolean
) {
let next = true;
function step(node: UnNode, parent: UnParent | undefined, index: number | undefined) {
if (filter && !filter(node, parent)) return;
if (unNodeIsParent(node)) {
for (let i = 0; i < node.children.length; i++) {
if (!next) break;
const child = node.children[i];
step(child, node, i);
}
node.children = node.children.filter((child) => child);
}
if (!next) return;
const signal = visit(node, parent, index);
next = signal === undefined || NEXT ? NEXT : STOP;
}
step(node, undefined, undefined);
}
export interface UnVisitor {
(node: UnNode | UnParent, parent: UnParent | undefined, index: number | undefined):
| boolean
| void;
}

View File

@ -0,0 +1,586 @@
import type { SourceLanguageCode, TargetLanguageCode } from 'deepl-node';
import type { MdNodeType } from './ast/mdast.js';
import { isBoolean } from './utils.js';
export interface ConfigBase {
/**
* Source's language code. Based on DeepL supported languages.
*/
sourceLanguage: SourceLanguageCode;
/**
* Output's languages code. Based on DeepL supported languages.
*/
outputLanguages: TargetLanguageCode[];
/**
* Sources and ouputs directories pairs. $langcode$ variable
* is provided to dynamically define directory.
*
* e.g. [ ["docs", "i18n/$langcode$/docs"], ["blog", "i18n/$langcode$/blog"] ]
*/
directories: [string, string][];
}
export interface Config extends ConfigBase {
/**
* Override current working directory, defaults to `process.cwd()`.
*/
cwd: string;
/**
* By default, all .md, .mdx, .json, and .yaml|.yml files inside
* source directories will be included.
*
* Define glob patterns to filter what files to include or exclude.
* But, the end result is still restricted by file types (.md, .mdx, .json).
*/
files: {
include?: string[];
exclude: string[];
};
/**
* Frontmatter fields.
*/
frontmatterFields: {
include: string[];
exclude: string[];
};
/**
* Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`.
*/
markdownNodes: {
default: boolean;
include: MdNodeType[];
exclude: MdNodeType[];
};
/**
* HTML elements to include and exlcude, down to the level of attributes
* and children. Include all HTML elements text content
* and some global attributes such as title and placeholder.
*/
htmlElements: {
include: Partial<{ [Tag in HtmlTag]: { children: boolean; attributes: string[] } }>;
exclude: HtmlTag[];
};
/**
* JSX components to include and exclude, down to the level of attributes
* and children. Include all JSX components text children
* and exclude all attributes by default.
*
* Support array, object, and jsx attribute value. For object and array value,
* you can specify the access path starting with the attribute name
* e.g. `items.description` to translate `items={[{description: "..."}]}.
*/
jsxComponents: {
default: boolean;
include: { [Name: string]: { children: boolean; attributes: string[] } };
exclude: string[];
};
/**
* JSON or YAML file properties to include and exclude.
* Exclude all properties by default.
*/
jsonOrYamlProperties: {
include: (string | number | symbol)[];
exclude: (string | number | symbol)[];
};
}
export interface UserConfig extends ConfigBase {
/**
* Override current working directory, defaults to `process.cwd()`.
*/
cwd?: string;
/**
* By default, all .md, .mdx, .json, and .yaml|.yml files inside
* source directories will be included.
*
* Define glob patterns to filter what files to include or exclude.
* But, the end result is still restricted by file types (.md, .mdx, .json).
*/
files?: {
include?: string[];
exclude?: string[];
};
/**
* Frontmatter fields.
*/
frontmatterFields?: {
include?: string[];
exclude?: string[];
};
/**
* Markdown node types to include or exclude based on MDAST. Defaults to exclude `code` and `link`.
*/
markdownNodes?: {
default?: boolean;
include?: MdNodeType[];
exclude?: MdNodeType[];
};
/**
* HTML elements to include and exlcude, down to the level of attributes
* and children. Include all HTML elements text content
* and some global attributes such as title and placeholder.
*/
htmlElements?: {
default?: boolean;
include?: Partial<{ [Tag in HtmlTag]: { children: boolean; attributes: string[] } }>;
exclude?: HtmlTag[];
};
/**
* JSX components to include and exclude, down to the level of attributes
* and children. Include all JSX components text children
* and exclude all attributes by default.
*
* Support array, object, and jsx attribute value. For object and array value,
* you can specify the access path starting with the attribute name
* e.g. `items.description` to translate `items={[{description: "..."}]}.
*/
jsxComponents?: {
default?: boolean;
include?: { [Name: string]: { children: boolean; attributes: string[] } };
exclude?: string[];
};
/**
* JSON or YAML file properties to include and exclude.
* Exclude all properties by default.
*/
jsonOrYamlProperties?: {
include?: string[];
exclude?: string[];
};
}
export type HtmlElementsConfig = { [Tag in HtmlTag]: { children: boolean; attributes: string[] } };
export const HTML_ELEMENTS_CONFIG: HtmlElementsConfig = getHtmlElementsConfig();
function getHtmlElementsConfig(): HtmlElementsConfig {
const includeChildren: HtmlTag[] = [
'a',
'abbr',
'address',
'article',
'aside',
'audio',
'b',
'bdi',
'bdo',
'blockquote',
'body',
'button',
'canvas',
'caption',
'cite',
'col',
'colgroup',
'data',
'datalist',
'dd',
'del',
'details',
'dfn',
'dialog',
'div',
'dl',
'dt',
'em',
'fieldset',
'figcaption',
'figure',
'footer',
'form',
'h1',
'h2',
'h3',
'h4',
'h5',
'h6',
'header',
'html',
'i',
'input',
'ins',
'label',
'legend',
'li',
'main',
'mark',
'meter',
'nav',
'ol',
'optgroup',
'output',
'p',
'progress',
'q',
'rp',
's',
'samp',
'section',
'select',
'small',
'span',
'strong',
'sub',
'summary',
'sup',
'table',
'tbody',
'td',
'template',
'text-area',
'tfoot',
'th',
'thead',
'time',
'title',
'tr',
'track',
'u',
'ul'
];
const excludeChildren: HtmlTag[] = [
'area',
'base',
'br',
'code',
'embed',
'head',
'hr',
'iframe',
'img',
'kbd',
'link',
'meta',
'noscript',
'object',
'param',
'picture',
'pre',
'rt',
'ruby',
'script',
'source',
'style',
'svg',
'var',
'video',
'qbr'
];
const config: Partial<HtmlElementsConfig> = {};
for (const tag of includeChildren) {
config[tag] = {
children: true,
attributes: ['title']
};
}
for (const tag of excludeChildren) {
config[tag] = {
children: false,
attributes: ['title']
};
}
return config as HtmlElementsConfig;
}
export const HTML_TAGS = Object.keys(HTML_ELEMENTS_CONFIG) as HtmlTag[];
export function isHtmlTag(name: string): name is HtmlTag {
return HTML_TAGS.includes(name as HtmlTag);
}
export function resolveConfig({
sourceLanguage,
outputLanguages,
directories,
cwd,
files,
markdownNodes,
frontmatterFields,
htmlElements,
jsxComponents,
jsonOrYamlProperties
}: UserConfig): Config {
return {
sourceLanguage,
outputLanguages,
directories,
cwd: cwd ?? '',
files: files
? {
include: files.include,
exclude: files.exclude ?? []
}
: { exclude: [] },
markdownNodes: markdownNodes
? {
default: isBoolean(markdownNodes.default) ? markdownNodes.default : true,
include: markdownNodes.include ?? [],
exclude: markdownNodes.exclude ?? ['code']
}
: { default: true, include: [], exclude: ['code'] },
frontmatterFields: frontmatterFields
? {
include: frontmatterFields.include ?? [],
exclude: frontmatterFields.exclude ?? []
}
: { include: [], exclude: [] },
htmlElements: htmlElements
? {
include: htmlElements.include
? (isBoolean(htmlElements.default) && htmlElements.default) ||
htmlElements.default === undefined
? { ...HTML_ELEMENTS_CONFIG, ...htmlElements.include }
: htmlElements.include
: isBoolean(htmlElements.default) && !htmlElements.default
? {}
: HTML_ELEMENTS_CONFIG,
exclude: htmlElements.exclude ?? []
}
: { include: HTML_ELEMENTS_CONFIG, exclude: [] },
jsxComponents: jsxComponents
? {
default: isBoolean(jsxComponents.default) ? jsxComponents.default : true,
include: jsxComponents.include ?? {},
exclude: jsxComponents.exclude ?? []
}
: { default: true, include: {}, exclude: [] },
jsonOrYamlProperties: jsonOrYamlProperties
? { include: jsonOrYamlProperties.include ?? [], exclude: jsonOrYamlProperties.exclude ?? [] }
: { include: [], exclude: [] }
};
}
export function isFrontmatterFieldIncluded({
field,
config
}: {
field: string;
config: Config;
}): boolean {
return (
!config.frontmatterFields.exclude.includes(field) &&
config.frontmatterFields.include.includes(field)
);
}
export function isMarkdownNodeIncluded({
type,
config
}: {
type: MdNodeType;
config: Config;
}): boolean {
return (
!config.markdownNodes.exclude.includes(type) &&
(config.markdownNodes.default || config.markdownNodes.include.includes(type))
);
}
export function isHtmlElementIncluded({ tag, config }: { tag: HtmlTag; config: Config }): boolean {
return (
!config.htmlElements.exclude.includes(tag) &&
Object.keys(config.htmlElements.include).includes(tag)
);
}
export function isHtmlElementAttributeIncluded({
tag,
attribute,
config
}: {
tag: HtmlTag;
attribute: string;
config: Config;
}): boolean {
return (
isHtmlElementIncluded({ tag, config }) &&
config.htmlElements.include[tag]!.attributes.includes(attribute)
);
}
export function isHtmlElementChildrenIncluded({
tag,
config
}: {
tag: HtmlTag;
config: Config;
}): boolean {
return isHtmlElementIncluded({ tag, config }) && config.htmlElements.include[tag]!.children;
}
export function isJsxComponentIncluded({
name,
config
}: {
name: string;
config: Config;
}): boolean {
return (
!config.jsxComponents.exclude.includes(name) &&
(config.jsxComponents.default || Object.keys(config.jsxComponents.include).includes(name))
);
}
export function isJsxComponentAttributeIncluded({
name,
attribute,
config
}: {
name: string;
attribute: string;
config: Config;
}): boolean {
return (
!config.jsxComponents.exclude.includes(name) &&
Object.keys(config.jsxComponents.include).includes(name) &&
config.jsxComponents.include[name].attributes.includes(attribute)
);
}
export function isJsxComponentChildrenIncluded({
name,
config
}: {
name: string;
config: Config;
}): boolean {
return (
!config.jsxComponents.exclude.includes(name) &&
((Object.keys(config.jsxComponents.include).includes(name) &&
config.jsxComponents.include[name].children) ||
(!Object.keys(config.jsxComponents.include).includes(name) && config.jsxComponents.default))
);
}
export function isJsonOrYamlPropertyIncluded({
property,
config
}: {
config: Config;
property: string | number | symbol;
}): boolean {
return (
!config.jsonOrYamlProperties.exclude.includes(property) &&
config.jsonOrYamlProperties.include.includes(property)
);
}
export type HtmlTag =
| 'a'
| 'abbr'
| 'address'
| 'article'
| 'aside'
| 'audio'
| 'b'
| 'bdi'
| 'bdo'
| 'blockquote'
| 'body'
| 'button'
| 'canvas'
| 'caption'
| 'cite'
| 'col'
| 'colgroup'
| 'data'
| 'datalist'
| 'dd'
| 'del'
| 'details'
| 'dfn'
| 'dialog'
| 'div'
| 'dl'
| 'dt'
| 'em'
| 'fieldset'
| 'figcaption'
| 'figure'
| 'footer'
| 'form'
| 'h1'
| 'h2'
| 'h3'
| 'h4'
| 'h5'
| 'h6'
| 'header'
| 'html'
| 'i'
| 'input'
| 'ins'
| 'label'
| 'legend'
| 'li'
| 'main'
| 'mark'
| 'meter'
| 'nav'
| 'ol'
| 'optgroup'
| 'output'
| 'p'
| 'progress'
| 'q'
| 'rp'
| 's'
| 'samp'
| 'section'
| 'select'
| 'small'
| 'span'
| 'strong'
| 'sub'
| 'summary'
| 'sup'
| 'table'
| 'tbody'
| 'td'
| 'template'
| 'text-area'
| 'tfoot'
| 'th'
| 'thead'
| 'time'
| 'title'
| 'tr'
| 'track'
| 'u'
| 'ul'
| 'area'
| 'base'
| 'br'
| 'code'
| 'embed'
| 'head'
| 'hr'
| 'iframe'
| 'img'
| 'kbd'
| 'link'
| 'meta'
| 'noscript'
| 'object'
| 'param'
| 'picture'
| 'pre'
| 'rt'
| 'ruby'
| 'script'
| 'source'
| 'style'
| 'svg'
| 'var'
| 'video'
| 'qbr';

View File

@ -0,0 +1,267 @@
import { parse as parseYaml } from 'yaml';
import {
EsJsxElement,
EsJsxIdentifier,
esNodeIs,
resolveEstreePropertyPath
} from './ast/estree.js';
import { eswalk } from './ast/eswalk.js';
import { mdNodeIs, mdNodeIsJsxElement, MdNodeType } from './ast/mdast.js';
import type { UnNode } from './ast/unist.js';
import { unwalk } from './ast/unwalk.js';
import {
type Config,
isHtmlTag,
isFrontmatterFieldIncluded,
isHtmlElementIncluded,
isHtmlElementAttributeIncluded,
isJsonOrYamlPropertyIncluded,
isJsxComponentIncluded,
isJsxComponentAttributeIncluded,
isMarkdownNodeIncluded,
isHtmlElementChildrenIncluded,
isJsxComponentChildrenIncluded
} from './config.js';
import { isArray, isEmptyArray, isEmptyString, isObject, isString } from './utils.js';
export function extractMdastStrings({
mdast,
config
}: {
mdast: UnNode;
config: Config;
}): string[] {
const strings: string[] = [];
unwalk(
mdast,
(node, __, _) => {
if (mdNodeIs(node, 'text')) {
pushTidyString({ array: strings, string: node.value });
return;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue;
if (
!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config })
)
continue;
if (isString(attribute.value)) {
strings.push(attribute.value.trim());
} else if (attribute.value?.data?.estree) {
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _) {
if (isString(esnode.value))
pushTidyString({ array: strings, string: esnode.value });
}
});
}
}
} else {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue;
const componentName: string = node.name;
const isAttributeIncluded = isJsxComponentAttributeIncluded({
name: componentName,
attribute: attribute.name,
config
});
if (isString(attribute.value)) {
if (!isAttributeIncluded) continue;
strings.push(attribute.value.trim());
} else if (attribute.value?.data?.estree) {
if (
!config.jsxComponents.include[componentName] ||
!config.jsxComponents.include[componentName].attributes.some(
(attrName) =>
attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)
)
continue;
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _) {
if (isString(esnode.value))
pushTidyString({ array: strings, string: esnode.value });
if (esnode.value === 'aye') console.log('passed');
},
JSXElement(esnode, _) {
const name = (esnode.openingElement.name as EsJsxIdentifier).name;
if (isHtmlTag(name)) {
if (
!isHtmlElementIncluded({ tag: name, config }) ||
!isHtmlElementChildrenIncluded({ tag: name, config })
)
return false;
} else if (
!isJsxComponentIncluded({ name: name, config }) ||
!isJsxComponentChildrenIncluded({ name: name, config })
)
return false;
},
JSXAttribute(esnode, parents) {
const name =
typeof esnode.name.name === 'string' ? esnode.name.name : esnode.name.name.name;
const parentName = (
(parents[parents.length - 1] as EsJsxElement).openingElement
.name as EsJsxIdentifier
).name;
if (isHtmlTag(parentName)) {
if (
!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config })
)
return false;
} else if (
!config.jsxComponents.include[name] ||
!config.jsxComponents.include[name].attributes.some(
(attrName) =>
attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)
) {
return false;
}
},
JSXText(esnode, _) {
pushTidyString({ array: strings, string: esnode.value });
},
Property(esnode, parents) {
if (!esNodeIs(esnode, 'Identifier')) return false;
const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name);
if (
!propertyPath ||
!isJsxComponentAttributeIncluded({
name: componentName,
attribute: propertyPath,
config
})
)
return false;
}
});
}
}
}
}
if (mdNodeIs(node, 'yaml')) {
if (isEmptyArray(config.frontmatterFields.include)) return;
if (isEmptyString(node.value)) return;
const object: Record<string, any> = parseYaml(node.value);
for (const field in object) {
if (!isFrontmatterFieldIncluded({ field, config })) continue;
const value = object[field];
if (isString(value)) {
strings.push(value);
continue;
}
if (isArray(value)) {
for (const item of value) {
if (!isString(item)) continue;
strings.push(item);
}
}
}
return;
}
},
(node, parent) => {
if (!isMarkdownNodeIncluded({ type: node.type as MdNodeType, config })) return false;
if (parent && mdNodeIsJsxElement(parent) && parent.name) {
if (isHtmlTag(parent.name)) {
if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false;
} else {
if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false;
}
return true;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
if (!isHtmlElementIncluded({ tag: node.name, config })) return false;
} else {
if (!isJsxComponentIncluded({ name: node.name, config })) return false;
}
return true;
}
return true;
}
);
return strings;
}
export function extractJsonOrYamlStrings({
source,
type = 'json',
config
}: {
source: string;
type?: 'json' | 'yaml';
config: Config;
}): string[] {
const strings: string[] = [];
if (isEmptyArray(config.jsonOrYamlProperties.include)) return strings;
const parsed = type === 'json' ? JSON.parse(source) : parseYaml(source);
process(parsed);
function process(value: unknown, property?: string) {
if (typeof value === 'string') {
if (property && isJsonOrYamlPropertyIncluded({ property, config })) strings.push(value);
return;
}
if (isArray(value)) {
for (const item of value) {
process(item);
}
return;
}
if (isObject(value)) {
for (const property in value) {
const item = (value as Record<string | number | symbol, unknown>)[property];
process(item, property);
}
return;
}
}
return strings;
}
function pushTidyString({ array, string }: { array: string[]; string: string }) {
if (!/^\s*$/.test(string)) {
array.push(string.replace(/(^\n|\r|\t|\v)+\s*/, '').replace(/\s+$/, ' '));
}
}

View File

@ -0,0 +1,42 @@
import prettier from 'prettier';
import { getMarkdown, getMdast, mdNodeIs } from './ast/mdast.js';
import { unwalk } from './ast/unwalk.js';
export async function format(markdown: string) {
/**
* `printWidth` is set to Infinity and `proseWrap` is set to never
* to avoid unnecessary linebreaks that break translation result
*/
const mdast = getMdast(
await prettier.format(markdown, {
parser: 'mdx',
printWidth: Infinity,
proseWrap: 'never',
useTabs: true
})
);
/**
* remove empty surface flow expression nodes that sometimes
* are produced by prettier
*/
unwalk(
mdast,
(node, parent, index) => {
if (mdNodeIs(node, 'mdxFlowExpression') && expressionIsEmpty(node.value)) {
(parent!.children[index!] as unknown) = undefined;
}
},
(node, parent) => {
delete node.position;
return mdNodeIs(parent, 'root');
}
);
return getMarkdown(mdast);
}
function expressionIsEmpty(text: string): boolean {
const regex = /^('|")\s*('|")$/;
return regex.test(text);
}

View File

@ -0,0 +1,69 @@
import type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';
import { getMarkdown, getMdast } from './ast/mdast.js';
import type { UserConfig } from './config.js';
import { resolveConfig } from './config.js';
import { extractMdastStrings } from './extract.js';
import { format } from './format.js';
import { replaceMdastStrings } from './replace.js';
import { translateStrings } from './translate.js';
/**
* Options to control which parts of the markdown are translated.
*/
export type TranslateOptions = Omit<UserConfig, 'sourceLanguage' | 'outputLanguages' | 'directories'> & {
/** DeepL API key. Falls back to `DEEPL_AUTH_KEY` env var if not provided. */
apiKey?: string;
/** DeepL translation options (tagHandling, splitSentences, formality, glossaryId, etc.) */
deeplOptions?: TranslateTextOptions;
};
/**
* Translate markdown/MDX content from one language to another using DeepL.
*
* Requires `DEEPL_AUTH_KEY` environment variable to be set.
*
* @param content - Markdown or MDX string to translate
* @param sourceLang - Source language code (e.g. 'en', 'de', 'fr')
* @param targetLang - Target language code (e.g. 'de', 'en-US', 'fr')
* @param options - Optional config to control extraction (frontmatter, jsx, html, etc.)
* @returns Translated markdown string
*
* @example
* ```ts
* import { translate } from 'deepmark';
*
* const result = await translate('# Hello World', 'en', 'de');
* console.log(result); // '# Hallo Welt'
* ```
*/
export async function translate(
content: string,
sourceLang: SourceLanguageCode,
targetLang: TargetLanguageCode,
options?: TranslateOptions
): Promise<string> {
const { apiKey, deeplOptions, ...configOptions } = options ?? {};
const config = resolveConfig({
sourceLanguage: sourceLang,
outputLanguages: [targetLang],
directories: [['', '']],
...configOptions
});
// Format, parse, extract translatable strings
const formatted = await format(content);
const mdast = getMdast(formatted);
const strings = extractMdastStrings({ mdast, config });
if (strings.length === 0) return content;
// Translate via DeepL
const translated = await translateStrings(strings, sourceLang, targetLang, apiKey, deeplOptions);
// Replace strings in the AST and serialize back to markdown
const result = replaceMdastStrings({ mdast, strings: translated, config });
return getMarkdown(result);
}
export type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';

View File

View File

@ -0,0 +1,296 @@
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
import {
EsJsxElement,
EsJsxIdentifier,
esNodeIs,
resolveEstreePropertyPath
} from './ast/estree.js';
import { eswalk } from './ast/eswalk.js';
import type { MdNodeType, MdRoot } from './ast/mdast.js';
import { mdNodeIs, mdNodeIsJsxElement } from './ast/mdast.js';
import { unwalk } from './ast/unwalk.js';
import {
type Config,
isHtmlTag,
isFrontmatterFieldIncluded,
isHtmlElementIncluded,
isHtmlElementAttributeIncluded,
isJsonOrYamlPropertyIncluded,
isJsxComponentIncluded,
isJsxComponentAttributeIncluded,
isMarkdownNodeIncluded,
isHtmlElementChildrenIncluded,
isJsxComponentChildrenIncluded
} from './config.js';
import { isArray, isEmptyArray, isEmptyString, isObject, isString } from './utils.js';
export function replaceMdastStrings({
mdast,
config,
strings
}: {
mdast: MdRoot;
strings: string[];
config: Config;
}): MdRoot {
strings = strings.reverse();
unwalk(
mdast,
(node, __, _) => {
if (mdNodeIs(node, 'text')) {
node.value = strings.pop()!;
return;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue;
if (
!isHtmlElementAttributeIncluded({ tag: node.name, attribute: attribute.name, config })
)
continue;
if (isString(attribute.value)) {
attribute.value = strings.pop();
} else if (attribute.value?.data?.estree) {
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _) {
if (isString(esnode.value)) esnode.value = strings.pop()!;
}
});
}
}
} else {
for (const attribute of node.attributes) {
if (!mdNodeIs(attribute, 'mdxJsxAttribute')) continue;
const componentName: string = node.name;
const isAttributeIncluded = isJsxComponentAttributeIncluded({
name: componentName,
attribute: attribute.name,
config
});
if (isString(attribute.value)) {
if (!isAttributeIncluded) continue;
attribute.value = strings.pop();
} else if (attribute.value?.data?.estree) {
if (
!config.jsxComponents.include[componentName] ||
!config.jsxComponents.include[componentName].attributes.some(
(attrName) =>
attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)
)
continue;
const estree = attribute.value.data.estree;
eswalk(estree, {
SimpleLiteral(esnode, _) {
if (isString(esnode.value)) esnode.value = strings.pop()!;
},
JSXElement(esnode, _) {
const name = (esnode.openingElement.name as EsJsxIdentifier).name;
if (isHtmlTag(name)) {
if (
!isHtmlElementIncluded({ tag: name, config }) ||
!isHtmlElementChildrenIncluded({ tag: name, config })
)
return false;
} else if (
!isJsxComponentIncluded({ name: name, config }) ||
!isJsxComponentChildrenIncluded({ name: name, config })
)
return false;
},
JSXAttribute(esnode, parents) {
const name =
typeof esnode.name.name === 'string' ? esnode.name.name : esnode.name.name.name;
const parentName = (
(parents[parents.length - 1] as EsJsxElement).openingElement
.name as EsJsxIdentifier
).name;
if (isHtmlTag(parentName)) {
if (
!isHtmlElementAttributeIncluded({ tag: parentName, attribute: name, config })
)
return false;
} else if (
!config.jsxComponents.include[name] ||
!config.jsxComponents.include[name].attributes.some(
(attrName) =>
attrName === attribute.name || attrName.startsWith(`${attribute.name}.`)
)
) {
return false;
}
},
JSXText(esnode, _) {
esnode.value = strings.pop()!;
},
Property(esnode, parents) {
if (!esNodeIs(esnode, 'Identifier')) return false;
const propertyPath = resolveEstreePropertyPath(esnode, parents, attribute.name);
if (
!propertyPath ||
!isJsxComponentAttributeIncluded({
name: componentName,
attribute: propertyPath,
config
})
)
return false;
}
});
}
}
}
}
if (mdNodeIs(node, 'yaml')) {
if (isEmptyArray(config.frontmatterFields.include)) return;
if (isEmptyString(node.value)) return;
const object: Record<string, any> = parseYaml(node.value);
for (const field in object) {
if (!isFrontmatterFieldIncluded({ field, config })) continue;
const value = object[field];
if (isString(value)) {
object[field] = strings.pop();
continue;
}
if (isArray(value)) {
for (const [index, item] of value.entries()) {
if (!isString(item)) continue;
value[index] = strings.pop();
}
}
}
return;
}
},
(node, parent) => {
if (!isMarkdownNodeIncluded({ type: node.type as MdNodeType, config })) return false;
if (parent && mdNodeIsJsxElement(parent) && parent.name) {
if (isHtmlTag(parent.name)) {
if (!isHtmlElementChildrenIncluded({ tag: parent.name, config })) return false;
} else {
if (!isJsxComponentChildrenIncluded({ name: parent.name, config })) return false;
}
return true;
}
if (mdNodeIsJsxElement(node) && node.name) {
if (isHtmlTag(node.name)) {
if (!isHtmlElementIncluded({ tag: node.name, config })) return false;
} else {
if (!isJsxComponentIncluded({ name: node.name, config })) return false;
}
return true;
}
return true;
}
);
return mdast;
}
export function replaceJsonOrYamlStrings({
source,
type = 'json',
strings,
config
}: {
source: string;
type?: 'json' | 'yaml';
strings: string[];
config: Config;
}): string {
if (isEmptyArray(config.jsonOrYamlProperties.include)) return source;
strings = strings.reverse();
const parsed = type === 'json' ? JSON.parse(source) : parseYaml(source);
process({ value: parsed });
function process(args: { value: unknown; parent?: never; property?: never; index?: never }): void;
function process(args: {
value: unknown;
parent: unknown[];
property?: string | number | symbol;
index: number;
}): void;
function process(args: {
value: unknown;
parent: Record<string | number | symbol, unknown>;
property: string | number | symbol;
index?: never;
}): void;
function process({
value,
parent,
property,
index
}: {
value: unknown;
parent?: unknown[] | Record<string | number | symbol, unknown>;
property?: string | number | symbol;
index?: number;
}) {
if (isArray(value)) {
for (const [index, item] of value.entries()) {
process({ value: item, parent: value, property, index });
}
return;
}
if (isObject(value)) {
for (const property in value) {
const item = (value as Record<string | number | symbol, unknown>)[property];
process({ value: item, parent: value, property });
}
return;
}
if (typeof value === 'string') {
if (property && isJsonOrYamlPropertyIncluded({ property, config })) {
if (isArray(parent) && index) {
parent[index] = strings.pop();
return;
}
if (isObject(parent)) {
parent[property] = strings.pop();
return;
}
}
return;
}
}
if (type === 'json') return JSON.stringify(parsed);
return stringifyYaml(parsed);
}

View File

@ -0,0 +1,62 @@
import type { SourceLanguageCode, TargetLanguageCode, TranslateTextOptions } from 'deepl-node';
import { Translator } from 'deepl-node';
const DEFAULT_BATCH_SIZE = 50;
const MAX_RETRIES = 3;
/**
* Translate an array of strings from sourceLang to targetLang using DeepL.
* Batches requests and retries on rate-limit (429) or server (5xx) errors.
*/
export async function translateStrings(
strings: string[],
sourceLang: SourceLanguageCode,
targetLang: TargetLanguageCode,
apiKey?: string,
deeplOptions?: TranslateTextOptions,
batchSize: number = DEFAULT_BATCH_SIZE
): Promise<string[]> {
if (strings.length === 0) return [];
const key = apiKey ?? process.env.DEEPL_AUTH_KEY;
if (!key) throw new Error('DeepL API key must be provided via options.apiKey or DEEPL_AUTH_KEY environment variable');
const deepl = new Translator(key);
const translations: string[] = new Array(strings.length).fill('');
const textOptions: TranslateTextOptions = {
tagHandling: 'html',
splitSentences: 'nonewlines',
...deeplOptions
};
for (let i = 0; i < strings.length; i += batchSize) {
const batch = strings.slice(i, i + batchSize);
const results = await retry(() =>
deepl.translateText(batch, sourceLang, targetLang, textOptions)
);
for (let j = 0; j < batch.length; j++) {
translations[i + j] = results[j].text;
}
}
return translations;
}
async function retry<T>(fn: () => Promise<T>, retries = MAX_RETRIES): Promise<T> {
for (let attempt = 0; ; attempt++) {
try {
return await fn();
} catch (err: any) {
const status = err?.statusCode ?? err?.status;
const retryable = status === 429 || status === 456 || (status >= 500 && status < 600);
if (!retryable || attempt >= retries) throw err;
const delay = Math.min(1000 * 2 ** attempt, 10_000);
await new Promise((r) => setTimeout(r, delay));
}
}
}

View File

@ -0,0 +1,27 @@
export function isArray(value: unknown): value is any[] {
return Array.isArray(value);
}
export function isBoolean(value: unknown): value is boolean {
return typeof value === 'boolean';
}
export function isEmptyArray(array: any[]): boolean {
return array.length === 0;
}
export function isEmptyObject(object: Object): boolean {
return Object.keys(object).length === 0;
}
export function isEmptyString(string: string): boolean {
return string.length === 0;
}
export function isObject(value: unknown): value is Record<string | number | symbol, unknown> {
return isArray(value) ? false : typeof value == 'object' ? true : false;
}
export function isString(value: unknown): value is string {
return typeof value === 'string';
}

View File

@ -0,0 +1,40 @@
// @ts-nocheck — vendored from mdast-util-html-comment, uses custom handler types
import type { Extension } from 'mdast-util-from-markdown';
import type { Options } from 'mdast-util-to-markdown';
export function htmlCommentFromMarkdown(): Extension {
return {
canContainEols: ['htmlComment'],
enter: {
htmlComment() {
this.buffer();
}
},
exit: {
htmlComment(token) {
const string = this.resume();
this.enter(
{
// @ts-ignore
type: 'htmlComment',
value: string.slice(0, -3)
},
token
);
this.exit(token);
}
}
};
}
export function htmlCommentToMarkdown(): Options {
return {
handlers: {
htmlComment(node) {
return `<!--${node.value as string}-->`;
}
}
};
}

View File

@ -0,0 +1,128 @@
// @ts-nocheck — vendored from micromark-extension-html-comment, uses custom token types
import { factorySpace } from 'micromark-factory-space';
import { markdownLineEnding } from 'micromark-util-character';
import { codes } from 'micromark-util-symbol/codes.js';
import { types } from 'micromark-util-symbol/types.js';
import type { Code, Extension, HtmlExtension, State, Tokenizer } from 'micromark-util-types';
export function htmlComment(): Extension {
return {
flow: {
[codes.lessThan]: { tokenize, concrete: true }
},
text: {
[codes.lessThan]: { tokenize }
}
};
}
export function htmlCommentToHtml(): HtmlExtension {
return {
enter: {
htmlComment() {
this.buffer();
}
},
exit: {
htmlComment() {
this.resume();
}
}
};
}
const tokenize: Tokenizer = (effects, ok, nok) => {
let value: string = '';
return start;
function start(code: Code): State | void {
effects.enter('htmlComment');
effects.enter('htmlCommentMarker');
effects.consume(code);
value += '<';
return open;
}
function open(code: Code): State | void {
if (value === '<' && code === codes.exclamationMark) {
effects.consume(code);
value += '!';
return open;
}
if (code === codes.dash) {
if (value === '<!') {
effects.consume(code);
value += '-';
return open;
}
if (value === '<!-') {
effects.consume(code);
effects.exit('htmlCommentMarker');
value += '-';
return inside;
}
}
return nok(code);
}
function inside(code: Code): State | void {
if (code === codes.eof) return nok(code);
if (markdownLineEnding(code)) {
effects.exit(types.data);
return atLineEnding(code);
}
if (code === codes.greaterThan) {
return close(code);
}
if (value === '<!--') {
effects.enter('htmlCommentString');
effects.enter(types.data);
}
effects.consume(code);
if (code === codes.dash) {
value += '-';
} else {
value += '*';
}
return inside;
}
function atLineEnding(code: Code): State | void {
effects.enter(types.lineEnding);
effects.consume(code);
effects.exit(types.lineEnding);
return factorySpace(effects, afterLinePrefix, types.linePrefix);
}
function afterLinePrefix(code: Code): State | void {
if (markdownLineEnding(code)) return atLineEnding(code);
effects.enter(types.data);
return inside(code);
}
function close(code: Code): State | void {
if (value.length >= 6 && value.slice(-2) === '--') {
effects.consume(code);
effects.exit(types.data);
effects.exit('htmlCommentString');
effects.exit('htmlComment');
value += '>';
return ok;
}
return nok(code);
}
};

View File

@ -0,0 +1,27 @@
{
"compilerOptions": {
"allowJs": true,
"checkJs": true,
"forceConsistentCasingInFileNames": true,
"module": "NodeNext",
"moduleResolution": "NodeNext",
"declaration": true,
"emitDeclarationOnly": true,
"lib": [
"ESNext"
],
"noEmit": false,
"outDir": "./dist",
"rootDir": "src",
"skipLibCheck": true,
"sourceMap": true,
"strict": true,
"target": "ESNext"
},
"include": [
"src/**/*.ts"
],
"exclude": [
"src/__test__/**"
]
}

View File

@ -0,0 +1,23 @@
import { readFileSync } from 'fs';
import np from 'path';
const CWD = process.cwd();
// Load .env into process.env for tests
try {
const env = readFileSync(np.resolve(CWD, '.env'), 'utf-8');
for (const line of env.split('\n')) {
const match = line.match(/^\s*([\w.-]+)\s*=\s*"?([^"]*)"?\s*$/);
if (match) process.env[match[1]] ??= match[2];
}
} catch { }
/** @type { import('vite').UserConfig } */
export default {
resolve: {
alias: {
$types: np.resolve(CWD, './src/types/index.js'),
$utils: np.resolve(CWD, './src/utils/index.js')
}
}
};

View File

@ -17,7 +17,7 @@ const defaultOptions = (yargs) => {
describe: 'Target file',
}).option('dst', {
describe: 'Path to the output file(s). Glob patters are supported',
default: '${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'
default: '${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'
}).option('srcLang', {
describe: 'Source language. Please run `osr-i18n info to see all supported languages`',
default: ''

View File

@ -18,7 +18,7 @@ export const options = (yargs) => {
describe: 'Path to the input file(s). Glob patters are supported',
}).option('dst', {
describe: 'Path to the output file(s). Glob patters are supported',
default: '${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'
default: '${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'
}).option('formality', {
describe: 'Formality: default|more|less',
default: 'default'

View File

@ -1,5 +1,7 @@
import { upload_document, check_document_status, download_document } from './deepl.js';
import * as fs from 'fs';
import * as path from 'path';
import { logger } from '../index.js';
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
export const translateDocument = async (src, dst, options) => {
const api_key = options.config?.deepl?.auth_key || options.api_key;
@ -10,6 +12,10 @@ export const translateDocument = async (src, dst, options) => {
if (!src) {
throw new Error('Source file missing for document translation.');
}
if (options.cache === false && fs.existsSync(dst)) {
logger.info(`Destination document ${dst} already exists. Skipping DeepL translation based on cache=false.`);
return fs.readFileSync(dst);
}
const targetLangMatch = (options.dstLang || '').toUpperCase();
if (!targetLangMatch) {
throw new Error('Target language missing for document translation.');
@ -28,33 +34,37 @@ export const translateDocument = async (src, dst, options) => {
deepLOptions.formality = options.formality;
}
try {
console.log(`Uploading document for translation: ${src}`);
logger.info(`Uploading document for translation: ${src}`);
const { document_id, document_key } = await upload_document(deepLOptions, src);
let isDone = false;
let pollInterval = 3000;
let pollInterval = 1500;
const startTime = Date.now();
const timeoutMs = 300000; // 5 minutes
while (!isDone) {
console.log(`Checking status for document_id: ${document_id}`);
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Document translation timed out after ${timeoutMs / 60000} minutes.`);
}
logger.info(`Checking status for document_id: ${document_id}`);
const statusRes = await check_document_status(api_key, document_id, document_key, is_free);
if (statusRes.status === 'done') {
isDone = true;
console.log(`Translation finished. Billed characters: ${statusRes.billed_characters}`);
logger.info(`Translation finished. Billed characters: ${statusRes.billed_characters}`);
}
else if (statusRes.status === 'error') {
throw new Error(`DeepL Document Translation failed: ${statusRes.message}`);
}
else {
const remaining = statusRes.seconds_remaining || 5;
const waitTime = Math.max(pollInterval, remaining * 1000);
console.log(`Status: ${statusRes.status}, waiting ${waitTime / 1000}s...`);
const waitTime = Math.round(pollInterval);
logger.info(`Status: ${statusRes.status}, waiting ${waitTime / 1000}s...`);
await delay(waitTime);
// Backoff slightly to ensure we don't spam if seconds_remaining is inaccurate
// Backoff slightly to ensure we don't spam
pollInterval = Math.min(pollInterval * 1.5, 10000);
}
}
console.log(`Downloading translated document...`);
logger.info(`Downloading translated document...`);
const fileBuffer = await download_document(api_key, document_id, document_key, is_free);
fs.mkdirSync(path.dirname(dst), { recursive: true });
fs.writeFileSync(dst, Buffer.from(fileBuffer));
console.log(`Saved translated document to ${dst}`);
return fileBuffer;
}
catch (err) {
@ -62,4 +72,4 @@ export const translateDocument = async (src, dst, options) => {
throw err;
}
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHJhbnNsYXRlX2RvY3VtZW50LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2xpYi90cmFuc2xhdGVfZG9jdW1lbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUNILGVBQWUsRUFDZixxQkFBcUIsRUFDckIsaUJBQWlCLEVBRXBCLE1BQU0sWUFBWSxDQUFBO0FBQ25CLE9BQU8sS0FBSyxFQUFFLE1BQU0sSUFBSSxDQUFBO0FBRXhCLE1BQU0sS0FBSyxHQUFHLENBQUMsRUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDLFVBQVUsQ0FBQyxPQUFPLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQztBQUU5RSxNQUFNLENBQUMsTUFBTSxpQkFBaUIsR0FBRyxLQUFLLEVBQ2xDLEdBQVcsRUFDWCxHQUFXLEVBQ1gsT0FBaUIsRUFDbkIsRUFBRTtJQUNBLE1BQU0sT0FBTyxHQUFHLE9BQU8sQ0FBQyxNQUFNLEVBQUUsS0FBSyxFQUFFLFFBQVEsSUFBSSxPQUFPLENBQUMsT0FBTyxDQUFDO0lBQ25FLE1BQU0sT0FBTyxHQUFHLE9BQU8sQ0FBQyxNQUFNLEVBQUUsS0FBSyxFQUFFLFFBQVEsSUFBSSxLQUFLLENBQUM7SUFFekQsSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ1gsTUFBTSxJQUFJLEtBQUssQ0FBQyw0QkFBNEIsQ0FBQyxDQUFDO0lBQ2xELENBQUM7SUFFRCxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7UUFDUCxNQUFNLElBQUksS0FBSyxDQUFDLCtDQUErQyxDQUFDLENBQUM7SUFDckUsQ0FBQztJQUNELE1BQU0sZUFBZSxHQUFHLENBQUMsT0FBTyxDQUFDLE9BQU8sSUFBSSxFQUFFLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQztJQUM5RCxJQUFJLENBQUMsZUFBZSxFQUFFLENBQUM7UUFDbEIsTUFBTSxJQUFJLEtBQUssQ0FBQyxtREFBbUQsQ0FBQyxDQUFDO0lBQzFFLENBQUM7SUFFRCwwQkFBMEI7SUFDMUIsTUFBTSxZQUFZLEdBQWtCO1FBQ2hDLFFBQVEsRUFBRSxPQUFPO1FBQ2pCLFFBQVEsRUFBRSxPQUFPO1FBQ2pCLFdBQVcsRUFBRSxlQUFzQjtRQUNuQyxJQUFJLEVBQUUsRUFBRTtLQUNYLENBQUM7SUFFRixJQUFJLE9BQU8sQ0FBQyxPQUFPLEVBQUUsQ0FBQztRQUNsQixZQUFZLENBQUMsV0FBVyxHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsV0FBVyxFQUFTLENBQUM7SUFDcEUsQ0FBQztJQUNELElBQUksT0FBTyxDQUFDLFNBQVMsSUFBSSxPQUFPLENBQUMsU0FBUyxLQUFLLFNBQVMsRUFBRSxDQUFDO1FBQ3ZELFlBQVksQ0FBQyxTQUFTLEdBQUcsT0FBTyxDQUFDLFNBQWdCLENBQUM7SUFDdEQsQ0FBQztJQUVELElBQUksQ0FBQztRQUNELE9BQU8sQ0FBQyxHQUFHLENBQUMsdUNBQXVDLEdBQUcsRUFBRSxDQUFDLENBQUM7UUFDMUQsTUFBTSxFQUFFLFdBQVcsRUFBRSxZQUFZLEVBQUUsR0FBRyxNQUFNLGVBQWUsQ0FBQyxZQUFZLEVBQUUsR0FBRyxDQUFDLENBQUM7UUFFL0UsSUFBSSxNQUFNLEdBQUcsS0FBSyxDQUFDO1FBQ25CLElBQUksWUFBWSxHQUFHLElBQUksQ0FBQztRQUV4QixPQUFPLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDYixPQUFPLENBQUMsR0FBRyxDQUFDLG9DQUFvQyxXQUFXLEVBQUUsQ0FBQyxDQUFDO1lBQy9ELE1BQU0sU0FBUyxHQUFHLE1BQU0scUJBQXFCLENBQUMsT0FBTyxFQUFFLFdBQVcsRUFBRSxZQUFZLEVBQUUsT0FBTyxDQUFDLENBQUM7WUFFM0YsSUFBSSxTQUFTLENBQUMsTUFBTSxLQUFLLE1BQU0sRUFBRSxDQUFDO2dCQUM5QixNQUFNLEdBQUcsSUFBSSxDQUFDO2dCQUNkLE9BQU8sQ0FBQyxHQUFHLENBQUMsNENBQTRDLFNBQVMsQ0FBQyxpQkFBaUIsRUFBRSxDQUFDLENBQUM7WUFDM0YsQ0FBQztpQkFBTSxJQUFJLFNBQVMsQ0FBQyxNQUFNLEtBQUssT0FBTyxFQUFFLENBQUM7Z0JBQ3RDLE1BQU0sSUFBSSxLQUFLLENBQUMsc0NBQXNDLFNBQVMsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDO1lBQy9FLENBQUM7aUJBQU0sQ0FBQztnQkFDSixNQUFNLFNBQVMsR0FBRyxTQUFTLENBQUMsaUJBQWlCLElBQUksQ0FBQyxDQUFDO2dCQUNuRCxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLFlBQVksRUFBRSxTQUFTLEdBQUcsSUFBSSxDQUFDLENBQUM7Z0JBQzFELE9BQU8sQ0FBQyxHQUFHLENBQUMsV0FBVyxTQUFTLENBQUMsTUFBTSxhQUFhLFFBQVEsR0FBRyxJQUFJLE1BQU0sQ0FBQyxDQUFDO2dCQUMzRSxNQUFNLEtBQUssQ0FBQyxRQUFRLENBQUMsQ0FBQztnQkFDdEIsOEVBQThFO2dCQUM5RSxZQUFZLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLEdBQUcsR0FBRyxFQUFFLEtBQUssQ0FBQyxDQUFDO1lBQ3ZELENBQUM7UUFDTCxDQUFDO1FBRUQsT0FBTyxDQUFDLEdBQUcsQ0FBQyxvQ0FBb0MsQ0FBQyxDQUFDO1FBQ2xELE1BQU0sVUFBVSxHQUFHLE1BQU0saUJBQWlCLENBQUMsT0FBTyxFQUFFLFdBQVcsRUFBRSxZQUFZLEVBQUUsT0FBTyxDQUFDLENBQUM7UUFFeEYsRUFBRSxDQUFDLGFBQWEsQ0FBQyxHQUFHLEVBQUUsTUFBTSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDO1FBQy9DLE9BQU8sQ0FBQyxHQUFHLENBQUMsZ0NBQWdDLEdBQUcsRUFBRSxDQUFDLENBQUM7UUFFbkQsT0FBTyxVQUFVLENBQUM7SUFFdEIsQ0FBQztJQUFDLE9BQU8sR0FBUSxFQUFFLENBQUM7UUFDaEIsT0FBTyxDQUFDLEtBQUssQ0FBQyxzQ0FBc0MsR0FBRyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUM7UUFDbkUsTUFBTSxHQUFHLENBQUM7SUFDZCxDQUFDO0FBQ0wsQ0FBQyxDQUFBIn0=
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHJhbnNsYXRlX2RvY3VtZW50LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2xpYi90cmFuc2xhdGVfZG9jdW1lbnQudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxFQUNILGVBQWUsRUFDZixxQkFBcUIsRUFDckIsaUJBQWlCLEVBRXBCLE1BQU0sWUFBWSxDQUFBO0FBQ25CLE9BQU8sS0FBSyxFQUFFLE1BQU0sSUFBSSxDQUFBO0FBQ3hCLE9BQU8sS0FBSyxJQUFJLE1BQU0sTUFBTSxDQUFBO0FBQzVCLE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxhQUFhLENBQUE7QUFFcEMsTUFBTSxLQUFLLEdBQUcsQ0FBQyxFQUFVLEVBQUUsRUFBRSxDQUFDLElBQUksT0FBTyxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUMsVUFBVSxDQUFDLE9BQU8sRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDO0FBRTlFLE1BQU0sQ0FBQyxNQUFNLGlCQUFpQixHQUFHLEtBQUssRUFDbEMsR0FBVyxFQUNYLEdBQVcsRUFDWCxPQUFpQixFQUNuQixFQUFFO0lBQ0EsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLE1BQU0sRUFBRSxLQUFLLEVBQUUsUUFBUSxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUM7SUFDbkUsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLE1BQU0sRUFBRSxLQUFLLEVBQUUsUUFBUSxJQUFJLEtBQUssQ0FBQztJQUV6RCxJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7UUFDWCxNQUFNLElBQUksS0FBSyxDQUFDLDRCQUE0QixDQUFDLENBQUM7SUFDbEQsQ0FBQztJQUVELElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztRQUNQLE1BQU0sSUFBSSxLQUFLLENBQUMsK0NBQStDLENBQUMsQ0FBQztJQUNyRSxDQUFDO0lBRUQsSUFBSSxPQUFPLENBQUMsS0FBSyxLQUFLLEtBQUssSUFBSSxFQUFFLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxFQUFFLENBQUM7UUFDaEQsTUFBTSxDQUFDLElBQUksQ0FBQyx3QkFBd0IsR0FBRyxtRUFBbUUsQ0FBQyxDQUFDO1FBQzVHLE9BQU8sRUFBRSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNoQyxDQUFDO0lBRUQsTUFBTSxlQUFlLEdBQUcsQ0FBQyxPQUFPLENBQUMsT0FBTyxJQUFJLEVBQUUsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDO0lBQzlELElBQUksQ0FBQyxlQUFlLEVBQUUsQ0FBQztRQUNuQixNQUFNLElBQUksS0FBSyxDQUFDLG1EQUFtRCxDQUFDLENBQUM7SUFDekUsQ0FBQztJQUVELDBCQUEwQjtJQUMxQixNQUFNLFlBQVksR0FBa0I7UUFDaEMsUUFBUSxFQUFFLE9BQU87UUFDakIsUUFBUSxFQUFFLE9BQU87UUFDakIsV0FBVyxFQUFFLGVBQXNCO1FBQ25DLElBQUksRUFBRSxFQUFFO0tBQ1gsQ0FBQztJQUVGLElBQUksT0FBTyxDQUFDLE9BQU8sRUFBRSxDQUFDO1FBQ2xCLFlBQVksQ0FBQyxXQUFXLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxXQUFXLEVBQVMsQ0FBQztJQUNwRSxDQUFDO0lBQ0QsSUFBSSxPQUFPLENBQUMsU0FBUyxJQUFJLE9BQU8sQ0FBQyxTQUFTLEtBQUssU0FBUyxFQUFFLENBQUM7UUFDdkQsWUFBWSxDQUFDLFNBQVMsR0FBRyxPQUFPLENBQUMsU0FBZ0IsQ0FBQztJQUN0RCxDQUFDO0lBRUQsSUFBSSxDQUFDO1FBQ0QsTUFBTSxDQUFDLElBQUksQ0FBQyx1Q0FBdUMsR0FBRyxFQUFFLENBQUMsQ0FBQztRQUMxRCxNQUFNLEVBQUUsV0FBVyxFQUFFLFlBQVksRUFBRSxHQUFHLE1BQU0sZUFBZSxDQUFDLFlBQVksRUFBRSxHQUFHLENBQUMsQ0FBQztRQUUvRSxJQUFJLE1BQU0sR0FBRyxLQUFLLENBQUM7UUFDbkIsSUFBSSxZQUFZLEdBQUcsSUFBSSxDQUFDO1FBQ3hCLE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQyxHQUFHLEVBQUUsQ0FBQztRQUM3QixNQUFNLFNBQVMsR0FBRyxNQUFNLENBQUMsQ0FBQyxZQUFZO1FBRXRDLE9BQU8sQ0FBQyxNQUFNLEVBQUUsQ0FBQztZQUNiLElBQUksSUFBSSxDQUFDLEdBQUcsRUFBRSxHQUFHLFNBQVMsR0FBRyxTQUFTLEVBQUUsQ0FBQztnQkFDckMsTUFBTSxJQUFJLEtBQUssQ0FBQyx3Q0FBd0MsU0FBUyxHQUFHLEtBQUssV0FBVyxDQUFDLENBQUM7WUFDMUYsQ0FBQztZQUVELE1BQU0sQ0FBQyxJQUFJLENBQUMsb0NBQW9DLFdBQVcsRUFBRSxDQUFDLENBQUM7WUFDL0QsTUFBTSxTQUFTLEdBQUcsTUFBTSxxQkFBcUIsQ0FBQyxPQUFPLEVBQUUsV0FBVyxFQUFFLFlBQVksRUFBRSxPQUFPLENBQUMsQ0FBQztZQUUzRixJQUFJLFNBQVMsQ0FBQyxNQUFNLEtBQUssTUFBTSxFQUFFLENBQUM7Z0JBQzlCLE1BQU0sR0FBRyxJQUFJLENBQUM7Z0JBQ2QsTUFBTSxDQUFDLElBQUksQ0FBQyw0Q0FBNEMsU0FBUyxDQUFDLGlCQUFpQixFQUFFLENBQUMsQ0FBQztZQUMzRixDQUFDO2lCQUFNLElBQUksU0FBUyxDQUFDLE1BQU0sS0FBSyxPQUFPLEVBQUUsQ0FBQztnQkFDdEMsTUFBTSxJQUFJLEtBQUssQ0FBQyxzQ0FBc0MsU0FBUyxDQUFDLE9BQU8sRUFBRSxDQUFDLENBQUM7WUFDL0UsQ0FBQztpQkFBTSxDQUFDO2dCQUNKLE1BQU0sUUFBUSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsWUFBWSxDQUFDLENBQUM7Z0JBQzFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsV0FBVyxTQUFTLENBQUMsTUFBTSxhQUFhLFFBQVEsR0FBRyxJQUFJLE1BQU0sQ0FBQyxDQUFDO2dCQUMzRSxNQUFNLEtBQUssQ0FBQyxRQUFRLENBQUMsQ0FBQztnQkFDdEIsMkNBQTJDO2dCQUMzQyxZQUFZLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxZQUFZLEdBQUcsR0FBRyxFQUFFLEtBQUssQ0FBQyxDQUFDO1lBQ3ZELENBQUM7UUFDTCxDQUFDO1FBQ0QsTUFBTSxDQUFDLElBQUksQ0FBQyxvQ0FBb0MsQ0FBQyxDQUFDO1FBQ2xELE1BQU0sVUFBVSxHQUFHLE1BQU0saUJBQWlCLENBQUMsT0FBTyxFQUFFLFdBQVcsRUFBRSxZQUFZLEVBQUUsT0FBTyxDQUFDLENBQUM7UUFDeEYsRUFBRSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxFQUFFLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7UUFDckQsRUFBRSxDQUFDLGFBQWEsQ0FBQyxHQUFHLEVBQUUsTUFBTSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDO1FBQy9DLE9BQU8sVUFBVSxDQUFDO0lBRXRCLENBQUM7SUFBQyxPQUFPLEdBQVEsRUFBRSxDQUFDO1FBQ2hCLE9BQU8sQ0FBQyxLQUFLLENBQUMsc0NBQXNDLEdBQUcsQ0FBQyxPQUFPLEVBQUUsQ0FBQyxDQUFDO1FBQ25FLE1BQU0sR0FBRyxDQUFDO0lBQ2QsQ0FBQztBQUNMLENBQUMsQ0FBQSJ9

View File

@ -6,7 +6,7 @@ export const TranslateOptionsSchema = z.object({
dry: z.boolean().default(false),
env_key: z.string().default('OSR-CONFIG'),
src: z.string().optional(),
dst: z.string().default('${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'),
dst: z.string().default('${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'),
formality: z.enum(formalityLevels).default('default'),
srcLang: z.string().default(''),
cache: z.boolean().default(false),

View File

@ -12,6 +12,7 @@
"@polymech/cache": "file:../cache",
"@polymech/commons": "file:../commons",
"@polymech/core": "file:../core",
"@polymech/deepl-mark": "file:../deepl-mark",
"@polymech/fs": "file:../fs",
"@polymech/log": "file:../log",
"@types/html-minifier-terser": "^7.0.2",
@ -122,6 +123,41 @@
"typescript": "^5.7.3"
}
},
"../deepl-mark": {
"name": "@polymech/deepl-mark",
"version": "0.3.0",
"license": "MIT",
"dependencies": {
"acorn": "^8.8.2",
"acorn-jsx": "^5.3.2",
"astring": "^1.8.4",
"deepl-node": "^1.24.0",
"mdast-util-from-markdown": "^1.3.0",
"mdast-util-frontmatter": "^1.0.1",
"mdast-util-gfm-table": "^1.0.7",
"mdast-util-mdx": "^2.0.1",
"mdast-util-to-markdown": "^1.5.0",
"micromark-extension-frontmatter": "^1.0.0",
"micromark-extension-gfm-table": "^1.0.7",
"micromark-extension-mdxjs": "^1.0.0",
"micromark-factory-space": "^1.0.0",
"micromark-util-character": "^1.1.0",
"micromark-util-symbol": "^1.0.1",
"micromark-util-types": "^1.0.2",
"prettier": "^2.8.3",
"yaml": "^2.2.1"
},
"devDependencies": {
"@types/estree": "^1.0.0",
"@types/mdast": "^3.0.10",
"@types/node": "^25.3.3",
"@types/prettier": "^2.7.2",
"@types/unist": "^2.0.6",
"esbuild": "^0.25.0",
"typescript": "^5.9.3",
"vitest": "^3.0.0"
}
},
"../fs": {
"name": "@polymech/fs",
"version": "0.13.41",
@ -332,6 +368,10 @@
"resolved": "../core",
"link": true
},
"node_modules/@polymech/deepl-mark": {
"resolved": "../deepl-mark",
"link": true
},
"node_modules/@polymech/fs": {
"resolved": "../fs",
"link": true

View File

@ -43,6 +43,7 @@
"@polymech/cache": "file:../cache",
"@polymech/commons": "file:../commons",
"@polymech/core": "file:../core",
"@polymech/deepl-mark": "file:../deepl-mark",
"@polymech/fs": "file:../fs",
"@polymech/log": "file:../log",
"@types/html-minifier-terser": "^7.0.2",
@ -75,6 +76,7 @@
},
"scripts": {
"test": "vitest run",
"test:document": "npm run test -- tests/document.e2e.test.ts",
"test-with-coverage": "vitest run --coverage",
"dev-test-watch": "vitest",
"lint": "tslint --project=./tsconfig.json",
@ -102,4 +104,4 @@
"vitest": "^4.1.2",
"webpack-cli": "^7.0.2"
}
}
}

View File

@ -1,8 +1,8 @@
{
"translate-es": {
"name": "Translate to EU Languages",
"name": "Translate to Spanish - Internal",
"command": "pm-i18n",
"args": "translate --alt=true --src=\"$(FullName)\" --dst=\"&{SRC_DIR}/&{SRC_NAME}_&{DST_LANG}.md\" --srcLang='en' --dstLang='es,de,it,fr'",
"args": "translate --alt=true --src=\"$(FullName)\" --dst=\"&{SRC_DIR}/&{SRC_NAME}_&{DST_LANG}.md\" --srcLang='en' --dstLang='es'",
"description": "Translate to EU"
}
}
}

View File

@ -20,7 +20,7 @@ const defaultOptions = (yargs: CLI.Argv) => {
describe: 'Target file',
}).option('dst', {
describe: 'Path to the output file(s). Glob patters are supported',
default: '${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'
default: '${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'
}).option('srcLang', {
describe: 'Source language. Please run `osr-i18n info to see all supported languages`',
default: ''

View File

@ -22,7 +22,7 @@ export const options = (yargs: CLI.Argv) => {
describe: 'Path to the input file(s). Glob patters are supported',
}).option('dst', {
describe: 'Path to the output file(s). Glob patters are supported',
default: '${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'
default: '${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'
}).option('formality', {
describe: 'Formality: default|more|less',
default: 'default'

View File

@ -6,6 +6,8 @@ import {
IDeepLOptions
} from './deepl.js'
import * as fs from 'fs'
import * as path from 'path'
import { logger } from '../index.js'
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@ -16,19 +18,25 @@ export const translateDocument = async (
) => {
const api_key = options.config?.deepl?.auth_key || options.api_key;
const is_free = options.config?.deepl?.free_api || false;
if (!api_key) {
throw new Error('DeepL auth_key is missing.');
}
if (!src) {
throw new Error('Source file missing for document translation.');
}
if (options.cache === false && fs.existsSync(dst)) {
logger.info(`Destination document ${dst} already exists. Skipping DeepL translation based on cache=false.`);
return fs.readFileSync(dst);
}
const targetLangMatch = (options.dstLang || '').toUpperCase();
if (!targetLangMatch) {
throw new Error('Target language missing for document translation.');
throw new Error('Target language missing for document translation.');
}
// Structure DeepL Options
const deepLOptions: IDeepLOptions = {
free_api: is_free,
@ -36,7 +44,7 @@ export const translateDocument = async (
target_lang: targetLangMatch as any,
text: ''
};
if (options.srcLang) {
deepLOptions.source_lang = options.srcLang.toUpperCase() as any;
}
@ -45,39 +53,41 @@ export const translateDocument = async (
}
try {
console.log(`Uploading document for translation: ${src}`);
logger.info(`Uploading document for translation: ${src}`);
const { document_id, document_key } = await upload_document(deepLOptions, src);
let isDone = false;
let pollInterval = 3000;
let pollInterval = 1500;
const startTime = Date.now();
const timeoutMs = 300000; // 5 minutes
while (!isDone) {
console.log(`Checking status for document_id: ${document_id}`);
if (Date.now() - startTime > timeoutMs) {
throw new Error(`Document translation timed out after ${timeoutMs / 60000} minutes.`);
}
logger.info(`Checking status for document_id: ${document_id}`);
const statusRes = await check_document_status(api_key, document_id, document_key, is_free);
if (statusRes.status === 'done') {
isDone = true;
console.log(`Translation finished. Billed characters: ${statusRes.billed_characters}`);
logger.info(`Translation finished. Billed characters: ${statusRes.billed_characters}`);
} else if (statusRes.status === 'error') {
throw new Error(`DeepL Document Translation failed: ${statusRes.message}`);
} else {
const remaining = statusRes.seconds_remaining || 5;
const waitTime = Math.max(pollInterval, remaining * 1000);
console.log(`Status: ${statusRes.status}, waiting ${waitTime / 1000}s...`);
const waitTime = Math.round(pollInterval);
logger.info(`Status: ${statusRes.status}, waiting ${waitTime / 1000}s...`);
await delay(waitTime);
// Backoff slightly to ensure we don't spam if seconds_remaining is inaccurate
pollInterval = Math.min(pollInterval * 1.5, 10000);
// Backoff slightly to ensure we don't spam
pollInterval = Math.min(pollInterval * 1.5, 10000);
}
}
console.log(`Downloading translated document...`);
logger.info(`Downloading translated document...`);
const fileBuffer = await download_document(api_key, document_id, document_key, is_free);
fs.mkdirSync(path.dirname(dst), { recursive: true });
fs.writeFileSync(dst, Buffer.from(fileBuffer));
console.log(`Saved translated document to ${dst}`);
return fileBuffer;
} catch (err: any) {
console.error(`Error during document translation: ${err.message}`);
throw err;

View File

@ -8,7 +8,7 @@ export const TranslateOptionsSchema = z.object({
dry: z.boolean().default(false),
env_key: z.string().default('OSR-CONFIG'),
src: z.string().optional(),
dst: z.string().default('${SRC_DIR}/${DST_LANG}/${SRC_NAME}${SRC_EXT}'),
dst: z.string().default('${SRC_DIR}/${SRC_NAME}_${DST_LANG}${SRC_EXT}'),
formality: z.enum(formalityLevels).default('default'),
srcLang: z.string().default(''),
cache: z.boolean().default(false),

View File

@ -12,27 +12,47 @@ describe('DeepL Document API Translation (Live)', () => {
it('should translate test.pdf natively as a document', async () => {
const srcPath = path.resolve(process.cwd(), 'tests/documents/test.pdf');
const dstPath = path.resolve(process.cwd(), 'tests/documents/test_de.pdf');
// Assert the test file exists before running
expect(fs.existsSync(srcPath)).toBe(true);
// Clean up any old output file
if (fs.existsSync(dstPath)) {
fs.unlinkSync(dstPath);
}
const { stdout, stderr } = await exec(`node ${CLI_PATH} translate --src="${srcPath}" --srcLang="en" --dstLang="de"`);
console.log(stdout, stderr);
// We verify the translation loop succeeds without errors.
expect(stderr).not.toMatch(/error/i);
}, 60000); // 60s timeout for real API call
// Verify that the output document got downloaded correctly
expect(fs.existsSync(dstPath)).toBe(true);
expect(fs.statSync(dstPath).size).toBeGreaterThan(0);
}, 120000); // 120s timeout for real API call
it('should translate test.xlsx as a document specifically requesting engine=document', async () => {
const srcPath = path.resolve(process.cwd(), 'tests/documents/test.xlsx');
const dstPath = path.resolve(process.cwd(), 'tests/documents/test_de.xlsx');
// Assert the test file exists before running
expect(fs.existsSync(srcPath)).toBe(true);
// Clean up any old output file
if (fs.existsSync(dstPath)) {
fs.unlinkSync(dstPath);
}
const { stdout, stderr } = await exec(`node ${CLI_PATH} translate --src="${srcPath}" --srcLang="en" --dstLang="de" --engine="document"`);
console.log(stdout, stderr);
// We verify the translation loop succeeds without errors.
expect(stderr).not.toMatch(/error/i);
}, 60000); // 60s timeout for real API call
// Verify that the output document got downloaded correctly
expect(fs.existsSync(dstPath)).toBe(true);
expect(fs.statSync(dstPath).size).toBeGreaterThan(0);
}, 120000); // 120s timeout for real API call
})

Binary file not shown.

Binary file not shown.

View File

@ -5,6 +5,12 @@
},
{
"path": "../commons"
},
{
"path": "../i18n"
},
{
"path": "../deepl-mark"
}
],
"settings": {}