implement plain syntax (#114)

* implement plain syntax

* public api

* plain syntax

* public api

* plain syntax

* Update test/parser.ts

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

* plain parser -> simple parser

* nomfm -> plain

Co-authored-by: syuilo <Syuilotan@yahoo.co.jp>

README.md

@@ -28,11 +28,11 @@ I'm @ai, A bot of misskey!
 https://github.com/syuilo/ai
 </center>`;
 
-// Generate a MFM tree from the MFM text.
+// Generate a MFM tree from the full MFM text.
 const mfmTree = mfm.parse(inputText);
 
-// Generate a MFM tree from the MFM plain text.
+// Generate a MFM tree from the simple MFM text.
-const plainMfmTree = mfm.parsePlain('I like the hot soup :soup:​');
+const simpleMfmTree = mfm.parseSimple('I like the hot soup :soup:​');
 
 // Reverse to a MFM text from the MFM tree.
 const text = mfm.toString(mfmTree);
@@ -62,9 +62,9 @@ full parser:
 npm run parse
 ```
 
-plain parser:
+simple parser:
 ```
-npm run parse-plain
+npm run parse-simple
 ```
 
 ## License

docs/api.md

@@ -36,13 +36,13 @@ console.log(JSON.stringify(nodes));
 // => [{"type":"bold","children":[{"type":"text","props":{"text":"<s>cannot nest</s>"}}]}]
 ```
 
-## parsePlain API
+## parseSimple API
 入力文字列からノードツリーを生成します。  
 絵文字コードとUnicode絵文字を利用可能です。  
 
 例:  
 ```ts
-const nodes = mfm.parsePlain('Hello :surprised_ai:');
+const nodes = mfm.parseSimple('Hello :surprised_ai:');
 console.log(JSON.stringify(nodes));
 // => [{"type":"text","props":{"text":"Hello "}},{"type":"emojiCode","props":{"name":"surprised_ai"}}]
 ```

etc/mfm-js.api.md

@@ -105,7 +105,7 @@ export type MfmHashtag = {
 };
 
 // @public (undocumented)
-export type MfmInline = MfmUnicodeEmoji | MfmEmojiCode | MfmBold | MfmSmall | MfmItalic | MfmStrike | MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmText;
+export type MfmInline = MfmUnicodeEmoji | MfmEmojiCode | MfmBold | MfmSmall | MfmItalic | MfmStrike | MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmPlain | MfmText;
 
 // @public (undocumented)
 export type MfmInlineCode = {
@@ -165,6 +165,13 @@ export type MfmMention = {
 // @public (undocumented)
 export type MfmNode = MfmBlock | MfmInline;
 
+// @public (undocumented)
+export type MfmPlain = {
+    type: 'plain';
+    props?: Record<string, unknown>;
+    children: MfmText[];
+};
+
 // @public (undocumented)
 export type MfmQuote = {
     type: 'quote';
@@ -182,6 +189,9 @@ export type MfmSearch = {
     children?: [];
 };
 
+// @public (undocumented)
+export type MfmSimpleNode = MfmUnicodeEmoji | MfmEmojiCode | MfmText;
+
 // @public (undocumented)
 export type MfmSmall = {
     type: 'small';
@@ -228,7 +238,7 @@ export type MfmUrl = {
 export const N_URL: (value: string, brackets?: boolean | undefined) => NodeType<'url'>;
 
 // @public (undocumented)
-export type NodeType<T extends MfmNode['type']> = T extends 'quote' ? MfmQuote : T extends 'search' ? MfmSearch : T extends 'blockCode' ? MfmCodeBlock : T extends 'mathBlock' ? MfmMathBlock : T extends 'center' ? MfmCenter : T extends 'unicodeEmoji' ? MfmUnicodeEmoji : T extends 'emojiCode' ? MfmEmojiCode : T extends 'bold' ? MfmBold : T extends 'small' ? MfmSmall : T extends 'italic' ? MfmItalic : T extends 'strike' ? MfmStrike : T extends 'inlineCode' ? MfmInlineCode : T extends 'mathInline' ? MfmMathInline : T extends 'mention' ? MfmMention : T extends 'hashtag' ? MfmHashtag : T extends 'url' ? MfmUrl : T extends 'link' ? MfmLink : T extends 'fn' ? MfmFn : T extends 'text' ? MfmText : never;
+export type NodeType<T extends MfmNode['type']> = T extends 'quote' ? MfmQuote : T extends 'search' ? MfmSearch : T extends 'blockCode' ? MfmCodeBlock : T extends 'mathBlock' ? MfmMathBlock : T extends 'center' ? MfmCenter : T extends 'unicodeEmoji' ? MfmUnicodeEmoji : T extends 'emojiCode' ? MfmEmojiCode : T extends 'bold' ? MfmBold : T extends 'small' ? MfmSmall : T extends 'italic' ? MfmItalic : T extends 'strike' ? MfmStrike : T extends 'inlineCode' ? MfmInlineCode : T extends 'mathInline' ? MfmMathInline : T extends 'mention' ? MfmMention : T extends 'hashtag' ? MfmHashtag : T extends 'url' ? MfmUrl : T extends 'link' ? MfmLink : T extends 'fn' ? MfmFn : T extends 'plain' ? MfmPlain : T extends 'text' ? MfmText : never;
 
 // @public (undocumented)
 export function parse(input: string, opts?: Partial<{
@@ -236,10 +246,11 @@ export function parse(input: string, opts?: Partial<{
     nestLimit: number;
 }>): MfmNode[];
 
-// Warning: (ae-forgotten-export) The symbol "MfmPlainNode" needs to be exported by the entry point index.d.ts
-//
 // @public (undocumented)
-export function parsePlain(input: string): MfmPlainNode[];
+export function parseSimple(input: string): MfmSimpleNode[];
+
+// @public (undocumented)
+export const PLAIN: (text: string) => NodeType<'plain'>;
 
 // @public (undocumented)
 export const QUOTE: (children: MfmNode[]) => NodeType<'quote'>;

package.json

@@ -7,13 +7,13 @@
   "scripts": {
     "build": "npm run tsc && npm run peg",
     "build-debug": "npm run tsc && npm run peg-debug",
-    "peg": "peggy --cache -o src/internal/parser.js --allowed-start-rules fullParser,plainParser src/internal/parser.pegjs && npm run peg-copy",
+    "peg": "peggy --cache -o src/internal/parser.js --allowed-start-rules fullParser,simpleParser src/internal/parser.pegjs && npm run peg-copy",
-    "peg-debug": "peggy --cache -o src/internal/parser.js --allowed-start-rules fullParser,inlineParser,plainParser --trace src/internal/parser.pegjs && npm run peg-copy",
+    "peg-debug": "peggy --cache -o src/internal/parser.js --allowed-start-rules fullParser,inlineParser,simpleParser --trace src/internal/parser.pegjs && npm run peg-copy",
     "peg-copy": "copyfiles -f src/internal/parser.js built/internal/",
     "tsc": "tsc",
     "tsd": "tsd",
     "parse": "node ./built/cli/parse",
-    "parse-plain": "node ./built/cli/parsePlain",
+    "parse-simple": "node ./built/cli/parseSimple",
     "api": "npx api-extractor run --local --verbose",
     "api-prod": "npx api-extractor run --verbose",
     "lint": "eslint . --ext .js,.jsx,.ts,.tsx",

src/api.ts

@@ -1,5 +1,5 @@
 import peg from 'peggy';
-import { MfmNode, MfmPlainNode } from './node';
+import { MfmNode, MfmSimpleNode } from './node';
 import { stringifyNode, stringifyTree, inspectOne } from './internal/util';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -18,10 +18,10 @@ export function parse(input: string, opts: Partial<{ fnNameList: string[]; nestL
 }
 
 /**
- * Generates a MfmNode tree of plain from the MFM string.
+ * Generates a MfmSimpleNode tree from the MFM string.
 */
-export function parsePlain(input: string): MfmPlainNode[] {
+export function parseSimple(input: string): MfmSimpleNode[] {
-	const nodes = parser.parse(input, { startRule: 'plainParser' });
+	const nodes = parser.parse(input, { startRule: 'simpleParser' });
 	return nodes;
 }
 

src/cli/parseSimple.ts

@@ -1,9 +1,9 @@
 import { performance } from 'perf_hooks';
 import inputLine, { InputCanceledError } from './misc/inputLine';
-import { parsePlain } from '..';
+import { parseSimple } from '..';
 
 async function entryPoint() {
-	console.log('intaractive plain parser');
+	console.log('intaractive simple parser');
 
 	while (true) {
 		let input: string;
@@ -26,7 +26,7 @@ async function entryPoint() {
 
 		try {
 			const parseTimeStart = performance.now();
-			const result = parsePlain(input);
+			const result = parseSimple(input);
 			const parseTimeEnd = performance.now();
 			console.log(JSON.stringify(result));
 			const parseTime = (parseTimeEnd - parseTimeStart).toFixed(3);

src/index.ts

@@ -1,6 +1,6 @@
 export {
 	parse,
-	parsePlain,
+	parseSimple,
 	toString,
 	inspect,
 	extract,
@@ -9,6 +9,7 @@ export {
 export {
 	NodeType,
 	MfmNode,
+	MfmSimpleNode,
 	MfmBlock,
 	MfmInline,
 } from './node';
@@ -35,6 +36,7 @@ export {
 	MfmUrl,
 	MfmLink,
 	MfmFn,
+	MfmPlain,
 	MfmText,
 } from './node';
 
@@ -60,5 +62,6 @@ export {
 	N_URL,
 	LINK,
 	FN,
+	PLAIN,
 	TEXT,
 } from './node';

src/internal/parser.pegjs

@@ -21,6 +21,7 @@
 		N_URL,
 		LINK,
 		FN,
+		PLAIN,
 		TEXT
 	} = require('../node');
 
@@ -101,8 +102,8 @@
 fullParser
 	= nodes:(&. @full)* { return mergeText(nodes); }
 
-plainParser
+simpleParser
-	= nodes:(&. @plain)* { return mergeText(nodes); }
+	= nodes:(&. @simple)* { return mergeText(nodes); }
 
 //
 // syntax list
@@ -126,6 +127,7 @@ full
 	/ hashtag
 	/ url
 	/ fn
+	/ plain
 	/ link
 	/ search // block
 	/ inlineText
@@ -144,6 +146,7 @@ inline
 	/ hashtag
 	/ url
 	/ fn
+	/ plain
 	/ link
 	/ inlineText
 
@@ -158,12 +161,13 @@ L_inline
 	/ inlineCode
 	/ mathInline
 	/ L_fn
+	/ plain
 	/ L_inlineText
 
-plain
+simple
 	= emojiCode
 	/ unicodeEmoji
-	/ plainText
+	/ simpleText
 
 //
 // block rules
@@ -553,6 +557,20 @@ fnArg
 	return { k: k, v: true };
 }
 
+// inline: plain
+
+plain
+	= "<plain>" LF? content:plainContent LF? "</plain>"
+{
+	return PLAIN(content);
+}
+
+plainContent
+	= (!(LF? "</plain>") .)+
+{
+	return text();
+}
+
 // inline: text
 
 inlineText
@@ -563,9 +581,9 @@ L_inlineText
 	= !(LF / _) [a-z0-9]i &italicAlt . { return text(); } // italic ignore
 	/ . /* text node */
 
-// inline: text (for plainParser)
+// inline: text (for simpleParser)
 
-plainText
+simpleText
 	= . /* text node */
 
 //

src/internal/util.ts

@@ -103,6 +103,9 @@ export function stringifyNode(node: MfmNode): string {
 			const args = (argFields.length > 0) ? '.' + argFields.join(',') : '';
 			return `$[${ node.props.name }${ args } ${ stringifyTree(node.children) }]`;
 		}
+		case 'plain': {
+			return `<plain>\n${ stringifyTree(node.children) }\n</plain>`;
+		}
 		case 'text': {
 			return node.props.text;
 		}

src/node.ts

@@ -1,6 +1,6 @@
 export type MfmNode = MfmBlock | MfmInline;
 
-export type MfmPlainNode = MfmUnicodeEmoji | MfmEmojiCode | MfmText;
+export type MfmSimpleNode = MfmUnicodeEmoji | MfmEmojiCode | MfmText;
 
 export type MfmBlock = MfmQuote | MfmSearch | MfmCodeBlock | MfmMathBlock | MfmCenter;
 
@@ -53,7 +53,7 @@ export type MfmCenter = {
 export const CENTER = (children: MfmInline[]): NodeType<'center'> => { return { type: 'center', children }; };
 
 export type MfmInline = MfmUnicodeEmoji | MfmEmojiCode | MfmBold | MfmSmall | MfmItalic | MfmStrike |
-	MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmText;
+	MfmInlineCode | MfmMathInline | MfmMention | MfmHashtag | MfmUrl | MfmLink | MfmFn | MfmPlain | MfmText;
 
 export type MfmUnicodeEmoji = {
 	type: 'unicodeEmoji';
@@ -173,6 +173,13 @@ export type MfmFn = {
 };
 export const FN = (name: string, args: MfmFn['props']['args'], children: MfmFn['children']): NodeType<'fn'> => { return { type: 'fn', props: { name, args }, children }; };
 
+export type MfmPlain = {
+	type: 'plain';
+	props?: Record<string, unknown>;
+	children: MfmText[];
+};
+export const PLAIN = (text: string): NodeType<'plain'> => { return { type: 'plain', children: [TEXT(text)] }; };
+
 export type MfmText = {
 	type: 'text';
 	props: {
@@ -201,5 +208,6 @@ export type NodeType<T extends MfmNode['type']> =
 	T extends 'url' ? MfmUrl :
 	T extends 'link' ? MfmLink :
 	T extends 'fn' ? MfmFn :
+	T extends 'plain' ? MfmPlain :
 	T extends 'text' ? MfmText :
 	never;

test/api.ts

@@ -147,6 +147,16 @@ after`;
 			assert.strictEqual(mfm.toString(mfm.parse(input)), '$[spin.speed=1s,alternate Hello]');
 		});
 
+		it('plain', () => {
+			const input = 'a\n<plain>\nHello\nworld\n</plain>\nb';
+			assert.strictEqual(mfm.toString(mfm.parse(input)), 'a\n<plain>\nHello\nworld\n</plain>\nb');
+		});
+
+		it('1 line plain', () => {
+			const input = 'a\n<plain>Hello</plain>\nb';
+			assert.strictEqual(mfm.toString(mfm.parse(input)), 'a\n<plain>\nHello\n</plain>\nb');
+		});
+
 		it('preserve url brackets', () => {
 			const input1 = 'https://github.com/syuilo/ai';
 			assert.strictEqual(mfm.toString(mfm.parse(input1)), input1);

test/parser.ts

@@ -1,27 +1,27 @@
 import assert from 'assert';
 import * as mfm from '../src/index';
 import {
-	TEXT, CENTER, FN, UNI_EMOJI, MENTION, EMOJI_CODE, HASHTAG, N_URL, BOLD, SMALL, ITALIC, STRIKE, QUOTE, MATH_BLOCK, SEARCH, CODE_BLOCK, LINK, INLINE_CODE, MATH_INLINE
+	TEXT, CENTER, FN, UNI_EMOJI, MENTION, EMOJI_CODE, HASHTAG, N_URL, BOLD, SMALL, ITALIC, STRIKE, QUOTE, MATH_BLOCK, SEARCH, CODE_BLOCK, LINK, INLINE_CODE, MATH_INLINE, PLAIN
 } from '../src/index';
 
-describe('PlainParser', () => {
+describe('SimpleParser', () => {
 	describe('text', () => {
 		it('basic', () => {
 			const input = 'abc';
 			const output = [TEXT('abc')];
-			assert.deepStrictEqual(mfm.parsePlain(input), output);
+			assert.deepStrictEqual(mfm.parseSimple(input), output);
 		});
 
 		it('ignore hashtag', () => {
 			const input = 'abc#abc';
 			const output = [TEXT('abc#abc')];
-			assert.deepStrictEqual(mfm.parsePlain(input), output);
+			assert.deepStrictEqual(mfm.parseSimple(input), output);
 		});
 
 		it('keycap number sign', () => {
 			const input = 'abc#️⃣abc';
 			const output = [TEXT('abc'), UNI_EMOJI('#️⃣'), TEXT('abc')];
-			assert.deepStrictEqual(mfm.parsePlain(input), output);
+			assert.deepStrictEqual(mfm.parseSimple(input), output);
 		});
 	});
 
@@ -29,20 +29,20 @@ describe('PlainParser', () => {
 		it('basic', () => {
 			const input = ':foo:';
 			const output = [EMOJI_CODE('foo')];
-			assert.deepStrictEqual(mfm.parsePlain(input), output);
+			assert.deepStrictEqual(mfm.parseSimple(input), output);
 		});
 
 		it('between texts', () => {
 			const input = 'foo:bar:baz';
 			const output = [TEXT('foo'), EMOJI_CODE('bar'), TEXT('baz')];
-			assert.deepStrictEqual(mfm.parsePlain(input), output);
+			assert.deepStrictEqual(mfm.parseSimple(input), output);
 		});
 	});
 
 	it('disallow other syntaxes', () => {
 		const input = 'foo **bar** baz';
 		const output = [TEXT('foo **bar** baz')];
-		assert.deepStrictEqual(mfm.parsePlain(input), output);
+		assert.deepStrictEqual(mfm.parseSimple(input), output);
 	});
 });
 
@@ -1138,6 +1138,28 @@ hoge`;
 		});
 	});
 
+	describe('plain', () => {
+		it('multiple line', () => {
+			const input = 'a\n<plain>\n**Hello**\nworld\n</plain>\nb';
+			const output = [
+				TEXT('a\n'),
+				PLAIN('**Hello**\nworld'),
+				TEXT('\nb')
+			];
+			assert.deepStrictEqual(mfm.parse(input), output);
+		});
+
+		it('single line', () => {
+			const input = 'a\n<plain>**Hello** world</plain>\nb';
+			const output = [
+				TEXT('a\n'),
+				PLAIN('**Hello** world'),
+				TEXT('\nb')
+			];
+			assert.deepStrictEqual(mfm.parse(input), output);
+		});
+	});
+
 	describe('nesting limit', () => {
 		describe('quote', () => {
 			it('basic', () => {