From 7cf838abda2e820bca1a6cef519f7d14f08cd301 Mon Sep 17 00:00:00 2001 From: marihachi Date: Sun, 16 Feb 2020 23:27:25 +0900 Subject: [PATCH] update iroiro --- LICENSE | 21 +++++++ README.md | 27 +++++++-- package.json | 19 ++++-- src/build.ts | 25 ++++++++ src/client/main-entry.ts | 13 ++--- src/index.ts | 9 +-- src/misc/generate-peg.ts | 32 ++++++++++ src/misc/inputLine.ts | 22 +++++++ src/parse.ts | 41 +++++++++++++ src/parser/core-parser.pegjs | 110 +++++++++++++---------------------- src/parser/index.ts | 14 +++++ src/parser/parser-utils.ts | 60 +++++++++++++++++++ src/parser/peg-parser.d.ts | 6 -- tsconfig.client.json | 4 +- tsconfig.json | 2 +- webpack.config.js | 2 +- 16 files changed, 301 insertions(+), 106 deletions(-) create mode 100644 LICENSE create mode 100644 src/build.ts create mode 100644 src/misc/generate-peg.ts create mode 100644 src/misc/inputLine.ts create mode 100644 src/parse.ts create mode 100644 src/parser/index.ts create mode 100644 src/parser/parser-utils.ts delete mode 100644 src/parser/peg-parser.d.ts diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d4a684f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2020 Marihachi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 2b29d66..784cefb 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,27 @@ # mfm-parser-pegjs ## Description -A trial of creating a MFM parser with peg.js +A MFM parser made with PEG.js (In developing) ## Installation ``` -npm i +npm i mfm-parser-pegjs ``` -## Build +## Usage +```ts +import { parse } from 'mfm-parser-pegjs'; + +// parse a MFM code +const result = parse('this is a ***MFM text***'); +``` + +## Usage (Repository) +### 1. Clone +``` +git clone https://github.com/marihachi/mfm-parser-pegjs.git +``` + +### 2. Build For production: ``` npm run build @@ -18,7 +32,10 @@ For development: npm run build-dev ``` -## Start +### Use Interactive interface ``` -npm start +npm parse ``` + +## License +MIT diff --git a/package.json b/package.json index 15d65ec..f0bd6a8 100644 --- a/package.json +++ b/package.json @@ -2,16 +2,17 @@ "private": true, "name": "mfm-parser-pegjs", "version": "0.1.0", + "description": "A MFM parser made with PEG.js", "main": "./built/index.js", "scripts": { - "build": "npm run peg && npm run tsc && npm run webpack", - "build-dev": "npm run peg-dev && npm run tsc && npm run webpack-dev", - "peg": "mkdirp ./built/parser && pegjs -o built/parser/core-parser.js src/parser/core-parser.pegjs", - "peg-dev": "mkdirp ./built/parser && pegjs -o built/parser/core-parser.js --trace src/parser/core-parser.pegjs", + "build": "npm run tsc && npm run peg && npm run webpack", + "build-dev": "npm run tsc && npm run peg-dev && npm run webpack-dev", + "peg": "node ./built/build", + "peg-dev": "node ./built/build trace", "tsc": "tsc", "webpack": "webpack --mode=production", "webpack-dev": "webpack --mode=development", - "start": "node ." + "parse": "node ./built/parse" }, "repository": { "type": "git", @@ -21,6 +22,7 @@ "devDependencies": { "@types/node": "^12.0.4", "@types/parsimmon": "^1.10.1", + "@types/pegjs": "^0.10.1", "mkdirp": "^0.5.1", "parsimmon": "^1.13.0", "pegjs": "^0.10.0", @@ -28,5 +30,10 @@ "typescript": "3.7.x", "webpack": "4.40.x", "webpack-cli": "3.3.x" - } + }, + "files": [ + "./built/index.js", + "./built/index.d.ts", + "./built/parser" + ] } diff --git a/src/build.ts b/src/build.ts new file mode 100644 index 0000000..0986692 --- /dev/null +++ b/src/build.ts @@ -0,0 +1,25 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { generateCode } from './misc/generate-peg'; + +async function entryPoint() { + // get arguments + let trace = false; + if (process.argv.some(i => i == 'trace')) { + trace = true; + } + + const srcPath = path.join(__dirname, '../src/parser/core-parser.pegjs'); + const destPath = path.join(__dirname, '../built/parser/core-parser.js'); + + // generate a code from PEG + const generatedCode = await generateCode(srcPath, trace); + + // write the generated code + await fs.writeFile(destPath, generatedCode, { encoding: 'utf8' }); +} +entryPoint() +.catch(err => { + console.log(err); + process.exit(1); +}); diff --git a/src/client/main-entry.ts b/src/client/main-entry.ts index ac3038f..36cf784 100644 --- a/src/client/main-entry.ts +++ b/src/client/main-entry.ts @@ -1,13 +1,10 @@ -import { PegParser } from '../parser/peg-parser'; +import { parse } from '../../built/index'; async function entryPoint() { - const coreParser: PegParser = require('../../built/parser/core-parser.js'); - - const input = '[hoge]'; - console.log('parsing input:', input); - const result = coreParser.parse(input); - console.log('parsing result:'); + const result = parse('abc'); console.log(JSON.stringify(result)); } entryPoint() -.catch(err => console.log(err)); +.catch(err => { + console.log(err); +}); diff --git a/src/index.ts b/src/index.ts index 9e1c11c..75aa8b9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1 @@ -import { PegParser } from './parser/peg-parser'; - -const coreParser: PegParser = require('./parser/core-parser'); -const input = '[hoge]'; -console.log('parsing input:', input); -const result = coreParser.parse(input); -console.log('parsing result:'); -console.log(JSON.stringify(result)); +export * from './parser'; diff --git a/src/misc/generate-peg.ts b/src/misc/generate-peg.ts new file mode 100644 index 0000000..0eb77c4 --- /dev/null +++ b/src/misc/generate-peg.ts @@ -0,0 +1,32 @@ +import { promises as fs } from 'fs'; +import peg from 'pegjs'; + +export async function generateParser(srcPath: string, trace?: boolean): Promise +{ + // read the parser source + const source = await fs.readFile(srcPath, 'utf8'); + + // generate a parser code + const generatedCode = peg.generate(source, { + allowedStartRules: ['root', 'all', 'inline'], + trace: trace + }); + + return generatedCode; +} + +export async function generateCode(srcPath: string, trace?: boolean): Promise +{ + // read the parser source + const source = await fs.readFile(srcPath, 'utf8'); + + // generate a parser code + const generatedCode = peg.generate(source, { + allowedStartRules: ['root', 'all', 'inline'], + output: 'source', + format: 'commonjs', + trace: trace + }); + + return generatedCode; +} diff --git a/src/misc/inputLine.ts b/src/misc/inputLine.ts new file mode 100644 index 0000000..0e82e30 --- /dev/null +++ b/src/misc/inputLine.ts @@ -0,0 +1,22 @@ +import readLine from 'readline'; + +export class InputCanceledError extends Error { + constructor(message?: string) { + super(message); + } +} + +export default function(message: string): Promise { + return new Promise((resolve, reject) => { + const rl = readLine.createInterface(process.stdin, process.stdout); + rl.question(message, (ans) => { + rl.close(); + resolve(ans); + }); + rl.on('SIGINT', () => { + console.log(''); + rl.close(); + reject(new InputCanceledError('SIGINT interrupted')); + }); + }); +} diff --git a/src/parse.ts b/src/parse.ts new file mode 100644 index 0000000..991c736 --- /dev/null +++ b/src/parse.ts @@ -0,0 +1,41 @@ +import inputLine, { InputCanceledError } from './misc/inputLine'; +import { parse } from './parser/index'; + +async function entryPoint() { + console.log('intaractive parser'); + + while (true) { + let input: string; + try { + input = await inputLine('> '); + } + catch (err) { + if (err instanceof InputCanceledError) { + console.log('bye.'); + return; + } + throw err; + } + + // replace special chars + input = input + .replace(/\\n/g, '\n') + .replace(/\\t/g, '\t'); + + let result: any; + try { + result = parse(input); + console.log(JSON.stringify(result)); + } + catch (err) { + console.log('parsing error:'); + console.log(err); + } + console.log(); + } +} +entryPoint() +.catch(err => { + console.log(err); + process.exit(1); +}); diff --git a/src/parser/core-parser.pegjs b/src/parser/core-parser.pegjs index cb6d103..fb84ae5 100644 --- a/src/parser/core-parser.pegjs +++ b/src/parser/core-parser.pegjs @@ -1,23 +1,23 @@ { - function buildList(head, others) { - return [ head, ...others ]; - } + const { + createTree, + mergeText + } = require('./parser-utils'); - function createTree(type, props, children) { - props = props || { }; - children = children || [ ]; - children = !Array.isArray(children) ? [children] : children; - - return { - node: { type, props }, - children: children - }; + function applyParser(input, rule) { + let parseFunc = peg$parse; + return parseFunc(input, rule ? { startRule : rule } : { }); } } root - = block - / inline + = ts:all* +{ + return mergeText(ts); +} + +all + = block / inline // plain // = @@ -30,6 +30,7 @@ block inline = big + / c:. { return createTree('text', { text: c }); } // block: title @@ -38,68 +39,35 @@ title = titleA / titleB titleA - = "【" content:titleA_content "】" + = "【" content:(!("】" ENDLINE) i:inline { return i; })+ "】" ENDLINE { return createTree('title', { }, content); } -titleA_content - = (inline / titleA_text)+ - -titleA_text - = s:$(titleA_char+) -{ - return createTree('text', { text: s }); -} - -titleA_char - = !(inline / "】") c:CHAR { return c; } - titleB - = "[" content: titleB_content "]" + = "[" content:(!("]" ENDLINE) i:inline { return i; })+ "]" ENDLINE { return createTree('title', { }, content); } -titleB_content - = (inline / titleB_text)+ - -titleB_text - = s:$(titleB_char+) -{ - return createTree('text', { text: s }); -} - -titleB_char - = !(inline / "]") c:CHAR { return c; } - // block: quote -// (handle the line as quote block if got a char ">" of the line head.) quote - = head:quote_line tail:(NEWLINE tree:quote_line { return tree; })* + = lines:quote_line+ { - const trees = [head, ...tail]; - console.log(trees.map(tree => tree.children));//.flat(); - return [head, ...tail].join('\n'); + const children = applyParser(lines.join('\n'), 'root'); + return createTree('quote', { }, children); } quote_line - = ">" content:quote_content &ENDLINE { return createTree('quote', { }, content); } - -// TODO: allow nesting -quote_content - = quote_text - -quote_text - = s:$(CHAR+) { return createTree('text', { text: s }); } + = ">" _? content:$(CHAR+) ENDLINE { return content; } // block: search search - = q:search_query sp:[ 　\t] key:search_keyToken &ENDLINE + = q:search_query sp:[ 　\t] key:search_keyToken ENDLINE { return createTree('search', { query: q, @@ -120,29 +88,36 @@ search_keyToken // block: blockCode blockCode - = "```" NEWLINE lines: (!("```" ENDLINE) line:blockCode_line NEWLINE { return line; } )* "```" &ENDLINE { return lines; } + = "```" NEWLINE lines: (!("```" ENDLINE) line:blockCode_line NEWLINE { return line; } )* "```" ENDLINE { return lines; } -// TODO: allow nesting blockCode_line - = t:$(CHAR*) { return t; } + = (!"```" (block / inline))+ // inline: big big - = "***" content:big_content "***" + = "***" content:(!"***" i:inline { return i; })+ "***" { return createTree('big', { }, content); } -big_content - = (big_text / inline)* -big_text - = s:$(big_char+) { return createTree('text', { text: s }); } +// inline: bold -big_char - = !("***") c:CHAR { return c; } +bold = bold_A / bold_B + +bold_A + = "**" content:(!"**" i:inline { return i; })+ "**" +{ + return createTree('bold', { }, content); +} + +bold_B + = "__" content:(!"__" i:inline { return i; })+ "__" +{ + return createTree('bold', { }, content); +} // Core rules @@ -159,8 +134,5 @@ NEWLINE EOF = !. -// __ "whitespaces" -// = _+ - -// _ "whitespace" -// = [ \t] +_ "whitespace" + = [ \t] diff --git a/src/parser/index.ts b/src/parser/index.ts new file mode 100644 index 0000000..b091ca2 --- /dev/null +++ b/src/parser/index.ts @@ -0,0 +1,14 @@ +import peg from 'pegjs'; +const coreParser: peg.Parser = require('./core-parser'); + +export interface Tree { + type: string; + props: Record; + children: Tree[]; +}; + +export function parse(input: string) { + let trees: Tree[]; + trees = coreParser.parse(input); + return trees; +} diff --git a/src/parser/parser-utils.ts b/src/parser/parser-utils.ts new file mode 100644 index 0000000..11c432a --- /dev/null +++ b/src/parser/parser-utils.ts @@ -0,0 +1,60 @@ +import { Tree } from '.'; + +export function createTree(type: string, props?: Record, children?: Tree[]): Tree { + props = props || { }; + children = children || [ ]; + children = !Array.isArray(children) ? [children] : children; + + return { type, props, children }; +} + +/** + * @param predicate specifies whether to group the previous item and the current item + * @returns grouped items +*/ +export function groupContinuous(arr: T[], predicate: (prev: T, current: T) => boolean): T[][] { + const dest: any[][] = []; + + for (let i = 0; i < arr.length; i++) { + if (i != 0 && predicate(arr[i - 1], arr[i])) { + dest[dest.length - 1].push(arr[i]); + } + else { + dest.push([arr[i]]); + } + } + + return dest; +} + +export function mergeGroupedTrees(groupedTrees: Tree[][]): Tree[] { + return groupedTrees.reduce((acc, val) => acc.concat(val), ([] as Tree[])); +} + +export function mergeText(trees: Tree[], recursive?: boolean): Tree[] { + let dest: Tree[]; + let groupes: Tree[][]; + + // group trees + groupes = groupContinuous(trees, (prev, current) => prev.type == current.type); + + // concatinate text + groupes = groupes.map((group) => { + if (group[0].type == 'text') { + return [ + createTree('text', { + text: group.map(i => i.props.text).join('') + }) + ]; + } + return group; + }); + + // merge groups + dest = mergeGroupedTrees(groupes); + + return dest.map(tree => { + // apply recursively to children + return createTree(tree.type, tree.props, recursive ? mergeText(tree.children) : tree.children); + }); +} diff --git a/src/parser/peg-parser.d.ts b/src/parser/peg-parser.d.ts deleted file mode 100644 index 8e316c8..0000000 --- a/src/parser/peg-parser.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface PegParserOptions { - startRule?: string; -} -export interface PegParser { - parse(input: string, options?: PegParserOptions): any; -} diff --git a/tsconfig.client.json b/tsconfig.client.json index ebfaf92..ff6f682 100644 --- a/tsconfig.client.json +++ b/tsconfig.client.json @@ -5,7 +5,7 @@ "module": "ESNext", "moduleResolution": "node", "outDir": "./built/client/", /* Redirect output structure to the directory. */ - "rootDir": "./src/client/", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + "rootDir": "./src/", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ "removeComments": false, /* Strict Type-Checking Options */ @@ -21,7 +21,7 @@ "experimentalDecorators": true, }, "include": [ - "src/client/**/*", + "src/**/*", ], "exclude": [ "node_modules", diff --git a/tsconfig.json b/tsconfig.json index 03be262..2a79d39 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ /* Basic Options */ "target": "es2017", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */ "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ - //"declaration": true, /* Generates corresponding '.d.ts' file. */ + "declaration": true, /* Generates corresponding '.d.ts' file. */ "outDir": "./built/", /* Redirect output structure to the directory. */ "rootDir": "./src/", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ "removeComments": true, /* Do not emit comments to output. */ diff --git a/webpack.config.js b/webpack.config.js index 3cca1c5..8dceb27 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -21,6 +21,6 @@ module.exports = { ] }, resolve: { - extensions: ['.ts'], + extensions: ['.ts', '.js'], }, };