Merge branch 'feat/matchnonascii' into 'develop'

Support non-ASCII characters in emoji code

See merge request TransFem-org/sfm-js!2

docs/CONTRIBUTING.en.md

@@ -28,3 +28,30 @@ Thank you for your PR! Before creating a PR, please check the following:
 
 Thanks for your cooperation 🤗
 
+## Testing with Sharkey
+
+Let's say you have Sharkey checked out at `~/src/Sharkey`, and this
+repository at `~/src/sfm-js`. You have made some modifications to this
+code, and want to run Sharkey with your modifications.
+
+```shell
+cd ~/src/Sharkey
+pnpm --filter=backend add ../../../sfm-js
+pnpm --filter=frontend add ../../../sfm-js
+```
+
+this will replace the "real" `@transfem-org/sfm-js` in the
+dependencies, with your local version. Remember to *not* commit the
+changes to `package.json` that this causes! (`pnpm link` should also
+work, but I couldn't figure it out)
+
+Then you'll need to tell Vite to handle that package specially. Edit
+`packages/frontend/vite.config.local-dev.ts` and add, before `define`:
+
+```json
+	optimizeDeps: {
+		include: ['@transfem-org/sfm-js'],
+	},
+```
+
+Now you can `pnpm dev` and test your changes.

src/internal/parser.ts

@@ -626,10 +626,10 @@ export const language = P.createLanguage({
 		return P.seq([
 			P.alt([P.lineBegin, side]),
 			mark,
-			P.regexp(/[a-z0-9_+-]+/i),
+			P.regexp(/[\p{Letter}\p{Number}\p{Mark}_+-]+/iu),
 			mark,
 			P.alt([P.lineEnd, side]),
-		], 2).map(name => M.EMOJI_CODE(name as string));
+		], 2).map(name => M.EMOJI_CODE((name as string).normalize('NFC')));
 	},
 
 	link: r => {

test/parser.ts

@@ -338,6 +338,30 @@ hoge`;
 			const output = [EMOJI_CODE('abc')];
 			assert.deepStrictEqual(mfm.parse(input), output);
 		});
+
+		test('non-ASCII', () => {
+			const input = ':taneŝima_ĝojas:, :मार्जारः:, :鹅:, :taneŝima_malsanas:, :แมว:, and :लक्षणा:';
+			const output = [
+				EMOJI_CODE('taneŝima_ĝojas'),
+				TEXT(', '),
+				EMOJI_CODE('मार्जारः'),
+				TEXT(', '),
+				EMOJI_CODE('鹅'),
+				TEXT(', '),
+				EMOJI_CODE('taneŝima_malsanas'),
+				TEXT(', '),
+				EMOJI_CODE('แมว'),
+				TEXT(', and '),
+				EMOJI_CODE('लक्षणा'),
+			];
+			assert.deepStrictEqual(mfm.parse(input), output);
+		});
+
+		test('non-ASCII normalization', () => {
+			const input = ":fo\u{0308}o:";
+			const output = [EMOJI_CODE("f\u{00F6}o")];
+			assert.deepStrictEqual(mfm.parse(input), output);
+		});
 	});
 
 	describe('unicode emoji', () => {