import { test } from 'node:test'; import assert from 'node:assert/strict'; import { tokenize, tokenStyle, SyntaxToken } from './syntaxHighlight'; const find = (tokens: SyntaxToken[], text: string) => tokens.find((t) => t.text === text); test('tokenize falls back to a single plain token for unsupported languages', () => { assert.deepEqual(tokenize('plain text here', 'unknownlang'), [ { text: 'plain text here', type: 'plain' }, ]); // empty lang is also unsupported assert.deepEqual(tokenize('hello', ''), [{ text: 'hello', type: 'plain' }]); }); test('tokenize returns an empty array for empty input', () => { assert.deepEqual(tokenize('', 'js'), []); }); test('tokenize strips a leading "language-" prefix', () => { const tokens = tokenize('const x', 'language-javascript'); assert.equal(find(tokens, 'const')?.type, 'kw'); }); test('tokenize classifies keywords, numbers, plain words and punctuation', () => { const tokens = tokenize('const x = 5;', 'js'); assert.deepEqual(tokens, [ { text: 'const', type: 'kw' }, { text: ' ', type: 'plain' }, { text: 'x', type: 'plain' }, { text: ' = ', type: 'plain' }, { text: '5', type: 'num' }, { text: ';', type: 'plain' }, ]); }); test('tokenize detects function-call identifiers by a following paren', () => { const tokens = tokenize('foo(1)', 'js'); assert.equal(find(tokens, 'foo')?.type, 'fn'); assert.equal(find(tokens, '1')?.type, 'num'); }); test('tokenize handles line comments up to the newline', () => { const tokens = tokenize('// hi\ncode', 'js'); assert.deepEqual(tokens[0], { text: '// hi', type: 'cmt' }); // the newline and the rest are plain assert.equal(find(tokens, 'code')?.type, 'plain'); }); test('tokenize handles block comments including the closing delimiter', () => { const tokens = tokenize('/* block */x', 'js'); assert.deepEqual(tokens[0], { text: '/* block */', type: 'cmt' }); assert.equal(find(tokens, 'x')?.type, 'plain'); }); test('tokenize captures string literals including escaped quotes', () => { assert.deepEqual(tokenize('"str"', 'js'), [{ text: '"str"', type: 'str' }]); assert.deepEqual(tokenize("'a\\'b'", 'js'), [{ text: "'a\\'b'", type: 'str' }]); }); test('tokenize treats # as a comment only in python', () => { // python: # after a space starts a comment const py = tokenize('a # not comment', 'python'); assert.equal( py.some((t) => t.type === 'cmt' && t.text === '# not comment'), true, ); // js: # is not a comment marker const js = tokenize('a # b', 'js'); assert.equal( js.some((t) => t.type === 'cmt'), false, ); }); test('tokenize uses the python keyword set for python', () => { const tokens = tokenize('def foo():', 'python'); assert.equal(find(tokens, 'def')?.type, 'kw'); assert.equal(find(tokens, 'foo')?.type, 'fn'); }); test('tokenize uses the rust keyword set for rust', () => { const tokens = tokenize('fn main() {}', 'rust'); assert.equal(find(tokens, 'fn')?.type, 'kw'); assert.equal(find(tokens, 'main')?.type, 'fn'); }); test('tokenize keeps a plain word that is a keyword in another language plain', () => { // "def" is a python keyword but not a js keyword const tokens = tokenize('def x', 'js'); assert.equal(find(tokens, 'def')?.type, 'plain'); }); test('tokenize re-concatenates to the original source', () => { const samples: Array<[string, string]> = [ ['const x = foo(42); // done', 'js'], ['def add(a, b):\n return a + b # sum', 'python'], ['let mut v = vec![1, 2, 3];', 'rust'], ]; for (const [code, lang] of samples) { assert.equal( tokenize(code, lang) .map((t) => t.text) .join(''), code, `roundtrip for ${lang}`, ); } }); test('tokenStyle returns distinct styles per token kind', () => { assert.deepEqual(tokenStyle('kw'), { color: 'var(--prism-keyword)' }); assert.deepEqual(tokenStyle('str'), { color: 'var(--prism-selector)' }); assert.deepEqual(tokenStyle('num'), { color: 'var(--prism-boolean)' }); assert.deepEqual(tokenStyle('cmt'), { color: 'var(--prism-comment)', fontStyle: 'italic' }); assert.deepEqual(tokenStyle('fn'), { color: 'var(--prism-atrule)' }); assert.deepEqual(tokenStyle('plain'), {}); });