cinny/src/app/utils/syntaxHighlight.test.ts

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { tokenize, tokenStyle, SyntaxToken } from './syntaxHighlight';

const find = (tokens: SyntaxToken[], text: string) => tokens.find((t) => t.text === text);

test('tokenize falls back to a single plain token for unsupported languages', () => {
  assert.deepEqual(tokenize('plain text here', 'unknownlang'), [
    { text: 'plain text here', type: 'plain' },
  ]);
  // empty lang is also unsupported
  assert.deepEqual(tokenize('hello', ''), [{ text: 'hello', type: 'plain' }]);
});

test('tokenize returns an empty array for empty input', () => {
  assert.deepEqual(tokenize('', 'js'), []);
});

test('tokenize strips a leading "language-" prefix', () => {
  const tokens = tokenize('const x', 'language-javascript');
  assert.equal(find(tokens, 'const')?.type, 'kw');
});

test('tokenize classifies keywords, numbers, plain words and punctuation', () => {
  const tokens = tokenize('const x = 5;', 'js');
  assert.deepEqual(tokens, [
    { text: 'const', type: 'kw' },
    { text: ' ', type: 'plain' },
    { text: 'x', type: 'plain' },
    { text: ' = ', type: 'plain' },
    { text: '5', type: 'num' },
    { text: ';', type: 'plain' },
  ]);
});

test('tokenize detects function-call identifiers by a following paren', () => {
  const tokens = tokenize('foo(1)', 'js');
  assert.equal(find(tokens, 'foo')?.type, 'fn');
  assert.equal(find(tokens, '1')?.type, 'num');
});

test('tokenize handles line comments up to the newline', () => {
  const tokens = tokenize('// hi\ncode', 'js');
  assert.deepEqual(tokens[0], { text: '// hi', type: 'cmt' });
  // the newline and the rest are plain
  assert.equal(find(tokens, 'code')?.type, 'plain');
});

test('tokenize handles block comments including the closing delimiter', () => {
  const tokens = tokenize('/* block */x', 'js');
  assert.deepEqual(tokens[0], { text: '/* block */', type: 'cmt' });
  assert.equal(find(tokens, 'x')?.type, 'plain');
});

test('tokenize captures string literals including escaped quotes', () => {
  assert.deepEqual(tokenize('"str"', 'js'), [{ text: '"str"', type: 'str' }]);
  assert.deepEqual(tokenize("'a\\'b'", 'js'), [{ text: "'a\\'b'", type: 'str' }]);
});

test('tokenize treats # as a comment only in python', () => {
  // python: # after a space starts a comment
  const py = tokenize('a # not comment', 'python');
  assert.equal(
    py.some((t) => t.type === 'cmt' && t.text === '# not comment'),
    true,
  );
  // js: # is not a comment marker
  const js = tokenize('a # b', 'js');
  assert.equal(
    js.some((t) => t.type === 'cmt'),
    false,
  );
});

test('tokenize uses the python keyword set for python', () => {
  const tokens = tokenize('def foo():', 'python');
  assert.equal(find(tokens, 'def')?.type, 'kw');
  assert.equal(find(tokens, 'foo')?.type, 'fn');
});

test('tokenize uses the rust keyword set for rust', () => {
  const tokens = tokenize('fn main() {}', 'rust');
  assert.equal(find(tokens, 'fn')?.type, 'kw');
  assert.equal(find(tokens, 'main')?.type, 'fn');
});

test('tokenize keeps a plain word that is a keyword in another language plain', () => {
  // "def" is a python keyword but not a js keyword
  const tokens = tokenize('def x', 'js');
  assert.equal(find(tokens, 'def')?.type, 'plain');
});

test('tokenize re-concatenates to the original source', () => {
  const samples: Array<[string, string]> = [
    ['const x = foo(42); // done', 'js'],
    ['def add(a, b):\n    return a + b  # sum', 'python'],
    ['let mut v = vec![1, 2, 3];', 'rust'],
  ];
  for (const [code, lang] of samples) {
    assert.equal(
      tokenize(code, lang)
        .map((t) => t.text)
        .join(''),
      code,
      `roundtrip for ${lang}`,
    );
  }
});

test('tokenStyle returns distinct styles per token kind', () => {
  assert.deepEqual(tokenStyle('kw'), { color: 'var(--prism-keyword)' });
  assert.deepEqual(tokenStyle('str'), { color: 'var(--prism-selector)' });
  assert.deepEqual(tokenStyle('num'), { color: 'var(--prism-boolean)' });
  assert.deepEqual(tokenStyle('cmt'), { color: 'var(--prism-comment)', fontStyle: 'italic' });
  assert.deepEqual(tokenStyle('fn'), { color: 'var(--prism-atrule)' });
  assert.deepEqual(tokenStyle('plain'), {});
});