Mark control characters in cells with a labelled chip

README.md

@@ -59,6 +59,7 @@ For files larger than 10 MB you get a quick menu to open the full file, preview
 - **Auto-Fit Columns** - Fit all columns to their content with one click. Double-click a resize handle to auto-fit a single column.
 - **Column Resize** - Drag column borders to adjust width manually
 - **Zoom** - Scale the whole grid from 60% to 200% with the toolbar buttons or keyboard shortcuts. The zoom level shows in the toolbar.
+- **Control Characters** - Invisible control characters inside a value show as a small chip with the character's abbreviation instead of an unnamed box, so you can tell which one it is. Hover it for the full name. The value itself is untouched, so editing, copy, save and export still carry the original character.
 - **Theme Integration** - Automatically adapts to your VS Code color theme (dark or light)
 
 ![CSV Grid Editor toolbar with auto-fit, zoom, find and replace, export and column profile buttons](https://raw.githubusercontent.com/Robin-Reiche/csv-grid-editor/master/images/toolbar.png)

media/webview.css

@@ -596,6 +596,29 @@ body.vscode-high-contrast #grid-container.cm-on {
     stroke: none;
 }
 
+/* ── Control character chip ── */
+/* A control character inside a cell — an in-value separator, a stray byte from
+   a legacy export — has no glyph in the UI font and draws as a tofu box. The
+   cell renderer replaces it with its ASCII abbreviation on a tinted chip; the
+   amber matches the date type badge below rather than the alarm-red of an
+   error, since the character is usually meaningful data, not corruption. */
+.csv-ctrl-char {
+    display: inline-block;
+    font-size: 0.75em;
+    font-weight: 700;
+    line-height: 1.5;
+    padding: 0 3px;
+    margin: 0 1px;
+    border-radius: 3px;
+    vertical-align: baseline;
+    background: rgba(204,167,0,0.18);
+    color: #e5bc4b;
+    font-family: var(--vscode-font-family, "Segoe UI", sans-serif);
+    /* Keeps the abbreviation out of any text selection dragged over the cell —
+       copying must yield the real character, never the label. */
+    user-select: none;
+}
+
 /* ── Column type badge icons ── */
 /* Attached to the label (a flex container) rather than the text, so the badge
    is its own non-shrinking flex item and never gets clipped or pushed out. */

src/webview/grid/builder.ts

@@ -3,6 +3,7 @@ import { getColumnType, scheduleRecomputeColTypes } from './column-type';
 import { createCombinedFilter } from './filter';
 import { dataRowIndexForNode } from './row-mapping';
 import { partitionFrozenRows, updateCountsDisplay } from './refresh';
+import { ControlCharCellRenderer } from './control-char-cell';
 import { refreshProfileIfOpen } from '../features/profile';
 import { pushUndo, notifyChange, updateButtons } from '../features/undo-redo';
 import { getFindCellClassRules } from '../features/find-replace';
@@ -161,6 +162,10 @@ export function buildGrid(): void {
             minWidth: 60,
             editable:     !IS_PREVIEW,
             sortable:     true,
+            // Display only — marks control characters that the UI font would
+            // otherwise draw as an anonymous tofu box. Not on the '#' gutter,
+            // which renders its own pin marker.
+            cellRenderer: ControlCharCellRenderer,
             filter:       createCombinedFilter(colType),
             resizable:    true,
             suppressMovable: false,

src/webview/grid/control-char-cell.ts

@@ -0,0 +1,62 @@
+import { hasControlChars, splitControlChars } from '../utils/control-chars';
+
+// ── Cell renderer: labelled control characters ───────────────────────────────
+// Draws the value with each control character replaced by a chip showing its
+// ASCII abbreviation. Display only — the stored value is untouched.
+//
+// Built by hand instead of returned as an HTML string: AG Grid inserts a string
+// result with innerHTML, which would execute markup coming from the file.
+//
+// refresh() returns true so AG Grid reuses this element rather than recreating
+// it. A recreated element breaks double-click-to-edit: range selection
+// force-refreshes cells on mousedown, and if the element the first click hit is
+// gone by the second, the browser fires no dblclick at all.
+
+function paint(host: HTMLElement, value: string): void {
+    host.textContent = '';
+
+    // The overwhelmingly common case: no control characters, one text node.
+    if (!hasControlChars(value)) {
+        host.textContent = value;
+        return;
+    }
+
+    for (const seg of splitControlChars(value)) {
+        if (seg.type === 'text') {
+            host.appendChild(document.createTextNode(seg.text));
+            continue;
+        }
+        const chip = document.createElement('span');
+        chip.className   = 'csv-ctrl-char';
+        chip.textContent = seg.abbr;
+        chip.title       = seg.label;
+        host.appendChild(chip);
+    }
+}
+
+const valueOf = (params: any): string => params.value == null ? '' : String(params.value);
+
+export class ControlCharCellRenderer {
+    private eGui!: HTMLSpanElement;
+    private value = '';
+
+    init(params: any): void {
+        this.eGui = document.createElement('span');
+        this.value = valueOf(params);
+        paint(this.eGui, this.value);
+    }
+
+    getGui(): HTMLElement {
+        return this.eGui;
+    }
+
+    refresh(params: any): boolean {
+        const next = valueOf(params);
+        // Unchanged value → leave the DOM alone, for the same reason.
+        if (next !== this.value) {
+            this.value = next;
+            paint(this.eGui, next);
+        }
+        return true;
+    }
+}

src/webview/utils/control-chars.ts

@@ -0,0 +1,92 @@
+// ── Control characters inside cell values ────────────────────────────────────
+// CSV cells legitimately carry C0 control characters: machine-generated data
+// uses them as in-value separators, and exports from legacy systems leave stray
+// bytes behind. None of them have a glyph in the UI font, so the grid used to
+// draw an anonymous tofu box — alarming, and useless for telling WHICH
+// character it is. Splitting the value here lets the grid draw a labelled chip
+// instead, the same idea as VS Code's renderControlCharacters.
+//
+// Pure module — no DOM, no AG Grid — so it is unit-testable in plain Node
+// (see test/control-chars.test.cjs).
+
+// Abbreviation plus the name the character is actually known by. For
+// U+001C..U+001F those are the ASCII names (FILE/GROUP/RECORD/UNIT SEPARATOR)
+// people recognise, not Unicode's own far less recognisable "INFORMATION
+// SEPARATOR FOUR..ONE".
+const CONTROL_NAMES: Record<number, [string, string]> = {
+    0x00: ['NUL', 'NULL'],
+    0x01: ['SOH', 'START OF HEADING'],
+    0x02: ['STX', 'START OF TEXT'],
+    0x03: ['ETX', 'END OF TEXT'],
+    0x04: ['EOT', 'END OF TRANSMISSION'],
+    0x05: ['ENQ', 'ENQUIRY'],
+    0x06: ['ACK', 'ACKNOWLEDGE'],
+    0x07: ['BEL', 'BELL'],
+    0x08: ['BS',  'BACKSPACE'],
+    0x0b: ['VT',  'LINE TABULATION'],
+    0x0c: ['FF',  'FORM FEED'],
+    0x0e: ['SO',  'SHIFT OUT'],
+    0x0f: ['SI',  'SHIFT IN'],
+    0x10: ['DLE', 'DATA LINK ESCAPE'],
+    0x11: ['DC1', 'DEVICE CONTROL ONE'],
+    0x12: ['DC2', 'DEVICE CONTROL TWO'],
+    0x13: ['DC3', 'DEVICE CONTROL THREE'],
+    0x14: ['DC4', 'DEVICE CONTROL FOUR'],
+    0x15: ['NAK', 'NEGATIVE ACKNOWLEDGE'],
+    0x16: ['SYN', 'SYNCHRONOUS IDLE'],
+    0x17: ['ETB', 'END OF TRANSMISSION BLOCK'],
+    0x18: ['CAN', 'CANCEL'],
+    0x19: ['EM',  'END OF MEDIUM'],
+    0x1a: ['SUB', 'SUBSTITUTE'],
+    0x1b: ['ESC', 'ESCAPE'],
+    0x1c: ['FS',  'FILE SEPARATOR'],
+    0x1d: ['GS',  'GROUP SEPARATOR'],
+    0x1e: ['RS',  'RECORD SEPARATOR'],
+    0x1f: ['US',  'UNIT SEPARATOR'],
+    0x7f: ['DEL', 'DELETE'],
+};
+
+// Tab, LF and CR are deliberately NOT marked. They are ordinary whitespace
+// inside a quoted field — parseCsv keeps them verbatim, and a Windows multi-line
+// cell would otherwise sprout a chip at every line break.
+export function isMarkedControlChar(code: number): boolean {
+    if (code === 0x09 || code === 0x0a || code === 0x0d) return false;
+    return code < 0x20 || code === 0x7f;
+}
+
+export function hasControlChars(value: string): boolean {
+    for (let i = 0; i < value.length; i++) {
+        if (isMarkedControlChar(value.charCodeAt(i))) return true;
+    }
+    return false;
+}
+
+export type CellSegment =
+    | { type: 'text';    text: string }
+    | { type: 'control'; abbr: string; label: string };
+
+function labelFor(code: number): { abbr: string; label: string } {
+    const hex   = 'U+' + code.toString(16).toUpperCase().padStart(4, '0');
+    const known = CONTROL_NAMES[code];
+    // Every code isMarkedControlChar accepts is in the table; the fallback only
+    // guards against the two drifting apart.
+    return known
+        ? { abbr: known[0], label: hex + ' ' + known[1] }
+        : { abbr: hex.slice(2), label: hex };
+}
+
+// Splits a cell value into runs of ordinary text and single control characters,
+// in order. Concatenating the text runs and the source characters reproduces the
+// input exactly — nothing is dropped, the grid only draws the pieces differently.
+export function splitControlChars(value: string): CellSegment[] {
+    const out: CellSegment[] = [];
+    let text = '';
+    for (let i = 0; i < value.length; i++) {
+        const code = value.charCodeAt(i);
+        if (!isMarkedControlChar(code)) { text += value[i]; continue; }
+        if (text !== '') { out.push({ type: 'text', text }); text = ''; }
+        out.push({ type: 'control', ...labelFor(code) });
+    }
+    if (text !== '') out.push({ type: 'text', text });
+    return out;
+}

test/control-chars.test.cjs

@@ -0,0 +1,118 @@
+// Tests for the control-character splitter (webview/utils/control-chars.ts),
+// which turns a cell value into text runs plus labelled control characters so
+// the grid can draw a chip instead of an anonymous tofu box.
+// The two things that must hold: the split is LOSSLESS (nothing is dropped or
+// reordered — the cell still shows the whole value), and ordinary whitespace
+// inside a quoted field (tab, LF, CR) is left alone so multi-line cells do not
+// sprout a chip at every line break.
+//
+// Run after `npm run compile` (or `tsc -p ./`):  node test/control-chars.test.cjs
+
+const assert = require('assert');
+const {
+    isMarkedControlChar,
+    hasControlChars,
+    splitControlChars,
+} = require('../out/webview/utils/control-chars.js');
+
+let failures = 0;
+function test(name, fn) {
+    try { fn(); console.log('  ✓ ' + name); }
+    catch (e) { failures++; console.error('  ✗ ' + name + '\n      ' + e.message); }
+}
+
+const ch = c => String.fromCharCode(c);
+const GS = ch(0x1d);
+
+console.log('control characters');
+
+// ── which characters are marked ──────────────────────────────────────────────
+
+test('C0 control characters are marked', () => {
+    assert.strictEqual(isMarkedControlChar(0x00), true);
+    assert.strictEqual(isMarkedControlChar(0x1d), true);
+    assert.strictEqual(isMarkedControlChar(0x1f), true);
+});
+
+test('DEL is marked', () => {
+    assert.strictEqual(isMarkedControlChar(0x7f), true);
+});
+
+test('tab, LF and CR are left alone — they are whitespace in a quoted field', () => {
+    assert.strictEqual(isMarkedControlChar(0x09), false);
+    assert.strictEqual(isMarkedControlChar(0x0a), false);
+    assert.strictEqual(isMarkedControlChar(0x0d), false);
+});
+
+test('printable characters are never marked', () => {
+    assert.strictEqual(isMarkedControlChar(0x20), false);
+    assert.strictEqual(isMarkedControlChar('A'.charCodeAt(0)), false);
+    assert.strictEqual(isMarkedControlChar('é'.charCodeAt(0)), false);
+});
+
+// ── hasControlChars ──────────────────────────────────────────────────────────
+
+test('hasControlChars spots a separator buried in a long value', () => {
+    assert.strictEqual(hasControlChars('4901234567894ABC' + GS + '17250131LOT42'), true);
+});
+
+test('hasControlChars is false for ordinary values, including multi-line ones', () => {
+    assert.strictEqual(hasControlChars('plain text'), false);
+    assert.strictEqual(hasControlChars('line1\r\nline2\tpadded'), false);
+    assert.strictEqual(hasControlChars(''), false);
+});
+
+// ── splitControlChars ────────────────────────────────────────────────────────
+
+test('a value with no control characters stays one text segment', () => {
+    assert.deepStrictEqual(splitControlChars('abc'), [{ type: 'text', text: 'abc' }]);
+});
+
+test('an empty value produces no segments', () => {
+    assert.deepStrictEqual(splitControlChars(''), []);
+});
+
+test('a control character is split out and labelled with its ASCII name', () => {
+    assert.deepStrictEqual(splitControlChars('a' + GS + 'b'), [
+        { type: 'text', text: 'a' },
+        { type: 'control', abbr: 'GS', label: 'U+001D GROUP SEPARATOR' },
+        { type: 'text', text: 'b' },
+    ]);
+});
+
+test('leading and trailing control characters do not emit empty text runs', () => {
+    assert.deepStrictEqual(splitControlChars(GS + 'x' + GS), [
+        { type: 'control', abbr: 'GS', label: 'U+001D GROUP SEPARATOR' },
+        { type: 'text', text: 'x' },
+        { type: 'control', abbr: 'GS', label: 'U+001D GROUP SEPARATOR' },
+    ]);
+});
+
+test('adjacent control characters each get their own segment', () => {
+    const segs = splitControlChars(ch(0x00) + ch(0x1f));
+    assert.deepStrictEqual(segs, [
+        { type: 'control', abbr: 'NUL', label: 'U+0000 NULL' },
+        { type: 'control', abbr: 'US',  label: 'U+001F UNIT SEPARATOR' },
+    ]);
+});
+
+test('the split is lossless — text runs plus one char per control segment rebuild the value', () => {
+    const raw = 'ab' + GS + 'cd' + ch(0x07) + '\tef\n' + ch(0x7f);
+    const rebuilt = splitControlChars(raw)
+        .map(s => s.type === 'text' ? s.text : '?')
+        .join('');
+    assert.strictEqual(rebuilt.length, raw.length, 'one placeholder per control char, text kept verbatim');
+    assert.strictEqual(rebuilt, 'ab?cd?\tef\n?');
+});
+
+test('every marked C0 character has a real abbreviation, not a hex fallback', () => {
+    for (let code = 0; code < 0x20; code++) {
+        if (!isMarkedControlChar(code)) continue;
+        const seg = splitControlChars(ch(code))[0];
+        assert.ok(/^[A-Z]{2,3}[0-9]?$/.test(seg.abbr), 'code ' + code + ' has abbr ' + seg.abbr);
+        assert.ok(seg.label.indexOf(' ') > 0, 'code ' + code + ' has a name in its label');
+    }
+});
+
+if (failures) { console.error('\n' + failures + ' test(s) failed'); process.exit(1); }
+console.log('\nAll tests passed');