Skip to content

Commit ec6f49e

Browse files
rajveermalviyarajveermalviya
committed
tools/content: Support surveying unimplemented KaTeX features
1 parent b0ad5a0 commit ec6f49e

File tree

2 files changed

+163
-1
lines changed

2 files changed

+163
-1
lines changed

‎tools/content/check-features

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ opt_verbose=
5050
opt_steps=()
5151
while (( $# )); do
5252
case "$1" in
53-
fetch|check) opt_steps+=("$1"); shift;;
53+
fetch|check|katex-check) opt_steps+=("$1"); shift;;
5454
--config) shift; opt_zuliprc="$1"; shift;;
5555
--verbose) opt_verbose=1; shift;;
5656
--help) usage; exit 0;;
@@ -98,11 +98,19 @@ run_check() {
9898
|| return 1
9999
}
100100

101+
run_katex_check() {
102+
flutter test tools/content/unimplemented_katex_test.dart \
103+
--dart-define=corpusDir="$opt_corpus_dir" \
104+
--dart-define=verbose="$opt_verbose" \
105+
|| return 1
106+
}
107+
101108
for step in "${opt_steps[@]}"; do
102109
echo "Running ${step}"
103110
case "${step}" in
104111
fetch) run_fetch ;;
105112
check) run_check ;;
113+
katex-check) run_katex_check ;;
106114
*) echo >&2 "Internal error: unknown step ${step}" ;;
107115
esac
108116
done

‎tools/content/unimplemented_katex_test.dart

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
// Override `flutter test`'s default timeout
2+
@Timeout(Duration(minutes: 10))
3+
library;
4+
5+
import 'dart:io';
6+
import 'dart:math';
7+
8+
import 'package:checks/checks.dart';
9+
import 'package:collection/collection.dart';
10+
import 'package:flutter/foundation.dart';
11+
import 'package:flutter_test/flutter_test.dart';
12+
import 'package:zulip/model/content.dart';
13+
import 'package:zulip/model/settings.dart';
14+
15+
import '../../test/model/binding.dart';
16+
import 'model.dart';
17+
18+
void main() async {
19+
TestZulipBinding.ensureInitialized();
20+
await testBinding.globalStore.settings.setBool(
21+
BoolGlobalSetting.renderKatex, true);
22+
23+
Future<void> checkForKatexFailuresInFile(File file) async {
24+
int totalMessageCount = 0;
25+
final Set<int> katexMessageIds = <int>{};
26+
final Set<int> failedKatexMessageIds = <int>{};
27+
int totalMathBlockNodes = 0;
28+
int failedMathBlockNodes = 0;
29+
int totalMathInlineNodes = 0;
30+
int failedMathInlineNodes = 0;
31+
32+
final failedMessageIdsByReason = <String, Set<int>>{};
33+
final failedMathNodesByReason = <String, List<MathNode>>{};
34+
35+
void walk(int messageId, DiagnosticsNode node) {
36+
final value = node.value;
37+
if (value is UnimplementedNode) return;
38+
39+
for (final child in node.getChildren()) {
40+
walk(messageId, child);
41+
}
42+
43+
if (value is! MathNode) return;
44+
katexMessageIds.add(messageId);
45+
switch (value) {
46+
case MathBlockNode(): totalMathBlockNodes++;
47+
case MathInlineNode(): totalMathInlineNodes++;
48+
}
49+
50+
if (value.nodes != null) return;
51+
failedKatexMessageIds.add(messageId);
52+
switch (value) {
53+
case MathBlockNode(): failedMathBlockNodes++;
54+
case MathInlineNode(): failedMathInlineNodes++;
55+
}
56+
57+
final hardFailReason = value.debugHardFailReason;
58+
final softFailReason = value.debugSoftFailReason;
59+
int failureCount = 0;
60+
61+
if (hardFailReason != null) {
62+
final firstLine = hardFailReason.stackTrace.toString().split('\n').first;
63+
final reason = 'hard fail: ${hardFailReason.error} "$firstLine"';
64+
(failedMessageIdsByReason[reason] ??= {}).add(messageId);
65+
(failedMathNodesByReason[reason] ??= []).add(value);
66+
failureCount++;
67+
}
68+
69+
if (softFailReason != null) {
70+
for (final cssClass in softFailReason.unsupportedCssClasses) {
71+
final reason = 'unsupported css class: $cssClass';
72+
(failedMessageIdsByReason[reason] ??= {}).add(messageId);
73+
(failedMathNodesByReason[reason] ??= []).add(value);
74+
failureCount++;
75+
}
76+
for (final cssProp in softFailReason.unsupportedInlineCssProperties) {
77+
final reason = 'unsupported inline css property: $cssProp';
78+
(failedMessageIdsByReason[reason] ??= {}).add(messageId);
79+
(failedMathNodesByReason[reason] ??= []).add(value);
80+
failureCount++;
81+
}
82+
}
83+
84+
if (failureCount == 0) {
85+
final reason = 'unknown';
86+
(failedMessageIdsByReason[reason] ??= {}).add(messageId);
87+
(failedMathNodesByReason[reason] ??= []).add(value);
88+
}
89+
}
90+
91+
await for (final message in readMessagesFromJsonl(file)) {
92+
totalMessageCount++;
93+
walk(message.id, parseContent(message.content).toDiagnosticsNode());
94+
}
95+
96+
final buf = StringBuffer();
97+
buf.writeln();
98+
buf.writeln('Out of $totalMessageCount total messages,'
99+
' ${katexMessageIds.length} of them were KaTeX containing messages'
100+
' and ${failedKatexMessageIds.length} of them failed.');
101+
buf.writeln('There were $totalMathBlockNodes math block nodes out of which $failedMathBlockNodes failed.');
102+
buf.writeln('There were $totalMathInlineNodes math inline nodes out of which $failedMathInlineNodes failed.');
103+
buf.writeln();
104+
105+
for (final MapEntry(key: reason, value: messageIds) in failedMessageIdsByReason.entries.sorted(
106+
(a, b) => b.value.length.compareTo(a.value.length),
107+
)) {
108+
final failedMathNodes = failedMathNodesByReason[reason]!;
109+
int oldestId = messageIds.reduce(min);
110+
int newestId = messageIds.reduce(max);
111+
112+
buf.writeln('Because of $reason:');
113+
buf.writeln(' ${messageIds.length} messages failed.');
114+
buf.writeln(' Oldest message: $oldestId, Newest message: $newestId');
115+
buf.writeln(' Message IDs (upto 100): ${messageIds.take(100).join(', ')}');
116+
buf.writeln(' TeX source (upto 30):');
117+
for (final node in failedMathNodes.take(30)) {
118+
final type = switch (node) {
119+
MathBlockNode() => 'block',
120+
MathInlineNode() => 'inline',
121+
};
122+
buf.writeln(' $type: "${node.texSource}"');
123+
}
124+
buf.writeln(' HTML (upto 10):');
125+
for (final node in failedMathNodes.take(10)) {
126+
buf.writeln(' ${node.debugHtmlText}');
127+
buf.writeln();
128+
}
129+
buf.writeln();
130+
}
131+
132+
check(failedKatexMessageIds.length, because: buf.toString()).equals(0);
133+
}
134+
135+
final corpusFiles = _getCorpusFiles();
136+
137+
if (corpusFiles.isEmpty) {
138+
throw Exception('No corpus found in directory "$_corpusDirPath" to check'
139+
' for katex failures.');
140+
}
141+
142+
group('Check for katex failures in', () {
143+
for (final file in corpusFiles) {
144+
test(file.path, () => checkForKatexFailuresInFile(file));
145+
}
146+
});
147+
}
148+
149+
const String _corpusDirPath = String.fromEnvironment('corpusDir');
150+
151+
Iterable<File> _getCorpusFiles() {
152+
final corpusDir = Directory(_corpusDirPath);
153+
return corpusDir.existsSync() ? corpusDir.listSync().whereType<File>() : [];
154+
}

0 commit comments

Comments
 (0)