Make build scripts ESM

Author: marijnh

.gitignore

@@ -12,6 +12,7 @@
 /html/[012]*.html
 /html/ejs.js
 /code/chapter/*
+/code/chapter_info.js
 /code/file_server.js
 /code/skillsharing.zip
 /code/solutions/20_3_a_public_space_on_the_web.zip

00_intro.md

@@ -114,7 +114,7 @@ Each line of the previous program contains a single instruction. It could be wri
 
 Although that is already more readable than the soup of bits, it is still rather obscure. Using names instead of numbers for the instructions and memory locations helps.
 
-```{lang: "text/plain"}
+```{lang: "null"}
  Set “total” to 0.
  Set “count” to 1.
 [loop]

02_program_structure.md

@@ -137,7 +137,7 @@ Binding names can be any word. Digits can be part of binding names—`catch22` i
 
 Words with a special meaning, such as `let`, are _((keyword))s_, and they may not be used as binding names. There are also a number of words that are "reserved for use" in ((future)) versions of JavaScript, which also can't be used as binding names. The full list of keywords and reserved words is rather long.
 
-```{lang: "text/plain"}
+```{lang: "null"}
 break case catch class const continue debugger default
 delete do else enum export extends false finally for
 function if implements import interface in instanceof let

04_data.md

@@ -940,7 +940,7 @@ JSON looks similar to JavaScript's way of writing arrays and objects, with a few
 
 A journal entry might look like this when represented as JSON data:
 
-```{lang: "application/json"}
+```{lang: "json"}
 {
   "squirrel": false,
   "events": ["work", "touched tree", "pizza", "running"]

05_higher_order.md

@@ -219,7 +219,7 @@ Though I can fluently read only Latin characters, I appreciate the fact that peo
 The example ((data set)) contains some pieces of information about the 140 scripts defined in Unicode. It is available in the [coding sandbox](https://eloquentjavascript.net/code#5) for this chapter[ ([_https://eloquentjavascript.net/code#5_](https://eloquentjavascript.net/code#5))]{if book} as the `SCRIPTS` binding. The binding contains an array of objects, each of which describes a script.
 
 
-```{lang: "application/json"}
+```{lang: "json"}
 {
   name: "Coptic",
   ranges: [[994, 1008], [11392, 11508], [11513, 11520]],

09_regexp.md

@@ -721,7 +721,7 @@ This makes use of the fact that the value of an ((assignment)) expression (`=`)
 
 To conclude the chapter, we'll look at a problem that calls for ((regular expression))s. Imagine we are writing a program to automatically collect information about our enemies from the ((Internet)). (We will not actually write that program here, just the part that reads the ((configuration)) file. Sorry.) The configuration file looks like this:
 
-```{lang: "text/plain"}
+```{lang: "null"}
 searchengine=https://duckduckgo.com/?q=$1
 spitefulness=9.7
 

10_modules.md

@@ -384,7 +384,7 @@ A package is a chunk of code that can be distributed on its own. NPM is a reposi
 
 These are the bindings that the project from [Chapter ?](robot) creates:
 
-```{lang: "text/plain"}
+```{lang: "null"}
 roads
 buildGraph
 roadGraph

12_language.md

@@ -61,7 +61,7 @@ Expressions of type `"value"` represent literal strings or numbers. Their `value
 
 The `>(x, 5)` part of the previous program would be represented like this:
 
-```{lang: "application/json"}
+```{lang: "json"}
 {
   type: "apply",
   operator: {type: "word", name: ">"},

13_browser.md

@@ -34,7 +34,7 @@ A _network ((protocol))_ describes a style of communication over a ((network)).
 
 For example, the _Hypertext Transfer Protocol_ (((HTTP))) is a protocol for retrieving named ((resource))s (chunks of information, such as web pages or pictures). It specifies that the side making the request should start with a line like this, naming the resource and the version of the protocol that it is trying to use:
 
-```{lang: "text/plain"}
+```{lang: "null"}
 GET /index.html HTTP/1.1
 ```
 

14_dom.md

@@ -513,7 +513,7 @@ When multiple rules define a value for the same property, the most recently read
 
 It is possible to target things other than ((tag)) names in CSS rules. A rule for `.abc` applies to all elements with `"abc"` in their `class` attribute. A rule for `#xyz` applies to the element with an `id` attribute of `"xyz"` (which should be unique within the document).
 
-```{lang: "text/css"}
+```{lang: "css"}
 .subtle {
   color: gray;
   font-size: 80%;

16_game.md

@@ -403,7 +403,7 @@ As mentioned, the background is drawn as a `<table>` element. This nicely corres
 
 The following ((CSS)) makes the table look like the background we want:
 
-```{lang: "text/css"}
+```{lang: "css"}
 .background    { background: rgb(52, 166, 251);
                  table-layout: fixed;
                  border-spacing: 0;              }
@@ -441,7 +441,7 @@ function drawActors(actors) {
 
 To give an element more than one class, we separate the class names by spaces. In the ((CSS)) code shown next, the `actor` class gives the actors their absolute position. Their type name is used as an extra class to give them a color. We don't have to define the `lava` class again because we're reusing the class for the lava grid squares we defined earlier.
 
-```{lang: "text/css"}
+```{lang: "css"}
 .actor  { position: absolute;            }
 .coin   { background: rgb(241, 229, 89); }
 .player { background: rgb(64, 64, 64);   }
@@ -465,7 +465,7 @@ DOMDisplay.prototype.syncState = function(state) {
 
 By adding the level's current status as a class name to the wrapper, we can style the player actor slightly differently when the game is won or lost by adding a ((CSS)) rule that takes effect only when the player has an ((ancestor element)) with a given class.
 
-```{lang: "text/css"}
+```{lang: "css"}
 .lost .player {
   background: rgb(160, 64, 64);
 }
@@ -484,7 +484,7 @@ After touching ((lava)), the player's color turns dark red, suggesting scorching
 
 We can't assume that the level always fits in the _viewport_—the element into which we draw the game. That is why the `scrollPlayerIntoView` call is needed. It ensures that if the level is protruding outside the viewport, we scroll that viewport to make sure the player is near its center. The following ((CSS)) gives the game's wrapping DOM element a maximum size and ensures that anything that sticks out of the element's box is not visible. We also give it a relative position so that the actors inside it are positioned relative to the level's top-left corner.
 
-```{lang: "text/css"}
+```{lang: "css"}
 .game {
   overflow: hidden;
   max-width: 600px;

20_node.md

@@ -186,7 +186,7 @@ In the `npm install` example, you could see a ((warning)) about the fact that th
 
 The robot simulation from [Chapter ?](robot), as modularized in the exercise in [Chapter ?](modules#modular_robot), might have a `package.json` file like this:
 
-```{lang: "application/json"}
+```{lang: "json"}
 {
   "author": "Marijn Haverbeke",
   "name": "eloquent-javascript-robot",

21_skillsharing.md

@@ -82,7 +82,7 @@ We will use ((JSON)) as the format of our request and response body. Like in the
 
 A `GET` request to `/talks` returns a JSON document like this:
 
-```{lang: "application/json"}
+```{lang: "json"}
 [{"title": "Unituning",
   "presenter": "Jamal",
   "summary": "Modifying your cycle for extra style",

Makefile

@@ -8,11 +8,11 @@ html: $(foreach CHAP,$(CHAPTERS),html/$(CHAP).html) html/ejs.js \
       code/skillsharing.zip code/solutions/20_3_a_public_space_on_the_web.zip html/code/chapter_info.js
 
 html/%.html: %.md
-	node src/render_html.js $< > $@
-	node src/build_code.js $<
+	node src/render_html.mjs $< > $@
+	node src/build_code.mjs $<
 
-html/code/chapter_info.js: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) code/solutions/* src/chapter_info.js
-	node src/chapter_info.js > html/code/chapter_info.js
+html/code/chapter_info.js: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) code/solutions/* src/chapter_info.mjs
+	node src/chapter_info.mjs > html/code/chapter_info.js
 
 html/ejs.js: node_modules/codemirror/dist/index.js \
 	     node_modules/@codemirror/view/dist/index.js \
@@ -34,8 +34,8 @@ code/solutions/20_3_a_public_space_on_the_web.zip: $(wildcard code/solutions/20_
 	cd code/solutions; zip 20_3_a_public_space_on_the_web.zip 20_3_a_public_space_on_the_web/*
 
 test: html
-	@for F in $(CHAPTERS); do echo Testing $$F:; node src/run_tests.js $$F.md; done
-	@node src/check_links.js
+	@for F in $(CHAPTERS); do echo Testing $$F:; node src/run_tests.mjs $$F.md; done
+	@node src/check_links.mjs
 	@echo Done.
 
 tex: $(foreach CHAP,$(CHAPTERS),pdf/$(CHAP).tex) pdf/hints.tex $(patsubst img/%.svg,img/generated/%.pdf,$(SVGS))
@@ -51,8 +51,8 @@ book_mobile.pdf: pdf/book_mobile.tex tex
 	cd pdf && sh build.sh book_mobile > /dev/null
 	mv pdf/book_mobile.pdf .	
 
-pdf/hints.tex: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) src/extract_hints.js
-	node src/extract_hints.js | node src/render_latex.js - > $@
+pdf/hints.tex: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) src/extract_hints.mjs
+	node src/extract_hints.mjs | node src/render_latex.mjs - > $@
 
 img/generated/%.pdf: img/%.svg
 	inkscape --export-pdf=$@ $<
@@ -61,21 +61,21 @@ pdf/%.tex: %.md
 	node src/render_latex.js $< > $@
 
 book.epub: epub/titlepage.xhtml epub/toc.xhtml epub/hints.xhtml $(foreach CHAP,$(CHAPTERS),epub/$(CHAP).xhtml) \
-           epub/content.opf.src epub/style.css src/add_images_to_epub.js
+           epub/content.opf.src epub/style.css src/add_images_to_epub.mjs
 	rm -f $@
 	grep '<img' epub/*.xhtml | sed -e 's/.*src="\([^"]*\)".*/\1/' | xargs -I{} rsync -R "{}" epub
-	node src/add_images_to_epub.js
+	node src/add_images_to_epub.mjs
 	cd epub; zip -X ../$@ mimetype
 	cd epub; zip -X ../$@ -r * -x mimetype -x *.src
 
 epub/toc.xhtml: epub/toc.xhtml.src $(foreach CHAP,$(CHAPTERS),epub/$(CHAP).xhtml) epub/hints.xhtml
-	node src/generate_epub_toc.js $^ > $@
+	node src/generate_epub_toc.mjs $^ > $@
 
 epub/%.xhtml: %.md src/render_html.js
 	node src/render_html.js --epub $< > $@
 
-epub/hints.xhtml: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) src/extract_hints.js src/render_html.js
-	node src/extract_hints.js | node src/render_html.js --epub - > $@
+epub/hints.xhtml: $(foreach CHAP,$(CHAPTERS),$(CHAP).md) src/extract_hints.mjs src/render_html.js
+	node src/extract_hints.mjs | node src/render_html.mjs --epub - > $@
 
 epubcheck: book.epub
 	epubcheck book.epub 2>&1 | grep -v 'img/.*\.svg'

html/css/ejs.css

@@ -316,7 +316,7 @@ blockquote footer:before {
 .tok-atom, .sandbox-output-bool {color: #106}
 .tok-number, .sandbox-output-number {color: #042}
 .tok-definition {color: #009}
-.tok-variableName2, .tok-propertyName {color: #027}
+.tok-variableName2 {color: #027}
 .tok-type {color: #072}
 .tok-comment {color: #740}
 .tok-string, .sandbox-output-string {color: #700}

src/add_images_to_epub.js renamed to src/add_images_to_epub.mjs

@@ -1,5 +1,5 @@
-const {readdirSync, lstatSync, readFileSync, writeFileSync} = require("fs")
-const path = require("path")
+import {readdirSync, lstatSync, readFileSync, writeFileSync} from "fs"
+import * as path from "path"
 
 let images = []
 function scanDir(dir) {

src/build_code.js renamed to src/build_code.mjs

@@ -1,6 +1,6 @@
-const fs = require("fs")
-const PJSON = require("./pseudo_json")
-const varify = require("./varify")
+import * as fs from "fs"
+import * as PJSON from "./pseudo_json.mjs"
+import varify from "./varify.mjs"
 
 let file = process.argv[2]
 let input = fs.readFileSync(file, "utf8")

src/chapter_info.js renamed to src/chapter_info.mjs

@@ -2,8 +2,9 @@
 // their starting code, and collects it into a big JSON object
 // together with the solution code.
 
-const PJSON = require("./pseudo_json")
-let fs = require("fs");
+import * as PJSON from "./pseudo_json.mjs"
+import * as fs from "fs"
+import jszip from "jszip"
 
 let output = [], failed = false;
 
@@ -193,7 +194,7 @@ function chapterZipFile(meta, chapter) {
   if (exists && files.every(file => fs.statSync("html/" + file).mtime < exists))
     return name;
 
-  let zip = new (require("jszip"));
+  let zip = new jszip;
   for (let file of files) {
     zip.file(chapter.id + "/" + file, fs.readFileSync("html/" + file));
   }

src/check_links.js renamed to src/check_links.mjs

@@ -1,4 +1,4 @@
-let {readdirSync, readFileSync} = require("fs")
+import {readdirSync, readFileSync} from "fs"
 
 let files = Object.create(null)
 for (let name of readdirSync(".")) {

src/client/code.mjs

@@ -73,7 +73,7 @@ class CodeSandbox {
 
     document.querySelector("#solution").addEventListener("click", () => {
       let context = this.editor.state.facet(contextFacet)
-      this.setEditorState(context.solution, {type: context.type})
+      this.setEditorState(context.solution, context)
     })
 
     this.parseFragment() || this.selectChapter(0, "box")

src/extract_hints.js renamed to src/extract_hints.mjs

@@ -1,4 +1,4 @@
-const {readdirSync, readFileSync} = require("fs")
+import {readdirSync, readFileSync} from "fs"
 
 process.stdout.write("# Exercise Hints\n\nThe hints below might help when you are stuck with one of the exercises in this book. They don't give away the entire solution, but rather try to help you find it yourself.\n\n");
 

src/generate_epub_toc.js renamed to src/generate_epub_toc.mjs

@@ -1,5 +1,5 @@
-const {readFileSync} = require("fs")
-const {basename} = require("path")
+import {readFileSync} from "fs"
+import {basename} from "path"
 
 let [template, ...chapters] = process.argv.slice(2)
 

src/markdown.js renamed to src/markdown.mjs

@@ -1,5 +1,5 @@
-const PJSON = require("./pseudo_json")
-const markdownIt = require("markdown-it")
+import * as PJSON from "./pseudo_json.mjs"
+import markdownIt from "markdown-it"
 
 function parseData(str) {
   let tag = /^\s*(\w+)\s*?/.exec(str), args
@@ -171,7 +171,7 @@ function plugin(md) {
   md.inline.ruler.before("strikethrough", "index_term", parseIndexTerm)
 }
 
-module.exports = markdownIt({html: true})
-  .use(plugin)
-  .use(require("markdown-it-sup"))
-  .use(require("markdown-it-sub"))
+import sup from "markdown-it-sup"
+import sub from "markdown-it-sub"
+
+export default markdownIt({html: true}).use(plugin).use(sup).use(sub)

src/pseudo_json.js renamed to src/pseudo_json.mjs

@@ -26,7 +26,7 @@ class Stream {
   }
 }
 
-exports.parse = function(str) {
+export function parse(str) {
   let stream = new Stream(str)
   stream.space()
   let value = parseValue(stream)

src/render_html.js renamed to src/render_html.mjs

@@ -1,11 +1,18 @@
-const PJSON = require("./pseudo_json")
-let fs = require("fs"), mold = new (require("mold-template"))
-let {transformTokens} = require("./transform")
-let CodeMirror = require("codemirror/addon/runmode/runmode.node.js")
-require("codemirror/mode/javascript/javascript.js")
-require("codemirror/mode/xml/xml.js")
-require("codemirror/mode/css/css.js")
-require("codemirror/mode/htmlmixed/htmlmixed.js")
+import * as PJSON from "./pseudo_json.mjs"
+import {transformTokens} from "./transform.mjs"
+import markdown from "./markdown.mjs"
+import * as fs from "fs"
+import {dirname} from "path"
+import {fileURLToPath} from "url"
+import moldTemplate from "mold-template"
+import {highlightCode, classHighlighter} from "@lezer/highlight"
+import {Tree} from "@lezer/common"
+import {html} from "@codemirror/lang-html"
+import {javascript} from "@codemirror/lang-javascript"
+import {css} from "@codemirror/lang-css"
+import {json} from "@codemirror/lang-json"
+
+const mold = new moldTemplate
 
 let file, epub = false
 for (let arg of process.argv.slice(2)) {
@@ -16,16 +23,17 @@ for (let arg of process.argv.slice(2)) {
 if (!file) throw new Error("No input file")
 let chapter = /^\d{2}_([^\.]+)/.exec(file) || [null, "hints"]
 
-let {tokens, metadata} = transformTokens(require("./markdown").parse(fs.readFileSync(file, "utf8"), {}), {
+let {tokens, metadata} = transformTokens(markdown.parse(fs.readFileSync(file, "utf8"), {}), {
   defined: epub ? ["book", "html"] : ["interactive", "html"],
   strip: epub ? "hints" : "",
   takeTitle: true,
   index: false
 })
 
 let close = epub ? "/" : ""
+const dir = dirname(fileURLToPath(import.meta.url))
 
-let chapters = fs.readdirSync(__dirname + "/..")
+let chapters = fs.readdirSync(dir + "/..")
     .filter(file => /^\d{2}_\w+\.md$/.test(file))
     .sort()
     .map(file => /^\d{2}_(\w+)\.md$/.exec(file)[1])
@@ -36,12 +44,21 @@ function escapeChar(ch) {
 }
 function escape(str) { return str.replace(/[<>&"]/g, escapeChar) }
 
+// FIXME http highlighting
+
+const parsers = {
+  css: css().language.parser,
+  html: html().language.parser,
+  javascript: javascript().language.parser,
+  json: json().language.parser
+}
+
 function highlight(lang, text) {
-  let result = ""
-  CodeMirror.runMode(text, lang, (text, style) => {
-    let esc = escape(text)
-    result += style ? `<span class="${style.replace(/^|\s+/g, "$&cm-")}">${esc}</span>` : esc
-  })
+  let result = "", parser = parsers[lang], tree = parser ? parser.parse(text) : Tree.empty
+  highlightCode(text, tree, classHighlighter, (code, cls) => {
+    let esc = escape(code)
+    result += cls ? `<span class="${cls}">${esc}</span>` : esc
+  }, () => result += "\n")
   return result
 }
 
@@ -184,6 +201,6 @@ if (chapter && (index = chapters.indexOf(chapter[1])) > -1) {
   metadata.page = {type: "hints"}
 }
 
-let template = mold.bake("chapter", fs.readFileSync(__dirname + `/${epub ? "epub_" : ""}chapter.html`, "utf8"))
+let template = mold.bake("chapter", fs.readFileSync(dir + `/${epub ? "epub_" : ""}chapter.html`, "utf8"))
 
 console.log(template(metadata))