make site work with the Cloudflare OpenNext adapter

update the site application so that it can be build using the
Cloudflare OpenNext adapter (`@opennextjs/cloudflare`) and thus
deployed on Cloudflare Workers

> [!Note]
> This is very experimental and full of hacks
> it's very much a work-in-progress right now

___

Co-authored-by: Brian Muenzenmeyer <[email protected]>
Co-authored-by: Igor Minar <[email protected]>

Author: 3 people

.gitignore

@@ -32,3 +32,10 @@ cache
 tsconfig.tsbuildinfo
 
 dist/
+
+# Ignore worker artifacts
+apps/site/.open-next
+apps/site/.wrangler
+
+# Pre-build generated files
+apps/site/.generated

apps/site/.cloudflare/empty.mjs

@@ -0,0 +1 @@
+export default {};

apps/site/.cloudflare/node/fs.mjs

@@ -0,0 +1,18 @@
+import { files } from '../../.generated/next.helpers.mjs';
+
+export function readdir(params, cb) {
+  console.log('fs#readdir', params);
+  cb(null, []);
+}
+
+export function exists(path, cb) {
+  const result =
+    files.includes(path) || files.includes(path.replace(/^\//, ''));
+  console.log('fs#exists', path, result);
+  cb(result);
+}
+
+export default {
+  readdir,
+  exists,
+};

apps/site/.cloudflare/node/fs/promises.mjs

@@ -0,0 +1,28 @@
+import { getCloudflareContext } from '@opennextjs/cloudflare';
+
+export async function readFile(path) {
+  console.log('fs/promies#readFile', path);
+
+  const { env } = await getCloudflareContext();
+
+  const text = await env.ASSETS.fetch(
+    new URL(path, 'https://jamesrocks/')
+  ).then(response => response.text());
+  return text;
+}
+
+export async function readdir(params) {
+  console.log('fs/promises#readdir', params);
+  return Promise.resolve([]);
+}
+
+export async function exists(...args) {
+  console.log('fs/promises#exists', args);
+  return Promise.resolve(false);
+}
+
+export default {
+  readdir,
+  exists,
+  readFile,
+};

apps/site/.cloudflare/opentelemetry.mjs

@@ -0,0 +1,7 @@
+// we shim @opentelemetry/api to the throwing shim so that it will throw right away, this is so that we throw inside the
+// try block here: https://github.com/vercel/next.js/blob/9e8266a7/packages/next/src/server/lib/trace/tracer.ts#L27-L31
+// causing the code to require the 'next/dist/compiled/@opentelemetry/api' module instead (which properly works)
+
+// IMPORTANT: we already do that in the open-next Cloudflare adapter, it shouldn't be necessary here too
+//            (https://github.com/opennextjs/opennextjs-cloudflare/issues/219 seems to be the same issue)
+throw new Error();

apps/site/.cloudflare/server-only.mjs

@@ -0,0 +1,4 @@
+// In our aliased fs code: apps/site/.cloudflare/node/fs/promises.mjs we are importing `getCloudflareContext`
+// from `@opennextjs/cloudflare`, this in turn imports from `server-only`, this aliasing makes it so that
+// server-only is not actually removed from the final bundle as it would otherwise cause an incorrect server
+// internal error

apps/site/CLOUDFLARE.md

@@ -0,0 +1,75 @@
+# nodejs.org on OpenNext for Cloudflare
+
+## Getting started
+
+To develop, build, preview, and deploy nodejs.org, execute the following commands to get started:
+
+```
+nvm use
+npm install
+cd apps/site
+```
+
+## Developing locally
+
+To develop locally, run the usual:
+
+```
+npm run dev
+```
+
+## Build nodejs.org production distribution using OpenNext
+
+To build you need connection to the Internet because the build system will try to fetch the following files:
+
+- https://nodejs.org/dist/index.json
+- https://raw.githubusercontent.com/nodejs/Release/master/schedule.json
+
+```
+npm run cf:build
+```
+
+## Preview a production build locally
+
+You can preview production build locally using [wrangler](https://developers.cloudflare.com/workers/wrangler/):
+
+```
+npm run cf:preview
+```
+
+## Deploying a build to production
+
+To build and deploy the application run:
+
+```
+npm run cf:deploy
+```
+
+The build is currently deployed to a dedicated "nodejs.org" (Cloudflare account id: 8ed4d03ac99f77561d0e8c9cbcc76cb6): https://nodejs-website.web-experiments.workers.dev
+
+You can monitor and configure the project at https://dash.cloudflare.com/8ed4d03ac99f77561d0e8c9cbcc76cb6/workers/services/view/nodejs-website/production
+
+## TODOs
+
+The following is an incomplete list of tasks and problems that still need to be resolved:
+
+- [x] update `@opennextjs/cloudflare` to the latest in `/apps/site/package.json`
+- [x] sort out issues with `eval` and MDX (Claudio is looking into this one)
+- [x] and undo edits in `./app/[locale]/[[...path]]/page.tsx`
+- [x] reimplement `getMarkdownFiles` in `next.helpers.mjs` to be generated at build time
+  - this can be accomplished either via a npm/turbo prebuild task, or possibly as part of next.js SSG/staticProps but
+  - [ ] we need to ensure that we don't end up accidentally downloading this big file to the client as part of hydration
+  - [x] once we have easy access to the list of files, we should roll back changes to `next-data/providers/blogData.ts`
+- [x] back out most changes from `next.dynamic.mjs`
+  - [x] instead of using runtime detection via `globalThis.navigator?.userAgent`, we should instead use `alias` feature in `wrangler.toml` to override the implementation of `node:fs` calls but only when running in workerd as we need the build to keep on running in node.js for SSG to work
+  - [x] could we reimplement the `existsAsync` call as sync `exists` which consults `getMarkdownFiles` from the task above?
+- [ ] remove symlink hack in `package.json#build:cloudflare`
+  - would it be possible to make the pages directory part of assets in a less hacky way?
+  - [ ] move these files under `.open-next/assets/cdn-cgi/pages` so that these raw md files are not publicly accessible as that could become a maintenance burden down the road.
+- [ ] wire up the changes with turborepo (right now just plain npm scripts are used)
+- [ ] reenable minification in `next.config.mjs`
+- [ ] remove as many `alias`es as possible from the `wrangler.toml` file
+      (the `alias`es that can't be removed should be fully investigated and documented)
+- [ ] fix flashes of unstyled content present on hard navigation
+- [x] enable caching
+- [x] fix routes for languages besides `en` 404ing

apps/site/app/[locale]/next-data/api-data/route.ts

@@ -21,7 +21,7 @@ const getPathnameForApiFile = (name: string, version: string) =>
 // for a digest and metadata of all API pages from the Node.js Website
 // @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
 export const GET = async () => {
-  const releases = provideReleaseData();
+  const releases = await provideReleaseData();
 
   const { versionWithPrefix } = releases.find(
     release => release.status === 'LTS'

apps/site/app/[locale]/next-data/release-data/route.ts

@@ -5,7 +5,7 @@ import { defaultLocale } from '@/next.locales.mjs';
 // for generating static data related to the Node.js Release Data
 // @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
 export const GET = async () => {
-  const releaseData = provideReleaseData();
+  const releaseData = await provideReleaseData();
 
   return Response.json(releaseData);
 };

apps/site/instrumentation.ts

@@ -6,7 +6,7 @@ import readline from 'node:readline';
 
 import graymatter from 'gray-matter';
 
-import { getMarkdownFiles } from '../../next.helpers.mjs';
+import { getMarkdownFiles } from '../../.generated/next.helpers.mjs';
 
 // gets the current blog path based on local module path
 const blogPath = join(process.cwd(), 'pages/en/blog');
@@ -63,6 +63,17 @@ const generateBlogData = async () => {
     '**/index.md',
   ]);
 
+  const result = {
+    /* generated at build time */
+  };
+
+  if (Object.keys(result).length > 0) {
+    return {
+      ...result,
+      posts: result.posts.map(post => ({ ...post, date: new Date(post.date) })),
+    };
+  }
+
   return new Promise(resolve => {
     const posts = [];
     const rawFrontmatter = [];

apps/site/next-data/generators/blogData.mjs

@@ -7,11 +7,15 @@ import { glob } from 'glob';
 
 import { availableLocaleCodes } from '../../next.locales.mjs';
 
+const preGeneratedDownloadSnippets = [
+  /* generated at build time */
+];
+
 /**
  * This method is used to generate the Node.js Website Download Snippets
  * for self-consumption during RSC and Static Builds
  */
-const generateDownloadSnippets = async () => {
+export const generateRawDownloadSnippets = async () => {
   /**
    * This generates all the Download Snippets for each available Locale
    *
@@ -37,6 +41,19 @@ const generateDownloadSnippets = async () => {
 
     return [locale, await Promise.all(snippets)];
   });
+  return await Promise.all(downloadSnippets);
+};
+
+/**
+ * This method is used to generate the Node.js Website Download Snippets
+ * for self-consumption during RSC and Static Builds
+ */
+const generateDownloadSnippets = async () => {
+  if (preGeneratedDownloadSnippets) {
+    return new Map(preGeneratedDownloadSnippets);
+  }
+
+  const downloadSnippets = generateRawDownloadSnippets();
 
   return new Map(await Promise.all(downloadSnippets));
 };

apps/site/next-data/generators/downloadSnippets.mjs

@@ -1,16 +1,16 @@
 import { cache } from 'react';
 
-import generateBlogData from '@/next-data/generators/blogData.mjs';
 import { BLOG_POSTS_PER_PAGE } from '@/next.constants.mjs';
-import type { BlogCategory, BlogPostsRSC } from '@/types';
+import type { BlogCategory, BlogPost, BlogPostsRSC } from '@/types';
+import generateBlogData from '@generated/blogData.mjs';
 
 const { categories, posts } = await generateBlogData();
 
 export const provideBlogCategories = cache(() => categories);
 
 export const provideBlogPosts = cache(
   (category: BlogCategory): BlogPostsRSC => {
-    const categoryPosts = posts
+    const categoryPosts = (posts as Array<BlogPost>)
       .filter(post => post.categories.includes(category))
       .sort((a, b) => b.date.getTime() - a.date.getTime());
 

apps/site/next-data/providers/blogData.ts

@@ -1,6 +1,6 @@
 import { cache } from 'react';
 
-import generateDownloadSnippets from '@/next-data/generators/downloadSnippets.mjs';
+import generateDownloadSnippets from '@generated/downloadSnippets.mjs';
 
 const downloadSnippets = await generateDownloadSnippets();
 

apps/site/next-data/providers/downloadSnippets.ts

@@ -2,8 +2,6 @@ import { cache } from 'react';
 
 import generateReleaseData from '@/next-data/generators/releaseData.mjs';
 
-const releaseData = await generateReleaseData();
-
-const provideReleaseData = cache(() => releaseData);
+const provideReleaseData = cache(() => generateReleaseData());
 
 export default provideReleaseData;

apps/site/next-data/providers/releaseData.ts

@@ -69,6 +69,15 @@ const nextConfig = {
   // as we already check it on the CI within each Pull Request
   // we also configure ESLint to run its lint checking on all files (next lint)
   eslint: { dirs: ['.'], ignoreDuringBuilds: true },
+  // Adds custom WebPack configuration to our Next.js setup
+  webpack: function (config) {
+    // CF hacking: temporarily disable minification to make debugging easier
+    config.optimization = {
+      minimize: false,
+    };
+
+    return config;
+  },
   experimental: {
     // Ensure that server-side code is also minified
     serverMinification: true,

apps/site/next.config.mjs

@@ -1,12 +1,15 @@
 'use strict';
 
+import { exists } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join, normalize, sep } from 'node:path';
 
 import matter from 'gray-matter';
 import { cache } from 'react';
 import { VFile } from 'vfile';
 
+import { getMarkdownFiles } from '@generated/next.helpers.mjs';
+
 import {
   BASE_PATH,
   BASE_URL,
@@ -19,7 +22,6 @@ import {
   IGNORED_ROUTES,
   PAGE_METADATA,
 } from './next.dynamic.constants.mjs';
-import { getMarkdownFiles } from './next.helpers.mjs';
 import { siteConfig } from './next.json.mjs';
 import { availableLocaleCodes, defaultLocale } from './next.locales.mjs';
 import { compile } from './next.mdx.compiler.mjs';
@@ -62,7 +64,7 @@ const getDynamicRouter = async () => {
 
   const websitePages = await getMarkdownFiles(
     process.cwd(),
-    `pages/${defaultLocale.code}`
+    `/pages/${defaultLocale.code}`
   );
 
   websitePages.forEach(filename => {
@@ -110,9 +112,20 @@ const getDynamicRouter = async () => {
 
     // This verifies if the given pathname actually exists on our Map
     // meaning that the route exists on the website and can be rendered
-    if (pathnameToFilename.has(normalizedPathname)) {
-      const filename = pathnameToFilename.get(normalizedPathname);
-      const filepath = join(process.cwd(), 'pages', locale, filename);
+    if (
+      pathnameToFilename.has(normalizedPathname) ||
+      pathnameToFilename.has(
+        `pages/en${normalizedPathname ? `/${normalizedPathname}` : ''}`
+      )
+    ) {
+      const filename = (
+        pathnameToFilename.get(normalizedPathname) ??
+        pathnameToFilename.get(
+          `pages/en${normalizedPathname ? `/${normalizedPathname}` : ''}`
+        )
+      ).replace(new RegExp(`^pages/en/`), '');
+
+      let filepath = join(process.cwd(), 'pages', locale, filename);
 
       // We verify if our Markdown cache already has a cache entry for a localized
       // version of this file, because if not, it means that either
@@ -143,6 +156,17 @@ const getDynamicRouter = async () => {
         return { source: fileLanguageContent, filename };
       }
 
+      const existsPromise = path =>
+        new Promise(resolve => exists(path, resolve));
+
+      // and return the current fetched result; If the file does not exist
+      // we fallback to the English source
+      if (await existsPromise(join(filepath, locale, filename))) {
+        filepath = join(filepath, locale, filename);
+
+        return { source: fileLanguageContent, filename };
+      }
+
       // Prevent infinite loops as if at this point the file does not exist with the default locale
       // then there must be an issue on the file system or there's an error on the mapping of paths to files
       if (locale === defaultLocale.code) {