make site work with the Cloudflare OpenNext adapter

update the site application so that it can be build using the
Cloudflare OpenNext adapter (`@opennextjs/cloudflare`) and thus
deployed on Cloudflare Workers

> [!Note]
> This is very experimental and currently very slow
> it's very much a work-in-progress right now

Author: dario-piotrowicz

.gitignore

@@ -32,3 +32,8 @@ cache
 tsconfig.tsbuildinfo
 
 dist/
+
+# Ignore worker artifacts
+apps/site/.open-next
+apps/site/.wrangler
+apps/site/.cloudflare/.asset-manifests

apps/site/.cloudflare/node/fs.mjs

@@ -0,0 +1,86 @@
+import fsPromises from 'fs/promises';
+
+import pagesManifest from '../.asset-manifests/pages.mjs';
+import snippetsManifest from '../.asset-manifests/snippets.mjs';
+
+export function readdir(path, options, cb) {
+  const withFileTypes = !!options.withFileTypes;
+
+  if (!withFileTypes) {
+    // TODO: also support withFileTypes false
+    throw new Error('fs#readdir please call readdir with withFileTypes true');
+  }
+
+  console.log('fs#readdir', path);
+
+  const result = findInDirentLikes(path);
+
+  const results =
+    !result || result.type !== 'directory'
+      ? []
+      : result.children.map(c => ({
+          name: c.name,
+          parentPath: c.parentPath,
+          path: c.path,
+          isFile: () => c.type === 'file',
+          isDirectory: () => c.type === 'directory',
+        }));
+
+  cb?.(null, results);
+}
+
+function findInDirentLikes(path) {
+  if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
+    return null;
+  }
+
+  // remove the leading `/`
+  path = path.slice(1);
+
+  const paths = path.split('/');
+
+  const manifestType = paths.shift();
+
+  const manifest = manifestType === 'pages' ? pagesManifest : snippetsManifest;
+
+  return recursivelyFindInDirentLikes(paths, manifest);
+  function recursivelyFindInDirentLikes(paths, direntLikes) {
+    const [current, ...restOfPaths] = paths;
+    const found = direntLikes.find(item => item.name === current);
+    if (!found) return null;
+    if (restOfPaths.length === 0) return found;
+    if (found.type !== 'directory') return null;
+    return recursivelyFindInDirentLikes(restOfPaths, found.children);
+  }
+}
+
+export function exists(path, cb) {
+  const result = existsImpl(path);
+  console.log('fs#exists', path, result);
+  cb(result);
+}
+
+export function existsSync(path) {
+  const result = existsImpl(path);
+  console.log('fs#existsSync', path, result);
+  return result;
+}
+
+function existsImpl(path) {
+  if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
+    return false;
+  }
+  return !!findInDirentLikes(path);
+}
+
+export function realpathSync() {
+  return true;
+}
+
+export default {
+  readdir,
+  exists,
+  existsSync,
+  realpathSync,
+  promises: fsPromises,
+};

apps/site/.cloudflare/node/fs/promises.mjs

@@ -0,0 +1,39 @@
+const cloudflareContextSymbol = Symbol.for('__cloudflare-context__');
+
+export async function readFile(path) {
+  console.log('fs/promies#readFile', path);
+
+  // This hack here is needed because readdir is not polyfilled correctly
+  // (if a directory contains a file inside a subdirectory that file is returned instead of the sub-directory
+  //  for example: readdir(`a`) where `a contains b/index.js` should return `b` as a directory instead currently
+  //  it returns `b/index.js` as a file)
+  path = path.replace(/^snippets\/.*?\/download\/snippets/, 'snippets');
+
+  // Same as above, hack to fix the wrong readdir polyfilling
+  path = path.replace(/\/blog\/blog\//, '/blog/');
+
+  // pages/en/blog/blog/vulnerability/september-2016-security-releases.md
+
+  const { env } = global[cloudflareContextSymbol];
+
+  const text = await env.ASSETS.fetch(
+    new URL(`/${path}`, 'https://jamesrocks/')
+  ).then(response => response.text());
+  return text;
+}
+
+export async function readdir(params) {
+  console.log('fs/promises#readdir', params);
+  return Promise.resolve([]);
+}
+
+export async function exists(...args) {
+  console.log('fs/promises#exists', args);
+  return Promise.resolve(false);
+}
+
+export default {
+  readdir,
+  exists,
+  readFile,
+};

apps/site/.cloudflare/prepare-build.mjs

@@ -0,0 +1,57 @@
+//@ts-check
+
+import nodeFs from 'node:fs/promises';
+import nodePath from 'node:path';
+
+await collectAndCopyDirToAssets('./pages');
+await collectAndCopyDirToAssets('./snippets');
+
+/**
+ * @param {string} path
+ * @returns {Promise<void>}
+ */
+async function collectAndCopyDirToAssets(path) {
+  await nodeFs.cp(path, nodePath.join('./.open-next/assets', path), {
+    recursive: true,
+    force: true,
+  });
+
+  const pagesChildren = await collectDirChildren(path);
+  await nodeFs.mkdir('./.cloudflare/.asset-manifests/', { recursive: true });
+  await nodeFs.writeFile(
+    `./.cloudflare/.asset-manifests/${nodePath.basename(path)}.mjs`,
+    `export default ${JSON.stringify(pagesChildren)}`
+  );
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<DirentLike[]>}
+ */
+async function collectDirChildren(path) {
+  const dirContent = await nodeFs.readdir(path, { withFileTypes: true });
+
+  return Promise.all(
+    dirContent.map(async item => {
+      const base = {
+        name: item.name,
+        parentPath: item.parentPath,
+      };
+      if (item.isFile()) {
+        return { ...base, type: 'file' };
+      } else {
+        const dirInfo = await collectDirChildren(
+          `${item.parentPath}/${item.name}`
+        );
+        return { ...base, type: 'directory', children: dirInfo };
+      }
+    })
+  );
+}
+
+/**
+ * @typedef {{ name: string, parentPath: string } } DirentLikeBase
+ * @typedef {DirentLikeBase & { type: 'file' }} DirentLikeFile
+ * @typedef {DirentLikeBase & { type: 'directory', children: DirentLike[] }} DirentLikeDir
+ * @typedef {DirentLikeFile|DirentLikeDir} DirentLike
+ */

apps/site/app/[locale]/feed/[feed]/route.ts

@@ -14,7 +14,7 @@ export const GET = async (_: Request, props: StaticParams) => {
   const params = await props.params;
 
   // Generate the Feed for the given feed type (blog, releases, etc)
-  const websiteFeed = provideWebsiteFeeds(params.feed);
+  const websiteFeed = await provideWebsiteFeeds(params.feed);
 
   return new NextResponse(websiteFeed, {
     headers: { 'Content-Type': 'application/xml' },

apps/site/app/[locale]/next-data/api-data/route.ts

@@ -21,7 +21,7 @@ const getPathnameForApiFile = (name: string, version: string) =>
 // for a digest and metadata of all API pages from the Node.js Website
 // @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
 export const GET = async () => {
-  const releases = provideReleaseData();
+  const releases = await provideReleaseData();
 
   const { versionWithPrefix } = releases.find(
     release => release.status === 'LTS'

apps/site/app/[locale]/next-data/blog-data/[category]/[page]/route.ts

@@ -20,12 +20,11 @@ export const GET = async (_: Request, props: StaticParams) => {
 
   const requestedPage = Number(params.page);
 
-  const data =
-    requestedPage >= 1
-      ? // This allows us to blindly get all blog posts from a given category
-        // if the page number is 0 or something smaller than 1
-        providePaginatedBlogPosts(params.category, requestedPage)
-      : provideBlogPosts(params.category);
+  const data = await (requestedPage >= 1
+    ? // This allows us to blindly get all blog posts from a given category
+      // if the page number is 0 or something smaller than 1
+      providePaginatedBlogPosts(params.category, requestedPage)
+    : provideBlogPosts(params.category));
 
   return Response.json(data, { status: data.posts.length ? 200 : 404 });
 };

apps/site/app/[locale]/next-data/download-snippets/route.tsx

@@ -11,7 +11,7 @@ export const GET = async (_: Request, props: StaticParams) => {
   const params = await props.params;
 
   // Retrieve all available Download snippets for a given locale if available
-  const snippets = provideDownloadSnippets(params.locale);
+  const snippets = await provideDownloadSnippets(params.locale);
 
   // We append always the default/fallback snippets when a result is found
   return Response.json(snippets, {

apps/site/app/[locale]/next-data/release-data/route.ts

@@ -5,7 +5,7 @@ import { defaultLocale } from '@/next.locales.mjs';
 // for generating static data related to the Node.js Release Data
 // @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
 export const GET = async () => {
-  const releaseData = provideReleaseData();
+  const releaseData = await provideReleaseData();
 
   return Response.json(releaseData);
 };

apps/site/app/[locale]/page.tsx

@@ -92,7 +92,7 @@ const getPage: FC<DynamicParams> = async props => {
   // Gets the current full pathname for a given path
   const pathname = dynamicRouter.getPathname(path);
 
-  const staticGeneratedLayout = DYNAMIC_ROUTES.get(pathname);
+  const staticGeneratedLayout = (await DYNAMIC_ROUTES()).get(pathname);
 
   // If the current pathname is a statically generated route
   // it means it does not have a Markdown file nor exists under the filesystem

apps/site/components/withDownloadSection.tsx

@@ -7,12 +7,15 @@ import getDownloadSnippets from '@/next-data/downloadSnippets';
 import getReleaseData from '@/next-data/releaseData';
 import { defaultLocale } from '@/next.locales.mjs';
 import { ReleaseProvider, ReleasesProvider } from '@/providers/releaseProvider';
+import type { DownloadSnippet } from '@/types';
 
 // By default the translated languages do not contain all the download snippets
 // Hence we always merge any translated snippet with the fallbacks for missing snippets
-const fallbackSnippets = await getDownloadSnippets(defaultLocale.code);
+let fallbackSnippets: Array<DownloadSnippet>;
 
 const WithDownloadSection: FC<PropsWithChildren> = async ({ children }) => {
+  fallbackSnippets ??= await getDownloadSnippets(defaultLocale.code);
+
   const locale = await getLocale();
   const releases = await getReleaseData();
   const snippets = await getDownloadSnippets(locale);

apps/site/next-data/downloadSnippets.ts

@@ -29,7 +29,7 @@ export default async function getDownloadSnippets(
     const { default: provideDownloadSnippets } = await import(
       '@/next-data/providers/downloadSnippets'
     );
-    return provideDownloadSnippets(lang)!;
+    return (await provideDownloadSnippets(lang))!;
   }
 
   // Applies the language to the URL, since this content is actually localized

apps/site/next-data/generators/blogData.mjs

@@ -1,8 +1,7 @@
 'use strict';
 
-import { createReadStream } from 'node:fs';
+import { readFile } from 'node:fs/promises';
 import { basename, extname, join } from 'node:path';
-import readline from 'node:readline';
 
 import graymatter from 'gray-matter';
 
@@ -63,50 +62,27 @@ const generateBlogData = async () => {
     '**/index.md',
   ]);
 
-  return new Promise(resolve => {
-    const posts = [];
-    const rawFrontmatter = [];
-
-    filenames.forEach(filename => {
-      // We create a stream for reading a file instead of reading the files
-      const _stream = createReadStream(join(blogPath, filename));
-
-      // We create a readline interface to read the file line-by-line
-      const _readLine = readline.createInterface({ input: _stream });
-
-      // Creates an array of the metadata based on the filename
-      // This prevents concurrency issues since the for-loop is synchronous
-      // and these event listeners are not
-      rawFrontmatter[filename] = [0, ''];
-
-      // We read line by line
-      _readLine.on('line', line => {
-        rawFrontmatter[filename][1] += `${line}\n`;
-
-        // We observe the frontmatter separators
-        if (line === '---') {
-          rawFrontmatter[filename][0] += 1;
-        }
-
-        // Once we have two separators we close the readLine and the stream
-        if (rawFrontmatter[filename][0] === 2) {
-          _readLine.close();
-          _stream.close();
-        }
-      });
-
-      // Then we parse gray-matter on the frontmatter
-      // This allows us to only read the frontmatter part of each file
-      // and optimise the read-process as we have thousands of markdown files
-      _readLine.on('close', () => {
-        posts.push(getFrontMatter(filename, rawFrontmatter[filename][1]));
-
-        if (posts.length === filenames.length) {
-          resolve({ categories: [...blogCategories], posts });
-        }
-      });
-    });
-  });
+  const posts = [];
+  // TODO: this should be done via a stream (like it originally was) instead of reading the whole file
+  //       (I went with `readFile` just because it's simpler for a POC, streams should work too)
+  for (const filename of filenames) {
+    const fileContents = await readFile(join(blogPath, filename), 'utf-8');
+
+    const frontmatterStart = fileContents.indexOf('---\n') + '---\n'.length;
+
+    const frontmatterEnd = fileContents
+      .slice(frontmatterStart)
+      .indexOf('---\n');
+
+    const rawFrontmatter =
+      '---\n' +
+      fileContents.slice(frontmatterStart, frontmatterEnd + '---\n'.length) +
+      '---\n';
+
+    posts.push(getFrontMatter(filename, rawFrontmatter));
+  }
+
+  return { categories: [...blogCategories], posts };
 };
 
 export default generateBlogData;