Skip to content

Commit 4ff9d79

Browse files
dario-piotrowiczdario-piotrowicz
committed
make site work with the Cloudflare OpenNext adapter
update the site application so that it can be build using the Cloudflare OpenNext adapter (`@opennextjs/cloudflare`) and thus deployed on Cloudflare Workers > [!Note] > This is very experimental and currently very slow > it's very much a work-in-progress right now
1 parent 0b2e7b6 commit 4ff9d79

24 files changed

+29851
-17413
lines changed

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,8 @@ cache
3232
tsconfig.tsbuildinfo
3333

3434
dist/
35+
36+
# Ignore worker artifacts
37+
apps/site/.open-next
38+
apps/site/.wrangler
39+
apps/site/.cloudflare/.asset-manifests

‎apps/site/.cloudflare/node/fs.mjs

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import fsPromises from 'fs/promises';
2+
3+
import pagesManifest from '../.asset-manifests/pages.mjs';
4+
import snippetsManifest from '../.asset-manifests/snippets.mjs';
5+
6+
export function readdir(path, options, cb) {
7+
const withFileTypes = !!options.withFileTypes;
8+
9+
if (!withFileTypes) {
10+
// TODO: also support withFileTypes false
11+
throw new Error('fs#readdir please call readdir with withFileTypes true');
12+
}
13+
14+
console.log('fs#readdir', path);
15+
16+
const result = findInDirentLikes(path);
17+
18+
const results =
19+
!result || result.type !== 'directory'
20+
? []
21+
: result.children.map(c => ({
22+
name: c.name,
23+
parentPath: c.parentPath,
24+
path: c.path,
25+
isFile: () => c.type === 'file',
26+
isDirectory: () => c.type === 'directory',
27+
}));
28+
29+
cb?.(null, results);
30+
}
31+
32+
function findInDirentLikes(path) {
33+
if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
34+
return null;
35+
}
36+
37+
// remove the leading `/`
38+
path = path.slice(1);
39+
40+
const paths = path.split('/');
41+
42+
const manifestType = paths.shift();
43+
44+
const manifest = manifestType === 'pages' ? pagesManifest : snippetsManifest;
45+
46+
return recursivelyFindInDirentLikes(paths, manifest);
47+
function recursivelyFindInDirentLikes(paths, direntLikes) {
48+
const [current, ...restOfPaths] = paths;
49+
const found = direntLikes.find(item => item.name === current);
50+
if (!found) return null;
51+
if (restOfPaths.length === 0) return found;
52+
if (found.type !== 'directory') return null;
53+
return recursivelyFindInDirentLikes(restOfPaths, found.children);
54+
}
55+
}
56+
57+
export function exists(path, cb) {
58+
const result = existsImpl(path);
59+
console.log('fs#exists', path, result);
60+
cb(result);
61+
}
62+
63+
export function existsSync(path) {
64+
const result = existsImpl(path);
65+
console.log('fs#existsSync', path, result);
66+
return result;
67+
}
68+
69+
function existsImpl(path) {
70+
if (!path.startsWith('/pages') && !path.startsWith('/snippets')) {
71+
return false;
72+
}
73+
return !!findInDirentLikes(path);
74+
}
75+
76+
export function realpathSync() {
77+
return true;
78+
}
79+
80+
export default {
81+
readdir,
82+
exists,
83+
existsSync,
84+
realpathSync,
85+
promises: fsPromises,
86+
};

‎apps/site/.cloudflare/node/fs/promises.mjs

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
const cloudflareContextSymbol = Symbol.for('__cloudflare-context__');
2+
3+
export async function readFile(path) {
4+
console.log('fs/promies#readFile', path);
5+
6+
const { env } = global[cloudflareContextSymbol];
7+
8+
const text = await env.ASSETS.fetch(
9+
new URL(`/${path}`, 'https://jamesrocks/')
10+
).then(response => response.text());
11+
return text;
12+
}
13+
14+
export async function readdir(params) {
15+
console.log('fs/promises#readdir', params);
16+
return Promise.resolve([]);
17+
}
18+
19+
export async function exists(...args) {
20+
console.log('fs/promises#exists', args);
21+
return Promise.resolve(false);
22+
}
23+
24+
export default {
25+
readdir,
26+
exists,
27+
readFile,
28+
};

‎apps/site/.cloudflare/prepare-build.mjs

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
//@ts-check
2+
3+
import nodeFs from 'node:fs/promises';
4+
import nodePath from 'node:path';
5+
6+
await collectAndCopyDirToAssets('./pages');
7+
await collectAndCopyDirToAssets('./snippets');
8+
9+
/**
10+
* @param {string} path
11+
* @returns {Promise<void>}
12+
*/
13+
async function collectAndCopyDirToAssets(path) {
14+
await nodeFs.cp(path, nodePath.join('./.open-next/assets', path), {
15+
recursive: true,
16+
force: true,
17+
});
18+
19+
const pagesChildren = await collectDirChildren(path);
20+
await nodeFs.mkdir('./.cloudflare/.asset-manifests/', { recursive: true });
21+
await nodeFs.writeFile(
22+
`./.cloudflare/.asset-manifests/${nodePath.basename(path)}.mjs`,
23+
`export default ${JSON.stringify(pagesChildren)}`
24+
);
25+
}
26+
27+
/**
28+
* @param {string} path
29+
* @returns {Promise<DirentLike[]>}
30+
*/
31+
async function collectDirChildren(path) {
32+
const dirContent = await nodeFs.readdir(path, { withFileTypes: true });
33+
34+
return Promise.all(
35+
dirContent.map(async item => {
36+
const base = {
37+
name: item.name,
38+
parentPath: item.parentPath,
39+
};
40+
if (item.isFile()) {
41+
return { ...base, type: 'file' };
42+
} else {
43+
const dirInfo = await collectDirChildren(
44+
`${item.parentPath}/${item.name}`
45+
);
46+
return { ...base, type: 'directory', children: dirInfo };
47+
}
48+
})
49+
);
50+
}
51+
52+
/**
53+
* @typedef {{ name: string, parentPath: string } } DirentLikeBase
54+
* @typedef {DirentLikeBase & { type: 'file' }} DirentLikeFile
55+
* @typedef {DirentLikeBase & { type: 'directory', children: DirentLike[] }} DirentLikeDir
56+
* @typedef {DirentLikeFile|DirentLikeDir} DirentLike
57+
*/

‎apps/site/app/[locale]/feed/[feed]/route.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export const GET = async (_: Request, props: StaticParams) => {
1414
const params = await props.params;
1515

1616
// Generate the Feed for the given feed type (blog, releases, etc)
17-
const websiteFeed = provideWebsiteFeeds(params.feed);
17+
const websiteFeed = await provideWebsiteFeeds(params.feed);
1818

1919
return new NextResponse(websiteFeed, {
2020
headers: { 'Content-Type': 'application/xml' },

‎apps/site/app/[locale]/next-data/api-data/route.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ const getPathnameForApiFile = (name: string, version: string) =>
2121
// for a digest and metadata of all API pages from the Node.js Website
2222
// @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
2323
export const GET = async () => {
24-
const releases = provideReleaseData();
24+
const releases = await provideReleaseData();
2525

2626
const { versionWithPrefix } = releases.find(
2727
release => release.status === 'LTS'

‎apps/site/app/[locale]/next-data/blog-data/[category]/[page]/route.ts

+5-6
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,11 @@ export const GET = async (_: Request, props: StaticParams) => {
2020

2121
const requestedPage = Number(params.page);
2222

23-
const data =
24-
requestedPage >= 1
25-
? // This allows us to blindly get all blog posts from a given category
26-
// if the page number is 0 or something smaller than 1
27-
providePaginatedBlogPosts(params.category, requestedPage)
28-
: provideBlogPosts(params.category);
23+
const data = await (requestedPage >= 1
24+
? // This allows us to blindly get all blog posts from a given category
25+
// if the page number is 0 or something smaller than 1
26+
providePaginatedBlogPosts(params.category, requestedPage)
27+
: provideBlogPosts(params.category));
2928

3029
return Response.json(data, { status: data.posts.length ? 200 : 404 });
3130
};

‎apps/site/app/[locale]/next-data/download-snippets/route.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export const GET = async (_: Request, props: StaticParams) => {
1111
const params = await props.params;
1212

1313
// Retrieve all available Download snippets for a given locale if available
14-
const snippets = provideDownloadSnippets(params.locale);
14+
const snippets = await provideDownloadSnippets(params.locale);
1515

1616
// We append always the default/fallback snippets when a result is found
1717
return Response.json(snippets, {

‎apps/site/app/[locale]/next-data/release-data/route.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { defaultLocale } from '@/next.locales.mjs';
55
// for generating static data related to the Node.js Release Data
66
// @see https://nextjs.org/docs/app/building-your-application/routing/router-handlers
77
export const GET = async () => {
8-
const releaseData = provideReleaseData();
8+
const releaseData = await provideReleaseData();
99

1010
return Response.json(releaseData);
1111
};

‎apps/site/app/[locale]/page.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ const getPage: FC<DynamicParams> = async props => {
9292
// Gets the current full pathname for a given path
9393
const pathname = dynamicRouter.getPathname(path);
9494

95-
const staticGeneratedLayout = DYNAMIC_ROUTES.get(pathname);
95+
const staticGeneratedLayout = (await DYNAMIC_ROUTES()).get(pathname);
9696

9797
// If the current pathname is a statically generated route
9898
// it means it does not have a Markdown file nor exists under the filesystem

‎apps/site/components/withDownloadSection.tsx

+4-1
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,15 @@ import getDownloadSnippets from '@/next-data/downloadSnippets';
77
import getReleaseData from '@/next-data/releaseData';
88
import { defaultLocale } from '@/next.locales.mjs';
99
import { ReleaseProvider, ReleasesProvider } from '@/providers/releaseProvider';
10+
import type { DownloadSnippet } from '@/types';
1011

1112
// By default the translated languages do not contain all the download snippets
1213
// Hence we always merge any translated snippet with the fallbacks for missing snippets
13-
const fallbackSnippets = await getDownloadSnippets(defaultLocale.code);
14+
let fallbackSnippets: Array<DownloadSnippet>;
1415

1516
const WithDownloadSection: FC<PropsWithChildren> = async ({ children }) => {
17+
fallbackSnippets ??= await getDownloadSnippets(defaultLocale.code);
18+
1619
const locale = await getLocale();
1720
const releases = await getReleaseData();
1821
const snippets = await getDownloadSnippets(locale);

‎apps/site/next-data/downloadSnippets.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export default async function getDownloadSnippets(
2929
const { default: provideDownloadSnippets } = await import(
3030
'@/next-data/providers/downloadSnippets'
3131
);
32-
return provideDownloadSnippets(lang)!;
32+
return (await provideDownloadSnippets(lang))!;
3333
}
3434

3535
// Applies the language to the URL, since this content is actually localized

‎apps/site/next-data/generators/blogData.mjs

+22-46
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
'use strict';
22

3-
import { createReadStream } from 'node:fs';
3+
import { readFile } from 'node:fs/promises';
44
import { basename, extname, join } from 'node:path';
5-
import readline from 'node:readline';
65

76
import graymatter from 'gray-matter';
87

@@ -63,50 +62,27 @@ const generateBlogData = async () => {
6362
'**/index.md',
6463
]);
6564

66-
return new Promise(resolve => {
67-
const posts = [];
68-
const rawFrontmatter = [];
69-
70-
filenames.forEach(filename => {
71-
// We create a stream for reading a file instead of reading the files
72-
const _stream = createReadStream(join(blogPath, filename));
73-
74-
// We create a readline interface to read the file line-by-line
75-
const _readLine = readline.createInterface({ input: _stream });
76-
77-
// Creates an array of the metadata based on the filename
78-
// This prevents concurrency issues since the for-loop is synchronous
79-
// and these event listeners are not
80-
rawFrontmatter[filename] = [0, ''];
81-
82-
// We read line by line
83-
_readLine.on('line', line => {
84-
rawFrontmatter[filename][1] += `${line}\n`;
85-
86-
// We observe the frontmatter separators
87-
if (line === '---') {
88-
rawFrontmatter[filename][0] += 1;
89-
}
90-
91-
// Once we have two separators we close the readLine and the stream
92-
if (rawFrontmatter[filename][0] === 2) {
93-
_readLine.close();
94-
_stream.close();
95-
}
96-
});
97-
98-
// Then we parse gray-matter on the frontmatter
99-
// This allows us to only read the frontmatter part of each file
100-
// and optimise the read-process as we have thousands of markdown files
101-
_readLine.on('close', () => {
102-
posts.push(getFrontMatter(filename, rawFrontmatter[filename][1]));
103-
104-
if (posts.length === filenames.length) {
105-
resolve({ categories: [...blogCategories], posts });
106-
}
107-
});
108-
});
109-
});
65+
const posts = [];
66+
// TODO: this should be done via a stream (like it originally was) instead of reading the whole file
67+
// (I went with `readFile` just because it's simpler for a POC, streams should work too)
68+
for (const filename of filenames) {
69+
const fileContents = await readFile(join(blogPath, filename), 'utf-8');
70+
71+
const frontmatterStart = fileContents.indexOf('---\n') + '---\n'.length;
72+
73+
const frontmatterEnd = fileContents
74+
.slice(frontmatterStart)
75+
.indexOf('---\n');
76+
77+
const rawFrontmatter =
78+
'---\n' +
79+
fileContents.slice(frontmatterStart, frontmatterEnd + '---\n'.length) +
80+
'---\n';
81+
82+
posts.push(getFrontMatter(filename, rawFrontmatter));
83+
}
84+
85+
return { categories: [...blogCategories], posts };
11086
};
11187

11288
export default generateBlogData;

‎apps/site/next-data/providers/blogData.ts

+14-7
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,22 @@ import { cache } from 'react';
22

33
import generateBlogData from '@/next-data/generators/blogData.mjs';
44
import { BLOG_POSTS_PER_PAGE } from '@/next.constants.mjs';
5-
import type { BlogCategory, BlogPostsRSC } from '@/types';
5+
import type { BlogCategory, BlogPostsRSC, BlogPost } from '@/types';
66

7-
const { categories, posts } = await generateBlogData();
7+
let blogData: {
8+
categories: Array<BlogCategory>;
9+
posts: Array<BlogPost>;
10+
};
811

9-
export const provideBlogCategories = cache(() => categories);
12+
export const provideBlogCategories = cache(async () => {
13+
blogData ??= await generateBlogData();
14+
return blogData.categories;
15+
});
1016

1117
export const provideBlogPosts = cache(
12-
(category: BlogCategory): BlogPostsRSC => {
13-
const categoryPosts = posts
18+
async (category: BlogCategory): Promise<BlogPostsRSC> => {
19+
blogData ??= await generateBlogData();
20+
const categoryPosts = blogData.posts
1421
.filter(post => post.categories.includes(category))
1522
.sort((a, b) => b.date.getTime() - a.date.getTime());
1623

@@ -32,8 +39,8 @@ export const provideBlogPosts = cache(
3239
);
3340

3441
export const providePaginatedBlogPosts = cache(
35-
(category: BlogCategory, page: number): BlogPostsRSC => {
36-
const { posts, pagination } = provideBlogPosts(category);
42+
async (category: BlogCategory, page: number): Promise<BlogPostsRSC> => {
43+
const { posts, pagination } = await provideBlogPosts(category);
3744

3845
// This autocorrects if invalid numbers are given to only allow
3946
// actual valid numbers to be provided

0 commit comments

Comments
 (0)