From b8c725de65bbc91c9a1d29e4a60148c86b7db3b1 Mon Sep 17 00:00:00 2001 From: Alexander Niebuhr Date: Mon, 2 Sep 2024 21:11:37 +0200 Subject: [PATCH 1/2] wip --- astro.config.ts | 1 + package.json | 1 + pnpm-lock.yaml | 86 +++++++++++++++++++++++++++++++-------- src/components/Tags.astro | 2 + src/content/config.ts | 21 ++++++++++ 5 files changed, 94 insertions(+), 17 deletions(-) diff --git a/astro.config.ts b/astro.config.ts index a9c4d0f..2fc8a02 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -13,6 +13,7 @@ export default defineConfig({ }, experimental: { serverIslands: true, + contentLayer: true, }, integrations: [ starlight({ diff --git a/package.json b/package.json index dfd8c2e..2ff0d0d 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "astro": "4.15.1", "semver": "7.6.3", "sharp": "0.33.5", + "@alexanderniebuhr/astro-monthly-blog-resource-loader": "0.0.1", "typescript": "5.5.4" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 29983ed..06708db 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,9 @@ importers: specifier: 5.5.4 version: 5.5.4 devDependencies: + '@alexanderniebuhr/astro-monthly-blog-resource-loader': + specifier: file:/Users/alexanderniebuhr/Developer/Projects/craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz + version: file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz(astro@4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4)) '@biomejs/biome': specifier: 1.8.3 version: 1.8.3 @@ -81,6 +84,12 @@ packages: '@actions/io@1.1.3': resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} + '@alexanderniebuhr/astro-monthly-blog-resource-loader@file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz': + resolution: {integrity: sha512-VqN3WoeRKDZWrPRATFx69hCg3l4TDV3XGKPinOWw/fjjRVnXKxDaNRz6vmo4SKHXg0QRqGDuvaeEYBPMLofnAg==, tarball: file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz} + version: 0.0.1 + peerDependencies: + astro: ^4.14.0 + '@ampproject/remapping@2.3.0': resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} @@ -2049,8 +2058,8 @@ packages: micromark-extension-mdx-expression@3.0.0: resolution: {integrity: sha512-sI0nwhUDz97xyzqJAbHQhp5TfaxEvZZZ2JDqUo+7NvyIYG6BZ5CPPqj2ogUoPJlmXHBnyZUzISg9+oUmU6tUjQ==} - micromark-extension-mdx-jsx@3.0.0: - resolution: {integrity: sha512-uvhhss8OGuzR4/N17L1JwvmJIpPhAd8oByMawEKx6NVdBCbesjH4t+vjEp3ZXft9DwvlKSD07fCeI44/N0Vf2w==} + micromark-extension-mdx-jsx@3.0.1: + resolution: {integrity: sha512-vNuFb9czP8QCtAQcEJn0UJQJZA8Dk6DXKBqx+bg/w0WGuSxDxNr7hErW89tHUY31dUW4NqEOWwmEUNhjTFmHkg==} micromark-extension-mdx-md@2.0.0: resolution: {integrity: sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==} @@ -2067,8 +2076,8 @@ packages: micromark-factory-label@2.0.0: resolution: {integrity: sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==} - micromark-factory-mdx-expression@2.0.1: - resolution: {integrity: sha512-F0ccWIUHRLRrYp5TC9ZYXmZo+p2AM13ggbsW4T0b5CRKP8KHVRB8t4pwtBgTxtjRmwrK0Irwm7vs2JOZabHZfg==} + micromark-factory-mdx-expression@2.0.2: + resolution: {integrity: sha512-5E5I2pFzJyg2CtemqAbcyCktpHXuJbABnsb32wX2U8IQKhhVFBqkcZR5LRm1WVoFqa4kTueZK4abep7wdo9nrw==} micromark-factory-space@2.0.0: resolution: {integrity: sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==} @@ -2286,8 +2295,8 @@ packages: resolution: {integrity: sha512-Q8qQfPiZ+THO/3ZrOrO0cJJKfpYCagtMUkXbnEfmgUjwXg6z/WBeOyS9APBBPCTSiDV+s4SwQGu8yFsiMRIudg==} engines: {node: '>=4'} - postcss@8.4.43: - resolution: {integrity: sha512-gJAQVYbh5R3gYm33FijzCZj7CHyQ3hWMgJMprLUlIYqCwTeZhBQ19wp0e9mA25BUbEvY5+EXuuaAjqQsrBxQBQ==} + postcss@8.4.44: + resolution: {integrity: sha512-Aweb9unOEpQ3ezu4Q00DPvvM2ZTUitJdNKeP/+uQgr1IBIqu574IaZoURId7BKtWMREwzKa9OgzPzezWGPWFQw==} engines: {node: ^10 || ^12 || >=14} preferred-pm@4.0.0: @@ -2639,9 +2648,15 @@ packages: unified@11.0.5: resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} + unist-util-filter@5.0.1: + resolution: {integrity: sha512-pHx7D4Zt6+TsfwylH9+lYhBhzyhEnCXs/lbq/Hstxno5z4gVdyc2WEW0asfjGKPyG4pEKrnBv5hdkO6+aRnQJw==} + unist-util-find-after@5.0.0: resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} + unist-util-inspect@8.1.0: + resolution: {integrity: sha512-mOlg8Mp33pR0eeFpo5d2902ojqFFOKMMG2hF8bmH7ZlhnmjFgh0NI3/ZDwdaBJNbvrS7LZFVrBVtIE9KZ9s7vQ==} + unist-util-is@6.0.0: resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} @@ -2657,6 +2672,9 @@ packages: unist-util-remove-position@5.0.0: resolution: {integrity: sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==} + unist-util-select@5.1.0: + resolution: {integrity: sha512-4A5mfokSHG/rNQ4g7gSbdEs+H586xyd24sdJqF1IWamqrLHvYb+DH48fzxowyOhOfK7YSqX+XlCojAyuuyyT2A==} + unist-util-stringify-position@4.0.0: resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} @@ -2980,6 +2998,20 @@ snapshots: '@actions/io@1.1.3': {} + '@alexanderniebuhr/astro-monthly-blog-resource-loader@file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz(astro@4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4))': + dependencies: + acorn: 8.12.1 + astro: 4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4) + mdast-util-from-markdown: 2.0.1 + mdast-util-mdx: 3.0.0 + micromark-extension-mdxjs: 3.0.0 + unist-util-filter: 5.0.1 + unist-util-inspect: 8.1.0 + unist-util-select: 5.1.0 + unist-util-visit: 5.0.0 + transitivePeerDependencies: + - supports-color + '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.5 @@ -3594,8 +3626,8 @@ snapshots: hast-util-to-html: 9.0.2 hast-util-to-text: 4.0.2 hastscript: 9.0.0 - postcss: 8.4.43 - postcss-nested: 6.2.0(postcss@8.4.43) + postcss: 8.4.44 + postcss-nested: 6.2.0(postcss@8.4.44) unist-util-visit: 5.0.0 unist-util-visit-parents: 6.0.1 @@ -5222,22 +5254,23 @@ snapshots: dependencies: '@types/estree': 1.0.5 devlop: 1.1.0 - micromark-factory-mdx-expression: 2.0.1 + micromark-factory-mdx-expression: 2.0.2 micromark-factory-space: 2.0.0 micromark-util-character: 2.1.0 micromark-util-events-to-acorn: 2.0.2 micromark-util-symbol: 2.0.0 micromark-util-types: 2.0.0 - micromark-extension-mdx-jsx@3.0.0: + micromark-extension-mdx-jsx@3.0.1: dependencies: '@types/acorn': 4.0.6 '@types/estree': 1.0.5 devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 - micromark-factory-mdx-expression: 2.0.1 + micromark-factory-mdx-expression: 2.0.2 micromark-factory-space: 2.0.0 micromark-util-character: 2.1.0 + micromark-util-events-to-acorn: 2.0.2 micromark-util-symbol: 2.0.0 micromark-util-types: 2.0.0 vfile-message: 4.0.2 @@ -5263,7 +5296,7 @@ snapshots: acorn: 8.12.1 acorn-jsx: 5.3.2(acorn@8.12.1) micromark-extension-mdx-expression: 3.0.0 - micromark-extension-mdx-jsx: 3.0.0 + micromark-extension-mdx-jsx: 3.0.1 micromark-extension-mdx-md: 2.0.0 micromark-extension-mdxjs-esm: 3.0.0 micromark-util-combine-extensions: 2.0.0 @@ -5282,10 +5315,11 @@ snapshots: micromark-util-symbol: 2.0.0 micromark-util-types: 2.0.0 - micromark-factory-mdx-expression@2.0.1: + micromark-factory-mdx-expression@2.0.2: dependencies: '@types/estree': 1.0.5 devlop: 1.1.0 + micromark-factory-space: 2.0.0 micromark-util-character: 2.1.0 micromark-util-events-to-acorn: 2.0.2 micromark-util-symbol: 2.0.0 @@ -5566,9 +5600,9 @@ snapshots: dependencies: find-up: 4.1.0 - postcss-nested@6.2.0(postcss@8.4.43): + postcss-nested@6.2.0(postcss@8.4.44): dependencies: - postcss: 8.4.43 + postcss: 8.4.44 postcss-selector-parser: 6.1.2 postcss-selector-parser@6.1.2: @@ -5576,7 +5610,7 @@ snapshots: cssesc: 3.0.0 util-deprecate: 1.0.2 - postcss@8.4.43: + postcss@8.4.44: dependencies: nanoid: 3.3.7 picocolors: 1.0.1 @@ -6024,11 +6058,21 @@ snapshots: trough: 2.2.0 vfile: 6.0.3 + unist-util-filter@5.0.1: + dependencies: + '@types/unist': 3.0.3 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 + unist-util-find-after@5.0.0: dependencies: '@types/unist': 3.0.3 unist-util-is: 6.0.0 + unist-util-inspect@8.1.0: + dependencies: + '@types/unist': 3.0.3 + unist-util-is@6.0.0: dependencies: '@types/unist': 3.0.3 @@ -6051,6 +6095,14 @@ snapshots: '@types/unist': 3.0.3 unist-util-visit: 5.0.0 + unist-util-select@5.1.0: + dependencies: + '@types/unist': 3.0.3 + css-selector-parser: 3.0.5 + devlop: 1.1.0 + nth-check: 2.1.1 + zwitch: 2.0.4 + unist-util-stringify-position@4.0.0: dependencies: '@types/unist': 3.0.3 @@ -6100,7 +6152,7 @@ snapshots: vite@5.4.2(@types/node@20.16.3): dependencies: esbuild: 0.21.5 - postcss: 8.4.43 + postcss: 8.4.44 rollup: 4.21.2 optionalDependencies: '@types/node': 20.16.3 diff --git a/src/components/Tags.astro b/src/components/Tags.astro index fd1ff34..44d7819 100644 --- a/src/components/Tags.astro +++ b/src/components/Tags.astro @@ -6,6 +6,8 @@ interface Props { tags?: string[]; } const allResources = await getCollection("resources"); +const allExtracted = await getCollection("automatedresources"); +console.log(allExtracted); const referer = Astro.request.headers.get("referer"); const tagsParams = diff --git a/src/content/config.ts b/src/content/config.ts index 7977ffd..3149d52 100644 --- a/src/content/config.ts +++ b/src/content/config.ts @@ -2,6 +2,7 @@ import { defineCollection, z } from 'astro:content'; import { docsSchema } from '@astrojs/starlight/schema'; import { minVersion, outside, validRange } from 'semver'; import pkg from '../../package.json'; +import { astroMonthlyBlogResourceLoader } from '@alexanderniebuhr/astro-monthly-blog-resource-loader'; const astroVersion = minVersion(pkg.dependencies.astro)?.version; @@ -36,7 +37,27 @@ const resourcesSchema = defineCollection({ }), }); +const automatedresources = defineCollection({ + loader: astroMonthlyBlogResourceLoader({ + urls: [ + 'https://raw.githubusercontent.com/withastro/astro.build/main/src/content/blog/whats-new-june-2024.mdx', + 'https://raw.githubusercontent.com/withastro/astro.build/main/src/content/blog/whats-new-july-2024.mdx', + 'https://raw.githubusercontent.com/withastro/astro.build/main/src/content/blog/whats-new-august-2024.mdx', + ], + exclude: [ + /github.com/, + /astro.build/, + /x.com/, + /2023.stateofjs.com/, + /astrolicious.dev/, + /astro-tips.dev/, + /reddit.com\/r\/withastro\/$/, + ], + }), +}); + export const collections = { docs: starlightSchema, resources: resourcesSchema, + automatedresources: automatedresources, }; From 02c8c25e07ef4e2976337d456b1500d7c63b92a4 Mon Sep 17 00:00:00 2001 From: Alexander Niebuhr Date: Sat, 7 Sep 2024 19:42:39 +0200 Subject: [PATCH 2/2] inline loader --- astro.config.ts | 1 + package.json | 4 +- pnpm-lock.yaml | 48 ++-------- src/content/config.ts | 2 +- src/utils/astro-monthly-blog-loader.ts | 119 +++++++++++++++++++++++++ src/utils/index.ts | 1 + 6 files changed, 134 insertions(+), 41 deletions(-) create mode 100644 src/utils/astro-monthly-blog-loader.ts create mode 100644 src/utils/index.ts diff --git a/astro.config.ts b/astro.config.ts index 2fc8a02..763f49b 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ experimental: { serverIslands: true, contentLayer: true, + contentIntellisense: true, }, integrations: [ starlight({ diff --git a/package.json b/package.json index 2ff0d0d..3b81c8e 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,6 @@ "astro": "4.15.1", "semver": "7.6.3", "sharp": "0.33.5", - "@alexanderniebuhr/astro-monthly-blog-resource-loader": "0.0.1", "typescript": "5.5.4" }, "devDependencies": { @@ -29,10 +28,13 @@ "@types/mdast": "4.0.4", "@types/node": "20.16.3", "@types/semver": "7.5.8", + "mdast-util-from-markdown": "^2.0.1", "mdast-util-mdx": "3.0.0", "mdast-util-to-markdown": "2.1.0", + "micromark-extension-mdxjs": "^3.0.0", "prettier": "3.3.3", "prettier-plugin-astro": "0.14.1", + "unist-util-select": "^5.1.0", "wrangler": "3.73.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06708db..5d16f49 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,9 +30,6 @@ importers: specifier: 5.5.4 version: 5.5.4 devDependencies: - '@alexanderniebuhr/astro-monthly-blog-resource-loader': - specifier: file:/Users/alexanderniebuhr/Developer/Projects/craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz - version: file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz(astro@4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4)) '@biomejs/biome': specifier: 1.8.3 version: 1.8.3 @@ -48,18 +45,27 @@ importers: '@types/semver': specifier: 7.5.8 version: 7.5.8 + mdast-util-from-markdown: + specifier: ^2.0.1 + version: 2.0.1 mdast-util-mdx: specifier: 3.0.0 version: 3.0.0 mdast-util-to-markdown: specifier: 2.1.0 version: 2.1.0 + micromark-extension-mdxjs: + specifier: ^3.0.0 + version: 3.0.0 prettier: specifier: 3.3.3 version: 3.3.3 prettier-plugin-astro: specifier: 0.14.1 version: 0.14.1 + unist-util-select: + specifier: ^5.1.0 + version: 5.1.0 wrangler: specifier: 3.73.0 version: 3.73.0(@cloudflare/workers-types@4.20240821.1) @@ -84,12 +90,6 @@ packages: '@actions/io@1.1.3': resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} - '@alexanderniebuhr/astro-monthly-blog-resource-loader@file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz': - resolution: {integrity: sha512-VqN3WoeRKDZWrPRATFx69hCg3l4TDV3XGKPinOWw/fjjRVnXKxDaNRz6vmo4SKHXg0QRqGDuvaeEYBPMLofnAg==, tarball: file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz} - version: 0.0.1 - peerDependencies: - astro: ^4.14.0 - '@ampproject/remapping@2.3.0': resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} @@ -2648,15 +2648,9 @@ packages: unified@11.0.5: resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} - unist-util-filter@5.0.1: - resolution: {integrity: sha512-pHx7D4Zt6+TsfwylH9+lYhBhzyhEnCXs/lbq/Hstxno5z4gVdyc2WEW0asfjGKPyG4pEKrnBv5hdkO6+aRnQJw==} - unist-util-find-after@5.0.0: resolution: {integrity: sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==} - unist-util-inspect@8.1.0: - resolution: {integrity: sha512-mOlg8Mp33pR0eeFpo5d2902ojqFFOKMMG2hF8bmH7ZlhnmjFgh0NI3/ZDwdaBJNbvrS7LZFVrBVtIE9KZ9s7vQ==} - unist-util-is@6.0.0: resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} @@ -2998,20 +2992,6 @@ snapshots: '@actions/io@1.1.3': {} - '@alexanderniebuhr/astro-monthly-blog-resource-loader@file:../../craftlions/md-loader/alexanderniebuhr-astro-monthly-blog-resource-loader-0.0.1.tgz(astro@4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4))': - dependencies: - acorn: 8.12.1 - astro: 4.15.1(@types/node@20.16.3)(rollup@4.21.2)(typescript@5.5.4) - mdast-util-from-markdown: 2.0.1 - mdast-util-mdx: 3.0.0 - micromark-extension-mdxjs: 3.0.0 - unist-util-filter: 5.0.1 - unist-util-inspect: 8.1.0 - unist-util-select: 5.1.0 - unist-util-visit: 5.0.0 - transitivePeerDependencies: - - supports-color - '@ampproject/remapping@2.3.0': dependencies: '@jridgewell/gen-mapping': 0.3.5 @@ -6058,21 +6038,11 @@ snapshots: trough: 2.2.0 vfile: 6.0.3 - unist-util-filter@5.0.1: - dependencies: - '@types/unist': 3.0.3 - unist-util-is: 6.0.0 - unist-util-visit-parents: 6.0.1 - unist-util-find-after@5.0.0: dependencies: '@types/unist': 3.0.3 unist-util-is: 6.0.0 - unist-util-inspect@8.1.0: - dependencies: - '@types/unist': 3.0.3 - unist-util-is@6.0.0: dependencies: '@types/unist': 3.0.3 diff --git a/src/content/config.ts b/src/content/config.ts index 3149d52..9f561ab 100644 --- a/src/content/config.ts +++ b/src/content/config.ts @@ -2,7 +2,7 @@ import { defineCollection, z } from 'astro:content'; import { docsSchema } from '@astrojs/starlight/schema'; import { minVersion, outside, validRange } from 'semver'; import pkg from '../../package.json'; -import { astroMonthlyBlogResourceLoader } from '@alexanderniebuhr/astro-monthly-blog-resource-loader'; +import { astroMonthlyBlogResourceLoader } from '../utils/index.js'; const astroVersion = minVersion(pkg.dependencies.astro)?.version; diff --git a/src/utils/astro-monthly-blog-loader.ts b/src/utils/astro-monthly-blog-loader.ts new file mode 100644 index 0000000..14f142c --- /dev/null +++ b/src/utils/astro-monthly-blog-loader.ts @@ -0,0 +1,119 @@ +import type { Loader } from 'astro/loaders'; + +import { AstroError } from 'astro/errors'; +import { z } from 'astro/zod'; +import { fromMarkdown } from 'mdast-util-from-markdown'; +import { mdxFromMarkdown } from 'mdast-util-mdx'; +import { mdxjs } from 'micromark-extension-mdxjs'; +import { selectAll } from 'unist-util-select'; + +const USER_CONFIG_SCHEMA = z.object({ + urls: z + .string() + .url() + .transform((url) => new URL(url)) + .array(), + exclude: z.instanceof(RegExp).array(), +}); + +type USER_CONFIG_TYPE = z.input; + +const SCHEMA = z.object({ + title: z.string(), + url: z.string().url(), + source_url: z.string().url().optional(), +}); + +async function fetchResources(params: { url: URL }) { + const path_segment = params.url.pathname.split('/').slice(-1)[0].replace('.mdx', ''); + const source_url = `https://astro.build/blog/${path_segment}/`; + const res = await fetch(params.url); + if (!res.ok || !res.body) { + throw new AstroError(`Failed to fetch the blog post from ${params.url.toString()}`); + } + const parsedRes = await res.text(); + + const tree = fromMarkdown(parsedRes, { + extensions: [mdxjs({})], + mdastExtensions: [mdxFromMarkdown()], + }); + + const treeLinks = selectAll('link', tree) as unknown[] as { + url: string; + children: { value: string }[]; + source_url: string; + }[]; + + const extractedLinks = new Array<{ + url: string; + title: string; + source_url: string; + }>(); + + for (const link of treeLinks) { + extractedLinks.push({ + url: link.url, + title: link.children[0].value, + source_url: source_url, + }); + } + + const videoGrid = selectAll('[name=YouTubeGrid]', tree); + if (videoGrid[0]) { + // @ts-expect-error + for (const videoLink of videoGrid[0].attributes[0].value.data.estree.body[0].expression + .elements) { + extractedLinks.push({ + url: videoLink.properties[0].value.value, + title: videoLink.properties[1].value.value, + source_url: source_url, + }); + } + } + + return extractedLinks; +} + +export function astroMonthlyBlogResourceLoader(config: USER_CONFIG_TYPE): Loader { + const PARSED_CONFIG = USER_CONFIG_SCHEMA.safeParse(config); + + if (!PARSED_CONFIG.success) { + throw new AstroError( + `The provided configuration for the Astro Monthly Blog Post loader is invalid.\n${PARSED_CONFIG.error.issues.map((issue) => issue.message).join('\n')}` + ); + } + + return { + name: 'astro-monthly-blog-links-loader', + schema: SCHEMA, + async load({ store, logger, parseData, generateDigest }) { + store.clear(); + for (const url of PARSED_CONFIG.data.urls) { + logger.info(`Getting all links from ${url.toString()}`); + const resources = await fetchResources({ url }); + + for (const extractedLink of resources) { + if (PARSED_CONFIG.data.exclude.some((exclude) => exclude.test(extractedLink.url))) + continue; + + const data = await parseData({ + id: extractedLink.url, + data: { + title: extractedLink.title, + url: extractedLink.url, + source_url: extractedLink.source_url, + }, + }); + + const digest = generateDigest(data); + + store.set({ + id: extractedLink.url, + data, + digest, + }); + } + } + }, + }; +} diff --git a/src/utils/index.ts b/src/utils/index.ts new file mode 100644 index 0000000..d78b531 --- /dev/null +++ b/src/utils/index.ts @@ -0,0 +1 @@ +export { astroMonthlyBlogResourceLoader } from './astro-monthly-blog-loader.ts';