From 83dedfe7108e71a5a4f22b01cde923ae8a221a28 Mon Sep 17 00:00:00 2001 From: Peter van der Heijden Date: Fri, 22 May 2026 23:23:14 +0200 Subject: [PATCH] feat: add @embedpdf/cli with doc search and reader commands --- .changeset/cli-search-improvements.md | 5 + .gitignore | 5 + packages/cli/README.md | 105 ++++++++++ packages/cli/package.json | 49 +++++ packages/cli/scripts/generate-manifest.mjs | 189 ++++++++++++++++++ packages/cli/src/bin.ts | 12 ++ packages/cli/src/commands/doc.ts | 92 +++++++++ packages/cli/src/commands/search.ts | 53 +++++ packages/cli/src/index.ts | 11 + packages/cli/src/lib/fetch.ts | 50 +++++ packages/cli/src/lib/manifest.ts | 30 +++ packages/cli/src/lib/search.ts | 52 +++++ packages/cli/tsconfig.json | 16 ++ packages/cli/vite.config.ts | 26 +++ website/package.json | 3 +- website/scripts/generate-embeddings.mjs | 138 +++++++++++++ website/src/app/api/search/route.ts | 179 +++++++++++++++++ .../content/docs/snippet/customizing-ui.mdx | 5 + website/src/content/docs/snippet/theme.mdx | 5 + 19 files changed, 1024 insertions(+), 1 deletion(-) create mode 100644 .changeset/cli-search-improvements.md create mode 100644 packages/cli/README.md create mode 100644 packages/cli/package.json create mode 100644 packages/cli/scripts/generate-manifest.mjs create mode 100644 packages/cli/src/bin.ts create mode 100644 packages/cli/src/commands/doc.ts create mode 100644 packages/cli/src/commands/search.ts create mode 100644 packages/cli/src/index.ts create mode 100644 packages/cli/src/lib/fetch.ts create mode 100644 packages/cli/src/lib/manifest.ts create mode 100644 packages/cli/src/lib/search.ts create mode 100644 packages/cli/tsconfig.json create mode 100644 packages/cli/vite.config.ts create mode 100644 website/scripts/generate-embeddings.mjs create mode 100644 website/src/app/api/search/route.ts diff --git a/.changeset/cli-search-improvements.md b/.changeset/cli-search-improvements.md new file mode 100644 index 000000000..3f6b1fb43 --- /dev/null +++ b/.changeset/cli-search-improvements.md @@ -0,0 +1,5 @@ +--- +"@embedpdf/cli": patch +--- + +Improve search-docs output: show descriptions in human-readable mode, reduce default limit from 10 to 5, and add minimum score threshold (0.25) to filter irrelevant results. diff --git a/.gitignore b/.gitignore index d9ba93d7d..e65eb1484 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,8 @@ config/tsup.frameworks.config.bundled* publish-summary.json **/.svelte-kit **/*/*.tgz +.env + +# Generated data files +packages/cli/src/data/manifest.json +website/src/data/embeddings.json diff --git a/packages/cli/README.md b/packages/cli/README.md new file mode 100644 index 000000000..4fc934d33 --- /dev/null +++ b/packages/cli/README.md @@ -0,0 +1,105 @@ +
+ + EmbedPDF logo + +

EmbedPDF

+ +NPM version License Join the community on GitHub + +
+ +# @embedpdf/cli + +CLI for searching and reading EmbedPDF documentation directly from your terminal. + +## Documentation + +For complete guides, examples, and full API reference, visit: + +**[Official Documentation](https://www.embedpdf.com/docs)** + +## Installation + +```bash +# npm +npm install -g @embedpdf/cli + +# pnpm +pnpm add -g @embedpdf/cli + +# or run directly +npx @embedpdf/cli +``` + +## Commands + +### `embedpdf search-docs ` + +Full-text search over the entire documentation corpus. + +```bash +# Basic search +embedpdf search-docs "zoom" + +# Filter by framework +embedpdf search-docs "getting started" --framework react + +# Filter by section +embedpdf search-docs "annotations" --section headless + +# JSON output (for LLM consumption) +embedpdf search-docs "rendering" --json --limit 5 +``` + +| Option | Description | +|---|---| +| `--json` | Output as JSON | +| `--framework ` | Filter by framework (`react`, `vue`, `svelte`) | +| `--section
` | Filter by section (`headless`, `viewer`, `snippet`, `engines`, `pdfium`) | +| `--limit ` | Maximum number of results (default: `10`) | + +### `embedpdf doc [path]` + +Fetch and display a documentation page. + +```bash +# Fetch a specific doc +embedpdf doc react/headless/getting-started + +# List all available pages +embedpdf doc --list + +# List docs for a specific framework +embedpdf doc --list --framework vue + +# Output as JSON +embedpdf doc react/headless/plugins/plugin-zoom --json +``` + +| Option | Description | +|---|---| +| `--json` | Output as JSON | +| `--list` | List all available doc pages | +| `--framework ` | Filter by framework (`react`, `vue`, `svelte`) | +| `--section
` | Filter by section (`headless`, `viewer`, `snippet`, `engines`, `pdfium`) | + +## Programmatic Usage + +The package also exports functions for use in Node.js: + +```typescript +import { searchDocs, fetchDocContent, getManifest } from '@embedpdf/cli'; + +// Search docs +const results = searchDocs('zoom', { framework: 'react', limit: 5 }); + +// Fetch a doc page +const content = await fetchDocContent('react/headless/getting-started'); + +// Get the full manifest +const manifest = getManifest(); +``` + +## License + +MIT – see the [LICENSE](https://github.com/embedpdf/embed-pdf-viewer/blob/main/packages/cli/LICENSE) file. diff --git a/packages/cli/package.json b/packages/cli/package.json new file mode 100644 index 000000000..6d6467edf --- /dev/null +++ b/packages/cli/package.json @@ -0,0 +1,49 @@ +{ + "name": "@embedpdf/cli", + "version": "2.14.3", + "private": false, + "description": "CLI for searching and reading EmbedPDF documentation.", + "type": "module", + "bin": { + "embedpdf": "./dist/bin.js" + }, + "main": "./dist/bin.cjs", + "module": "./dist/bin.js", + "exports": { + ".": { + "types": "./dist/bin.d.ts", + "import": "./dist/bin.js", + "require": "./dist/bin.cjs" + } + }, + "scripts": { + "generate-manifest": "node scripts/generate-manifest.mjs", + "build": "pnpm run generate-manifest && pnpm run clean && vite build", + "clean": "rimraf dist" + }, + "files": [ + "dist", + "README.md" + ], + "repository": { + "type": "git", + "url": "https://github.com/embedpdf/embed-pdf-viewer", + "directory": "packages/cli" + }, + "homepage": "https://www.embedpdf.com/docs", + "bugs": { + "url": "https://github.com/embedpdf/embed-pdf-viewer/issues" + }, + "author": "EmbedPDF", + "license": "MIT", + "dependencies": { + "commander": "^13.1.0" + }, + "devDependencies": { + "@embedpdf/build": "workspace:*", + "typescript": "^5.0.0" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/cli/scripts/generate-manifest.mjs b/packages/cli/scripts/generate-manifest.mjs new file mode 100644 index 000000000..533e6d01b --- /dev/null +++ b/packages/cli/scripts/generate-manifest.mjs @@ -0,0 +1,189 @@ +/** + * Scans website/src/content/docs/ for all MDX files, + * extracts frontmatter (title, description, searchable) and full text content, + * and writes src/lib/manifest.json. + * + * Run: node scripts/generate-manifest.mjs + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const CONTENT_DIR = path.resolve(__dirname, '../../../website/src/content/docs'); +const TOOLS_DIR = path.resolve(__dirname, '../../../website/src/content/tools'); +const OUTPUT_PATH = path.resolve(__dirname, '../src/data/manifest.json'); + +function parseFrontmatter(content) { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!match) return { fm: {}, body: content }; + + const fm = {}; + for (const line of match[1].split('\n')) { + const colonIdx = line.indexOf(':'); + if (colonIdx === -1) continue; + const key = line.slice(0, colonIdx).trim(); + let value = line.slice(colonIdx + 1).trim(); + + // Strip surrounding quotes + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + // Parse booleans + if (value === 'true') value = true; + else if (value === 'false') value = false; + + fm[key] = value; + } + + // Everything after the frontmatter block + const body = content.slice(match[0].length); + return { fm, body }; +} + +/** + * Strip MDX/JSX syntax from doc content, returning plain searchable text. + */ +function stripMdx(content) { + let text = content; + + // Remove import statements + text = text.replace(/^import\s+.*?;\s*$/gm, ''); + + // Remove JSX component wrappers but keep text children + // e.g. ... + text = text.replace(/]*>[\s\S]*?<\/ExampleWrapper>/g, ''); + + // Remove Callout tags but keep content + text = text.replace(/<\/?Callout[^>]*>/g, ''); + + // Remove self-closing JSX tags + text = text.replace(/<[A-Z][a-zA-Z]*\s*[^>]*\/>/g, ''); + + // Remove remaining JSX open/close tags (keep text content) + text = text.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, ''); + + // Remove fenced code blocks (they're code examples, not prose) + text = text.replace(/```[\s\S]*?```/g, ''); + + // Remove inline code backticks but keep the text + text = text.replace(/`([^`]+)`/g, '$1'); + + // Remove markdown link syntax, keep text + text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); + + // Remove markdown image syntax + text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, ''); + + // Remove HTML comments + text = text.replace(//g, ''); + + // Remove markdown heading markers but keep text + text = text.replace(/^#{1,6}\s+/gm, ''); + + // Remove markdown bold/italic markers + text = text.replace(/\*{1,3}([^*]+)\*{1,3}/g, '$1'); + text = text.replace(/_{1,3}([^_]+)_{1,3}/g, '$1'); + + // Remove horizontal rules + text = text.replace(/^[-*_]{3,}\s*$/gm, ''); + + // Remove markdown list markers + text = text.replace(/^\s*[-*+]\s+/gm, ''); + text = text.replace(/^\s*\d+\.\s+/gm, ''); + + // Collapse multiple blank lines + text = text.replace(/\n{3,}/g, '\n\n'); + + return text.trim(); +} + +function inferMeta(relativePath, collection = 'docs') { + const parts = relativePath.replace(/\.mdx$/, '').split('/'); + + let framework = null; + let section = null; + + if (collection === 'tools') { + section = 'tools'; + } else if (['react', 'vue', 'svelte'].includes(parts[0])) { + framework = parts[0]; + if (['headless', 'viewer'].includes(parts[1])) { + section = parts[1]; + } + } else if (parts[0] === 'snippet') { + section = 'snippet'; + } else if (parts[0] === 'engines') { + section = 'engines'; + } else if (parts[0] === 'pdfium') { + section = 'pdfium'; + } + + const docPath = relativePath.replace(/\.mdx$/, ''); + const urlPrefix = collection === 'tools' ? 'tools' : 'docs'; + const url = `https://www.embedpdf.com/${urlPrefix}/${docPath}`; + + return { docPath, url, framework, section }; +} + +function scanCollection(dir, collection) { + if (!fs.existsSync(dir)) return []; + const entries = fs.readdirSync(dir, { recursive: true }); + const docs = []; + + for (const entry of entries) { + const entryStr = String(entry); + if (!entryStr.endsWith('.mdx')) continue; + + const fullPath = path.join(dir, entryStr); + const raw = fs.readFileSync(fullPath, 'utf-8'); + const { fm, body } = parseFrontmatter(raw); + + // Skip non-searchable pages + if (fm.searchable === false) continue; + + // Skip index pages (they're usually just navigation wrappers) + const basename = path.basename(entryStr, '.mdx'); + if (basename === 'index') continue; + + const meta = inferMeta(entryStr, collection); + const content = stripMdx(body); + + docs.push({ + path: meta.docPath, + title: fm.title || basename, + description: fm.description || '', + url: meta.url, + framework: meta.framework, + section: meta.section, + content, + }); + } + + return docs; +} + +function scanDocs() { + const docs = [ + ...scanCollection(CONTENT_DIR, 'docs'), + ...scanCollection(TOOLS_DIR, 'tools'), + ]; + + // Sort by path for deterministic output + docs.sort((a, b) => a.path.localeCompare(b.path)); + return docs; +} + +const docs = scanDocs(); + +fs.writeFileSync(OUTPUT_PATH, JSON.stringify({ docs }, null, 2) + '\n'); + +const totalContentChars = docs.reduce((sum, d) => sum + d.content.length, 0); +console.log( + `[generate-manifest] Wrote ${docs.length} docs (${(totalContentChars / 1024).toFixed(0)}KB of content) to ${path.relative(process.cwd(), OUTPUT_PATH)}`, +); diff --git a/packages/cli/src/bin.ts b/packages/cli/src/bin.ts new file mode 100644 index 000000000..9e2d0700a --- /dev/null +++ b/packages/cli/src/bin.ts @@ -0,0 +1,12 @@ +import { Command } from 'commander'; +import { registerDocCommand } from './commands/doc'; +import { registerSearchCommand } from './commands/search'; + +const program = new Command(); + +program.name('embedpdf').description('The official CLI for the EmbedPDF project').version('2.14.3'); + +registerDocCommand(program); +registerSearchCommand(program); + +program.parse(); diff --git a/packages/cli/src/commands/doc.ts b/packages/cli/src/commands/doc.ts new file mode 100644 index 000000000..15d6adab4 --- /dev/null +++ b/packages/cli/src/commands/doc.ts @@ -0,0 +1,92 @@ +import { Command } from 'commander'; +import { findDoc, listDocs } from '../lib/manifest'; +import { fetchDocContent } from '../lib/fetch'; + +export function registerDocCommand(program: Command) { + program + .command('doc [path]') + .description('Fetch and display a documentation page') + .option('--json', 'Output as JSON') + .option('--list', 'List all available doc pages') + .option('--framework ', 'Filter by framework (react, vue, svelte)') + .option('--section
', 'Filter by section (headless, viewer, snippet, engines, pdfium)') + .action( + async ( + docPath: string | undefined, + opts: { + json?: boolean; + list?: boolean; + framework?: string; + section?: string; + }, + ) => { + if (opts.list) { + const docs = listDocs({ + framework: opts.framework, + section: opts.section, + }); + + if (opts.json) { + console.log(JSON.stringify(docs, null, 2)); + return; + } + + if (docs.length === 0) { + console.log('No docs found matching the given filters.'); + return; + } + + console.log(`\nFound ${docs.length} doc(s):\n`); + for (const doc of docs) { + const meta = [doc.framework, doc.section].filter(Boolean).join('/'); + console.log(` ${doc.path}`); + console.log(` ${doc.title}${meta ? ` (${meta})` : ''}`); + console.log(); + } + return; + } + + if (!docPath) { + console.error('Error: Please provide a doc path, or use --list to see available pages.'); + console.error('Example: embedpdf doc react/headless/getting-started'); + process.exit(1); + } + + const entry = findDoc(docPath); + if (!entry) { + console.error(`Error: Doc not found: "${docPath}"`); + console.error('Use "embedpdf doc --list" to see available pages.'); + process.exit(1); + } + + console.error(`Fetching: ${entry.url}\n`); + + const content = await fetchDocContent(docPath); + if (!content) { + console.error(`Error: Could not fetch doc content from GitHub.`); + process.exit(1); + } + + if (opts.json) { + console.log( + JSON.stringify( + { + path: entry.path, + title: entry.title, + description: entry.description, + url: entry.url, + framework: entry.framework, + section: entry.section, + content, + }, + null, + 2, + ), + ); + return; + } + + console.log(content); + }, + ); +} diff --git a/packages/cli/src/commands/search.ts b/packages/cli/src/commands/search.ts new file mode 100644 index 000000000..a9970d94d --- /dev/null +++ b/packages/cli/src/commands/search.ts @@ -0,0 +1,53 @@ +import { Command } from 'commander'; +import { searchDocs } from '../lib/search'; + +export function registerSearchCommand(program: Command) { + program + .command('search-docs ') + .description('Search EmbedPDF documentation') + .option('--json', 'Output as JSON (for LLM consumption)') + .option('--framework ', 'Filter by framework (react, vue, svelte)') + .option('--section
', 'Filter by section (headless, viewer, snippet, engines, pdfium)') + .option('--limit ', 'Maximum number of results', '5') + .action( + async ( + query: string, + opts: { + json?: boolean; + framework?: string; + section?: string; + limit: string; + }, + ) => { + const limit = parseInt(opts.limit, 10); + + const results = await searchDocs(query, { + framework: opts.framework, + section: opts.section, + limit, + }); + + if (opts.json) { + console.log(JSON.stringify(results, null, 2)); + return; + } + + if (results.total === 0) { + console.log(`No results found for "${query}".`); + return; + } + + console.log(`\nFound ${results.total} result(s) for "${query}":\n`); + + for (let i = 0; i < results.results.length; i++) { + const r = results.results[i]; + const num = String(i + 1).padStart(2, ' '); + const meta = [r.framework, r.section].filter(Boolean).join('/'); + console.log(` ${num}. ${r.title}${meta ? ` (${meta})` : ''}`); + console.log(` ${r.description}`); + console.log(` ${r.url}`); + console.log(); + } + }, + ); +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts new file mode 100644 index 000000000..54db3c4ea --- /dev/null +++ b/packages/cli/src/index.ts @@ -0,0 +1,11 @@ +export interface DocEntry { + path: string; + title: string; + description: string; + url: string; + framework: string | null; + section: string | null; + content: string; +} + +export type { SearchResult, SearchResponse } from './lib/search'; diff --git a/packages/cli/src/lib/fetch.ts b/packages/cli/src/lib/fetch.ts new file mode 100644 index 000000000..e3468e800 --- /dev/null +++ b/packages/cli/src/lib/fetch.ts @@ -0,0 +1,50 @@ +const REPO = 'embedpdf/embed-pdf-viewer'; +const BRANCH = 'main'; +const DOCS_ROOT = 'website/src/content/docs'; + +/** + * Fetch raw MDX content for a doc page from GitHub. + */ +export async function fetchDocContent(docPath: string): Promise { + const normalized = docPath.replace(/^\/|\/$/g, '').replace(/\.mdx$/, ''); + const url = `https://raw.githubusercontent.com/${REPO}/${BRANCH}/${DOCS_ROOT}/${normalized}.mdx`; + + const res = await fetch(url); + if (!res.ok) return null; + + const raw = await res.text(); + return stripMdx(raw); +} + +/** + * Strip frontmatter and MDX-specific imports/components from content, + * returning clean Markdown suitable for terminal display. + */ +function stripMdx(content: string): string { + // Remove YAML frontmatter + let text = content.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, ''); + + // Remove import statements + text = text.replace(/^import\s+.*?;\s*$/gm, ''); + + // Remove JSX component wrappers (keep their children) + // e.g. ... + text = text.replace(/]*>[\s\S]*?<\/ExampleWrapper>/g, '[Interactive Example]'); + + // Convert to blockquote-style + text = text.replace(/]*>([\s\S]*?)<\/Callout>/g, (_match, inner) => { + const lines = inner.trim().split('\n'); + return lines.map((l: string) => `> ${l}`).join('\n'); + }); + + // Remove remaining self-closing JSX tags + text = text.replace(/<[A-Z][a-zA-Z]*\s*[^>]*\/>/g, ''); + + // Remove remaining JSX component open/close tags (but keep text content) + text = text.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, ''); + + // Collapse multiple blank lines + text = text.replace(/\n{3,}/g, '\n\n'); + + return text.trim(); +} diff --git a/packages/cli/src/lib/manifest.ts b/packages/cli/src/lib/manifest.ts new file mode 100644 index 000000000..0d8ef605d --- /dev/null +++ b/packages/cli/src/lib/manifest.ts @@ -0,0 +1,30 @@ +import type { DocEntry } from '../index'; +import manifestData from '../data/manifest.json'; + +export interface Manifest { + docs: DocEntry[]; +} + +export function getManifest(): Manifest { + return manifestData as Manifest; +} + +export function findDoc(docPath: string): DocEntry | undefined { + const manifest = getManifest(); + const normalized = docPath.replace(/^\/|\/$/g, '').replace(/\.mdx$/, ''); + return manifest.docs.find((d) => d.path === normalized); +} + +export function listDocs(options?: { framework?: string; section?: string }): DocEntry[] { + const manifest = getManifest(); + let docs = manifest.docs; + + if (options?.framework) { + docs = docs.filter((d) => d.framework === options.framework); + } + if (options?.section) { + docs = docs.filter((d) => d.section === options.section); + } + + return docs; +} diff --git a/packages/cli/src/lib/search.ts b/packages/cli/src/lib/search.ts new file mode 100644 index 000000000..ae87cdbae --- /dev/null +++ b/packages/cli/src/lib/search.ts @@ -0,0 +1,52 @@ +/** + * Searches EmbedPDF documentation via the search API. + * The API uses pre-computed vector embeddings for semantic search. + */ + +const SEARCH_API_URL = + process.env.EMBEDPDF_API_URL || 'https://www.embedpdf.com/api/search'; + +export interface SearchResult { + path: string; + title: string; + description: string; + url: string; + framework: string | null; + section: string | null; + score: number; +} + +export interface SearchResponse { + query: string; + total: number; + results: SearchResult[]; +} + +export async function searchDocs( + query: string, + options?: { + framework?: string; + section?: string; + limit?: number; + }, +): Promise { + const response = await fetch(SEARCH_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + query, + framework: options?.framework, + section: options?.section, + limit: options?.limit ?? 10, + }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`Search API error (${response.status}): ${error}`); + } + + return response.json(); +} diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json new file mode 100644 index 000000000..793cbdb1f --- /dev/null +++ b/packages/cli/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "declaration": true, + "esModuleInterop": true, + "skipLibCheck": true, + "rootDir": "src", + "outDir": "dist", + "resolveJsonModule": true + }, + "include": ["src"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/cli/vite.config.ts b/packages/cli/vite.config.ts new file mode 100644 index 000000000..18d65fc51 --- /dev/null +++ b/packages/cli/vite.config.ts @@ -0,0 +1,26 @@ +import { defineConfig } from 'vite'; +import path from 'node:path'; + +export default defineConfig({ + build: { + outDir: 'dist', + emptyOutDir: true, + sourcemap: true, + target: 'node20', + minify: false, + lib: { + entry: { + bin: path.resolve(__dirname, 'src/bin.ts'), + }, + formats: ['es', 'cjs'], + fileName: (format, entryName) => `${entryName}.${format === 'es' ? 'js' : 'cjs'}`, + }, + rollupOptions: { + external: [/^node:/, 'commander'], + output: { + dir: path.resolve(__dirname, 'dist'), + banner: (chunk) => (chunk.name === 'bin' ? '#!/usr/bin/env node' : ''), + }, + }, + }, +}); diff --git a/website/package.json b/website/package.json index 91a7d16bd..1ea41a7d7 100644 --- a/website/package.json +++ b/website/package.json @@ -9,7 +9,8 @@ "postbuild": "pagefind --site .next/server/app --output-path public/_pagefind", "start": "next start", "clean": "rm -rf .next node_modules/.cache", - "copy-wasm": "ts-node tools/copy-wasm.ts" + "copy-wasm": "ts-node tools/copy-wasm.ts", + "generate-embeddings": "node scripts/generate-embeddings.mjs" }, "dependencies": { "@dnd-kit/core": "^6.3.1", diff --git a/website/scripts/generate-embeddings.mjs b/website/scripts/generate-embeddings.mjs new file mode 100644 index 000000000..802bcf79e --- /dev/null +++ b/website/scripts/generate-embeddings.mjs @@ -0,0 +1,138 @@ +/** + * Generates vector embeddings for all docs using OpenRouter's text-embedding-3-small. + * Reads the CLI manifest, batches docs, calls OpenRouter, and writes embeddings.json. + * + * Run: OPENROUTER_API_KEY=... node scripts/generate-embeddings.mjs + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const MANIFEST_PATH = path.resolve( + __dirname, + '../../packages/cli/src/data/manifest.json', +); +const OUTPUT_PATH = path.resolve(__dirname, '../src/data/embeddings.json'); + +const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY; +if (!OPENROUTER_API_KEY) { + console.error( + 'Error: OPENROUTER_API_KEY environment variable is required.\n' + + 'Usage: OPENROUTER_API_KEY=sk-or-... node scripts/generate-embeddings.mjs', + ); + process.exit(1); +} + +const MODEL = 'openai/text-embedding-3-small'; +const BATCH_SIZE = 50; +const MAX_CHARS = 25000; // ~6000 tokens, well within 8191 token limit + +/** + * Call OpenRouter embeddings API for a batch of texts. + */ +async function generateEmbeddings(texts) { + const response = await fetch('https://openrouter.ai/api/v1/embeddings', { + method: 'POST', + headers: { + Authorization: `Bearer ${OPENROUTER_API_KEY}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://www.embedpdf.com', + 'X-Title': 'EmbedPDF Docs', + }, + body: JSON.stringify({ + input: texts, + model: MODEL, + }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`OpenRouter API error (${response.status}): ${error}`); + } + + const data = await response.json(); + // Sort by index to ensure order matches input + return data.data + .sort((a, b) => a.index - b.index) + .map((d) => d.embedding); +} + +/** + * Prepare the text to embed for a single doc. + */ +function prepareText(doc) { + let text = `${doc.title}\n${doc.description}\n\n${doc.content}`; + if (text.length > MAX_CHARS) { + text = text.slice(0, MAX_CHARS); + } + return text; +} + +async function main() { + if (!fs.existsSync(MANIFEST_PATH)) { + console.error( + `Error: Manifest not found at ${MANIFEST_PATH}.\n` + + 'Run "pnpm run generate-manifest" in packages/cli first.', + ); + process.exit(1); + } + + const manifest = JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf-8')); + const docs = manifest.docs; + + console.log(`[generate-embeddings] Processing ${docs.length} docs...`); + + const allEmbeddings = []; + + for (let i = 0; i < docs.length; i += BATCH_SIZE) { + const batch = docs.slice(i, i + BATCH_SIZE); + const texts = batch.map(prepareText); + + const batchNum = Math.floor(i / BATCH_SIZE) + 1; + const totalBatches = Math.ceil(docs.length / BATCH_SIZE); + console.log( + `[generate-embeddings] Embedding batch ${batchNum}/${totalBatches} (${batch.length} docs)...`, + ); + + const embeddings = await generateEmbeddings(texts); + allEmbeddings.push(...embeddings); + + // Small delay between batches + if (i + BATCH_SIZE < docs.length) { + await new Promise((r) => setTimeout(r, 500)); + } + } + + const output = { + model: MODEL, + generatedAt: new Date().toISOString(), + docs: docs.map((doc, i) => ({ + path: doc.path, + title: doc.title, + description: doc.description, + url: doc.url, + framework: doc.framework, + section: doc.section, + embedding: allEmbeddings[i], + })), + }; + + const outputDir = path.dirname(OUTPUT_PATH); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + fs.writeFileSync(OUTPUT_PATH, JSON.stringify(output)); + + const fileSizeKB = (fs.statSync(OUTPUT_PATH).size / 1024).toFixed(0); + console.log( + `[generate-embeddings] Wrote ${docs.length} doc embeddings (${fileSizeKB}KB) to ${path.relative(process.cwd(), OUTPUT_PATH)}`, + ); +} + +main().catch((err) => { + console.error('[generate-embeddings] Fatal error:', err); + process.exit(1); +}); diff --git a/website/src/app/api/search/route.ts b/website/src/app/api/search/route.ts new file mode 100644 index 000000000..49d48a174 --- /dev/null +++ b/website/src/app/api/search/route.ts @@ -0,0 +1,179 @@ +import { NextRequest, NextResponse } from 'next/server' +import { readFileSync } from 'fs' +import { join } from 'path' + +export const dynamic = 'force-dynamic' + +const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY +const MODEL = 'openai/text-embedding-3-small' + +interface EmbeddingDoc { + path: string + title: string + description: string + url: string + framework: string | null + section: string | null + embedding: number[] +} + +interface EmbeddingsData { + model: string + generatedAt: string + docs: EmbeddingDoc[] +} + +let cachedData: EmbeddingsData | null = null + +function getEmbeddingsData(): EmbeddingsData { + if (!cachedData) { + try { + const filePath = join(process.cwd(), 'src/data/embeddings.json') + cachedData = JSON.parse(readFileSync(filePath, 'utf-8')) + } catch { + cachedData = { model: MODEL, generatedAt: '', docs: [] } + } + } + return cachedData +} + +function cosineSimilarity(a: number[], b: number[]): number { + let dotProduct = 0 + let normA = 0 + let normB = 0 + for (let i = 0; i < a.length; i++) { + dotProduct += a[i] * b[i] + normA += a[i] * a[i] + normB += b[i] * b[i] + } + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)) +} + +async function embedQuery(query: string): Promise { + const response = await fetch('https://openrouter.ai/api/v1/embeddings', { + method: 'POST', + headers: { + Authorization: `Bearer ${OPENROUTER_API_KEY}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': 'https://www.embedpdf.com', + 'X-Title': 'EmbedPDF Docs', + }, + body: JSON.stringify({ + input: query, + model: MODEL, + }), + }) + + if (!response.ok) { + const error = await response.text() + throw new Error(`OpenRouter API error (${response.status}): ${error}`) + } + + const data = await response.json() + return data.data[0].embedding +} + +async function search( + query: string, + options: { framework?: string; section?: string; limit: number }, +) { + if (!OPENROUTER_API_KEY) { + throw new Error('OPENROUTER_API_KEY not configured') + } + + const data = getEmbeddingsData() + + if (data.docs.length === 0) { + throw new Error('Embeddings not yet generated') + } + + const queryEmbedding = await embedQuery(query) + + let docs = data.docs + + if (options.framework) { + docs = docs.filter((d) => d.framework === options.framework) + } + if (options.section) { + docs = docs.filter((d) => d.section === options.section) + } + + const scored = docs.map((doc) => ({ + path: doc.path, + title: doc.title, + description: doc.description, + url: doc.url, + framework: doc.framework, + section: doc.section, + score: + Math.round(cosineSimilarity(queryEmbedding, doc.embedding) * 1000) / + 1000, + })) + + scored.sort((a, b) => b.score - a.score) + + const filtered = scored.filter(r => r.score >= 0.25) + + return { + query, + total: filtered.slice(0, options.limit).length, + results: filtered.slice(0, options.limit), + } +} + +export async function GET(request: NextRequest) { + try { + const { searchParams } = request.nextUrl + const query = searchParams.get('q') + const framework = searchParams.get('framework') ?? undefined + const section = searchParams.get('section') ?? undefined + const limit = parseInt(searchParams.get('limit') ?? '10', 10) + + if (!query) { + return NextResponse.json( + { error: 'Missing required query parameter: q' }, + { status: 400 }, + ) + } + + const results = await search(query, { framework, section, limit }) + return NextResponse.json(results) + } catch (error) { + console.error('Search API error:', error) + const message = + error instanceof Error ? error.message : 'Internal server error' + const status = message.includes('not configured') + ? 503 + : message.includes('not yet generated') + ? 503 + : 500 + return NextResponse.json({ error: message }, { status }) + } +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json() + const { query, framework, section, limit = 10 } = body + + if (!query || typeof query !== 'string') { + return NextResponse.json( + { error: 'Missing or invalid query parameter' }, + { status: 400 }, + ) + } + + const results = await search(query, { framework, section, limit }) + return NextResponse.json(results) + } catch (error) { + console.error('Search API error:', error) + const message = + error instanceof Error ? error.message : 'Internal server error' + const status = message.includes('not configured') + ? 503 + : message.includes('not yet generated') + ? 503 + : 500 + return NextResponse.json({ error: message }, { status }) + } +} diff --git a/website/src/content/docs/snippet/customizing-ui.mdx b/website/src/content/docs/snippet/customizing-ui.mdx index fc4d709de..4afcd1810 100644 --- a/website/src/content/docs/snippet/customizing-ui.mdx +++ b/website/src/content/docs/snippet/customizing-ui.mdx @@ -1,3 +1,8 @@ +--- +title: "Customizing the UI" +description: "Customize toolbar buttons, menus, layout, and appearance of the EmbedPDF Snippet PDF viewer. Add custom commands, icons, and UI schema modifications at runtime." +--- + # Customizing the UI The EmbedPDF snippet provides a powerful, declarative way to customize the user interface. You can add custom buttons, modify toolbars, create menus, and register custom icons—all at runtime after the viewer initializes. diff --git a/website/src/content/docs/snippet/theme.mdx b/website/src/content/docs/snippet/theme.mdx index 6bb1e6970..1600c563c 100644 --- a/website/src/content/docs/snippet/theme.mdx +++ b/website/src/content/docs/snippet/theme.mdx @@ -1,3 +1,8 @@ +--- +title: "Customizing the Theme" +description: "Customize the EmbedPDF Snippet viewer's appearance with light, dark, and system color modes. Override colors for backgrounds, text, accents, borders, and semantic states." +--- + # Customizing the Theme The EmbedPDF viewer snippet includes a robust theming system that supports Light, Dark, and System modes. You can customize every color aspect of the viewer—from backgrounds and borders to brand accents—by passing a `theme` object during initialization.