diff --git a/packages/config/src/api/client.js b/packages/config/src/api/client.js index d5d1fe4926..e19e4ff945 100644 --- a/packages/config/src/api/client.js +++ b/packages/config/src/api/client.js @@ -3,13 +3,18 @@ import { NetlifyAPI } from 'netlify' import { removeUndefined } from '../utils/remove_falsy.js' // Retrieve Netlify API client, if an access token was passed -export const getApiClient = function ({ token, offline, testOpts = {}, host, scheme, pathPrefix }) { +export const getApiClient = function ({ cache, token, offline, testOpts = {}, host, scheme, pathPrefix }) { if (!token || offline) { return } // TODO: find less intrusive way to mock HTTP requests - const parameters = removeUndefined({ scheme: testOpts.scheme || scheme, host: testOpts.host || host, pathPrefix }) + const parameters = removeUndefined({ + cache, + scheme: testOpts.scheme || scheme, + host: testOpts.host || host, + pathPrefix, + }) const api = new NetlifyAPI(token, parameters) return api } diff --git a/packages/config/src/api/site_info.ts b/packages/config/src/api/site_info.ts index f0229d1c7b..a3d0fefec3 100644 --- a/packages/config/src/api/site_info.ts +++ b/packages/config/src/api/site_info.ts @@ -60,6 +60,7 @@ export const getSiteInfo = async function ({ const integrations = mode === 'buildbot' && !offline ? await getIntegrations({ + api, siteId, testOpts, offline, @@ -74,10 +75,9 @@ export const getSiteInfo = async function ({ return { siteInfo, accounts: [], addons: [], integrations } } - const [siteInfo, accounts, addons, integrations] = await Promise.all([ + const [siteInfo, accounts, integrations] = await Promise.all([ getSite(api, siteId, siteFeatureFlagPrefix), getAccounts(api), - getAddons(api, siteId), getIntegrations({ siteId, testOpts, offline, accountId, token, featureFlags, extensionApiBaseUrl, mode }), ]) @@ -87,7 +87,7 @@ export const getSiteInfo = async function ({ siteInfo.build_settings.env = envelope } - return { siteInfo, accounts, addons, integrations } + return { siteInfo, accounts, addons: [], integrations } } const getSite = async function (api: NetlifyAPI, siteId: string, siteFeatureFlagPrefix: string) { @@ -128,20 +128,8 @@ const getAccounts = async function (api: NetlifyAPI): Promise } } -const getAddons = async function (api: NetlifyAPI, siteId: string) { - if (siteId === undefined) { - return [] - } - - try { - const addons = await (api as any).listServiceInstancesForSite({ siteId }) - return Array.isArray(addons) ? addons : [] - } catch (error) { - throwUserError(`Failed retrieving addons for site ${siteId}: ${error.message}. ${ERROR_CALL_TO_ACTION}`) - } -} - type GetIntegrationsOpts = { + api?: NetlifyAPI siteId?: string accountId?: string testOpts: TestOptions @@ -153,6 +141,7 @@ type GetIntegrationsOpts = { } const getIntegrations = async function ({ + api, siteId, accountId, testOpts, @@ -196,6 +185,16 @@ const getIntegrations = async function ({ : `${baseUrl}site/${siteId}/integrations/safe` try { + // Even though integrations don't come through the Netlify API, we can + // still leverage the API cache if one is being used. + if (api?.cache) { + const response = await api.cache.get(url, 'get', {}) + + if (response !== null && Array.isArray(response.body)) { + return response.body + } + } + const requestOptions = {} as RequestInit // This is used to identify where the request is coming from diff --git a/packages/config/src/main.ts b/packages/config/src/main.ts index 918e047e0a..19b8ad901a 100644 --- a/packages/config/src/main.ts +++ b/packages/config/src/main.ts @@ -50,6 +50,7 @@ export type Config = { */ export const resolveConfig = async function (opts): Promise { const { + apiCache, cachedConfig, cachedConfigPath, host, @@ -63,7 +64,7 @@ export const resolveConfig = async function (opts): Promise { ...optsA } = addDefaultOpts(opts) as $TSFixMe // `api` is not JSON-serializable, so we cannot cache it inside `cachedConfig` - const api = getApiClient({ token, offline, host, scheme, pathPrefix, testOpts }) + const api = getApiClient({ token, offline, host, scheme, pathPrefix, testOpts, cache: apiCache }) const parsedCachedConfig = await getCachedConfig({ cachedConfig, cachedConfigPath, token, api }) // If there is a cached config, use it. The exception is when a default config, diff --git a/packages/js-client/src/buffered_response.ts b/packages/js-client/src/buffered_response.ts new file mode 100644 index 0000000000..78b20cfd9f --- /dev/null +++ b/packages/js-client/src/buffered_response.ts @@ -0,0 +1,33 @@ +import { getHeadersObject, HeadersObject } from './headers.js' +import { getResponseType } from './methods/response_type.js' + +type JSONResponse = { type: 'json'; body: any } +type TextResponse = { type: 'text'; body: string } + +/** + * An HTTP response that has been fully read. The body has been buffered and so + * it can be read multiple times and serialized to disk. + */ +export type BufferedResponse = { headers: HeadersObject; timestamp: number } & (JSONResponse | TextResponse) + +/** + * Consumes an HTTP response and returns a `BufferedResponse` object. + */ +export const getBufferedResponse = async (res: Response): Promise => { + const headers = getHeadersObject(res.headers) + const data = { + headers, + timestamp: Date.now(), + } + const type = getResponseType(res) + + if (type === 'json') { + return { + ...data, + type: 'json', + body: await res.json(), + } + } + + return { ...data, type: 'text', body: await res.text() } +} diff --git a/packages/js-client/src/cache.test.ts b/packages/js-client/src/cache.test.ts new file mode 100644 index 0000000000..21ad08ffdd --- /dev/null +++ b/packages/js-client/src/cache.test.ts @@ -0,0 +1,198 @@ +import { promises as fs } from 'fs' +import path from 'path' + +import test, { ExecutionContext } from 'ava' + +import { APICache } from '../lib/index.js' + +const dateNow = Date.now +const globalFetch = globalThis.fetch + +test.afterEach(() => { + Date.now = dateNow + globalThis.fetch = globalFetch +}) + +const getMockFetch = (t: ExecutionContext, mocks: Record Response>) => { + const calls: Record = {} + + const mockFetch = async (input: URL | RequestInfo) => { + for (const url in mocks) { + if (input.toString() === url) { + calls[url] = calls[url] ?? 0 + calls[url]++ + + return mocks[url]() + } + } + + t.fail(`Unexpected fetch call: ${input}`) + + return new Response(null, { status: 400 }) + } + + return { calls, mockFetch } +} + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms) + }) + +test.serial('Returns response from cache if within TTL', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 30, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 10 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 1) +}) + +test.serial('Returns response from cache if outside of TTL but within SWR', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 60, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 45 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 1) + + await sleep(10) + + t.is(calls[mockEndpoint], 2) + + const cacheKey = cache.getCacheKey(mockEndpoint, 'get', {}) + t.is(cache.entries[cacheKey].timestamp, future) +}) + +test.serial('Returns fresh response if outside of TTL and SWR', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 60, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 90 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 2) +}) + +test.serial('Uses disk fallback', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const fsPath = await fs.mkdtemp('netlify-js-client-test') + const cache = new APICache({ + fsPath, + ttl: 30, + swr: 60, + }) + + t.teardown(async () => { + await fs.rm(fsPath, { recursive: true }) + }) + + const now = Date.now() + const cacheKey = cache.getCacheKey(mockEndpoint, 'get', {}) + const filePath = path.join(fsPath, cacheKey) + const file = { + body: mockResponse, + headers: { + 'content-type': 'application/json', + }, + timestamp: now - 20, + type: 'json', + } + + await fs.writeFile(filePath, JSON.stringify(file)) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + t.falsy(calls[mockEndpoint]) + + const future = now + 20 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.falsy(calls[mockEndpoint]) + + await sleep(10) + + t.is(calls[mockEndpoint], 1) + + const newFile = await fs.readFile(filePath, 'utf8') + const data = JSON.parse(newFile) + + t.deepEqual(data.body, mockResponse) + t.deepEqual(data.headers, { 'content-type': 'application/json' }) + t.is(data.timestamp, future) +}) diff --git a/packages/js-client/src/cache.ts b/packages/js-client/src/cache.ts new file mode 100644 index 0000000000..d51207fe50 --- /dev/null +++ b/packages/js-client/src/cache.ts @@ -0,0 +1,155 @@ +import { createHash } from 'crypto' +import { promises as fs } from 'fs' +import { join } from 'path' + +import { BufferedResponse, getBufferedResponse } from './buffered_response.js' +import { HeadersObject } from './headers.js' + +const DEFAULT_TTL = 30_000 +const DEFAULT_SWR = 120_000 + +interface APIOptions { + fsPath?: string + ttl?: number + swr?: number +} + +export class APICache { + entries: Record + fetches: Record> + fsPath?: string + fsPathSetup?: Promise + ttl: number + swr: number + + constructor({ fsPath, ttl = DEFAULT_TTL, swr = DEFAULT_SWR }: APIOptions) { + this.entries = {} + this.fetches = {} + this.fsPath = fsPath + this.fsPathSetup = fsPath ? fs.mkdir(fsPath, { recursive: true }) : Promise.resolve() + this.ttl = ttl + this.swr = swr + } + + private async addToCache(key: string, res: BufferedResponse) { + this.entries[key] = res + + await this.saveToDisk(key, res) + } + + private async getCached(key: string) { + if (this.entries[key]) { + return this.entries[key] + } + + const fromDisk = await this.loadFromDisk(key) + + if (fromDisk) { + this.entries[key] = fromDisk + + return this.entries[key] + } + } + + private fetchAndAddToCache(url: string, method: string, headers: HeadersObject) { + const key = this.getCacheKey(url, method, headers) + + if (!this.fetches[key]) { + this.fetches[key] = fetch(url, { + headers, + method, + }) + .then(async (res) => { + delete this.fetches[key] + + const bufferedRes = await getBufferedResponse(res) + + if (res.status === 200) { + this.addToCache(key, bufferedRes) + } + + return bufferedRes + }) + .catch(() => { + delete this.fetches[key] + + return null + }) + } + + return this.fetches[key] + } + + private async loadFromDisk(key: string) { + if (!this.fsPath) { + return + } + + const filePath = join(this.fsPath, key) + + try { + const file = await fs.readFile(filePath, 'utf8') + const data = JSON.parse(file) as BufferedResponse + + if (data.type !== 'json' && data.type !== 'text') { + throw new Error('Unsupported response type') + } + + return data + } catch { + // no-op + } + } + + private async saveToDisk(key: string, res: BufferedResponse) { + if (!this.fsPath) { + return + } + + const data = { + ...res, + timestamp: Date.now(), + } + + try { + await this.fsPathSetup + await fs.writeFile(join(this.fsPath, key), JSON.stringify(data)) + } catch { + // no-op + } + } + + async get(url: string, method: string, headers: HeadersObject) { + const key = this.getCacheKey(url, method, headers) + const cached = await this.getCached(key) + + if (cached) { + const currentTimestamp = Date.now() + + if (this.ttl > 0 && currentTimestamp - cached.timestamp <= this.ttl) { + return cached + } + + if (this.swr > 0 && currentTimestamp - cached.timestamp <= this.swr) { + setTimeout(() => { + this.fetchAndAddToCache(url, method, headers) + }, 0) + + return cached + } + } + + return this.fetchAndAddToCache(url, method, headers) + } + + getCacheKey(url: string, method: string, headers: HeadersObject) { + const headersInKey = { + authorization: headers.Authorization, + } + const hash = createHash('md5') + .update(JSON.stringify({ url, method, headers: headersInKey })) + .digest('hex') + + return hash + } +} diff --git a/packages/js-client/src/headers.ts b/packages/js-client/src/headers.ts new file mode 100644 index 0000000000..1b9c6547f9 --- /dev/null +++ b/packages/js-client/src/headers.ts @@ -0,0 +1,14 @@ +export type HeadersObject = Record + +/** + * Serializes a `Headers` object into a plain object. + */ +export const getHeadersObject = (headers: Headers) => { + const obj: HeadersObject = {} + + for (const [key, value] of headers.entries()) { + obj[key] = value + } + + return obj +} diff --git a/packages/js-client/src/index.ts b/packages/js-client/src/index.ts index 189267fa49..2e40575e57 100644 --- a/packages/js-client/src/index.ts +++ b/packages/js-client/src/index.ts @@ -1,5 +1,6 @@ import pWaitFor from 'p-wait-for' +import { APICache } from './cache.js' import { getMethods } from './methods/index.js' import { openApiSpec } from './open_api.js' import { getOperations } from './operations.js' @@ -27,6 +28,7 @@ type APIOptions = { * Global params are only sent of the OpenAPI spec specifies the provided params. */ globalParams?: Record + cache?: APICache } /** @@ -48,6 +50,7 @@ export interface NetlifyAPI extends DynamicMethods {} export class NetlifyAPI { #accessToken: string | undefined | null = null + cache?: APICache defaultHeaders: Record = { accept: 'application/json', } @@ -73,7 +76,10 @@ export class NetlifyAPI { this.accessToken = options.accessToken || accessTokenInput || null this.defaultHeaders['User-agent'] = options.userAgent || 'netlify/js-client' + this.cache = options.cache + const methods = getMethods({ + cache: this.cache, basePath: this.basePath, defaultHeaders: this.defaultHeaders, agent: this.agent, @@ -128,3 +134,5 @@ export class NetlifyAPI { } export const methods = getOperations() + +export { APICache } diff --git a/packages/js-client/src/methods/index.js b/packages/js-client/src/methods/index.js index d78df9a09a..67d03a12a1 100644 --- a/packages/js-client/src/methods/index.js +++ b/packages/js-client/src/methods/index.js @@ -10,24 +10,26 @@ import { getUrl } from './url.js' // For each OpenAPI operation, add a corresponding method. // The `operationId` is the method name. -export const getMethods = function ({ basePath, defaultHeaders, agent, globalParams }) { +export const getMethods = function ({ basePath, cache, defaultHeaders, agent, globalParams }) { const operations = getOperations() - const methods = operations.map((method) => getMethod({ method, basePath, defaultHeaders, agent, globalParams })) + const methods = operations.map((method) => + getMethod({ method, basePath, defaultHeaders, agent, globalParams, cache }), + ) return Object.assign({}, ...methods) } -const getMethod = function ({ method, basePath, defaultHeaders, agent, globalParams }) { +const getMethod = function ({ method, basePath, defaultHeaders, agent, globalParams, cache }) { return { [method.operationId](params, opts) { - return callMethod({ method, basePath, defaultHeaders, agent, globalParams, params, opts }) + return callMethod({ method, basePath, defaultHeaders, agent, globalParams, params, opts, cache }) }, } } -const callMethod = async function ({ method, basePath, defaultHeaders, agent, globalParams, params, opts }) { +const callMethod = async function ({ method, basePath, defaultHeaders, agent, globalParams, params, opts, cache }) { const requestParams = { ...globalParams, ...params } const url = getUrl(method, basePath, requestParams) - const response = await makeRequestOrRetry({ url, method, defaultHeaders, agent, requestParams, opts }) + const response = await makeRequestOrRetry({ url, method, defaultHeaders, agent, requestParams, opts, cache }) const parsedResponse = await parseResponse(response) return parsedResponse @@ -70,11 +72,11 @@ const addAgent = function (agent, opts) { return opts } -const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, requestParams, opts }) { +const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, requestParams, opts, cache }) { // Using a loop is simpler here for (let index = 0; index <= MAX_RETRY; index++) { const optsA = getOpts({ method, defaultHeaders, agent, requestParams, opts }) - const { response, error } = await makeRequest(url, optsA) + const { response, error } = await makeRequest(url, optsA, cache) if (shouldRetry({ response, error, method }) && index !== MAX_RETRY) { await waitForRetry(response) @@ -89,8 +91,13 @@ const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, } } -const makeRequest = async function (url, opts) { +const makeRequest = async function (url, opts, cache) { try { + if (cache) { + const response = await cache.get(url, opts.method, opts.headers) + return { response } + } + const response = await fetch(url, opts) return { response } } catch (error) { diff --git a/packages/js-client/src/methods/response.js b/packages/js-client/src/methods/response.ts similarity index 74% rename from packages/js-client/src/methods/response.js rename to packages/js-client/src/methods/response.ts index 5316c46aa0..c76361b8f1 100644 --- a/packages/js-client/src/methods/response.js +++ b/packages/js-client/src/methods/response.ts @@ -1,9 +1,17 @@ import { JSONHTTPError, TextHTTPError } from 'micro-api-client' +import { BufferedResponse } from '../buffered_response.js' import omit from '../omit.js' + +import { getResponseType, ResponseType } from './response_type.js' + // Read and parse the HTTP response -export const parseResponse = async function (response) { +export const parseResponse = async function (response: BufferedResponse | Response) { + if (!(response instanceof Response)) { + return response.body + } + const responseType = getResponseType(response) const textResponse = await response.text() @@ -17,17 +25,7 @@ export const parseResponse = async function (response) { return parsedResponse } -const getResponseType = function ({ headers }) { - const contentType = headers.get('Content-Type') - - if (contentType != null && contentType.includes('json')) { - return 'json' - } - - return 'text' -} - -const parseJsonResponse = function (response, textResponse, responseType) { +const parseJsonResponse = function (response: Response, textResponse: string, responseType: ResponseType) { if (responseType === 'text') { return textResponse } diff --git a/packages/js-client/src/methods/response_type.ts b/packages/js-client/src/methods/response_type.ts new file mode 100644 index 0000000000..1064309b67 --- /dev/null +++ b/packages/js-client/src/methods/response_type.ts @@ -0,0 +1,11 @@ +export type ResponseType = 'json' | 'text' + +export const getResponseType = function ({ headers }): ResponseType { + const contentType = headers.get('Content-Type') + + if (contentType != null && contentType.includes('json')) { + return 'json' + } + + return 'text' +}