diff --git a/README.md b/README.md index 1feb600e38..9561826d16 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,7 @@ Grist can be configured in many ways. Here are the main environment variables it | GRIST_MAX_PARALLEL_REQUESTS_PER_DOC| max number of concurrent API requests allowed per document (default is 10, set to 0 for unlimited) | | GRIST_MAX_UPLOAD_ATTACHMENT_MB | max allowed size for attachments (0 or empty for unlimited). | | GRIST_MAX_UPLOAD_IMPORT_MB | max allowed size for imports (except .grist files) (0 or empty for unlimited). | +| GRIST_MAX_API_REQUEST_BODY_MB | max allowed size for API request bodies in MB (0 or empty for unlimited, defaults to 1). | | GRIST_OFFER_ALL_LANGUAGES | if set, all translated langauages are offered to the user (by default, only languages with a special 'good enough' key set are offered to user). | | GRIST_ORG_IN_PATH | if true, encode org in path rather than domain | | GRIST_PAGE_TITLE_SUFFIX | a string to append to the end of the `` in HTML documents. Defaults to `" - Grist"`. Set to `_blank` for no suffix at all. | diff --git a/app/common/gristUrls.ts b/app/common/gristUrls.ts index 7e4f074ac7..f7e4e3e5c7 100644 --- a/app/common/gristUrls.ts +++ b/app/common/gristUrls.ts @@ -879,6 +879,9 @@ export interface GristLoadConfig { // Max upload allowed for attachments, in bytes; 0 or omitted for unlimited. maxUploadSizeAttachment?: number; + // Max allowed size for API request bodies, in bytes; 0 or omitted for unlimited. + maxApiRequestBodySize?: number; + // Pre-fetched call to getDoc for the doc being loaded. getDoc?: {[id: string]: Document}; diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 3416e281f4..6aeea223a1 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1,107 +1,177 @@ -import {ApiError} from 'app/common/ApiError'; -import {ICustomWidget} from 'app/common/CustomWidget'; -import {delay} from 'app/common/delay'; -import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes, - GristLoadConfig, IGristUrlState, isOrgInPathOnly, LatestVersionAvailable, parseSubdomain, - sanitizePathTail} from 'app/common/gristUrls'; -import {getOrgUrlInfo} from 'app/common/gristUrls'; -import {isAffirmative} from 'app/common/gutil'; -import {UserProfile} from 'app/common/LoginSessionAPI'; -import {SandboxInfo} from 'app/common/SandboxInfo'; -import {tbind} from 'app/common/tbind'; -import * as version from 'app/common/version'; -import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer'; -import {Document} from 'app/gen-server/entity/Document'; -import {Organization} from 'app/gen-server/entity/Organization'; -import {User} from 'app/gen-server/entity/User'; -import {Workspace} from 'app/gen-server/entity/Workspace'; -import {ActivationsManager} from 'app/gen-server/lib/ActivationsManager'; -import {DocApiForwarder} from 'app/gen-server/lib/DocApiForwarder'; -import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; -import {Doom} from 'app/gen-server/lib/Doom'; -import {HomeDBManager, UserChange} from 'app/gen-server/lib/homedb/HomeDBManager'; -import {Housekeeper} from 'app/gen-server/lib/Housekeeper'; -import {Usage} from 'app/gen-server/lib/Usage'; -import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens'; -import {createSandbox} from 'app/server/lib/ActiveDoc'; -import {attachAppEndpoint} from 'app/server/lib/AppEndpoint'; -import {appSettings} from 'app/server/lib/AppSettings'; -import {attachEarlyEndpoints} from 'app/server/lib/attachEarlyEndpoints'; +import { ApiError } from "app/common/ApiError"; +import { ICustomWidget } from "app/common/CustomWidget"; +import { delay } from "app/common/delay"; +import { + encodeUrl, + getSlugIfNeeded, + GristDeploymentType, + GristDeploymentTypes, + GristLoadConfig, + IGristUrlState, + isOrgInPathOnly, + LatestVersionAvailable, + parseSubdomain, + sanitizePathTail, +} from "app/common/gristUrls"; +import { getOrgUrlInfo } from "app/common/gristUrls"; +import { isAffirmative } from "app/common/gutil"; +import { UserProfile } from "app/common/LoginSessionAPI"; +import { SandboxInfo } from "app/common/SandboxInfo"; +import { tbind } from "app/common/tbind"; +import * as version from "app/common/version"; +import { ApiServer, getOrgFromRequest } from "app/gen-server/ApiServer"; +import { Document } from "app/gen-server/entity/Document"; +import { Organization } from "app/gen-server/entity/Organization"; +import { User } from "app/gen-server/entity/User"; +import { Workspace } from "app/gen-server/entity/Workspace"; +import { ActivationsManager } from "app/gen-server/lib/ActivationsManager"; +import { DocApiForwarder } from "app/gen-server/lib/DocApiForwarder"; +import { getDocWorkerMap } from "app/gen-server/lib/DocWorkerMap"; +import { Doom } from "app/gen-server/lib/Doom"; +import { + HomeDBManager, + UserChange, +} from "app/gen-server/lib/homedb/HomeDBManager"; +import { Housekeeper } from "app/gen-server/lib/Housekeeper"; +import { Usage } from "app/gen-server/lib/Usage"; +import { AccessTokens, IAccessTokens } from "app/server/lib/AccessTokens"; +import { createSandbox } from "app/server/lib/ActiveDoc"; +import { attachAppEndpoint } from "app/server/lib/AppEndpoint"; +import { appSettings } from "app/server/lib/AppSettings"; +import { attachEarlyEndpoints } from "app/server/lib/attachEarlyEndpoints"; import { AttachmentStoreProvider, checkAvailabilityAttachmentStoreOptions, getConfiguredAttachmentStoreConfigs, - IAttachmentStoreProvider -} from 'app/server/lib/AttachmentStoreProvider'; -import {addRequestUser, getUser, getUserId, isAnonymousUser, - isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer'; -import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer'; -import {forceSessionChange} from 'app/server/lib/BrowserSession'; -import {Comm} from 'app/server/lib/Comm'; -import {ConfigBackendAPI} from 'app/server/lib/ConfigBackendAPI'; -import {IGristCoreConfig} from 'app/server/lib/configCore'; -import {getAndClearSignupStateCookie} from 'app/server/lib/cookieUtils'; -import {create} from 'app/server/lib/create'; -import {createSavedDoc} from 'app/server/lib/createSavedDoc'; -import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect'; -import {addDocApiRoutes} from 'app/server/lib/DocApi'; -import {DocManager} from 'app/server/lib/DocManager'; -import {getSqliteMode} from 'app/server/lib/DocStorage'; -import {DocWorker} from 'app/server/lib/DocWorker'; -import {DocWorkerLoadTracker, getDocWorkerLoadTracker} from 'app/server/lib/DocWorkerLoadTracker'; -import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; -import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap'; -import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg'; -import {addGoogleAuthEndpoint} from 'app/server/lib/GoogleAuth'; -import {createGristJobs, GristJobs} from 'app/server/lib/GristJobs'; -import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer, - RequestWithGrist} from 'app/server/lib/GristServer'; -import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions'; -import {IAssistant} from 'app/server/lib/IAssistant'; -import {IAuditLogger} from 'app/server/lib/IAuditLogger'; -import {IBilling} from 'app/server/lib/IBilling'; -import {IDocNotificationManager} from 'app/server/lib/IDocNotificationManager'; -import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; -import {EmitNotifier, INotifier} from 'app/server/lib/INotifier'; -import {InstallAdmin} from 'app/server/lib/InstallAdmin'; -import log, {logAsJson} from 'app/server/lib/log'; -import {disableCache} from 'app/server/lib/middleware'; -import {IPermitStore} from 'app/server/lib/Permit'; -import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places'; -import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'; -import {PluginManager} from 'app/server/lib/PluginManager'; -import * as ProcessMonitor from 'app/server/lib/ProcessMonitor'; -import { createPubSubManager, IPubSubManager } from 'app/server/lib/PubSubManager'; -import {adaptServerUrl, getOrgUrl, getOriginUrl, getScope, integerParam, isParameterOn, optIntegerParam, - optStringParam, RequestWithGristInfo, stringArrayParam, stringParam, TEST_HTTPS_OFFSET, - trustOrigin} from 'app/server/lib/requestUtils'; -import {buildScimRouter} from 'app/server/lib/scim'; -import {ISendAppPageOptions, makeGristConfig, makeMessagePage, makeSendAppPage} from 'app/server/lib/sendAppPage'; -import {getDatabaseUrl, listenPromise, timeoutReached} from 'app/server/lib/serverUtils'; -import {Sessions} from 'app/server/lib/Sessions'; -import * as shutdown from 'app/server/lib/shutdown'; -import {TagChecker} from 'app/server/lib/TagChecker'; -import {ITelemetry} from 'app/server/lib/Telemetry'; -import {startTestingHooks} from 'app/server/lib/TestingHooks'; -import {getTestLoginSystem} from 'app/server/lib/TestLogin'; -import {UpdateManager} from 'app/server/lib/UpdateManager'; -import {addUploadRoute} from 'app/server/lib/uploads'; -import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository'; -import {setupLocale} from 'app/server/localization'; -import axios from 'axios'; -import express from 'express'; -import * as fse from 'fs-extra'; -import * as http from 'http'; -import * as https from 'https'; -import {i18n} from 'i18next'; -import i18Middleware from 'i18next-http-middleware'; -import mapValues = require('lodash/mapValues'); -import pick = require('lodash/pick'); -import morganLogger from 'morgan'; -import {AddressInfo} from 'net'; -import fetch from 'node-fetch'; -import * as path from 'path'; -import * as serveStatic from 'serve-static'; + IAttachmentStoreProvider, +} from "app/server/lib/AttachmentStoreProvider"; +import { + addRequestUser, + getUser, + getUserId, + isAnonymousUser, + isSingleUserMode, + redirectToLoginUnconditionally, +} from "app/server/lib/Authorizer"; +import { + redirectToLogin, + RequestWithLogin, + signInStatusMiddleware, +} from "app/server/lib/Authorizer"; +import { forceSessionChange } from "app/server/lib/BrowserSession"; +import { Comm } from "app/server/lib/Comm"; +import { ConfigBackendAPI } from "app/server/lib/ConfigBackendAPI"; +import { IGristCoreConfig } from "app/server/lib/configCore"; +import { getAndClearSignupStateCookie } from "app/server/lib/cookieUtils"; +import { create } from "app/server/lib/create"; +import { createSavedDoc } from "app/server/lib/createSavedDoc"; +import { addDiscourseConnectEndpoints } from "app/server/lib/DiscourseConnect"; +import { addDocApiRoutes } from "app/server/lib/DocApi"; +import { DocManager } from "app/server/lib/DocManager"; +import { getSqliteMode } from "app/server/lib/DocStorage"; +import { DocWorker } from "app/server/lib/DocWorker"; +import { + DocWorkerLoadTracker, + getDocWorkerLoadTracker, +} from "app/server/lib/DocWorkerLoadTracker"; +import { DocWorkerInfo, IDocWorkerMap } from "app/server/lib/DocWorkerMap"; +import { + expressWrap, + jsonErrorHandler, + secureJsonErrorHandler, +} from "app/server/lib/expressWrap"; +import { Hosts, RequestWithOrg } from "app/server/lib/extractOrg"; +import { addGoogleAuthEndpoint } from "app/server/lib/GoogleAuth"; +import { createGristJobs, GristJobs } from "app/server/lib/GristJobs"; +import { + DocTemplate, + GristLoginMiddleware, + GristLoginSystem, + GristServer, + RequestWithGrist, +} from "app/server/lib/GristServer"; +import { initGristSessions, SessionStore } from "app/server/lib/gristSessions"; +import { IAssistant } from "app/server/lib/IAssistant"; +import { IAuditLogger } from "app/server/lib/IAuditLogger"; +import { IBilling } from "app/server/lib/IBilling"; +import { IDocNotificationManager } from "app/server/lib/IDocNotificationManager"; +import { IDocStorageManager } from "app/server/lib/IDocStorageManager"; +import { EmitNotifier, INotifier } from "app/server/lib/INotifier"; +import { InstallAdmin } from "app/server/lib/InstallAdmin"; +import log, { logAsJson } from "app/server/lib/log"; +import { disableCache } from "app/server/lib/middleware"; +import { IPermitStore } from "app/server/lib/Permit"; +import { + getAppPathTo, + getAppRoot, + getInstanceRoot, + getUnpackedAppRoot, +} from "app/server/lib/places"; +import { + addPluginEndpoints, + limitToPlugins, +} from "app/server/lib/PluginEndpoint"; +import { PluginManager } from "app/server/lib/PluginManager"; +import * as ProcessMonitor from "app/server/lib/ProcessMonitor"; +import { + createPubSubManager, + IPubSubManager, +} from "app/server/lib/PubSubManager"; +import { + adaptServerUrl, + getOrgUrl, + getOriginUrl, + getScope, + integerParam, + isParameterOn, + optIntegerParam, + optStringParam, + RequestWithGristInfo, + stringArrayParam, + stringParam, + TEST_HTTPS_OFFSET, + trustOrigin, +} from "app/server/lib/requestUtils"; +import { buildScimRouter } from "app/server/lib/scim"; +import { + ISendAppPageOptions, + makeGristConfig, + makeMessagePage, + makeSendAppPage, +} from "app/server/lib/sendAppPage"; +import { + getDatabaseUrl, + listenPromise, + timeoutReached, +} from "app/server/lib/serverUtils"; +import { Sessions } from "app/server/lib/Sessions"; +import * as shutdown from "app/server/lib/shutdown"; +import { TagChecker } from "app/server/lib/TagChecker"; +import { ITelemetry } from "app/server/lib/Telemetry"; +import { startTestingHooks } from "app/server/lib/TestingHooks"; +import { getTestLoginSystem } from "app/server/lib/TestLogin"; +import { UpdateManager } from "app/server/lib/UpdateManager"; +import { addUploadRoute } from "app/server/lib/uploads"; +import { + buildWidgetRepository, + getWidgetsInPlugins, + IWidgetRepository, +} from "app/server/lib/WidgetRepository"; +import { setupLocale } from "app/server/localization"; +import axios from "axios"; +import express from "express"; +import * as fse from "fs-extra"; +import * as http from "http"; +import * as https from "https"; +import { i18n } from "i18next"; +import i18Middleware from "i18next-http-middleware"; +import mapValues = require("lodash/mapValues"); +import pick = require("lodash/pick"); +import morganLogger from "morgan"; +import { AddressInfo } from "net"; +import fetch from "node-fetch"; +import * as path from "path"; +import * as serveStatic from "serve-static"; // Health checks are a little noisy in the logs, so we don't show them all. // We show the first N health checks: @@ -114,7 +184,7 @@ const HEALTH_CHECK_LOG_SHOW_EVERY_N = 100; const DOC_ID_NEW_USER_INFO = process.env.DOC_ID_NEW_USER_INFO; // PubSub channel we use to inform all servers when a new available Grist version is detected. -const latestVersionChannel = 'latestVersionAvailable'; +const latestVersionChannel = "latestVersionAvailable"; export interface FlexServerOptions { dataDir?: string; @@ -152,8 +222,8 @@ export class FlexServer implements GristServer { private _comm: Comm; private _deploymentType: GristDeploymentType; private _dbManager: HomeDBManager; - private _defaultBaseDomain: string|undefined; - private _pluginUrl: string|undefined; + private _defaultBaseDomain: string | undefined; + private _pluginUrl: string | undefined; private _pluginUrlReady: boolean = false; private _servesPlugins?: boolean; private _bundledWidgets?: ICustomWidget[]; @@ -170,20 +240,23 @@ export class FlexServer implements GristServer { private _storageManager: IDocStorageManager; private _auditLogger: IAuditLogger; private _telemetry: ITelemetry; - private _processMonitorStop?: () => void; // Callback to stop the ProcessMonitor + private _processMonitorStop?: () => void; // Callback to stop the ProcessMonitor private _docWorkerMap: IDocWorkerMap; private _docWorkerLoadTracker?: DocWorkerLoadTracker; private _widgetRepository: IWidgetRepository; - private _docNotificationManager: IDocNotificationManager|undefined|false = false; - private _pubSubManager: IPubSubManager = createPubSubManager(process.env.REDIS_URL); + private _docNotificationManager: IDocNotificationManager | undefined | false = + false; + private _pubSubManager: IPubSubManager = createPubSubManager( + process.env.REDIS_URL + ); private _assistant?: IAssistant; private _accessTokens: IAccessTokens; - private _internalPermitStore: IPermitStore; // store for permits that stay within our servers - private _externalPermitStore: IPermitStore; // store for permits that pass through outside servers + private _internalPermitStore: IPermitStore; // store for permits that stay within our servers + private _externalPermitStore: IPermitStore; // store for permits that pass through outside servers private _disabled: boolean = false; private _disableExternalStorage: boolean = false; - private _healthy: boolean = true; // becomes false if a serious error has occurred and - // server cannot do its work. + private _healthy: boolean = true; // becomes false if a serious error has occurred and + // server cannot do its work. private _healthCheckCounter: number = 0; private _hasTestingHooks: boolean = false; private _loginMiddleware: GristLoginMiddleware; @@ -200,10 +273,23 @@ export class FlexServer implements GristServer { private _redirectToLoginUnconditionally: express.RequestHandler | null; private _redirectToOrgMiddleware: express.RequestHandler; private _redirectToHostMiddleware: express.RequestHandler; - private _getLoginRedirectUrl: (req: express.Request, target: URL) => Promise<string>; - private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>; - private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>; - private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>; + private _getLoginRedirectUrl: ( + req: express.Request, + target: URL + ) => Promise<string>; + private _getSignUpRedirectUrl: ( + req: express.Request, + target: URL + ) => Promise<string>; + private _getLogoutRedirectUrl: ( + req: express.Request, + nextUrl: URL + ) => Promise<string>; + private _sendAppPage: ( + req: express.Request, + resp: express.Response, + options: ISendAppPageOptions + ) => Promise<void>; private _getLoginSystem: () => Promise<GristLoginSystem>; // Set once ready() is called private _isReady: boolean = false; @@ -213,34 +299,44 @@ export class FlexServer implements GristServer { private _emitNotifier: EmitNotifier = new EmitNotifier(); private _latestVersionAvailable?: LatestVersionAvailable; - constructor(public port: number, public name: string = 'flexServer', - public readonly options: FlexServerOptions = {}) { + constructor( + public port: number, + public name: string = "flexServer", + public readonly options: FlexServerOptions = {} + ) { this._getLoginSystem = create.getLoginSystem.bind(create); this.settings = options.settings; this.app = express(); - this.app.set('port', port); + this.app.set("port", port); this.appRoot = getAppRoot(); this.host = process.env.GRIST_HOST || "localhost"; - log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`); - this.info.push(['appRoot', this.appRoot]); + log.info( + `== Grist version is ${version.version} (commit ${version.gitcommit})` + ); + this.info.push(["appRoot", this.appRoot]); // Initialize locales files. this.i18Instance = setupLocale(this.appRoot); if (Array.isArray(this.i18Instance.options.preload)) { - this.info.push(['i18:locale', this.i18Instance.options.preload.join(",")]); + this.info.push([ + "i18:locale", + this.i18Instance.options.preload.join(","), + ]); } if (Array.isArray(this.i18Instance.options.ns)) { - this.info.push(['i18:namespace', this.i18Instance.options.ns.join(",")]); + this.info.push(["i18:namespace", this.i18Instance.options.ns.join(",")]); } // Add language detection middleware. this.app.use(i18Middleware.handle(this.i18Instance)); // This directory hold Grist documents. - let docsRoot = path.resolve((this.options && this.options.dataDir) || - process.env.GRIST_DATA_DIR || - getAppPathTo(this.appRoot, 'samples')); + let docsRoot = path.resolve( + (this.options && this.options.dataDir) || + process.env.GRIST_DATA_DIR || + getAppPathTo(this.appRoot, "samples") + ); // In testing, it can be useful to separate out document roots used // by distinct FlexServers. - if (process.env.GRIST_TEST_ADD_PORT_TO_DOCS_ROOT === 'true') { + if (process.env.GRIST_TEST_ADD_PORT_TO_DOCS_ROOT === "true") { docsRoot = path.resolve(docsRoot, String(port)); } // Create directory if it doesn't exist. @@ -249,27 +345,36 @@ export class FlexServer implements GristServer { // to simply fail if the docs root directory does not exist. fse.mkdirpSync(docsRoot); this.docsRoot = fse.realpathSync(docsRoot); - this.info.push(['docsRoot', this.docsRoot]); + this.info.push(["docsRoot", this.docsRoot]); this._deploymentType = this.create.deploymentType(); if (process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE) { - this._deploymentType = GristDeploymentTypes.check(process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE); + this._deploymentType = GristDeploymentTypes.check( + process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE + ); } const homeUrl = process.env.APP_HOME_URL; // The "base domain" is only a thing if orgs are encoded as a subdomain. - if (process.env.GRIST_ORG_IN_PATH === 'true' || process.env.GRIST_SINGLE_ORG) { - this._defaultBaseDomain = options.baseDomain || (homeUrl && new URL(homeUrl).hostname); + if ( + process.env.GRIST_ORG_IN_PATH === "true" || + process.env.GRIST_SINGLE_ORG + ) { + this._defaultBaseDomain = + options.baseDomain || (homeUrl && new URL(homeUrl).hostname); } else { - this._defaultBaseDomain = options.baseDomain || (homeUrl && parseSubdomain(new URL(homeUrl).hostname).base); + this._defaultBaseDomain = + options.baseDomain || + (homeUrl && parseSubdomain(new URL(homeUrl).hostname).base); } - this.info.push(['defaultBaseDomain', this._defaultBaseDomain]); + this.info.push(["defaultBaseDomain", this._defaultBaseDomain]); this._pluginUrl = options.pluginUrl || process.env.APP_UNTRUSTED_URL; // We don't bother unsubscribing because that's automatic when we close this._pubSubManager. void this.getPubSubManager().subscribe(latestVersionChannel, (message) => { - const latestVersionAvailable: LatestVersionAvailable = JSON.parse(message); - log.debug('FlexServer: setting latest version', latestVersionAvailable); + const latestVersionAvailable: LatestVersionAvailable = + JSON.parse(message); + log.debug("FlexServer: setting latest version", latestVersionAvailable); this.setLatestVersionAvailable(latestVersionAvailable); }); @@ -282,7 +387,7 @@ export class FlexServer implements GristServer { this.electronServerMethods = { onDocOpen(cb) { // currently only a stub. - cb(''); + cb(""); }, async getUserConfig() { return userConfig; @@ -291,8 +396,8 @@ export class FlexServer implements GristServer { userConfig = obj; }, onBackupMade() { - log.info('backup skipped'); - } + log.info("backup skipped"); + }, }; this.app.use((req, res, next) => { @@ -321,8 +426,11 @@ export class FlexServer implements GristServer { * via Notifier are incompatible for this reason). */ public getDefaultHomeUrl(): string { - const homeUrl = process.env.APP_HOME_URL || (this._has('api') && this.getOwnUrl()); - if (!homeUrl) { throw new Error("need APP_HOME_URL"); } + const homeUrl = + process.env.APP_HOME_URL || (this._has("api") && this.getOwnUrl()); + if (!homeUrl) { + throw new Error("need APP_HOME_URL"); + } return homeUrl; } @@ -340,7 +448,7 @@ export class FlexServer implements GristServer { * If relPath is given, returns that path relative to homeUrl. If omitted, note that * getHomeUrl() will still return a URL ending in "/". */ - public getHomeUrl(req: express.Request, relPath: string = ''): string { + public getHomeUrl(req: express.Request, relPath: string = ""): string { // Get the default home url. const homeUrl = new URL(relPath, this.getDefaultHomeUrl()); adaptServerUrl(homeUrl, req as RequestWithOrg); @@ -350,7 +458,7 @@ export class FlexServer implements GristServer { /** * Same as getHomeUrl, but for requesting internally. */ - public getHomeInternalUrl(relPath: string = ''): string { + public getHomeInternalUrl(relPath: string = ""): string { const homeUrl = new URL(relPath, this.getDefaultHomeInternalUrl()); return homeUrl.href; } @@ -361,7 +469,10 @@ export class FlexServer implements GristServer { * specifically with custom domains (perhaps we might limit which docs can be accessed * based on domain). */ - public async getHomeUrlByDocId(docId: string, relPath: string = ''): Promise<string> { + public async getHomeUrlByDocId( + docId: string, + relPath: string = "" + ): Promise<string> { return new URL(relPath, this.getDefaultHomeInternalUrl()).href; } @@ -369,7 +480,9 @@ export class FlexServer implements GristServer { // number the client expects when communicating with the server if there are intermediaries. public getOwnPort(): number { // Get the port from the server in case it was started with port 0. - return this.server ? (this.server.address() as AddressInfo).port : this.port; + return this.server + ? (this.server.address() as AddressInfo).port + : this.port; } /** @@ -383,27 +496,42 @@ export class FlexServer implements GristServer { * Get a url to an org that should be accessible by all signed-in users. For now, this * returns the base URL of the personal org (typically docs[-s]). */ - public getMergedOrgUrl(req: RequestWithLogin, pathname: string = '/'): string { - return this._getOrgRedirectUrl(req, this._dbManager.mergedOrgDomain(), pathname); + public getMergedOrgUrl( + req: RequestWithLogin, + pathname: string = "/" + ): string { + return this._getOrgRedirectUrl( + req, + this._dbManager.mergedOrgDomain(), + pathname + ); } public getPermitStore(): IPermitStore { - if (!this._internalPermitStore) { throw new Error('no permit store available'); } + if (!this._internalPermitStore) { + throw new Error("no permit store available"); + } return this._internalPermitStore; } public getExternalPermitStore(): IPermitStore { - if (!this._externalPermitStore) { throw new Error('no permit store available'); } + if (!this._externalPermitStore) { + throw new Error("no permit store available"); + } return this._externalPermitStore; } public getSessions(): Sessions { - if (!this._sessions) { throw new Error('no sessions available'); } + if (!this._sessions) { + throw new Error("no sessions available"); + } return this._sessions; } public getComm(): Comm { - if (!this._comm) { throw new Error('no Comm available'); } + if (!this._comm) { + throw new Error("no Comm available"); + } return this._comm; } @@ -412,42 +540,58 @@ export class FlexServer implements GristServer { } public getHosts(): Hosts { - if (!this._hosts) { throw new Error('no hosts available'); } + if (!this._hosts) { + throw new Error("no hosts available"); + } return this._hosts; } public getActivations(): ActivationsManager { - if (!this._activations) { throw new Error('no activations available'); } + if (!this._activations) { + throw new Error("no activations available"); + } return this._activations; } public getHomeDBManager(): HomeDBManager { - if (!this._dbManager) { throw new Error('no home db available'); } + if (!this._dbManager) { + throw new Error("no home db available"); + } return this._dbManager; } public getStorageManager(): IDocStorageManager { - if (!this._storageManager) { throw new Error('no storage manager available'); } + if (!this._storageManager) { + throw new Error("no storage manager available"); + } return this._storageManager; } public getAuditLogger(): IAuditLogger { - if (!this._auditLogger) { throw new Error('no audit logger available'); } + if (!this._auditLogger) { + throw new Error("no audit logger available"); + } return this._auditLogger; } public getDocManager(): DocManager { - if (!this._docManager) { throw new Error('no document manager available'); } + if (!this._docManager) { + throw new Error("no document manager available"); + } return this._docManager; } public getTelemetry(): ITelemetry { - if (!this._telemetry) { throw new Error('no telemetry available'); } + if (!this._telemetry) { + throw new Error("no telemetry available"); + } return this._telemetry; } public getWidgetRepository(): IWidgetRepository { - if (!this._widgetRepository) { throw new Error('no widget repository available'); } + if (!this._widgetRepository) { + throw new Error("no widget repository available"); + } return this._widgetRepository; } @@ -458,16 +602,19 @@ export class FlexServer implements GristServer { public getNotifier(): INotifier { // We only warn if we are in a server that doesn't configure notifiers (i.e. not a home // server). But actually having a working notifier isn't required. - if (!this._has('notifier')) { throw new Error('no notifier available'); } + if (!this._has("notifier")) { + throw new Error("no notifier available"); + } // Expose a wrapper around it that emits actions. return this._emitNotifier; } - public getDocNotificationManager(): IDocNotificationManager|undefined { + public getDocNotificationManager(): IDocNotificationManager | undefined { if (this._docNotificationManager === false) { // The special value of 'false' is used to create only on first call. Afterwards, // the value may be undefined, but no longer false. - this._docNotificationManager = this.create.createDocNotificationManager(this); + this._docNotificationManager = + this.create.createDocNotificationManager(this); } return this._docNotificationManager; } @@ -481,12 +628,16 @@ export class FlexServer implements GristServer { } public getInstallAdmin(): InstallAdmin { - if (!this._installAdmin) { throw new Error('no InstallAdmin available'); } + if (!this._installAdmin) { + throw new Error("no InstallAdmin available"); + } return this._installAdmin; } public getAccessTokens() { - if (this._accessTokens) { return this._accessTokens; } + if (this._accessTokens) { + return this._accessTokens; + } this.addDocWorkerMap(); const cli = this._docWorkerMap.getRedisClient(); this._accessTokens = new AccessTokens(cli); @@ -494,40 +645,57 @@ export class FlexServer implements GristServer { } public getUpdateManager() { - if (!this._updateManager) { throw new Error('no UpdateManager available'); } + if (!this._updateManager) { + throw new Error("no UpdateManager available"); + } return this._updateManager; } public getBilling(): IBilling { if (!this._billing) { - if (!this._dbManager) { throw new Error("need dbManager"); } + if (!this._dbManager) { + throw new Error("need dbManager"); + } this._billing = this.create.Billing(this._dbManager, this); } return this._billing; } - public sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void> { - if (!this._sendAppPage) { throw new Error('no _sendAppPage method available'); } + public sendAppPage( + req: express.Request, + resp: express.Response, + options: ISendAppPageOptions + ): Promise<void> { + if (!this._sendAppPage) { + throw new Error("no _sendAppPage method available"); + } return this._sendAppPage(req, resp, options); } public addLogging() { - if (this._check('logging')) { return; } - if (!this._httpLoggingEnabled()) { return; } + if (this._check("logging")) { + return; + } + if (!this._httpLoggingEnabled()) { + return; + } // Add a timestamp token that matches exactly the formatting of non-morgan logs. - morganLogger.token('logTime', (req: Request) => log.timestamp()); + morganLogger.token("logTime", (req: Request) => log.timestamp()); // Add an optional gristInfo token that can replace the url, if the url is sensitive. - morganLogger.token('gristInfo', (req: RequestWithGristInfo) => - req.gristInfo || req.originalUrl || req.url); - morganLogger.token('host', (req: express.Request) => req.get('host')); - morganLogger.token('body', (req: express.Request) => - req.is('application/json') ? JSON.stringify(req.body) : undefined + morganLogger.token( + "gristInfo", + (req: RequestWithGristInfo) => req.gristInfo || req.originalUrl || req.url + ); + morganLogger.token("host", (req: express.Request) => req.get("host")); + morganLogger.token("body", (req: express.Request) => + req.is("application/json") ? JSON.stringify(req.body) : undefined ); // For debugging, be careful not to enable logging in production (may log sensitive data) const shouldLogBody = isAffirmative(process.env.GRIST_LOG_HTTP_BODY); - const msg = `:logTime :host :method :gristInfo ${shouldLogBody ? ':body ' : ''}` + + const msg = + `:logTime :host :method :gristInfo ${shouldLogBody ? ":body " : ""}` + ":status :response-time ms - :res[content-length]"; // In hosted Grist, render json so logs retain more organization. function outputJson(tokens: any, req: any, res: any) { @@ -538,18 +706,23 @@ export class FlexServer implements GristServer { path: tokens.gristInfo(req, res), ...(shouldLogBody ? { body: tokens.body(req, res) } : {}), status: tokens.status(req, res), - timeMs: parseFloat(tokens['response-time'](req, res)) || undefined, - contentLength: parseInt(tokens.res(req, res, 'content-length'), 10) || undefined, + timeMs: parseFloat(tokens["response-time"](req, res)) || undefined, + contentLength: + parseInt(tokens.res(req, res, "content-length"), 10) || undefined, altSessionId: req.altSessionId, }); } - this.app.use(morganLogger(logAsJson ? outputJson : msg, { - skip: this._shouldSkipRequestLogging.bind(this) - })); + this.app.use( + morganLogger(logAsJson ? outputJson : msg, { + skip: this._shouldSkipRequestLogging.bind(this), + }) + ); } public addHealthCheck() { - if (this._check('health')) { return; } + if (this._check("health")) { + return; + } // Health check endpoint. if called with /hooks, testing hooks are required in order to be // considered healthy. Testing hooks are used only in server started for tests, and // /status/hooks allows the tests to wait for them to be ready. @@ -557,26 +730,35 @@ export class FlexServer implements GristServer { // If redis=1 query parameter is included, status will include the status of the Redis connection. // If docWorkerRegistered=1 query parameter is included, status will include the status of the // doc worker registration in Redis. - this.app.get('/status(/hooks)?', async (req, res) => { - const checks = new Map<string, Promise<boolean>|boolean>(); - const timeout = optIntegerParam(req.query.timeout, 'timeout') || 10_000; + this.app.get("/status(/hooks)?", async (req, res) => { + const checks = new Map<string, Promise<boolean> | boolean>(); + const timeout = optIntegerParam(req.query.timeout, "timeout") || 10_000; // Check that the given promise resolves with no error within our timeout. - const asyncCheck = async (promise: Promise<unknown>|undefined) => { - if (!promise || await timeoutReached(timeout, promise) === true) { + const asyncCheck = async (promise: Promise<unknown> | undefined) => { + if (!promise || (await timeoutReached(timeout, promise)) === true) { return false; } - return promise.then(() => true, () => false); // Success => true, rejection => false + return promise.then( + () => true, + () => false + ); // Success => true, rejection => false }; - if (req.path.endsWith('/hooks')) { - checks.set('hooks', this._hasTestingHooks); + if (req.path.endsWith("/hooks")) { + checks.set("hooks", this._hasTestingHooks); } if (isParameterOn(req.query.db)) { - checks.set('db', asyncCheck(this._dbManager.connection.query('SELECT 1'))); + checks.set( + "db", + asyncCheck(this._dbManager.connection.query("SELECT 1")) + ); } if (isParameterOn(req.query.redis)) { - checks.set('redis', asyncCheck(this._docWorkerMap.getRedisClient()?.pingAsync())); + checks.set( + "redis", + asyncCheck(this._docWorkerMap.getRedisClient()?.pingAsync()) + ); } if (isParameterOn(req.query.docWorkerRegistered) && this.worker) { // Only check whether the doc worker is registered if we have a worker. @@ -584,40 +766,50 @@ export class FlexServer implements GristServer { // be checked with the 'redis' parameter (the user may want to avoid // removing workers when connection is unstable). if (this._docWorkerMap.getRedisClient()?.connected) { - checks.set('docWorkerRegistered', asyncCheck( - this._docWorkerMap.isWorkerRegistered(this.worker).then(isRegistered => { - if (!isRegistered) { throw new Error('doc worker not registered'); } - return isRegistered; - }) - )); + checks.set( + "docWorkerRegistered", + asyncCheck( + this._docWorkerMap + .isWorkerRegistered(this.worker) + .then((isRegistered) => { + if (!isRegistered) { + throw new Error("doc worker not registered"); + } + return isRegistered; + }) + ) + ); } } if (isParameterOn(req.query.ready)) { - checks.set('ready', this._isReady); + checks.set("ready", this._isReady); } - let extra = ''; + let extra = ""; let ok = true; let statuses: string[] = []; // If we had any extra check, collect their status to report them. if (checks.size > 0) { const results = await Promise.all(checks.values()); - ok = ok && results.every(r => r === true); - statuses = Array.from(checks.keys(), (key, i) => `${key} ${results[i] ? 'ok' : 'not ok'}`); + ok = ok && results.every((r) => r === true); + statuses = Array.from( + checks.keys(), + (key, i) => `${key} ${results[i] ? "ok" : "not ok"}` + ); extra = ` (${statuses.join(", ")})`; } const overallOk = ok && this._healthy; - if ((this._healthCheckCounter % 100) === 0 || !overallOk) { + if (this._healthCheckCounter % 100 === 0 || !overallOk) { log.rawDebug(`Healthcheck result`, { - host: req.get('host'), + host: req.get("host"), path: req.path, query: req.query, ok, statuses, healthy: this._healthy, overallOk, - previousSuccessfulChecks: this._healthCheckCounter + previousSuccessfulChecks: this._healthCheckCounter, }); } @@ -625,7 +817,7 @@ export class FlexServer implements GristServer { this._healthCheckCounter++; res.status(200).send(`Grist ${this.name} is alive${extra}.`); } else { - this._healthCheckCounter = 0; // reset counter if we ever go internally unhealthy. + this._healthCheckCounter = 0; // reset counter if we ever go internally unhealthy. res.status(500).send(`Grist ${this.name} is unhealthy${extra}.`); } }); @@ -658,17 +850,19 @@ export class FlexServer implements GristServer { * */ public addBootPage() { - if (this._check('boot')) { return; } - this.app.get('/boot(/*)?', async (req, res) => { + if (this._check("boot")) { + return; + } + this.app.get("/boot(/*)?", async (req, res) => { // Doing a good redirect is actually pretty subtle and we might // get it wrong, so just say /boot got moved. - res.send('The /boot/KEY page is now /admin?boot-key=KEY'); + res.send("The /boot/KEY page is now /admin?boot-key=KEY"); }); } - public getBootKey(): string|undefined { - return appSettings.section('boot').flag('key').readString({ - envVar: 'GRIST_BOOT_KEY' + public getBootKey(): string | undefined { + return appSettings.section("boot").flag("key").readString({ + envVar: "GRIST_BOOT_KEY", }); } @@ -678,34 +872,40 @@ export class FlexServer implements GristServer { // If ready() hasn't been called yet, don't continue, and // give a clear error. This is to avoid exposing the service // in a partially configured form. - return res.status(503).json({error: 'Service unavailable during start up'}); + return res + .status(503) + .json({ error: "Service unavailable during start up" }); } next(); }); } public testAddRouter() { - if (this._check('router')) { return; } - this.app.get('/test/router', (req, res) => { - const act = optStringParam(req.query.act, 'act') || 'none'; - const port = stringParam(req.query.port, 'port'); // port is trusted in mock; in prod it is not. - if (act === 'add' || act === 'remove') { + if (this._check("router")) { + return; + } + this.app.get("/test/router", (req, res) => { + const act = optStringParam(req.query.act, "act") || "none"; + const port = stringParam(req.query.port, "port"); // port is trusted in mock; in prod it is not. + if (act === "add" || act === "remove") { const host = `localhost:${port}`; return res.status(200).json({ act, host, url: `http://${host}`, - message: 'ok', + message: "ok", }); } - return res.status(500).json({error: 'unrecognized action'}); + return res.status(500).json({ error: "unrecognized action" }); }); } public addCleanup() { - if (this._check('cleanup')) { return; } + if (this._check("cleanup")) { + return; + } // Set up signal handlers. Note that nodemon sends SIGUSR2 to restart node. - shutdown.cleanupOnSignals('SIGINT', 'SIGTERM', 'SIGHUP', 'SIGUSR2'); + shutdown.cleanupOnSignals("SIGINT", "SIGTERM", "SIGHUP", "SIGUSR2"); // We listen for uncaughtExceptions / unhandledRejections, but do exit when they happen. It is // a strong recommendation, which seems best to follow @@ -717,22 +917,24 @@ export class FlexServer implements GristServer { // Note that this event catches also 'unhandledRejection' (origin should be either // 'uncaughtException' or 'unhandledRejection'). - process.on('uncaughtException', (err, origin) => { + process.on("uncaughtException", (err, origin) => { log.error(`UNHANDLED ERROR ${origin} (${counter}):`, err); if (counter === 0) { // Only call shutdown once. It's async and could in theory fail, in which case it would be // another unhandledRejection, and would get caught and reported by this same handler. - void(shutdown.exit(1)); + void shutdown.exit(1); } counter++; }); } public addTagChecker() { - if (this._check('tag', '!org')) { return; } + if (this._check("tag", "!org")) { + return; + } // Handle requests that start with /v/TAG/ and set .tag property on them. this.tag = version.gitcommit; - this.info.push(['tag', this.tag]); + this.info.push(["tag", this.tag]); this.tagChecker = new TagChecker(this.tag); this.app.use(this.tagChecker.inspectTag); } @@ -744,65 +946,98 @@ export class FlexServer implements GristServer { * TODO: determine what the prefix should be, and check it, to catch bugs. */ public stripDocWorkerIdPathPrefixIfPresent() { - if (this._check('strip_dw', '!tag', '!org')) { return; } + if (this._check("strip_dw", "!tag", "!org")) { + return; + } this.app.use((req, resp, next) => { const match = req.url.match(/^\/dw\/([-a-zA-Z0-9]+)([/?].*)?$/); - if (match) { req.url = sanitizePathTail(match[2]); } + if (match) { + req.url = sanitizePathTail(match[2]); + } next(); }); } public addOrg() { - if (this._check('org', 'homedb', 'hosts')) { return; } + if (this._check("org", "homedb", "hosts")) { + return; + } this.app.use(this._hosts.extractOrg); } public setDirectory() { - if (this._check('dir')) { return; } + if (this._check("dir")) { + return; + } process.chdir(getUnpackedAppRoot(this.appRoot)); } public get instanceRoot() { if (!this._instanceRoot) { this._instanceRoot = getInstanceRoot(); - this.info.push(['instanceRoot', this._instanceRoot]); + this.info.push(["instanceRoot", this._instanceRoot]); } return this._instanceRoot; } public addStaticAndBowerDirectories() { - if (this._check('static_and_bower', 'dir')) { return; } + if (this._check("static_and_bower", "dir")) { + return; + } this.addTagChecker(); // Grist has static help files, which may be useful for standalone app, // but for hosted grist the latest help is at support.getgrist.com. Redirect // to this page for the benefit of crawlers which currently rank the static help // page link highly for historic reasons. - this.app.use(/^\/help\//, expressWrap(async (req, res) => { - res.redirect('https://support.getgrist.com'); - })); + this.app.use( + /^\/help\//, + expressWrap(async (req, res) => { + res.redirect("https://support.getgrist.com"); + }) + ); // If there is a directory called "static_ext", serve material from there // as well. This isn't used in grist-core but is handy for extensions such // as an Electron app. - const staticExtDir = getAppPathTo(this.appRoot, 'static') + '_ext'; - const staticExtApp = fse.existsSync(staticExtDir) ? - express.static(staticExtDir, serveAnyOrigin) : null; - const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), serveAnyOrigin); - const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), serveAnyOrigin); + const staticExtDir = getAppPathTo(this.appRoot, "static") + "_ext"; + const staticExtApp = fse.existsSync(staticExtDir) + ? express.static(staticExtDir, serveAnyOrigin) + : null; + const staticApp = express.static( + getAppPathTo(this.appRoot, "static"), + serveAnyOrigin + ); + const bowerApp = express.static( + getAppPathTo(this.appRoot, "bower_components"), + serveAnyOrigin + ); if (process.env.GRIST_LOCALES_DIR) { - const locales = express.static(process.env.GRIST_LOCALES_DIR, serveAnyOrigin); + const locales = express.static( + process.env.GRIST_LOCALES_DIR, + serveAnyOrigin + ); this.app.use("/locales", this.tagChecker.withTag(locales)); } - if (staticExtApp) { this.app.use(this.tagChecker.withTag(staticExtApp)); } + if (staticExtApp) { + this.app.use(this.tagChecker.withTag(staticExtApp)); + } this.app.use(this.tagChecker.withTag(staticApp)); this.app.use(this.tagChecker.withTag(bowerApp)); } // Some tests rely on testFOO.html files being served. public addAssetsForTests() { - if (this._check('testAssets', 'dir')) { return; } + if (this._check("testAssets", "dir")) { + return; + } // Serve test[a-z]*.html for test purposes. - this.app.use(/^\/(test[a-z]*.html)$/i, expressWrap(async (req, res) => - res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + this.app.use( + /^\/(test[a-z]*.html)$/i, + expressWrap(async (req, res) => + res.sendFile(req.params[0], { + root: getAppPathTo(this.appRoot, "static"), + }) + ) + ); } // Plugin operation relies currently on grist-plugin-api.js being available, @@ -810,45 +1045,73 @@ export class FlexServer implements GristServer { // host. The assets should be available without version tags, but not // at the root level - we nest them in /plugins/assets. public async addAssetsForPlugins() { - if (this._check('pluginUntaggedAssets', 'dir')) { return; } - this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) => - res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + if (this._check("pluginUntaggedAssets", "dir")) { + return; + } + this.app.use( + /^\/(grist-plugin-api.js)$/, + expressWrap(async (req, res) => + res.sendFile(req.params[0], { + root: getAppPathTo(this.appRoot, "static"), + }) + ) + ); // Plugins get access to static resources without a tag this.app.use( - '/plugins/assets', - limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'static')))); + "/plugins/assets", + limitToPlugins(this, express.static(getAppPathTo(this.appRoot, "static"))) + ); this.app.use( - '/plugins/assets', - limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'bower_components')))); + "/plugins/assets", + limitToPlugins( + this, + express.static(getAppPathTo(this.appRoot, "bower_components")) + ) + ); // Serve custom-widget.html message for anyone. - this.app.use(/^\/(custom-widget.html)$/, expressWrap(async (req, res) => - res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + this.app.use( + /^\/(custom-widget.html)$/, + expressWrap(async (req, res) => + res.sendFile(req.params[0], { + root: getAppPathTo(this.appRoot, "static"), + }) + ) + ); this.addOrg(); addPluginEndpoints(this, await this._addPluginManager()); // Serve bundled custom widgets on the plugin endpoint. - const places = getWidgetsInPlugins(this, ''); + const places = getWidgetsInPlugins(this, ""); if (places.length > 0) { // For all widgets served in place, replace any copies of // grist-plugin-api.js with this app's version of it. // This is perhaps a bit rude, but beats the alternative // of either using inconsistent bundled versions, or // requiring network access. - this.app.use(/^\/widgets\/.*\/(grist-plugin-api.js)$/, expressWrap(async (req, res) => - res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); + this.app.use( + /^\/widgets\/.*\/(grist-plugin-api.js)$/, + expressWrap(async (req, res) => + res.sendFile(req.params[0], { + root: getAppPathTo(this.appRoot, "static"), + }) + ) + ); } for (const place of places) { this.app.use( - '/widgets/' + place.pluginId, this.tagChecker.withTag( + "/widgets/" + place.pluginId, + this.tagChecker.withTag( limitToPlugins(this, express.static(place.dir, serveAnyOrigin)) - ) - ); + ) + ); } } // Prepare cache for managing org-to-host relationship. public addHosts() { - if (this._check('hosts', 'homedb')) { return; } + if (this._check("hosts", "homedb")) { + return; + } this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this); } @@ -859,18 +1122,21 @@ export class FlexServer implements GristServer { */ public async hardDeleteDoc(docId: string) { if (!this._internalPermitStore) { - throw new Error('permit store not available'); + throw new Error("permit store not available"); } // In general, documents can only be manipulated with the coordination of the // document worker to which they are assigned. - const permitKey = await this._internalPermitStore.setPermit({docId}); + const permitKey = await this._internalPermitStore.setPermit({ docId }); try { - const result = await fetch(await this.getHomeUrlByDocId(docId, `/api/docs/${docId}`), { - method: 'DELETE', - headers: { - Permit: permitKey + const result = await fetch( + await this.getHomeUrlByDocId(docId, `/api/docs/${docId}`), + { + method: "DELETE", + headers: { + Permit: permitKey, + }, } - }); + ); if (result.status !== 200) { throw new ApiError((await result.json()).error, result.status); } @@ -880,13 +1146,22 @@ export class FlexServer implements GristServer { } public async initHomeDBManager() { - if (this._check('homedb')) { return; } - this._dbManager = new HomeDBManager(this, this._emitNotifier, this._pubSubManager); + if (this._check("homedb")) { + return; + } + this._dbManager = new HomeDBManager( + this, + this._emitNotifier, + this._pubSubManager + ); this._dbManager.setPrefix(process.env.GRIST_ID_PREFIX || ""); await this._dbManager.connect(); await this._dbManager.initializeSpecialIds(); // Report which database we are using, without sensitive credentials. - this.info.push(['database', getDatabaseUrl(this._dbManager.connection.options, false)]); + this.info.push([ + "database", + getDatabaseUrl(this._dbManager.connection.options, false), + ]); // If the installation appears to be new, give it an id and a creation date. this._activations = new ActivationsManager(this._dbManager); await this._activations.current(); @@ -894,47 +1169,70 @@ export class FlexServer implements GristServer { } public addDocWorkerMap() { - if (this._check('map')) { return; } + if (this._check("map")) { + return; + } this._docWorkerMap = getDocWorkerMap(); - this._internalPermitStore = this._docWorkerMap.getPermitStore('internal'); - this._externalPermitStore = this._docWorkerMap.getPermitStore('external'); + this._internalPermitStore = this._docWorkerMap.getPermitStore("internal"); + this._externalPermitStore = this._docWorkerMap.getPermitStore("external"); } // Set up the main express middleware used. For a single user setup, without logins, // all this middleware is currently a no-op. public addAccessMiddleware() { - if (this._check('middleware', 'map', 'loginMiddleware', isSingleUserMode() ? null : 'hosts')) { return; } + if ( + this._check( + "middleware", + "map", + "loginMiddleware", + isSingleUserMode() ? null : "hosts" + ) + ) { + return; + } if (!isSingleUserMode()) { - const skipSession = appSettings.section('login').flag('skipSession').readBool({ - envVar: 'GRIST_IGNORE_SESSION', - }); + const skipSession = appSettings + .section("login") + .flag("skipSession") + .readBool({ + envVar: "GRIST_IGNORE_SESSION", + }); // Middleware to redirect landing pages to preferred host this._redirectToHostMiddleware = this._hosts.redirectHost; // Middleware to add the userId to the express request object. - this._userIdMiddleware = expressWrap(addRequestUser.bind( - null, this._dbManager, this._internalPermitStore, - { - overrideProfile: this._loginMiddleware.overrideProfile?.bind(this._loginMiddleware), - // Set this to false to stop Grist using a cookie for authentication purposes. + this._userIdMiddleware = expressWrap( + addRequestUser.bind(null, this._dbManager, this._internalPermitStore, { + overrideProfile: this._loginMiddleware.overrideProfile?.bind( + this._loginMiddleware + ), + // Set this to false to stop Grist using a cookie for authentication purposes. skipSession, gristServer: this, - } - )); + }) + ); this._trustOriginsMiddleware = expressWrap(trustOriginHandler); // middleware to authorize doc access to the app. Note that this requires the userId // to be set on the request by _userIdMiddleware. - this._docPermissionsMiddleware = expressWrap((...args) => this._docWorker.assertDocAccess(...args)); - this._redirectToLoginWithExceptionsMiddleware = redirectToLogin(true, - this._getLoginRedirectUrl, - this._getSignUpRedirectUrl, - this._dbManager); - this._redirectToLoginWithoutExceptionsMiddleware = redirectToLogin(false, - this._getLoginRedirectUrl, - this._getSignUpRedirectUrl, - this._dbManager); - this._redirectToLoginUnconditionally = redirectToLoginUnconditionally(this._getLoginRedirectUrl, - this._getSignUpRedirectUrl); + this._docPermissionsMiddleware = expressWrap((...args) => + this._docWorker.assertDocAccess(...args) + ); + this._redirectToLoginWithExceptionsMiddleware = redirectToLogin( + true, + this._getLoginRedirectUrl, + this._getSignUpRedirectUrl, + this._dbManager + ); + this._redirectToLoginWithoutExceptionsMiddleware = redirectToLogin( + false, + this._getLoginRedirectUrl, + this._getSignUpRedirectUrl, + this._dbManager + ); + this._redirectToLoginUnconditionally = redirectToLoginUnconditionally( + this._getLoginRedirectUrl, + this._getSignUpRedirectUrl + ); this._redirectToOrgMiddleware = tbind(this._redirectToOrg, this); } else { this._userIdMiddleware = noop; @@ -947,7 +1245,7 @@ export class FlexServer implements GristServer { this._docPermissionsMiddleware = noop; this._redirectToLoginWithExceptionsMiddleware = noop; this._redirectToLoginWithoutExceptionsMiddleware = noop; - this._redirectToLoginUnconditionally = null; // there is no way to log in. + this._redirectToLoginUnconditionally = null; // there is no way to log in. this._redirectToOrgMiddleware = noop; this._redirectToHostMiddleware = noop; } @@ -957,7 +1255,9 @@ export class FlexServer implements GristServer { * Add middleware common to all API endpoints (including forwarding ones). */ public addApiMiddleware() { - if (this._check('api-mw', 'middleware')) { return; } + if (this._check("api-mw", "middleware")) { + return; + } // API endpoints need req.userId and need to support requests from different subdomains. this.app.use("/api", this._userIdMiddleware); this.app.use("/api", this._trustOriginsMiddleware); @@ -968,26 +1268,32 @@ export class FlexServer implements GristServer { * Add error-handling middleware common to all API endpoints. */ public addApiErrorHandlers() { - if (this._check('api-error', 'api-mw')) { return; } + if (this._check("api-error", "api-mw")) { + return; + } // add a final not-found handler for api this.app.use("/api", (req, res) => { - res.status(404).send({error: `not found: ${req.originalUrl}`}); + res.status(404).send({ error: `not found: ${req.originalUrl}` }); }); // Add a final error handler for /api endpoints that reports errors as JSON. - this.app.use('/api/auth', secureJsonErrorHandler); - this.app.use('/api', jsonErrorHandler); + this.app.use("/api/auth", secureJsonErrorHandler); + this.app.use("/api", jsonErrorHandler); } public addWidgetRepository() { - if (this._check('widgets')) { return; } + if (this._check("widgets")) { + return; + } this._widgetRepository = buildWidgetRepository(this); } public addHomeApi() { - if (this._check('api', 'homedb', 'json', 'api-mw')) { return; } + if (this._check("api", "homedb", "json", "api-mw")) { + return; + } // ApiServer's constructor adds endpoints to the app. // tslint:disable-next-line:no-unused-expression @@ -995,26 +1301,31 @@ export class FlexServer implements GristServer { } public addScimApi() { - if (this._check('scim', 'api', 'homedb', 'json', 'api-mw')) { return; } + if (this._check("scim", "api", "homedb", "json", "api-mw")) { + return; + } - const scimRouter = isAffirmative(process.env.GRIST_ENABLE_SCIM) ? - buildScimRouter(this._dbManager, this._installAdmin) : - () => { - throw new ApiError('SCIM API is not enabled', 501); - }; + const scimRouter = isAffirmative(process.env.GRIST_ENABLE_SCIM) + ? buildScimRouter(this._dbManager, this._installAdmin) + : () => { + throw new ApiError("SCIM API is not enabled", 501); + }; - this.app.use('/api/scim', scimRouter); + this.app.use("/api/scim", scimRouter); } - public addBillingApi() { - if (this._check('billing-api', 'homedb', 'json', 'api-mw')) { return; } + if (this._check("billing-api", "homedb", "json", "api-mw")) { + return; + } this.getBilling().addEndpoints(this.app); this.getBilling().addEventHandlers(); } public addBillingMiddleware() { - if (this._check('activation', 'homedb')) { return; } + if (this._check("activation", "homedb")) { + return; + } this.getBilling().addMiddleware?.(this.app); } @@ -1024,11 +1335,13 @@ export class FlexServer implements GristServer { * service for dealing with client errors. */ public addLogEndpoint() { - if (this._check('log-endpoint', 'json', 'api-mw')) { return; } + if (this._check("log-endpoint", "json", "api-mw")) { + return; + } - this.app.post('/api/log', async (req, resp) => { + this.app.post("/api/log", async (req, resp) => { const mreq = req as RequestWithLogin; - log.rawWarn('client error', { + log.rawWarn("client error", { event: req.body.event, docId: req.body.docId, page: req.body.page, @@ -1043,13 +1356,17 @@ export class FlexServer implements GristServer { } public addAuditLogger() { - if (this._check('audit-logger', 'homedb')) { return; } + if (this._check("audit-logger", "homedb")) { + return; + } this._auditLogger = this.create.AuditLogger(this._dbManager, this); } public async addTelemetry() { - if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; } + if (this._check("telemetry", "homedb", "json", "api-mw")) { + return; + } this._telemetry = this.create.Telemetry(this._dbManager, this); this._telemetry.addEndpoints(this.app); @@ -1062,51 +1379,105 @@ export class FlexServer implements GristServer { public async close() { this._processMonitorStop?.(); await this._updateManager?.clear(); - if (this.usage) { await this.usage.close(); } - if (this._hosts) { this._hosts.close(); } + if (this.usage) { + await this.usage.close(); + } + if (this._hosts) { + this._hosts.close(); + } this._emitNotifier.removeAllListeners(); this._dbManager?.clearCaches(); this._installAdmin?.clearCaches(); - if (this.server) { this.server.close(); } - if (this.httpsServer) { this.httpsServer.close(); } - if (this.housekeeper) { await this.housekeeper.stop(); } - if (this._jobs) { await this._jobs.stop(); } + if (this.server) { + this.server.close(); + } + if (this.httpsServer) { + this.httpsServer.close(); + } + if (this.housekeeper) { + await this.housekeeper.stop(); + } + if (this._jobs) { + await this._jobs.stop(); + } await this._shutdown(); - if (this._accessTokens) { await this._accessTokens.close(); } + if (this._accessTokens) { + await this._accessTokens.close(); + } // Do this after _shutdown, since DocWorkerMap is used during shutdown. - if (this._docWorkerMap) { await this._docWorkerMap.close(); } - if (this._sessionStore) { await this._sessionStore.close(); } - if (this._auditLogger) { await this._auditLogger.close(); } - if (this._billing) { await this._billing.close?.(); } + if (this._docWorkerMap) { + await this._docWorkerMap.close(); + } + if (this._sessionStore) { + await this._sessionStore.close(); + } + if (this._auditLogger) { + await this._auditLogger.close(); + } + if (this._billing) { + await this._billing.close?.(); + } await this._pubSubManager.close(); } public addDocApiForwarder() { - if (this._check('doc_api_forwarder', '!json', 'homedb', 'api-mw', 'map')) { return; } - const docApiForwarder = new DocApiForwarder(this._docWorkerMap, this._dbManager, this); + if (this._check("doc_api_forwarder", "!json", "homedb", "api-mw", "map")) { + return; + } + const docApiForwarder = new DocApiForwarder( + this._docWorkerMap, + this._dbManager, + this + ); docApiForwarder.addEndpoints(this.app); } public addJsonSupport() { - if (this._check('json')) { return; } - this.app.use(express.json({limit: '1mb'})); // Increase from the default 100kb + if (this._check("json")) { + return; + } + const limit = this.getApiRequestBodyLimit(); + this.app.use(express.json({ limit })); // Configurable limit, defaulting to 1mb + } + + /** + * Get the API request body limit as a string suitable for Express middleware. + * Uses appSettings for consistent configuration handling. + */ + public getApiRequestBodyLimit(): string { + const limitMB = appSettings + .section("docApi") + .flag("maxRequestBodyMB") + .readInt({ + envVar: "GRIST_MAX_API_REQUEST_BODY_MB", + defaultValue: 1, + minValue: 1, + }); + return `${limitMB}mb`; } public addSessions() { - if (this._check('sessions', 'loginMiddleware')) { return; } + if (this._check("sessions", "loginMiddleware")) { + return; + } this.addTagChecker(); this.addOrg(); // Create the sessionStore and related objects. - const {sessions, sessionMiddleware, sessionStore} = initGristSessions(getUnpackedAppRoot(this.instanceRoot), this); + const { sessions, sessionMiddleware, sessionStore } = initGristSessions( + getUnpackedAppRoot(this.instanceRoot), + this + ); this.app.use(sessionMiddleware); this.app.use(signInStatusMiddleware); // Create an endpoint for making cookies during testing. - this.app.get('/test/session', async (req, res) => { + this.app.get("/test/session", async (req, res) => { const mreq = req as RequestWithLogin; forceSessionChange(mreq.session); - res.status(200).send(`Grist ${this.name} is alive and is interested in you.`); + res + .status(200) + .send(`Grist ${this.name} is alive and is interested in you.`); }); this._sessions = sessions; @@ -1115,9 +1486,9 @@ export class FlexServer implements GristServer { // Close connections and stop accepting new connections. Remove server from any lists // it may be in. - public async stopListening(mode: 'crash'|'clean' = 'clean') { + public async stopListening(mode: "crash" | "clean" = "clean") { if (!this._disabled) { - if (mode === 'clean') { + if (mode === "clean") { await this._shutdown(); this._disabled = true; } else { @@ -1128,17 +1499,22 @@ export class FlexServer implements GristServer { } } this.server.close(); - if (this.httpsServer) { this.httpsServer.close(); } + if (this.httpsServer) { + this.httpsServer.close(); + } } } - public async createWorkerUrl(): Promise<{url: string, host: string}> { + public async createWorkerUrl(): Promise<{ url: string; host: string }> { if (!process.env.GRIST_ROUTER_URL) { - throw new Error('No service available to create worker url'); + throw new Error("No service available to create worker url"); } - const w = await axios.get(process.env.GRIST_ROUTER_URL, - {params: {act: 'add', port: this.getOwnPort()}}); - log.info(`DocWorker registered itself via ${process.env.GRIST_ROUTER_URL} as ${w.data.url}`); + const w = await axios.get(process.env.GRIST_ROUTER_URL, { + params: { act: "add", port: this.getOwnPort() }, + }); + log.info( + `DocWorker registered itself via ${process.env.GRIST_ROUTER_URL} as ${w.data.url}` + ); const statusUrl = `${w.data.url}/status`; // We now wait for the worker to be available from the url that clients will // use to connect to it. This may take some time. The main delay is the @@ -1152,7 +1528,9 @@ export class FlexServer implements GristServer { await axios.get(statusUrl); return w.data; } catch (err) { - log.debug(`While waiting for ${statusUrl} got error ${(err as Error).message}`); + log.debug( + `While waiting for ${statusUrl} got error ${(err as Error).message}` + ); } } throw new Error(`Cannot connect to ${statusUrl}`); @@ -1160,15 +1538,23 @@ export class FlexServer implements GristServer { // Accept new connections again. Add server to any lists it needs to be in to get work. public async restartListening() { - if (!this._docWorkerMap) { throw new Error('expected to have DocWorkerMap'); } - await this.stopListening('clean'); + if (!this._docWorkerMap) { + throw new Error("expected to have DocWorkerMap"); + } + await this.stopListening("clean"); if (this._disabled) { if (this._storageManager) { this._storageManager.testReopenStorage(); } this._comm.setServerActivation(true); if (this.worker) { - await this._startServers(this.server, this.httpsServer, this.name, this.port, false); + await this._startServers( + this.server, + this.httpsServer, + this.name, + this.port, + false + ); await this._addSelfAsWorker(this._docWorkerMap); this._docWorkerLoadTracker?.start(); } @@ -1178,101 +1564,130 @@ export class FlexServer implements GristServer { public async addLandingPages() { // TODO: check if isSingleUserMode() path can be removed from this method - if (this._check('landing', 'map', isSingleUserMode() ? null : 'homedb')) { return; } + if (this._check("landing", "map", isSingleUserMode() ? null : "homedb")) { + return; + } this.addSessions(); // Initialize _sendAppPage helper. this._sendAppPage = makeSendAppPage({ server: this, - staticDir: getAppPathTo(this.appRoot, 'static'), + staticDir: getAppPathTo(this.appRoot, "static"), tag: this.tag, testLogin: isTestLoginAllowed(), baseDomain: this._defaultBaseDomain, }); - const forceLogin = appSettings.section('login').flag('forced').readBool({ - envVar: 'GRIST_FORCE_LOGIN', + const forceLogin = appSettings.section("login").flag("forced").readBool({ + envVar: "GRIST_FORCE_LOGIN", }); - const forcedLoginMiddleware = forceLogin ? this._redirectToLoginWithoutExceptionsMiddleware : noop; - - const welcomeNewUser: express.RequestHandler = isSingleUserMode() ? - (req, res, next) => next() : - expressWrap(async (req, res, next) => { - const mreq = req as RequestWithLogin; - const user = getUser(req); - if (user && user.isFirstTimeUser) { - log.debug(`welcoming user: ${user.name}`); - // Reset isFirstTimeUser flag. - await this._dbManager.updateUser(user.id, {isFirstTimeUser: false}); - - // This is a good time to set some other flags, for showing a page with welcome question(s) - // to this new user and recording their sign-up with Google Tag Manager. These flags are also - // scoped to the user, but isFirstTimeUser has a dedicated DB field because it predates userPrefs. - // Note that the updateOrg() method handles all levels of prefs (for user, user+org, or org). - await this._dbManager.updateOrg(getScope(req), 0, {userPrefs: { - showNewUserQuestions: true, - recordSignUpEvent: true - }}); - - // Give a chance to the login system to react to the first visit after signup. - this._loginMiddleware.onFirstVisit?.(req); - - // If the assistant needs to perform some work (e.g. redirect to a new document with a - // particular prompt pre-filled), do it now. - // - // TODO: break out this and other parts of `welcomeNewUser` into separate Express middleware. - // `onFirstVisit` may send a response, which is why we awkwardly check `headersSent` wasn't - // set before resuming the current middleware. This wouldn't be necessary if `onFirstVisit` - // was a proper Express middleware that called `next` when not sending a response. - if (this._assistant?.version === 2 && this._assistant.onFirstVisit) { - await this._assistant.onFirstVisit(req, res); - if (res.headersSent) { - return; + const forcedLoginMiddleware = forceLogin + ? this._redirectToLoginWithoutExceptionsMiddleware + : noop; + + const welcomeNewUser: express.RequestHandler = isSingleUserMode() + ? (req, res, next) => next() + : expressWrap(async (req, res, next) => { + const mreq = req as RequestWithLogin; + const user = getUser(req); + if (user && user.isFirstTimeUser) { + log.debug(`welcoming user: ${user.name}`); + // Reset isFirstTimeUser flag. + await this._dbManager.updateUser(user.id, { + isFirstTimeUser: false, + }); + + // This is a good time to set some other flags, for showing a page with welcome question(s) + // to this new user and recording their sign-up with Google Tag Manager. These flags are also + // scoped to the user, but isFirstTimeUser has a dedicated DB field because it predates userPrefs. + // Note that the updateOrg() method handles all levels of prefs (for user, user+org, or org). + await this._dbManager.updateOrg(getScope(req), 0, { + userPrefs: { + showNewUserQuestions: true, + recordSignUpEvent: true, + }, + }); + + // Give a chance to the login system to react to the first visit after signup. + this._loginMiddleware.onFirstVisit?.(req); + + // If the assistant needs to perform some work (e.g. redirect to a new document with a + // particular prompt pre-filled), do it now. + // + // TODO: break out this and other parts of `welcomeNewUser` into separate Express middleware. + // `onFirstVisit` may send a response, which is why we awkwardly check `headersSent` wasn't + // set before resuming the current middleware. This wouldn't be necessary if `onFirstVisit` + // was a proper Express middleware that called `next` when not sending a response. + if ( + this._assistant?.version === 2 && + this._assistant.onFirstVisit + ) { + await this._assistant.onFirstVisit(req, res); + if (res.headersSent) { + return; + } } - } - // If we need to copy an unsaved document or template as part of sign-up, do so now - // and redirect to it. - const docId = await this._maybeCopyDocToHomeWorkspace(mreq, res); - if (docId) { - return res.redirect(this.getMergedOrgUrl(mreq, `/doc/${docId}`)); - } + // If we need to copy an unsaved document or template as part of sign-up, do so now + // and redirect to it. + const docId = await this._maybeCopyDocToHomeWorkspace(mreq, res); + if (docId) { + return res.redirect(this.getMergedOrgUrl(mreq, `/doc/${docId}`)); + } - const domain = mreq.org ?? null; - if (!process.env.GRIST_SINGLE_ORG && this._dbManager.isMergedOrg(domain)) { - // We're logging in for the first time on the merged org; if the user has - // access to other team sites, forward the user to a page that lists all - // the teams they have access to. - const result = await this._dbManager.getMergedOrgs(user.id, user.id, domain); - const orgs = this._dbManager.unwrapQueryResult(result); - if (orgs.length > 1 && mreq.path === '/') { - // Only forward if the request is for the home page. - return res.redirect(this.getMergedOrgUrl(mreq, '/welcome/teams')); + const domain = mreq.org ?? null; + if ( + !process.env.GRIST_SINGLE_ORG && + this._dbManager.isMergedOrg(domain) + ) { + // We're logging in for the first time on the merged org; if the user has + // access to other team sites, forward the user to a page that lists all + // the teams they have access to. + const result = await this._dbManager.getMergedOrgs( + user.id, + user.id, + domain + ); + const orgs = this._dbManager.unwrapQueryResult(result); + if (orgs.length > 1 && mreq.path === "/") { + // Only forward if the request is for the home page. + return res.redirect( + this.getMergedOrgUrl(mreq, "/welcome/teams") + ); + } } } - } - if (mreq.org && mreq.org.startsWith('o-')) { - // We are on a team site without a custom subdomain. - const orgInfo = this._dbManager.unwrapQueryResult(await this._dbManager.getOrg({userId: user.id}, mreq.org)); - - // If the user is a billing manager for the org, and the org - // is supposed to have a custom subdomain, forward the user - // to a page to set it. - - // TODO: this is more or less a hack for AppSumo signup flow, - // and could be removed if/when signup flow is revamped. - - // If "welcomeNewUser" is ever added to billing pages, we'd need - // to avoid a redirect loop. - - if (orgInfo.billingAccount.isManager && orgInfo.billingAccount.getFeatures().vanityDomain) { - const prefix = isOrgInPathOnly(req.hostname) ? `/o/${mreq.org}` : ''; - return res.redirect(`${prefix}/billing/payment?billingTask=signUpLite`); + if (mreq.org && mreq.org.startsWith("o-")) { + // We are on a team site without a custom subdomain. + const orgInfo = this._dbManager.unwrapQueryResult( + await this._dbManager.getOrg({ userId: user.id }, mreq.org) + ); + + // If the user is a billing manager for the org, and the org + // is supposed to have a custom subdomain, forward the user + // to a page to set it. + + // TODO: this is more or less a hack for AppSumo signup flow, + // and could be removed if/when signup flow is revamped. + + // If "welcomeNewUser" is ever added to billing pages, we'd need + // to avoid a redirect loop. + + if ( + orgInfo.billingAccount.isManager && + orgInfo.billingAccount.getFeatures().vanityDomain + ) { + const prefix = isOrgInPathOnly(req.hostname) + ? `/o/${mreq.org}` + : ""; + return res.redirect( + `${prefix}/billing/payment?billingTask=signUpLite` + ); + } } - } - next(); - }); + next(); + }); attachAppEndpoint({ app: this.app, @@ -1282,7 +1697,7 @@ export class FlexServer implements GristServer { forcedLoginMiddleware, this._redirectToLoginWithExceptionsMiddleware, this._redirectToOrgMiddleware, - welcomeNewUser + welcomeNewUser, ], docMiddleware: [ // Same as middleware, except without login redirect middleware. @@ -1290,39 +1705,49 @@ export class FlexServer implements GristServer { this._userIdMiddleware, forcedLoginMiddleware, this._redirectToOrgMiddleware, - welcomeNewUser - ], - formMiddleware: [ - this._userIdMiddleware, - forcedLoginMiddleware, + welcomeNewUser, ], + formMiddleware: [this._userIdMiddleware, forcedLoginMiddleware], forceLogin: this._redirectToLoginUnconditionally, docWorkerMap: isSingleUserMode() ? null : this._docWorkerMap, sendAppPage: this._sendAppPage, dbManager: this._dbManager, - plugins : (await this._addPluginManager()).getPlugins(), + plugins: (await this._addPluginManager()).getPlugins(), gristServer: this, }); } public async addLoginMiddleware() { - if (this._check('loginMiddleware')) { return; } + if (this._check("loginMiddleware")) { + return; + } // TODO: We could include a third mock provider of login/logout URLs for better tests. Or we // could create a mock SAML identity provider for testing this using the SAML flow. const loginSystem = await this.resolveLoginSystem(); this._loginMiddleware = await loginSystem.getMiddleware(this); - this._getLoginRedirectUrl = tbind(this._loginMiddleware.getLoginRedirectUrl, this._loginMiddleware); - this._getSignUpRedirectUrl = tbind(this._loginMiddleware.getSignUpRedirectUrl, this._loginMiddleware); - this._getLogoutRedirectUrl = tbind(this._loginMiddleware.getLogoutRedirectUrl, this._loginMiddleware); + this._getLoginRedirectUrl = tbind( + this._loginMiddleware.getLoginRedirectUrl, + this._loginMiddleware + ); + this._getSignUpRedirectUrl = tbind( + this._loginMiddleware.getSignUpRedirectUrl, + this._loginMiddleware + ); + this._getLogoutRedirectUrl = tbind( + this._loginMiddleware.getLogoutRedirectUrl, + this._loginMiddleware + ); const wildcardMiddleware = this._loginMiddleware.getWildcardMiddleware?.(); if (wildcardMiddleware?.length) { - this.app.use(wildcardMiddleware); + this.app.use(wildcardMiddleware); } } public addComm() { - if (this._check('comm', 'start', 'homedb', 'loginMiddleware')) { return; } + if (this._check("comm", "start", "homedb", "loginMiddleware")) { + return; + } this._comm = new Comm(this.server, { settings: {}, sessions: this._sessions, @@ -1338,39 +1763,62 @@ export class FlexServer implements GristServer { * are used by the client libraries. */ public addClientSecrets() { - if (this._check('clientSecret')) { return; } - this.app.get('/client-secret.js', expressWrap(async (req, res) => { - const config = this.getGristConfig(); - // Currently we are exposing only Google keys. - // Those keys are eventually visible by the client, but should be usable - // only from Grist's domains. - const secrets = { - googleClientId: config.googleClientId, - }; - res.set('Content-Type', 'application/javascript'); - res.status(200); - res.send(` + if (this._check("clientSecret")) { + return; + } + this.app.get( + "/client-secret.js", + expressWrap(async (req, res) => { + const config = this.getGristConfig(); + // Currently we are exposing only Google keys. + // Those keys are eventually visible by the client, but should be usable + // only from Grist's domains. + const secrets = { + googleClientId: config.googleClientId, + }; + res.set("Content-Type", "application/javascript"); + res.status(200); + res.send(` window.gristClientSecret = ${JSON.stringify(secrets)} `); - })); + }) + ); } public async addLoginRoutes() { - if (this._check('login', 'org', 'sessions', 'homedb', 'hosts')) { return; } + if (this._check("login", "org", "sessions", "homedb", "hosts")) { + return; + } // TODO: We do NOT want Comm here at all, it's only being used for handling sessions, which // should be factored out of it. this.addComm(); - const signinMiddleware = this._loginMiddleware.getLoginOrSignUpMiddleware ? - this._loginMiddleware.getLoginOrSignUpMiddleware() : - []; - this.app.get('/login', ...signinMiddleware, expressWrap(this._redirectToLoginOrSignup.bind(this, { - signUp: false, - }))); - this.app.get('/signup', ...signinMiddleware, expressWrap(this._redirectToLoginOrSignup.bind(this, { - signUp: true, - }))); - this.app.get('/signin', ...signinMiddleware, expressWrap(this._redirectToLoginOrSignup.bind(this, {}))); + const signinMiddleware = this._loginMiddleware.getLoginOrSignUpMiddleware + ? this._loginMiddleware.getLoginOrSignUpMiddleware() + : []; + this.app.get( + "/login", + ...signinMiddleware, + expressWrap( + this._redirectToLoginOrSignup.bind(this, { + signUp: false, + }) + ) + ); + this.app.get( + "/signup", + ...signinMiddleware, + expressWrap( + this._redirectToLoginOrSignup.bind(this, { + signUp: true, + }) + ) + ); + this.app.get( + "/signin", + ...signinMiddleware, + expressWrap(this._redirectToLoginOrSignup.bind(this, {})) + ); if (isTestLoginAllowed()) { // This is an endpoint for the dev environment that lets you log in as anyone. @@ -1378,29 +1826,43 @@ export class FlexServer implements GristServer { // and localhost:8080/o/<org>/test/login. Only available when GRIST_TEST_LOGIN is set. // Handy when without network connectivity to reach Cognito. - log.warn("Adding a /test/login endpoint because GRIST_TEST_LOGIN is set. " + - "Users will be able to login as anyone."); - - this.app.get('/test/login', expressWrap(async (req, res) => { - log.warn("Serving unauthenticated /test/login endpoint, made available because GRIST_TEST_LOGIN is set."); + log.warn( + "Adding a /test/login endpoint because GRIST_TEST_LOGIN is set. " + + "Users will be able to login as anyone." + ); - // Query parameter is called "username" for compatibility with Cognito. - const email = optStringParam(req.query.username, 'username'); - if (email) { - const redirect = optStringParam(req.query.next, 'next'); - const profile: UserProfile = { - email, - name: optStringParam(req.query.name, 'name') || email, - }; - const url = new URL(redirect || getOrgUrl(req)); - // Make sure we update session for org we'll be redirecting to. - const {org} = await this._hosts.getOrgInfoFromParts(url.hostname, url.pathname); - const scopedSession = this._sessions.getOrCreateSessionFromRequest(req, { org }); - await scopedSession.updateUserProfile(req, profile); - this._sessions.clearCacheIfNeeded({email, org}); - if (redirect) { return res.redirect(redirect); } - } - res.send(`<!doctype html> + this.app.get( + "/test/login", + expressWrap(async (req, res) => { + log.warn( + "Serving unauthenticated /test/login endpoint, made available because GRIST_TEST_LOGIN is set." + ); + + // Query parameter is called "username" for compatibility with Cognito. + const email = optStringParam(req.query.username, "username"); + if (email) { + const redirect = optStringParam(req.query.next, "next"); + const profile: UserProfile = { + email, + name: optStringParam(req.query.name, "name") || email, + }; + const url = new URL(redirect || getOrgUrl(req)); + // Make sure we update session for org we'll be redirecting to. + const { org } = await this._hosts.getOrgInfoFromParts( + url.hostname, + url.pathname + ); + const scopedSession = this._sessions.getOrCreateSessionFromRequest( + req, + { org } + ); + await scopedSession.updateUserProfile(req, profile); + this._sessions.clearCacheIfNeeded({ email, org }); + if (redirect) { + return res.redirect(redirect); + } + } + res.send(`<!doctype html> <html><body> <div class="modal-content-desktop"> <h1>A Very Credulous Login Page</h1> @@ -1412,28 +1874,41 @@ export class FlexServer implements GristServer { <div>Email <input type=text name=username placeholder=email /></div> <div>Name <input type=text name=name placeholder=name /></div> <div>Dummy password <input type=text name=password placeholder=unused ></div> - <input type=hidden name=next value="${req.query.next || ''}"> + <input type=hidden name=next value="${req.query.next || ""}"> <div><input type=submit name=signInSubmitButton value=login></div> </form> </div> </body></html> `); - })); + }) + ); } - this.app.get('/logout', ...this._logoutMiddleware(), expressWrap(async (req, resp) => { - const signedOutUrl = new URL(getOrgUrl(req) + 'signed-out'); - const redirectUrl = await this._getLogoutRedirectUrl(req, signedOutUrl); - resp.redirect(redirectUrl); - })); + this.app.get( + "/logout", + ...this._logoutMiddleware(), + expressWrap(async (req, resp) => { + const signedOutUrl = new URL(getOrgUrl(req) + "signed-out"); + const redirectUrl = await this._getLogoutRedirectUrl(req, signedOutUrl); + resp.redirect(redirectUrl); + }) + ); // Add a static "signed-out" page. This is where logout typically lands (e.g. after redirecting // through SAML). - this.app.get('/signed-out', expressWrap((req, resp) => - this._sendAppPage(req, resp, {path: 'error.html', status: 200, config: {errPage: 'signed-out'}}))); + this.app.get( + "/signed-out", + expressWrap((req, resp) => + this._sendAppPage(req, resp, { + path: "error.html", + status: 200, + config: { errPage: "signed-out" }, + }) + ) + ); const comment = await this._loginMiddleware.addEndpoints(this.app); - this.info.push(['loginMiddlewareComment', comment]); + this.info.push(["loginMiddlewareComment", comment]); addDiscourseConnectEndpoints(this.app, { userIdMiddleware: this._userIdMiddleware, @@ -1442,10 +1917,15 @@ export class FlexServer implements GristServer { } public async addTestingHooks(workerServers?: FlexServer[]) { - this._check('testinghooks', 'comm'); + this._check("testinghooks", "comm"); if (process.env.GRIST_TESTING_SOCKET) { - await startTestingHooks(process.env.GRIST_TESTING_SOCKET, this.port, this._comm, this, - workerServers || []); + await startTestingHooks( + process.env.GRIST_TESTING_SOCKET, + this.port, + this._comm, + this, + workerServers || [] + ); this._hasTestingHooks = true; } } @@ -1462,10 +1942,20 @@ export class FlexServer implements GristServer { // Add document-related endpoints and related support. public async addDoc() { - this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ? - null : 'homedb', 'api-mw', 'map', 'telemetry'); + this._check( + "doc", + "start", + "tag", + "json", + isSingleUserMode() ? null : "homedb", + "api-mw", + "map", + "telemetry" + ); // add handlers for cleanup, if we are in charge of the doc manager. - if (!this._docManager) { this.addCleanup(); } + if (!this._docManager) { + this.addCleanup(); + } await this.addLoginMiddleware(); this.addComm(); // Check SQLite mode so it shows up in initial configuration readout @@ -1475,52 +1965,80 @@ export class FlexServer implements GristServer { await this.create.configure?.(); if (!isSingleUserMode()) { - const externalStorage = appSettings.section('externalStorage'); - const haveExternalStorage = Object.values(externalStorage.nested) - .some(storage => storage.flag('active').getAsBool()); - const disabled = externalStorage.flag('disable') - .read({ envVar: 'GRIST_DISABLE_S3' }).getAsBool(); + const externalStorage = appSettings.section("externalStorage"); + const haveExternalStorage = Object.values(externalStorage.nested).some( + (storage) => storage.flag("active").getAsBool() + ); + const disabled = externalStorage + .flag("disable") + .read({ envVar: "GRIST_DISABLE_S3" }) + .getAsBool(); if (disabled || !haveExternalStorage) { this._disableExternalStorage = true; - externalStorage.flag('active').set(false); + externalStorage.flag("active").set(false); } await this.create.checkBackend?.(); const workers = this._docWorkerMap; const docWorkerId = await this._addSelfAsWorker(workers); const storageManager = await this.create.createHostedDocStorageManager( - this, this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, + this, + this.docsRoot, + docWorkerId, + this._disableExternalStorage, + workers, + this._dbManager, this.create.ExternalStorage.bind(this.create) ); this._storageManager = storageManager; } else { - const samples = getAppPathTo(this.appRoot, 'public_samples'); + const samples = getAppPathTo(this.appRoot, "public_samples"); const storageManager = await this.create.createLocalDocStorageManager( - this.docsRoot, samples, this._comm, undefined, this); + this.docsRoot, + samples, + this._comm, + undefined, + this + ); this._storageManager = storageManager; } const pluginManager = await this._addPluginManager(); - const allStoreOptions = Object.values(this.create.getAttachmentStoreOptions()); - const checkedStoreOptions = await checkAvailabilityAttachmentStoreOptions(allStoreOptions); + const allStoreOptions = Object.values( + this.create.getAttachmentStoreOptions() + ); + const checkedStoreOptions = await checkAvailabilityAttachmentStoreOptions( + allStoreOptions + ); log.info("Attachment store backend availability", { - available: checkedStoreOptions.available.map(option => option.name), - unavailable: checkedStoreOptions.unavailable.map(option => option.name), + available: checkedStoreOptions.available.map((option) => option.name), + unavailable: checkedStoreOptions.unavailable.map((option) => option.name), }); - this._attachmentStoreProvider = this._attachmentStoreProvider || new AttachmentStoreProvider( - await getConfiguredAttachmentStoreConfigs(), - (await this.getActivations().current()).id, - ); - this._docManager = this._docManager || new DocManager(this._storageManager, - pluginManager, - this._dbManager, - this._attachmentStoreProvider, - this); + this._attachmentStoreProvider = + this._attachmentStoreProvider || + new AttachmentStoreProvider( + await getConfiguredAttachmentStoreConfigs(), + (await this.getActivations().current()).id + ); + this._docManager = + this._docManager || + new DocManager( + this._storageManager, + pluginManager, + this._dbManager, + this._attachmentStoreProvider, + this + ); const docManager = this._docManager; - shutdown.addCleanupHandler(null, this._shutdown.bind(this), 25000, 'FlexServer._shutdown'); + shutdown.addCleanupHandler( + null, + this._shutdown.bind(this), + 25000, + "FlexServer._shutdown" + ); if (!isSingleUserMode()) { this._docWorkerLoadTracker = getDocWorkerLoadTracker( @@ -1537,13 +2055,16 @@ export class FlexServer implements GristServer { this._docWorkerLoadTracker.start(); } this._comm.registerMethods({ - openDoc: docManager.openDoc.bind(docManager), + openDoc: docManager.openDoc.bind(docManager), }); this._serveDocPage(); } // Attach docWorker endpoints and Comm methods. - const docWorker = new DocWorker(this._dbManager, {comm: this._comm, gristServer: this}); + const docWorker = new DocWorker(this._dbManager, { + comm: this._comm, + gristServer: this, + }); this._docWorker = docWorker; // Register the websocket comm functions associated with the docworker. @@ -1554,50 +2075,61 @@ export class FlexServer implements GristServer { const docAccessMiddleware = [ this._userIdMiddleware, this._docPermissionsMiddleware, - this.tagChecker.requireTag + this.tagChecker.requireTag, ]; this._addSupportPaths(docAccessMiddleware); if (!isSingleUserMode()) { - addDocApiRoutes(this.app, docWorker, this._docWorkerMap, docManager, this._dbManager, - this._attachmentStoreProvider, this); + addDocApiRoutes( + this.app, + docWorker, + this._docWorkerMap, + docManager, + this._dbManager, + this._attachmentStoreProvider, + this + ); } } public async getSandboxInfo(): Promise<SandboxInfo> { - if (this._sandboxInfo) { return this._sandboxInfo; } + if (this._sandboxInfo) { + return this._sandboxInfo; + } - const flavor = process.env.GRIST_SANDBOX_FLAVOR || 'unknown'; - const info = this._sandboxInfo = { + const flavor = process.env.GRIST_SANDBOX_FLAVOR || "unknown"; + const info = (this._sandboxInfo = { flavor, - configured: flavor !== 'unsandboxed', + configured: flavor !== "unsandboxed", functional: false, effective: false, sandboxed: false, - lastSuccessfulStep: 'none', - } as SandboxInfo; + lastSuccessfulStep: "none", + } as SandboxInfo); // Only meaningful on instances that handle documents. - if (!this._docManager) { return info; } + if (!this._docManager) { + return info; + } try { const sandbox = createSandbox({ server: this, - docId: 'test', // The id is just used in logging - no - // document is created or read at this level. - preferredPythonVersion: '3', + docId: "test", // The id is just used in logging - no + // document is created or read at this level. + preferredPythonVersion: "3", }); info.flavor = sandbox.getFlavor(); - info.configured = info.flavor !== 'unsandboxed'; - info.lastSuccessfulStep = 'create'; - const result = await sandbox.pyCall('get_version'); - if (typeof result !== 'number') { + info.configured = info.flavor !== "unsandboxed"; + info.lastSuccessfulStep = "create"; + const result = await sandbox.pyCall("get_version"); + if (typeof result !== "number") { throw new Error(`Expected a number: ${result}`); } - info.lastSuccessfulStep = 'use'; + info.lastSuccessfulStep = "use"; await sandbox.shutdown(); - info.lastSuccessfulStep = 'all'; + info.lastSuccessfulStep = "all"; info.functional = true; - info.effective = ![ 'skip', 'unsandboxed' ].includes(info.flavor); + info.effective = !["skip", "unsandboxed"].includes(info.flavor); } catch (e) { info.error = String(e); } @@ -1610,8 +2142,8 @@ export class FlexServer implements GristServer { } public disableExternalStorage() { - if (this.deps.has('doc')) { - throw new Error('disableExternalStorage called too late'); + if (this.deps.has("doc")) { + throw new Error("disableExternalStorage called too late"); } this._disableExternalStorage = true; } @@ -1621,7 +2153,7 @@ export class FlexServer implements GristServer { const permitStore = this.getPermitStore(); const notifier = this.getNotifier(); const loginSystem = await this.resolveLoginSystem(); - const homeUrl = this.getHomeInternalUrl().replace(/\/$/, ''); + const homeUrl = this.getHomeInternalUrl().replace(/\/$/, ""); return new Doom(dbManager, permitStore, notifier, loginSystem, homeUrl); } @@ -1629,72 +2161,101 @@ export class FlexServer implements GristServer { const middleware = [ this._redirectToHostMiddleware, this._userIdMiddleware, - this._redirectToLoginWithoutExceptionsMiddleware + this._redirectToLoginWithoutExceptionsMiddleware, ]; - this.app.get('/account', ...middleware, expressWrap(async (req, resp) => { - return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}}); - })); + this.app.get( + "/account", + ...middleware, + expressWrap(async (req, resp) => { + return this._sendAppPage(req, resp, { + path: "app.html", + status: 200, + config: {}, + }); + }) + ); if (isAffirmative(process.env.GRIST_ACCOUNT_CLOSE)) { - this.app.delete('/api/doom/account', expressWrap(async (req, resp) => { - // Make sure we have a valid user authenticated user here. - const userId = getUserId(req); - - // Make sure we are deleting the correct user account (and not the anonymous user) - const requestedUser = integerParam(req.query.userid, 'userid'); - if (requestedUser !== userId || isAnonymousUser(req)) { - // This probably shouldn't happen, but if user has already deleted the account and tries to do it - // once again in a second tab, we might end up here. In that case we are returning false to indicate - // that account wasn't deleted. - return resp.status(200).json(false); - } + this.app.delete( + "/api/doom/account", + expressWrap(async (req, resp) => { + // Make sure we have a valid user authenticated user here. + const userId = getUserId(req); + + // Make sure we are deleting the correct user account (and not the anonymous user) + const requestedUser = integerParam(req.query.userid, "userid"); + if (requestedUser !== userId || isAnonymousUser(req)) { + // This probably shouldn't happen, but if user has already deleted the account and tries to do it + // once again in a second tab, we might end up here. In that case we are returning false to indicate + // that account wasn't deleted. + return resp.status(200).json(false); + } - // We are a valid user, we can proceed with the deletion. Note that we will - // delete user as an admin, as we need to remove other resources that user - // might not have access to. + // We are a valid user, we can proceed with the deletion. Note that we will + // delete user as an admin, as we need to remove other resources that user + // might not have access to. - // Reuse Doom cli tool for account deletion. It won't allow to delete account if it has access - // to other (not public) team sites. - const doom = await this.getDoomTool(); - const {data} = await doom.deleteUser(userId); - if (data) { this._logDeleteUserEvents(req as RequestWithLogin, data); } - return resp.status(200).json(true); - })); + // Reuse Doom cli tool for account deletion. It won't allow to delete account if it has access + // to other (not public) team sites. + const doom = await this.getDoomTool(); + const { data } = await doom.deleteUser(userId); + if (data) { + this._logDeleteUserEvents(req as RequestWithLogin, data); + } + return resp.status(200).json(true); + }) + ); - this.app.get('/account-deleted', ...this._logoutMiddleware(), expressWrap((req, resp) => { - return this._sendAppPage(req, resp, {path: 'error.html', status: 200, config: {errPage: 'account-deleted'}}); - })); + this.app.get( + "/account-deleted", + ...this._logoutMiddleware(), + expressWrap((req, resp) => { + return this._sendAppPage(req, resp, { + path: "error.html", + status: 200, + config: { errPage: "account-deleted" }, + }); + }) + ); - this.app.delete('/api/doom/org', expressWrap(async (req, resp) => { - const mreq = req as RequestWithLogin; - const orgDomain = getOrgFromRequest(req); - if (!orgDomain) { throw new ApiError("Cannot determine organization", 400); } + this.app.delete( + "/api/doom/org", + expressWrap(async (req, resp) => { + const mreq = req as RequestWithLogin; + const orgDomain = getOrgFromRequest(req); + if (!orgDomain) { + throw new ApiError("Cannot determine organization", 400); + } - if (this._dbManager.isMergedOrg(orgDomain)) { - throw new ApiError("Cannot delete a personal site", 400); - } + if (this._dbManager.isMergedOrg(orgDomain)) { + throw new ApiError("Cannot delete a personal site", 400); + } - // Get org from the server. - const query = await this._dbManager.getOrg(getScope(mreq), orgDomain); - const org = this._dbManager.unwrapQueryResult(query); + // Get org from the server. + const query = await this._dbManager.getOrg(getScope(mreq), orgDomain); + const org = this._dbManager.unwrapQueryResult(query); - if (!org || org.ownerId) { - // This shouldn't happen, but just in case test it. - throw new ApiError("Cannot delete an org with an owner", 400); - } + if (!org || org.ownerId) { + // This shouldn't happen, but just in case test it. + throw new ApiError("Cannot delete an org with an owner", 400); + } - if (!org.billingAccount.isManager) { - throw new ApiError("Only billing manager can delete a team site", 403); - } + if (!org.billingAccount.isManager) { + throw new ApiError( + "Only billing manager can delete a team site", + 403 + ); + } - // Reuse Doom cli tool for org deletion. Note, this removes everything as a super user. - const deletedOrg = structuredClone(org); - const doom = await this.getDoomTool(); - await doom.deleteOrg(org.id); - this._logDeleteSiteEvents(mreq, deletedOrg); - return resp.status(200).send(); - })); + // Reuse Doom cli tool for org deletion. Note, this removes everything as a super user. + const deletedOrg = structuredClone(org); + const doom = await this.getDoomTool(); + await doom.deleteOrg(org.id); + this._logDeleteSiteEvents(mreq, deletedOrg); + return resp.status(200).send(); + }) + ); } } @@ -1702,7 +2263,7 @@ export class FlexServer implements GristServer { const middleware = [ this._redirectToHostMiddleware, this._userIdMiddleware, - this._redirectToLoginWithoutExceptionsMiddleware + this._redirectToLoginWithoutExceptionsMiddleware, ]; this.getBilling().addPages(this.app, middleware); @@ -1713,7 +2274,9 @@ export class FlexServer implements GristServer { * we need to get these webhooks in before the bodyParser is added to parse json. */ public addEarlyWebhooks() { - if (this._check('webhooks', 'homedb', '!json')) { return; } + if (this._check("webhooks", "homedb", "!json")) { + return; + } this.getBilling().addWebhooks(this.app); } @@ -1725,9 +2288,17 @@ export class FlexServer implements GristServer { ]; // These are some special-purpose welcome pages, with no middleware. - this.app.get(/\/welcome\/(signup|verify|teams|select-account)/, expressWrap(async (req, resp, next) => { - return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}, googleTagManager: 'anon'}); - })); + this.app.get( + /\/welcome\/(signup|verify|teams|select-account)/, + expressWrap(async (req, resp, next) => { + return this._sendAppPage(req, resp, { + path: "app.html", + status: 200, + config: {}, + googleTagManager: "anon", + }); + }) + ); /** * A nuanced redirecting endpoint. For example, on docs.getgrist.com it does: @@ -1736,18 +2307,23 @@ export class FlexServer implements GristServer { * 3) If logged out but has a cookie -> /login, then 1 or 2 * 4) If entirely unknown -> /signup */ - this.app.get('/welcome/start', [ - this._redirectToHostMiddleware, - this._userIdMiddleware, - ], expressWrap(async (req, resp, next) => { - if (isAnonymousUser(req)) { - return this._redirectToLoginOrSignup({ - nextUrl: new URL(getOrgUrl(req, '/welcome/start')), - }, req, resp); - } + this.app.get( + "/welcome/start", + [this._redirectToHostMiddleware, this._userIdMiddleware], + expressWrap(async (req, resp, next) => { + if (isAnonymousUser(req)) { + return this._redirectToLoginOrSignup( + { + nextUrl: new URL(getOrgUrl(req, "/welcome/start")), + }, + req, + resp + ); + } - await this._redirectToHomeOrWelcomePage(req as RequestWithLogin, resp); - })); + await this._redirectToHomeOrWelcomePage(req as RequestWithLogin, resp); + }) + ); /** * Like /welcome/start, but doesn't redirect anonymous users to sign in. @@ -1760,84 +2336,124 @@ export class FlexServer implements GristServer { * 2) If logged in and has team sites -> https://docs.getgrist.com/welcome/teams * 3) If logged out -> https://docs.getgrist.com/ */ - this.app.get('/welcome/home', [ - this._redirectToHostMiddleware, - this._userIdMiddleware, - ], expressWrap(async (req, resp) => { - const mreq = req as RequestWithLogin; - if (isAnonymousUser(req)) { - return resp.redirect(this.getMergedOrgUrl(mreq)); - } + this.app.get( + "/welcome/home", + [this._redirectToHostMiddleware, this._userIdMiddleware], + expressWrap(async (req, resp) => { + const mreq = req as RequestWithLogin; + if (isAnonymousUser(req)) { + return resp.redirect(this.getMergedOrgUrl(mreq)); + } - await this._redirectToHomeOrWelcomePage(mreq, resp, {redirectToMergedOrg: true}); - })); - - this.app.post('/welcome/info', ...middleware, expressWrap(async (req, resp, next) => { - const userId = getUserId(req); - const user = getUser(req); - const orgName = stringParam(req.body.org_name, 'org_name'); - const orgRole = stringParam(req.body.org_role, 'org_role'); - const useCases = stringArrayParam(req.body.use_cases, 'use_cases'); - const useOther = stringParam(req.body.use_other, 'use_other'); - const row = { - UserID: userId, - Name: user.name, - Email: user.loginEmail, - org_name: orgName, - org_role: orgRole, - use_cases: ['L', ...useCases], - use_other: useOther, - }; - try { - await this._recordNewUserInfo(row); - } catch (e) { - // If we failed to record, at least log the data, so we could potentially recover it. - log.rawWarn(`Failed to record new user info: ${e.message}`, {newUserQuestions: row}); - } - const nonOtherUseCases = useCases.filter(useCase => useCase !== 'Other'); - for (const useCase of [...nonOtherUseCases, ...(useOther ? [`Other - ${useOther}`] : [])]) { - this.getTelemetry().logEvent(req as RequestWithLogin, 'answeredUseCaseQuestion', { - full: { - userId, - useCase, - }, + await this._redirectToHomeOrWelcomePage(mreq, resp, { + redirectToMergedOrg: true, }); - } + }) + ); + + this.app.post( + "/welcome/info", + ...middleware, + expressWrap(async (req, resp, next) => { + const userId = getUserId(req); + const user = getUser(req); + const orgName = stringParam(req.body.org_name, "org_name"); + const orgRole = stringParam(req.body.org_role, "org_role"); + const useCases = stringArrayParam(req.body.use_cases, "use_cases"); + const useOther = stringParam(req.body.use_other, "use_other"); + const row = { + UserID: userId, + Name: user.name, + Email: user.loginEmail, + org_name: orgName, + org_role: orgRole, + use_cases: ["L", ...useCases], + use_other: useOther, + }; + try { + await this._recordNewUserInfo(row); + } catch (e) { + // If we failed to record, at least log the data, so we could potentially recover it. + log.rawWarn(`Failed to record new user info: ${e.message}`, { + newUserQuestions: row, + }); + } + const nonOtherUseCases = useCases.filter( + (useCase) => useCase !== "Other" + ); + for (const useCase of [ + ...nonOtherUseCases, + ...(useOther ? [`Other - ${useOther}`] : []), + ]) { + this.getTelemetry().logEvent( + req as RequestWithLogin, + "answeredUseCaseQuestion", + { + full: { + userId, + useCase, + }, + } + ); + } - resp.status(200).send(); - }), jsonErrorHandler); // Add a final error handler that reports errors as JSON. + resp.status(200).send(); + }), + jsonErrorHandler + ); // Add a final error handler that reports errors as JSON. } public finalizeEndpoints() { this.addApiErrorHandlers(); // add a final non-found handler for other content. - this.app.use("/", expressWrap((req, resp) => { - if (this._sendAppPage) { - return this._sendAppPage(req, resp, {path: 'error.html', status: 404, config: {errPage: 'not-found'}}); - } else { - return resp.status(404).json({error: 'not found'}); - } - })); + this.app.use( + "/", + expressWrap((req, resp) => { + if (this._sendAppPage) { + return this._sendAppPage(req, resp, { + path: "error.html", + status: 404, + config: { errPage: "not-found" }, + }); + } else { + return resp.status(404).json({ error: "not found" }); + } + }) + ); // add a final error handler - this.app.use(async (err: any, req: express.Request, resp: express.Response, next: express.NextFunction) => { - // Delegate to default error handler when headers have already been sent, as express advises - // at https://expressjs.com/en/guide/error-handling.html#the-default-error-handler. - // Also delegates if no _sendAppPage method has been configured. - if (resp.headersSent || !this._sendAppPage) { return next(err); } - try { - const errPage = ( - err.status === 403 ? 'access-denied' : - err.status === 404 ? 'not-found' : - 'other-error' - ); - const config = {errPage, errMessage: err.message || err}; - await this._sendAppPage(req, resp, {path: 'error.html', status: err.status || 400, config}); - } catch (error) { - return next(error); + this.app.use( + async ( + err: any, + req: express.Request, + resp: express.Response, + next: express.NextFunction + ) => { + // Delegate to default error handler when headers have already been sent, as express advises + // at https://expressjs.com/en/guide/error-handling.html#the-default-error-handler. + // Also delegates if no _sendAppPage method has been configured. + if (resp.headersSent || !this._sendAppPage) { + return next(err); + } + try { + const errPage = + err.status === 403 + ? "access-denied" + : err.status === 404 + ? "not-found" + : "other-error"; + const config = { errPage, errMessage: err.message || err }; + await this._sendAppPage(req, resp, { + path: "error.html", + status: err.status || 400, + config, + }); + } catch (error) { + return next(error); + } } - }); + ); } /** @@ -1845,7 +2461,7 @@ export class FlexServer implements GristServer { */ public servesPlugins() { if (this._servesPlugins === undefined) { - throw new Error('do not know if server will serve plugins'); + throw new Error("do not know if server will serve plugins"); } return this._servesPlugins; } @@ -1863,25 +2479,25 @@ export class FlexServer implements GristServer { */ public getPluginUrl() { if (!this._pluginUrlReady) { - throw new Error('looked at plugin url too early'); + throw new Error("looked at plugin url too early"); } return this._pluginUrl; } public getPlugins() { if (!this._pluginManager) { - throw new Error('plugin manager not available'); + throw new Error("plugin manager not available"); } return this._pluginManager.getPlugins(); } - public async finalizePlugins(userPort: number|null) { + public async finalizePlugins(userPort: number | null) { if (isAffirmative(process.env.GRIST_TRUST_PLUGINS)) { this._pluginUrl = this.getDefaultHomeUrl(); } else if (userPort !== null) { // If plugin content is served from same host but on different port, // run webserver on that port - const ports = await this.startCopy('pluginServer', userPort); + const ports = await this.startCopy("pluginServer", userPort); // If Grist is running on a desktop, directly on the host, it // can be convenient to leave the user port free for the OS to // allocate by using GRIST_UNTRUSTED_PORT=0. But we do need to @@ -1892,8 +2508,8 @@ export class FlexServer implements GristServer { this._pluginUrl = url.href; } } - this.info.push(['pluginUrl', this._pluginUrl]); - this.info.push(['willServePlugins', this._servesPlugins]); + this.info.push(["pluginUrl", this._pluginUrl]); + this.info.push(["willServePlugins", this._servesPlugins]); this._pluginUrlReady = true; const repo = buildWidgetRepository(this, { localOnly: true }); this._bundledWidgets = await repo.getWidgets(); @@ -1901,7 +2517,7 @@ export class FlexServer implements GristServer { public getBundledWidgets(): ICustomWidget[] { if (!this._bundledWidgets) { - throw new Error('bundled widgets accessed too early'); + throw new Error("bundled widgets accessed too early"); } return this._bundledWidgets; } @@ -1912,50 +2528,67 @@ export class FlexServer implements GristServer { } for (const item of appSettings.describeAll()) { const txt = - ((item.value !== undefined) ? String(item.value) : '-') + - (item.foundInEnvVar ? ` [${item.foundInEnvVar}]` : '') + - (item.usedDefault ? ' [default]' : '') + - ((item.wouldFindInEnvVar && !item.foundInEnvVar) ? ` [${item.wouldFindInEnvVar}]` : ''); + (item.value !== undefined ? String(item.value) : "-") + + (item.foundInEnvVar ? ` [${item.foundInEnvVar}]` : "") + + (item.usedDefault ? " [default]" : "") + + (item.wouldFindInEnvVar && !item.foundInEnvVar + ? ` [${item.wouldFindInEnvVar}]` + : ""); log.info("== %s: %s", item.name, txt); } } public setReady(value: boolean) { - if(value) { - log.debug('FlexServer is ready'); + if (value) { + log.debug("FlexServer is ready"); } else { - log.debug('FlexServer is no longer ready'); + log.debug("FlexServer is no longer ready"); } this._isReady = value; } public checkOptionCombinations() { // Check for some bad combinations we should warn about. - const allowedWebhookDomains = appSettings.section('integrations').flag('allowedWebhookDomains').readString({ - envVar: 'ALLOWED_WEBHOOK_DOMAINS', - }); - const proxy = appSettings.section('integrations').flag('proxy').readString({ - envVar: 'GRIST_HTTPS_PROXY', + const allowedWebhookDomains = appSettings + .section("integrations") + .flag("allowedWebhookDomains") + .readString({ + envVar: "ALLOWED_WEBHOOK_DOMAINS", + }); + const proxy = appSettings.section("integrations").flag("proxy").readString({ + envVar: "GRIST_HTTPS_PROXY", }); // If all webhook targets are accepted, and no proxy is defined, issue // a warning. This warning can be removed by explicitly setting the proxy // to the empty string. - if (allowedWebhookDomains === '*' && proxy === undefined) { - log.warn("Setting an ALLOWED_WEBHOOK_DOMAINS wildcard without a GRIST_HTTPS_PROXY exposes your internal network"); + if (allowedWebhookDomains === "*" && proxy === undefined) { + log.warn( + "Setting an ALLOWED_WEBHOOK_DOMAINS wildcard without a GRIST_HTTPS_PROXY exposes your internal network" + ); } } public async start() { - if (this._check('start')) { return; } + if (this._check("start")) { + return; + } const servers = this._createServers(); this.server = servers.server; this.httpsServer = servers.httpsServer; - await this._startServers(this.server, this.httpsServer, this.name, this.port, true); + await this._startServers( + this.server, + this.httpsServer, + this.name, + this.port, + true + ); } public addNotifier() { - if (this._check('notifier', 'start', 'homedb')) { return; } + if (this._check("notifier", "start", "homedb")) { + return; + } // TODO: make Notifier aware of base domains, rather than sending emails with default // base domain. // Most notifications are ultimately triggered by requests with a base domain in them, @@ -1972,7 +2605,9 @@ export class FlexServer implements GristServer { } public addAssistant() { - if (this._check('assistant')) { return; } + if (this._check("assistant")) { + return; + } this._assistant = this.create.Assistant(this); if (this._assistant?.version === 2) { this._assistant?.addEndpoints?.(this.app); @@ -1990,26 +2625,35 @@ export class FlexServer implements GristServer { /** * Get a url for a team site. */ - public async getOrgUrl(orgKey: string|number): Promise<string> { + public async getOrgUrl(orgKey: string | number): Promise<string> { const org = await this.getOrg(orgKey); return this.getResourceUrl(org); } - public async getOrg(orgKey: string|number) { - if (!this._dbManager) { throw new Error('database missing'); } - const org = await this._dbManager.getOrg({ - userId: this._dbManager.getPreviewerUserId(), - showAll: true - }, orgKey); + public async getOrg(orgKey: string | number) { + if (!this._dbManager) { + throw new Error("database missing"); + } + const org = await this._dbManager.getOrg( + { + userId: this._dbManager.getPreviewerUserId(), + showAll: true, + }, + orgKey + ); return this._dbManager.unwrapQueryResult(org); } /** * Get a url for an organization, workspace, or document. */ - public async getResourceUrl(resource: Organization|Workspace|Document, - purpose?: 'api'|'html'): Promise<string> { - if (!this._dbManager) { throw new Error('database missing'); } + public async getResourceUrl( + resource: Organization | Workspace | Document, + purpose?: "api" | "html" + ): Promise<string> { + if (!this._dbManager) { + throw new Error("database missing"); + } const gristConfig = this.getGristConfig(); const state: IGristUrlState = {}; let org: Organization; @@ -2023,36 +2667,64 @@ export class FlexServer implements GristServer { state.doc = resource.urlId || resource.id; state.slug = getSlugIfNeeded(resource); } - state.org = this._dbManager.normalizeOrgDomain(org.id, org.domain, org.ownerId); - state.api = purpose === 'api'; - if (!gristConfig.homeUrl) { throw new Error('Computing a resource URL requires a home URL'); } + state.org = this._dbManager.normalizeOrgDomain( + org.id, + org.domain, + org.ownerId + ); + state.api = purpose === "api"; + if (!gristConfig.homeUrl) { + throw new Error("Computing a resource URL requires a home URL"); + } return encodeUrl(gristConfig, state, new URL(gristConfig.homeUrl)); } public addUsage() { - if (this._check('usage', 'start', 'homedb')) { return; } + if (this._check("usage", "start", "homedb")) { + return; + } this.usage = new Usage(this._dbManager); } public async addHousekeeper() { - if (this._check('housekeeper', 'start', 'homedb', 'map', 'json', 'api-mw')) { return; } + if ( + this._check("housekeeper", "start", "homedb", "map", "json", "api-mw") + ) { + return; + } const store = this._docWorkerMap; - this.housekeeper = new Housekeeper(this._dbManager, this, this._internalPermitStore, store); + this.housekeeper = new Housekeeper( + this._dbManager, + this, + this._internalPermitStore, + store + ); this.housekeeper.addEndpoints(this.app); await this.housekeeper.start(); } - public async startCopy(name2: string, port2: number): Promise<{ - serverPort: number, - httpsServerPort?: number, - }>{ + public async startCopy( + name2: string, + port2: number + ): Promise<{ + serverPort: number; + httpsServerPort?: number; + }> { const servers = this._createServers(); - return this._startServers(servers.server, servers.httpsServer, name2, port2, true); + return this._startServers( + servers.server, + servers.httpsServer, + name2, + port2, + true + ); } public addGoogleAuthEndpoint() { - if (this._check('google-auth')) { return; } - const messagePage = makeMessagePage(getAppPathTo(this.appRoot, 'static')); + if (this._check("google-auth")) { + return; + } + const messagePage = makeMessagePage(getAppPathTo(this.appRoot, "static")); addGoogleAuthEndpoint(this.app, messagePage); } @@ -2069,7 +2741,9 @@ export class FlexServer implements GristServer { * handful of endpoints need relaxed parsing (e.g. /configs). */ public addEarlyApi() { - if (this._check('early-api', 'api-mw', 'homedb', '!json')) { return; } + if (this._check("early-api", "api-mw", "homedb", "!json")) { + return; + } attachEarlyEndpoints({ app: this.app, @@ -2080,7 +2754,8 @@ export class FlexServer implements GristServer { public addConfigEndpoints() { // Need to be an admin to change the Grist config - const requireInstallAdmin = this.getInstallAdmin().getMiddlewareRequireAdmin(); + const requireInstallAdmin = + this.getInstallAdmin().getMiddlewareRequireAdmin(); const configBackendAPI = new ConfigBackendAPI(); configBackendAPI.addEndpoints(this.app, requireInstallAdmin); @@ -2093,34 +2768,50 @@ export class FlexServer implements GristServer { return this._latestVersionAvailable; } - public setLatestVersionAvailable(latestVersionAvailable: LatestVersionAvailable): void { - log.info(`Setting ${latestVersionAvailable.version} as the latest available version`); + public setLatestVersionAvailable( + latestVersionAvailable: LatestVersionAvailable + ): void { + log.info( + `Setting ${latestVersionAvailable.version} as the latest available version` + ); this._latestVersionAvailable = latestVersionAvailable; } - public async publishLatestVersionAvailable(latestVersionAvailable: LatestVersionAvailable): Promise<void> { - log.info(`Publishing ${latestVersionAvailable.version} as the latest available version`); + public async publishLatestVersionAvailable( + latestVersionAvailable: LatestVersionAvailable + ): Promise<void> { + log.info( + `Publishing ${latestVersionAvailable.version} as the latest available version` + ); try { - await this.getPubSubManager().publish(latestVersionChannel, JSON.stringify(latestVersionAvailable)); - } catch(error) { - log.error(`Error publishing latest version`, {error, latestVersionAvailable}); + await this.getPubSubManager().publish( + latestVersionChannel, + JSON.stringify(latestVersionAvailable) + ); + } catch (error) { + log.error(`Error publishing latest version`, { + error, + latestVersionAvailable, + }); } } // Get the HTML template sent for document pages. public async getDocTemplate(): Promise<DocTemplate> { - const page = await fse.readFile(path.join(getAppPathTo(this.appRoot, 'static'), - 'app.html'), 'utf8'); + const page = await fse.readFile( + path.join(getAppPathTo(this.appRoot, "static"), "app.html"), + "utf8" + ); return { page, - tag: this.tag + tag: this.tag, }; } public getTag(): string { if (!this.tag) { - throw new Error('getTag called too early'); + throw new Error("getTag called too early"); } return this.tag; } @@ -2145,15 +2836,18 @@ export class FlexServer implements GristServer { } public resolveLoginSystem() { - return isTestLoginAllowed() ? - getTestLoginSystem() : this._getLoginSystem(); + return isTestLoginAllowed() ? getTestLoginSystem() : this._getLoginSystem(); } public addUpdatesCheck() { - if (this._check('update', 'json')) { return; } + if (this._check("update", "json")) { + return; + } // For now we only are active for sass deployments. - if (this._deploymentType !== 'saas') { return; } + if (this._deploymentType !== "saas") { + return; + } this._updateManager = new UpdateManager(this.app, this); this._updateManager.addEndpoints(); @@ -2168,11 +2862,13 @@ export class FlexServer implements GristServer { } public onUserChange(callback: (change: UserChange) => Promise<void>) { - this._emitNotifier.on('userChange', callback); + this._emitNotifier.on("userChange", callback); } - public onStreamingDestinationsChange(callback: (orgId?: number) => Promise<void>) { - this._emitNotifier.on('streamingDestinationsChange', callback); + public onStreamingDestinationsChange( + callback: (orgId?: number) => Promise<void> + ) { + this._emitNotifier.on("streamingDestinationsChange", callback); } public async getSigninUrl( @@ -2183,8 +2879,8 @@ export class FlexServer implements GristServer { params?: Record<string, string | undefined>; } ) { - let {nextUrl, signUp} = options; - const {params = {}} = options; + let { nextUrl, signUp } = options; + const { params = {} } = options; const mreq = req as RequestWithLogin; @@ -2194,15 +2890,17 @@ export class FlexServer implements GristServer { // Redirect to the requested URL after successful login. if (!nextUrl) { - const nextPath = optStringParam(req.query.next, 'next'); + const nextPath = optStringParam(req.query.next, "next"); nextUrl = new URL(getOrgUrl(req, nextPath)); } if (signUp === undefined) { // Like redirectToLogin in Authorizer, redirect to sign up if it doesn't look like the // user has ever logged in on this browser. - signUp = (mreq.session.users === undefined); + signUp = mreq.session.users === undefined; } - const getRedirectUrl = signUp ? this._getSignUpRedirectUrl : this._getLoginRedirectUrl; + const getRedirectUrl = signUp + ? this._getSignUpRedirectUrl + : this._getLoginRedirectUrl; const url = new URL(await getRedirectUrl(req, nextUrl)); for (const [key, value] of Object.entries(params)) { if (value !== undefined) { @@ -2214,24 +2912,42 @@ export class FlexServer implements GristServer { // Adds endpoints that support imports and exports. private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) { - if (!this._docWorker) { throw new Error("need DocWorker"); } + if (!this._docWorker) { + throw new Error("need DocWorker"); + } - const basicMiddleware = [this._userIdMiddleware, this.tagChecker.requireTag]; + const basicMiddleware = [ + this._userIdMiddleware, + this.tagChecker.requireTag, + ]; // Add the handling for the /upload route. Most uploads are meant for a DocWorker: they are put // in temporary files, and the DocWorker needs to be on the same machine to have access to them. // This doesn't check for doc access permissions because the request isn't tied to a document. - addUploadRoute(this, this.app, this._docWorkerMap, this._trustOriginsMiddleware, ...basicMiddleware); + addUploadRoute( + this, + this.app, + this._docWorkerMap, + this._trustOriginsMiddleware, + ...basicMiddleware + ); - this.app.get('/attachment', ...docAccessMiddleware, - expressWrap(async (req, res) => this._docWorker.getAttachment(req, res))); + this.app.get( + "/attachment", + ...docAccessMiddleware, + expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)) + ); } - private _check(part: string, ...precedents: Array<string|null>) { - if (this.deps.has(part)) { return true; } + private _check(part: string, ...precedents: Array<string | null>) { + if (this.deps.has(part)) { + return true; + } for (const precedent of precedents) { - if (!precedent) { continue; } - if (precedent[0] === '!') { + if (!precedent) { + continue; + } + if (precedent[0] === "!") { const antecedent = precedent.slice(1); if (this._has(antecedent)) { throw new Error(`${part} is needed before ${antecedent}`); @@ -2255,7 +2971,6 @@ export class FlexServer implements GristServer { // it always will be. In testing, we may disconnect and reconnect the // worker. We only need to determine docWorkerId and this.worker once. if (!this.worker) { - if (process.env.GRIST_ROUTER_URL) { // register ourselves with the load balancer first. const w = await this.createWorkerUrl(); @@ -2267,15 +2982,17 @@ export class FlexServer implements GristServer { internalUrl: url, }; } else { - const url = (process.env.APP_DOC_URL || this.getOwnUrl()) + `/v/${this.tag}/`; + const url = + (process.env.APP_DOC_URL || this.getOwnUrl()) + `/v/${this.tag}/`; this.worker = { // The worker id should be unique to this worker. - id: process.env.GRIST_DOC_WORKER_ID || `testDocWorkerId_${this.port}`, + id: + process.env.GRIST_DOC_WORKER_ID || `testDocWorkerId_${this.port}`, publicUrl: url, internalUrl: process.env.APP_DOC_INTERNAL_URL || url, }; } - this.info.push(['docWorkerId', this.worker.id]); + this.info.push(["docWorkerId", this.worker.id]); if (process.env.GRIST_WORKER_GROUP) { this.worker.group = process.env.GRIST_WORKER_GROUP; @@ -2294,23 +3011,35 @@ export class FlexServer implements GristServer { return this.worker.id; } - private async _removeSelfAsWorker(workers: IDocWorkerMap, docWorkerId: string) { + private async _removeSelfAsWorker( + workers: IDocWorkerMap, + docWorkerId: string + ) { this._healthy = false; this._docWorkerLoadTracker?.stop(); await workers.removeWorker(docWorkerId); if (process.env.GRIST_ROUTER_URL) { - await axios.get(process.env.GRIST_ROUTER_URL, - {params: {act: 'remove', port: this.getOwnPort()}}); - log.info(`DocWorker unregistered itself via ${process.env.GRIST_ROUTER_URL}`); + await axios.get(process.env.GRIST_ROUTER_URL, { + params: { act: "remove", port: this.getOwnPort() }, + }); + log.info( + `DocWorker unregistered itself via ${process.env.GRIST_ROUTER_URL}` + ); } } // Called when server is shutting down. Save any state that needs saving, and // disentangle ourselves from outside world. private async _shutdown(): Promise<void> { - if (!this.worker) { return; } - if (!this._storageManager) { return; } - if (!this._docWorkerMap) { return; } // but this should never happen. + if (!this.worker) { + return; + } + if (!this._storageManager) { + return; + } + if (!this._docWorkerMap) { + return; + } // but this should never happen. const workers = this._docWorkerMap; @@ -2324,49 +3053,54 @@ export class FlexServer implements GristServer { let assignments = await workers.getAssignments(this.worker.id); let retries: number = 0; while (assignments.length > 0 && retries < 3) { - await Promise.all(assignments.map(async assignment => { - log.info("FlexServer shutdown assignment", assignment); - try { - // Start sending the doc to S3 if needed. - const flushOp = this._storageManager.closeDocument(assignment); - - // Get access to the clients of this document. This has the side - // effect of waiting for the ActiveDoc to finish initialization. - // This could include loading it from S3, an operation we could - // potentially abort as an optimization. - // TODO: abort any s3 loading as an optimization. - const docPromise = this._docManager.getActiveDoc(assignment); - const doc = docPromise && await docPromise; - - await flushOp; - // At this instant, S3 and local document should be the same. - - // We'd now like to make sure (synchronously) that: - // - we never output anything new to S3 about this document. - // - we never output anything new to user about this document. - // There could be asynchronous operations going on related to - // these documents, but if we can make sure that their effects - // do not reach the outside world then we can ignore them. - if (doc) { - doc.docClients.interruptAllClients(); - doc.setMuted(); - } + await Promise.all( + assignments.map(async (assignment) => { + log.info("FlexServer shutdown assignment", assignment); + try { + // Start sending the doc to S3 if needed. + const flushOp = this._storageManager.closeDocument(assignment); + + // Get access to the clients of this document. This has the side + // effect of waiting for the ActiveDoc to finish initialization. + // This could include loading it from S3, an operation we could + // potentially abort as an optimization. + // TODO: abort any s3 loading as an optimization. + const docPromise = this._docManager.getActiveDoc(assignment); + const doc = docPromise && (await docPromise); + + await flushOp; + // At this instant, S3 and local document should be the same. + + // We'd now like to make sure (synchronously) that: + // - we never output anything new to S3 about this document. + // - we never output anything new to user about this document. + // There could be asynchronous operations going on related to + // these documents, but if we can make sure that their effects + // do not reach the outside world then we can ignore them. + if (doc) { + doc.docClients.interruptAllClients(); + doc.setMuted(); + } - // Release this document for other workers to pick up. - // There is a small window of time here in which a client - // could reconnect to us. The muted ActiveDoc will result - // in them being dropped again. - await workers.releaseAssignment(this.worker.id, assignment); - } catch (err) { - log.info("problem dealing with assignment", assignment, err); - } - })); + // Release this document for other workers to pick up. + // There is a small window of time here in which a client + // could reconnect to us. The muted ActiveDoc will result + // in them being dropped again. + await workers.releaseAssignment(this.worker.id, assignment); + } catch (err) { + log.info("problem dealing with assignment", assignment, err); + } + }) + ); // Check for any assignments that slipped through at the last minute. assignments = await workers.getAssignments(this.worker.id); retries++; } if (assignments.length > 0) { - log.error("FlexServer shutdown failed to release assignments:", assignments); + log.error( + "FlexServer shutdown failed to release assignments:", + assignments + ); } await this._removeSelfAsWorker(workers, this.worker.id); @@ -2385,9 +3119,15 @@ export class FlexServer implements GristServer { * Middleware that redirects a request with a userId but without an org to an org-specific URL, * after looking up the first org for this userId in DB. */ - private async _redirectToOrg(req: express.Request, resp: express.Response, next: express.NextFunction) { + private async _redirectToOrg( + req: express.Request, + resp: express.Response, + next: express.NextFunction + ) { const mreq = req as RequestWithLogin; - if (mreq.org || !mreq.userId) { return next(); } + if (mreq.org || !mreq.userId) { + return next(); + } // Redirect anonymous users to the merged org. if (!mreq.userIsAuthorized) { @@ -2399,8 +3139,12 @@ export class FlexServer implements GristServer { // We have a userId, but the request is for an unknown org. Redirect to an org that's // available to the user. This matters in dev, and in prod when visiting a generic URL, which // will here redirect to e.g. the user's personal org. - const result = await this._dbManager.getMergedOrgs(mreq.userId, mreq.userId, null); - const orgs = (result.status === 200) ? result.data : null; + const result = await this._dbManager.getMergedOrgs( + mreq.userId, + mreq.userId, + null + ); + const orgs = result.status === 200 ? result.data : null; const subdomain = orgs && orgs.length > 0 ? orgs[0].domain : null; const redirectUrl = subdomain && this._getOrgRedirectUrl(mreq, subdomain); if (redirectUrl) { @@ -2415,9 +3159,17 @@ export class FlexServer implements GristServer { * subdomain either in the hostname or in the path. Optionally passing pathname overrides url's * path. */ - private _getOrgRedirectUrl(req: RequestWithLogin, subdomain: string, pathname: string = req.originalUrl): string { + private _getOrgRedirectUrl( + req: RequestWithLogin, + subdomain: string, + pathname: string = req.originalUrl + ): string { const config = this.getGristConfig(); - const {hostname, orgInPath} = getOrgUrlInfo(subdomain, req.get('host')!, config); + const { hostname, orgInPath } = getOrgUrlInfo( + subdomain, + req.get("host")!, + config + ); const redirectUrl = new URL(pathname, getOriginUrl(req)); if (hostname) { redirectUrl.hostname = hostname; @@ -2428,13 +3180,16 @@ export class FlexServer implements GristServer { return redirectUrl.href; } - // Create and initialize the plugin manager private async _addPluginManager() { - if (this._pluginManager) { return this._pluginManager; } + if (this._pluginManager) { + return this._pluginManager; + } // Only used as {userRoot}/plugins as a place for plugins in addition to {appRoot}/plugins - const userRoot = path.resolve(process.env.GRIST_USER_ROOT || getAppPathTo(this.appRoot, '.grist')); - this.info.push(['userRoot', userRoot]); + const userRoot = path.resolve( + process.env.GRIST_USER_ROOT || getAppPathTo(this.appRoot, ".grist") + ); + this.info.push(["userRoot", userRoot]); // Some custom widgets may be included as an npm package called @gristlabs/grist-widget. // The package doesn't actually contain node code, but should be in the same vicinity // as other packages that do, so we can use require.resolve on one of them to find it. @@ -2442,12 +3197,18 @@ export class FlexServer implements GristServer { // a larger project like grist-electron. // TODO: maybe add a little node code to @gristlabs/grist-widget so it can be resolved // directly? - const gristLabsModules = path.dirname(path.dirname(require.resolve('@gristlabs/express-session'))); - const bundledRoot = isAffirmative(process.env.GRIST_SKIP_BUNDLED_WIDGETS) ? undefined : path.join( - gristLabsModules, 'grist-widget', 'dist' + const gristLabsModules = path.dirname( + path.dirname(require.resolve("@gristlabs/express-session")) + ); + const bundledRoot = isAffirmative(process.env.GRIST_SKIP_BUNDLED_WIDGETS) + ? undefined + : path.join(gristLabsModules, "grist-widget", "dist"); + this.info.push(["bundledRoot", bundledRoot]); + const pluginManager = new PluginManager( + this.appRoot, + userRoot, + bundledRoot ); - this.info.push(['bundledRoot', bundledRoot]); - const pluginManager = new PluginManager(this.appRoot, userRoot, bundledRoot); // `initialize()` is asynchronous and reads plugins manifests; if PluginManager is used before it // finishes, it will act as if there are no plugins. // ^ I think this comment was here to justify calling initialize without waiting for @@ -2462,17 +3223,27 @@ export class FlexServer implements GristServer { // Serve the static app.html file. // TODO: We should be the ones to fill in the base href here to ensure that the browser fetches // the correct version of static files for this app.html. - this.app.get('/:docId/app.html', this._userIdMiddleware, expressWrap(async (req, res) => { - res.json(await this.getDocTemplate()); - })); + this.app.get( + "/:docId/app.html", + this._userIdMiddleware, + expressWrap(async (req, res) => { + res.json(await this.getDocTemplate()); + }) + ); } // Check whether logger should skip a line. Careful, req and res are morgan-specific // types, not Express. - private _shouldSkipRequestLogging(req: {url: string}, res: {statusCode: number}) { - if (req.url === '/status' && [200, 304].includes(res.statusCode) && - this._healthCheckCounter > HEALTH_CHECK_LOG_SHOW_FIRST_N && - this._healthCheckCounter % HEALTH_CHECK_LOG_SHOW_EVERY_N !== 1) { + private _shouldSkipRequestLogging( + req: { url: string }, + res: { statusCode: number } + ) { + if ( + req.url === "/status" && + [200, 304].includes(res.statusCode) && + this._healthCheckCounter > HEALTH_CHECK_LOG_SHOW_FIRST_N && + this._healthCheckCounter % HEALTH_CHECK_LOG_SHOW_EVERY_N !== 1 + ) { return true; } return false; @@ -2485,30 +3256,56 @@ export class FlexServer implements GristServer { if (TEST_HTTPS_OFFSET) { const certFile = process.env.GRIST_TEST_SSL_CERT; const privateKeyFile = process.env.GRIST_TEST_SSL_KEY; - if (!certFile) { throw new Error('Set GRIST_TEST_SSL_CERT to location of certificate file'); } - if (!privateKeyFile) { throw new Error('Set GRIST_TEST_SSL_KEY to location of private key file'); } + if (!certFile) { + throw new Error( + "Set GRIST_TEST_SSL_CERT to location of certificate file" + ); + } + if (!privateKeyFile) { + throw new Error( + "Set GRIST_TEST_SSL_KEY to location of private key file" + ); + } log.debug(`https support: reading cert from ${certFile}`); log.debug(`https support: reading private key from ${privateKeyFile}`); - httpsServer = logServer(https.createServer({ - ...getServerFlags(), - key: fse.readFileSync(privateKeyFile, 'utf8'), - cert: fse.readFileSync(certFile, 'utf8'), - }, this.app)); + httpsServer = logServer( + https.createServer( + { + ...getServerFlags(), + key: fse.readFileSync(privateKeyFile, "utf8"), + cert: fse.readFileSync(certFile, "utf8"), + }, + this.app + ) + ); } - return {server, httpsServer}; + return { server, httpsServer }; } - private async _startServers(server: http.Server, httpsServer: https.Server|undefined, - name: string, port: number, verbose: boolean) { + private async _startServers( + server: http.Server, + httpsServer: https.Server | undefined, + name: string, + port: number, + verbose: boolean + ) { await listenPromise(server.listen(port, this.host)); const serverPort = (server.address() as AddressInfo).port; - if (verbose) { log.info(`${name} available at ${this.host}:${serverPort}`); } - let httpsServerPort: number|undefined; + if (verbose) { + log.info(`${name} available at ${this.host}:${serverPort}`); + } + let httpsServerPort: number | undefined; if (TEST_HTTPS_OFFSET && httpsServer) { - if (port === 0) { throw new Error('cannot use https with OS-assigned port'); } + if (port === 0) { + throw new Error("cannot use https with OS-assigned port"); + } httpsServerPort = port + TEST_HTTPS_OFFSET; await listenPromise(httpsServer.listen(httpsServerPort, this.host)); - if (verbose) { log.info(`${name} available at https://${this.host}:${httpsServerPort}`); } + if (verbose) { + log.info( + `${name} available at https://${this.host}:${httpsServerPort}` + ); + } } return { serverPort, @@ -2519,11 +3316,13 @@ export class FlexServer implements GristServer { private async _recordNewUserInfo(row: object) { const urlId = DOC_ID_NEW_USER_INFO; // If nowhere to record data, return immediately. - if (!urlId) { return; } - let body: string|undefined; - let permitKey: string|undefined; + if (!urlId) { + return; + } + let body: string | undefined; + let permitKey: string | undefined; try { - body = JSON.stringify(mapValues(row, value => [value])); + body = JSON.stringify(mapValues(row, (value) => [value])); // Take an extra step to translate the special urlId to a docId. This is helpful to // allow the same urlId to be used in production and in test. We need the docId for the @@ -2532,18 +3331,27 @@ export class FlexServer implements GristServer { // TODO With proper forms support, we could give an origin-based permission to submit a // form to this doc, and do it from the client directly. const previewerUserId = this._dbManager.getPreviewerUserId(); - const docAuth = await this._dbManager.getDocAuthCached({urlId, userId: previewerUserId}); + const docAuth = await this._dbManager.getDocAuthCached({ + urlId, + userId: previewerUserId, + }); const docId = docAuth.docId; if (!docId) { throw new Error(`Can't resolve ${urlId}: ${docAuth.error}`); } - permitKey = await this._internalPermitStore.setPermit({docId}); - const res = await fetch(await this.getHomeUrlByDocId(docId, `/api/docs/${docId}/tables/Responses/data`), { - method: 'POST', - headers: {'Permit': permitKey, 'Content-Type': 'application/json'}, - body, - }); + permitKey = await this._internalPermitStore.setPermit({ docId }); + const res = await fetch( + await this.getHomeUrlByDocId( + docId, + `/api/docs/${docId}/tables/Responses/data` + ), + { + method: "POST", + headers: { Permit: permitKey, "Content-Type": "application/json" }, + body, + } + ); if (res.status !== 200) { throw new Error(`API call failed with ${res.status}`); } @@ -2568,7 +3376,8 @@ export class FlexServer implements GristServer { nextUrl?: URL; params?: Record<string, string | undefined>; }, - req: express.Request, resp: express.Response, + req: express.Request, + resp: express.Response ) { const url = await this.getSigninUrl(req, options); resp.redirect(url); @@ -2577,9 +3386,9 @@ export class FlexServer implements GristServer { private async _redirectToHomeOrWelcomePage( mreq: RequestWithLogin, resp: express.Response, - options: {redirectToMergedOrg?: boolean} = {} + options: { redirectToMergedOrg?: boolean } = {} ) { - const {redirectToMergedOrg} = options; + const { redirectToMergedOrg } = options; const userId = getUserId(mreq); const domain = getOrgFromRequest(mreq); const orgs = this._dbManager.unwrapQueryResult( @@ -2588,9 +3397,11 @@ export class FlexServer implements GristServer { }) ); if (orgs.length > 1) { - resp.redirect(getOrgUrl(mreq, '/welcome/teams')); + resp.redirect(getOrgUrl(mreq, "/welcome/teams")); } else { - resp.redirect(redirectToMergedOrg ? this.getMergedOrgUrl(mreq) : getOrgUrl(mreq)); + resp.redirect( + redirectToMergedOrg ? this.getMergedOrgUrl(mreq) : getOrgUrl(mreq) + ); } } @@ -2603,20 +3414,25 @@ export class FlexServer implements GristServer { private async _maybeCopyDocToHomeWorkspace( req: RequestWithLogin, resp: express.Response - ): Promise<string|null> { + ): Promise<string | null> { const state = getAndClearSignupStateCookie(req, resp); if (!state) { return null; } - const {srcDocId} = state; - if (!srcDocId) { return null; } + const { srcDocId } = state; + if (!srcDocId) { + return null; + } let newDocId: string | null = null; try { - newDocId = await createSavedDoc(this, req, {srcDocId}); + newDocId = await createSavedDoc(this, req, { srcDocId }); } catch (e) { - log.error(`FlexServer failed to copy doc ${srcDocId} to Home workspace`, e); + log.error( + `FlexServer failed to copy doc ${srcDocId} to Home workspace`, + e + ); } return newDocId; } @@ -2632,15 +3448,18 @@ export class FlexServer implements GristServer { // SAML logout in theory uses userSession, so clear it AFTER we compute the URL. // Express-session will save these changes. const expressSession = (req as RequestWithLogin).session; - if (expressSession) { expressSession.users = []; expressSession.orgToUser = {}; } + if (expressSession) { + expressSession.users = []; + expressSession.orgToUser = {}; + } await scopedSession.clearScopedSession(req); // TODO: limit cache clearing to specific user. this._sessions.clearCacheIfNeeded(); next(); }); - const pluggedMiddleware = this._loginMiddleware.getLogoutMiddleware ? - this._loginMiddleware.getLogoutMiddleware() : - []; + const pluggedMiddleware = this._loginMiddleware.getLogoutMiddleware + ? this._loginMiddleware.getLogoutMiddleware() + : []; return [...pluggedMiddleware, sessionClearMiddleware]; } @@ -2653,19 +3472,28 @@ export class FlexServer implements GristServer { * and throws an exception if GRIST_LOG_SKIP_HTTP and GRIST_LOG_HTTP are both set to make the server crash. */ private _httpLoggingEnabled(): boolean { - const deprecatedOptionEnablesLog = process.env.GRIST_LOG_SKIP_HTTP === ''; + const deprecatedOptionEnablesLog = process.env.GRIST_LOG_SKIP_HTTP === ""; const isGristLogHttpEnabled = isAffirmative(process.env.GRIST_LOG_HTTP); - if (process.env.GRIST_LOG_HTTP !== undefined && process.env.GRIST_LOG_SKIP_HTTP !== undefined) { - throw new Error('Both GRIST_LOG_HTTP and GRIST_LOG_SKIP_HTTP are set. ' + - 'Please remove GRIST_LOG_SKIP_HTTP and set GRIST_LOG_HTTP to the value you actually want.'); + if ( + process.env.GRIST_LOG_HTTP !== undefined && + process.env.GRIST_LOG_SKIP_HTTP !== undefined + ) { + throw new Error( + "Both GRIST_LOG_HTTP and GRIST_LOG_SKIP_HTTP are set. " + + "Please remove GRIST_LOG_SKIP_HTTP and set GRIST_LOG_HTTP to the value you actually want." + ); } if (process.env.GRIST_LOG_SKIP_HTTP !== undefined) { - const expectedGristLogHttpVal = deprecatedOptionEnablesLog ? "true" : "false"; + const expectedGristLogHttpVal = deprecatedOptionEnablesLog + ? "true" + : "false"; - log.warn(`Setting env variable GRIST_LOG_SKIP_HTTP="${process.env.GRIST_LOG_SKIP_HTTP}" ` - + `is deprecated in favor of GRIST_LOG_HTTP="${expectedGristLogHttpVal}"`); + log.warn( + `Setting env variable GRIST_LOG_SKIP_HTTP="${process.env.GRIST_LOG_SKIP_HTTP}" ` + + `is deprecated in favor of GRIST_LOG_HTTP="${expectedGristLogHttpVal}"` + ); } return isGristLogHttpEnabled || deprecatedOptionEnablesLog; @@ -2743,23 +3571,32 @@ function getServerFlags(): https.ServerOptions { // so imports don't get interrupted too early (but Grist should // probably change how long uploads are done). - const requestTimeoutMs = appSettings.section('server').flag('requestTimeoutMs').requireInt({ - envVar: 'GRIST_REQUEST_TIMEOUT_MS', - defaultValue: 306000, - }); + const requestTimeoutMs = appSettings + .section("server") + .flag("requestTimeoutMs") + .requireInt({ + envVar: "GRIST_REQUEST_TIMEOUT_MS", + defaultValue: 306000, + }); flags.requestTimeout = requestTimeoutMs; - const headersTimeoutMs = appSettings.section('server').flag('headersTimeoutMs').requireInt({ - envVar: 'GRIST_HEADERS_TIMEOUT_MS', - defaultValue: 306000, - }); + const headersTimeoutMs = appSettings + .section("server") + .flag("headersTimeoutMs") + .requireInt({ + envVar: "GRIST_HEADERS_TIMEOUT_MS", + defaultValue: 306000, + }); flags.headersTimeout = headersTimeoutMs; // Likewise keepAlive - const keepAliveTimeoutMs = appSettings.section('server').flag('keepAliveTimeoutMs').requireInt({ - envVar: 'GRIST_KEEP_ALIVE_TIMEOUT_MS', - defaultValue: 305000, - }); + const keepAliveTimeoutMs = appSettings + .section("server") + .flag("keepAliveTimeoutMs") + .requireInt({ + envVar: "GRIST_KEEP_ALIVE_TIMEOUT_MS", + defaultValue: 305000, + }); flags.keepAliveTimeout = keepAliveTimeoutMs; return flags; @@ -2768,9 +3605,13 @@ function getServerFlags(): https.ServerOptions { /** * log some properties of the server. */ -function logServer<T extends https.Server|http.Server>(server: T): T { - log.info("Server timeouts: requestTimeout %s keepAliveTimeout %s headersTimeout %s", - server.requestTimeout, server.keepAliveTimeout, server.headersTimeout); +function logServer<T extends https.Server | http.Server>(server: T): T { + log.info( + "Server timeouts: requestTimeout %s keepAliveTimeout %s headersTimeout %s", + server.requestTimeout, + server.keepAliveTimeout, + server.headersTimeout + ); return server; } @@ -2781,25 +3622,41 @@ function isTestLoginAllowed() { // Check OPTIONS requests for allowed origins, and return heads to allow the browser to proceed // with a POST (or other method) request. -function trustOriginHandler(req: express.Request, res: express.Response, next: express.NextFunction) { - res.header("Access-Control-Allow-Methods", "GET, PATCH, PUT, POST, DELETE, OPTIONS"); +function trustOriginHandler( + req: express.Request, + res: express.Response, + next: express.NextFunction +) { + res.header( + "Access-Control-Allow-Methods", + "GET, PATCH, PUT, POST, DELETE, OPTIONS" + ); if (trustOrigin(req, res)) { res.header("Access-Control-Allow-Credentials", "true"); - res.header("Access-Control-Allow-Headers", "Authorization, Content-Type, X-Requested-With"); + res.header( + "Access-Control-Allow-Headers", + "Authorization, Content-Type, X-Requested-With" + ); } else { // Any origin is allowed, but if it isn't trusted, then we don't allow credentials, // i.e. no Cookie or Authorization header. res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "Content-Type, X-Requested-With"); + res.header( + "Access-Control-Allow-Headers", + "Content-Type, X-Requested-With" + ); if (req.get("Cookie") || req.get("Authorization")) { // In practice we don't expect to actually reach this point, // as the browser should not include credentials in preflight (OPTIONS) requests, // and should block real requests with credentials based on the preflight response. // But having this means not having to rely on our understanding of browsers and CORS too much. - throw new ApiError("Credentials not supported for cross-origin requests", 403); + throw new ApiError( + "Credentials not supported for cross-origin requests", + 403 + ); } } - if ('OPTIONS' === req.method) { + if ("OPTIONS" === req.method) { res.sendStatus(200); } else { next(); @@ -2818,5 +3675,5 @@ export interface ElectronServerMethods { const serveAnyOrigin: serveStatic.ServeStaticOptions = { setHeaders: (res, filepath, stat) => { res.setHeader("Access-Control-Allow-Origin", "*"); - } + }, }; diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts index 5fc58201c7..a4586bea34 100644 --- a/app/server/lib/GristServer.ts +++ b/app/server/lib/GristServer.ts @@ -1,40 +1,53 @@ -import { ICustomWidget } from 'app/common/CustomWidget'; -import { GristDeploymentType, GristLoadConfig, LatestVersionAvailable } from 'app/common/gristUrls'; -import { LocalPlugin } from 'app/common/plugin'; -import { SandboxInfo } from 'app/common/SandboxInfo'; -import { UserProfile } from 'app/common/UserAPI'; -import { Document } from 'app/gen-server/entity/Document'; -import { Organization } from 'app/gen-server/entity/Organization'; -import { User } from 'app/gen-server/entity/User'; -import { Workspace } from 'app/gen-server/entity/Workspace'; -import { ActivationsManager } from 'app/gen-server/lib/ActivationsManager'; -import { Doom } from 'app/gen-server/lib/Doom'; -import { HomeDBManager, UserChange } from 'app/gen-server/lib/homedb/HomeDBManager'; -import { IAccessTokens } from 'app/server/lib/AccessTokens'; -import { RequestWithLogin } from 'app/server/lib/Authorizer'; -import { Comm } from 'app/server/lib/Comm'; -import { create } from 'app/server/lib/create'; -import { DocManager } from 'app/server/lib/DocManager'; -import { Hosts } from 'app/server/lib/extractOrg'; -import { GristJobs } from 'app/server/lib/GristJobs'; -import { IAssistant } from 'app/server/lib/IAssistant'; -import { createNullAuditLogger, IAuditLogger } from 'app/server/lib/IAuditLogger'; -import { IBilling } from 'app/server/lib/IBilling'; -import { ICreate } from 'app/server/lib/ICreate'; -import { IDocStorageManager } from 'app/server/lib/IDocStorageManager'; -import { IDocNotificationManager } from 'app/server/lib/IDocNotificationManager'; -import { INotifier } from 'app/server/lib/INotifier'; -import { InstallAdmin } from 'app/server/lib/InstallAdmin'; -import { IPermitStore } from 'app/server/lib/Permit'; -import { IPubSubManager } from 'app/server/lib/PubSubManager'; -import { ISendAppPageOptions } from 'app/server/lib/sendAppPage'; -import { fromCallback } from 'app/server/lib/serverUtils'; -import { Sessions } from 'app/server/lib/Sessions'; -import { ITelemetry } from 'app/server/lib/Telemetry'; -import { IWidgetRepository } from 'app/server/lib/WidgetRepository'; -import { IGristCoreConfig, loadGristCoreConfig } from "app/server/lib/configCore"; -import * as express from 'express'; -import { IncomingMessage } from 'http'; +import { ICustomWidget } from "app/common/CustomWidget"; +import { + GristDeploymentType, + GristLoadConfig, + LatestVersionAvailable, +} from "app/common/gristUrls"; +import { LocalPlugin } from "app/common/plugin"; +import { SandboxInfo } from "app/common/SandboxInfo"; +import { UserProfile } from "app/common/UserAPI"; +import { Document } from "app/gen-server/entity/Document"; +import { Organization } from "app/gen-server/entity/Organization"; +import { User } from "app/gen-server/entity/User"; +import { Workspace } from "app/gen-server/entity/Workspace"; +import { ActivationsManager } from "app/gen-server/lib/ActivationsManager"; +import { Doom } from "app/gen-server/lib/Doom"; +import { + HomeDBManager, + UserChange, +} from "app/gen-server/lib/homedb/HomeDBManager"; +import { IAccessTokens } from "app/server/lib/AccessTokens"; +import { RequestWithLogin } from "app/server/lib/Authorizer"; +import { Comm } from "app/server/lib/Comm"; +import { create } from "app/server/lib/create"; +import { DocManager } from "app/server/lib/DocManager"; +import { Hosts } from "app/server/lib/extractOrg"; +import { GristJobs } from "app/server/lib/GristJobs"; +import { IAssistant } from "app/server/lib/IAssistant"; +import { + createNullAuditLogger, + IAuditLogger, +} from "app/server/lib/IAuditLogger"; +import { IBilling } from "app/server/lib/IBilling"; +import { ICreate } from "app/server/lib/ICreate"; +import { IDocStorageManager } from "app/server/lib/IDocStorageManager"; +import { IDocNotificationManager } from "app/server/lib/IDocNotificationManager"; +import { INotifier } from "app/server/lib/INotifier"; +import { InstallAdmin } from "app/server/lib/InstallAdmin"; +import { IPermitStore } from "app/server/lib/Permit"; +import { IPubSubManager } from "app/server/lib/PubSubManager"; +import { ISendAppPageOptions } from "app/server/lib/sendAppPage"; +import { fromCallback } from "app/server/lib/serverUtils"; +import { Sessions } from "app/server/lib/Sessions"; +import { ITelemetry } from "app/server/lib/Telemetry"; +import { IWidgetRepository } from "app/server/lib/WidgetRepository"; +import { + IGristCoreConfig, + loadGristCoreConfig, +} from "app/server/lib/configCore"; +import * as express from "express"; +import { IncomingMessage } from "http"; /** * @@ -58,10 +71,12 @@ export interface GristServer extends StorageCoordinator { getHomeInternalUrl(relPath?: string): string; getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>; getOwnUrl(): string; - getOrgUrl(orgKey: string|number): Promise<string>; + getOrgUrl(orgKey: string | number): Promise<string>; getMergedOrgUrl(req: RequestWithLogin, pathname?: string): string; - getResourceUrl(resource: Organization|Workspace|Document, - purpose?: 'api'|'html'): Promise<string>; + getResourceUrl( + resource: Organization | Workspace | Document, + purpose?: "api" | "html" + ): Promise<string>; getGristConfig(): GristLoadConfig; getPermitStore(): IPermitStore; getExternalPermitStore(): IPermitStore; @@ -78,38 +93,52 @@ export interface GristServer extends StorageCoordinator { getWidgetRepository(): IWidgetRepository; hasNotifier(): boolean; getNotifier(): INotifier; - getDocNotificationManager(): IDocNotificationManager|undefined; + getDocNotificationManager(): IDocNotificationManager | undefined; getPubSubManager(): IPubSubManager; - getAssistant(): IAssistant|undefined; + getAssistant(): IAssistant | undefined; getDocTemplate(): Promise<DocTemplate>; getTag(): string; - sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void>; + sendAppPage( + req: express.Request, + resp: express.Response, + options: ISendAppPageOptions + ): Promise<void>; getAccessTokens(): IAccessTokens; resolveLoginSystem(): Promise<GristLoginSystem>; - getPluginUrl(): string|undefined; + getPluginUrl(): string | undefined; getPlugins(): LocalPlugin[]; servesPlugins(): boolean; getBundledWidgets(): ICustomWidget[]; - getBootKey(): string|undefined; + getBootKey(): string | undefined; getSandboxInfo(): Promise<SandboxInfo>; getInfo(key: string): any; getJobs(): GristJobs; getBilling(): IBilling; getDoomTool(): Promise<Doom>; - getLatestVersionAvailable(): LatestVersionAvailable|undefined; - setLatestVersionAvailable(latestVersionAvailable: LatestVersionAvailable): void - publishLatestVersionAvailable(latestVersionAvailable: LatestVersionAvailable): Promise<void>; + getLatestVersionAvailable(): LatestVersionAvailable | undefined; + setLatestVersionAvailable( + latestVersionAvailable: LatestVersionAvailable + ): void; + publishLatestVersionAvailable( + latestVersionAvailable: LatestVersionAvailable + ): Promise<void>; + getApiRequestBodyLimit(): string; setRestrictedMode(restrictedMode?: boolean): void; getDocManager(): DocManager; isRestrictedMode(): boolean; onUserChange(callback: (change: UserChange) => Promise<void>): void; - onStreamingDestinationsChange(callback: (orgId?: number) => Promise<void>): void; + onStreamingDestinationsChange( + callback: (orgId?: number) => Promise<void> + ): void; setReady(value: boolean): void; - getSigninUrl(req: express.Request, options: { - signUp?: boolean; - nextUrl?: URL; - params?: Record<string, string | undefined>; - }): Promise<string>; + getSigninUrl( + req: express.Request, + options: { + signUp?: boolean; + nextUrl?: URL; + params?: Record<string, string | undefined>; + } + ): Promise<string>; } export interface GristLoginSystem { @@ -133,7 +162,9 @@ export interface GristLoginMiddleware { // is identified by a session cookie. When given, overrideProfile() will be called first to // extract the profile from each request. Result can be a profile, or null if anonymous // (sessions will then not be used), or undefined to fall back to using session info. - overrideProfile?(req: express.Request|IncomingMessage): Promise<UserProfile|null|undefined>; + overrideProfile?( + req: express.Request | IncomingMessage + ): Promise<UserProfile | null | undefined>; // Called on first visit to an app page after a signup, for reporting or telemetry purposes. onFirstVisit?(req: express.Request): void; } @@ -141,13 +172,19 @@ export interface GristLoginMiddleware { /** * Set the user in the current session. */ -export async function setUserInSession(req: express.Request, gristServer: GristServer, profile: UserProfile) { - const scopedSession = gristServer.getSessions().getOrCreateSessionFromRequest(req); +export async function setUserInSession( + req: express.Request, + gristServer: GristServer, + profile: UserProfile +) { + const scopedSession = gristServer + .getSessions() + .getOrCreateSessionFromRequest(req); // Make sure session is up to date before operating on it. // Behavior on a completely fresh session is a little awkward currently. const reqSession = (req as any).session; if (reqSession?.save) { - await fromCallback(cb => reqSession.save(cb)); + await fromCallback((cb) => reqSession.save(cb)); } await scopedSession.updateUserProfile(req, profile); } @@ -157,8 +194,8 @@ export interface RequestWithGrist extends express.Request { } export interface DocTemplate { - page: string, - tag: string, + page: string; + tag: string; } /** @@ -169,70 +206,198 @@ export function createDummyGristServer(): GristServer { return { create, settings: loadGristCoreConfig(), - getHost() { return 'localhost:4242'; }, - getHomeUrl() { return 'http://localhost:4242'; }, - getHomeInternalUrl() { return 'http://localhost:4242'; }, - getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); }, - getMergedOrgUrl() { return 'http://localhost:4242'; }, - getOwnUrl() { return 'http://localhost:4242'; }, - getPermitStore() { throw new Error('no permit store'); }, - getExternalPermitStore() { throw new Error('no external permit store'); }, - getGristConfig() { return { homeUrl: '', timestampMs: 0, serveSameOrigin: true, checkForLatestVersion: false }; }, - getOrgUrl() { return Promise.resolve(''); }, - getResourceUrl() { return Promise.resolve(''); }, - getSessions() { throw new Error('no sessions'); }, - getComm() { throw new Error('no comms'); }, - getDeploymentType() { return 'core'; }, - getHosts() { throw new Error('no hosts'); }, - getActivations() { throw new Error('no activations'); }, - getInstallAdmin() { throw new Error('no install admin'); }, - getHomeDBManager() { throw new Error('no db'); }, - getStorageManager() { throw new Error('no storage manager'); }, - getAuditLogger() { return createNullAuditLogger(); }, - getTelemetry() { return createDummyTelemetry(); }, - getWidgetRepository() { throw new Error('no widget repository'); }, - getNotifier() { throw new Error('no notifier'); }, - getDocNotificationManager(): IDocNotificationManager|undefined { return undefined; }, - getPubSubManager(): IPubSubManager { throw new Error('no PubSubManager'); }, - hasNotifier() { return false; }, - getAssistant() { return undefined; }, - getDocTemplate() { throw new Error('no doc template'); }, - getTag() { return 'tag'; }, - sendAppPage() { return Promise.resolve(); }, - getAccessTokens() { throw new Error('no access tokens'); }, - resolveLoginSystem() { throw new Error('no login system'); }, - getPluginUrl() { return undefined; }, - servesPlugins() { return false; }, - getPlugins() { return []; }, - getBundledWidgets() { return []; }, - getBootKey() { return undefined; }, - getSandboxInfo() { throw new Error('no sandbox'); }, - getInfo(key: string) { return undefined; }, - getJobs(): GristJobs { throw new Error('no job system'); }, - getBilling() { throw new Error('no billing'); }, - getDoomTool() { throw new Error('no doom tool'); }, - getLatestVersionAvailable() { throw new Error('no version checking'); }, - setLatestVersionAvailable() { /* do nothing */ }, - publishLatestVersionAvailable() { return Promise.resolve(); }, - setRestrictedMode() { /* do nothing */ }, - getDocManager() { throw new Error('no DocManager'); }, - isRestrictedMode() { return false; }, - onUserChange() { /* do nothing */ }, - onStreamingDestinationsChange() { /* do nothing */ }, - hardDeleteDoc() { return Promise.resolve(); }, - setReady() { /* do nothing */ }, - getSigninUrl() { return Promise.resolve(''); }, + getHost() { + return "localhost:4242"; + }, + getHomeUrl() { + return "http://localhost:4242"; + }, + getHomeInternalUrl() { + return "http://localhost:4242"; + }, + getHomeUrlByDocId() { + return Promise.resolve("http://localhost:4242"); + }, + getMergedOrgUrl() { + return "http://localhost:4242"; + }, + getOwnUrl() { + return "http://localhost:4242"; + }, + getPermitStore() { + throw new Error("no permit store"); + }, + getExternalPermitStore() { + throw new Error("no external permit store"); + }, + getGristConfig() { + return { + homeUrl: "", + timestampMs: 0, + serveSameOrigin: true, + checkForLatestVersion: false, + }; + }, + getOrgUrl() { + return Promise.resolve(""); + }, + getResourceUrl() { + return Promise.resolve(""); + }, + getSessions() { + throw new Error("no sessions"); + }, + getComm() { + throw new Error("no comms"); + }, + getDeploymentType() { + return "core"; + }, + getHosts() { + throw new Error("no hosts"); + }, + getActivations() { + throw new Error("no activations"); + }, + getInstallAdmin() { + throw new Error("no install admin"); + }, + getHomeDBManager() { + throw new Error("no db"); + }, + getStorageManager() { + throw new Error("no storage manager"); + }, + getAuditLogger() { + return createNullAuditLogger(); + }, + getTelemetry() { + return createDummyTelemetry(); + }, + getWidgetRepository() { + throw new Error("no widget repository"); + }, + getNotifier() { + throw new Error("no notifier"); + }, + getDocNotificationManager(): IDocNotificationManager | undefined { + return undefined; + }, + getPubSubManager(): IPubSubManager { + throw new Error("no PubSubManager"); + }, + hasNotifier() { + return false; + }, + getAssistant() { + return undefined; + }, + getDocTemplate() { + throw new Error("no doc template"); + }, + getTag() { + return "tag"; + }, + sendAppPage() { + return Promise.resolve(); + }, + getAccessTokens() { + throw new Error("no access tokens"); + }, + resolveLoginSystem() { + throw new Error("no login system"); + }, + getPluginUrl() { + return undefined; + }, + servesPlugins() { + return false; + }, + getPlugins() { + return []; + }, + getBundledWidgets() { + return []; + }, + getBootKey() { + return undefined; + }, + getSandboxInfo() { + throw new Error("no sandbox"); + }, + getInfo(key: string) { + return undefined; + }, + getJobs(): GristJobs { + throw new Error("no job system"); + }, + getBilling() { + throw new Error("no billing"); + }, + getDoomTool() { + throw new Error("no doom tool"); + }, + getLatestVersionAvailable() { + throw new Error("no version checking"); + }, + setLatestVersionAvailable() { + /* do nothing */ + }, + publishLatestVersionAvailable() { + return Promise.resolve(); + }, + getApiRequestBodyLimit() { + return "1mb"; + }, + setRestrictedMode() { + /* do nothing */ + }, + getDocManager() { + throw new Error("no DocManager"); + }, + isRestrictedMode() { + return false; + }, + onUserChange() { + /* do nothing */ + }, + onStreamingDestinationsChange() { + /* do nothing */ + }, + hardDeleteDoc() { + return Promise.resolve(); + }, + setReady() { + /* do nothing */ + }, + getSigninUrl() { + return Promise.resolve(""); + }, }; } export function createDummyTelemetry(): ITelemetry { return { - addEndpoints() { /* do nothing */ }, - start() { return Promise.resolve(); }, - logEvent() { /* do nothing */ }, - logEventAsync() { return Promise.resolve(); }, - shouldLogEvent() { return false; }, - getTelemetryConfig() { return undefined; }, - fetchTelemetryPrefs() { return Promise.resolve(); }, + addEndpoints() { + /* do nothing */ + }, + start() { + return Promise.resolve(); + }, + logEvent() { + /* do nothing */ + }, + logEventAsync() { + return Promise.resolve(); + }, + shouldLogEvent() { + return false; + }, + getTelemetryConfig() { + return undefined; + }, + fetchTelemetryPrefs() { + return Promise.resolve(); + }, }; } diff --git a/app/server/lib/attachEarlyEndpoints.ts b/app/server/lib/attachEarlyEndpoints.ts index fb433b8fa0..b62efbd8ef 100644 --- a/app/server/lib/attachEarlyEndpoints.ts +++ b/app/server/lib/attachEarlyEndpoints.ts @@ -68,7 +68,9 @@ export function attachEarlyEndpoints(options: AttachOptions) { return gristServer.sendAppPage(req, res, { path: "app.html", status: 200, - config: {adminControls: gristServer.create.areAdminControlsAvailable()}, + config: { + adminControls: gristServer.create.areAdminControlsAvailable(), + }, }); }) ); @@ -89,7 +91,7 @@ export function attachEarlyEndpoints(options: AttachOptions) { expressWrap(async (req, res) => { const mreq = req as RequestWithLogin; const meta = { - host: mreq.get('host'), + host: mreq.get("host"), path: mreq.path, email: mreq.user?.loginEmail, }; @@ -100,7 +102,10 @@ export function attachEarlyEndpoints(options: AttachOptions) { // can restart us. log.rawDebug(`Restart[${mreq.method}] finishing:`, meta); if (process.send && process.env.GRIST_RUNNING_UNDER_SUPERVISOR) { - log.rawDebug(`Restart[${mreq.method}] requesting supervisor to restart home server:`, meta); + log.rawDebug( + `Restart[${mreq.method}] requesting supervisor to restart home server:`, + meta + ); process.send({ action: "restart" }); } }); @@ -124,8 +129,8 @@ export function attachEarlyEndpoints(options: AttachOptions) { expressWrap(async (_req, res) => { const activation = await gristServer.getActivations().current(); const telemetryPrefs = await getTelemetryPrefs( - gristServer.getHomeDBManager(), - activation + gristServer.getHomeDBManager(), + activation ); return sendOkReply(null, res, { telemetry: telemetryPrefs, @@ -136,7 +141,7 @@ export function attachEarlyEndpoints(options: AttachOptions) { app.patch( "/api/install/prefs", - json({ limit: "1mb" }), + json({ limit: gristServer.getApiRequestBodyLimit() }), expressWrap(async (req, res) => { const props = { prefs: req.body }; const activation = await gristServer.getActivations().current(); @@ -160,7 +165,10 @@ export function attachEarlyEndpoints(options: AttachOptions) { "/api/install/updates", expressWrap(async (_req, res) => { try { - const updateData = await updateGristServerLatestVersion(gristServer, true); + const updateData = await updateGristServerLatestVersion( + gristServer, + true + ); res.json(updateData); } catch (error) { res.status(error.status); @@ -188,7 +196,7 @@ export function attachEarlyEndpoints(options: AttachOptions) { app.put( "/api/install/configs/:key", - json({ limit: "1mb", strict: false }), + json({ limit: gristServer.getApiRequestBodyLimit(), strict: false }), hasValidConfig, expressWrap(async (req, res) => { const key = stringParam(req.params.key, "key") as ConfigKey; @@ -235,7 +243,7 @@ export function attachEarlyEndpoints(options: AttachOptions) { app.put( "/api/orgs/:oid/configs/:key", - json({ limit: "1mb", strict: false }), + json({ limit: gristServer.getApiRequestBodyLimit(), strict: false }), hasValidConfig, expressWrap(async (req, res) => { const key = stringParam(req.params.key, "key") as ConfigKey; diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts index 64159dbfa8..7fd877397c 100644 --- a/app/server/lib/sendAppPage.ts +++ b/app/server/lib/sendAppPage.ts @@ -1,4 +1,4 @@ -import {AssistantConfig} from 'app/common/Assistant'; +import { AssistantConfig } from "app/common/Assistant"; import { commonUrls, Features, @@ -11,30 +11,37 @@ import { getTermsOfServiceUrl, getWebinarsUrl, GristLoadConfig, - IFeature -} from 'app/common/gristUrls'; -import {isAffirmative} from 'app/common/gutil'; -import {getTagManagerSnippet} from 'app/common/tagManager'; -import {Document} from 'app/common/UserAPI'; -import {AttachedCustomWidgets, IAttachedCustomWidget} from "app/common/widgetTypes"; -import {appSettings} from "app/server/lib/AppSettings"; -import {SUPPORT_EMAIL} from 'app/gen-server/lib/homedb/HomeDBManager'; -import {isAnonymousUser, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer'; -import {RequestWithOrg} from 'app/server/lib/extractOrg'; -import {GristServer} from 'app/server/lib/GristServer'; + IFeature, +} from "app/common/gristUrls"; +import { isAffirmative } from "app/common/gutil"; +import { getTagManagerSnippet } from "app/common/tagManager"; +import { Document } from "app/common/UserAPI"; +import { + AttachedCustomWidgets, + IAttachedCustomWidget, +} from "app/common/widgetTypes"; +import { appSettings } from "app/server/lib/AppSettings"; +import { SUPPORT_EMAIL } from "app/gen-server/lib/homedb/HomeDBManager"; +import { + isAnonymousUser, + isSingleUserMode, + RequestWithLogin, +} from "app/server/lib/Authorizer"; +import { RequestWithOrg } from "app/server/lib/extractOrg"; +import { GristServer } from "app/server/lib/GristServer"; import { getOnboardingTutorialDocId, getTemplateOrg, - getUserPresenceMaxUsers -} from 'app/server/lib/gristSettings'; -import {getSupportedEngineChoices} from 'app/server/lib/serverUtils'; -import {readLoadedLngs, readLoadedNamespaces} from 'app/server/localization'; -import * as express from 'express'; -import * as fse from 'fs-extra'; -import * as handlebars from 'handlebars'; -import jsesc from 'jsesc'; -import * as path from 'path'; -import difference from 'lodash/difference'; + getUserPresenceMaxUsers, +} from "app/server/lib/gristSettings"; +import { getSupportedEngineChoices } from "app/server/lib/serverUtils"; +import { readLoadedLngs, readLoadedNamespaces } from "app/server/localization"; +import * as express from "express"; +import * as fse from "fs-extra"; +import * as handlebars from "handlebars"; +import jsesc from "jsesc"; +import * as path from "path"; +import difference from "lodash/difference"; const { escapeExpression } = handlebars.Utils; @@ -45,52 +52,74 @@ const { escapeExpression } = handlebars.Utils; * @param key The key of the translation (which will be prefixed by `sendAppPage`) * @param args The args to pass to the translation string (optional) */ -const translate = (req: express.Request, key: string, args?: any) => req.t(`sendAppPage.${key}`, args)?.toString(); +const translate = (req: express.Request, key: string, args?: any) => + req.t(`sendAppPage.${key}`, args)?.toString(); -const GRIST_FEATURE_FORM_FRAMING = appSettings.section("features").flag("formFraming") +const GRIST_FEATURE_FORM_FRAMING = appSettings + .section("features") + .flag("formFraming") .requireString({ - envVar: 'GRIST_FEATURE_FORM_FRAMING', - defaultValue: 'border', - acceptedValues: ['border', 'minimal'], + envVar: "GRIST_FEATURE_FORM_FRAMING", + defaultValue: "border", + acceptedValues: ["border", "minimal"], }); +/** + * Get the maximum API request body size in bytes using appSettings. + * Returns undefined if not set to maintain the same behavior as before. + */ +function getMaxApiRequestBodySize(): number | undefined { + const limitMB = appSettings + .section("docApi") + .flag("maxRequestBodyMB") + .readInt({ + envVar: "GRIST_MAX_API_REQUEST_BODY_MB", + minValue: 1, + }); + return limitMB ? limitMB * 1024 * 1024 : undefined; +} + export interface ISendAppPageOptions { - path: string; // Ignored if .content is present (set to "" for clarity). + path: string; // Ignored if .content is present (set to "" for clarity). content?: string; status: number; config: Partial<GristLoadConfig>; - tag?: string; // If present, override version tag. + tag?: string; // If present, override version tag. // If present, enable Google Tag Manager on this page (if GOOGLE_TAG_MANAGER_ID env var is set). // Used on the welcome page to track sign-ups. We don't intend to use it for in-app analytics. // Set to true to insert tracker unconditionally; false to omit it; "anon" to insert // it only when the user is not logged in. - googleTagManager?: true | false | 'anon'; + googleTagManager?: true | false | "anon"; } export interface MakeGristConfigOptions { - homeUrl: string|null; + homeUrl: string | null; extra: Partial<GristLoadConfig>; baseDomain?: string; req?: express.Request; - server?: GristServer|null; + server?: GristServer | null; } -export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfig { - const {homeUrl, extra, baseDomain, req, server} = options; +export function makeGristConfig( + options: MakeGristConfigOptions +): GristLoadConfig { + const { homeUrl, extra, baseDomain, req, server } = options; // .invalid is a TLD the IETF promises will never exist. - const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid'; - const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") || - (homeUrl && new URL(homeUrl).hostname === 'localhost') || false; - const mreq = req as RequestWithOrg|undefined; + const pluginUrl = process.env.APP_UNTRUSTED_URL || "http://plugins.invalid"; + const pathOnly = + process.env.GRIST_ORG_IN_PATH === "true" || + (homeUrl && new URL(homeUrl).hostname === "localhost") || + false; + const mreq = req as RequestWithOrg | undefined; - // Configure form framing behavior. + // Configure form framing behavior. return { homeUrl, org: process.env.GRIST_SINGLE_ORG || (mreq && mreq.org), baseDomain, - // True if no subdomains or separate servers are defined for the home servers or doc workers. + // True if no subdomains or separate servers are defined for the home servers or doc workers. serveSameOrigin: !baseDomain && pathOnly, singleOrg: process.env.GRIST_SINGLE_ORG, helpCenterUrl: getHelpCenterUrl(), @@ -101,7 +130,9 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi contactSupportUrl: getContactSupportUrl(), pathOnly, supportAnon: shouldSupportAnon(), - enableAnonPlayground: isAffirmative(process.env.GRIST_ANON_PLAYGROUND ?? true), + enableAnonPlayground: isAffirmative( + process.env.GRIST_ANON_PLAYGROUND ?? true + ), supportEngines: getSupportedEngineChoices(), features: getFeatures(), pageTitleSuffix: configuredPageTitleSuffix(), @@ -110,16 +141,23 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi googleClientId: process.env.GOOGLE_CLIENT_ID, googleDriveScope: process.env.GOOGLE_DRIVE_SCOPE, helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID_V2, - maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined, - maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined, + maxUploadSizeImport: + Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024 || undefined, + maxUploadSizeAttachment: + Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024 || + undefined, + maxApiRequestBodySize: getMaxApiRequestBodySize(), timestampMs: Date.now(), - enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL) || - ((server?.getBundledWidgets().length || 0) > 0), + enableWidgetRepository: + Boolean(process.env.GRIST_WIDGET_LIST_URL) || + (server?.getBundledWidgets().length || 0) > 0, survey: Boolean(process.env.DOC_ID_NEW_USER_INFO), tagManagerId: process.env.GOOGLE_TAG_MANAGER_ID, - activation: (req as RequestWithLogin|undefined)?.activation, + activation: (req as RequestWithLogin | undefined)?.activation, latestVersionAvailable: server?.getLatestVersionAvailable(), - automaticVersionCheckingAllowed: isAffirmative(process.env.GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING), + automaticVersionCheckingAllowed: isAffirmative( + process.env.GRIST_ALLOW_AUTOMATIC_VERSION_CHECKING + ), enableCustomCss: isAffirmative(process.env.APP_STATIC_INCLUDE_CUSTOM_CSS), supportedLngs: readLoadedLngs(req?.i18n), namespaces: readLoadedNamespaces(req?.i18n), @@ -127,9 +165,13 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi permittedCustomWidgets: getPermittedCustomWidgets(server), supportEmail: SUPPORT_EMAIL, userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale, - telemetry: server?.getTelemetry().getTelemetryConfig(req as RequestWithLogin | undefined), + telemetry: server + ?.getTelemetry() + .getTelemetryConfig(req as RequestWithLogin | undefined), deploymentType: server?.getDeploymentType(), - forceEnableEnterprise: isAffirmative(process.env.GRIST_FORCE_ENABLE_ENTERPRISE), + forceEnableEnterprise: isAffirmative( + process.env.GRIST_FORCE_ENABLE_ENTERPRISE + ), templateOrg: getTemplateOrg(), onboardingTutorialDocId: getOnboardingTutorialDocId(), canCloseAccount: isAffirmative(process.env.GRIST_ACCOUNT_CLOSE), @@ -149,32 +191,55 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi */ export function makeMessagePage(staticDir: string) { return async (req: express.Request, resp: express.Response, message: any) => { - const fileContent = await fse.readFile(path.join(staticDir, "message.html"), 'utf8'); + const fileContent = await fse.readFile( + path.join(staticDir, "message.html"), + "utf8" + ); const content = fileContent.replace( "<!-- INSERT MESSAGE -->", - `<script>window.message = ${jsesc(message, {isScriptContext: true, json: true})};</script>` + `<script>window.message = ${jsesc(message, { + isScriptContext: true, + json: true, + })};</script>` ); - resp.status(200).type('html').send(content); + resp.status(200).type("html").send(content); }; } -export type SendAppPageFunction = - (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>; +export type SendAppPageFunction = ( + req: express.Request, + resp: express.Response, + options: ISendAppPageOptions +) => Promise<void>; /** * Send a simple template page, read from file at pagePath (relative to static/), with certain * placeholders replaced. */ -export function makeSendAppPage({ server, staticDir, tag, testLogin, baseDomain }: { - server: GristServer, staticDir: string, tag: string, testLogin?: boolean, baseDomain?: string +export function makeSendAppPage({ + server, + staticDir, + tag, + testLogin, + baseDomain, +}: { + server: GristServer; + staticDir: string; + tag: string; + testLogin?: boolean; + baseDomain?: string; }): SendAppPageFunction { - // If env var GRIST_INCLUDE_CUSTOM_SCRIPT_URL is set, load it in a <script> tag on all app pages. const customScriptUrl = process.env.GRIST_INCLUDE_CUSTOM_SCRIPT_URL; - const insertCustomScript: string = customScriptUrl ? - `<script src="${customScriptUrl}" crossorigin="anonymous"></script>` : ''; - - return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => { + const insertCustomScript: string = customScriptUrl + ? `<script src="${customScriptUrl}" crossorigin="anonymous"></script>` + : ""; + + return async ( + req: express.Request, + resp: express.Response, + options: ISendAppPageOptions + ) => { const config = makeGristConfig({ homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null, extra: options.config, @@ -185,46 +250,70 @@ export function makeSendAppPage({ server, staticDir, tag, testLogin, baseDomain // We could cache file contents in memory, but the filesystem does caching too, and compared // to that, the performance gain is unlikely to be meaningful. So keep it simple here. - const fileContent = options.content || await fse.readFile(path.join(staticDir, options.path), 'utf8'); + const fileContent = + options.content || + (await fse.readFile(path.join(staticDir, options.path), "utf8")); - const needTagManager = (options.googleTagManager === 'anon' && isAnonymousUser(req)) || + const needTagManager = + (options.googleTagManager === "anon" && isAnonymousUser(req)) || options.googleTagManager === true; - const tagManagerSnippet = needTagManager ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) : ''; + const tagManagerSnippet = needTagManager + ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) + : ""; const staticTag = options.tag || tag; // If boot tag is used, serve assets locally, otherwise respect // APP_STATIC_URL. - const staticOrigin = staticTag === 'boot' ? '' : (process.env.APP_STATIC_URL || ''); + const staticOrigin = + staticTag === "boot" ? "" : process.env.APP_STATIC_URL || ""; const staticBaseUrl = `${staticOrigin}/v/${staticTag}/`; const customHeadHtmlSnippet = server.create.getExtraHeadHtml?.() ?? ""; - const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : ""; + const warning = testLogin + ? '<div class="dev_warning">Authentication is not enforced</div>' + : ""; // Preload all languages that will be used or are requested by client. const preloads = req.languages - .filter(lng => (readLoadedLngs(req.i18n)).includes(lng)) - .map(lng => lng.replace('-', '_')) + .filter((lng) => readLoadedLngs(req.i18n).includes(lng)) + .map((lng) => lng.replace("-", "_")) .map((lng) => - readLoadedNamespaces(req.i18n).map((ns) => - `<link rel="preload" href="locales/${lng}.${ns}.json" as="fetch" type="application/json" crossorigin>` - ).join("\n") - ).join('\n'); + readLoadedNamespaces(req.i18n) + .map( + (ns) => + `<link rel="preload" href="locales/${lng}.${ns}.json" as="fetch" type="application/json" crossorigin>` + ) + .join("\n") + ) + .join("\n"); const content = fileContent .replace("<!-- INSERT WARNING -->", warning) - .replace("<!-- INSERT TITLE -->", getDocName(config) ?? escapeExpression(translate(req, 'Loading...'))) + .replace( + "<!-- INSERT TITLE -->", + getDocName(config) ?? escapeExpression(translate(req, "Loading...")) + ) .replace("<!-- INSERT META -->", getPageMetadataHtmlSnippet(req, config)) - .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server.getGristConfig())) - .replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet) + .replace( + "<!-- INSERT TITLE SUFFIX -->", + getPageTitleSuffix(server.getGristConfig()) + ) + .replace( + "<!-- INSERT BASE -->", + `<base href="${staticBaseUrl}">` + tagManagerSnippet + ) .replace("<!-- INSERT LOCALE -->", preloads) .replace("<!-- INSERT CUSTOM -->", customHeadHtmlSnippet) .replace("<!-- INSERT CUSTOM SCRIPT -->", insertCustomScript) .replace( "<!-- INSERT CONFIG -->", - `<script>window.gristConfig = ${jsesc(config, {isScriptContext: true, json: true})};</script>` + `<script>window.gristConfig = ${jsesc(config, { + isScriptContext: true, + json: true, + })};</script>` ); logVisitedPageTelemetryEvent(req as RequestWithLogin, { server, pagePath: options.path, docId: config.assignmentId, }); - resp.status(options.status).type('html').send(content); + resp.status(options.status).type("html").send(content); }; } @@ -234,25 +323,28 @@ interface LogVisitedPageEventOptions { docId?: string; } -function logVisitedPageTelemetryEvent(req: RequestWithLogin, options: LogVisitedPageEventOptions) { - const {server, pagePath, docId} = options; +function logVisitedPageTelemetryEvent( + req: RequestWithLogin, + options: LogVisitedPageEventOptions +) { + const { server, pagePath, docId } = options; // Construct a fake URL and append the utm_* parameters from the original URL. // We avoid using the original URL here because it may contain sensitive identifiers, // such as link key parameters and site/doc ids. - const url = new URL('fake', server.getMergedOrgUrl(req)); + const url = new URL("fake", server.getMergedOrgUrl(req)); for (const [key, value] of Object.entries(req.query)) { - if (key.startsWith('utm_')) { + if (key.startsWith("utm_")) { url.searchParams.set(key, String(value)); } } - server.getTelemetry().logEvent(req, 'visitedPage', { + server.getTelemetry().logEvent(req, "visitedPage", { full: { docIdDigest: docId, url: url.toString(), path: pagePath, - userAgent: req.headers['user-agent'], + userAgent: req.headers["user-agent"], userId: req.userId, altSessionId: req.altSessionId, }, @@ -265,22 +357,27 @@ function shouldSupportAnon() { } function getFeatures(): IFeature[] { - const disabledFeatures = process.env.GRIST_HIDE_UI_ELEMENTS?.split(',') ?? []; - const enabledFeatures = process.env.GRIST_UI_FEATURES?.split(',') ?? Features.values; + const disabledFeatures = process.env.GRIST_HIDE_UI_ELEMENTS?.split(",") ?? []; + const enabledFeatures = + process.env.GRIST_UI_FEATURES?.split(",") ?? Features.values; return Features.checkAll(difference(enabledFeatures, disabledFeatures)); } -function getAssistantConfig(gristServer?: GristServer|null): AssistantConfig|undefined { +function getAssistantConfig( + gristServer?: GristServer | null +): AssistantConfig | undefined { const assistant = gristServer?.getAssistant(); if (!assistant) { return undefined; } - const {provider, version} = assistant; - return {provider, version}; + const { provider, version } = assistant; + return { provider, version }; } -function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCustomWidget[] { +function getPermittedCustomWidgets( + gristServer?: GristServer | null +): IAttachedCustomWidget[] { if (!process.env.PERMITTED_CUSTOM_WIDGETS && gristServer) { // The PERMITTED_CUSTOM_WIDGETS environment variable is a bit of // a drag. If there are bundled widgets that overlap with widgets @@ -291,14 +388,17 @@ function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCus for (const widget of widgets) { // Permitted custom widgets are identified so many ways across the // code! Why? TODO: cut down on identifiers. - const name = widget.widgetId.replace('@gristlabs/widget-', 'custom.'); + const name = widget.widgetId.replace("@gristlabs/widget-", "custom."); if (names.has(name)) { namesFound.push(name as IAttachedCustomWidget); } } return AttachedCustomWidgets.checkAll(namesFound); } - const widgetsList = process.env.PERMITTED_CUSTOM_WIDGETS?.split(',').map(widgetName=>`custom.${widgetName}`) ?? []; + const widgetsList = + process.env.PERMITTED_CUSTOM_WIDGETS?.split(",").map( + (widgetName) => `custom.${widgetName}` + ) ?? []; return AttachedCustomWidgets.checkAll(widgetsList); } @@ -313,7 +413,7 @@ function configuredPageTitleSuffix() { * Note: The string returned is escaped and safe to insert into HTML. * */ -function getDocName(config: GristLoadConfig): string|null { +function getDocName(config: GristLoadConfig): string | null { const maybeDoc = getDocFromConfig(config); return maybeDoc && escapeExpression(maybeDoc.name); @@ -328,7 +428,10 @@ function getDocName(config: GristLoadConfig): string|null { * * Note: The string returned is escaped and safe to insert into HTML. */ -function getPageMetadataHtmlSnippet(req: express.Request, config: GristLoadConfig): string { +function getPageMetadataHtmlSnippet( + req: express.Request, + config: GristLoadConfig +): string { const metadataElements: string[] = []; const maybeDoc = getDocFromConfig(config); @@ -337,32 +440,46 @@ function getPageMetadataHtmlSnippet(req: express.Request, config: GristLoadConfi } metadataElements.push('<meta property="og:type" content="website">'); - metadataElements.push('<meta name="twitter:card" content="summary_large_image">'); + metadataElements.push( + '<meta name="twitter:card" content="summary_large_image">' + ); - const description = maybeDoc?.options?.description ? - escapeExpression(maybeDoc.options.description) : - translate(req, 'og-description'); + const description = maybeDoc?.options?.description + ? escapeExpression(maybeDoc.options.description) + : translate(req, "og-description"); metadataElements.push(`<meta name="description" content="${description}">`); - metadataElements.push(`<meta property="og:description" content="${description}">`); - metadataElements.push(`<meta name="twitter:description" content="${description}">`); - - const openGraphPreviewImage = process.env.GRIST_OPEN_GRAPH_PREVIEW_IMAGE || commonUrls.openGraphPreviewImage; - const image = escapeExpression(maybeDoc?.options?.icon ?? openGraphPreviewImage); + metadataElements.push( + `<meta property="og:description" content="${description}">` + ); + metadataElements.push( + `<meta name="twitter:description" content="${description}">` + ); + + const openGraphPreviewImage = + process.env.GRIST_OPEN_GRAPH_PREVIEW_IMAGE || + commonUrls.openGraphPreviewImage; + const image = escapeExpression( + maybeDoc?.options?.icon ?? openGraphPreviewImage + ); metadataElements.push(`<meta name="thumbnail" content="${image}">`); metadataElements.push(`<meta property="og:image" content="${image}">`); metadataElements.push(`<meta name="twitter:image" content="${image}">`); const maybeDocTitle = getDocName(config); - const title = maybeDocTitle ? maybeDocTitle + getPageTitleSuffix(config) : translate(req, 'og-title'); + const title = maybeDocTitle + ? maybeDocTitle + getPageTitleSuffix(config) + : translate(req, "og-title"); // NB: We don't generate the content of the <title> tag here. metadataElements.push(`<meta property="og:title" content="${title}">`); metadataElements.push(`<meta name="twitter:title" content="${title}">`); - return metadataElements.join('\n'); + return metadataElements.join("\n"); } function getDocFromConfig(config: GristLoadConfig): Document | null { - if (!config.getDoc || !config.assignmentId) { return null; } + if (!config.getDoc || !config.assignmentId) { + return null; + } return config.getDoc[config.assignmentId] ?? null; } diff --git a/test/nbrowser/testServer.ts b/test/nbrowser/testServer.ts index 8d16e5958a..68c3690431 100644 --- a/test/nbrowser/testServer.ts +++ b/test/nbrowser/testServer.ts @@ -119,6 +119,7 @@ export class TestServerMerged extends EventEmitter implements IMochaServer { // Set low limits for uploads, for testing. GRIST_MAX_UPLOAD_IMPORT_MB: '1', GRIST_MAX_UPLOAD_ATTACHMENT_MB: '2', + GRIST_MAX_API_REQUEST_BODY_MB: '1', // The following line only matters for testing with non-localhost URLs, which some tests do. GRIST_SERVE_SAME_ORIGIN: 'true', // Run with HOME_PORT, STATIC_PORT, DOC_PORT, DOC_WORKER_COUNT in the environment to override.