Skip to content

Commit 916a663

Browse files
authored
chore: Prepare for the switch of backend plugins to dynamic plugins (redhat-developer#601)
* Add `/dynamic-plugins` to the monorepo Signed-off-by: David Festal <[email protected]> * Allow duplicate plugin-specific app-config values if they are equal. Signed-off-by: David Festal <[email protected]> * Use a `dynamic-plugins.default.yaml` file in the container image if nothing is specified in the Helm chart. Signed-off-by: David Festal <[email protected]> * Securely support links in dynamic plugin archives Signed-off-by: David Festal <[email protected]> * Support optional in the main index.ts file * Skip .eslintrc.js files in lint-stage Signed-off-by: David Festal <[email protected]> * Add changeset Signed-off-by: David Festal <[email protected]> * Fox review comment Signed-off-by: David Festal <[email protected]> * Fix new review comment Signed-off-by: David Festal <[email protected]> --------- Signed-off-by: David Festal <[email protected]>
1 parent 4af2b85 commit 916a663

16 files changed

+175
-44
lines changed

.changeset/thin-frogs-jog.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'backend': patch
3+
'app': patch
4+
---
5+
6+
Prepare the showcase application for the switch of most plugins from static to dynamic loading.

.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,8 @@ site
5252
# Cypress
5353
**/cypress/downloads
5454
**/cypress/screenshots
55+
56+
# Dynamic plugins root content
57+
dynamic-plugins-root/*
58+
!dynamic-plugins-root/.gitkeep
59+
dynamic-plugins/*/dist-dynamic/src

app-config.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -339,3 +339,6 @@ enabled:
339339
permission: ${PERMISSION_ENABLED}
340340
metrics: ${METRICS_ENABLED}
341341
aap: ${AAP_ENABLED}
342+
343+
dynamicPlugins:
344+
rootDirectory: dynamic-plugins-root

docker/Dockerfile

+18-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ RUN chmod +x $YARN
4646
# Stage 2 - Install dependencies
4747
FROM skeleton AS deps
4848

49+
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/
4950
COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./
5051
COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json
5152
COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json
@@ -66,7 +67,18 @@ RUN rm app-config.yaml && mv app-config.example.yaml app-config.yaml
6667
RUN $YARN build --filter=backend
6768

6869
# Build dynamic plugins
69-
RUN $YARN --cwd ./dynamic-plugins export-dynamic
70+
RUN $YARN export-dynamic
71+
RUN $YARN clean-dynamic-sources
72+
RUN mkdir -p dynamic-plugins-root && \
73+
cd dynamic-plugins-root && \
74+
rm -Rf * && \
75+
for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \
76+
if [ -d $pkg ]; then \
77+
archive=$(npm pack $pkg) && \
78+
tar -xzf "$archive" && rm "$archive" && \
79+
mv package $(echo $archive | sed -e 's:\.tgz$::'); \
80+
fi; \
81+
done
7082

7183
# Stage 4 - Build the actual backend image and install production dependencies
7284
FROM skeleton AS cleanup
@@ -84,6 +96,7 @@ RUN tar xzf $TARBALL_PATH/skeleton.tar.gz; tar xzf $TARBALL_PATH/bundle.tar.gz;
8496
# Copy app-config files needed in runtime
8597
# Upstream only
8698
COPY $EXTERNAL_SOURCE_NESTED/app-config*.yaml ./
99+
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins.default.yaml ./
87100

88101
# Install production dependencies
89102
# hadolint ignore=DL3059
@@ -118,6 +131,10 @@ RUN chmod a+r ./install-dynamic-plugins.py
118131
COPY --from=build $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
119132
RUN chmod -R a+r ./dynamic-plugins/
120133

134+
# Copy default dynamic plugins root
135+
COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/
136+
RUN chmod -R a+r ./dynamic-plugins-root/
137+
121138
# The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
122139
# The upstream backstage image does not account for this and it causes the container to fail at runtime.
123140
RUN fix-permissions ./

docker/brew.Dockerfile

+18-3
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ COPY $EXTERNAL_SOURCE_NESTED/.yarnrc.yml ./
4646
RUN chmod +x $YARN
4747

4848
# Stage 2 - Install dependencies
49+
COPY $EXTERNAL_SOURCE_NESTED/dynamic-plugins/ ./dynamic-plugins/
4950
COPY $EXTERNAL_SOURCE_NESTED/package.json $EXTERNAL_SOURCE_NESTED/yarn.lock ./
5051
COPY $EXTERNAL_SOURCE_NESTED/packages/app/package.json ./packages/app/package.json
5152
COPY $EXTERNAL_SOURCE_NESTED/packages/backend/package.json ./packages/backend/package.json
@@ -108,8 +109,18 @@ RUN git config --global --add safe.directory ./
108109
RUN $YARN build --filter=backend
109110

110111
# Build dynamic plugins
111-
# hadolint ignore=DL3059
112-
RUN $YARN --cwd ./dynamic-plugins export-dynamic
112+
RUN $YARN export-dynamic
113+
RUN $YARN clean-dynamic-sources
114+
RUN mkdir -p dynamic-plugins-root && \
115+
cd dynamic-plugins-root && \
116+
rm -Rf * && \
117+
for pkg in $CONTAINER_SOURCE/dynamic-plugins/*/dist-dynamic; do \
118+
if [ -d $pkg ]; then \
119+
archive=$(npm pack $pkg) && \
120+
tar -xzf "$archive" && rm "$archive" && \
121+
mv package $(echo $archive | sed -e 's:\.tgz$::'); \
122+
fi; \
123+
done
113124

114125
# Stage 4 - Build the actual backend image and install production dependencies
115126

@@ -161,7 +172,7 @@ RUN microdnf update -y && \
161172
pip3.11 install --user --no-cache-dir -r requirements.txt -r requirements-build.txt; \
162173
popd >/dev/null; \
163174
microdnf clean all; rm -fr $CONTAINER_SOURCE/upstream2
164-
175+
165176
# Downstream only - copy from builder, not cleanup stage
166177
COPY --from=builder --chown=1001:1001 $CONTAINER_SOURCE/ ./
167178

@@ -173,6 +184,10 @@ RUN chmod a+r ./install-dynamic-plugins.py
173184
COPY --from=builder $CONTAINER_SOURCE/dynamic-plugins/ ./dynamic-plugins/
174185
RUN chmod -R a+r ./dynamic-plugins/
175186

187+
# Copy default dynamic plugins root
188+
COPY --from=build $CONTAINER_SOURCE/dynamic-plugins-root/ ./dynamic-plugins-root/
189+
RUN chmod -R a+r ./dynamic-plugins-root/
190+
176191
# The fix-permissions script is important when operating in environments that dynamically use a random UID at runtime, such as OpenShift.
177192
# The upstream backstage image does not account for this and it causes the container to fail at runtime.
178193
RUN fix-permissions ./

docker/install-dynamic-plugins.py

+75-13
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,61 @@
1+
#
2+
# Copyright (c) 2023 Red Hat, Inc.
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
116
import os
217
import sys
318
import yaml
419
import tarfile
520
import shutil
621
import subprocess
722

23+
# This script is used to install dynamic plugins in the Backstage application,
24+
# and is available in the container image to be called at container initialization,
25+
# for example in an init container when using Kubernetes.
26+
#
27+
# It expects, as the only argument, the path to the root directory where
28+
# the dynamic plugins will be installed.
29+
#
30+
# Additionally The MAX_ENTRY_SIZE environment variable can be defined to set
31+
# the maximum size of a file in the archive (default: 10MB).
32+
#
33+
# It expects the `dynamic-plugins.yaml` file to be present in the current directory and
34+
# to contain the list of plugins to install along with their optional configuration.
35+
#
36+
# The `dynamic-plugins.yaml` file must be a list of objects with the following properties:
37+
# - `package`: the NPM package to install (either a package name or a path to a local package)
38+
# - `pluginConfig`: an optional plugin-specific configuration fragment
39+
#
40+
# For each package mentioned in the `dynamic-plugins.yaml` file, the script will:
41+
# - call `npm pack` to get the package archive and extract it in the dynamic plugins root directory
42+
# - merge the plugin-specific configuration fragment in a global configuration file named `app-config.dynamic-plugins.yaml`
43+
#
44+
845
class InstallException(Exception):
946
"""Exception class from which every exception in this library will derive."""
1047
pass
1148

12-
def merge(source, destination):
49+
def merge(source, destination, prefix = ''):
1350
for key, value in source.items():
1451
if isinstance(value, dict):
1552
# get node or create one
1653
node = destination.setdefault(key, {})
17-
merge(value, node)
54+
merge(value, node, key + '.')
1855
else:
1956
# if key exists in destination trigger an error
20-
if key in destination:
21-
raise InstallException('Config key ' + key + ' defined for 2 dynamic plugins')
57+
if key in destination and destination[key] != value:
58+
raise InstallException("Config key '" + prefix + key + "' defined differently for 2 dynamic plugins")
2259

2360
destination[key] = value
2461

@@ -28,16 +65,20 @@ def main():
2865
dynamicPluginsRoot = sys.argv[1]
2966
maxEntrySize = int(os.environ.get('MAX_ENTRY_SIZE', 10000000))
3067

31-
dynamicPluginsFile = os.path.join(dynamicPluginsRoot, 'dynamic-plugins.yaml')
68+
dynamicPluginsFile = 'dynamic-plugins.yaml'
69+
dynamicPluginsDefaultFile = 'dynamic-plugins.default.yaml'
3270
dynamicPluginsGlobalConfigFile = os.path.join(dynamicPluginsRoot, 'app-config.dynamic-plugins.yaml')
3371

3472
# test if file dynamic-plugins.yaml exists
3573
if not os.path.isfile(dynamicPluginsFile):
36-
print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.')
37-
with open(dynamicPluginsGlobalConfigFile, 'w') as file:
38-
file.write('')
39-
file.close()
40-
exit(0)
74+
print(f'No {dynamicPluginsFile} file found, trying {dynamicPluginsDefaultFile} file.')
75+
dynamicPluginsFile = dynamicPluginsDefaultFile
76+
if not os.path.isfile(dynamicPluginsFile):
77+
print(f'No {dynamicPluginsFile} file found. Skipping dynamic plugins installation.')
78+
with open(dynamicPluginsGlobalConfigFile, 'w') as file:
79+
file.write('')
80+
file.close()
81+
exit(0)
4182

4283
with open(dynamicPluginsFile, 'r') as file:
4384
plugins = yaml.safe_load(file)
@@ -74,6 +115,7 @@ def main():
74115

75116
archive = os.path.join(dynamicPluginsRoot, completed.stdout.decode('utf-8').strip())
76117
directory = archive.replace('.tgz', '')
118+
directoryRealpath = os.path.realpath(directory)
77119

78120
print('\t==> Removing previous plugin directory', directory, flush=True)
79121
shutil.rmtree(directory, ignore_errors=True, onerror=None)
@@ -90,13 +132,33 @@ def main():
90132
if member.size > maxEntrySize:
91133
raise InstallException('Zip bomb detected in ' + member.name)
92134

93-
# Remove the `package/` prefix from the file name
94-
member.name = member.name[8:]
135+
member.name = member.name.removeprefix('package/')
95136
file.extract(member, path=directory)
96137
elif member.isdir():
97138
print('\t\tSkipping directory entry', member.name, flush=True)
139+
elif member.islnk() or member.issym():
140+
if not member.linkpath.startswith('package/'):
141+
raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath)
142+
143+
member.name = member.name.removeprefix('package/')
144+
member.linkpath = member.linkpath.removeprefix('package/')
145+
146+
realpath = os.path.realpath(os.path.join(directory, *os.path.split(member.linkname)))
147+
if not realpath.startswith(directoryRealpath):
148+
raise InstallException('NPM package archive contains a link outside of the archive: ' + member.name + ' -> ' + member.linkpath)
149+
150+
file.extract(member, path=directory)
98151
else:
99-
raise InstallException('NPM package archive contains a non regular file: ' + member.name)
152+
if member.type == tarfile.CHRTYPE:
153+
type_str = "character device"
154+
elif member.type == tarfile.BLKTYPE:
155+
type_str = "block device"
156+
elif member.type == tarfile.FIFOTYPE:
157+
type_str = "FIFO"
158+
else:
159+
type_str = "unknown"
160+
161+
raise InstallException('NPM package archive contains a non regular file: ' + member.name + ' - ' + type_str)
100162

101163
file.close()
102164

dynamic-plugins-root/.gitkeep

Whitespace-only changes.

dynamic-plugins.default.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- package: ./dynamic-plugins/scaffolder-backend-module-utils-wrapped/dist-dynamic

dynamic-plugins/.eslintignore

-4
This file was deleted.

dynamic-plugins/.eslintrc

-6
This file was deleted.

dynamic-plugins/.gitkeep

Whitespace-only changes.

dynamic-plugins/package.json

-6
This file was deleted.

package.json

+6-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
"start-backend": "turbo run start --filter=backend",
1313
"build": "turbo run build",
1414
"tsc": "tsc",
15+
"export-dynamic": "turbo run export-dynamic --concurrency 1",
16+
"clean-dynamic-sources": "turbo run clean-dynamic-sources",
1517
"clean": "turbo run clean",
1618
"test": "turbo run test",
1719
"test:e2e": "turbo run test:e2e",
@@ -29,7 +31,8 @@
2931
"workspaces": {
3032
"packages": [
3133
"packages/*",
32-
"plugins/*"
34+
"plugins/*",
35+
"dynamic-plugins/*"
3336
]
3437
},
3538
"devDependencies": {
@@ -51,7 +54,8 @@
5154
"prettier": "@spotify/prettier-config",
5255
"lint-staged": {
5356
"*": "yarn run prettier:fix",
54-
"*.{js,jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix"
57+
"*.{jsx,ts,tsx,mjs,cjs}": "yarn run lint -- -- --fix",
58+
"!(.eslintrc).js": "yarn run lint -- -- --fix"
5559
},
5660
"packageManager": "[email protected]"
5761
}

packages/backend/src/index.ts

+35-8
Original file line numberDiff line numberDiff line change
@@ -113,12 +113,28 @@ type AddPlugin = {
113113
isOptional?: false;
114114
} & AddPluginBase;
115115

116+
type OptionalPluginOptions = {
117+
key?: string;
118+
path?: string;
119+
};
120+
116121
type AddOptionalPlugin = {
117122
isOptional: true;
118123
config: Config;
119-
options?: { key?: string; path?: string };
124+
options?: OptionalPluginOptions;
120125
} & AddPluginBase;
121126

127+
const OPTIONAL_DYNAMIC_PLUGINS: { [key: string]: OptionalPluginOptions } = {
128+
techdocs: {},
129+
argocd: {},
130+
sonarqube: {},
131+
kubernetes: {},
132+
'azure-devops': { key: 'enabled.azureDevOps' },
133+
jenkins: {},
134+
ocm: {},
135+
gitlab: {},
136+
} as const satisfies { [key: string]: OptionalPluginOptions };
137+
122138
async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise<void> {
123139
const { isOptional, plugin, apiRouter, createEnv, router, options } = args;
124140

@@ -132,10 +148,13 @@ async function addPlugin(args: AddPlugin | AddOptionalPlugin): Promise<void> {
132148
);
133149
apiRouter.use(options?.path ?? `/${plugin}`, await router(pluginEnv));
134150
console.log(`Using backend plugin ${plugin}...`);
151+
} else if (isOptional) {
152+
console.log(`Backend plugin ${plugin} is disabled`);
135153
}
136154
}
137155

138156
type AddRouterBase = {
157+
isOptional?: boolean;
139158
name: string;
140159
service: ServiceBuilder;
141160
root: string;
@@ -284,13 +303,21 @@ async function main() {
284303
if (plugin.installer.kind === 'legacy') {
285304
const pluginRouter = plugin.installer.router;
286305
if (pluginRouter !== undefined) {
287-
const pluginEnv = useHotMemoize(module, () =>
288-
createEnv(pluginRouter.pluginID),
289-
);
290-
apiRouter.use(
291-
`/${pluginRouter.pluginID}`,
292-
await pluginRouter.createPlugin(pluginEnv),
293-
);
306+
let optionals = {};
307+
if (pluginRouter.pluginID in OPTIONAL_DYNAMIC_PLUGINS) {
308+
optionals = {
309+
isOptional: true,
310+
config: config,
311+
options: OPTIONAL_DYNAMIC_PLUGINS[pluginRouter.pluginID],
312+
};
313+
}
314+
await addPlugin({
315+
plugin: pluginRouter.pluginID,
316+
apiRouter,
317+
createEnv,
318+
router: pluginRouter.createPlugin,
319+
...optionals,
320+
});
294321
}
295322
}
296323
}

tsconfig.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
"packages/*/src",
55
"plugins/*/src",
66
"plugins/*/dev",
7-
"plugins/*/migrations"
7+
"plugins/*/migrations",
8+
"dynamic-plugins/*/src"
89
],
910
"exclude": ["node_modules"],
1011
"compilerOptions": {

0 commit comments

Comments
 (0)