Skip to content

Commit 4fa2672

Browse files
0xi4ochungyau97
0xi4o
and
chungyau97
authored
NVIDIA NIM fixes (#4215)
* fix: udpate label to "NVIDIA NIM API Key" * test: update tag from ":latest" to ":1.8.0-rtx" * test: add image URL path "nvcr.io/nim/" * fix/nvidia-nim-2 (#4208) * fix: update nim-container-manager * feat: add "DeepSeek R1 Distill Llama 8B" * fix/nidia-nim-3 (#4209) * chore: add error message NVIDIA NIM is not installed. * chore: standardize NVIDIA NGC API Key * chore: capitalize Nvidia to NVIDIA * chore: generalize error message for chat models * fix/nvidia-nim-4-yau (#4212) * test: nimRelaxMemConstraints and hostPort * test: add logger for hostPort and nimRelaxMemConstraints * test: nim-container-manager version 1.0.9 * test: parseInt nimRelaxMemConstraints * test: update nim-container-manager version to 1.0.10 * chore: update nim-container-manager version to 1.0.11 * Update start container behaviour - show existing containers and give users the choice * Go back to previous step when clicking start new so user can change port number * Update condition for showing existing container dialog * Fix start new in different port not working * Update get container controller * Update again * fix: generalize error message for chat models * Update getContainer controller * Fix incorrect image check in getContainer controller * Update existing container dialog text * Fix styles in container exists dialog for nvidia nim --------- Co-authored-by: chungyau97 <[email protected]> Co-authored-by: Ong Chung Yau <[email protected]>
1 parent 7867489 commit 4fa2672

File tree

10 files changed

+458
-252
lines changed

10 files changed

+458
-252
lines changed

‎packages/components/credentials/NvdiaNIMApi.credential.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { INodeParams, INodeCredential } from '../src/Interface'
1+
import { INodeCredential, INodeParams } from '../src/Interface'
22

33
class NvidiaNIMApi implements INodeCredential {
44
label: string
@@ -8,12 +8,12 @@ class NvidiaNIMApi implements INodeCredential {
88
inputs: INodeParams[]
99

1010
constructor() {
11-
this.label = 'Nvdia NIM API Key'
11+
this.label = 'NVIDIA NGC API Key'
1212
this.name = 'nvidiaNIMApi'
1313
this.version = 1.0
1414
this.inputs = [
1515
{
16-
label: 'Nvidia NIM API Key',
16+
label: 'NVIDIA NGC API Key',
1717
name: 'nvidiaNIMApiKey',
1818
type: 'password'
1919
}

‎packages/components/nodes/chatmodels/ChatNvdiaNIM/ChatNvdiaNIM.ts

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { ChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
21
import { BaseCache } from '@langchain/core/caches'
2+
import { ChatOpenAI, ChatOpenAIFields } from '@langchain/openai'
33
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
44
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
55

@@ -16,13 +16,13 @@ class ChatNvdiaNIM_ChatModels implements INode {
1616
inputs: INodeParams[]
1717

1818
constructor() {
19-
this.label = 'Chat Nvidia NIM'
20-
this.name = 'chatNvidiaNIM'
21-
this.version = 1.0
22-
this.type = 'ChatNvidiaNIM'
19+
this.label = 'Chat NVIDIA NIM'
20+
this.name = 'Chat NVIDIA NIM'
21+
this.version = 1.1
22+
this.type = 'Chat NVIDIA NIM'
2323
this.icon = 'nvdia.svg'
2424
this.category = 'Chat Models'
25-
this.description = 'Wrapper around Nvdia NIM Inference API'
25+
this.description = 'Wrapper around NVIDIA NIM Inference API'
2626
this.baseClasses = [this.type, ...getBaseClasses(ChatOpenAI)]
2727
this.credential = {
2828
label: 'Connect Credential',
@@ -153,7 +153,7 @@ class ChatNvdiaNIM_ChatModels implements INode {
153153
try {
154154
parsedBaseOptions = typeof baseOptions === 'object' ? baseOptions : JSON.parse(baseOptions)
155155
} catch (exception) {
156-
throw new Error("Invalid JSON in the ChatNvidiaNIM's baseOptions: " + exception)
156+
throw new Error("Invalid JSON in the Chat NVIDIA NIM's baseOptions: " + exception)
157157
}
158158
}
159159

‎packages/server/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@
9797
"multer": "^1.4.5-lts.1",
9898
"multer-s3": "^3.0.1",
9999
"mysql2": "^3.11.3",
100-
"nim-container-manager": "^1.0.5",
100+
"flowise-nim-container-manager": "^1.0.11",
101101
"openai": "^4.82.0",
102102
"pg": "^8.11.1",
103103
"posthog-node": "^3.5.0",

‎packages/server/src/controllers/nvidia-nim/index.ts

+51-9
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import axios from 'axios'
2-
import { Request, Response, NextFunction } from 'express'
2+
import { NextFunction, Request, Response } from 'express'
33

4-
const { NimContainerManager } = require('nim-container-manager')
4+
const { NimContainerManager } = require('flowise-nim-container-manager')
55

66
const getToken = async (req: Request, res: Response, next: NextFunction) => {
77
try {
@@ -55,7 +55,13 @@ const startContainer = async (req: Request, res: Response, next: NextFunction) =
5555
try {
5656
const imageTag = req.body.imageTag
5757
const apiKey = req.body.apiKey
58-
await NimContainerManager.startContainer(imageTag, apiKey)
58+
const hostPort = req.body.hostPort
59+
const nimRelaxMemConstraints = parseInt(req.body.nimRelaxMemConstraints)
60+
// Validate nimRelaxMemConstraints
61+
if (isNaN(nimRelaxMemConstraints) || (nimRelaxMemConstraints !== 0 && nimRelaxMemConstraints !== 1)) {
62+
return res.status(400).send('nimRelaxMemConstraints must be 0 or 1')
63+
}
64+
await NimContainerManager.startContainer(imageTag, apiKey, hostPort, nimRelaxMemConstraints)
5965
return res.send(`Starting container ${imageTag}`)
6066
} catch (error) {
6167
next(error)
@@ -79,17 +85,51 @@ const getImage = async (req: Request, res: Response, next: NextFunction) => {
7985
const getContainer = async (req: Request, res: Response, next: NextFunction) => {
8086
try {
8187
const imageTag = req.body.imageTag
88+
const port = req.body.port
89+
90+
// First check if the image exists
8291
const images = await NimContainerManager.userImageLibrary()
8392
const image = images.find((img: any) => img.tag === imageTag)
8493
if (!image) {
8594
return res.status(404).send(`Image ${imageTag} not found`)
8695
}
87-
if (!image.container) {
88-
return res.status(404).send(`Container of ${imageTag} not found`)
96+
97+
const containers = await NimContainerManager.listRunningContainers()
98+
const portInUse = containers.find((cont: any) => cont.port === port)
99+
if (portInUse) {
100+
const isModelContainer = portInUse.image === image.tag
101+
if (isModelContainer) {
102+
portInUse.image = image.name
103+
return res.json(portInUse)
104+
} else {
105+
return res.status(409).send({
106+
message: `Port ${port} is already in use by another container`,
107+
container: portInUse
108+
})
109+
}
89110
}
90-
const container = image.container
91-
container.image = image.name
92-
return res.json(container)
111+
112+
// If no container found with matching port, return 404
113+
return res.status(404).send(`Container of ${imageTag} with port ${port} not found`)
114+
} catch (error) {
115+
next(error)
116+
}
117+
}
118+
119+
const listRunningContainers = async (req: Request, res: Response, next: NextFunction) => {
120+
try {
121+
const containers = await NimContainerManager.listRunningContainers()
122+
return res.json(containers)
123+
} catch (error) {
124+
next(error)
125+
}
126+
}
127+
128+
const stopContainer = async (req: Request, res: Response, next: NextFunction) => {
129+
try {
130+
const containerId = req.body.containerId
131+
const containerInfo = await NimContainerManager.stopContainer(containerId)
132+
return res.json(containerInfo)
93133
} catch (error) {
94134
next(error)
95135
}
@@ -102,5 +142,7 @@ export default {
102142
pullImage,
103143
startContainer,
104144
getImage,
105-
getContainer
145+
getContainer,
146+
listRunningContainers,
147+
stopContainer
106148
}

‎packages/server/src/middlewares/errors/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { InternalFlowiseError } from '../../errors/internalFlowiseError'
55
// we need eslint because we have to pass next arg for the error middleware
66
// eslint-disable-next-line
77
async function errorHandlerMiddleware(err: InternalFlowiseError, req: Request, res: Response, next: NextFunction) {
8+
if (err.message.includes('401 Incorrect API key provided'))
9+
err.message = '401 Invalid model key or Incorrect local model configuration.'
810
let displayedError = {
911
statusCode: err.statusCode || StatusCodes.INTERNAL_SERVER_ERROR,
1012
success: false,

‎packages/server/src/routes/nvidia-nim/index.ts

+2
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ const router = express.Router()
66
router.get('/preload', nimController.preload)
77
router.get('/get-token', nimController.getToken)
88
router.get('/download-installer', nimController.downloadInstaller)
9+
router.get('/list-running-containers', nimController.listRunningContainers)
910
router.post('/pull-image', nimController.pullImage)
1011
router.post('/start-container', nimController.startContainer)
12+
router.post('/stop-container', nimController.stopContainer)
1113
router.post('/get-image', nimController.getImage)
1214
router.post('/get-container', nimController.getContainer)
1315

‎packages/server/src/utils/SSEStreamer.ts

+1
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ export class SSEStreamer implements IServerSideEventStreamer {
166166
}
167167

168168
streamErrorEvent(chatId: string, msg: string) {
169+
if (msg.includes('401 Incorrect API key provided')) msg = '401 Invalid model key or Incorrect local model configuration.'
169170
const client = this.clients[chatId]
170171
if (client) {
171172
const clientResponse = {

0 commit comments

Comments
 (0)