Skip to content

Commit fdeb40d

Browse files
committed
Remove local model support
This was causing more issues than bringing value. Local in-browser models are too small to follow litlytics prompts properly. Hence I'll be removing them for now. Will probably re-add once we have better/smaller models we can run in browser.
1 parent d994877 commit fdeb40d

File tree

10 files changed

+36
-219
lines changed

10 files changed

+36
-219
lines changed

app/components/ui/Overlay.tsx

+5-8
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import {
2929
DropdownMenu,
3030
DropdownShortcut,
3131
} from '~/components/catalyst/dropdown';
32-
import { configAtom, pipelineUndoAtom, webllmAtom } from '~/store/store';
32+
import { configAtom, pipelineUndoAtom } from '~/store/store';
3333
import { useLitlytics } from '~/store/WithLitLytics';
3434
import { Field, FieldGroup, Label } from '../catalyst/fieldset';
3535
import { Input } from '../catalyst/input';
@@ -65,7 +65,6 @@ export function OverlayUI() {
6565
const { litlytics, pipeline, setPipeline, setPipelineStatus } =
6666
useLitlytics();
6767
const litlyticsConfig = useAtomValue(configAtom);
68-
const webllm = useAtomValue(webllmAtom);
6968
const { undo, redo, canUndo, canRedo } = useAtomValue(pipelineUndoAtom);
7069
const [isOpen, setIsOpen] = useState(false);
7170
const [isSaveOpen, setIsSaveOpen] = useState(false);
@@ -83,10 +82,10 @@ export function OverlayUI() {
8382

8483
useEffect(() => {
8584
// show settings if key not set
86-
if (!litlyticsConfig.llmKey?.length && webllm.loadProgress !== 1) {
85+
if (!litlyticsConfig.llmKey?.length) {
8786
setIsSettingsOpen(true);
8887
}
89-
}, [litlyticsConfig, webllm]);
88+
}, [litlyticsConfig]);
9089

9190
const closeHelp = () => {
9291
setIsHelpOpen(false);
@@ -182,7 +181,7 @@ export function OverlayUI() {
182181

183182
const closeSettings = () => {
184183
// disallow closing if no key set
185-
if (!litlyticsConfig.llmKey?.length && webllm.loadProgress !== 1) {
184+
if (!litlyticsConfig.llmKey?.length) {
186185
return;
187186
}
188187
setIsSettingsOpen(false);
@@ -368,9 +367,7 @@ export function OverlayUI() {
368367
size="2xl"
369368
open={isSettingsOpen}
370369
onClose={closeSettings}
371-
canClose={
372-
Boolean(litlyticsConfig.llmKey?.length) || webllm.loadProgress === 1
373-
}
370+
canClose={Boolean(litlyticsConfig.llmKey?.length)}
374371
topClassName="z-20"
375372
>
376373
<DialogTitle className="!text-lg">Settings</DialogTitle>

app/components/ui/Recommended.tsx

-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { Badge } from '../catalyst/badge';
66
export const recommendedForProvider: Record<LLMProviders, LLMModel> = {
77
anthropic: 'claude-3-5-sonnet-20240620',
88
gemini: 'gemini-1.5-flash-latest',
9-
local: 'Llama-3.2-3B-Instruct-q4f16_1-MLC',
109
openai: 'gpt-4o-mini',
1110
ollama: '',
1211
};
@@ -38,10 +37,6 @@ export function Recommended() {
3837
<span className="text-sm opacity-60">Best free tier:</span>
3938
<br /> <Badge>Gemini</Badge> gemini-1.5-flash
4039
</li>
41-
<li>
42-
<span className="text-sm opacity-60">Best for privacy:</span>
43-
<br /> <Badge>Local</Badge> Llama-3.2-3B-Instruct
44-
</li>
4540
</ul>
4641

4742
<p className="text-xs">

app/components/ui/Settings.tsx

+25-124
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,23 @@
11
import { CurrencyDollarIcon, XCircleIcon } from '@heroicons/react/24/solid';
2-
import { MLCEngine } from '@mlc-ai/web-llm';
32
import { useAtom } from 'jotai';
43
import {
54
LLMModelsList,
65
LLMProvider,
76
LLMProviders,
87
LLMProvidersList,
9-
localModelSizes,
108
modelCosts,
119
} from 'litlytics';
1210
import _ from 'lodash';
1311
import { useEffect, useMemo, useState } from 'react';
14-
import { configAtom, webllmAtom } from '~/store/store';
12+
import { configAtom } from '~/store/store';
1513
import { Badge } from '../catalyst/badge';
16-
import { Button } from '../catalyst/button';
1714
import { Description, Field, FieldGroup, Label } from '../catalyst/fieldset';
1815
import { Input } from '../catalyst/input';
1916
import { Select } from '../catalyst/select';
20-
import { Spinner } from '../Spinner';
2117
import { ProviderKeysHint, providerNames } from './ProviderKeys';
2218
import { Recommended, recommendedForProvider } from './Recommended';
2319

24-
export function Settings({ close }: { close: () => void }) {
25-
const [webllm, setWebllm] = useAtom(webllmAtom);
20+
export function Settings({ close: _close }: { close: () => void }) {
2621
const [config, setConfig] = useAtom(configAtom);
2722
const [providers, setProviders] = useState<LLMProviders[]>([]);
2823

@@ -34,67 +29,11 @@ export function Settings({ close }: { close: () => void }) {
3429
const prov: LLMProviders[] = structuredClone(
3530
LLMProvidersList
3631
) as unknown as LLMProviders[];
37-
try {
38-
if (!navigator.gpu) {
39-
setProviders(prov);
40-
return;
41-
}
42-
43-
const adapter = await navigator.gpu.requestAdapter();
44-
if (!adapter) {
45-
setProviders(prov);
46-
return;
47-
}
48-
49-
const device = await adapter.requestDevice();
50-
if (!device) {
51-
setProviders(prov);
52-
return;
53-
}
54-
55-
setProviders(['local', ...prov]);
56-
} catch (err) {
57-
console.error(err);
58-
setProviders(prov);
59-
}
32+
setProviders(prov);
6033
}
6134
loadProviders();
6235
}, []);
6336

64-
const loadLocalModel = () => {
65-
if (!config.model) {
66-
return;
67-
}
68-
// create engine
69-
const engine = new MLCEngine();
70-
engine.setInitProgressCallback((initProgress) => {
71-
const textProgress = initProgress.text.split('[').at(1)?.split(']').at(0);
72-
const isFetching = initProgress.text.includes('Fetching');
73-
const isFinished = initProgress.text.includes('Finish');
74-
const [leftProgress, rightProgress] = textProgress?.split('/') ?? [
75-
'0',
76-
'1',
77-
];
78-
const loadProgress = parseInt(leftProgress) / parseInt(rightProgress);
79-
// console.log({
80-
// fetchProgress: isFetching ? initProgress.progress : 1,
81-
// loadProgress,
82-
// status: initProgress.text,
83-
// });
84-
setWebllm({
85-
engine,
86-
fetchProgress: isFetching ? initProgress.progress : 1,
87-
loadProgress: isFinished ? 1 : loadProgress,
88-
status: initProgress.text,
89-
});
90-
if (isFinished) {
91-
close();
92-
}
93-
});
94-
engine.reload(config.model);
95-
setWebllm({ engine, fetchProgress: 0, loadProgress: 0, status: '' });
96-
};
97-
9837
return (
9938
<div className="max-w-full">
10039
<FieldGroup>
@@ -114,7 +53,7 @@ export function Settings({ close }: { close: () => void }) {
11453
>
11554
{providers.map((prov) => (
11655
<option key={prov} value={prov}>
117-
{prov == 'local' ? 'local' : providerNames[prov]}
56+
{providerNames[prov]}
11857
</option>
11958
))}
12059
</Select>
@@ -155,7 +94,7 @@ export function Settings({ close }: { close: () => void }) {
15594
))}
15695
</Select>
15796
)}
158-
{config.provider !== 'local' && config.provider !== 'ollama' && (
97+
{config.provider !== 'ollama' && (
15998
<Description className="flex gap-2">
16099
<Badge title="Input price (USD) per million tokens">
161100
Input: <CurrencyDollarIcon className="w-3 h-3" />{' '}
@@ -190,7 +129,7 @@ export function Settings({ close }: { close: () => void }) {
190129
</Field>
191130
)}
192131

193-
{config.provider !== 'local' && config.provider !== 'ollama' && (
132+
{config.provider !== 'ollama' && (
194133
<Field>
195134
<Label>API Key</Label>
196135
<Input
@@ -211,67 +150,29 @@ export function Settings({ close }: { close: () => void }) {
211150
)}
212151
</Field>
213152
)}
214-
215-
{config.provider === 'local' && (
216-
<Field className="flex flex-col">
217-
<div className="flex items-center gap-2">
218-
<Button
219-
onClick={loadLocalModel}
220-
title={webllm.status ?? ''}
221-
disabled={webllm.fetchProgress !== -1}
222-
>
223-
{webllm.fetchProgress > -1 && webllm.fetchProgress < 1 ? (
224-
<>
225-
<Spinner className="w-3 h-3" />
226-
Fetching {Math.round(webllm.fetchProgress * 100)}%
227-
</>
228-
) : webllm.loadProgress > -1 && webllm.loadProgress < 1 ? (
229-
<>
230-
<Spinner className="w-3 h-3" />
231-
Loading {Math.round(webllm.loadProgress * 100)}%
232-
</>
233-
) : webllm.loadProgress === 1 ? (
234-
'Model loaded'
235-
) : (
236-
'Load model'
237-
)}
238-
</Button>
239-
<Description>
240-
Download and use selected model locally. This model size is:{' '}
241-
<strong>{localModelSizes[config.model!]}mb</strong>
242-
</Description>
243-
</div>
244-
<Description className="mt-4 mx-1 !text-xs">
245-
Please note that local model are worse at generating pipelines
246-
that their larger counterparts.
247-
</Description>
248-
</Field>
249-
)}
250153
</FieldGroup>
251154

252-
{!config.llmKey?.length &&
253-
config.provider !== 'local' &&
254-
config.provider !== 'ollama' && (
255-
<>
256-
<ProviderKeysHint provider={provider as LLMProvider} />
257-
<div className="rounded-md bg-red-50 dark:bg-red-900 p-4 mt-4">
258-
<div className="flex">
259-
<div className="flex-shrink-0">
260-
<XCircleIcon
261-
aria-hidden="true"
262-
className="h-5 w-5 text-red-400 dark:text-red-200"
263-
/>
264-
</div>
265-
<div className="ml-3">
266-
<h3 className="text-sm font-medium text-red-800 dark:text-red-200">
267-
Choose a provider, model and set a key for the provider of
268-
your choice to get started!
269-
</h3>
270-
</div>
155+
{!config.llmKey?.length && config.provider !== 'ollama' && (
156+
<>
157+
<ProviderKeysHint provider={provider as LLMProvider} />
158+
<div className="rounded-md bg-red-50 dark:bg-red-900 p-4 mt-4">
159+
<div className="flex">
160+
<div className="flex-shrink-0">
161+
<XCircleIcon
162+
aria-hidden="true"
163+
className="h-5 w-5 text-red-400 dark:text-red-200"
164+
/>
165+
</div>
166+
<div className="ml-3">
167+
<h3 className="text-sm font-medium text-red-800 dark:text-red-200">
168+
Choose a provider, model and set a key for the provider of
169+
your choice to get started!
170+
</h3>
271171
</div>
272172
</div>
273-
</>
274-
)}
173+
</div>
174+
</>
175+
)}
275176
</div>
276177
);
277178
}

app/store/WithLitLytics.tsx

+1-7
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import {
66
litlyticsAtom,
77
pipelineAtom,
88
pipelineStatusAtom,
9-
webllmAtom,
109
} from './store';
1110

1211
const LitLyticsContext = createContext<{
@@ -28,7 +27,6 @@ const LitLyticsContext = createContext<{
2827
});
2928

3029
export function WithLitLytics({ children }: { children: React.ReactNode }) {
31-
const webllm = useAtomValue(webllmAtom);
3230
const config = useAtomValue(configAtom);
3331
const litlytics = useAtomValue(litlyticsAtom);
3432
const [pipeline, setPipeline] = useAtom(pipelineAtom);
@@ -45,11 +43,7 @@ export function WithLitLytics({ children }: { children: React.ReactNode }) {
4543
) {
4644
litlytics.importConfig(config);
4745
}
48-
// assign webllm engine
49-
if (litlytics.engine !== webllm.engine) {
50-
litlytics.engine = webllm.engine;
51-
}
52-
}, [config, webllm, litlytics]);
46+
}, [config, litlytics]);
5347

5448
// update pipeline on changes
5549
useEffect(() => {

app/store/store.ts

-14
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import {
55
emptyPipeline,
66
LitLytics,
77
LitLyticsConfig,
8-
MLCEngine,
98
Pipeline,
109
PipelineStatus,
1110
} from 'litlytics';
@@ -43,16 +42,3 @@ export const litlyticsAtom = atom<LitLytics>(
4342
key: '',
4443
})
4544
);
46-
47-
// webllm instance
48-
export const webllmAtom = atom<{
49-
engine?: MLCEngine;
50-
fetchProgress: number;
51-
loadProgress: number;
52-
status: string;
53-
}>({
54-
engine: undefined,
55-
fetchProgress: -1,
56-
loadProgress: -1,
57-
status: '',
58-
});

packages/litlytics/bun.lockb

-718 Bytes
Binary file not shown.

packages/litlytics/engine/runPrompt.ts

-12
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,22 @@
1-
import type { MLCEngine } from '@mlc-ai/web-llm';
21
import type { CoreMessage, CoreTool } from 'ai';
32
import type { LLMProviders } from '../litlytics';
43
import { executeOnLLM } from '../llm/llm';
54
import type { LLMModel, LLMProvider, LLMRequest } from '../llm/types';
6-
import { runWithWebLLM } from '../webllm/webllm.client';
75

86
export interface RunPromptFromMessagesArgs {
97
provider: LLMProviders;
108
key: string;
119
model: LLMModel;
12-
engine?: MLCEngine;
1310
messages: CoreMessage[];
1411
args?: Record<string, CoreTool>;
1512
}
1613
export const runPromptFromMessages = async ({
1714
provider,
1815
key,
1916
model,
20-
engine,
2117
messages,
2218
args,
2319
}: RunPromptFromMessagesArgs) => {
24-
// execute locally
25-
if (provider === 'local') {
26-
return runWithWebLLM({ engine, messages, args });
27-
}
28-
2920
const req: LLMRequest = {
3021
provider: provider as LLMProvider,
3122
key,
@@ -43,7 +34,6 @@ export interface RunPromptArgs {
4334
provider: LLMProviders;
4435
key: string;
4536
model: LLMModel;
46-
engine?: MLCEngine;
4737
system: string;
4838
user: string;
4939
args?: Record<string, CoreTool>;
@@ -52,7 +42,6 @@ export const runPrompt = async ({
5242
provider,
5343
key,
5444
model,
55-
engine,
5645
system,
5746
user,
5847
args,
@@ -68,7 +57,6 @@ export const runPrompt = async ({
6857
provider,
6958
key,
7059
model,
71-
engine,
7260
messages,
7361
args,
7462
});

0 commit comments

Comments
 (0)