Remove local model support

This was causing more issues than bringing value.
Local in-browser models are too small to follow litlytics prompts
properly. Hence I'll be removing them for now.
Will probably re-add once we have better/smaller models we can run in
browser.

Author: yamalight

app/components/ui/Overlay.tsx

@@ -29,7 +29,7 @@ import {
   DropdownMenu,
   DropdownShortcut,
 } from '~/components/catalyst/dropdown';
-import { configAtom, pipelineUndoAtom, webllmAtom } from '~/store/store';
+import { configAtom, pipelineUndoAtom } from '~/store/store';
 import { useLitlytics } from '~/store/WithLitLytics';
 import { Field, FieldGroup, Label } from '../catalyst/fieldset';
 import { Input } from '../catalyst/input';
@@ -65,7 +65,6 @@ export function OverlayUI() {
   const { litlytics, pipeline, setPipeline, setPipelineStatus } =
     useLitlytics();
   const litlyticsConfig = useAtomValue(configAtom);
-  const webllm = useAtomValue(webllmAtom);
   const { undo, redo, canUndo, canRedo } = useAtomValue(pipelineUndoAtom);
   const [isOpen, setIsOpen] = useState(false);
   const [isSaveOpen, setIsSaveOpen] = useState(false);
@@ -83,10 +82,10 @@ export function OverlayUI() {
 
   useEffect(() => {
     // show settings if key not set
-    if (!litlyticsConfig.llmKey?.length && webllm.loadProgress !== 1) {
+    if (!litlyticsConfig.llmKey?.length) {
       setIsSettingsOpen(true);
     }
-  }, [litlyticsConfig, webllm]);
+  }, [litlyticsConfig]);
 
   const closeHelp = () => {
     setIsHelpOpen(false);
@@ -182,7 +181,7 @@ export function OverlayUI() {
 
   const closeSettings = () => {
     // disallow closing if no key set
-    if (!litlyticsConfig.llmKey?.length && webllm.loadProgress !== 1) {
+    if (!litlyticsConfig.llmKey?.length) {
       return;
     }
     setIsSettingsOpen(false);
@@ -368,9 +367,7 @@ export function OverlayUI() {
         size="2xl"
         open={isSettingsOpen}
         onClose={closeSettings}
-        canClose={
-          Boolean(litlyticsConfig.llmKey?.length) || webllm.loadProgress === 1
-        }
+        canClose={Boolean(litlyticsConfig.llmKey?.length)}
         topClassName="z-20"
       >
         <DialogTitle className="!text-lg">Settings</DialogTitle>

app/components/ui/Recommended.tsx

@@ -6,7 +6,6 @@ import { Badge } from '../catalyst/badge';
 export const recommendedForProvider: Record<LLMProviders, LLMModel> = {
   anthropic: 'claude-3-5-sonnet-20240620',
   gemini: 'gemini-1.5-flash-latest',
-  local: 'Llama-3.2-3B-Instruct-q4f16_1-MLC',
   openai: 'gpt-4o-mini',
   ollama: '',
 };
@@ -38,10 +37,6 @@ export function Recommended() {
               <span className="text-sm opacity-60">Best free tier:</span>
               <br /> <Badge>Gemini</Badge> gemini-1.5-flash
             </li>
-            <li>
-              <span className="text-sm opacity-60">Best for privacy:</span>
-              <br /> <Badge>Local</Badge> Llama-3.2-3B-Instruct
-            </li>
           </ul>
 
           <p className="text-xs">

app/components/ui/Settings.tsx

@@ -1,28 +1,23 @@
 import { CurrencyDollarIcon, XCircleIcon } from '@heroicons/react/24/solid';
-import { MLCEngine } from '@mlc-ai/web-llm';
 import { useAtom } from 'jotai';
 import {
   LLMModelsList,
   LLMProvider,
   LLMProviders,
   LLMProvidersList,
-  localModelSizes,
   modelCosts,
 } from 'litlytics';
 import _ from 'lodash';
 import { useEffect, useMemo, useState } from 'react';
-import { configAtom, webllmAtom } from '~/store/store';
+import { configAtom } from '~/store/store';
 import { Badge } from '../catalyst/badge';
-import { Button } from '../catalyst/button';
 import { Description, Field, FieldGroup, Label } from '../catalyst/fieldset';
 import { Input } from '../catalyst/input';
 import { Select } from '../catalyst/select';
-import { Spinner } from '../Spinner';
 import { ProviderKeysHint, providerNames } from './ProviderKeys';
 import { Recommended, recommendedForProvider } from './Recommended';
 
-export function Settings({ close }: { close: () => void }) {
-  const [webllm, setWebllm] = useAtom(webllmAtom);
+export function Settings({ close: _close }: { close: () => void }) {
   const [config, setConfig] = useAtom(configAtom);
   const [providers, setProviders] = useState<LLMProviders[]>([]);
 
@@ -34,67 +29,11 @@ export function Settings({ close }: { close: () => void }) {
       const prov: LLMProviders[] = structuredClone(
         LLMProvidersList
       ) as unknown as LLMProviders[];
-      try {
-        if (!navigator.gpu) {
-          setProviders(prov);
-          return;
-        }
-
-        const adapter = await navigator.gpu.requestAdapter();
-        if (!adapter) {
-          setProviders(prov);
-          return;
-        }
-
-        const device = await adapter.requestDevice();
-        if (!device) {
-          setProviders(prov);
-          return;
-        }
-
-        setProviders(['local', ...prov]);
-      } catch (err) {
-        console.error(err);
-        setProviders(prov);
-      }
+      setProviders(prov);
     }
     loadProviders();
   }, []);
 
-  const loadLocalModel = () => {
-    if (!config.model) {
-      return;
-    }
-    // create engine
-    const engine = new MLCEngine();
-    engine.setInitProgressCallback((initProgress) => {
-      const textProgress = initProgress.text.split('[').at(1)?.split(']').at(0);
-      const isFetching = initProgress.text.includes('Fetching');
-      const isFinished = initProgress.text.includes('Finish');
-      const [leftProgress, rightProgress] = textProgress?.split('/') ?? [
-        '0',
-        '1',
-      ];
-      const loadProgress = parseInt(leftProgress) / parseInt(rightProgress);
-      // console.log({
-      //   fetchProgress: isFetching ? initProgress.progress : 1,
-      //   loadProgress,
-      //   status: initProgress.text,
-      // });
-      setWebllm({
-        engine,
-        fetchProgress: isFetching ? initProgress.progress : 1,
-        loadProgress: isFinished ? 1 : loadProgress,
-        status: initProgress.text,
-      });
-      if (isFinished) {
-        close();
-      }
-    });
-    engine.reload(config.model);
-    setWebllm({ engine, fetchProgress: 0, loadProgress: 0, status: '' });
-  };
-
   return (
     <div className="max-w-full">
       <FieldGroup>
@@ -114,7 +53,7 @@ export function Settings({ close }: { close: () => void }) {
             >
               {providers.map((prov) => (
                 <option key={prov} value={prov}>
-                  {prov == 'local' ? 'local' : providerNames[prov]}
+                  {providerNames[prov]}
                 </option>
               ))}
             </Select>
@@ -155,7 +94,7 @@ export function Settings({ close }: { close: () => void }) {
               ))}
             </Select>
           )}
-          {config.provider !== 'local' && config.provider !== 'ollama' && (
+          {config.provider !== 'ollama' && (
             <Description className="flex gap-2">
               <Badge title="Input price (USD) per million tokens">
                 Input: <CurrencyDollarIcon className="w-3 h-3" />{' '}
@@ -190,7 +129,7 @@ export function Settings({ close }: { close: () => void }) {
           </Field>
         )}
 
-        {config.provider !== 'local' && config.provider !== 'ollama' && (
+        {config.provider !== 'ollama' && (
           <Field>
             <Label>API Key</Label>
             <Input
@@ -211,67 +150,29 @@ export function Settings({ close }: { close: () => void }) {
             )}
           </Field>
         )}
-
-        {config.provider === 'local' && (
-          <Field className="flex flex-col">
-            <div className="flex items-center gap-2">
-              <Button
-                onClick={loadLocalModel}
-                title={webllm.status ?? ''}
-                disabled={webllm.fetchProgress !== -1}
-              >
-                {webllm.fetchProgress > -1 && webllm.fetchProgress < 1 ? (
-                  <>
-                    <Spinner className="w-3 h-3" />
-                    Fetching {Math.round(webllm.fetchProgress * 100)}%
-                  </>
-                ) : webllm.loadProgress > -1 && webllm.loadProgress < 1 ? (
-                  <>
-                    <Spinner className="w-3 h-3" />
-                    Loading {Math.round(webllm.loadProgress * 100)}%
-                  </>
-                ) : webllm.loadProgress === 1 ? (
-                  'Model loaded'
-                ) : (
-                  'Load model'
-                )}
-              </Button>
-              <Description>
-                Download and use selected model locally. This model size is:{' '}
-                <strong>{localModelSizes[config.model!]}mb</strong>
-              </Description>
-            </div>
-            <Description className="mt-4 mx-1 !text-xs">
-              Please note that local model are worse at generating pipelines
-              that their larger counterparts.
-            </Description>
-          </Field>
-        )}
       </FieldGroup>
 
-      {!config.llmKey?.length &&
-        config.provider !== 'local' &&
-        config.provider !== 'ollama' && (
-          <>
-            <ProviderKeysHint provider={provider as LLMProvider} />
-            <div className="rounded-md bg-red-50 dark:bg-red-900 p-4 mt-4">
-              <div className="flex">
-                <div className="flex-shrink-0">
-                  <XCircleIcon
-                    aria-hidden="true"
-                    className="h-5 w-5 text-red-400 dark:text-red-200"
-                  />
-                </div>
-                <div className="ml-3">
-                  <h3 className="text-sm font-medium text-red-800 dark:text-red-200">
-                    Choose a provider, model and set a key for the provider of
-                    your choice to get started!
-                  </h3>
-                </div>
+      {!config.llmKey?.length && config.provider !== 'ollama' && (
+        <>
+          <ProviderKeysHint provider={provider as LLMProvider} />
+          <div className="rounded-md bg-red-50 dark:bg-red-900 p-4 mt-4">
+            <div className="flex">
+              <div className="flex-shrink-0">
+                <XCircleIcon
+                  aria-hidden="true"
+                  className="h-5 w-5 text-red-400 dark:text-red-200"
+                />
+              </div>
+              <div className="ml-3">
+                <h3 className="text-sm font-medium text-red-800 dark:text-red-200">
+                  Choose a provider, model and set a key for the provider of
+                  your choice to get started!
+                </h3>
               </div>
             </div>
-          </>
-        )}
+          </div>
+        </>
+      )}
     </div>
   );
 }

app/store/WithLitLytics.tsx

@@ -6,7 +6,6 @@ import {
   litlyticsAtom,
   pipelineAtom,
   pipelineStatusAtom,
-  webllmAtom,
 } from './store';
 
 const LitLyticsContext = createContext<{
@@ -28,7 +27,6 @@ const LitLyticsContext = createContext<{
 });
 
 export function WithLitLytics({ children }: { children: React.ReactNode }) {
-  const webllm = useAtomValue(webllmAtom);
   const config = useAtomValue(configAtom);
   const litlytics = useAtomValue(litlyticsAtom);
   const [pipeline, setPipeline] = useAtom(pipelineAtom);
@@ -45,11 +43,7 @@ export function WithLitLytics({ children }: { children: React.ReactNode }) {
     ) {
       litlytics.importConfig(config);
     }
-    // assign webllm engine
-    if (litlytics.engine !== webllm.engine) {
-      litlytics.engine = webllm.engine;
-    }
-  }, [config, webllm, litlytics]);
+  }, [config, litlytics]);
 
   // update pipeline on changes
   useEffect(() => {

app/store/store.ts

@@ -5,7 +5,6 @@ import {
   emptyPipeline,
   LitLytics,
   LitLyticsConfig,
-  MLCEngine,
   Pipeline,
   PipelineStatus,
 } from 'litlytics';
@@ -43,16 +42,3 @@ export const litlyticsAtom = atom<LitLytics>(
     key: '',
   })
 );
-
-// webllm instance
-export const webllmAtom = atom<{
-  engine?: MLCEngine;
-  fetchProgress: number;
-  loadProgress: number;
-  status: string;
-}>({
-  engine: undefined,
-  fetchProgress: -1,
-  loadProgress: -1,
-  status: '',
-});

packages/litlytics/bun.lockb

@@ -1,31 +1,22 @@
-import type { MLCEngine } from '@mlc-ai/web-llm';
 import type { CoreMessage, CoreTool } from 'ai';
 import type { LLMProviders } from '../litlytics';
 import { executeOnLLM } from '../llm/llm';
 import type { LLMModel, LLMProvider, LLMRequest } from '../llm/types';
-import { runWithWebLLM } from '../webllm/webllm.client';
 
 export interface RunPromptFromMessagesArgs {
   provider: LLMProviders;
   key: string;
   model: LLMModel;
-  engine?: MLCEngine;
   messages: CoreMessage[];
   args?: Record<string, CoreTool>;
 }
 export const runPromptFromMessages = async ({
   provider,
   key,
   model,
-  engine,
   messages,
   args,
 }: RunPromptFromMessagesArgs) => {
-  // execute locally
-  if (provider === 'local') {
-    return runWithWebLLM({ engine, messages, args });
-  }
-
   const req: LLMRequest = {
     provider: provider as LLMProvider,
     key,
@@ -43,7 +34,6 @@ export interface RunPromptArgs {
   provider: LLMProviders;
   key: string;
   model: LLMModel;
-  engine?: MLCEngine;
   system: string;
   user: string;
   args?: Record<string, CoreTool>;
@@ -52,7 +42,6 @@ export const runPrompt = async ({
   provider,
   key,
   model,
-  engine,
   system,
   user,
   args,
@@ -68,7 +57,6 @@ export const runPrompt = async ({
     provider,
     key,
     model,
-    engine,
     messages,
     args,
   });