Remove default max_tokens and allow configuring llm args from step

Author: yamalight

packages/litlytics/engine/step/runLLMStep.ts

@@ -79,7 +79,7 @@ export async function runLLMStep({
 
   const user = input;
   const startTime = performance.now();
-  const res = await litlytics.runPrompt({ system, user });
+  const res = await litlytics.runPrompt({ system, user, args: step.llmArgs });
   const endTime = performance.now();
   // replace existing result if present
   const existingResult = doc?.processingResults.find(

packages/litlytics/llm/llm.ts

@@ -80,7 +80,6 @@ export async function executeOnLLM({
   // set key
   const modelObj = getModel({ provider, model, key });
   const { text, usage } = await generateText({
-    maxTokens: 4096,
     ...modelArgs,
     model: modelObj,
     messages,

packages/litlytics/step/Step.ts

@@ -1,4 +1,4 @@
-import type { LanguageModelUsage } from 'ai';
+import type { CoreTool, LanguageModelUsage } from 'ai';
 import type { Doc } from '../doc/Document';
 
 export interface StepResult {
@@ -44,7 +44,10 @@ export interface SourceStep extends BaseStep {
 export interface ProcessingStep extends BaseStep {
   type: ProcessingStepTypes;
   input?: StepInput;
+  // llm
   prompt?: string;
+  llmArgs?: Record<string, CoreTool>;
+  // code
   code?: string;
   codeExplanation?: string;
 }