add new models for evals

Author: seanmcguire12

evals/index.eval.ts

@@ -39,6 +39,7 @@ import { google } from "@ai-sdk/google";
 import { anthropic } from "@ai-sdk/anthropic";
 import { groq } from "@ai-sdk/groq";
 import { cerebras } from "@ai-sdk/cerebras";
+import { openai } from "@ai-sdk/openai";
 dotenv.config();
 
 /**
@@ -274,14 +275,12 @@ const generateFilteredTestcases = (): Testcase[] => {
 
           // Execute the task
           let llmClient: LLMClient;
-          if (input.modelName.startsWith("gpt")) {
-            llmClient = new CustomOpenAIClient({
-              modelName: input.modelName as AvailableModel,
-              client: wrapOpenAI(
-                new OpenAI({
-                  apiKey: process.env.OPENAI_API_KEY,
-                }),
-              ),
+          if (
+            input.modelName.startsWith("gpt") ||
+            input.modelName.startsWith("o")
+          ) {
+            llmClient = new AISdkClient({
+              model: wrapAISDKModel(openai(input.modelName)),
             });
           } else if (input.modelName.startsWith("gemini")) {
             llmClient = new AISdkClient({

evals/taskConfig.ts

@@ -23,13 +23,18 @@ const ALL_EVAL_MODELS = [
   "gemini-2.5-pro-exp-03-25",
   "gemini-1.5-pro",
   "gemini-1.5-flash-8b",
+  "gemini-2.5-flash-preview-04-17",
+  "gemini-2.5-pro-preview-03-25",
   // ANTHROPIC
   "claude-3-5-sonnet-latest",
   "claude-3-7-sonnet-latest",
   // OPENAI
   "gpt-4o-mini",
   "gpt-4o",
   "gpt-4.5-preview",
+  "o3",
+  "o3-mini",
+  "o4-mini",
   // TOGETHER - META
   "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
   "meta-llama/Llama-3.3-70B-Instruct-Turbo",
@@ -95,7 +100,7 @@ if (filterByEvalName && !tasksByName[filterByEvalName]) {
  */
 const DEFAULT_EVAL_MODELS = process.env.EVAL_MODELS
   ? process.env.EVAL_MODELS.split(",")
-  : ["claude-3-5-sonnet-latest", "gpt-4o-mini", "gpt-4o"];
+  : ["gemini-2.5-pro-preview-03-25", "o3"];
 
 /**
  * getModelList: