Adding an Eval Type

// src/main/eval-executors/rule-executor.ts (simplified)
export async function executeRule(
  config: RuleConfig,
  output: string
): Promise<RuleResult> { … }

// src/main/eval-executors/metric-executor.ts (simplified)
export function executeMetric(
  config: MetricConfig,
  metrics: MetricInput
): MetricResult { … }

// src/shared/eval-types.ts — extend EvalType union
export type EvalType = 'llm_judge' | 'rule' | 'similarity' | 'metric' | 'my_custom'

// Add config interface
export interface MyCustomConfig {
  expectedFormat: string // example field
  caseSensitive: boolean
}

// EvalConfig union — add your config
export type EvalConfig =
  | RuleConfig
  | LlmJudgeConfig
  | SimilarityConfig
  | MetricConfig
  | MyCustomConfig

// src/main/eval-executors/my-custom-executor.ts
import type { MyCustomConfig } from '../../shared/eval-types'
import log from 'electron-log/node'

export interface MyCustomResult {
  passed: boolean
  detail: string
  error?: string
}

export function executeMyCustom(config: MyCustomConfig, output: string): MyCustomResult {
  try {
    const target = config.caseSensitive
      ? config.expectedFormat
      : config.expectedFormat.toLowerCase()
    const actual = config.caseSensitive ? output : output.toLowerCase()

    const passed = actual.includes(target)
    return {
      passed,
      detail: passed
        ? `Output contains expected format "${config.expectedFormat}"`
        : `Output does not contain "${config.expectedFormat}"`,
    }
  } catch (err) {
    log.error('MyCustomExecutor error:', err)
    return { passed: false, detail: '', error: String(err) }
  }
}

// src/workers/eval-worker.ts — inside the switch on eval.type
case 'my_custom': {
  const { executeMyCustom } = await import('../main/eval-executors/my-custom-executor')
  const result = executeMyCustom(eval.config as MyCustomConfig, run.outputResponse)
  score = result.passed ? 1 : 0
  passed = result.passed
  detail = result.detail
  error = result.error
  break
}

// src/main/eval-executors/my-custom-executor.test.ts
import { executeMyCustom } from './my-custom-executor'

describe('executeMyCustom', () => {
  it('passes when output contains expected format', () => {
    const result = executeMyCustom(
      { expectedFormat: 'Conclusion', caseSensitive: false },
      'In Conclusion, the analysis shows…'
    )
    expect(result.passed).toBe(true)
  })

  it('fails when expected format is absent', () => {
    const result = executeMyCustom(
      { expectedFormat: 'Conclusion', caseSensitive: false },
      'The analysis shows mixed results.'
    )
    expect(result.passed).toBe(false)
  })
})