From 8b0096d8709e90a7706b14e74627a4533f930ab9 Mon Sep 17 00:00:00 2001
From: Justin Butterfield <Justin.Butterfield@improving.com>
Date: Thu, 2 Jul 2026 11:44:48 -0400
Subject: [PATCH] feat: add file-based Devin provider to fix ENAMETOOLONG on
 Windows

Add devinProvider.js as a promptfoo custom provider that writes prompts
to a temp file and passes them via --prompt-file instead of as a CLI
argument. This avoids the Windows ENAMETOOLONG error when prompts exceed
the 32k character command-line limit.

The provider also uses async spawn and parses token usage from Devin's
--export output. The provider entry in promptfooconfig.yaml is added
but commented out, ready to be enabled when needed.
---
 devinProvider.js     | 119 +++++++++++++++++++++++++++++++++++++++++++
 promptfooconfig.yaml |   6 +++
 2 files changed, 125 insertions(+)
 create mode 100644 devinProvider.js

diff --git a/devinProvider.js b/devinProvider.js
new file mode 100644
index 0000000..30b1239
--- /dev/null
+++ b/devinProvider.js
@@ -0,0 +1,119 @@
+const { spawn } = require('child_process');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+class DevinProvider {
+  constructor(options) {
+    this.providerId = options.id || 'devin-custom';
+    this.config = options.config || {};
+  }
+
+  id() {
+    return this.providerId;
+  }
+
+  async callApi(prompt, context, options) {
+    const model = this.config.model || 'SWE-1.6';
+
+    let isGraderMode = false;
+    try {
+      const parsed = JSON.parse(prompt);
+      if (Array.isArray(parsed) && parsed.length > 0 && parsed[0].role) {
+        isGraderMode = true;
+      }
+    } catch (e) {
+      // Not JSON, provider mode
+    }
+
+    let fullPrompt;
+
+    if (isGraderMode) {
+      let systemMsg, userMsg;
+      try {
+        const messages = JSON.parse(prompt);
+        const systemMessage = messages.find(m => m.role === 'system');
+        const userMessage = messages.find(m => m.role === 'user');
+
+        if (systemMessage && userMessage) {
+          systemMsg = systemMessage.content;
+          userMsg = userMessage.content;
+        } else {
+          throw new Error('Missing system or user message');
+        }
+      } catch (e) {
+        systemMsg = 'You are an evaluator. Respond with only valid JSON: {"pass": bool, "score": 0.0-1.0, "reason": "string"}';
+        userMsg = prompt;
+      }
+      fullPrompt = `${systemMsg}\n\n${userMsg}`;
+    } else {
+      fullPrompt = prompt;
+    }
+
+    const tmpFile = path.join(os.tmpdir(), `devin-prompt-${Date.now()}-${process.pid}.txt`);
+    const exportFile = path.join(os.tmpdir(), `devin-export-${Date.now()}-${process.pid}.json`);
+
+    try {
+      fs.writeFileSync(tmpFile, fullPrompt, 'utf8');
+
+      const args = ['-p', '--prompt-file', tmpFile, '--model', model, '--export', exportFile];
+      if (!isGraderMode) {
+        args.push('--permission-mode', 'auto');
+      }
+
+      const result = await new Promise((resolve, reject) => {
+        const child = spawn('devin', args, {
+          encoding: 'utf8',
+          stdio: ['pipe', 'pipe', 'pipe'],
+        });
+
+        let stdout = '';
+        let stderr = '';
+
+        child.stdout.on('data', (data) => { stdout += data; });
+        child.stderr.on('data', (data) => { stderr += data; });
+
+        child.on('error', (err) => {
+          resolve({ error: err.message, tokenUsage: parseTokenUsage(exportFile) });
+        });
+
+        child.on('close', (code) => {
+          if (code !== 0) {
+            resolve({ error: stdout || stderr || `devin exited with code ${code}`, tokenUsage: parseTokenUsage(exportFile) });
+          } else {
+            resolve({ output: stdout, tokenUsage: parseTokenUsage(exportFile) });
+          }
+        });
+      });
+
+      return result;
+    } finally {
+      try {
+        fs.unlinkSync(tmpFile);
+        fs.unlinkSync(exportFile);
+      } catch (e) {
+        // Ignore cleanup errors
+      }
+    }
+  }
+}
+
+function parseTokenUsage(exportFilePath) {
+  try {
+    const data = JSON.parse(fs.readFileSync(exportFilePath, 'utf8'));
+    const agentSteps = (data.steps || []).filter(s => s.source === 'agent');
+    let promptTokens = 0, completionTokens = 0;
+    for (const step of agentSteps) {
+      const m = step.metadata?.metrics;
+      if (m) {
+        promptTokens += (m.input_tokens || 0) + (m.cache_read_tokens || 0) + (m.cache_creation_tokens || 0);
+        completionTokens += (m.output_tokens || 0);
+      }
+    }
+    return { total: promptTokens + completionTokens, prompt: promptTokens, completion: completionTokens };
+  } catch {
+    return { total: 0, prompt: 0, completion: 0 };
+  }
+}
+
+module.exports = DevinProvider;
diff --git a/promptfooconfig.yaml b/promptfooconfig.yaml
index dca66be..4bd1046 100644
--- a/promptfooconfig.yaml
+++ b/promptfooconfig.yaml
@@ -46,6 +46,12 @@ providers:
     config:
       model: 'SWE-1.6'
 
+  # Devin CLI provider — file-based (fixes ENAMETOOLONG on Windows for prompts >32k)
+  #- id: 'file://devinProvider.js'
+  #  label: 'Devin SWE-1.6 (file provider)'
+  #  config:
+  #    model: 'SWE-1.6'
+
   # Devin CLI provider — For Testing a Second Model
   # MUST Use Label to differentiate providers with same ID (first in wins otherwise)
   # - id: 'exec: node ./devin.js'