初始化 kimi-eyes:给 KimiCode 非多模态模型补上视觉能力
This commit is contained in:
commit
ed8f7ae977
12 files changed
+1510
No files matched your search
@@ -0,0 +1 @@
|
||||
{"zhipuai/glm-4.6v":{"name":"GLM-4.6V","image":true},"zhipuai/glm-5":{"name":"GLM-5","image":false},"zhipuai/glm-4.5-air":{"name":"GLM-4.5-Air","image":false},"zhipuai/glm-5.1":{"name":"GLM-5.1","image":false},"zhipuai/glm-4.7-flash":{"name":"GLM-4.7-Flash","image":false},"zhipuai/glm-5.2":{"name":"GLM-5.2","image":false},"zhipuai/glm-4.7-flashx":{"name":"GLM-4.7-FlashX","image":false},"zhipuai/glm-4.6":{"name":"GLM-4.6","image":false},"zhipuai/glm-4.5":{"name":"GLM-4.5","image":false},"zhipuai/glm-4.5v":{"name":"GLM-4.5V","image":true},"zhipuai/glm-4.7":{"name":"GLM-4.7","image":false},"zhipuai/glm-5-turbo":{"name":"GLM-5-Turbo","image":false},"zhipuai/glm-5v-turbo":{"name":"GLM-5V-Turbo","image":true},"zhipuai/glm-4.5-flash":{"name":"GLM-4.5-Flash","image":false},"microsoft/mai-code-1-flash":{"name":"MAI-Code-1-Flash","image":false},"cohere/command-r7b-arabic-02-2025":{"name":"Command R7B Arabic","image":false},"cohere/command-r-08-2024":{"name":"Command R","image":false},"cohere/command-a-plus-05-2026":{"name":"Command A Plus","image":true},"cohere/command-a-translate-08-2025":{"name":"Command A Translate","image":false},"cohere/c4ai-aya-expanse-8b":{"name":"Aya Expanse 8B","image":false},"cohere/c4ai-aya-vision-32b":{"name":"Aya Vision 32B","image":true},"cohere/command-a-03-2025":{"name":"Command A","image":false},"cohere/command-r-plus-08-2024":{"name":"Command R+","image":false},"cohere/command-a-reasoning-08-2025":{"name":"Command A Reasoning","image":false},"cohere/command-a-vision-07-2025":{"name":"Command A Vision","image":true},"cohere/command-r7b-12-2024":{"name":"Command R7B","image":false},"cohere/c4ai-aya-vision-8b":{"name":"Aya Vision 8B","image":true},"cohere/north-mini-code-1-0":{"name":"North Mini Code","image":false},"cohere/c4ai-aya-expanse-32b":{"name":"Aya Expanse 32B","image":false},"nvidia/mistral-nemotron":{"name":"Mistral Nemotron","image":false},"nvidia/nemotron-nano-12b-v2-vl":{"name":"Nemotron Nano 12B v2 VL","image":true},"nvidia/nemotron-3-nano-30b-a3b":{"name":"Nemotron 3 Nano 30B A3B","image":false},"nvidia/llama-3.3-nemotron-super-49b-v1.5":{"name":"Llama 3.3 Nemotron Super 49B v1.5","image":false},"nvidia/llama-3.1-nemotron-safety-guard-8b-v3":{"name":"Llama 3.1 Nemotron Safety Guard 8B v3","image":false},"nvidia/llama-3.3-nemotron-super-49b-v1":{"name":"Llama 3.3 Nemotron Super 49B v1","image":false},"nvidia/llama-3.1-nemotron-70b-instruct":{"name":"Llama 3.1 Nemotron 70B Instruct","image":false},"nvidia/nemotron-3-super-120b-a12b":{"name":"Nemotron 3 Super 120B A12B","image":false},"nvidia/nemotron-3-content-safety":{"name":"Nemotron 3 Content Safety","image":false},"nvidia/nemotron-3-nano-omni-30b-a3b-reasoning":{"name":"Nemotron 3 Nano Omni 30B A3B Reasoning","image":true},"nvidia/llama-3.1-nemotron-ultra-253b":{"name":"Llama 3.1 Nemotron Ultra 253B","image":false},"nvidia/nemotron-voicechat":{"name":"Nemotron VoiceChat","image":false},"nvidia/llama-nemotron-embed-vl-1b-v2":{"name":"Llama Nemotron Embed VL 1B v2","image":true},"nvidia/nemotron-content-safety-reasoning-4b":{"name":"Nemotron Content Safety Reasoning 4B","image":false},"nvidia/nemotron-3-ultra-550b-a55b":{"name":"Nemotron 3 Ultra 550B A55B","image":false},"nvidia/llama-nemotron-rerank-vl-1b-v2":{"name":"Llama Nemotron Rerank VL 1B v2","image":true},"nvidia/nemotron-mini-4b-instruct":{"name":"Nemotron Mini 4B Instruct","image":false},"nvidia/nemotron-nano-9b-v2":{"name":"Nemotron Nano 9B v2","image":false},"nvidia/nemotron-cascade-2-30b-a3b":{"name":"Nemotron Cascade 2 30B A3B","image":false},"nvidia/nemotron-3.5-content-safety":{"name":"Nemotron 3.5 Content Safety","image":true},"google/gemini-2.5-computer-use-preview-10-2025":{"name":"Gemini 2.5 Computer Use Preview","image":true},"google/deep-research-preview-04-2026":{"name":"Gemini Deep Research Preview","image":true},"google/gemini-3.1-flash-tts-preview":{"name":"Gemini 3.1 Flash TTS Preview","image":false},"google/gemini-flash-latest":{"name":"Gemini Flash Latest","image":true},"google/gemini-embedding-2":{"name":"Gemini Embedding 2","image":true},"google/lyria-3-pro-preview":{"name":"Lyria 3 Pro Preview","image":true},"google/gemini-3.5-flash":{"name":"Gemini 3.5 Flash","image":true},"google/gemini-2.5-flash":{"name":"Gemini 2.5 Flash","image":true},"google/gemini-3.5-flash-lite":{"name":"Gemini 3.5 Flash Lite","image":true},"google/lyria-3-clip-preview":{"name":"Lyria 3 Clip Preview","image":true},"google/gemini-omni-flash-preview":{"name":"Gemini Omni Flash Preview","image":true},"google/veo-3.1-generate-preview":{"name":"Veo 3.1 Preview","image":true},"google/gemini-2.5-pro-tts":{"name":"Gemini 2.5 Pro TTS","image":false},"google/deep-research-max-preview-04-2026":{"name":"Deep Research Max Preview","image":true},"google/gemini-3-pro-image-preview":{"name":"Nano Banana Pro","image":true},"google/gemini-3.1-flash-lite-preview":{"name":"Gemini 3.1 Flash Lite Preview","image":true},"google/gemini-2.5-flash-tts":{"name":"Gemini 2.5 Flash TTS","imLine truncated
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
// kimi-eyes MCP stdio server — zero-dependency.
|
||||
// Speaks JSON-RPC 2.0 over LSP-style framing (Content-Length headers) on stdio.
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
VERSION,
|
||||
getConfig,
|
||||
requireConfigured,
|
||||
callVLM,
|
||||
sniffImage,
|
||||
clipboardImageToTemp,
|
||||
lookupModelCapability,
|
||||
} from './vision.mjs';
|
||||
|
||||
const TOOLS = [
|
||||
{
|
||||
name: 'read_image',
|
||||
description:
|
||||
'Analyze a local image file using the configured vision API and return a text description. ' +
|
||||
'Use when the user provides an image path or an @ file reference and your model cannot see images directly.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
path: {
|
||||
type: 'string',
|
||||
description: 'Absolute path to a local image file (png, jpg, jpeg, webp, gif, bmp).',
|
||||
},
|
||||
prompt: {
|
||||
type: 'string',
|
||||
description: 'Optional instruction describing what to analyze (default: describe the image).',
|
||||
},
|
||||
},
|
||||
required: ['path'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'read_clipboard_image',
|
||||
description:
|
||||
'Capture the image currently in the system clipboard (e.g. a screenshot) and analyze it with the ' +
|
||||
'configured vision API. Use when the user says they just took a screenshot or copied an image and no file path is given.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
prompt: {
|
||||
type: 'string',
|
||||
description: 'Optional instruction describing what to analyze (default: describe the image).',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tools
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* When the user declares their main model (config.mainModel) and the models.dev
|
||||
* database says it supports image input, the tools are unnecessary — the model can
|
||||
* see images natively. Refuse with a hint instead of wasting a VLM call.
|
||||
*/
|
||||
function assertNeedsExternalVision(cfg) {
|
||||
if (!cfg.mainModel) return;
|
||||
const cap = lookupModelCapability(cfg.mainModel);
|
||||
if (cap.known && cap.image) {
|
||||
throw new Error(
|
||||
`Model "${cfg.mainModel}" supports native image input (per the models.dev database), ` +
|
||||
`so the kimi-eyes tools are not needed. Paste the image directly and the model will see it. ` +
|
||||
`If you still want kimi-eyes active, remove "mainModel" from the kimi-eyes config.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function readImageTool(args) {
|
||||
const p = args?.path;
|
||||
if (typeof p !== 'string' || !p.trim()) {
|
||||
throw new Error('Missing required parameter: path');
|
||||
}
|
||||
const cfg = getConfig();
|
||||
assertNeedsExternalVision(cfg);
|
||||
const abs = path.resolve(p.trim());
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(abs);
|
||||
} catch {
|
||||
throw new Error(`File not found: ${abs}`);
|
||||
}
|
||||
if (!stat.isFile()) throw new Error(`Not a file: ${abs}`);
|
||||
const img = sniffImage(abs);
|
||||
if (!img) {
|
||||
throw new Error(
|
||||
`Unsupported or invalid image file: ${abs} (supported: png, jpg, jpeg, webp, gif, bmp)`
|
||||
);
|
||||
}
|
||||
requireConfigured(cfg);
|
||||
return callVLM({ ...img, prompt: args?.prompt }, cfg);
|
||||
}
|
||||
|
||||
async function readClipboardImageTool(args) {
|
||||
const cfg = getConfig();
|
||||
assertNeedsExternalVision(cfg);
|
||||
requireConfigured(cfg);
|
||||
let tmp = null;
|
||||
try {
|
||||
tmp = clipboardImageToTemp();
|
||||
const img = sniffImage(tmp);
|
||||
if (!img) throw new Error('Clipboard does not contain a valid image.');
|
||||
return callVLM({ ...img, prompt: args?.prompt }, cfg);
|
||||
} finally {
|
||||
if (tmp) {
|
||||
try {
|
||||
fs.unlinkSync(tmp);
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function callTool(name, args) {
|
||||
if (name === 'read_image') return readImageTool(args);
|
||||
if (name === 'read_clipboard_image') return readClipboardImageTool(args);
|
||||
throw new Error(`Unknown tool: ${name}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// JSON-RPC over stdio (MCP framing)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function send(obj) {
|
||||
const body = Buffer.from(JSON.stringify(obj), 'utf8');
|
||||
process.stdout.write(`Content-Length: ${body.length}\r\n\r\n`);
|
||||
process.stdout.write(body);
|
||||
}
|
||||
|
||||
function sendError(id, code, message) {
|
||||
send({ jsonrpc: '2.0', id, error: { code, message } });
|
||||
}
|
||||
|
||||
async function handleRequest(msg) {
|
||||
const { id, method, params } = msg;
|
||||
try {
|
||||
switch (method) {
|
||||
case 'initialize':
|
||||
send({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
result: {
|
||||
protocolVersion: params?.protocolVersion || '2025-06-18',
|
||||
capabilities: { tools: { listChanged: false } },
|
||||
serverInfo: { name: 'kimi-eyes', version: VERSION },
|
||||
},
|
||||
});
|
||||
return;
|
||||
case 'ping':
|
||||
send({ jsonrpc: '2.0', id, result: {} });
|
||||
return;
|
||||
case 'tools/list':
|
||||
send({ jsonrpc: '2.0', id, result: { tools: TOOLS } });
|
||||
return;
|
||||
case 'tools/call': {
|
||||
const name = params?.name;
|
||||
const args = params?.arguments || {};
|
||||
try {
|
||||
const text = await callTool(name, args);
|
||||
send({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
result: { content: [{ type: 'text', text }] },
|
||||
});
|
||||
} catch (err) {
|
||||
send({
|
||||
jsonrpc: '2.0',
|
||||
id,
|
||||
result: {
|
||||
content: [{ type: 'text', text: err.message || String(err) }],
|
||||
isError: true,
|
||||
},
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
default:
|
||||
sendError(id, -32601, `Method not found: ${method}`);
|
||||
}
|
||||
} catch (err) {
|
||||
sendError(id, -32603, err.message || String(err));
|
||||
}
|
||||
}
|
||||
|
||||
function handleMessage(msg) {
|
||||
if (!msg || typeof msg !== 'object' || typeof msg.method !== 'string') return;
|
||||
const isRequest = msg.id !== undefined && msg.id !== null;
|
||||
if (isRequest) {
|
||||
handleRequest(msg).catch((err) => sendError(msg.id, -32603, err.message || String(err)));
|
||||
}
|
||||
// Notifications (e.g. notifications/initialized) are intentionally ignored.
|
||||
}
|
||||
|
||||
// --- framing parser ---------------------------------------------------------
|
||||
|
||||
let buf = Buffer.alloc(0);
|
||||
|
||||
function tryParseFrame() {
|
||||
const headerEnd = buf.indexOf('\r\n\r\n');
|
||||
if (headerEnd === -1) return null;
|
||||
const header = buf.subarray(0, headerEnd).toString('ascii');
|
||||
const m = /Content-Length:\s*(\d+)/i.exec(header);
|
||||
if (!m) return null;
|
||||
const len = Number(m[1]);
|
||||
const bodyStart = headerEnd + 4;
|
||||
if (buf.length < bodyStart + len) return null;
|
||||
const body = buf.subarray(bodyStart, bodyStart + len).toString('utf8');
|
||||
buf = buf.subarray(bodyStart + len);
|
||||
return body;
|
||||
}
|
||||
|
||||
function pump() {
|
||||
for (;;) {
|
||||
const body = tryParseFrame();
|
||||
if (body === null) break;
|
||||
let msg;
|
||||
try {
|
||||
msg = JSON.parse(body);
|
||||
} catch (err) {
|
||||
console.error(`[kimi-eyes] invalid JSON frame: ${err.message}`);
|
||||
continue;
|
||||
}
|
||||
handleMessage(msg);
|
||||
}
|
||||
}
|
||||
|
||||
process.stdin.on('data', (chunk) => {
|
||||
buf = Buffer.concat([buf, chunk]);
|
||||
pump();
|
||||
});
|
||||
|
||||
process.stdin.on('end', () => process.exit(0));
|
||||
|
||||
console.error(`[kimi-eyes] MCP server started (v${VERSION}, node ${process.version})`);
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
// kimi-eyes vision core — shared by the MCP server and the setup wizard.
|
||||
// Zero-dependency. Node 18+ (native fetch, AbortSignal.timeout).
|
||||
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const MODELS_DB_PATH = path.join(__dirname, 'models-db.json');
|
||||
|
||||
export const VERSION = '1.0.0';
|
||||
|
||||
export const EXT_MEDIA = {
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
webp: 'image/webp',
|
||||
gif: 'image/gif',
|
||||
bmp: 'image/bmp',
|
||||
};
|
||||
|
||||
export const DEFAULT_PROMPT = 'Describe this image in detail.';
|
||||
|
||||
// A minimal 1x1 PNG used for setup verification.
|
||||
export const ONE_PX_PNG_BASE64 =
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Configuration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Resolution order: environment variables (VISION_*) > config.json.
|
||||
// protocol resolves to 'openai' by default when nothing is set.
|
||||
export function getConfig(env = process.env) {
|
||||
const home = env.KIMI_CODE_HOME || path.join(os.homedir(), '.kimi-code');
|
||||
const cfgPath = path.join(home, 'kimi-eyes', 'config.json');
|
||||
let file = {};
|
||||
try {
|
||||
file = JSON.parse(fs.readFileSync(cfgPath, 'utf8'));
|
||||
} catch {
|
||||
// no config file yet — fine
|
||||
}
|
||||
const envProtocol = (env.VISION_API_PROTOCOL || '').toLowerCase();
|
||||
const protocol =
|
||||
envProtocol === 'openai' || envProtocol === 'anthropic'
|
||||
? envProtocol
|
||||
: file.protocol || 'openai';
|
||||
return {
|
||||
configPath: cfgPath,
|
||||
protocol,
|
||||
apiKey: env.VISION_API_KEY || file.apiKey || '',
|
||||
baseUrl: env.VISION_API_URL || file.baseUrl || '',
|
||||
model: env.VISION_MODEL || file.model || '',
|
||||
mainModel: env.VISION_MAIN_MODEL || file.mainModel || '',
|
||||
};
|
||||
}
|
||||
|
||||
export function checkConfig(cfg) {
|
||||
const missing = [];
|
||||
if (!cfg.apiKey) missing.push('VISION_API_KEY');
|
||||
if (!cfg.baseUrl) missing.push('VISION_API_URL');
|
||||
if (!cfg.model) missing.push('VISION_MODEL');
|
||||
return missing;
|
||||
}
|
||||
|
||||
export function requireConfigured(cfg) {
|
||||
const missing = checkConfig(cfg);
|
||||
if (missing.length > 0) {
|
||||
throw new Error(
|
||||
`Vision API is not configured. Missing: ${missing.join(', ')}. ` +
|
||||
`Run "node setup.mjs" in the kimi-eyes plugin directory (or set the environment variables).`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Models.dev capability database (see scripts/sync-models.mjs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let modelsDbCache = null; // null = not loaded yet, undefined = unavailable
|
||||
|
||||
/** Load the slimmed models.dev database; returns null if unavailable. */
|
||||
export function loadModelsDb() {
|
||||
if (modelsDbCache !== null) return modelsDbCache;
|
||||
try {
|
||||
modelsDbCache = JSON.parse(fs.readFileSync(MODELS_DB_PATH, 'utf8'));
|
||||
} catch {
|
||||
modelsDbCache = undefined;
|
||||
}
|
||||
return modelsDbCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up whether a model name supports image input, using the models.dev cache.
|
||||
* Matching: exact id → provider/id suffix → case-insensitive display name.
|
||||
* @returns {{known: boolean, image: boolean}}
|
||||
*/
|
||||
export function lookupModelCapability(modelName) {
|
||||
const db = loadModelsDb();
|
||||
if (!db) return { known: false, image: false };
|
||||
const n = String(modelName || '').trim();
|
||||
if (!n) return { known: false, image: false };
|
||||
if (db[n]) return { known: true, image: !!db[n].image };
|
||||
const lower = n.toLowerCase();
|
||||
for (const [id, m] of Object.entries(db)) {
|
||||
if (id.toLowerCase().endsWith(`/${lower}`)) {
|
||||
return { known: true, image: !!m.image };
|
||||
}
|
||||
if (typeof m.name === 'string' && m.name.toLowerCase() === lower) {
|
||||
return { known: true, image: !!m.image };
|
||||
}
|
||||
}
|
||||
return { known: false, image: false };
|
||||
}
|
||||
|
||||
/** Text tag used by the setup wizard: exact capability when known, keyword hint otherwise. */
|
||||
export function modelCapabilityTag(modelName, visionHintRe) {
|
||||
const cap = lookupModelCapability(modelName);
|
||||
if (cap.known) return cap.image ? '✓视觉' : '文本';
|
||||
return visionHintRe && visionHintRe.test(modelName) ? '★疑似' : '';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function openaiEndpoint(baseUrl) {
|
||||
let b = String(baseUrl || '').trim().replace(/\/+$/, '');
|
||||
if (!b) return '';
|
||||
if (/\/chat\/completions$/i.test(b)) return b;
|
||||
if (/\/v\d+$/i.test(b)) return `${b}/chat/completions`;
|
||||
return `${b}/v1/chat/completions`;
|
||||
}
|
||||
|
||||
export function anthropicEndpoint(baseUrl) {
|
||||
let b = String(baseUrl || '').trim().replace(/\/+$/, '');
|
||||
if (!b) return '';
|
||||
if (/\/v1\/messages$/i.test(b)) return b;
|
||||
if (/\/v\d+$/i.test(b)) return `${b}/messages`;
|
||||
return `${b}/v1/messages`;
|
||||
}
|
||||
|
||||
export function modelsEndpoint(baseUrl, protocol) {
|
||||
let b = String(baseUrl || '').trim().replace(/\/+$/, '');
|
||||
if (!b) return '';
|
||||
if (/\/v\d+$/i.test(b)) return `${b}/models`;
|
||||
return `${b}/v1/models`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Vision API calls
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FETCH_TIMEOUT_MS = Number(process.env.VISION_FETCH_TIMEOUT_MS) || 60000;
|
||||
const MAX_TOKENS = Number(process.env.VISION_MAX_TOKENS) || 1024;
|
||||
|
||||
/**
|
||||
* Call the user-configured VLM with a base64 image.
|
||||
* @param {{imageBase64:string, mediaType:string, prompt?:string}} input
|
||||
* @param {{protocol:string, apiKey:string, baseUrl:string, model:string}} cfg
|
||||
* @returns {Promise<string>} text description
|
||||
*/
|
||||
export async function callVLM({ imageBase64, mediaType, prompt }, cfg) {
|
||||
requireConfigured(cfg);
|
||||
if (cfg.protocol === 'anthropic') {
|
||||
return callAnthropic(imageBase64, mediaType, prompt, cfg);
|
||||
}
|
||||
return callOpenAI(imageBase64, mediaType, prompt, cfg);
|
||||
}
|
||||
|
||||
async function callOpenAI(imageBase64, mediaType, prompt, cfg) {
|
||||
const endpoint = openaiEndpoint(cfg.baseUrl);
|
||||
const body = {
|
||||
model: cfg.model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: prompt || DEFAULT_PROMPT },
|
||||
{ type: 'image_url', image_url: { url: `data:${mediaType};base64,${imageBase64}` } },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const res = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${cfg.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`Vision API HTTP ${res.status}: ${(await safeText(res)).slice(0, 300)}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
const content = data?.choices?.[0]?.message?.content;
|
||||
if (typeof content === 'string' && content.trim()) return content.trim();
|
||||
if (Array.isArray(content)) {
|
||||
const text = content
|
||||
.filter((p) => p && p.type === 'text' && typeof p.text === 'string')
|
||||
.map((p) => p.text)
|
||||
.join('\n')
|
||||
.trim();
|
||||
if (text) return text;
|
||||
}
|
||||
throw new Error(`Unexpected response from vision API: ${JSON.stringify(data).slice(0, 300)}`);
|
||||
}
|
||||
|
||||
async function callAnthropic(imageBase64, mediaType, prompt, cfg) {
|
||||
const endpoint = anthropicEndpoint(cfg.baseUrl);
|
||||
const body = {
|
||||
model: cfg.model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image', source: { type: 'base64', media_type: mediaType, data: imageBase64 } },
|
||||
{ type: 'text', text: prompt || DEFAULT_PROMPT },
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const res = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': cfg.apiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`Vision API HTTP ${res.status}: ${(await safeText(res)).slice(0, 300)}`);
|
||||
}
|
||||
const data = await res.json();
|
||||
const texts = (Array.isArray(data?.content) ? data.content : [])
|
||||
.filter((p) => p && p.type === 'text' && typeof p.text === 'string')
|
||||
.map((p) => p.text)
|
||||
.join('\n')
|
||||
.trim();
|
||||
if (texts) return texts;
|
||||
throw new Error(`Unexpected response from vision API: ${JSON.stringify(data).slice(0, 300)}`);
|
||||
}
|
||||
|
||||
async function safeText(res) {
|
||||
try {
|
||||
return await res.text();
|
||||
} catch {
|
||||
return '(could not read response body)';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Image sniffing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Validate that a file is a supported image and return base64 + media type.
|
||||
* @returns {{imageBase64:string, mediaType:string}|null}
|
||||
*/
|
||||
export function sniffImage(absPath) {
|
||||
const ext = path.extname(absPath).toLowerCase().replace('.', '');
|
||||
if (!EXT_MEDIA[ext]) return null;
|
||||
let buf;
|
||||
try {
|
||||
buf = fs.readFileSync(absPath);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (buf.length === 0) return null;
|
||||
const ascii4 = buf.toString('ascii', 0, 4);
|
||||
let ok = false;
|
||||
if (ext === 'png') ok = buf.subarray(0, 8).toString('hex').startsWith('89504e47');
|
||||
else if (ext === 'jpg' || ext === 'jpeg') ok = buf.subarray(0, 3).toString('hex') === 'ffd8ff';
|
||||
else if (ext === 'gif') ok = ascii4 === 'GIF8';
|
||||
else if (ext === 'bmp') ok = ascii4 === 'BM';
|
||||
else if (ext === 'webp') {
|
||||
ok = buf.length > 12 && ascii4 === 'RIFF' && buf.toString('ascii', 8, 12) === 'WEBP';
|
||||
}
|
||||
if (!ok) return null;
|
||||
return { imageBase64: buf.toString('base64'), mediaType: EXT_MEDIA[ext] };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clipboard capture
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Grab the image currently in the system clipboard and save it to a temp PNG.
|
||||
* @returns {string} path to the temp PNG file
|
||||
*/
|
||||
export function clipboardImageToTemp() {
|
||||
const out = path.join(
|
||||
os.tmpdir(),
|
||||
`kimi-eyes-clip-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`
|
||||
);
|
||||
if (process.platform === 'win32') {
|
||||
const script = [
|
||||
'Add-Type -AssemblyName System.Windows.Forms;',
|
||||
'$img = [System.Windows.Forms.Clipboard]::GetImage();',
|
||||
"if ($img -eq $null) { Write-Error 'No image in clipboard'; exit 1 };",
|
||||
`$img.Save('${out.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png)`,
|
||||
].join(' ');
|
||||
execFileSync('powershell', ['-NoProfile', '-NonInteractive', '-Command', script], {
|
||||
stdio: 'pipe',
|
||||
timeout: 20000,
|
||||
});
|
||||
} else if (process.platform === 'darwin') {
|
||||
try {
|
||||
execFileSync('pngpaste', [out], { stdio: 'pipe', timeout: 20000 });
|
||||
} catch (e) {
|
||||
throw new Error('Clipboard capture failed. Install "pngpaste" (brew install pngpaste). ' + e.message);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
const png = execFileSync('wl-paste', ['-t', 'image/png', '--no-newline'], {
|
||||
stdio: 'pipe',
|
||||
timeout: 15000,
|
||||
});
|
||||
fs.writeFileSync(out, png);
|
||||
} catch (e1) {
|
||||
try {
|
||||
const png = execFileSync('xclip', ['-selection', 'clipboard', '-t', 'image/png', '-o'], {
|
||||
stdio: 'pipe',
|
||||
timeout: 15000,
|
||||
});
|
||||
fs.writeFileSync(out, png);
|
||||
} catch (e2) {
|
||||
throw new Error(
|
||||
'Clipboard capture failed. Need "wl-paste" (Wayland) or "xclip" (X11). ' +
|
||||
`${e1.message}; ${e2.message}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
Reference in new issue
Block a user