feat(mcp-server): add nexus_vision tool and fix upload MIME detection

- New tool nexus_vision: describe an image via a vision chat model
  (default google/gemini-3.1-flash-lite) and record result in journal.
- NordRouter /media/upload now detects MIME from file extension so
  data: URLs use the correct type (fixes HTTP 415 for images/audio/video).
- HttpClient supports FormData bodies (undici) for future multipart use.
This commit is contained in:
OpenCode
2026-08-18 11:50:29 +07:00
parent c65f9106df
commit b2f57cc361
5 changed files with 135 additions and 6 deletions
@@ -1,11 +1,13 @@
import { randomUUID } from 'crypto';
import {
NexusError,
NordRouterClient,
buildEmbeddedContent,
createLogger,
type Capability,
type MediaResult,
} from '@nexusai/core';
import * as fs from 'fs';
import type { AppContext } from './context.js';
import {
ImageSchema,
@@ -17,6 +19,7 @@ import {
TtsSchema,
UsageStatsSchema,
VideoSchema,
VisionSchema,
WebSearchSchema,
} from './schemas.js';
@@ -70,6 +73,8 @@ export class Handlers {
return this.upload(MediaUploadSchema.parse(args));
case 'nexus_usage_stats':
return this.usage(UsageStatsSchema.parse(args));
case 'nexus_vision':
return this.vision(VisionSchema.parse(args));
default:
throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`);
}
@@ -273,4 +278,90 @@ export class Handlers {
log.debug('usage stats requested');
return { content: [jsonBlock(payload)] };
}
private async vision(args: {
provider?: string;
image: string;
prompt: string;
model?: string;
}): Promise<McpResult> {
const providerName = this.providerName(args.provider);
const row = this.ctx.providersStore.get(providerName);
const apiKey = this.ctx.providersStore.getApiKey(providerName);
const baseUrl = row?.baseUrl || process.env[`${providerName.toUpperCase()}_BASE_URL`] || 'https://nordrouter.com';
if (!apiKey) {
throw new NexusError('CONFIG', `No API key for provider "${providerName}"`);
}
let imageUrl = args.image;
if (!args.image.startsWith('http') && !args.image.startsWith('data:')) {
if (!fs.existsSync(args.image)) {
throw new NexusError('IO', `Image file not found: ${args.image}`);
}
const buf = fs.readFileSync(args.image);
const ext = args.image.split('.').pop()?.toLowerCase() || 'png';
const mime = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : ext === 'webp' ? 'image/webp' : 'image/png';
imageUrl = `data:${mime};base64,${buf.toString('base64')}`;
}
const client = new NordRouterClient(apiKey, baseUrl);
const started = Date.now();
const id = randomUUID();
const model = args.model || 'google/gemini-3.1-flash-lite';
try {
const res = await client.http.requestJson<{
choices?: Array<{ message?: { content?: string } }>;
usage?: { total_cost?: number };
}>({
method: 'POST',
path: '/v1/chat/completions',
body: {
model,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: args.prompt },
{ type: 'image_url', image_url: { url: imageUrl } },
],
},
],
},
});
const text = res.choices?.[0]?.message?.content?.trim() || '';
this.ctx.journal.record({
id,
provider: providerName,
capability: 'vision',
model,
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
status: 'done',
costUsd: res.usage?.total_cost ?? null,
filePath: null,
resultUrl: null,
error: null,
durationMs: Date.now() - started,
createdAt: new Date().toISOString(),
});
return { content: [jsonBlock({ provider: providerName, model, description: text })] };
} catch (err) {
const e = NexusError.from(err);
this.ctx.journal.record({
id,
provider: providerName,
capability: 'vision',
model,
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
status: 'failed',
costUsd: null,
filePath: null,
resultUrl: null,
error: `${e.code}: ${e.message}`,
durationMs: Date.now() - started,
createdAt: new Date().toISOString(),
});
throw e;
}
}
}
@@ -112,6 +112,19 @@ export const WebSearchSchema = z.object({
maxResults: z.number().int().positive().max(50).optional(),
});
export const VisionSchema = z.object({
provider,
image: z.string().describe('Image URL, local file path, or base64/data-URL.'),
prompt: z
.string()
.default('Describe the scene in this image in detail, as if writing a prompt for video generation.')
.describe('What to ask the vision model about the image.'),
model: z
.string()
.optional()
.describe('Vision-capable chat model, e.g. google/gemini-3.1-flash-lite, openai/gpt-5.4-mini.'),
});
export const MediaModelsSchema = z.object({
provider,
type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'),
+6
View File
@@ -9,6 +9,7 @@ import {
TtsSchema,
UsageStatsSchema,
VideoSchema,
VisionSchema,
WebSearchSchema,
} from './schemas.js';
@@ -75,4 +76,9 @@ export const TOOLS: ToolDef[] = [
'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.',
UsageStatsSchema
),
make(
'nexus_vision',
'Describe an image using a vision-capable chat model. Returns a text description useful for prompts.',
VisionSchema
),
];