feat(mcp-server): add nexus_vision tool and fix upload MIME detection

- New tool nexus_vision: describe an image via a vision chat model
  (default google/gemini-3.1-flash-lite) and record result in journal.
- NordRouter /media/upload now detects MIME from file extension so
  data: URLs use the correct type (fixes HTTP 415 for images/audio/video).
- HttpClient supports FormData bodies (undici) for future multipart use.
This commit is contained in:
OpenCode
2026-08-18 11:50:29 +07:00
parent c65f9106df
commit b2f57cc361
5 changed files with 135 additions and 6 deletions
@@ -136,7 +136,21 @@ export class NordRouterMediaProvider implements MediaProvider {
throw new NexusError('IO', `Upload file not found: ${input.path}`); throw new NexusError('IO', `Upload file not found: ${input.path}`);
} }
const buf = fs.readFileSync(input.path); const buf = fs.readFileSync(input.path);
const mime = input.mimeType || 'application/octet-stream'; const ext = input.path.split('.').pop()?.toLowerCase() || '';
const mimeByExt: Record<string, string> = {
png: 'image/png',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
webp: 'image/webp',
gif: 'image/gif',
mp4: 'video/mp4',
mov: 'video/quicktime',
wav: 'audio/wav',
mp3: 'audio/mpeg',
ogg: 'audio/ogg',
m4a: 'audio/mp4',
};
const mime = input.mimeType || mimeByExt[ext] || 'application/octet-stream';
const name = input.name || input.path.split('/').pop() || 'file'; const name = input.name || input.path.split('/').pop() || 'file';
const dataUrl = `data:${mime};base64,${buf.toString('base64')}`; const dataUrl = `data:${mime};base64,${buf.toString('base64')}`;
const res = await this.client.http.requestJson<{ url?: string }>({ const res = await this.client.http.requestJson<{ url?: string }>({
+10 -5
View File
@@ -1,4 +1,4 @@
import { fetch, ProxyAgent, type RequestInit, type Response } from 'undici'; import { fetch, ProxyAgent, FormData, type RequestInit, type Response } from 'undici';
import { NexusError, codeFromHttpStatus } from './errors.js'; import { NexusError, codeFromHttpStatus } from './errors.js';
import { sleep } from './polling.js'; import { sleep } from './polling.js';
@@ -83,10 +83,15 @@ export class HttpClient {
const headers: Record<string, string> = { ...this.opts.defaultHeaders, ...options.headers }; const headers: Record<string, string> = { ...this.opts.defaultHeaders, ...options.headers };
let bodyStr: string | undefined; let bodyStr: string | undefined;
let bodyForm: FormData | undefined;
if (options.body !== undefined) { if (options.body !== undefined) {
bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body); if (options.body instanceof FormData) {
if (!headers['Content-Type'] && !headers['content-type']) { bodyForm = options.body;
headers['Content-Type'] = 'application/json'; } else {
bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body);
if (!headers['Content-Type'] && !headers['content-type']) {
headers['Content-Type'] = 'application/json';
}
} }
} }
@@ -96,7 +101,7 @@ export class HttpClient {
const init: RequestInit & { dispatcher?: ProxyAgent } = { const init: RequestInit & { dispatcher?: ProxyAgent } = {
method, method,
headers, headers,
body: bodyStr, body: bodyForm ?? bodyStr,
signal: AbortSignal.timeout(timeoutMs), signal: AbortSignal.timeout(timeoutMs),
}; };
if (this.proxyAgent) init.dispatcher = this.proxyAgent; if (this.proxyAgent) init.dispatcher = this.proxyAgent;
@@ -1,11 +1,13 @@
import { randomUUID } from 'crypto'; import { randomUUID } from 'crypto';
import { import {
NexusError, NexusError,
NordRouterClient,
buildEmbeddedContent, buildEmbeddedContent,
createLogger, createLogger,
type Capability, type Capability,
type MediaResult, type MediaResult,
} from '@nexusai/core'; } from '@nexusai/core';
import * as fs from 'fs';
import type { AppContext } from './context.js'; import type { AppContext } from './context.js';
import { import {
ImageSchema, ImageSchema,
@@ -17,6 +19,7 @@ import {
TtsSchema, TtsSchema,
UsageStatsSchema, UsageStatsSchema,
VideoSchema, VideoSchema,
VisionSchema,
WebSearchSchema, WebSearchSchema,
} from './schemas.js'; } from './schemas.js';
@@ -70,6 +73,8 @@ export class Handlers {
return this.upload(MediaUploadSchema.parse(args)); return this.upload(MediaUploadSchema.parse(args));
case 'nexus_usage_stats': case 'nexus_usage_stats':
return this.usage(UsageStatsSchema.parse(args)); return this.usage(UsageStatsSchema.parse(args));
case 'nexus_vision':
return this.vision(VisionSchema.parse(args));
default: default:
throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`); throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`);
} }
@@ -273,4 +278,90 @@ export class Handlers {
log.debug('usage stats requested'); log.debug('usage stats requested');
return { content: [jsonBlock(payload)] }; return { content: [jsonBlock(payload)] };
} }
private async vision(args: {
provider?: string;
image: string;
prompt: string;
model?: string;
}): Promise<McpResult> {
const providerName = this.providerName(args.provider);
const row = this.ctx.providersStore.get(providerName);
const apiKey = this.ctx.providersStore.getApiKey(providerName);
const baseUrl = row?.baseUrl || process.env[`${providerName.toUpperCase()}_BASE_URL`] || 'https://nordrouter.com';
if (!apiKey) {
throw new NexusError('CONFIG', `No API key for provider "${providerName}"`);
}
let imageUrl = args.image;
if (!args.image.startsWith('http') && !args.image.startsWith('data:')) {
if (!fs.existsSync(args.image)) {
throw new NexusError('IO', `Image file not found: ${args.image}`);
}
const buf = fs.readFileSync(args.image);
const ext = args.image.split('.').pop()?.toLowerCase() || 'png';
const mime = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : ext === 'webp' ? 'image/webp' : 'image/png';
imageUrl = `data:${mime};base64,${buf.toString('base64')}`;
}
const client = new NordRouterClient(apiKey, baseUrl);
const started = Date.now();
const id = randomUUID();
const model = args.model || 'google/gemini-3.1-flash-lite';
try {
const res = await client.http.requestJson<{
choices?: Array<{ message?: { content?: string } }>;
usage?: { total_cost?: number };
}>({
method: 'POST',
path: '/v1/chat/completions',
body: {
model,
messages: [
{
role: 'user',
content: [
{ type: 'text', text: args.prompt },
{ type: 'image_url', image_url: { url: imageUrl } },
],
},
],
},
});
const text = res.choices?.[0]?.message?.content?.trim() || '';
this.ctx.journal.record({
id,
provider: providerName,
capability: 'vision',
model,
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
status: 'done',
costUsd: res.usage?.total_cost ?? null,
filePath: null,
resultUrl: null,
error: null,
durationMs: Date.now() - started,
createdAt: new Date().toISOString(),
});
return { content: [jsonBlock({ provider: providerName, model, description: text })] };
} catch (err) {
const e = NexusError.from(err);
this.ctx.journal.record({
id,
provider: providerName,
capability: 'vision',
model,
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
status: 'failed',
costUsd: null,
filePath: null,
resultUrl: null,
error: `${e.code}: ${e.message}`,
durationMs: Date.now() - started,
createdAt: new Date().toISOString(),
});
throw e;
}
}
} }
@@ -112,6 +112,19 @@ export const WebSearchSchema = z.object({
maxResults: z.number().int().positive().max(50).optional(), maxResults: z.number().int().positive().max(50).optional(),
}); });
export const VisionSchema = z.object({
provider,
image: z.string().describe('Image URL, local file path, or base64/data-URL.'),
prompt: z
.string()
.default('Describe the scene in this image in detail, as if writing a prompt for video generation.')
.describe('What to ask the vision model about the image.'),
model: z
.string()
.optional()
.describe('Vision-capable chat model, e.g. google/gemini-3.1-flash-lite, openai/gpt-5.4-mini.'),
});
export const MediaModelsSchema = z.object({ export const MediaModelsSchema = z.object({
provider, provider,
type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'), type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'),
+6
View File
@@ -9,6 +9,7 @@ import {
TtsSchema, TtsSchema,
UsageStatsSchema, UsageStatsSchema,
VideoSchema, VideoSchema,
VisionSchema,
WebSearchSchema, WebSearchSchema,
} from './schemas.js'; } from './schemas.js';
@@ -75,4 +76,9 @@ export const TOOLS: ToolDef[] = [
'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.', 'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.',
UsageStatsSchema UsageStatsSchema
), ),
make(
'nexus_vision',
'Describe an image using a vision-capable chat model. Returns a text description useful for prompts.',
VisionSchema
),
]; ];