feat(mcp-server): add nexus_vision tool and fix upload MIME detection
- New tool nexus_vision: describe an image via a vision chat model (default google/gemini-3.1-flash-lite) and record result in journal. - NordRouter /media/upload now detects MIME from file extension so data: URLs use the correct type (fixes HTTP 415 for images/audio/video). - HttpClient supports FormData bodies (undici) for future multipart use.
This commit is contained in:
@@ -136,7 +136,21 @@ export class NordRouterMediaProvider implements MediaProvider {
|
||||
throw new NexusError('IO', `Upload file not found: ${input.path}`);
|
||||
}
|
||||
const buf = fs.readFileSync(input.path);
|
||||
const mime = input.mimeType || 'application/octet-stream';
|
||||
const ext = input.path.split('.').pop()?.toLowerCase() || '';
|
||||
const mimeByExt: Record<string, string> = {
|
||||
png: 'image/png',
|
||||
jpg: 'image/jpeg',
|
||||
jpeg: 'image/jpeg',
|
||||
webp: 'image/webp',
|
||||
gif: 'image/gif',
|
||||
mp4: 'video/mp4',
|
||||
mov: 'video/quicktime',
|
||||
wav: 'audio/wav',
|
||||
mp3: 'audio/mpeg',
|
||||
ogg: 'audio/ogg',
|
||||
m4a: 'audio/mp4',
|
||||
};
|
||||
const mime = input.mimeType || mimeByExt[ext] || 'application/octet-stream';
|
||||
const name = input.name || input.path.split('/').pop() || 'file';
|
||||
const dataUrl = `data:${mime};base64,${buf.toString('base64')}`;
|
||||
const res = await this.client.http.requestJson<{ url?: string }>({
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { fetch, ProxyAgent, type RequestInit, type Response } from 'undici';
|
||||
import { fetch, ProxyAgent, FormData, type RequestInit, type Response } from 'undici';
|
||||
import { NexusError, codeFromHttpStatus } from './errors.js';
|
||||
import { sleep } from './polling.js';
|
||||
|
||||
@@ -83,12 +83,17 @@ export class HttpClient {
|
||||
|
||||
const headers: Record<string, string> = { ...this.opts.defaultHeaders, ...options.headers };
|
||||
let bodyStr: string | undefined;
|
||||
let bodyForm: FormData | undefined;
|
||||
if (options.body !== undefined) {
|
||||
if (options.body instanceof FormData) {
|
||||
bodyForm = options.body;
|
||||
} else {
|
||||
bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body);
|
||||
if (!headers['Content-Type'] && !headers['content-type']) {
|
||||
headers['Content-Type'] = 'application/json';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let lastError: Error | undefined;
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
@@ -96,7 +101,7 @@ export class HttpClient {
|
||||
const init: RequestInit & { dispatcher?: ProxyAgent } = {
|
||||
method,
|
||||
headers,
|
||||
body: bodyStr,
|
||||
body: bodyForm ?? bodyStr,
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
};
|
||||
if (this.proxyAgent) init.dispatcher = this.proxyAgent;
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import {
|
||||
NexusError,
|
||||
NordRouterClient,
|
||||
buildEmbeddedContent,
|
||||
createLogger,
|
||||
type Capability,
|
||||
type MediaResult,
|
||||
} from '@nexusai/core';
|
||||
import * as fs from 'fs';
|
||||
import type { AppContext } from './context.js';
|
||||
import {
|
||||
ImageSchema,
|
||||
@@ -17,6 +19,7 @@ import {
|
||||
TtsSchema,
|
||||
UsageStatsSchema,
|
||||
VideoSchema,
|
||||
VisionSchema,
|
||||
WebSearchSchema,
|
||||
} from './schemas.js';
|
||||
|
||||
@@ -70,6 +73,8 @@ export class Handlers {
|
||||
return this.upload(MediaUploadSchema.parse(args));
|
||||
case 'nexus_usage_stats':
|
||||
return this.usage(UsageStatsSchema.parse(args));
|
||||
case 'nexus_vision':
|
||||
return this.vision(VisionSchema.parse(args));
|
||||
default:
|
||||
throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`);
|
||||
}
|
||||
@@ -273,4 +278,90 @@ export class Handlers {
|
||||
log.debug('usage stats requested');
|
||||
return { content: [jsonBlock(payload)] };
|
||||
}
|
||||
|
||||
private async vision(args: {
|
||||
provider?: string;
|
||||
image: string;
|
||||
prompt: string;
|
||||
model?: string;
|
||||
}): Promise<McpResult> {
|
||||
const providerName = this.providerName(args.provider);
|
||||
const row = this.ctx.providersStore.get(providerName);
|
||||
const apiKey = this.ctx.providersStore.getApiKey(providerName);
|
||||
const baseUrl = row?.baseUrl || process.env[`${providerName.toUpperCase()}_BASE_URL`] || 'https://nordrouter.com';
|
||||
if (!apiKey) {
|
||||
throw new NexusError('CONFIG', `No API key for provider "${providerName}"`);
|
||||
}
|
||||
|
||||
let imageUrl = args.image;
|
||||
if (!args.image.startsWith('http') && !args.image.startsWith('data:')) {
|
||||
if (!fs.existsSync(args.image)) {
|
||||
throw new NexusError('IO', `Image file not found: ${args.image}`);
|
||||
}
|
||||
const buf = fs.readFileSync(args.image);
|
||||
const ext = args.image.split('.').pop()?.toLowerCase() || 'png';
|
||||
const mime = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : ext === 'webp' ? 'image/webp' : 'image/png';
|
||||
imageUrl = `data:${mime};base64,${buf.toString('base64')}`;
|
||||
}
|
||||
|
||||
const client = new NordRouterClient(apiKey, baseUrl);
|
||||
const started = Date.now();
|
||||
const id = randomUUID();
|
||||
const model = args.model || 'google/gemini-3.1-flash-lite';
|
||||
|
||||
try {
|
||||
const res = await client.http.requestJson<{
|
||||
choices?: Array<{ message?: { content?: string } }>;
|
||||
usage?: { total_cost?: number };
|
||||
}>({
|
||||
method: 'POST',
|
||||
path: '/v1/chat/completions',
|
||||
body: {
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: args.prompt },
|
||||
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const text = res.choices?.[0]?.message?.content?.trim() || '';
|
||||
this.ctx.journal.record({
|
||||
id,
|
||||
provider: providerName,
|
||||
capability: 'vision',
|
||||
model,
|
||||
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
|
||||
status: 'done',
|
||||
costUsd: res.usage?.total_cost ?? null,
|
||||
filePath: null,
|
||||
resultUrl: null,
|
||||
error: null,
|
||||
durationMs: Date.now() - started,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
return { content: [jsonBlock({ provider: providerName, model, description: text })] };
|
||||
} catch (err) {
|
||||
const e = NexusError.from(err);
|
||||
this.ctx.journal.record({
|
||||
id,
|
||||
provider: providerName,
|
||||
capability: 'vision',
|
||||
model,
|
||||
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
|
||||
status: 'failed',
|
||||
costUsd: null,
|
||||
filePath: null,
|
||||
resultUrl: null,
|
||||
error: `${e.code}: ${e.message}`,
|
||||
durationMs: Date.now() - started,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,6 +112,19 @@ export const WebSearchSchema = z.object({
|
||||
maxResults: z.number().int().positive().max(50).optional(),
|
||||
});
|
||||
|
||||
export const VisionSchema = z.object({
|
||||
provider,
|
||||
image: z.string().describe('Image URL, local file path, or base64/data-URL.'),
|
||||
prompt: z
|
||||
.string()
|
||||
.default('Describe the scene in this image in detail, as if writing a prompt for video generation.')
|
||||
.describe('What to ask the vision model about the image.'),
|
||||
model: z
|
||||
.string()
|
||||
.optional()
|
||||
.describe('Vision-capable chat model, e.g. google/gemini-3.1-flash-lite, openai/gpt-5.4-mini.'),
|
||||
});
|
||||
|
||||
export const MediaModelsSchema = z.object({
|
||||
provider,
|
||||
type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'),
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
TtsSchema,
|
||||
UsageStatsSchema,
|
||||
VideoSchema,
|
||||
VisionSchema,
|
||||
WebSearchSchema,
|
||||
} from './schemas.js';
|
||||
|
||||
@@ -75,4 +76,9 @@ export const TOOLS: ToolDef[] = [
|
||||
'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.',
|
||||
UsageStatsSchema
|
||||
),
|
||||
make(
|
||||
'nexus_vision',
|
||||
'Describe an image using a vision-capable chat model. Returns a text description useful for prompts.',
|
||||
VisionSchema
|
||||
),
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user