feat(mcp-server): add nexus_vision tool and fix upload MIME detection
- New tool nexus_vision: describe an image via a vision chat model (default google/gemini-3.1-flash-lite) and record result in journal. - NordRouter /media/upload now detects MIME from file extension so data: URLs use the correct type (fixes HTTP 415 for images/audio/video). - HttpClient supports FormData bodies (undici) for future multipart use.
This commit is contained in:
@@ -136,7 +136,21 @@ export class NordRouterMediaProvider implements MediaProvider {
|
|||||||
throw new NexusError('IO', `Upload file not found: ${input.path}`);
|
throw new NexusError('IO', `Upload file not found: ${input.path}`);
|
||||||
}
|
}
|
||||||
const buf = fs.readFileSync(input.path);
|
const buf = fs.readFileSync(input.path);
|
||||||
const mime = input.mimeType || 'application/octet-stream';
|
const ext = input.path.split('.').pop()?.toLowerCase() || '';
|
||||||
|
const mimeByExt: Record<string, string> = {
|
||||||
|
png: 'image/png',
|
||||||
|
jpg: 'image/jpeg',
|
||||||
|
jpeg: 'image/jpeg',
|
||||||
|
webp: 'image/webp',
|
||||||
|
gif: 'image/gif',
|
||||||
|
mp4: 'video/mp4',
|
||||||
|
mov: 'video/quicktime',
|
||||||
|
wav: 'audio/wav',
|
||||||
|
mp3: 'audio/mpeg',
|
||||||
|
ogg: 'audio/ogg',
|
||||||
|
m4a: 'audio/mp4',
|
||||||
|
};
|
||||||
|
const mime = input.mimeType || mimeByExt[ext] || 'application/octet-stream';
|
||||||
const name = input.name || input.path.split('/').pop() || 'file';
|
const name = input.name || input.path.split('/').pop() || 'file';
|
||||||
const dataUrl = `data:${mime};base64,${buf.toString('base64')}`;
|
const dataUrl = `data:${mime};base64,${buf.toString('base64')}`;
|
||||||
const res = await this.client.http.requestJson<{ url?: string }>({
|
const res = await this.client.http.requestJson<{ url?: string }>({
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { fetch, ProxyAgent, type RequestInit, type Response } from 'undici';
|
import { fetch, ProxyAgent, FormData, type RequestInit, type Response } from 'undici';
|
||||||
import { NexusError, codeFromHttpStatus } from './errors.js';
|
import { NexusError, codeFromHttpStatus } from './errors.js';
|
||||||
import { sleep } from './polling.js';
|
import { sleep } from './polling.js';
|
||||||
|
|
||||||
@@ -83,12 +83,17 @@ export class HttpClient {
|
|||||||
|
|
||||||
const headers: Record<string, string> = { ...this.opts.defaultHeaders, ...options.headers };
|
const headers: Record<string, string> = { ...this.opts.defaultHeaders, ...options.headers };
|
||||||
let bodyStr: string | undefined;
|
let bodyStr: string | undefined;
|
||||||
|
let bodyForm: FormData | undefined;
|
||||||
if (options.body !== undefined) {
|
if (options.body !== undefined) {
|
||||||
|
if (options.body instanceof FormData) {
|
||||||
|
bodyForm = options.body;
|
||||||
|
} else {
|
||||||
bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body);
|
bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body);
|
||||||
if (!headers['Content-Type'] && !headers['content-type']) {
|
if (!headers['Content-Type'] && !headers['content-type']) {
|
||||||
headers['Content-Type'] = 'application/json';
|
headers['Content-Type'] = 'application/json';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let lastError: Error | undefined;
|
let lastError: Error | undefined;
|
||||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||||
@@ -96,7 +101,7 @@ export class HttpClient {
|
|||||||
const init: RequestInit & { dispatcher?: ProxyAgent } = {
|
const init: RequestInit & { dispatcher?: ProxyAgent } = {
|
||||||
method,
|
method,
|
||||||
headers,
|
headers,
|
||||||
body: bodyStr,
|
body: bodyForm ?? bodyStr,
|
||||||
signal: AbortSignal.timeout(timeoutMs),
|
signal: AbortSignal.timeout(timeoutMs),
|
||||||
};
|
};
|
||||||
if (this.proxyAgent) init.dispatcher = this.proxyAgent;
|
if (this.proxyAgent) init.dispatcher = this.proxyAgent;
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
import { randomUUID } from 'crypto';
|
import { randomUUID } from 'crypto';
|
||||||
import {
|
import {
|
||||||
NexusError,
|
NexusError,
|
||||||
|
NordRouterClient,
|
||||||
buildEmbeddedContent,
|
buildEmbeddedContent,
|
||||||
createLogger,
|
createLogger,
|
||||||
type Capability,
|
type Capability,
|
||||||
type MediaResult,
|
type MediaResult,
|
||||||
} from '@nexusai/core';
|
} from '@nexusai/core';
|
||||||
|
import * as fs from 'fs';
|
||||||
import type { AppContext } from './context.js';
|
import type { AppContext } from './context.js';
|
||||||
import {
|
import {
|
||||||
ImageSchema,
|
ImageSchema,
|
||||||
@@ -17,6 +19,7 @@ import {
|
|||||||
TtsSchema,
|
TtsSchema,
|
||||||
UsageStatsSchema,
|
UsageStatsSchema,
|
||||||
VideoSchema,
|
VideoSchema,
|
||||||
|
VisionSchema,
|
||||||
WebSearchSchema,
|
WebSearchSchema,
|
||||||
} from './schemas.js';
|
} from './schemas.js';
|
||||||
|
|
||||||
@@ -70,6 +73,8 @@ export class Handlers {
|
|||||||
return this.upload(MediaUploadSchema.parse(args));
|
return this.upload(MediaUploadSchema.parse(args));
|
||||||
case 'nexus_usage_stats':
|
case 'nexus_usage_stats':
|
||||||
return this.usage(UsageStatsSchema.parse(args));
|
return this.usage(UsageStatsSchema.parse(args));
|
||||||
|
case 'nexus_vision':
|
||||||
|
return this.vision(VisionSchema.parse(args));
|
||||||
default:
|
default:
|
||||||
throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`);
|
throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`);
|
||||||
}
|
}
|
||||||
@@ -273,4 +278,90 @@ export class Handlers {
|
|||||||
log.debug('usage stats requested');
|
log.debug('usage stats requested');
|
||||||
return { content: [jsonBlock(payload)] };
|
return { content: [jsonBlock(payload)] };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async vision(args: {
|
||||||
|
provider?: string;
|
||||||
|
image: string;
|
||||||
|
prompt: string;
|
||||||
|
model?: string;
|
||||||
|
}): Promise<McpResult> {
|
||||||
|
const providerName = this.providerName(args.provider);
|
||||||
|
const row = this.ctx.providersStore.get(providerName);
|
||||||
|
const apiKey = this.ctx.providersStore.getApiKey(providerName);
|
||||||
|
const baseUrl = row?.baseUrl || process.env[`${providerName.toUpperCase()}_BASE_URL`] || 'https://nordrouter.com';
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new NexusError('CONFIG', `No API key for provider "${providerName}"`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let imageUrl = args.image;
|
||||||
|
if (!args.image.startsWith('http') && !args.image.startsWith('data:')) {
|
||||||
|
if (!fs.existsSync(args.image)) {
|
||||||
|
throw new NexusError('IO', `Image file not found: ${args.image}`);
|
||||||
|
}
|
||||||
|
const buf = fs.readFileSync(args.image);
|
||||||
|
const ext = args.image.split('.').pop()?.toLowerCase() || 'png';
|
||||||
|
const mime = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : ext === 'webp' ? 'image/webp' : 'image/png';
|
||||||
|
imageUrl = `data:${mime};base64,${buf.toString('base64')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new NordRouterClient(apiKey, baseUrl);
|
||||||
|
const started = Date.now();
|
||||||
|
const id = randomUUID();
|
||||||
|
const model = args.model || 'google/gemini-3.1-flash-lite';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await client.http.requestJson<{
|
||||||
|
choices?: Array<{ message?: { content?: string } }>;
|
||||||
|
usage?: { total_cost?: number };
|
||||||
|
}>({
|
||||||
|
method: 'POST',
|
||||||
|
path: '/v1/chat/completions',
|
||||||
|
body: {
|
||||||
|
model,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: 'user',
|
||||||
|
content: [
|
||||||
|
{ type: 'text', text: args.prompt },
|
||||||
|
{ type: 'image_url', image_url: { url: imageUrl } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const text = res.choices?.[0]?.message?.content?.trim() || '';
|
||||||
|
this.ctx.journal.record({
|
||||||
|
id,
|
||||||
|
provider: providerName,
|
||||||
|
capability: 'vision',
|
||||||
|
model,
|
||||||
|
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
|
||||||
|
status: 'done',
|
||||||
|
costUsd: res.usage?.total_cost ?? null,
|
||||||
|
filePath: null,
|
||||||
|
resultUrl: null,
|
||||||
|
error: null,
|
||||||
|
durationMs: Date.now() - started,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
return { content: [jsonBlock({ provider: providerName, model, description: text })] };
|
||||||
|
} catch (err) {
|
||||||
|
const e = NexusError.from(err);
|
||||||
|
this.ctx.journal.record({
|
||||||
|
id,
|
||||||
|
provider: providerName,
|
||||||
|
capability: 'vision',
|
||||||
|
model,
|
||||||
|
inputSummary: summarize({ prompt: args.prompt, image: args.image }),
|
||||||
|
status: 'failed',
|
||||||
|
costUsd: null,
|
||||||
|
filePath: null,
|
||||||
|
resultUrl: null,
|
||||||
|
error: `${e.code}: ${e.message}`,
|
||||||
|
durationMs: Date.now() - started,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,6 +112,19 @@ export const WebSearchSchema = z.object({
|
|||||||
maxResults: z.number().int().positive().max(50).optional(),
|
maxResults: z.number().int().positive().max(50).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const VisionSchema = z.object({
|
||||||
|
provider,
|
||||||
|
image: z.string().describe('Image URL, local file path, or base64/data-URL.'),
|
||||||
|
prompt: z
|
||||||
|
.string()
|
||||||
|
.default('Describe the scene in this image in detail, as if writing a prompt for video generation.')
|
||||||
|
.describe('What to ask the vision model about the image.'),
|
||||||
|
model: z
|
||||||
|
.string()
|
||||||
|
.optional()
|
||||||
|
.describe('Vision-capable chat model, e.g. google/gemini-3.1-flash-lite, openai/gpt-5.4-mini.'),
|
||||||
|
});
|
||||||
|
|
||||||
export const MediaModelsSchema = z.object({
|
export const MediaModelsSchema = z.object({
|
||||||
provider,
|
provider,
|
||||||
type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'),
|
type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'),
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
TtsSchema,
|
TtsSchema,
|
||||||
UsageStatsSchema,
|
UsageStatsSchema,
|
||||||
VideoSchema,
|
VideoSchema,
|
||||||
|
VisionSchema,
|
||||||
WebSearchSchema,
|
WebSearchSchema,
|
||||||
} from './schemas.js';
|
} from './schemas.js';
|
||||||
|
|
||||||
@@ -75,4 +76,9 @@ export const TOOLS: ToolDef[] = [
|
|||||||
'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.',
|
'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.',
|
||||||
UsageStatsSchema
|
UsageStatsSchema
|
||||||
),
|
),
|
||||||
|
make(
|
||||||
|
'nexus_vision',
|
||||||
|
'Describe an image using a vision-capable chat model. Returns a text description useful for prompts.',
|
||||||
|
VisionSchema
|
||||||
|
),
|
||||||
];
|
];
|
||||||
|
|||||||
Reference in New Issue
Block a user