From b2f57cc361f23a08d1f0900e202216f3b57acfcc Mon Sep 17 00:00:00 2001 From: OpenCode Date: Tue, 18 Aug 2026 11:50:29 +0700 Subject: [PATCH] feat(mcp-server): add nexus_vision tool and fix upload MIME detection - New tool nexus_vision: describe an image via a vision chat model (default google/gemini-3.1-flash-lite) and record result in journal. - NordRouter /media/upload now detects MIME from file extension so data: URLs use the correct type (fixes HTTP 415 for images/audio/video). - HttpClient supports FormData bodies (undici) for future multipart use. --- .../core/src/providers/nordrouter/media.ts | 16 +++- NexusAI/packages/core/src/shared/http.ts | 15 ++- NexusAI/packages/mcp-server/src/handlers.ts | 91 +++++++++++++++++++ NexusAI/packages/mcp-server/src/schemas.ts | 13 +++ NexusAI/packages/mcp-server/src/tools.ts | 6 ++ 5 files changed, 135 insertions(+), 6 deletions(-) diff --git a/NexusAI/packages/core/src/providers/nordrouter/media.ts b/NexusAI/packages/core/src/providers/nordrouter/media.ts index 805b8a9..0a8316d 100644 --- a/NexusAI/packages/core/src/providers/nordrouter/media.ts +++ b/NexusAI/packages/core/src/providers/nordrouter/media.ts @@ -136,7 +136,21 @@ export class NordRouterMediaProvider implements MediaProvider { throw new NexusError('IO', `Upload file not found: ${input.path}`); } const buf = fs.readFileSync(input.path); - const mime = input.mimeType || 'application/octet-stream'; + const ext = input.path.split('.').pop()?.toLowerCase() || ''; + const mimeByExt: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + webp: 'image/webp', + gif: 'image/gif', + mp4: 'video/mp4', + mov: 'video/quicktime', + wav: 'audio/wav', + mp3: 'audio/mpeg', + ogg: 'audio/ogg', + m4a: 'audio/mp4', + }; + const mime = input.mimeType || mimeByExt[ext] || 'application/octet-stream'; const name = input.name || input.path.split('/').pop() || 'file'; const dataUrl = `data:${mime};base64,${buf.toString('base64')}`; const res = await this.client.http.requestJson<{ url?: string }>({ diff --git a/NexusAI/packages/core/src/shared/http.ts b/NexusAI/packages/core/src/shared/http.ts index 5104a6b..d51b897 100644 --- a/NexusAI/packages/core/src/shared/http.ts +++ b/NexusAI/packages/core/src/shared/http.ts @@ -1,4 +1,4 @@ -import { fetch, ProxyAgent, type RequestInit, type Response } from 'undici'; +import { fetch, ProxyAgent, FormData, type RequestInit, type Response } from 'undici'; import { NexusError, codeFromHttpStatus } from './errors.js'; import { sleep } from './polling.js'; @@ -83,10 +83,15 @@ export class HttpClient { const headers: Record = { ...this.opts.defaultHeaders, ...options.headers }; let bodyStr: string | undefined; + let bodyForm: FormData | undefined; if (options.body !== undefined) { - bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body); - if (!headers['Content-Type'] && !headers['content-type']) { - headers['Content-Type'] = 'application/json'; + if (options.body instanceof FormData) { + bodyForm = options.body; + } else { + bodyStr = typeof options.body === 'string' ? options.body : JSON.stringify(options.body); + if (!headers['Content-Type'] && !headers['content-type']) { + headers['Content-Type'] = 'application/json'; + } } } @@ -96,7 +101,7 @@ export class HttpClient { const init: RequestInit & { dispatcher?: ProxyAgent } = { method, headers, - body: bodyStr, + body: bodyForm ?? bodyStr, signal: AbortSignal.timeout(timeoutMs), }; if (this.proxyAgent) init.dispatcher = this.proxyAgent; diff --git a/NexusAI/packages/mcp-server/src/handlers.ts b/NexusAI/packages/mcp-server/src/handlers.ts index b585bbc..2bf3f39 100644 --- a/NexusAI/packages/mcp-server/src/handlers.ts +++ b/NexusAI/packages/mcp-server/src/handlers.ts @@ -1,11 +1,13 @@ import { randomUUID } from 'crypto'; import { NexusError, + NordRouterClient, buildEmbeddedContent, createLogger, type Capability, type MediaResult, } from '@nexusai/core'; +import * as fs from 'fs'; import type { AppContext } from './context.js'; import { ImageSchema, @@ -17,6 +19,7 @@ import { TtsSchema, UsageStatsSchema, VideoSchema, + VisionSchema, WebSearchSchema, } from './schemas.js'; @@ -70,6 +73,8 @@ export class Handlers { return this.upload(MediaUploadSchema.parse(args)); case 'nexus_usage_stats': return this.usage(UsageStatsSchema.parse(args)); + case 'nexus_vision': + return this.vision(VisionSchema.parse(args)); default: throw new NexusError('NOT_FOUND', `Unknown tool: ${name}`); } @@ -273,4 +278,90 @@ export class Handlers { log.debug('usage stats requested'); return { content: [jsonBlock(payload)] }; } + + private async vision(args: { + provider?: string; + image: string; + prompt: string; + model?: string; + }): Promise { + const providerName = this.providerName(args.provider); + const row = this.ctx.providersStore.get(providerName); + const apiKey = this.ctx.providersStore.getApiKey(providerName); + const baseUrl = row?.baseUrl || process.env[`${providerName.toUpperCase()}_BASE_URL`] || 'https://nordrouter.com'; + if (!apiKey) { + throw new NexusError('CONFIG', `No API key for provider "${providerName}"`); + } + + let imageUrl = args.image; + if (!args.image.startsWith('http') && !args.image.startsWith('data:')) { + if (!fs.existsSync(args.image)) { + throw new NexusError('IO', `Image file not found: ${args.image}`); + } + const buf = fs.readFileSync(args.image); + const ext = args.image.split('.').pop()?.toLowerCase() || 'png'; + const mime = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : ext === 'webp' ? 'image/webp' : 'image/png'; + imageUrl = `data:${mime};base64,${buf.toString('base64')}`; + } + + const client = new NordRouterClient(apiKey, baseUrl); + const started = Date.now(); + const id = randomUUID(); + const model = args.model || 'google/gemini-3.1-flash-lite'; + + try { + const res = await client.http.requestJson<{ + choices?: Array<{ message?: { content?: string } }>; + usage?: { total_cost?: number }; + }>({ + method: 'POST', + path: '/v1/chat/completions', + body: { + model, + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: args.prompt }, + { type: 'image_url', image_url: { url: imageUrl } }, + ], + }, + ], + }, + }); + const text = res.choices?.[0]?.message?.content?.trim() || ''; + this.ctx.journal.record({ + id, + provider: providerName, + capability: 'vision', + model, + inputSummary: summarize({ prompt: args.prompt, image: args.image }), + status: 'done', + costUsd: res.usage?.total_cost ?? null, + filePath: null, + resultUrl: null, + error: null, + durationMs: Date.now() - started, + createdAt: new Date().toISOString(), + }); + return { content: [jsonBlock({ provider: providerName, model, description: text })] }; + } catch (err) { + const e = NexusError.from(err); + this.ctx.journal.record({ + id, + provider: providerName, + capability: 'vision', + model, + inputSummary: summarize({ prompt: args.prompt, image: args.image }), + status: 'failed', + costUsd: null, + filePath: null, + resultUrl: null, + error: `${e.code}: ${e.message}`, + durationMs: Date.now() - started, + createdAt: new Date().toISOString(), + }); + throw e; + } + } } diff --git a/NexusAI/packages/mcp-server/src/schemas.ts b/NexusAI/packages/mcp-server/src/schemas.ts index 595b920..25c24f3 100644 --- a/NexusAI/packages/mcp-server/src/schemas.ts +++ b/NexusAI/packages/mcp-server/src/schemas.ts @@ -112,6 +112,19 @@ export const WebSearchSchema = z.object({ maxResults: z.number().int().positive().max(50).optional(), }); +export const VisionSchema = z.object({ + provider, + image: z.string().describe('Image URL, local file path, or base64/data-URL.'), + prompt: z + .string() + .default('Describe the scene in this image in detail, as if writing a prompt for video generation.') + .describe('What to ask the vision model about the image.'), + model: z + .string() + .optional() + .describe('Vision-capable chat model, e.g. google/gemini-3.1-flash-lite, openai/gpt-5.4-mini.'), +}); + export const MediaModelsSchema = z.object({ provider, type: z.enum(['image', 'video', 'music', 'audio', 'upscale']).optional().describe('Filter by model type.'), diff --git a/NexusAI/packages/mcp-server/src/tools.ts b/NexusAI/packages/mcp-server/src/tools.ts index 351cbd8..e0e87ad 100644 --- a/NexusAI/packages/mcp-server/src/tools.ts +++ b/NexusAI/packages/mcp-server/src/tools.ts @@ -9,6 +9,7 @@ import { TtsSchema, UsageStatsSchema, VideoSchema, + VisionSchema, WebSearchSchema, } from './schemas.js'; @@ -75,4 +76,9 @@ export const TOOLS: ToolDef[] = [ 'Show usage and spend recorded in the local journal (totals by provider, capability, day) and optionally recent generations.', UsageStatsSchema ), + make( + 'nexus_vision', + 'Describe an image using a vision-capable chat model. Returns a text description useful for prompts.', + VisionSchema + ), ];