feat: STT transcripts, model catalog+fuzzy resolve, UI audio player & auto-refresh

Core:
- ModelCatalog: periodic background sync of provider models + fuzzy resolve.
  Misspelled model ids map to the nearest match (cheapest on ties); no match
  raises ModelResolutionError with cheapest-first suggestions for the agent.

mcp-server:
- nexus_stt now writes the transcript to a Markdown file in the media dir so it
  appears in the gallery and is readable in the modal.
- media handler resolves model via catalog before generating; returns a note
  when substituting, or a BAD_REQUEST with available models when nothing fits.
- context wires a ModelCatalog per media provider with auto-sync.

admin:
- /media serves correct Content-Type (incl. text/markdown for transcripts).
- Gallery: audio play button on cards, dedicated audio player in modal,
  text/transcript tiles that fetch and render the .md in the modal.
- Header: auto-refresh toggle (5/10/30s) that refreshes the active tab
  without a full page reload; pauses while a modal is open.
This commit is contained in:
OpenCode
2026-08-18 12:25:42 +07:00
parent b2f57cc361
commit 8bd2e89907
8 changed files with 767 additions and 20 deletions
+67 -12
View File
@@ -41,18 +41,46 @@ $('#themeToggle').addEventListener('click', () => {
applyTheme(localStorage.getItem('nexus-theme') || 'dark');
// --- Tabs ---
let activeTab = 'usage';
function refreshActiveTab() {
if (activeTab === 'usage') return loadStats();
if (activeTab === 'gallery') return loadGallery();
if (activeTab === 'providers') return loadProviders();
}
$$('nav button').forEach((btn) => {
btn.addEventListener('click', () => {
$$('nav button').forEach((b) => b.classList.remove('active'));
btn.classList.add('active');
$$('main section').forEach((s) => s.classList.add('hidden'));
$(`#tab-${btn.dataset.tab}`).classList.remove('hidden');
if (btn.dataset.tab === 'usage') loadStats();
if (btn.dataset.tab === 'gallery') loadGallery();
if (btn.dataset.tab === 'providers') loadProviders();
activeTab = btn.dataset.tab;
refreshActiveTab();
});
});
// --- Auto-refresh (no page reload) ---
let autoTimer = null;
function isModalOpen() { return $('#modalRoot').innerHTML.trim() !== ''; }
function setupAutoRefresh() {
if (autoTimer) { clearInterval(autoTimer); autoTimer = null; }
const on = $('#autoRefresh').checked;
localStorage.setItem('nexus-autorefresh', on ? '1' : '0');
localStorage.setItem('nexus-autorefresh-int', $('#autoRefreshInterval').value);
if (!on) return;
const ms = parseInt($('#autoRefreshInterval').value, 10) || 10000;
autoTimer = setInterval(() => {
// Don't disrupt the user while a modal is open or an audio is playing.
if (isModalOpen()) return;
if (document.hidden) return;
refreshActiveTab();
}, ms);
}
$('#autoRefresh').checked = localStorage.getItem('nexus-autorefresh') === '1';
$('#autoRefreshInterval').value = localStorage.getItem('nexus-autorefresh-int') || '10000';
$('#autoRefresh').addEventListener('change', setupAutoRefresh);
$('#autoRefreshInterval').addEventListener('change', setupAutoRefresh);
setupAutoRefresh();
// --- Usage ---
async function loadStats() {
const range = $('#statsRange').value;
@@ -98,19 +126,33 @@ async function loadGallery() {
$$('[data-view]').forEach((el) => el.onclick = () => openModal(el.dataset.view));
$$('[data-del]').forEach((b) => b.onclick = (e) => { e.stopPropagation(); delGeneration(b.dataset.del); });
$$('[data-play]').forEach((b) => b.onclick = (e) => { e.stopPropagation(); openModal(b.dataset.play); });
}
function isAudio(it) { return it.capability === 'tts' || it.capability === 'music'; }
function isText(it) { return it.capability === 'stt' || (it.mediaUrl && /\.md$/i.test(it.mediaUrl)) || it.capability === 'vision'; }
function renderTile(it) {
const cost = it.costUsd != null ? `$${usd(it.costUsd)}` : '';
let media = '';
if (it.mediaUrl) {
if (it.capability === 'image') media = `<div class="thumb" data-view="${it.id}"><img src="${it.mediaUrl}" loading="lazy" /></div>`;
else if (it.capability === 'tts' || it.capability === 'music') media = `<div style="padding:10px"><audio controls src="${it.mediaUrl}"></audio></div>`;
else if (it.capability === 'video') media = `<div class="thumb" data-view="${it.id}"><video style="width:100%;height:150px;background:#000" src="${it.mediaUrl}"></video></div>`;
else media = `<a href="${it.mediaUrl}" target="_blank" style="display:block;padding:20px;text-align:center">Открыть файл</a>`;
if (it.capability === 'image') {
media = `<div class="thumb" data-view="${it.id}"><img src="${it.mediaUrl}" loading="lazy" /></div>`;
} else if (isAudio(it)) {
media = `<div class="thumb audio-thumb" data-view="${it.id}">
<button class="play-btn" data-play="${it.id}" data-src="${it.mediaUrl}" title="Прослушать">▶</button>
</div>`;
} else if (it.capability === 'video') {
media = `<div class="thumb" data-view="${it.id}"><video style="width:100%;height:150px;background:#000" src="${it.mediaUrl}"></video></div>`;
} else if (isText(it)) {
media = `<div class="thumb text-thumb" data-view="${it.id}"><span class="doc-icon">📄</span></div>`;
} else {
media = `<a href="${it.mediaUrl}" target="_blank" style="display:block;padding:20px;text-align:center">Открыть файл</a>`;
}
} else {
const status = it.status === 'failed' ? '⚠ ошибка' : 'нет локального файла';
media = `<div style="height:150px;display:flex;align-items:center;justify-content:center" class="muted">${status}</div>`;
media = `<div class="thumb text-thumb ${it.status === 'failed' ? 'err' : ''}" data-view="${it.id}">
<span class="muted">${status}</span></div>`;
}
return `<div class="tile">
<button class="del" data-del="${it.id}" title="Удалить">✕</button>
@@ -143,10 +185,23 @@ async function openModal(id) {
catch (e) { return toast('Ошибка: ' + e.message); }
let body = '';
if (rec.mediaUrl && rec.capability === 'image') body = `<img src="${rec.mediaUrl}" />`;
else if (rec.mediaUrl && rec.capability === 'video') body = `<video controls autoplay src="${rec.mediaUrl}"></video>`;
else if (rec.mediaUrl && (rec.capability === 'tts' || rec.capability === 'music')) body = `<audio controls autoplay src="${rec.mediaUrl}"></audio>`;
else body = `<div class="muted">Нет предпросмотра для этого типа.</div>`;
if (rec.mediaUrl && rec.capability === 'image') {
body = `<img src="${rec.mediaUrl}" />`;
} else if (rec.mediaUrl && rec.capability === 'video') {
body = `<video controls autoplay src="${rec.mediaUrl}"></video>`;
} else if (rec.mediaUrl && (rec.capability === 'tts' || rec.capability === 'music')) {
body = `<div class="audio-player">
<div class="audio-visual">🎵</div>
<audio controls autoplay src="${rec.mediaUrl}"></audio>
</div>`;
} else if (rec.mediaUrl && /\.md$/i.test(rec.mediaUrl)) {
// Transcript / text document — fetch and render.
let text = '';
try { text = await (await fetch(rec.mediaUrl)).text(); } catch { text = '(не удалось загрузить текст)'; }
body = `<div class="doc-view">${esc(text)}</div>`;
} else {
body = `<div class="muted">Нет предпросмотра для этого типа.</div>`;
}
const kv = `
<div class="kv">Модель: ${esc(rec.model)}</div>
+29
View File
@@ -85,6 +85,26 @@
.modal-body img, .modal-body video { max-width: 88vw; max-height: 72vh; display: block; margin: 0 auto; }
.modal-body audio { width: 60vw; max-width: 600px; }
.kv { color: var(--muted); font-size: 12px; margin-top: 10px; word-break: break-all; }
/* Auto-refresh control */
.autorefresh { display: flex; align-items: center; gap: 6px; font-size: 12px; color: var(--muted); }
.autorefresh select { width: auto; padding: 4px 6px; }
.autorefresh input[type="checkbox"] { width: auto; }
/* Audio card thumb + play button */
.audio-thumb, .text-thumb { height: 150px; display: flex; align-items: center; justify-content: center;
cursor: pointer; background: linear-gradient(135deg, var(--panel2), var(--panel)); }
.text-thumb.err { background: rgba(242,109,109,.08); }
.play-btn { width: 56px; height: 56px; border-radius: 50%; border: none; cursor: pointer;
background: var(--accent); color: #fff; font-size: 20px; line-height: 1;
box-shadow: 0 6px 18px var(--shadow); transition: transform .1s; }
.play-btn:hover { transform: scale(1.08); }
.doc-icon { font-size: 42px; opacity: .8; }
/* Modal audio player */
.audio-player { display: flex; flex-direction: column; align-items: center; gap: 16px; padding: 20px; }
.audio-visual { font-size: 64px; }
/* Transcript / doc view */
.doc-view { white-space: pre-wrap; word-break: break-word; font: 13px/1.6 ui-monospace, SFMono-Regular, Menlo, monospace;
background: var(--panel2); border: 1px solid var(--line); border-radius: 8px; padding: 16px;
max-width: 80vw; max-height: 70vh; overflow: auto; text-align: left; }
</style>
</head>
<body>
@@ -92,6 +112,14 @@
<h1>NexusAI Admin</h1>
<span class="badge" id="cryptoBadge"></span>
<span class="spacer"></span>
<label class="autorefresh" title="Автообновление активной вкладки">
<input type="checkbox" id="autoRefresh" /> Авто-обновление
<select id="autoRefreshInterval">
<option value="5000">5с</option>
<option value="10000" selected>10с</option>
<option value="30000">30с</option>
</select>
</label>
<button id="themeToggle" title="Переключить тему">🌙 Тёмная</button>
</header>
<nav>
@@ -141,6 +169,7 @@
<option value="music">music</option>
<option value="video">video</option>
<option value="stt">stt</option>
<option value="vision">vision</option>
<option value="search">search</option>
</select>
</div>
+9
View File
@@ -27,6 +27,15 @@ async function main(): Promise<void> {
if (!full.startsWith(mediaDir + path.sep) || !fs.existsSync(full)) {
return reply.code(404).send({ error: 'not found' });
}
const ext = path.extname(full).toLowerCase();
const CT: Record<string, string> = {
'.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
'.webp': 'image/webp', '.gif': 'image/gif',
'.mp4': 'video/mp4', '.webm': 'video/webm',
'.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.ogg': 'audio/ogg', '.m4a': 'audio/mp4',
'.md': 'text/markdown; charset=utf-8', '.txt': 'text/plain; charset=utf-8', '.json': 'application/json',
};
if (CT[ext]) reply.header('Content-Type', CT[ext]);
const stream = fs.createReadStream(full);
reply.header('Cache-Control', 'private, max-age=60');
return reply.send(stream);
+1
View File
@@ -11,5 +11,6 @@ export * from './media/storage.js';
export * from './media/content.js';
export * from './providers/types.js';
export * from './providers/registry.js';
export * from './providers/model-catalog.js';
export * from './providers/nordrouter/index.js';
export * from './providers/local-stt/whisper.js';
@@ -0,0 +1,205 @@
import { createLogger } from '../shared/logger.js';
import type { Capability, MediaProvider, ModelInfo } from './types.js';
const log = createLogger('model-catalog');
/** Map a NordRouter model `type` to our Capability. */
export function typeToCapability(type?: string): Capability | undefined {
switch ((type || '').toLowerCase()) {
case 'audio':
return 'tts';
case 'image':
return 'image';
case 'video':
return 'video';
case 'music':
return 'music';
default:
return undefined;
}
}
export interface ResolveResult {
/** Exact match found. */
exact: boolean;
/** The chosen model id (may differ from the requested one when fuzzy-matched). */
model: string;
/** Human note explaining substitution, if any. */
note?: string;
/** Alternatives in the same category (for error messages / agent guidance). */
suggestions?: Array<{ id: string; label?: string; estUsd?: number }>;
}
export class ModelResolutionError extends Error {
constructor(
message: string,
public readonly capability: Capability,
public readonly suggestions: Array<{ id: string; label?: string; estUsd?: number }>
) {
super(message);
this.name = 'ModelResolutionError';
}
}
/** Normalize an id/label for fuzzy comparison. */
function norm(s: string): string {
return s.toLowerCase().replace(/^(image|video|audio|music)\//, '').replace(/[^a-z0-9]+/g, '');
}
/** Length of the longest common subsequence. */
function lcs(a: string, b: string): number {
const m = a.length;
const n = b.length;
if (!m || !n) return 0;
let prev = new Array(n + 1).fill(0);
let cur = new Array(n + 1).fill(0);
for (let i = 1; i <= m; i++) {
for (let j = 1; j <= n; j++) {
cur[j] = a[i - 1] === b[j - 1] ? prev[j - 1] + 1 : Math.max(prev[j], cur[j - 1]);
}
[prev, cur] = [cur, prev];
}
return prev[n];
}
/** Cheap similarity in [0,1] combining token overlap and character-level LCS. */
function similarity(a: string, b: string): number {
const na = norm(a);
const nb = norm(b);
if (!na || !nb) return 0;
if (na === nb) return 1;
if (na.includes(nb) || nb.includes(na)) return 0.85;
// Token overlap on the original strings.
const ta = new Set(a.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
const tb = new Set(b.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean));
let inter = 0;
for (const t of ta) if (tb.has(t)) inter += 1;
const tokenScore = ta.size && tb.size ? inter / Math.max(ta.size, tb.size) : 0;
// Character-level LCS on normalized strings (handles minor typos / punctuation).
const charScore = lcs(na, nb) / Math.max(na.length, nb.length);
return Math.max(tokenScore, charScore);
}
/**
* Caches a provider's model catalog and resolves (possibly misspelled) model ids
* to the nearest valid model within the same capability, preferring the cheapest
* when several are similar. Periodically refreshed in the background.
*/
export class ModelCatalog {
private models: ModelInfo[] = [];
private lastSync = 0;
private syncing?: Promise<void>;
private timer?: NodeJS.Timeout;
constructor(
private readonly provider: MediaProvider,
private readonly opts: { ttlMs?: number; minSimilarity?: number } = {}
) {}
get ttlMs(): number {
return this.opts.ttlMs ?? 10 * 60 * 1000;
}
get minSimilarity(): number {
return this.opts.minSimilarity ?? 0.4;
}
/** Start periodic background sync. Safe to call once. */
startAutoSync(intervalMs = 30 * 60 * 1000): void {
if (this.timer) return;
// Fire once soon, then on interval. Unref so it never blocks process exit.
this.timer = setInterval(() => void this.sync().catch(() => {}), intervalMs);
if (typeof this.timer.unref === 'function') this.timer.unref();
void this.sync().catch(() => {});
}
stopAutoSync(): void {
if (this.timer) {
clearInterval(this.timer);
this.timer = undefined;
}
}
async sync(force = false): Promise<void> {
if (!this.provider.listModels) return;
if (!force && Date.now() - this.lastSync < this.ttlMs && this.models.length) return;
if (this.syncing) return this.syncing;
this.syncing = (async () => {
try {
const models = await this.provider.listModels!();
this.models = models;
this.lastSync = Date.now();
log.debug(`synced ${models.length} models for provider "${this.provider.name}"`);
} catch (err) {
log.warn('model sync failed', err instanceof Error ? err.message : String(err));
} finally {
this.syncing = undefined;
}
})();
return this.syncing;
}
/** All known models, syncing lazily if the cache is empty/stale. */
async all(): Promise<ModelInfo[]> {
await this.sync();
return this.models;
}
/** Models for a given capability. */
async byCapability(capability: Capability): Promise<ModelInfo[]> {
const all = await this.all();
return all.filter((m) => typeToCapability(m.type) === capability);
}
/**
* Resolve a requested model id for a capability.
* - Exact id match → return as-is.
* - Otherwise pick the most similar model above `minSimilarity`; when several
* are equally close, prefer the cheapest.
* - If nothing is close enough, throw ModelResolutionError with suggestions.
*/
async resolve(requested: string, capability: Capability): Promise<ResolveResult> {
const pool = await this.byCapability(capability);
// If we couldn't load a catalog at all, don't block the call — trust the id.
if (!pool.length) {
return { exact: true, model: requested, note: 'catalog unavailable; passing id through' };
}
const exact = pool.find((m) => m.id === requested);
if (exact) return { exact: true, model: exact.id };
const scored = pool
.map((m) => ({ m, score: Math.max(similarity(requested, m.id), similarity(requested, m.label || '')) }))
.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
return (a.m.estUsd ?? Infinity) - (b.m.estUsd ?? Infinity); // tie-break: cheapest
});
const best = scored[0];
const suggestions = pool
.slice()
.sort((a, b) => (a.estUsd ?? Infinity) - (b.estUsd ?? Infinity))
.slice(0, 10)
.map((m) => ({ id: m.id, label: m.label, estUsd: m.estUsd }));
if (best && best.score >= this.minSimilarity) {
return {
exact: false,
model: best.m.id,
note: `Model "${requested}" not found; using closest match "${best.m.id}"${
best.m.estUsd != null ? ` (est $${best.m.estUsd})` : ''
}.`,
suggestions,
};
}
throw new ModelResolutionError(
`No model matching "${requested}" for capability "${capability}". Ask the user to pick one of the available models.`,
capability,
suggestions
);
}
}
+11 -1
View File
@@ -3,6 +3,7 @@ import {
KeyCipher,
KeyResolver,
MediaStorage,
ModelCatalog,
ProviderRegistry,
ProvidersStore,
createNordRouterProviders,
@@ -20,6 +21,8 @@ export interface AppContext {
storage: MediaStorage;
journal: Journal;
providersStore: ProvidersStore;
/** Model catalogs keyed by provider name (for sync + fuzzy resolution). */
catalogs: Map<string, ModelCatalog>;
/** Effective default provider name after resolving DB/env. */
defaultProvider: string;
}
@@ -47,6 +50,7 @@ export function buildContext(): AppContext {
const resolved = resolver.resolveAll(BUILTINS);
const defaultProvider = resolver.resolveDefault(resolved);
const catalogs = new Map<string, ModelCatalog>();
for (const p of resolved) {
if (!p.enabled) {
@@ -69,6 +73,12 @@ export function buildContext(): AppContext {
Object.defineProperty(nr.search, 'name', { value: p.name });
registry.registerMedia(nr.media);
registry.registerSearch(nr.search);
// Model catalog with periodic background sync + fuzzy resolution.
if (p.apiKey) {
const catalog = new ModelCatalog(nr.media);
catalog.startAutoSync();
catalogs.set(p.name, catalog);
}
log.info(`registered provider "${p.name}" (media + search, key: ${p.keySource})`);
} catch (err) {
log.warn(`failed to register "${p.name}"`, err instanceof Error ? err.message : String(err));
@@ -82,5 +92,5 @@ export function buildContext(): AppContext {
log.info('registered local STT provider');
log.info(`default provider: ${defaultProvider}`);
return { config, registry, storage, journal, providersStore, defaultProvider };
return { config, registry, storage, journal, providersStore, catalogs, defaultProvider };
}
+73 -7
View File
@@ -1,5 +1,6 @@
import { randomUUID } from 'crypto';
import {
ModelResolutionError,
NexusError,
NordRouterClient,
buildEmbeddedContent,
@@ -8,6 +9,7 @@ import {
type MediaResult,
} from '@nexusai/core';
import * as fs from 'fs';
import * as path from 'path';
import type { AppContext } from './context.js';
import {
ImageSchema,
@@ -89,13 +91,36 @@ export class Handlers {
const started = Date.now();
const id = randomUUID();
// Resolve the model against the synced catalog: exact, nearest-cheapest, or
// a helpful error listing available models for the agent to relay.
let model = args.model;
let resolutionNote: string | undefined;
const catalog = this.ctx.catalogs.get(providerName);
if (catalog) {
try {
const r = await catalog.resolve(args.model, capability);
model = r.model;
resolutionNote = r.note;
} catch (err) {
if (err instanceof ModelResolutionError) {
throw new NexusError('BAD_REQUEST', err.message, {
capability: err.capability,
requested: args.model,
available: err.suggestions,
hint: 'Ask the user to choose one of the available models (sorted cheapest first).',
});
}
throw err;
}
}
try {
const result = await provider.generate(capability, { model: args.model, input: args.input });
const result = await provider.generate(capability, { model, input: args.input });
this.ctx.journal.record({
id,
provider: providerName,
capability,
model: args.model,
model,
inputSummary: summarize(args.input),
status: 'done',
costUsd: result.costUsd ?? null,
@@ -105,14 +130,14 @@ export class Handlers {
durationMs: Date.now() - started,
createdAt: new Date().toISOString(),
});
return this.mediaResult(result, args.output?.embed ?? false);
return this.mediaResult(result, args.output?.embed ?? false, resolutionNote);
} catch (err) {
const e = NexusError.from(err);
this.ctx.journal.record({
id,
provider: providerName,
capability,
model: args.model,
model,
inputSummary: summarize(args.input),
status: 'failed',
costUsd: null,
@@ -126,7 +151,7 @@ export class Handlers {
}
}
private mediaResult(result: MediaResult, embed: boolean): McpResult {
private mediaResult(result: MediaResult, embed: boolean, note?: string): McpResult {
const content: McpContent[] = [
jsonBlock({
provider: result.provider,
@@ -138,6 +163,7 @@ export class Handlers {
result_url: result.resultUrl,
result_url_2: result.resultUrl2,
cost_usd: result.costUsd,
...(note ? { note } : {}),
}),
];
@@ -159,6 +185,9 @@ export class Handlers {
const id = randomUUID();
try {
const res = await provider.transcribe(args.input);
// Persist the transcript as a Markdown file so it shows up in the gallery
// and can be read in the modal viewer.
const transcriptPath = this.writeTranscript(id, res, args.input);
this.ctx.journal.record({
id,
provider: providerName,
@@ -167,8 +196,7 @@ export class Handlers {
inputSummary: summarize({ ...args.input, audioBase64: args.input.audioBase64 ? '[base64]' : undefined }),
status: 'done',
costUsd: 0,
// STT does not produce a local media file; the source audio path stays in inputSummary.
filePath: null,
filePath: transcriptPath,
resultUrl: null,
error: null,
durationMs: Date.now() - started,
@@ -205,6 +233,44 @@ export class Handlers {
}
}
/** Write the STT transcript to a Markdown file in the media dir. Returns the path or null. */
private writeTranscript(
id: string,
res: { text: string; language?: string; durationSec?: number; segments?: Array<{ start: number; end: number; text: string }> },
input: { audioPath?: string; audioUrl?: string; model?: string }
): string | null {
try {
const source = input.audioPath || input.audioUrl || '(inline audio)';
const lines: string[] = [
'# Транскрипция',
'',
`- **Источник:** ${source}`,
`- **Язык:** ${res.language ?? 'auto'}`,
`- **Длительность:** ${res.durationSec != null ? res.durationSec + ' сек' : '—'}`,
`- **Модель:** ${input.model || 'default'}`,
`- **Создано:** ${new Date().toISOString()}`,
'',
'## Текст',
'',
res.text || '_(пусто)_',
];
if (res.segments?.length) {
lines.push('', '## Сегменты', '');
for (const s of res.segments) {
lines.push(`- \`[${s.start.toFixed(2)}${s.end.toFixed(2)}]\` ${s.text.trim()}`);
}
}
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
const filePath = path.join(this.ctx.config.mediaDir, `transcript-${stamp}-${id.slice(0, 8)}.md`);
fs.mkdirSync(this.ctx.config.mediaDir, { recursive: true });
fs.writeFileSync(filePath, lines.join('\n'), 'utf8');
return filePath;
} catch (err) {
log.warn('failed to write transcript', err instanceof Error ? err.message : String(err));
return null;
}
}
private async search(args: {
provider?: string;
query: string;