native first
This commit is contained in:
Илья Глазунов
@@ -612,7 +612,7 @@ export class CheatingDaddyApp extends LitElement {
|
||||
|
||||
async handleStart() {
|
||||
const prefs = await cheatingDaddy.storage.getPreferences();
|
||||
const providerMode = prefs.providerMode || "byok";
|
||||
const providerMode = prefs.providerMode || "local";
|
||||
|
||||
if (providerMode === "local") {
|
||||
const success = await cheatingDaddy.initializeLocal(this.selectedProfile);
|
||||
|
||||
@@ -1,143 +1,160 @@
|
||||
import { html, css, LitElement } from '../../assets/lit-core-2.7.4.min.js';
|
||||
import { unifiedPageStyles } from './sharedPageStyles.js';
|
||||
import { html, css, LitElement } from "../../assets/lit-core-2.7.4.min.js";
|
||||
import { unifiedPageStyles } from "./sharedPageStyles.js";
|
||||
|
||||
export class AICustomizeView extends LitElement {
|
||||
static styles = [
|
||||
unifiedPageStyles,
|
||||
css`
|
||||
.unified-page {
|
||||
height: 100%;
|
||||
}
|
||||
.unified-wrap {
|
||||
height: 100%;
|
||||
}
|
||||
section.surface {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.form-grid {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.form-group.vertical {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
textarea.control {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
overflow-y: auto;
|
||||
min-height: 0;
|
||||
}
|
||||
`,
|
||||
static styles = [
|
||||
unifiedPageStyles,
|
||||
css`
|
||||
.unified-page {
|
||||
height: 100%;
|
||||
}
|
||||
.unified-wrap {
|
||||
height: 100%;
|
||||
}
|
||||
section.surface {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.form-grid {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.form-group.vertical {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
textarea.control {
|
||||
flex: 1;
|
||||
resize: none;
|
||||
overflow-y: auto;
|
||||
min-height: 0;
|
||||
}
|
||||
`,
|
||||
];
|
||||
|
||||
static properties = {
|
||||
selectedProfile: { type: String },
|
||||
onProfileChange: { type: Function },
|
||||
_context: { state: true },
|
||||
_providerMode: { state: true },
|
||||
};
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.selectedProfile = "interview";
|
||||
this.onProfileChange = () => {};
|
||||
this._context = "";
|
||||
this._providerMode = "local";
|
||||
this._loadFromStorage();
|
||||
}
|
||||
|
||||
async _loadFromStorage() {
|
||||
try {
|
||||
const prefs = await cheatingDaddy.storage.getPreferences();
|
||||
this._context = prefs.customPrompt || "";
|
||||
this._providerMode = prefs.providerMode || "local";
|
||||
this.requestUpdate();
|
||||
} catch (error) {
|
||||
console.error("Error loading AI customize storage:", error);
|
||||
}
|
||||
}
|
||||
|
||||
_handleProfileChange(e) {
|
||||
this.onProfileChange(e.target.value);
|
||||
}
|
||||
|
||||
async _handleProviderModeChange(e) {
|
||||
this._providerMode = e.target.value;
|
||||
await cheatingDaddy.storage.updatePreference(
|
||||
"providerMode",
|
||||
this._providerMode,
|
||||
);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveContext(val) {
|
||||
this._context = val;
|
||||
await cheatingDaddy.storage.updatePreference("customPrompt", val);
|
||||
}
|
||||
|
||||
_getProfileName(profile) {
|
||||
const names = {
|
||||
interview: "Job Interview",
|
||||
sales: "Sales Call",
|
||||
meeting: "Business Meeting",
|
||||
presentation: "Presentation",
|
||||
negotiation: "Negotiation",
|
||||
exam: "Exam Assistant",
|
||||
};
|
||||
return names[profile] || profile;
|
||||
}
|
||||
|
||||
render() {
|
||||
const profiles = [
|
||||
{ value: "interview", label: "Job Interview" },
|
||||
{ value: "sales", label: "Sales Call" },
|
||||
{ value: "meeting", label: "Business Meeting" },
|
||||
{ value: "presentation", label: "Presentation" },
|
||||
{ value: "negotiation", label: "Negotiation" },
|
||||
{ value: "exam", label: "Exam Assistant" },
|
||||
];
|
||||
|
||||
static properties = {
|
||||
selectedProfile: { type: String },
|
||||
onProfileChange: { type: Function },
|
||||
_context: { state: true },
|
||||
_providerMode: { state: true },
|
||||
};
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.selectedProfile = 'interview';
|
||||
this.onProfileChange = () => {};
|
||||
this._context = '';
|
||||
this._providerMode = 'byok';
|
||||
this._loadFromStorage();
|
||||
}
|
||||
|
||||
async _loadFromStorage() {
|
||||
try {
|
||||
const prefs = await cheatingDaddy.storage.getPreferences();
|
||||
this._context = prefs.customPrompt || '';
|
||||
this._providerMode = prefs.providerMode || 'byok';
|
||||
this.requestUpdate();
|
||||
} catch (error) {
|
||||
console.error('Error loading AI customize storage:', error);
|
||||
}
|
||||
}
|
||||
|
||||
_handleProfileChange(e) {
|
||||
this.onProfileChange(e.target.value);
|
||||
}
|
||||
|
||||
async _handleProviderModeChange(e) {
|
||||
this._providerMode = e.target.value;
|
||||
await cheatingDaddy.storage.updatePreference('providerMode', this._providerMode);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveContext(val) {
|
||||
this._context = val;
|
||||
await cheatingDaddy.storage.updatePreference('customPrompt', val);
|
||||
}
|
||||
|
||||
_getProfileName(profile) {
|
||||
const names = {
|
||||
interview: 'Job Interview',
|
||||
sales: 'Sales Call',
|
||||
meeting: 'Business Meeting',
|
||||
presentation: 'Presentation',
|
||||
negotiation: 'Negotiation',
|
||||
exam: 'Exam Assistant',
|
||||
};
|
||||
return names[profile] || profile;
|
||||
}
|
||||
|
||||
render() {
|
||||
const profiles = [
|
||||
{ value: 'interview', label: 'Job Interview' },
|
||||
{ value: 'sales', label: 'Sales Call' },
|
||||
{ value: 'meeting', label: 'Business Meeting' },
|
||||
{ value: 'presentation', label: 'Presentation' },
|
||||
{ value: 'negotiation', label: 'Negotiation' },
|
||||
{ value: 'exam', label: 'Exam Assistant' },
|
||||
];
|
||||
|
||||
return html`
|
||||
<div class="unified-page">
|
||||
<div class="unified-wrap">
|
||||
<div>
|
||||
<div class="page-title">AI Context</div>
|
||||
</div>
|
||||
|
||||
<section class="surface">
|
||||
<div class="form-grid">
|
||||
<div class="form-group">
|
||||
<label class="form-label">Regime</label>
|
||||
<select class="control" .value=${this._providerMode} @change=${this._handleProviderModeChange}>
|
||||
<option value="byok">BYOK (API Keys)</option>
|
||||
<option value="local">Local AI (Ollama)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="form-label">Profile</label>
|
||||
<select class="control" .value=${this.selectedProfile} @change=${this._handleProfileChange}>
|
||||
${profiles.map(profile => html`<option value=${profile.value}>${profile.label}</option>`)}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group vertical">
|
||||
<label class="form-label">Custom Instructions</label>
|
||||
<textarea
|
||||
class="control"
|
||||
placeholder="Resume details, role requirements, constraints..."
|
||||
.value=${this._context}
|
||||
@input=${e => this._saveContext(e.target.value)}
|
||||
></textarea>
|
||||
<div class="form-help">Sent as context at session start. Keep it short.</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
return html`
|
||||
<div class="unified-page">
|
||||
<div class="unified-wrap">
|
||||
<div>
|
||||
<div class="page-title">AI Context</div>
|
||||
</div>
|
||||
|
||||
<section class="surface">
|
||||
<div class="form-grid">
|
||||
<div class="form-group">
|
||||
<label class="form-label">Regime</label>
|
||||
<select
|
||||
class="control"
|
||||
.value=${this._providerMode}
|
||||
@change=${this._handleProviderModeChange}
|
||||
>
|
||||
<option value="byok">BYOK (API Keys)</option>
|
||||
<option value="local">Local AI (LM Studio)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="form-label">Profile</label>
|
||||
<select
|
||||
class="control"
|
||||
.value=${this.selectedProfile}
|
||||
@change=${this._handleProfileChange}
|
||||
>
|
||||
${profiles.map(
|
||||
(profile) =>
|
||||
html`<option value=${profile.value}>
|
||||
${profile.label}
|
||||
</option>`,
|
||||
)}
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group vertical">
|
||||
<label class="form-label">Custom Instructions</label>
|
||||
<textarea
|
||||
class="control"
|
||||
placeholder="Resume details, role requirements, constraints..."
|
||||
.value=${this._context}
|
||||
@input=${(e) => this._saveContext(e.target.value)}
|
||||
></textarea>
|
||||
<div class="form-help">
|
||||
Sent as context at session start. Keep it short.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
customElements.define('ai-customize-view', AICustomizeView);
|
||||
customElements.define("ai-customize-view", AICustomizeView);
|
||||
|
||||
@@ -208,7 +208,7 @@ export class CustomizeView extends LitElement {
|
||||
this.onImageQualityChange = () => {};
|
||||
this.onLayoutModeChange = () => {};
|
||||
this.googleSearchEnabled = true;
|
||||
this.providerMode = "byok";
|
||||
this.providerMode = "local";
|
||||
this.isClearing = false;
|
||||
this.isRestoring = false;
|
||||
this.clearStatusMessage = "";
|
||||
@@ -232,7 +232,7 @@ export class CustomizeView extends LitElement {
|
||||
cheatingDaddy.storage.getKeybinds(),
|
||||
]);
|
||||
this.googleSearchEnabled = prefs.googleSearchEnabled ?? true;
|
||||
this.providerMode = prefs.providerMode || "byok";
|
||||
this.providerMode = prefs.providerMode || "local";
|
||||
this.backgroundTransparency = prefs.backgroundTransparency ?? 0.8;
|
||||
this.fontSize = prefs.fontSize ?? 20;
|
||||
this.audioMode = prefs.audioMode ?? "speaker_only";
|
||||
@@ -664,7 +664,7 @@ export class CustomizeView extends LitElement {
|
||||
@change=${this.handleProviderModeChange}
|
||||
>
|
||||
<option value="byok">BYOK (API Keys)</option>
|
||||
<option value="local">Local AI (Ollama)</option>
|
||||
<option value="local">Local AI (LM Studio)</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+165
-138
@@ -501,6 +501,11 @@ export class MainView extends LitElement {
|
||||
_whisperModel: { state: true },
|
||||
_customWhisperModel: { state: true },
|
||||
_showLocalHelp: { state: true },
|
||||
_localLlmBaseUrl: { state: true },
|
||||
_localLlmModel: { state: true },
|
||||
_localLlmApiKey: { state: true },
|
||||
_localSttUrl: { state: true },
|
||||
_localSttLanguage: { state: true },
|
||||
};
|
||||
|
||||
constructor() {
|
||||
@@ -513,7 +518,7 @@ export class MainView extends LitElement {
|
||||
this.whisperDownloading = false;
|
||||
this.whisperProgress = null;
|
||||
|
||||
this._mode = "byok";
|
||||
this._mode = "local";
|
||||
this._token = "";
|
||||
this._geminiKey = "";
|
||||
this._groqKey = "";
|
||||
@@ -528,6 +533,11 @@ export class MainView extends LitElement {
|
||||
this._tokenError = false;
|
||||
this._keyError = false;
|
||||
this._showLocalHelp = false;
|
||||
this._localLlmBaseUrl = "http://127.0.0.1:1234/v1";
|
||||
this._localLlmModel = "";
|
||||
this._localLlmApiKey = "";
|
||||
this._localSttUrl = "ws://127.0.0.1:8765/v1/asr/stream";
|
||||
this._localSttLanguage = "en-US";
|
||||
this._ollamaHost = "http://127.0.0.1:11434";
|
||||
this._ollamaModel = "llama3.1";
|
||||
this._whisperModel = "Xenova/whisper-small";
|
||||
@@ -549,7 +559,7 @@ export class MainView extends LitElement {
|
||||
cheatingDaddy.storage.getCredentials().catch(() => ({})),
|
||||
]);
|
||||
|
||||
this._mode = prefs.providerMode || "byok";
|
||||
this._mode = prefs.providerMode || "local";
|
||||
|
||||
// Load keys
|
||||
this._token = "";
|
||||
@@ -571,6 +581,13 @@ export class MainView extends LitElement {
|
||||
this._responseProvider = prefs.responseProvider || "gemini";
|
||||
|
||||
// Load local AI settings
|
||||
this._localLlmBaseUrl =
|
||||
prefs.localLlmBaseUrl || "http://127.0.0.1:1234/v1";
|
||||
this._localLlmModel = prefs.localLlmModel || "";
|
||||
this._localLlmApiKey = prefs.localLlmApiKey || "";
|
||||
this._localSttUrl =
|
||||
prefs.localSttUrl || "ws://127.0.0.1:8765/v1/asr/stream";
|
||||
this._localSttLanguage = prefs.localSttLanguage || "en-US";
|
||||
this._ollamaHost = prefs.ollamaHost || "http://127.0.0.1:11434";
|
||||
this._ollamaModel = prefs.ollamaModel || "llama3.1";
|
||||
this._whisperModel = prefs.whisperModel || "Xenova/whisper-small";
|
||||
@@ -917,6 +934,51 @@ export class MainView extends LitElement {
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveLocalLlmBaseUrl(val) {
|
||||
this._localLlmBaseUrl = val;
|
||||
await cheatingDaddy.storage.updatePreference("localLlmBaseUrl", val);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveLocalLlmModel(val) {
|
||||
this._localLlmModel = val;
|
||||
await cheatingDaddy.storage.updatePreference("localLlmModel", val);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveLocalLlmApiKey(val) {
|
||||
this._localLlmApiKey = val;
|
||||
await cheatingDaddy.storage.updatePreference("localLlmApiKey", val);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveLocalSttUrl(val) {
|
||||
this._localSttUrl = val;
|
||||
await cheatingDaddy.storage.updatePreference("localSttUrl", val);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
async _saveLocalSttLanguage(val) {
|
||||
this._localSttLanguage = val;
|
||||
await cheatingDaddy.storage.updatePreference("localSttLanguage", val);
|
||||
this.requestUpdate();
|
||||
}
|
||||
|
||||
_isLoopbackUrl(value) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
return (
|
||||
hostname === "localhost" ||
|
||||
hostname === "127.0.0.1" ||
|
||||
hostname === "::1" ||
|
||||
hostname === "[::1]"
|
||||
);
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async _saveOllamaHost(val) {
|
||||
this._ollamaHost = val;
|
||||
await cheatingDaddy.storage.updatePreference("ollamaHost", val);
|
||||
@@ -1002,8 +1064,13 @@ export class MainView extends LitElement {
|
||||
return;
|
||||
}
|
||||
} else if (this._mode === "local") {
|
||||
// Local mode doesn't need API keys, just Ollama host
|
||||
if (!this._ollamaHost.trim()) {
|
||||
if (
|
||||
!this._localLlmBaseUrl.trim() ||
|
||||
!this._localLlmModel.trim() ||
|
||||
!this._localSttUrl.trim()
|
||||
) {
|
||||
this._keyError = true;
|
||||
this.requestUpdate();
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -1258,105 +1325,88 @@ export class MainView extends LitElement {
|
||||
// ── Local AI mode ──
|
||||
|
||||
_renderLocalMode() {
|
||||
const llmIsLocal = this._isLoopbackUrl(this._localLlmBaseUrl);
|
||||
const sttIsLocal = this._isLoopbackUrl(this._localSttUrl);
|
||||
|
||||
return html`
|
||||
<div class="form-group">
|
||||
<label class="form-label">Ollama Host</label>
|
||||
<label class="form-label">LM Studio Base URL</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="http://127.0.0.1:11434"
|
||||
.value=${this._ollamaHost}
|
||||
@input=${(e) => this._saveOllamaHost(e.target.value)}
|
||||
/>
|
||||
<div class="form-hint">Ollama must be running locally</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label class="form-label">Ollama Model</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="llama3.1"
|
||||
.value=${this._ollamaModel}
|
||||
@input=${(e) => this._saveOllamaModel(e.target.value)}
|
||||
placeholder="http://127.0.0.1:1234/v1"
|
||||
.value=${this._localLlmBaseUrl}
|
||||
@input=${(e) => this._saveLocalLlmBaseUrl(e.target.value)}
|
||||
class=${this._keyError && !this._localLlmBaseUrl.trim()
|
||||
? "error"
|
||||
: ""}
|
||||
/>
|
||||
<div class="form-hint">
|
||||
Run
|
||||
<code
|
||||
style="font-family: var(--font-mono); font-size: 11px; background: var(--bg-elevated); padding: 1px 4px; border-radius: 3px;"
|
||||
>ollama pull ${this._ollamaModel}</code
|
||||
>
|
||||
first
|
||||
LM Studio local server endpoint for OpenAI-compatible chat
|
||||
${!llmIsLocal
|
||||
? html`<span style="color: var(--warning, #d97706);">
|
||||
· not a localhost URL
|
||||
</span>`
|
||||
: ""}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<div class="whisper-label-row">
|
||||
<label class="form-label">Whisper Model</label>
|
||||
${this.whisperDownloading
|
||||
? html`<div class="whisper-spinner"></div>`
|
||||
<label class="form-label">LM Studio Model ID</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="gemma-4 or the exact loaded model id"
|
||||
.value=${this._localLlmModel}
|
||||
@input=${(e) => this._saveLocalLlmModel(e.target.value)}
|
||||
class=${this._keyError && !this._localLlmModel.trim() ? "error" : ""}
|
||||
/>
|
||||
<div class="form-hint">
|
||||
Manual only: use the model identifier shown in LM Studio
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label class="form-label">LM Studio API Key</label>
|
||||
<input
|
||||
type="password"
|
||||
placeholder="Optional for most local LM Studio setups"
|
||||
.value=${this._localLlmApiKey}
|
||||
@input=${(e) => this._saveLocalLlmApiKey(e.target.value)}
|
||||
/>
|
||||
<div class="form-hint">
|
||||
Leave blank unless your local server requires a token
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label class="form-label">Nemotron ASR Sidecar URL</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="ws://127.0.0.1:8765/v1/asr/stream"
|
||||
.value=${this._localSttUrl}
|
||||
@input=${(e) => this._saveLocalSttUrl(e.target.value)}
|
||||
class=${this._keyError && !this._localSttUrl.trim() ? "error" : ""}
|
||||
/>
|
||||
<div class="form-hint">
|
||||
External streaming STT service that accepts 16 kHz mono PCM
|
||||
${!sttIsLocal
|
||||
? html`<span style="color: var(--warning, #d97706);">
|
||||
· not a localhost URL
|
||||
</span>`
|
||||
: ""}
|
||||
</div>
|
||||
<select
|
||||
.value=${this._whisperModel}
|
||||
@change=${(e) => this._saveWhisperModel(e.target.value)}
|
||||
>
|
||||
<option
|
||||
value="Xenova/whisper-tiny"
|
||||
?selected=${this._whisperModel === "Xenova/whisper-tiny"}
|
||||
>
|
||||
Tiny (fastest, least accurate)
|
||||
</option>
|
||||
<option
|
||||
value="Xenova/whisper-base"
|
||||
?selected=${this._whisperModel === "Xenova/whisper-base"}
|
||||
>
|
||||
Base
|
||||
</option>
|
||||
<option
|
||||
value="Xenova/whisper-small"
|
||||
?selected=${this._whisperModel === "Xenova/whisper-small"}
|
||||
>
|
||||
Small (recommended)
|
||||
</option>
|
||||
<option
|
||||
value="Xenova/whisper-medium"
|
||||
?selected=${this._whisperModel === "Xenova/whisper-medium"}
|
||||
>
|
||||
Medium (most accurate, slowest)
|
||||
</option>
|
||||
<option
|
||||
value="__custom__"
|
||||
?selected=${this._whisperModel === "__custom__"}
|
||||
>
|
||||
Custom HuggingFace model...
|
||||
</option>
|
||||
</select>
|
||||
${this._whisperModel === "__custom__"
|
||||
? html`
|
||||
<input
|
||||
type="text"
|
||||
placeholder="e.g. onnx-community/whisper-large-v3-turbo"
|
||||
.value=${this._customWhisperModel}
|
||||
@change=${(e) => this._saveCustomWhisperModel(e.target.value)}
|
||||
@input=${(e) => {
|
||||
this._customWhisperModel = e.target.value;
|
||||
}}
|
||||
style="margin-top: 6px;"
|
||||
/>
|
||||
<div class="form-hint">
|
||||
Enter a HuggingFace model ID compatible with
|
||||
@huggingface/transformers speech-to-text pipeline
|
||||
</div>
|
||||
`
|
||||
: html`
|
||||
<div class="form-hint">
|
||||
${this.whisperDownloading
|
||||
? "Downloading model..."
|
||||
: "Downloaded automatically on first use"}
|
||||
</div>
|
||||
`}
|
||||
${this.whisperDownloading && this.whisperProgress
|
||||
? this._renderWhisperProgress()
|
||||
: ""}
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label class="form-label">STT Language</label>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="en-US"
|
||||
.value=${this._localSttLanguage}
|
||||
@input=${(e) => this._saveLocalSttLanguage(e.target.value)}
|
||||
/>
|
||||
<div class="form-hint">
|
||||
First local sidecar target is English streaming ASR
|
||||
</div>
|
||||
</div>
|
||||
|
||||
${this._renderStartButton()}
|
||||
@@ -1458,74 +1508,51 @@ export class MainView extends LitElement {
|
||||
return html`
|
||||
<div class="help-content">
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">What is Ollama?</div>
|
||||
<div class="help-section-title">LM Studio</div>
|
||||
<div class="help-section-text">
|
||||
Ollama lets you run large language models locally on your machine.
|
||||
Everything stays on your computer — no data leaves your device.
|
||||
LM Studio runs the answer model locally and exposes an
|
||||
OpenAI-compatible server for streaming responses.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Install Ollama</div>
|
||||
<div class="help-section-title">Start LM Studio server</div>
|
||||
<div class="help-section-text">
|
||||
Download from
|
||||
<span
|
||||
class="help-link"
|
||||
@click=${() => this.onExternalLink("https://ollama.com/download")}
|
||||
>ollama.com/download</span
|
||||
>
|
||||
and install it.
|
||||
Download LM Studio, load a model, then start the local server from
|
||||
the Developer tab. The default endpoint is:
|
||||
</div>
|
||||
<code class="help-code">http://127.0.0.1:1234/v1</code>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Model ID</div>
|
||||
<div class="help-section-text">
|
||||
Enter the exact model identifier shown by LM Studio. Gemma 4 is the
|
||||
reference target, but any loaded compatible model can be used.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Ollama must be running</div>
|
||||
<div class="help-section-title">Nemotron ASR sidecar</div>
|
||||
<div class="help-section-text">
|
||||
Ollama needs to be running before you start a session. If it's not
|
||||
running, open your terminal and type:
|
||||
Speech-to-text runs as a separate local streaming service. The app
|
||||
connects to:
|
||||
</div>
|
||||
<code class="help-code">ollama serve</code>
|
||||
<code class="help-code">ws://127.0.0.1:8765/v1/asr/stream</code>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Pull a model</div>
|
||||
<div class="help-section-title">Screenshots</div>
|
||||
<div class="help-section-text">
|
||||
Download a model before first use:
|
||||
</div>
|
||||
<code class="help-code">ollama pull gemma3:4b</code>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Recommended models</div>
|
||||
<div class="help-models">
|
||||
<div class="help-model">
|
||||
<span class="help-model-name">gemma3:4b</span
|
||||
><span>4B — fast, multimodal (images + text)</span>
|
||||
</div>
|
||||
<div class="help-model">
|
||||
<span class="help-model-name">mistral-small</span
|
||||
><span>8B — solid all-rounder, text only</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="help-section-text">
|
||||
gemma3:4b and above supports images — screenshots will work with
|
||||
these models.
|
||||
Manual screenshots are sent to the same local LM Studio model. Use a
|
||||
vision-capable model for screen analysis.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-warn">
|
||||
Avoid "thinking" models (e.g. deepseek-r1, qwq). Local inference is
|
||||
already slower — a thinking model adds extra delay before
|
||||
responding.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="help-section">
|
||||
<div class="help-section-title">Whisper</div>
|
||||
<div class="help-section-text">
|
||||
The Whisper speech-to-text model is downloaded automatically the
|
||||
first time you start a session. This is a one-time download.
|
||||
Non-local endpoints are allowed, but they may send audio transcripts
|
||||
or screenshots outside this machine.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -30,7 +30,15 @@ const DEFAULT_PREFERENCES = {
|
||||
fontSize: "medium",
|
||||
backgroundTransparency: 0.8,
|
||||
googleSearchEnabled: false,
|
||||
providerMode: "local",
|
||||
responseProvider: "gemini",
|
||||
llmProvider: "lmstudio",
|
||||
sttProvider: "nemotron-sidecar",
|
||||
localLlmBaseUrl: "http://127.0.0.1:1234/v1",
|
||||
localLlmModel: "",
|
||||
localLlmApiKey: "",
|
||||
localSttUrl: "ws://127.0.0.1:8765/v1/asr/stream",
|
||||
localSttLanguage: "en-US",
|
||||
ollamaHost: "http://127.0.0.1:11434",
|
||||
ollamaModel: "llama3.1",
|
||||
whisperModel: "Xenova/whisper-small",
|
||||
|
||||
+2
-2
@@ -1073,7 +1073,7 @@ function setupGeminiIpcHandlers(geminiSessionRef) {
|
||||
"initialize-local",
|
||||
async (
|
||||
event,
|
||||
ollamaHost,
|
||||
localConfigOrOllamaHost,
|
||||
ollamaModel,
|
||||
whisperModel,
|
||||
profile,
|
||||
@@ -1081,7 +1081,7 @@ function setupGeminiIpcHandlers(geminiSessionRef) {
|
||||
) => {
|
||||
currentProviderMode = "local";
|
||||
const success = await getLocalAi().initializeLocalSession(
|
||||
ollamaHost,
|
||||
localConfigOrOllamaHost,
|
||||
ollamaModel,
|
||||
whisperModel,
|
||||
profile,
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
const { EventEmitter } = require("events");
|
||||
const WebSocket = require("ws");
|
||||
|
||||
const DEFAULT_LOCAL_LLM_BASE_URL = "http://127.0.0.1:1234/v1";
|
||||
const DEFAULT_LOCAL_STT_URL = "ws://127.0.0.1:8765/v1/asr/stream";
|
||||
const DEFAULT_LOCAL_STT_LANGUAGE = "en-US";
|
||||
|
||||
function normalizeOpenAiBaseUrl(baseUrl) {
|
||||
const trimmed = (baseUrl || DEFAULT_LOCAL_LLM_BASE_URL)
|
||||
.trim()
|
||||
.replace(/\/+$/, "");
|
||||
if (!trimmed) return DEFAULT_LOCAL_LLM_BASE_URL;
|
||||
return trimmed.endsWith("/v1") ? trimmed : `${trimmed}/v1`;
|
||||
}
|
||||
|
||||
function isLoopbackUrl(value) {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
return (
|
||||
hostname === "localhost" ||
|
||||
hostname === "127.0.0.1" ||
|
||||
hostname === "::1" ||
|
||||
hostname === "[::1]"
|
||||
);
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function buildChatMessages({
|
||||
systemPrompt,
|
||||
history = [],
|
||||
userText,
|
||||
imageBase64,
|
||||
}) {
|
||||
const messages = [];
|
||||
|
||||
if (systemPrompt && systemPrompt.trim()) {
|
||||
messages.push({ role: "system", content: systemPrompt.trim() });
|
||||
}
|
||||
|
||||
for (const message of history) {
|
||||
if (!message || !message.role || !message.content) continue;
|
||||
if (
|
||||
message.role !== "user" &&
|
||||
message.role !== "assistant" &&
|
||||
message.role !== "system"
|
||||
)
|
||||
continue;
|
||||
messages.push({
|
||||
role: message.role,
|
||||
content: String(message.content),
|
||||
});
|
||||
}
|
||||
|
||||
const text = (userText || "").trim();
|
||||
if (imageBase64) {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: { url: `data:image/jpeg;base64,${imageBase64}` },
|
||||
},
|
||||
],
|
||||
});
|
||||
} else if (text) {
|
||||
messages.push({ role: "user", content: text });
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
function parseChatCompletionSseLine(line) {
|
||||
if (!line.startsWith("data: ")) return null;
|
||||
|
||||
const data = line.slice(6).trim();
|
||||
if (!data || data === "[DONE]") return null;
|
||||
|
||||
const parsed = JSON.parse(data);
|
||||
return parsed.choices?.[0]?.delta?.content || "";
|
||||
}
|
||||
|
||||
async function streamLmStudioChat({
|
||||
baseUrl,
|
||||
apiKey,
|
||||
model,
|
||||
messages,
|
||||
temperature = 0.7,
|
||||
maxTokens = 2048,
|
||||
onToken,
|
||||
}) {
|
||||
if (!model || !model.trim()) {
|
||||
throw new Error("LM Studio model id is required");
|
||||
}
|
||||
|
||||
const normalizedBaseUrl = normalizeOpenAiBaseUrl(baseUrl);
|
||||
const response = await fetch(`${normalizedBaseUrl}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(apiKey && apiKey.trim()
|
||||
? { Authorization: `Bearer ${apiKey.trim()}` }
|
||||
: {}),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model.trim(),
|
||||
messages,
|
||||
stream: true,
|
||||
temperature,
|
||||
max_tokens: maxTokens,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => "");
|
||||
if (response.status === 400 && /image|vision|multimodal/i.test(errorText)) {
|
||||
throw new Error(
|
||||
"The selected local model does not appear to support image input",
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`LM Studio error ${response.status}: ${errorText || response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("LM Studio response did not include a stream body");
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
let fullText = "";
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split(/\r?\n/);
|
||||
buffer = lines.pop() || "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
let token = "";
|
||||
try {
|
||||
token = parseChatCompletionSseLine(trimmed) || "";
|
||||
} catch (_) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (token) {
|
||||
fullText += token;
|
||||
if (onToken) onToken(token, fullText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fullText;
|
||||
}
|
||||
|
||||
class NemotronSidecarClient extends EventEmitter {
|
||||
constructor({
|
||||
url = DEFAULT_LOCAL_STT_URL,
|
||||
language = DEFAULT_LOCAL_STT_LANGUAGE,
|
||||
sampleRate = 16000,
|
||||
channels = 1,
|
||||
encoding = "pcm_s16le",
|
||||
} = {}) {
|
||||
super();
|
||||
this.url = url;
|
||||
this.language = language;
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.encoding = encoding;
|
||||
this.socket = null;
|
||||
this.connected = false;
|
||||
}
|
||||
|
||||
connect() {
|
||||
if (this.connected && this.socket?.readyState === WebSocket.OPEN) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = new WebSocket(this.url);
|
||||
this.socket = socket;
|
||||
|
||||
const cleanup = () => {
|
||||
socket.removeListener("open", onOpen);
|
||||
socket.removeListener("error", onErrorBeforeOpen);
|
||||
};
|
||||
|
||||
const onOpen = () => {
|
||||
cleanup();
|
||||
this.connected = true;
|
||||
this.emit("connected");
|
||||
this._sendStart();
|
||||
resolve();
|
||||
};
|
||||
|
||||
const onErrorBeforeOpen = (error) => {
|
||||
cleanup();
|
||||
this.connected = false;
|
||||
this.emit("error", error);
|
||||
reject(error);
|
||||
};
|
||||
|
||||
socket.once("open", onOpen);
|
||||
socket.once("error", onErrorBeforeOpen);
|
||||
|
||||
socket.on("message", (data) => this._handleMessage(data));
|
||||
socket.on("close", (code, reason) => {
|
||||
this.connected = false;
|
||||
this.emit("close", { code, reason: reason?.toString?.() || "" });
|
||||
});
|
||||
socket.on("error", (error) => {
|
||||
this.connected = false;
|
||||
this.emit("error", error);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
_sendStart() {
|
||||
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) return;
|
||||
this.socket.send(
|
||||
JSON.stringify({
|
||||
type: "start",
|
||||
sampleRate: this.sampleRate,
|
||||
channels: this.channels,
|
||||
encoding: this.encoding,
|
||||
language: this.language,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
_handleMessage(data) {
|
||||
let message;
|
||||
try {
|
||||
message = JSON.parse(data.toString("utf8"));
|
||||
} catch (error) {
|
||||
this.emit(
|
||||
"error",
|
||||
new Error(`Invalid ASR sidecar message: ${error.message}`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!message || !message.type) return;
|
||||
|
||||
if (message.type === "partial") {
|
||||
this.emit("partial", message.text || "");
|
||||
} else if (message.type === "final") {
|
||||
this.emit("final", message.text || "");
|
||||
} else if (message.type === "ready") {
|
||||
this.emit("ready", message);
|
||||
} else if (message.type === "error") {
|
||||
this.emit(
|
||||
"error",
|
||||
new Error(message.error || message.message || "ASR sidecar error"),
|
||||
);
|
||||
} else {
|
||||
this.emit(message.type, message);
|
||||
}
|
||||
}
|
||||
|
||||
sendAudio(pcm16kBuffer) {
|
||||
if (!this.socket || this.socket.readyState !== WebSocket.OPEN) return false;
|
||||
if (!pcm16kBuffer || pcm16kBuffer.length === 0) return false;
|
||||
this.socket.send(pcm16kBuffer, { binary: true });
|
||||
return true;
|
||||
}
|
||||
|
||||
close() {
|
||||
if (!this.socket) return;
|
||||
try {
|
||||
if (this.socket.readyState === WebSocket.OPEN) {
|
||||
this.socket.send(JSON.stringify({ type: "stop" }));
|
||||
}
|
||||
this.socket.close();
|
||||
} catch (_) {
|
||||
// Best-effort close.
|
||||
}
|
||||
this.connected = false;
|
||||
this.socket = null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_LOCAL_LLM_BASE_URL,
|
||||
DEFAULT_LOCAL_STT_LANGUAGE,
|
||||
DEFAULT_LOCAL_STT_URL,
|
||||
NemotronSidecarClient,
|
||||
buildChatMessages,
|
||||
isLoopbackUrl,
|
||||
normalizeOpenAiBaseUrl,
|
||||
parseChatCompletionSseLine,
|
||||
streamLmStudioChat,
|
||||
};
|
||||
+361
-36
@@ -5,6 +5,16 @@ const {
|
||||
initializeNewSession,
|
||||
saveConversationTurn,
|
||||
} = require("./gemini");
|
||||
const {
|
||||
DEFAULT_LOCAL_LLM_BASE_URL,
|
||||
DEFAULT_LOCAL_STT_LANGUAGE,
|
||||
DEFAULT_LOCAL_STT_URL,
|
||||
NemotronSidecarClient,
|
||||
buildChatMessages,
|
||||
isLoopbackUrl,
|
||||
normalizeOpenAiBaseUrl,
|
||||
streamLmStudioChat,
|
||||
} = require("./localProviders");
|
||||
const { fork } = require("child_process");
|
||||
const path = require("path");
|
||||
const { getSystemNode } = require("./nodeDetect");
|
||||
@@ -19,6 +29,8 @@ let whisperReady = false;
|
||||
let localConversationHistory = [];
|
||||
let currentSystemPrompt = null;
|
||||
let isLocalActive = false;
|
||||
let localConfig = null;
|
||||
let nemotronClient = null;
|
||||
|
||||
// Set when we intentionally kill the worker to suppress crash handling
|
||||
let whisperShuttingDown = false;
|
||||
@@ -63,6 +75,58 @@ const MAX_SPEECH_BUFFER_BYTES = 16000 * 2 * 30; // 960,000 bytes
|
||||
// Audio resampling buffer
|
||||
let resampleRemainder = Buffer.alloc(0);
|
||||
|
||||
function resolveLocalSessionConfig(
|
||||
configOrHost,
|
||||
model,
|
||||
whisperModel,
|
||||
profile,
|
||||
customPrompt,
|
||||
) {
|
||||
if (configOrHost && typeof configOrHost === "object") {
|
||||
return {
|
||||
llmProvider: configOrHost.llmProvider || "lmstudio",
|
||||
sttProvider: configOrHost.sttProvider || "nemotron-sidecar",
|
||||
localLlmBaseUrl:
|
||||
configOrHost.localLlmBaseUrl || DEFAULT_LOCAL_LLM_BASE_URL,
|
||||
localLlmModel: configOrHost.localLlmModel || "",
|
||||
localLlmApiKey: configOrHost.localLlmApiKey || "",
|
||||
localSttUrl: configOrHost.localSttUrl || DEFAULT_LOCAL_STT_URL,
|
||||
localSttLanguage:
|
||||
configOrHost.localSttLanguage || DEFAULT_LOCAL_STT_LANGUAGE,
|
||||
ollamaHost: configOrHost.ollamaHost || "http://127.0.0.1:11434",
|
||||
ollamaModel: configOrHost.ollamaModel || "llama3.1",
|
||||
whisperModel: configOrHost.whisperModel || "Xenova/whisper-small",
|
||||
profile: configOrHost.profile || profile || "interview",
|
||||
customPrompt: configOrHost.customPrompt || customPrompt || "",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
llmProvider: "ollama",
|
||||
sttProvider: "whisper",
|
||||
localLlmBaseUrl: DEFAULT_LOCAL_LLM_BASE_URL,
|
||||
localLlmModel: "",
|
||||
localLlmApiKey: "",
|
||||
localSttUrl: DEFAULT_LOCAL_STT_URL,
|
||||
localSttLanguage: DEFAULT_LOCAL_STT_LANGUAGE,
|
||||
ollamaHost: configOrHost || "http://127.0.0.1:11434",
|
||||
ollamaModel: model || "llama3.1",
|
||||
whisperModel: whisperModel || "Xenova/whisper-small",
|
||||
profile: profile || "interview",
|
||||
customPrompt: customPrompt || "",
|
||||
};
|
||||
}
|
||||
|
||||
function stripThinkingTags(text) {
|
||||
return (text || "").replace(/<think>[\s\S]*?<\/think>/g, "").trim();
|
||||
}
|
||||
|
||||
function trimLocalHistory(maxMessages = 40) {
|
||||
if (localConversationHistory.length > maxMessages) {
|
||||
localConversationHistory = localConversationHistory.slice(-maxMessages);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Audio Resampling (24kHz → 16kHz) ──
|
||||
|
||||
function resample24kTo16k(inputBuffer) {
|
||||
@@ -550,6 +614,66 @@ async function transcribeAudio(pcm16kBuffer) {
|
||||
});
|
||||
}
|
||||
|
||||
// ── Nemotron Sidecar Streaming STT ──
|
||||
|
||||
async function connectNemotronSidecar(config) {
|
||||
closeNemotronSidecar();
|
||||
|
||||
nemotronClient = new NemotronSidecarClient({
|
||||
url: config.localSttUrl,
|
||||
language: config.localSttLanguage,
|
||||
});
|
||||
|
||||
nemotronClient.on("connected", () => {
|
||||
sendToRenderer("update-status", "ASR sidecar connected");
|
||||
});
|
||||
|
||||
nemotronClient.on("ready", () => {
|
||||
console.log("[LocalAI] Nemotron sidecar ready");
|
||||
sendToRenderer("update-status", "ASR sidecar ready - Listening...");
|
||||
});
|
||||
|
||||
nemotronClient.on("partial", (text) => {
|
||||
if (!text || !text.trim()) return;
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Transcribing... " + text.trim().slice(-80),
|
||||
);
|
||||
});
|
||||
|
||||
nemotronClient.on("final", (text) => {
|
||||
const transcription = (text || "").trim();
|
||||
if (!transcription) return;
|
||||
sendToRenderer("update-status", "Generating response...");
|
||||
handleFinalTranscription(transcription).catch((error) => {
|
||||
console.error("[LocalAI] Final transcript handler error:", error);
|
||||
sendToRenderer("update-status", "Local AI error: " + error.message);
|
||||
});
|
||||
});
|
||||
|
||||
nemotronClient.on("close", ({ code }) => {
|
||||
if (!isLocalActive) return;
|
||||
console.warn("[LocalAI] Nemotron sidecar disconnected:", code);
|
||||
sendToRenderer("update-status", "ASR sidecar disconnected");
|
||||
});
|
||||
|
||||
nemotronClient.on("error", (error) => {
|
||||
console.error("[LocalAI] Nemotron sidecar error:", error);
|
||||
sendToRenderer("update-status", "ASR sidecar error: " + error.message);
|
||||
});
|
||||
|
||||
sendToRenderer("update-status", "Connecting to ASR sidecar...");
|
||||
await nemotronClient.connect();
|
||||
}
|
||||
|
||||
function closeNemotronSidecar() {
|
||||
if (nemotronClient) {
|
||||
nemotronClient.removeAllListeners();
|
||||
nemotronClient.close();
|
||||
nemotronClient = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Speech End Handler ──
|
||||
|
||||
async function handleSpeechEnd(audioData) {
|
||||
@@ -578,7 +702,7 @@ async function handleSpeechEnd(audioData) {
|
||||
}
|
||||
|
||||
sendToRenderer("update-status", "Generating response...");
|
||||
await sendToOllama(transcription);
|
||||
await handleFinalTranscription(transcription);
|
||||
} catch (error) {
|
||||
console.error("[LocalAI] handleSpeechEnd error:", error);
|
||||
sendToRenderer(
|
||||
@@ -588,6 +712,144 @@ async function handleSpeechEnd(audioData) {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleFinalTranscription(transcription) {
|
||||
if (!localConfig) {
|
||||
await sendToOllama(transcription);
|
||||
return;
|
||||
}
|
||||
|
||||
if (localConfig.llmProvider === "lmstudio") {
|
||||
await sendToLmStudio(transcription);
|
||||
return;
|
||||
}
|
||||
|
||||
await sendToOllama(transcription);
|
||||
}
|
||||
|
||||
// ── LM Studio Chat (OpenAI-compatible) ──
|
||||
|
||||
function getLmStudioConfig() {
|
||||
return {
|
||||
baseUrl: normalizeOpenAiBaseUrl(
|
||||
localConfig?.localLlmBaseUrl || DEFAULT_LOCAL_LLM_BASE_URL,
|
||||
),
|
||||
apiKey: localConfig?.localLlmApiKey || "",
|
||||
model: localConfig?.localLlmModel || "",
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyLmStudioConnection(config) {
|
||||
if (!config.localLlmModel || !config.localLlmModel.trim()) {
|
||||
sendToRenderer("update-status", "LM Studio model id is required");
|
||||
return false;
|
||||
}
|
||||
|
||||
const baseUrl = normalizeOpenAiBaseUrl(config.localLlmBaseUrl);
|
||||
|
||||
if (!isLoopbackUrl(baseUrl)) {
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Warning: LM Studio endpoint is not localhost",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${baseUrl}/models`, {
|
||||
headers: {
|
||||
...(config.localLlmApiKey && config.localLlmApiKey.trim()
|
||||
? { Authorization: `Bearer ${config.localLlmApiKey.trim()}` }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(`HTTP ${response.status}${text ? ": " + text : ""}`);
|
||||
}
|
||||
|
||||
console.log("[LocalAI] LM Studio connection verified");
|
||||
sendToRenderer("update-status", "LM Studio reachable");
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error("[LocalAI] Cannot connect to LM Studio:", error);
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Cannot connect to LM Studio: " + error.message,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendToLmStudio(userText, imageBase64 = null) {
|
||||
const config = getLmStudioConfig();
|
||||
if (!config.model || !config.model.trim()) {
|
||||
sendToRenderer("update-status", "LM Studio model id is required");
|
||||
return { success: false, error: "LM Studio model id is required" };
|
||||
}
|
||||
|
||||
if (!userText || !userText.trim()) {
|
||||
return { success: false, error: "Empty prompt" };
|
||||
}
|
||||
|
||||
const previousHistory = localConversationHistory.slice(-20);
|
||||
const messages = buildChatMessages({
|
||||
systemPrompt: currentSystemPrompt || "You are a helpful assistant.",
|
||||
history: previousHistory,
|
||||
userText: userText.trim(),
|
||||
imageBase64,
|
||||
});
|
||||
|
||||
localConversationHistory.push({
|
||||
role: "user",
|
||||
content: userText.trim(),
|
||||
});
|
||||
trimLocalHistory();
|
||||
|
||||
try {
|
||||
console.log(
|
||||
`[LocalAI] Sending to LM Studio (${config.model}):`,
|
||||
userText.substring(0, 100) + "...",
|
||||
);
|
||||
|
||||
let isFirst = true;
|
||||
const fullText = await streamLmStudioChat({
|
||||
baseUrl: config.baseUrl,
|
||||
apiKey: config.apiKey,
|
||||
model: config.model,
|
||||
messages,
|
||||
onToken: (_token, accumulated) => {
|
||||
const cleaned = stripThinkingTags(accumulated);
|
||||
if (!cleaned) return;
|
||||
sendToRenderer(isFirst ? "new-response" : "update-response", cleaned);
|
||||
isFirst = false;
|
||||
},
|
||||
});
|
||||
|
||||
const cleanedResponse = stripThinkingTags(fullText);
|
||||
if (cleanedResponse && cleanedResponse !== fullText) {
|
||||
sendToRenderer("update-response", cleanedResponse);
|
||||
}
|
||||
|
||||
if (cleanedResponse) {
|
||||
localConversationHistory.push({
|
||||
role: "assistant",
|
||||
content: cleanedResponse,
|
||||
});
|
||||
trimLocalHistory();
|
||||
|
||||
saveConversationTurn(userText, cleanedResponse);
|
||||
}
|
||||
|
||||
console.log("[LocalAI] LM Studio response completed");
|
||||
sendToRenderer("update-status", "Listening...");
|
||||
return { success: true, text: cleanedResponse, model: config.model };
|
||||
} catch (error) {
|
||||
console.error("[LocalAI] LM Studio error:", error);
|
||||
sendToRenderer("update-status", "LM Studio error: " + error.message);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Ollama Chat ──
|
||||
|
||||
async function sendToOllama(transcription) {
|
||||
@@ -658,53 +920,95 @@ async function sendToOllama(transcription) {
|
||||
// ── Public API ──
|
||||
|
||||
async function initializeLocalSession(
|
||||
ollamaHost,
|
||||
configOrOllamaHost,
|
||||
model,
|
||||
whisperModel,
|
||||
profile,
|
||||
customPrompt,
|
||||
) {
|
||||
console.log("[LocalAI] Initializing local session:", {
|
||||
ollamaHost,
|
||||
const config = resolveLocalSessionConfig(
|
||||
configOrOllamaHost,
|
||||
model,
|
||||
whisperModel,
|
||||
profile,
|
||||
customPrompt,
|
||||
);
|
||||
|
||||
console.log("[LocalAI] Initializing local session:", {
|
||||
llmProvider: config.llmProvider,
|
||||
sttProvider: config.sttProvider,
|
||||
localLlmBaseUrl: config.localLlmBaseUrl,
|
||||
localLlmModel: config.localLlmModel,
|
||||
localSttUrl: config.localSttUrl,
|
||||
profile: config.profile,
|
||||
});
|
||||
|
||||
sendToRenderer("session-initializing", true);
|
||||
|
||||
try {
|
||||
closeNemotronSidecar();
|
||||
isLocalActive = false;
|
||||
|
||||
// Setup system prompt
|
||||
currentSystemPrompt = getSystemPrompt(profile, customPrompt, false);
|
||||
currentSystemPrompt = getSystemPrompt(
|
||||
config.profile,
|
||||
config.customPrompt,
|
||||
false,
|
||||
);
|
||||
|
||||
// Initialize Ollama client
|
||||
ollamaClient = new Ollama({ host: ollamaHost });
|
||||
ollamaModel = model;
|
||||
localConfig = config;
|
||||
ollamaClient = null;
|
||||
ollamaModel = null;
|
||||
|
||||
// Test Ollama connection
|
||||
try {
|
||||
await ollamaClient.list();
|
||||
console.log("[LocalAI] Ollama connection verified");
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[LocalAI] Cannot connect to Ollama at",
|
||||
ollamaHost,
|
||||
":",
|
||||
error.message,
|
||||
);
|
||||
sendToRenderer("session-initializing", false);
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Cannot connect to Ollama at " + ollamaHost,
|
||||
);
|
||||
return false;
|
||||
if (config.llmProvider === "lmstudio") {
|
||||
const lmStudioReady = await verifyLmStudioConnection(config);
|
||||
if (!lmStudioReady) {
|
||||
sendToRenderer("session-initializing", false);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// Initialize Ollama client fallback
|
||||
ollamaClient = new Ollama({ host: config.ollamaHost });
|
||||
ollamaModel = config.ollamaModel;
|
||||
|
||||
try {
|
||||
await ollamaClient.list();
|
||||
console.log("[LocalAI] Ollama connection verified");
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[LocalAI] Cannot connect to Ollama at",
|
||||
config.ollamaHost,
|
||||
":",
|
||||
error.message,
|
||||
);
|
||||
sendToRenderer("session-initializing", false);
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Cannot connect to Ollama at " + config.ollamaHost,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Load Whisper model
|
||||
const pipeline = await loadWhisperPipeline(whisperModel);
|
||||
if (!pipeline) {
|
||||
sendToRenderer("session-initializing", false);
|
||||
return false;
|
||||
if (config.sttProvider === "nemotron-sidecar") {
|
||||
try {
|
||||
await connectNemotronSidecar(config);
|
||||
} catch (error) {
|
||||
console.error("[LocalAI] Cannot connect to ASR sidecar:", error);
|
||||
sendToRenderer("session-initializing", false);
|
||||
sendToRenderer(
|
||||
"update-status",
|
||||
"Cannot connect to ASR sidecar: " + error.message,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// Load Whisper model fallback
|
||||
const pipeline = await loadWhisperPipeline(config.whisperModel);
|
||||
if (!pipeline) {
|
||||
sendToRenderer("session-initializing", false);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Reset VAD state
|
||||
@@ -716,7 +1020,7 @@ async function initializeLocalSession(
|
||||
localConversationHistory = [];
|
||||
|
||||
// Initialize conversation session
|
||||
initializeNewSession(profile, customPrompt);
|
||||
initializeNewSession(config.profile, config.customPrompt);
|
||||
|
||||
isLocalActive = true;
|
||||
sendToRenderer("session-initializing", false);
|
||||
@@ -737,14 +1041,22 @@ function processLocalAudio(monoChunk24k) {
|
||||
|
||||
// Resample from 24kHz to 16kHz
|
||||
const pcm16k = resample24kTo16k(monoChunk24k);
|
||||
if (pcm16k.length > 0) {
|
||||
processVAD(pcm16k);
|
||||
if (pcm16k.length === 0) return;
|
||||
|
||||
if (localConfig?.sttProvider === "nemotron-sidecar") {
|
||||
if (!nemotronClient || !nemotronClient.sendAudio(pcm16k)) {
|
||||
sendToRenderer("update-status", "ASR sidecar is not connected");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
processVAD(pcm16k);
|
||||
}
|
||||
|
||||
function closeLocalSession() {
|
||||
console.log("[LocalAI] Closing local session");
|
||||
isLocalActive = false;
|
||||
closeNemotronSidecar();
|
||||
isSpeaking = false;
|
||||
speechBuffers = [];
|
||||
silenceFrameCount = 0;
|
||||
@@ -753,6 +1065,7 @@ function closeLocalSession() {
|
||||
localConversationHistory = [];
|
||||
ollamaClient = null;
|
||||
ollamaModel = null;
|
||||
localConfig = null;
|
||||
currentSystemPrompt = null;
|
||||
// Note: whisperWorker is kept alive to avoid reloading model on next session
|
||||
// To fully clean up, call killWhisperWorker()
|
||||
@@ -762,14 +1075,17 @@ function isLocalSessionActive() {
|
||||
return isLocalActive;
|
||||
}
|
||||
|
||||
// ── Send text directly to Ollama (for manual text input) ──
|
||||
// ── Send text directly to the active local LLM ──
|
||||
|
||||
async function sendLocalText(text) {
|
||||
if (!isLocalActive || !ollamaClient) {
|
||||
if (!isLocalActive) {
|
||||
return { success: false, error: "No active local session" };
|
||||
}
|
||||
|
||||
try {
|
||||
if (localConfig?.llmProvider === "lmstudio") {
|
||||
return await sendToLmStudio(text);
|
||||
}
|
||||
await sendToOllama(text);
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
@@ -778,10 +1094,19 @@ async function sendLocalText(text) {
|
||||
}
|
||||
|
||||
async function sendLocalImage(base64Data, prompt) {
|
||||
if (!isLocalActive || !ollamaClient) {
|
||||
if (!isLocalActive) {
|
||||
return { success: false, error: "No active local session" };
|
||||
}
|
||||
|
||||
if (localConfig?.llmProvider === "lmstudio") {
|
||||
sendToRenderer("update-status", "Analyzing image locally...");
|
||||
return await sendToLmStudio(prompt, base64Data);
|
||||
}
|
||||
|
||||
if (!ollamaClient) {
|
||||
return { success: false, error: "No active Ollama session" };
|
||||
}
|
||||
|
||||
try {
|
||||
console.log("[LocalAI] Sending image to Ollama");
|
||||
sendToRenderer("update-status", "Analyzing image...");
|
||||
|
||||
+19
-13
@@ -177,19 +177,22 @@ async function initializeGemini(profile = "interview", language = "en-US") {
|
||||
|
||||
async function initializeLocal(profile = "interview") {
|
||||
const prefs = await storage.getPreferences();
|
||||
const ollamaHost = prefs.ollamaHost || "http://127.0.0.1:11434";
|
||||
const ollamaModel = prefs.ollamaModel || "llama3.1";
|
||||
const whisperModel = prefs.whisperModel || "Xenova/whisper-small";
|
||||
const customPrompt = prefs.customPrompt || "";
|
||||
|
||||
const success = await ipcRenderer.invoke(
|
||||
"initialize-local",
|
||||
ollamaHost,
|
||||
ollamaModel,
|
||||
whisperModel,
|
||||
const localConfig = {
|
||||
llmProvider: prefs.llmProvider || "lmstudio",
|
||||
sttProvider: prefs.sttProvider || "nemotron-sidecar",
|
||||
localLlmBaseUrl: prefs.localLlmBaseUrl || "http://127.0.0.1:1234/v1",
|
||||
localLlmModel: prefs.localLlmModel || "",
|
||||
localLlmApiKey: prefs.localLlmApiKey || "",
|
||||
localSttUrl: prefs.localSttUrl || "ws://127.0.0.1:8765/v1/asr/stream",
|
||||
localSttLanguage: prefs.localSttLanguage || "en-US",
|
||||
ollamaHost: prefs.ollamaHost || "http://127.0.0.1:11434",
|
||||
ollamaModel: prefs.ollamaModel || "llama3.1",
|
||||
whisperModel: prefs.whisperModel || "Xenova/whisper-small",
|
||||
profile,
|
||||
customPrompt,
|
||||
);
|
||||
customPrompt: prefs.customPrompt || "",
|
||||
};
|
||||
|
||||
const success = await ipcRenderer.invoke("initialize-local", localConfig);
|
||||
if (success) {
|
||||
cheatingDaddy.setStatus("Local AI Live");
|
||||
return true;
|
||||
@@ -1090,7 +1093,10 @@ const theme = {
|
||||
// Determine if theme is light or dark
|
||||
const lightThemes = ["light", "sepia"];
|
||||
const isLightTheme = lightThemes.includes(themeName);
|
||||
document.body.setAttribute("data-theme-type", isLightTheme ? "light" : "dark");
|
||||
document.body.setAttribute(
|
||||
"data-theme-type",
|
||||
isLightTheme ? "light" : "dark",
|
||||
);
|
||||
|
||||
// New design tokens (used by components)
|
||||
root.style.setProperty("--text-primary", colors.text);
|
||||
|
||||
Reference in New Issue
Block a user