initial commit
This commit is contained in:
Илья Глазунов
@@ -0,0 +1,605 @@
|
||||
const { GoogleGenAI, Modality } = require('@google/genai');
|
||||
const { BrowserWindow, ipcMain } = require('electron');
|
||||
const { spawn } = require('child_process');
|
||||
const { saveDebugAudio } = require('../audioUtils');
|
||||
const { getSystemPrompt } = require('./prompts');
|
||||
const { getAvailableModel, incrementLimitCount, getApiKey } = require('../storage');
|
||||
|
||||
// Conversation tracking variables
|
||||
let currentSessionId = null;
|
||||
let currentTranscription = '';
|
||||
let conversationHistory = [];
|
||||
let screenAnalysisHistory = [];
|
||||
let currentProfile = null;
|
||||
let currentCustomPrompt = null;
|
||||
let isInitializingSession = false;
|
||||
|
||||
function formatSpeakerResults(results) {
|
||||
let text = '';
|
||||
for (const result of results) {
|
||||
if (result.transcript && result.speakerId) {
|
||||
const speakerLabel = result.speakerId === 1 ? 'Interviewer' : 'Candidate';
|
||||
text += `[${speakerLabel}]: ${result.transcript}\n`;
|
||||
}
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
module.exports.formatSpeakerResults = formatSpeakerResults;
|
||||
|
||||
// Audio capture variables
|
||||
let systemAudioProc = null;
|
||||
let messageBuffer = '';
|
||||
|
||||
// Reconnection variables
|
||||
let isUserClosing = false;
|
||||
let sessionParams = null;
|
||||
let reconnectAttempts = 0;
|
||||
const MAX_RECONNECT_ATTEMPTS = 3;
|
||||
const RECONNECT_DELAY = 2000;
|
||||
|
||||
function sendToRenderer(channel, data) {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
if (windows.length > 0) {
|
||||
windows[0].webContents.send(channel, data);
|
||||
}
|
||||
}
|
||||
|
||||
// Build context message for session restoration
|
||||
function buildContextMessage() {
|
||||
const lastTurns = conversationHistory.slice(-20);
|
||||
const validTurns = lastTurns.filter(turn => turn.transcription?.trim() && turn.ai_response?.trim());
|
||||
|
||||
if (validTurns.length === 0) return null;
|
||||
|
||||
const contextLines = validTurns.map(turn =>
|
||||
`[Interviewer]: ${turn.transcription.trim()}\n[Your answer]: ${turn.ai_response.trim()}`
|
||||
);
|
||||
|
||||
return `Session reconnected. Here's the conversation so far:\n\n${contextLines.join('\n\n')}\n\nContinue from here.`;
|
||||
}
|
||||
|
||||
// Conversation management functions
|
||||
function initializeNewSession(profile = null, customPrompt = null) {
|
||||
currentSessionId = Date.now().toString();
|
||||
currentTranscription = '';
|
||||
conversationHistory = [];
|
||||
screenAnalysisHistory = [];
|
||||
currentProfile = profile;
|
||||
currentCustomPrompt = customPrompt;
|
||||
console.log('New conversation session started:', currentSessionId, 'profile:', profile);
|
||||
|
||||
// Save initial session with profile context
|
||||
if (profile) {
|
||||
sendToRenderer('save-session-context', {
|
||||
sessionId: currentSessionId,
|
||||
profile: profile,
|
||||
customPrompt: customPrompt || ''
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function saveConversationTurn(transcription, aiResponse) {
|
||||
if (!currentSessionId) {
|
||||
initializeNewSession();
|
||||
}
|
||||
|
||||
const conversationTurn = {
|
||||
timestamp: Date.now(),
|
||||
transcription: transcription.trim(),
|
||||
ai_response: aiResponse.trim(),
|
||||
};
|
||||
|
||||
conversationHistory.push(conversationTurn);
|
||||
console.log('Saved conversation turn:', conversationTurn);
|
||||
|
||||
// Send to renderer to save in IndexedDB
|
||||
sendToRenderer('save-conversation-turn', {
|
||||
sessionId: currentSessionId,
|
||||
turn: conversationTurn,
|
||||
fullHistory: conversationHistory,
|
||||
});
|
||||
}
|
||||
|
||||
function saveScreenAnalysis(prompt, response, model) {
|
||||
if (!currentSessionId) {
|
||||
initializeNewSession();
|
||||
}
|
||||
|
||||
const analysisEntry = {
|
||||
timestamp: Date.now(),
|
||||
prompt: prompt,
|
||||
response: response.trim(),
|
||||
model: model
|
||||
};
|
||||
|
||||
screenAnalysisHistory.push(analysisEntry);
|
||||
console.log('Saved screen analysis:', analysisEntry);
|
||||
|
||||
// Send to renderer to save
|
||||
sendToRenderer('save-screen-analysis', {
|
||||
sessionId: currentSessionId,
|
||||
analysis: analysisEntry,
|
||||
fullHistory: screenAnalysisHistory,
|
||||
profile: currentProfile,
|
||||
customPrompt: currentCustomPrompt
|
||||
});
|
||||
}
|
||||
|
||||
function getCurrentSessionData() {
|
||||
return {
|
||||
sessionId: currentSessionId,
|
||||
history: conversationHistory,
|
||||
};
|
||||
}
|
||||
|
||||
async function getEnabledTools() {
|
||||
const tools = [];
|
||||
|
||||
// Check if Google Search is enabled (default: true)
|
||||
const googleSearchEnabled = await getStoredSetting('googleSearchEnabled', 'true');
|
||||
console.log('Google Search enabled:', googleSearchEnabled);
|
||||
|
||||
if (googleSearchEnabled === 'true') {
|
||||
tools.push({ googleSearch: {} });
|
||||
console.log('Added Google Search tool');
|
||||
} else {
|
||||
console.log('Google Search tool disabled');
|
||||
}
|
||||
|
||||
return tools;
|
||||
}
|
||||
|
||||
async function getStoredSetting(key, defaultValue) {
|
||||
try {
|
||||
const windows = BrowserWindow.getAllWindows();
|
||||
if (windows.length > 0) {
|
||||
// Wait a bit for the renderer to be ready
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
// Try to get setting from renderer process localStorage
|
||||
const value = await windows[0].webContents.executeJavaScript(`
|
||||
(function() {
|
||||
try {
|
||||
if (typeof localStorage === 'undefined') {
|
||||
console.log('localStorage not available yet for ${key}');
|
||||
return '${defaultValue}';
|
||||
}
|
||||
const stored = localStorage.getItem('${key}');
|
||||
console.log('Retrieved setting ${key}:', stored);
|
||||
return stored || '${defaultValue}';
|
||||
} catch (e) {
|
||||
console.error('Error accessing localStorage for ${key}:', e);
|
||||
return '${defaultValue}';
|
||||
}
|
||||
})()
|
||||
`);
|
||||
return value;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting stored setting for', key, ':', error.message);
|
||||
}
|
||||
console.log('Using default value for', key, ':', defaultValue);
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
async function initializeGeminiSession(apiKey, customPrompt = '', profile = 'interview', language = 'en-US', isReconnect = false) {
|
||||
if (isInitializingSession) {
|
||||
console.log('Session initialization already in progress');
|
||||
return false;
|
||||
}
|
||||
|
||||
isInitializingSession = true;
|
||||
if (!isReconnect) {
|
||||
sendToRenderer('session-initializing', true);
|
||||
}
|
||||
|
||||
// Store params for reconnection
|
||||
if (!isReconnect) {
|
||||
sessionParams = { apiKey, customPrompt, profile, language };
|
||||
reconnectAttempts = 0;
|
||||
}
|
||||
|
||||
const client = new GoogleGenAI({
|
||||
vertexai: false,
|
||||
apiKey: apiKey,
|
||||
httpOptions: { apiVersion: 'v1alpha' },
|
||||
});
|
||||
|
||||
// Get enabled tools first to determine Google Search status
|
||||
const enabledTools = await getEnabledTools();
|
||||
const googleSearchEnabled = enabledTools.some(tool => tool.googleSearch);
|
||||
|
||||
const systemPrompt = getSystemPrompt(profile, customPrompt, googleSearchEnabled);
|
||||
|
||||
// Initialize new conversation session only on first connect
|
||||
if (!isReconnect) {
|
||||
initializeNewSession(profile, customPrompt);
|
||||
}
|
||||
|
||||
try {
|
||||
const session = await client.live.connect({
|
||||
model: 'gemini-2.5-flash-native-audio-preview-09-2025',
|
||||
callbacks: {
|
||||
onopen: function () {
|
||||
sendToRenderer('update-status', 'Live session connected');
|
||||
},
|
||||
onmessage: function (message) {
|
||||
console.log('----------------', message);
|
||||
|
||||
// Handle input transcription (what was spoken)
|
||||
if (message.serverContent?.inputTranscription?.results) {
|
||||
currentTranscription += formatSpeakerResults(message.serverContent.inputTranscription.results);
|
||||
} else if (message.serverContent?.inputTranscription?.text) {
|
||||
const text = message.serverContent.inputTranscription.text;
|
||||
if (text.trim() !== '') {
|
||||
currentTranscription += text;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle AI model response via output transcription (native audio model)
|
||||
if (message.serverContent?.outputTranscription?.text) {
|
||||
const text = message.serverContent.outputTranscription.text;
|
||||
if (text.trim() === '') return; // Ignore empty transcriptions
|
||||
const isNewResponse = messageBuffer === '';
|
||||
messageBuffer += text;
|
||||
sendToRenderer(isNewResponse ? 'new-response' : 'update-response', messageBuffer);
|
||||
}
|
||||
|
||||
if (message.serverContent?.generationComplete) {
|
||||
// Only send/save if there's actual content
|
||||
if (messageBuffer.trim() !== '') {
|
||||
sendToRenderer('update-response', messageBuffer);
|
||||
|
||||
// Save conversation turn when we have both transcription and AI response
|
||||
if (currentTranscription) {
|
||||
saveConversationTurn(currentTranscription, messageBuffer);
|
||||
currentTranscription = ''; // Reset for next turn
|
||||
}
|
||||
}
|
||||
messageBuffer = '';
|
||||
}
|
||||
|
||||
if (message.serverContent?.turnComplete) {
|
||||
sendToRenderer('update-status', 'Listening...');
|
||||
}
|
||||
},
|
||||
onerror: function (e) {
|
||||
console.log('Session error:', e.message);
|
||||
sendToRenderer('update-status', 'Error: ' + e.message);
|
||||
},
|
||||
onclose: function (e) {
|
||||
console.log('Session closed:', e.reason);
|
||||
|
||||
// Don't reconnect if user intentionally closed
|
||||
if (isUserClosing) {
|
||||
isUserClosing = false;
|
||||
sendToRenderer('update-status', 'Session closed');
|
||||
return;
|
||||
}
|
||||
|
||||
// Attempt reconnection
|
||||
if (sessionParams && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
|
||||
attemptReconnect();
|
||||
} else {
|
||||
sendToRenderer('update-status', 'Session closed');
|
||||
}
|
||||
},
|
||||
},
|
||||
config: {
|
||||
responseModalities: [Modality.AUDIO],
|
||||
proactivity: { proactiveAudio: true },
|
||||
outputAudioTranscription: {},
|
||||
tools: enabledTools,
|
||||
// Enable speaker diarization
|
||||
inputAudioTranscription: {
|
||||
enableSpeakerDiarization: true,
|
||||
minSpeakerCount: 2,
|
||||
maxSpeakerCount: 2,
|
||||
},
|
||||
contextWindowCompression: { slidingWindow: {} },
|
||||
speechConfig: { languageCode: language },
|
||||
systemInstruction: {
|
||||
parts: [{ text: systemPrompt }],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
isInitializingSession = false;
|
||||
if (!isReconnect) {
|
||||
sendToRenderer('session-initializing', false);
|
||||
}
|
||||
return session;
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize Gemini session:', error);
|
||||
isInitializingSession = false;
|
||||
if (!isReconnect) {
|
||||
sendToRenderer('session-initializing', false);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function attemptReconnect() {
|
||||
reconnectAttempts++;
|
||||
console.log(`Reconnection attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS}`);
|
||||
|
||||
// Clear stale buffers
|
||||
messageBuffer = '';
|
||||
currentTranscription = '';
|
||||
|
||||
sendToRenderer('update-status', `Reconnecting... (${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS})`);
|
||||
|
||||
// Wait before attempting
|
||||
await new Promise(resolve => setTimeout(resolve, RECONNECT_DELAY));
|
||||
|
||||
try {
|
||||
const session = await initializeGeminiSession(
|
||||
sessionParams.apiKey,
|
||||
sessionParams.customPrompt,
|
||||
sessionParams.profile,
|
||||
sessionParams.language,
|
||||
true // isReconnect
|
||||
);
|
||||
|
||||
if (session && global.geminiSessionRef) {
|
||||
global.geminiSessionRef.current = session;
|
||||
|
||||
// Restore context from conversation history via text message
|
||||
const contextMessage = buildContextMessage();
|
||||
if (contextMessage) {
|
||||
try {
|
||||
console.log('Restoring conversation context...');
|
||||
await session.sendRealtimeInput({ text: contextMessage });
|
||||
} catch (contextError) {
|
||||
console.error('Failed to restore context:', contextError);
|
||||
// Continue without context - better than failing
|
||||
}
|
||||
}
|
||||
|
||||
// Don't reset reconnectAttempts here - let it reset on next fresh session
|
||||
sendToRenderer('update-status', 'Reconnected! Listening...');
|
||||
console.log('Session reconnected successfully');
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Reconnection attempt ${reconnectAttempts} failed:`, error);
|
||||
}
|
||||
|
||||
// If we still have attempts left, try again
|
||||
if (reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
|
||||
return attemptReconnect();
|
||||
}
|
||||
|
||||
// Max attempts reached - notify frontend
|
||||
console.log('Max reconnection attempts reached');
|
||||
sendToRenderer('reconnect-failed', {
|
||||
message: 'Tried 3 times to reconnect. Must be upstream/network issues. Try restarting or download updated app from site.',
|
||||
});
|
||||
sessionParams = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
function killExistingSystemAudioDump() {
|
||||
return new Promise(resolve => {
|
||||
console.log('Checking for existing SystemAudioDump processes...');
|
||||
|
||||
// Kill any existing SystemAudioDump processes
|
||||
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
|
||||
stdio: 'ignore',
|
||||
});
|
||||
|
||||
killProc.on('close', code => {
|
||||
if (code === 0) {
|
||||
console.log('Killed existing SystemAudioDump processes');
|
||||
} else {
|
||||
console.log('No existing SystemAudioDump processes found');
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
|
||||
killProc.on('error', err => {
|
||||
console.log('Error checking for existing processes (this is normal):', err.message);
|
||||
resolve();
|
||||
});
|
||||
|
||||
// Timeout after 2 seconds
|
||||
setTimeout(() => {
|
||||
killProc.kill();
|
||||
resolve();
|
||||
}, 2000);
|
||||
});
|
||||
}
|
||||
|
||||
async function startMacOSAudioCapture(geminiSessionRef) {
|
||||
if (process.platform !== 'darwin') return false;
|
||||
|
||||
// Kill any existing SystemAudioDump processes first
|
||||
await killExistingSystemAudioDump();
|
||||
|
||||
console.log('Starting macOS audio capture with SystemAudioDump...');
|
||||
|
||||
const { app } = require('electron');
|
||||
const path = require('path');
|
||||
|
||||
let systemAudioPath;
|
||||
if (app.isPackaged) {
|
||||
systemAudioPath = path.join(process.resourcesPath, 'SystemAudioDump');
|
||||
} else {
|
||||
systemAudioPath = path.join(__dirname, '../assets', 'SystemAudioDump');
|
||||
}
|
||||
|
||||
console.log('SystemAudioDump path:', systemAudioPath);
|
||||
|
||||
const spawnOptions = {
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: {
|
||||
...process.env,
|
||||
},
|
||||
};
|
||||
|
||||
systemAudioProc = spawn(systemAudioPath, [], spawnOptions);
|
||||
|
||||
if (!systemAudioProc.pid) {
|
||||
console.error('Failed to start SystemAudioDump');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('SystemAudioDump started with PID:', systemAudioProc.pid);
|
||||
|
||||
const CHUNK_DURATION = 0.1;
|
||||
const SAMPLE_RATE = 24000;
|
||||
const BYTES_PER_SAMPLE = 2;
|
||||
const CHANNELS = 2;
|
||||
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
|
||||
|
||||
let audioBuffer = Buffer.alloc(0);
|
||||
|
||||
systemAudioProc.stdout.on('data', data => {
|
||||
audioBuffer = Buffer.concat([audioBuffer, data]);
|
||||
|
||||
while (audioBuffer.length >= CHUNK_SIZE) {
|
||||
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
|
||||
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
|
||||
|
||||
const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk;
|
||||
const base64Data = monoChunk.toString('base64');
|
||||
sendAudioToGemini(base64Data, geminiSessionRef);
|
||||
|
||||
if (process.env.DEBUG_AUDIO) {
|
||||
console.log(`Processed audio chunk: ${chunk.length} bytes`);
|
||||
saveDebugAudio(monoChunk, 'system_audio');
|
||||
}
|
||||
}
|
||||
|
||||
const maxBufferSize = SAMPLE_RATE * BYTES_PER_SAMPLE * 1;
|
||||
if (audioBuffer.length > maxBufferSize) {
|
||||
audioBuffer = audioBuffer.slice(-maxBufferSize);
|
||||
}
|
||||
});
|
||||
|
||||
systemAudioProc.stderr.on('data', data => {
|
||||
console.error('SystemAudioDump stderr:', data.toString());
|
||||
});
|
||||
|
||||
systemAudioProc.on('close', code => {
|
||||
console.log('SystemAudioDump process closed with code:', code);
|
||||
systemAudioProc = null;
|
||||
});
|
||||
|
||||
systemAudioProc.on('error', err => {
|
||||
console.error('SystemAudioDump process error:', err);
|
||||
systemAudioProc = null;
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
function convertStereoToMono(stereoBuffer) {
|
||||
const samples = stereoBuffer.length / 4;
|
||||
const monoBuffer = Buffer.alloc(samples * 2);
|
||||
|
||||
for (let i = 0; i < samples; i++) {
|
||||
const leftSample = stereoBuffer.readInt16LE(i * 4);
|
||||
monoBuffer.writeInt16LE(leftSample, i * 2);
|
||||
}
|
||||
|
||||
return monoBuffer;
|
||||
}
|
||||
|
||||
function stopMacOSAudioCapture() {
|
||||
if (systemAudioProc) {
|
||||
console.log('Stopping SystemAudioDump...');
|
||||
systemAudioProc.kill('SIGTERM');
|
||||
systemAudioProc = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function sendAudioToGemini(base64Data, geminiSessionRef) {
|
||||
if (!geminiSessionRef.current) return;
|
||||
|
||||
try {
|
||||
process.stdout.write('.');
|
||||
await geminiSessionRef.current.sendRealtimeInput({
|
||||
audio: {
|
||||
data: base64Data,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error sending audio to Gemini:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function sendImageToGeminiHttp(base64Data, prompt) {
|
||||
// Get available model based on rate limits
|
||||
const model = getAvailableModel();
|
||||
|
||||
const apiKey = getApiKey();
|
||||
if (!apiKey) {
|
||||
return { success: false, error: 'No API key configured' };
|
||||
}
|
||||
|
||||
try {
|
||||
const ai = new GoogleGenAI({ apiKey: apiKey });
|
||||
|
||||
const contents = [
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: 'image/jpeg',
|
||||
data: base64Data,
|
||||
},
|
||||
},
|
||||
{ text: prompt },
|
||||
];
|
||||
|
||||
console.log(`Sending image to ${model} (streaming)...`);
|
||||
const response = await ai.models.generateContentStream({
|
||||
model: model,
|
||||
contents: contents,
|
||||
});
|
||||
|
||||
// Increment count after successful call
|
||||
incrementLimitCount(model);
|
||||
|
||||
// Stream the response
|
||||
let fullText = '';
|
||||
let isFirst = true;
|
||||
for await (const chunk of response) {
|
||||
const chunkText = chunk.text;
|
||||
if (chunkText) {
|
||||
fullText += chunkText;
|
||||
// Send to renderer - new response for first chunk, update for subsequent
|
||||
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullText);
|
||||
isFirst = false;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Image response completed from ${model}`);
|
||||
|
||||
// Save screen analysis to history
|
||||
saveScreenAnalysis(prompt, fullText, model);
|
||||
|
||||
return { success: true, text: fullText, model: model };
|
||||
} catch (error) {
|
||||
console.error('Error sending image to Gemini HTTP:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
initializeGeminiSession,
|
||||
getEnabledTools,
|
||||
getStoredSetting,
|
||||
sendToRenderer,
|
||||
initializeNewSession,
|
||||
saveConversationTurn,
|
||||
getCurrentSessionData,
|
||||
killExistingSystemAudioDump,
|
||||
startMacOSAudioCapture,
|
||||
convertStereoToMono,
|
||||
stopMacOSAudioCapture,
|
||||
sendAudioToGemini,
|
||||
sendImageToGeminiHttp,
|
||||
formatSpeakerResults,
|
||||
};
|
||||
Reference in New Issue
Block a user