initial commit

This commit is contained in:
Илья Глазунов
2026-01-14 22:57:19 +03:00
commit beb9034cd4
50 changed files with 17257 additions and 0 deletions
+453
View File
@@ -0,0 +1,453 @@
const { BrowserWindow, ipcMain } = require('electron');
const { getSystemPrompt } = require('./prompts');
const { getAvailableModel, incrementLimitCount, getApiKey, getOpenAICredentials, getOpenAISDKCredentials, getPreferences } = require('../storage');
// Import provider implementations
const geminiProvider = require('./gemini');
const openaiRealtimeProvider = require('./openai-realtime');
const openaiSdkProvider = require('./openai-sdk');
// Conversation tracking (shared across providers)
let currentSessionId = null;
let conversationHistory = [];
let screenAnalysisHistory = [];
let currentProfile = null;
let currentCustomPrompt = null;
let currentProvider = 'gemini'; // 'gemini', 'openai-realtime', or 'openai-sdk'
let providerConfig = {};
function sendToRenderer(channel, data) {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
windows[0].webContents.send(channel, data);
}
}
function initializeNewSession(profile = null, customPrompt = null) {
currentSessionId = Date.now().toString();
conversationHistory = [];
screenAnalysisHistory = [];
currentProfile = profile;
currentCustomPrompt = customPrompt;
console.log('New conversation session started:', currentSessionId, 'profile:', profile, 'provider:', currentProvider);
if (profile) {
sendToRenderer('save-session-context', {
sessionId: currentSessionId,
profile: profile,
customPrompt: customPrompt || '',
provider: currentProvider,
});
}
}
function saveConversationTurn(transcription, aiResponse) {
if (!currentSessionId) {
initializeNewSession();
}
const conversationTurn = {
timestamp: Date.now(),
transcription: transcription.trim(),
ai_response: aiResponse.trim(),
};
conversationHistory.push(conversationTurn);
console.log('Saved conversation turn:', conversationTurn);
sendToRenderer('save-conversation-turn', {
sessionId: currentSessionId,
turn: conversationTurn,
fullHistory: conversationHistory,
});
}
function saveScreenAnalysis(prompt, response, model) {
if (!currentSessionId) {
initializeNewSession();
}
const analysisEntry = {
timestamp: Date.now(),
prompt: prompt,
response: response.trim(),
model: model,
provider: currentProvider,
};
screenAnalysisHistory.push(analysisEntry);
console.log('Saved screen analysis:', analysisEntry);
sendToRenderer('save-screen-analysis', {
sessionId: currentSessionId,
analysis: analysisEntry,
fullHistory: screenAnalysisHistory,
profile: currentProfile,
customPrompt: currentCustomPrompt,
});
}
function getCurrentSessionData() {
return {
sessionId: currentSessionId,
history: conversationHistory,
provider: currentProvider,
};
}
// Get provider configuration from storage
async function getStoredSetting(key, defaultValue) {
try {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
await new Promise(resolve => setTimeout(resolve, 100));
const value = await windows[0].webContents.executeJavaScript(`
(function() {
try {
if (typeof localStorage === 'undefined') {
return '${defaultValue}';
}
const stored = localStorage.getItem('${key}');
return stored || '${defaultValue}';
} catch (e) {
return '${defaultValue}';
}
})()
`);
return value;
}
} catch (error) {
console.error('Error getting stored setting for', key, ':', error.message);
}
return defaultValue;
}
// Initialize AI session based on selected provider
async function initializeAISession(customPrompt = '', profile = 'interview', language = 'en-US') {
// Read provider from file-based storage (preferences.json)
const prefs = getPreferences();
const provider = prefs.aiProvider || 'gemini';
currentProvider = provider;
console.log('Initializing AI session with provider:', provider);
// Check if Google Search is enabled for system prompt
const googleSearchEnabled = prefs.googleSearchEnabled ?? true;
const systemPrompt = getSystemPrompt(profile, customPrompt, googleSearchEnabled);
if (provider === 'openai-realtime') {
// Get OpenAI Realtime configuration
const creds = getOpenAICredentials();
if (!creds.apiKey) {
sendToRenderer('update-status', 'OpenAI API key not configured');
return false;
}
providerConfig = {
apiKey: creds.apiKey,
baseUrl: creds.baseUrl || null,
model: creds.model,
systemPrompt,
language,
isReconnect: false,
};
initializeNewSession(profile, customPrompt);
try {
await openaiRealtimeProvider.initializeOpenAISession(providerConfig, conversationHistory);
return true;
} catch (error) {
console.error('Failed to initialize OpenAI Realtime session:', error);
sendToRenderer('update-status', 'Failed to connect to OpenAI Realtime');
return false;
}
} else if (provider === 'openai-sdk') {
// Get OpenAI SDK configuration (for BotHub, etc.)
const creds = getOpenAISDKCredentials();
if (!creds.apiKey) {
sendToRenderer('update-status', 'OpenAI SDK API key not configured');
return false;
}
providerConfig = {
apiKey: creds.apiKey,
baseUrl: creds.baseUrl || null,
model: creds.model,
visionModel: creds.visionModel,
whisperModel: creds.whisperModel,
};
initializeNewSession(profile, customPrompt);
try {
await openaiSdkProvider.initializeOpenAISDK(providerConfig);
openaiSdkProvider.setSystemPrompt(systemPrompt);
sendToRenderer('update-status', 'Ready (OpenAI SDK)');
return true;
} catch (error) {
console.error('Failed to initialize OpenAI SDK:', error);
sendToRenderer('update-status', 'Failed to initialize OpenAI SDK: ' + error.message);
return false;
}
} else {
// Use Gemini (default)
const apiKey = getApiKey();
if (!apiKey) {
sendToRenderer('update-status', 'Gemini API key not configured');
return false;
}
const session = await geminiProvider.initializeGeminiSession(apiKey, customPrompt, profile, language);
if (session && global.geminiSessionRef) {
global.geminiSessionRef.current = session;
return true;
}
return false;
}
}
// Send audio to appropriate provider
async function sendAudioContent(data, mimeType, isSystemAudio = true) {
if (currentProvider === 'openai-realtime') {
return await openaiRealtimeProvider.sendAudioToOpenAI(data);
} else if (currentProvider === 'openai-sdk') {
// OpenAI SDK buffers audio and transcribes on flush
return await openaiSdkProvider.processAudioChunk(data, mimeType);
} else {
// Gemini
if (!global.geminiSessionRef?.current) {
return { success: false, error: 'No active Gemini session' };
}
try {
const marker = isSystemAudio ? '.' : ',';
process.stdout.write(marker);
await global.geminiSessionRef.current.sendRealtimeInput({
audio: { data, mimeType },
});
return { success: true };
} catch (error) {
console.error('Error sending audio to Gemini:', error);
return { success: false, error: error.message };
}
}
}
// Send image to appropriate provider
async function sendImageContent(data, prompt) {
if (currentProvider === 'openai-realtime') {
const creds = getOpenAICredentials();
const result = await openaiRealtimeProvider.sendImageToOpenAI(data, prompt, {
apiKey: creds.apiKey,
baseUrl: creds.baseUrl,
model: creds.model,
});
if (result.success) {
saveScreenAnalysis(prompt, result.text, result.model);
}
return result;
} else if (currentProvider === 'openai-sdk') {
const result = await openaiSdkProvider.sendImageMessage(data, prompt);
if (result.success) {
saveScreenAnalysis(prompt, result.text, result.model);
}
return result;
} else {
// Use Gemini HTTP API
const result = await geminiProvider.sendImageToGeminiHttp(data, prompt);
// Screen analysis is saved inside sendImageToGeminiHttp for Gemini
return result;
}
}
// Send text message to appropriate provider
async function sendTextMessage(text) {
if (currentProvider === 'openai-realtime') {
return await openaiRealtimeProvider.sendTextToOpenAI(text);
} else if (currentProvider === 'openai-sdk') {
const result = await openaiSdkProvider.sendTextMessage(text);
if (result.success && result.text) {
saveConversationTurn(text, result.text);
}
return result;
} else {
// Gemini
if (!global.geminiSessionRef?.current) {
return { success: false, error: 'No active Gemini session' };
}
try {
console.log('Sending text message to Gemini:', text);
await global.geminiSessionRef.current.sendRealtimeInput({ text: text.trim() });
return { success: true };
} catch (error) {
console.error('Error sending text to Gemini:', error);
return { success: false, error: error.message };
}
}
}
// Close session for appropriate provider
async function closeSession() {
try {
if (currentProvider === 'openai-realtime') {
openaiRealtimeProvider.closeOpenAISession();
} else if (currentProvider === 'openai-sdk') {
openaiSdkProvider.closeOpenAISDK();
} else {
geminiProvider.stopMacOSAudioCapture();
if (global.geminiSessionRef?.current) {
await global.geminiSessionRef.current.close();
global.geminiSessionRef.current = null;
}
}
return { success: true };
} catch (error) {
console.error('Error closing session:', error);
return { success: false, error: error.message };
}
}
// Setup IPC handlers
function setupAIProviderIpcHandlers(geminiSessionRef) {
// Store reference for Gemini
global.geminiSessionRef = geminiSessionRef;
// Listen for conversation turn save requests from providers
ipcMain.on('save-conversation-turn-data', (event, { transcription, response }) => {
saveConversationTurn(transcription, response);
});
ipcMain.handle('initialize-ai-session', async (event, customPrompt, profile, language) => {
return await initializeAISession(customPrompt, profile, language);
});
ipcMain.handle('send-audio-content', async (event, { data, mimeType }) => {
return await sendAudioContent(data, mimeType, true);
});
ipcMain.handle('send-mic-audio-content', async (event, { data, mimeType }) => {
return await sendAudioContent(data, mimeType, false);
});
ipcMain.handle('send-image-content', async (event, { data, prompt }) => {
return await sendImageContent(data, prompt);
});
ipcMain.handle('send-text-message', async (event, text) => {
return await sendTextMessage(text);
});
ipcMain.handle('close-session', async event => {
return await closeSession();
});
// macOS system audio
ipcMain.handle('start-macos-audio', async event => {
if (process.platform !== 'darwin') {
return {
success: false,
error: 'macOS audio capture only available on macOS',
};
}
try {
if (currentProvider === 'gemini') {
const success = await geminiProvider.startMacOSAudioCapture(global.geminiSessionRef);
return { success };
} else if (currentProvider === 'openai-sdk') {
const success = await openaiSdkProvider.startMacOSAudioCapture();
return { success };
} else if (currentProvider === 'openai-realtime') {
// OpenAI Realtime uses WebSocket, handle differently if needed
return {
success: false,
error: 'OpenAI Realtime uses WebSocket for audio',
};
}
return {
success: false,
error: 'Unknown provider: ' + currentProvider,
};
} catch (error) {
console.error('Error starting macOS audio capture:', error);
return { success: false, error: error.message };
}
});
ipcMain.handle('stop-macos-audio', async event => {
try {
if (currentProvider === 'gemini') {
geminiProvider.stopMacOSAudioCapture();
} else if (currentProvider === 'openai-sdk') {
openaiSdkProvider.stopMacOSAudioCapture();
}
return { success: true };
} catch (error) {
console.error('Error stopping macOS audio capture:', error);
return { success: false, error: error.message };
}
});
// Session management
ipcMain.handle('get-current-session', async event => {
try {
return { success: true, data: getCurrentSessionData() };
} catch (error) {
console.error('Error getting current session:', error);
return { success: false, error: error.message };
}
});
ipcMain.handle('start-new-session', async event => {
try {
initializeNewSession();
return { success: true, sessionId: currentSessionId };
} catch (error) {
console.error('Error starting new session:', error);
return { success: false, error: error.message };
}
});
ipcMain.handle('update-google-search-setting', async (event, enabled) => {
try {
console.log('Google Search setting updated to:', enabled);
return { success: true };
} catch (error) {
console.error('Error updating Google Search setting:', error);
return { success: false, error: error.message };
}
});
// Provider switching
ipcMain.handle('switch-ai-provider', async (event, provider) => {
try {
console.log('Switching AI provider to:', provider);
currentProvider = provider;
return { success: true };
} catch (error) {
console.error('Error switching provider:', error);
return { success: false, error: error.message };
}
});
}
module.exports = {
setupAIProviderIpcHandlers,
initializeAISession,
sendAudioContent,
sendImageContent,
sendTextMessage,
closeSession,
getCurrentSessionData,
initializeNewSession,
saveConversationTurn,
};
+605
View File
@@ -0,0 +1,605 @@
const { GoogleGenAI, Modality } = require('@google/genai');
const { BrowserWindow, ipcMain } = require('electron');
const { spawn } = require('child_process');
const { saveDebugAudio } = require('../audioUtils');
const { getSystemPrompt } = require('./prompts');
const { getAvailableModel, incrementLimitCount, getApiKey } = require('../storage');
// Conversation tracking variables
let currentSessionId = null;
let currentTranscription = '';
let conversationHistory = [];
let screenAnalysisHistory = [];
let currentProfile = null;
let currentCustomPrompt = null;
let isInitializingSession = false;
function formatSpeakerResults(results) {
let text = '';
for (const result of results) {
if (result.transcript && result.speakerId) {
const speakerLabel = result.speakerId === 1 ? 'Interviewer' : 'Candidate';
text += `[${speakerLabel}]: ${result.transcript}\n`;
}
}
return text;
}
module.exports.formatSpeakerResults = formatSpeakerResults;
// Audio capture variables
let systemAudioProc = null;
let messageBuffer = '';
// Reconnection variables
let isUserClosing = false;
let sessionParams = null;
let reconnectAttempts = 0;
const MAX_RECONNECT_ATTEMPTS = 3;
const RECONNECT_DELAY = 2000;
function sendToRenderer(channel, data) {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
windows[0].webContents.send(channel, data);
}
}
// Build context message for session restoration
function buildContextMessage() {
const lastTurns = conversationHistory.slice(-20);
const validTurns = lastTurns.filter(turn => turn.transcription?.trim() && turn.ai_response?.trim());
if (validTurns.length === 0) return null;
const contextLines = validTurns.map(turn =>
`[Interviewer]: ${turn.transcription.trim()}\n[Your answer]: ${turn.ai_response.trim()}`
);
return `Session reconnected. Here's the conversation so far:\n\n${contextLines.join('\n\n')}\n\nContinue from here.`;
}
// Conversation management functions
function initializeNewSession(profile = null, customPrompt = null) {
currentSessionId = Date.now().toString();
currentTranscription = '';
conversationHistory = [];
screenAnalysisHistory = [];
currentProfile = profile;
currentCustomPrompt = customPrompt;
console.log('New conversation session started:', currentSessionId, 'profile:', profile);
// Save initial session with profile context
if (profile) {
sendToRenderer('save-session-context', {
sessionId: currentSessionId,
profile: profile,
customPrompt: customPrompt || ''
});
}
}
function saveConversationTurn(transcription, aiResponse) {
if (!currentSessionId) {
initializeNewSession();
}
const conversationTurn = {
timestamp: Date.now(),
transcription: transcription.trim(),
ai_response: aiResponse.trim(),
};
conversationHistory.push(conversationTurn);
console.log('Saved conversation turn:', conversationTurn);
// Send to renderer to save in IndexedDB
sendToRenderer('save-conversation-turn', {
sessionId: currentSessionId,
turn: conversationTurn,
fullHistory: conversationHistory,
});
}
function saveScreenAnalysis(prompt, response, model) {
if (!currentSessionId) {
initializeNewSession();
}
const analysisEntry = {
timestamp: Date.now(),
prompt: prompt,
response: response.trim(),
model: model
};
screenAnalysisHistory.push(analysisEntry);
console.log('Saved screen analysis:', analysisEntry);
// Send to renderer to save
sendToRenderer('save-screen-analysis', {
sessionId: currentSessionId,
analysis: analysisEntry,
fullHistory: screenAnalysisHistory,
profile: currentProfile,
customPrompt: currentCustomPrompt
});
}
function getCurrentSessionData() {
return {
sessionId: currentSessionId,
history: conversationHistory,
};
}
async function getEnabledTools() {
const tools = [];
// Check if Google Search is enabled (default: true)
const googleSearchEnabled = await getStoredSetting('googleSearchEnabled', 'true');
console.log('Google Search enabled:', googleSearchEnabled);
if (googleSearchEnabled === 'true') {
tools.push({ googleSearch: {} });
console.log('Added Google Search tool');
} else {
console.log('Google Search tool disabled');
}
return tools;
}
async function getStoredSetting(key, defaultValue) {
try {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
// Wait a bit for the renderer to be ready
await new Promise(resolve => setTimeout(resolve, 100));
// Try to get setting from renderer process localStorage
const value = await windows[0].webContents.executeJavaScript(`
(function() {
try {
if (typeof localStorage === 'undefined') {
console.log('localStorage not available yet for ${key}');
return '${defaultValue}';
}
const stored = localStorage.getItem('${key}');
console.log('Retrieved setting ${key}:', stored);
return stored || '${defaultValue}';
} catch (e) {
console.error('Error accessing localStorage for ${key}:', e);
return '${defaultValue}';
}
})()
`);
return value;
}
} catch (error) {
console.error('Error getting stored setting for', key, ':', error.message);
}
console.log('Using default value for', key, ':', defaultValue);
return defaultValue;
}
async function initializeGeminiSession(apiKey, customPrompt = '', profile = 'interview', language = 'en-US', isReconnect = false) {
if (isInitializingSession) {
console.log('Session initialization already in progress');
return false;
}
isInitializingSession = true;
if (!isReconnect) {
sendToRenderer('session-initializing', true);
}
// Store params for reconnection
if (!isReconnect) {
sessionParams = { apiKey, customPrompt, profile, language };
reconnectAttempts = 0;
}
const client = new GoogleGenAI({
vertexai: false,
apiKey: apiKey,
httpOptions: { apiVersion: 'v1alpha' },
});
// Get enabled tools first to determine Google Search status
const enabledTools = await getEnabledTools();
const googleSearchEnabled = enabledTools.some(tool => tool.googleSearch);
const systemPrompt = getSystemPrompt(profile, customPrompt, googleSearchEnabled);
// Initialize new conversation session only on first connect
if (!isReconnect) {
initializeNewSession(profile, customPrompt);
}
try {
const session = await client.live.connect({
model: 'gemini-2.5-flash-native-audio-preview-09-2025',
callbacks: {
onopen: function () {
sendToRenderer('update-status', 'Live session connected');
},
onmessage: function (message) {
console.log('----------------', message);
// Handle input transcription (what was spoken)
if (message.serverContent?.inputTranscription?.results) {
currentTranscription += formatSpeakerResults(message.serverContent.inputTranscription.results);
} else if (message.serverContent?.inputTranscription?.text) {
const text = message.serverContent.inputTranscription.text;
if (text.trim() !== '') {
currentTranscription += text;
}
}
// Handle AI model response via output transcription (native audio model)
if (message.serverContent?.outputTranscription?.text) {
const text = message.serverContent.outputTranscription.text;
if (text.trim() === '') return; // Ignore empty transcriptions
const isNewResponse = messageBuffer === '';
messageBuffer += text;
sendToRenderer(isNewResponse ? 'new-response' : 'update-response', messageBuffer);
}
if (message.serverContent?.generationComplete) {
// Only send/save if there's actual content
if (messageBuffer.trim() !== '') {
sendToRenderer('update-response', messageBuffer);
// Save conversation turn when we have both transcription and AI response
if (currentTranscription) {
saveConversationTurn(currentTranscription, messageBuffer);
currentTranscription = ''; // Reset for next turn
}
}
messageBuffer = '';
}
if (message.serverContent?.turnComplete) {
sendToRenderer('update-status', 'Listening...');
}
},
onerror: function (e) {
console.log('Session error:', e.message);
sendToRenderer('update-status', 'Error: ' + e.message);
},
onclose: function (e) {
console.log('Session closed:', e.reason);
// Don't reconnect if user intentionally closed
if (isUserClosing) {
isUserClosing = false;
sendToRenderer('update-status', 'Session closed');
return;
}
// Attempt reconnection
if (sessionParams && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
attemptReconnect();
} else {
sendToRenderer('update-status', 'Session closed');
}
},
},
config: {
responseModalities: [Modality.AUDIO],
proactivity: { proactiveAudio: true },
outputAudioTranscription: {},
tools: enabledTools,
// Enable speaker diarization
inputAudioTranscription: {
enableSpeakerDiarization: true,
minSpeakerCount: 2,
maxSpeakerCount: 2,
},
contextWindowCompression: { slidingWindow: {} },
speechConfig: { languageCode: language },
systemInstruction: {
parts: [{ text: systemPrompt }],
},
},
});
isInitializingSession = false;
if (!isReconnect) {
sendToRenderer('session-initializing', false);
}
return session;
} catch (error) {
console.error('Failed to initialize Gemini session:', error);
isInitializingSession = false;
if (!isReconnect) {
sendToRenderer('session-initializing', false);
}
return null;
}
}
async function attemptReconnect() {
reconnectAttempts++;
console.log(`Reconnection attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS}`);
// Clear stale buffers
messageBuffer = '';
currentTranscription = '';
sendToRenderer('update-status', `Reconnecting... (${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS})`);
// Wait before attempting
await new Promise(resolve => setTimeout(resolve, RECONNECT_DELAY));
try {
const session = await initializeGeminiSession(
sessionParams.apiKey,
sessionParams.customPrompt,
sessionParams.profile,
sessionParams.language,
true // isReconnect
);
if (session && global.geminiSessionRef) {
global.geminiSessionRef.current = session;
// Restore context from conversation history via text message
const contextMessage = buildContextMessage();
if (contextMessage) {
try {
console.log('Restoring conversation context...');
await session.sendRealtimeInput({ text: contextMessage });
} catch (contextError) {
console.error('Failed to restore context:', contextError);
// Continue without context - better than failing
}
}
// Don't reset reconnectAttempts here - let it reset on next fresh session
sendToRenderer('update-status', 'Reconnected! Listening...');
console.log('Session reconnected successfully');
return true;
}
} catch (error) {
console.error(`Reconnection attempt ${reconnectAttempts} failed:`, error);
}
// If we still have attempts left, try again
if (reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
return attemptReconnect();
}
// Max attempts reached - notify frontend
console.log('Max reconnection attempts reached');
sendToRenderer('reconnect-failed', {
message: 'Tried 3 times to reconnect. Must be upstream/network issues. Try restarting or download updated app from site.',
});
sessionParams = null;
return false;
}
function killExistingSystemAudioDump() {
return new Promise(resolve => {
console.log('Checking for existing SystemAudioDump processes...');
// Kill any existing SystemAudioDump processes
const killProc = spawn('pkill', ['-f', 'SystemAudioDump'], {
stdio: 'ignore',
});
killProc.on('close', code => {
if (code === 0) {
console.log('Killed existing SystemAudioDump processes');
} else {
console.log('No existing SystemAudioDump processes found');
}
resolve();
});
killProc.on('error', err => {
console.log('Error checking for existing processes (this is normal):', err.message);
resolve();
});
// Timeout after 2 seconds
setTimeout(() => {
killProc.kill();
resolve();
}, 2000);
});
}
async function startMacOSAudioCapture(geminiSessionRef) {
if (process.platform !== 'darwin') return false;
// Kill any existing SystemAudioDump processes first
await killExistingSystemAudioDump();
console.log('Starting macOS audio capture with SystemAudioDump...');
const { app } = require('electron');
const path = require('path');
let systemAudioPath;
if (app.isPackaged) {
systemAudioPath = path.join(process.resourcesPath, 'SystemAudioDump');
} else {
systemAudioPath = path.join(__dirname, '../assets', 'SystemAudioDump');
}
console.log('SystemAudioDump path:', systemAudioPath);
const spawnOptions = {
stdio: ['ignore', 'pipe', 'pipe'],
env: {
...process.env,
},
};
systemAudioProc = spawn(systemAudioPath, [], spawnOptions);
if (!systemAudioProc.pid) {
console.error('Failed to start SystemAudioDump');
return false;
}
console.log('SystemAudioDump started with PID:', systemAudioProc.pid);
const CHUNK_DURATION = 0.1;
const SAMPLE_RATE = 24000;
const BYTES_PER_SAMPLE = 2;
const CHANNELS = 2;
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
let audioBuffer = Buffer.alloc(0);
systemAudioProc.stdout.on('data', data => {
audioBuffer = Buffer.concat([audioBuffer, data]);
while (audioBuffer.length >= CHUNK_SIZE) {
const chunk = audioBuffer.slice(0, CHUNK_SIZE);
audioBuffer = audioBuffer.slice(CHUNK_SIZE);
const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk;
const base64Data = monoChunk.toString('base64');
sendAudioToGemini(base64Data, geminiSessionRef);
if (process.env.DEBUG_AUDIO) {
console.log(`Processed audio chunk: ${chunk.length} bytes`);
saveDebugAudio(monoChunk, 'system_audio');
}
}
const maxBufferSize = SAMPLE_RATE * BYTES_PER_SAMPLE * 1;
if (audioBuffer.length > maxBufferSize) {
audioBuffer = audioBuffer.slice(-maxBufferSize);
}
});
systemAudioProc.stderr.on('data', data => {
console.error('SystemAudioDump stderr:', data.toString());
});
systemAudioProc.on('close', code => {
console.log('SystemAudioDump process closed with code:', code);
systemAudioProc = null;
});
systemAudioProc.on('error', err => {
console.error('SystemAudioDump process error:', err);
systemAudioProc = null;
});
return true;
}
function convertStereoToMono(stereoBuffer) {
const samples = stereoBuffer.length / 4;
const monoBuffer = Buffer.alloc(samples * 2);
for (let i = 0; i < samples; i++) {
const leftSample = stereoBuffer.readInt16LE(i * 4);
monoBuffer.writeInt16LE(leftSample, i * 2);
}
return monoBuffer;
}
function stopMacOSAudioCapture() {
if (systemAudioProc) {
console.log('Stopping SystemAudioDump...');
systemAudioProc.kill('SIGTERM');
systemAudioProc = null;
}
}
async function sendAudioToGemini(base64Data, geminiSessionRef) {
if (!geminiSessionRef.current) return;
try {
process.stdout.write('.');
await geminiSessionRef.current.sendRealtimeInput({
audio: {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
},
});
} catch (error) {
console.error('Error sending audio to Gemini:', error);
}
}
async function sendImageToGeminiHttp(base64Data, prompt) {
// Get available model based on rate limits
const model = getAvailableModel();
const apiKey = getApiKey();
if (!apiKey) {
return { success: false, error: 'No API key configured' };
}
try {
const ai = new GoogleGenAI({ apiKey: apiKey });
const contents = [
{
inlineData: {
mimeType: 'image/jpeg',
data: base64Data,
},
},
{ text: prompt },
];
console.log(`Sending image to ${model} (streaming)...`);
const response = await ai.models.generateContentStream({
model: model,
contents: contents,
});
// Increment count after successful call
incrementLimitCount(model);
// Stream the response
let fullText = '';
let isFirst = true;
for await (const chunk of response) {
const chunkText = chunk.text;
if (chunkText) {
fullText += chunkText;
// Send to renderer - new response for first chunk, update for subsequent
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullText);
isFirst = false;
}
}
console.log(`Image response completed from ${model}`);
// Save screen analysis to history
saveScreenAnalysis(prompt, fullText, model);
return { success: true, text: fullText, model: model };
} catch (error) {
console.error('Error sending image to Gemini HTTP:', error);
return { success: false, error: error.message };
}
}
module.exports = {
initializeGeminiSession,
getEnabledTools,
getStoredSetting,
sendToRenderer,
initializeNewSession,
saveConversationTurn,
getCurrentSessionData,
killExistingSystemAudioDump,
startMacOSAudioCapture,
convertStereoToMono,
stopMacOSAudioCapture,
sendAudioToGemini,
sendImageToGeminiHttp,
formatSpeakerResults,
};
+402
View File
@@ -0,0 +1,402 @@
const { BrowserWindow } = require('electron');
const WebSocket = require('ws');
// OpenAI Realtime API implementation
// Documentation: https://platform.openai.com/docs/api-reference/realtime
let ws = null;
let isUserClosing = false;
let sessionParams = null;
let reconnectAttempts = 0;
const MAX_RECONNECT_ATTEMPTS = 3;
const RECONNECT_DELAY = 2000;
// Message buffer for accumulating responses
let messageBuffer = '';
let currentTranscription = '';
function sendToRenderer(channel, data) {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
windows[0].webContents.send(channel, data);
}
}
function buildContextMessage(conversationHistory) {
const lastTurns = conversationHistory.slice(-20);
const validTurns = lastTurns.filter(turn => turn.transcription?.trim() && turn.ai_response?.trim());
if (validTurns.length === 0) return null;
const contextLines = validTurns.map(turn => `User: ${turn.transcription.trim()}\nAssistant: ${turn.ai_response.trim()}`);
return `Session reconnected. Here's the conversation so far:\n\n${contextLines.join('\n\n')}\n\nContinue from here.`;
}
async function initializeOpenAISession(config, conversationHistory = []) {
const { apiKey, baseUrl, systemPrompt, model, language, isReconnect } = config;
if (!isReconnect) {
sessionParams = config;
reconnectAttempts = 0;
sendToRenderer('session-initializing', true);
}
// Use custom baseURL or default OpenAI endpoint
const wsUrl = baseUrl || 'wss://api.openai.com/v1/realtime';
const fullUrl = `${wsUrl}?model=${model || 'gpt-4o-realtime-preview-2024-12-17'}`;
return new Promise((resolve, reject) => {
try {
ws = new WebSocket(fullUrl, {
headers: {
Authorization: `Bearer ${apiKey}`,
'OpenAI-Beta': 'realtime=v1',
},
});
ws.on('open', () => {
console.log('OpenAI Realtime connection established');
// Configure session
const sessionConfig = {
type: 'session.update',
session: {
modalities: ['text', 'audio'],
instructions: systemPrompt,
voice: 'alloy',
input_audio_format: 'pcm16',
output_audio_format: 'pcm16',
input_audio_transcription: {
model: 'whisper-1',
},
turn_detection: {
type: 'server_vad',
threshold: 0.5,
prefix_padding_ms: 300,
silence_duration_ms: 500,
},
temperature: 0.8,
max_response_output_tokens: 4096,
},
};
ws.send(JSON.stringify(sessionConfig));
// Restore context if reconnecting
if (isReconnect && conversationHistory.length > 0) {
const contextMessage = buildContextMessage(conversationHistory);
if (contextMessage) {
ws.send(
JSON.stringify({
type: 'conversation.item.create',
item: {
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: contextMessage }],
},
})
);
ws.send(JSON.stringify({ type: 'response.create' }));
}
}
sendToRenderer('update-status', 'Connected to OpenAI');
if (!isReconnect) {
sendToRenderer('session-initializing', false);
}
resolve(ws);
});
ws.on('message', data => {
try {
const event = JSON.parse(data.toString());
handleOpenAIEvent(event);
} catch (error) {
console.error('Error parsing OpenAI message:', error);
}
});
ws.on('error', error => {
console.error('OpenAI WebSocket error:', error);
sendToRenderer('update-status', 'Error: ' + error.message);
reject(error);
});
ws.on('close', (code, reason) => {
console.log(`OpenAI WebSocket closed: ${code} - ${reason}`);
if (isUserClosing) {
isUserClosing = false;
sendToRenderer('update-status', 'Session closed');
return;
}
// Attempt reconnection
if (sessionParams && reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
attemptReconnect(conversationHistory);
} else {
sendToRenderer('update-status', 'Session closed');
}
});
} catch (error) {
console.error('Failed to initialize OpenAI session:', error);
if (!isReconnect) {
sendToRenderer('session-initializing', false);
}
reject(error);
}
});
}
function handleOpenAIEvent(event) {
console.log('OpenAI event:', event.type);
switch (event.type) {
case 'session.created':
console.log('Session created:', event.session.id);
break;
case 'session.updated':
console.log('Session updated');
sendToRenderer('update-status', 'Listening...');
break;
case 'input_audio_buffer.speech_started':
console.log('Speech started');
break;
case 'input_audio_buffer.speech_stopped':
console.log('Speech stopped');
break;
case 'conversation.item.input_audio_transcription.completed':
if (event.transcript) {
currentTranscription += event.transcript;
console.log('Transcription:', event.transcript);
}
break;
case 'response.audio_transcript.delta':
if (event.delta) {
const isNewResponse = messageBuffer === '';
messageBuffer += event.delta;
sendToRenderer(isNewResponse ? 'new-response' : 'update-response', messageBuffer);
}
break;
case 'response.audio_transcript.done':
console.log('Audio transcript complete');
break;
case 'response.text.delta':
if (event.delta) {
const isNewResponse = messageBuffer === '';
messageBuffer += event.delta;
sendToRenderer(isNewResponse ? 'new-response' : 'update-response', messageBuffer);
}
break;
case 'response.done':
if (messageBuffer.trim() !== '') {
sendToRenderer('update-response', messageBuffer);
// Send conversation turn to be saved
if (currentTranscription) {
sendToRenderer('save-conversation-turn-data', {
transcription: currentTranscription,
response: messageBuffer,
});
currentTranscription = '';
}
}
messageBuffer = '';
sendToRenderer('update-status', 'Listening...');
break;
case 'error':
console.error('OpenAI error:', event.error);
sendToRenderer('update-status', 'Error: ' + event.error.message);
break;
default:
// console.log('Unhandled event type:', event.type);
break;
}
}
async function attemptReconnect(conversationHistory) {
reconnectAttempts++;
console.log(`Reconnection attempt ${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS}`);
messageBuffer = '';
currentTranscription = '';
sendToRenderer('update-status', `Reconnecting... (${reconnectAttempts}/${MAX_RECONNECT_ATTEMPTS})`);
await new Promise(resolve => setTimeout(resolve, RECONNECT_DELAY));
try {
const newConfig = { ...sessionParams, isReconnect: true };
ws = await initializeOpenAISession(newConfig, conversationHistory);
sendToRenderer('update-status', 'Reconnected! Listening...');
console.log('OpenAI session reconnected successfully');
return true;
} catch (error) {
console.error(`Reconnection attempt ${reconnectAttempts} failed:`, error);
if (reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {
return attemptReconnect(conversationHistory);
}
console.log('Max reconnection attempts reached');
sendToRenderer('reconnect-failed', {
message: 'Tried 3 times to reconnect to OpenAI. Check your connection and API key.',
});
sessionParams = null;
return false;
}
}
async function sendAudioToOpenAI(base64Data) {
if (!ws || ws.readyState !== WebSocket.OPEN) {
console.error('WebSocket not connected');
return { success: false, error: 'No active connection' };
}
try {
ws.send(
JSON.stringify({
type: 'input_audio_buffer.append',
audio: base64Data,
})
);
return { success: true };
} catch (error) {
console.error('Error sending audio to OpenAI:', error);
return { success: false, error: error.message };
}
}
async function sendTextToOpenAI(text) {
if (!ws || ws.readyState !== WebSocket.OPEN) {
console.error('WebSocket not connected');
return { success: false, error: 'No active connection' };
}
try {
// Create a conversation item with user text
ws.send(
JSON.stringify({
type: 'conversation.item.create',
item: {
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: text }],
},
})
);
// Trigger response generation
ws.send(JSON.stringify({ type: 'response.create' }));
return { success: true };
} catch (error) {
console.error('Error sending text to OpenAI:', error);
return { success: false, error: error.message };
}
}
async function sendImageToOpenAI(base64Data, prompt, config) {
const { apiKey, baseUrl, model } = config;
// OpenAI doesn't support images in Realtime API yet, use standard Chat Completions
const apiEndpoint = baseUrl ? `${baseUrl.replace('wss://', 'https://').replace('/v1/realtime', '')}/v1/chat/completions` : 'https://api.openai.com/v1/chat/completions';
try {
const response = await fetch(apiEndpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${apiKey}`,
},
body: JSON.stringify({
model: model || 'gpt-4o',
messages: [
{
role: 'user',
content: [
{ type: 'text', text: prompt },
{
type: 'image_url',
image_url: {
url: `data:image/jpeg;base64,${base64Data}`,
},
},
],
},
],
max_tokens: 4096,
stream: true,
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`OpenAI API error: ${response.status} - ${error}`);
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let fullText = '';
let isFirst = true;
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value);
const lines = chunk.split('\n').filter(line => line.trim().startsWith('data: '));
for (const line of lines) {
const data = line.replace('data: ', '');
if (data === '[DONE]') continue;
try {
const json = JSON.parse(data);
const content = json.choices[0]?.delta?.content;
if (content) {
fullText += content;
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullText);
isFirst = false;
}
} catch (e) {
// Skip invalid JSON
}
}
}
return { success: true, text: fullText, model: model || 'gpt-4o' };
} catch (error) {
console.error('Error sending image to OpenAI:', error);
return { success: false, error: error.message };
}
}
function closeOpenAISession() {
isUserClosing = true;
sessionParams = null;
if (ws) {
ws.close();
ws = null;
}
}
module.exports = {
initializeOpenAISession,
sendAudioToOpenAI,
sendTextToOpenAI,
sendImageToOpenAI,
closeOpenAISession,
};
+561
View File
@@ -0,0 +1,561 @@
const { BrowserWindow } = require('electron');
const fs = require('fs');
const path = require('path');
const os = require('os');
const { spawn } = require('child_process');
// OpenAI SDK will be loaded dynamically
let OpenAI = null;
// OpenAI SDK-based provider (for BotHub, Azure, and other OpenAI-compatible APIs)
// This uses the standard Chat Completions API with Whisper for transcription
let openaiClient = null;
let currentConfig = null;
let conversationMessages = [];
let isProcessing = false;
// macOS audio capture
let systemAudioProc = null;
let audioBuffer = Buffer.alloc(0);
let transcriptionTimer = null;
const TRANSCRIPTION_INTERVAL_MS = 3000; // Transcribe every 3 seconds
const MIN_AUDIO_DURATION_MS = 500; // Minimum audio duration to transcribe
const SAMPLE_RATE = 24000;
function sendToRenderer(channel, data) {
const windows = BrowserWindow.getAllWindows();
if (windows.length > 0) {
windows[0].webContents.send(channel, data);
}
}
async function initializeOpenAISDK(config) {
const { apiKey, baseUrl, model } = config;
if (!apiKey) {
throw new Error('OpenAI API key is required');
}
// Dynamic import for ES module
if (!OpenAI) {
const openaiModule = await import('openai');
OpenAI = openaiModule.default;
}
const clientConfig = {
apiKey: apiKey,
};
// Use custom baseURL if provided
if (baseUrl && baseUrl.trim() !== '') {
clientConfig.baseURL = baseUrl;
}
openaiClient = new OpenAI(clientConfig);
currentConfig = config;
conversationMessages = [];
console.log('OpenAI SDK initialized with baseURL:', clientConfig.baseURL || 'default');
sendToRenderer('update-status', 'Ready (OpenAI SDK)');
return true;
}
function setSystemPrompt(systemPrompt) {
// Clear conversation and set system prompt
conversationMessages = [];
if (systemPrompt) {
conversationMessages.push({
role: 'system',
content: systemPrompt,
});
}
}
// Create WAV file from raw PCM data
function createWavBuffer(pcmBuffer, sampleRate = 24000, numChannels = 1, bitsPerSample = 16) {
const byteRate = sampleRate * numChannels * (bitsPerSample / 8);
const blockAlign = numChannels * (bitsPerSample / 8);
const dataSize = pcmBuffer.length;
const headerSize = 44;
const fileSize = headerSize + dataSize - 8;
const wavBuffer = Buffer.alloc(headerSize + dataSize);
// RIFF header
wavBuffer.write('RIFF', 0);
wavBuffer.writeUInt32LE(fileSize, 4);
wavBuffer.write('WAVE', 8);
// fmt chunk
wavBuffer.write('fmt ', 12);
wavBuffer.writeUInt32LE(16, 16); // fmt chunk size
wavBuffer.writeUInt16LE(1, 20); // audio format (1 = PCM)
wavBuffer.writeUInt16LE(numChannels, 22);
wavBuffer.writeUInt32LE(sampleRate, 24);
wavBuffer.writeUInt32LE(byteRate, 28);
wavBuffer.writeUInt16LE(blockAlign, 32);
wavBuffer.writeUInt16LE(bitsPerSample, 34);
// data chunk
wavBuffer.write('data', 36);
wavBuffer.writeUInt32LE(dataSize, 40);
// Copy PCM data
pcmBuffer.copy(wavBuffer, 44);
return wavBuffer;
}
async function transcribeAudio(audioBuffer, mimeType = 'audio/wav') {
if (!openaiClient) {
throw new Error('OpenAI client not initialized');
}
try {
// Save audio buffer to temp file (OpenAI SDK requires file path)
const tempDir = os.tmpdir();
const tempFile = path.join(tempDir, `audio_${Date.now()}.wav`);
// Convert base64 to buffer if needed
let buffer = audioBuffer;
if (typeof audioBuffer === 'string') {
buffer = Buffer.from(audioBuffer, 'base64');
}
// Create proper WAV file with header
const wavBuffer = createWavBuffer(buffer, SAMPLE_RATE, 1, 16);
fs.writeFileSync(tempFile, wavBuffer);
const transcription = await openaiClient.audio.transcriptions.create({
file: fs.createReadStream(tempFile),
model: currentConfig.whisperModel || 'whisper-1',
response_format: 'text',
});
// Clean up temp file
try {
fs.unlinkSync(tempFile);
} catch (e) {
// Ignore cleanup errors
}
return transcription;
} catch (error) {
console.error('Transcription error:', error);
throw error;
}
}
async function sendTextMessage(text) {
if (!openaiClient) {
return { success: false, error: 'OpenAI client not initialized' };
}
if (isProcessing) {
return { success: false, error: 'Already processing a request' };
}
isProcessing = true;
try {
// Add user message to conversation
conversationMessages.push({
role: 'user',
content: text,
});
sendToRenderer('update-status', 'Thinking...');
const stream = await openaiClient.chat.completions.create({
model: currentConfig.model || 'gpt-4o',
messages: conversationMessages,
stream: true,
max_tokens: 4096,
});
let fullResponse = '';
let isFirst = true;
for await (const chunk of stream) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
fullResponse += content;
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullResponse);
isFirst = false;
}
}
// Add assistant response to conversation
conversationMessages.push({
role: 'assistant',
content: fullResponse,
});
sendToRenderer('update-status', 'Ready');
isProcessing = false;
return { success: true, text: fullResponse };
} catch (error) {
console.error('Chat completion error:', error);
sendToRenderer('update-status', 'Error: ' + error.message);
isProcessing = false;
return { success: false, error: error.message };
}
}
async function sendImageMessage(base64Image, prompt) {
if (!openaiClient) {
return { success: false, error: 'OpenAI client not initialized' };
}
if (isProcessing) {
return { success: false, error: 'Already processing a request' };
}
isProcessing = true;
try {
sendToRenderer('update-status', 'Analyzing image...');
const messages = [
...conversationMessages,
{
role: 'user',
content: [
{ type: 'text', text: prompt },
{
type: 'image_url',
image_url: {
url: `data:image/jpeg;base64,${base64Image}`,
},
},
],
},
];
const stream = await openaiClient.chat.completions.create({
model: currentConfig.visionModel || currentConfig.model || 'gpt-4o',
messages: messages,
stream: true,
max_tokens: 4096,
});
let fullResponse = '';
let isFirst = true;
for await (const chunk of stream) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
fullResponse += content;
sendToRenderer(isFirst ? 'new-response' : 'update-response', fullResponse);
isFirst = false;
}
}
// Add to conversation history (text only for follow-ups)
conversationMessages.push({
role: 'user',
content: prompt,
});
conversationMessages.push({
role: 'assistant',
content: fullResponse,
});
sendToRenderer('update-status', 'Ready');
isProcessing = false;
return { success: true, text: fullResponse, model: currentConfig.visionModel || currentConfig.model };
} catch (error) {
console.error('Vision error:', error);
sendToRenderer('update-status', 'Error: ' + error.message);
isProcessing = false;
return { success: false, error: error.message };
}
}
// Process audio chunk and get response
// This accumulates audio and transcribes when silence is detected
let audioChunks = [];
let lastAudioTime = 0;
const SILENCE_THRESHOLD_MS = 1500; // 1.5 seconds of silence
async function processAudioChunk(base64Audio, mimeType) {
if (!openaiClient) {
return { success: false, error: 'OpenAI client not initialized' };
}
const now = Date.now();
const buffer = Buffer.from(base64Audio, 'base64');
// Add to audio buffer
audioChunks.push(buffer);
lastAudioTime = now;
// Check for silence (no new audio for SILENCE_THRESHOLD_MS)
// This is a simple approach - in production you'd want proper VAD
return { success: true, buffering: true };
}
async function flushAudioAndTranscribe() {
if (audioChunks.length === 0) {
return { success: true, text: '' };
}
try {
// Combine all audio chunks
const combinedBuffer = Buffer.concat(audioChunks);
audioChunks = [];
// Transcribe
const transcription = await transcribeAudio(combinedBuffer);
if (transcription && transcription.trim()) {
// Send to chat
const response = await sendTextMessage(transcription);
return {
success: true,
transcription: transcription,
response: response.text,
};
}
return { success: true, text: '' };
} catch (error) {
console.error('Flush audio error:', error);
return { success: false, error: error.message };
}
}
function clearConversation() {
const systemMessage = conversationMessages.find(m => m.role === 'system');
conversationMessages = systemMessage ? [systemMessage] : [];
audioChunks = [];
}
function closeOpenAISDK() {
stopMacOSAudioCapture();
openaiClient = null;
currentConfig = null;
conversationMessages = [];
audioChunks = [];
isProcessing = false;
sendToRenderer('update-status', 'Disconnected');
}
// ============ macOS Audio Capture ============
async function killExistingSystemAudioDump() {
return new Promise(resolve => {
const { exec } = require('child_process');
exec('pkill -f SystemAudioDump', error => {
// Ignore errors (process might not exist)
setTimeout(resolve, 100);
});
});
}
function convertStereoToMono(stereoBuffer) {
const samples = stereoBuffer.length / 4;
const monoBuffer = Buffer.alloc(samples * 2);
for (let i = 0; i < samples; i++) {
const leftSample = stereoBuffer.readInt16LE(i * 4);
monoBuffer.writeInt16LE(leftSample, i * 2);
}
return monoBuffer;
}
// Calculate RMS (Root Mean Square) volume level of audio buffer
function calculateRMS(buffer) {
const samples = buffer.length / 2;
if (samples === 0) return 0;
let sumSquares = 0;
for (let i = 0; i < samples; i++) {
const sample = buffer.readInt16LE(i * 2);
sumSquares += sample * sample;
}
return Math.sqrt(sumSquares / samples);
}
// Check if audio contains speech (simple VAD based on volume threshold)
function hasSpeech(buffer, threshold = 500) {
const rms = calculateRMS(buffer);
return rms > threshold;
}
async function transcribeBufferedAudio() {
if (audioBuffer.length === 0 || isProcessing) {
return;
}
// Calculate audio duration
const bytesPerSample = 2;
const audioDurationMs = (audioBuffer.length / bytesPerSample / SAMPLE_RATE) * 1000;
if (audioDurationMs < MIN_AUDIO_DURATION_MS) {
return; // Not enough audio
}
// Check if there's actual speech in the audio (Voice Activity Detection)
if (!hasSpeech(audioBuffer)) {
// Clear buffer if it's just silence/noise
audioBuffer = Buffer.alloc(0);
return;
}
// Take current buffer and reset
const currentBuffer = audioBuffer;
audioBuffer = Buffer.alloc(0);
try {
console.log(`Transcribing ${audioDurationMs.toFixed(0)}ms of audio...`);
sendToRenderer('update-status', 'Transcribing...');
const transcription = await transcribeAudio(currentBuffer, 'audio/wav');
if (transcription && transcription.trim() && transcription.trim().length > 2) {
console.log('Transcription:', transcription);
sendToRenderer('update-status', 'Processing...');
// Send to chat
await sendTextMessage(transcription);
}
sendToRenderer('update-status', 'Listening...');
} catch (error) {
console.error('Transcription error:', error);
sendToRenderer('update-status', 'Listening...');
}
}
async function startMacOSAudioCapture() {
if (process.platform !== 'darwin') return false;
// Kill any existing SystemAudioDump processes first
await killExistingSystemAudioDump();
console.log('Starting macOS audio capture with SystemAudioDump for OpenAI SDK...');
const { app } = require('electron');
let systemAudioPath;
if (app.isPackaged) {
systemAudioPath = path.join(process.resourcesPath, 'SystemAudioDump');
} else {
systemAudioPath = path.join(__dirname, '../assets', 'SystemAudioDump');
}
console.log('SystemAudioDump path:', systemAudioPath);
const spawnOptions = {
stdio: ['ignore', 'pipe', 'pipe'],
env: {
...process.env,
},
};
systemAudioProc = spawn(systemAudioPath, [], spawnOptions);
if (!systemAudioProc.pid) {
console.error('Failed to start SystemAudioDump');
return false;
}
console.log('SystemAudioDump started with PID:', systemAudioProc.pid);
const CHUNK_DURATION = 0.1;
const BYTES_PER_SAMPLE = 2;
const CHANNELS = 2;
const CHUNK_SIZE = SAMPLE_RATE * BYTES_PER_SAMPLE * CHANNELS * CHUNK_DURATION;
let tempBuffer = Buffer.alloc(0);
systemAudioProc.stdout.on('data', data => {
tempBuffer = Buffer.concat([tempBuffer, data]);
while (tempBuffer.length >= CHUNK_SIZE) {
const chunk = tempBuffer.slice(0, CHUNK_SIZE);
tempBuffer = tempBuffer.slice(CHUNK_SIZE);
// Convert stereo to mono
const monoChunk = CHANNELS === 2 ? convertStereoToMono(chunk) : chunk;
// Add to audio buffer for transcription
audioBuffer = Buffer.concat([audioBuffer, monoChunk]);
}
// Limit buffer size (max 30 seconds of audio)
const maxBufferSize = SAMPLE_RATE * BYTES_PER_SAMPLE * 30;
if (audioBuffer.length > maxBufferSize) {
audioBuffer = audioBuffer.slice(-maxBufferSize);
}
});
systemAudioProc.stderr.on('data', data => {
console.error('SystemAudioDump stderr:', data.toString());
});
systemAudioProc.on('close', code => {
console.log('SystemAudioDump process closed with code:', code);
systemAudioProc = null;
stopTranscriptionTimer();
});
systemAudioProc.on('error', err => {
console.error('SystemAudioDump process error:', err);
systemAudioProc = null;
stopTranscriptionTimer();
});
// Start periodic transcription
startTranscriptionTimer();
sendToRenderer('update-status', 'Listening...');
return true;
}
function startTranscriptionTimer() {
stopTranscriptionTimer();
transcriptionTimer = setInterval(transcribeBufferedAudio, TRANSCRIPTION_INTERVAL_MS);
}
function stopTranscriptionTimer() {
if (transcriptionTimer) {
clearInterval(transcriptionTimer);
transcriptionTimer = null;
}
}
function stopMacOSAudioCapture() {
stopTranscriptionTimer();
if (systemAudioProc) {
console.log('Stopping SystemAudioDump for OpenAI SDK...');
systemAudioProc.kill('SIGTERM');
systemAudioProc = null;
}
audioBuffer = Buffer.alloc(0);
}
module.exports = {
initializeOpenAISDK,
setSystemPrompt,
transcribeAudio,
sendTextMessage,
sendImageMessage,
processAudioChunk,
flushAudioAndTranscribe,
clearConversation,
closeOpenAISDK,
startMacOSAudioCapture,
stopMacOSAudioCapture,
};
+225
View File
@@ -0,0 +1,225 @@
const profilePrompts = {
interview: {
intro: `You are an AI-powered interview assistant, designed to act as a discreet on-screen teleprompter. Your mission is to help the user excel in their job interview by providing concise, impactful, and ready-to-speak answers or key talking points. Analyze the ongoing interview dialogue and, crucially, the 'User-provided context' below.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-3 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for key points and emphasis
- Use bullet points (-) for lists when appropriate
- Focus on the most essential information only`,
searchUsage: `**SEARCH TOOL USAGE:**
- If the interviewer mentions **recent events, news, or current trends** (anything from the last 6 months), **ALWAYS use Google search** to get up-to-date information
- If they ask about **company-specific information, recent acquisitions, funding, or leadership changes**, use Google search first
- If they mention **new technologies, frameworks, or industry developments**, search for the latest information
- After searching, provide a **concise, informed response** based on the real-time data`,
content: `Focus on delivering the most essential information the user needs. Your suggestions should be direct and immediately usable.
To help the user 'crack' the interview in their specific field:
1. Heavily rely on the 'User-provided context' (e.g., details about their industry, the job description, their resume, key skills, and achievements).
2. Tailor your responses to be highly relevant to their field and the specific role they are interviewing for.
Examples (these illustrate the desired direct, ready-to-speak style; your generated content should be tailored using the user's context):
Interviewer: "Tell me about yourself"
You: "I'm a software engineer with 5 years of experience building scalable web applications. I specialize in React and Node.js, and I've led development teams at two different startups. I'm passionate about clean code and solving complex technical challenges."
Interviewer: "What's your experience with React?"
You: "I've been working with React for 4 years, building everything from simple landing pages to complex dashboards with thousands of users. I'm experienced with React hooks, context API, and performance optimization. I've also worked with Next.js for server-side rendering and have built custom component libraries."
Interviewer: "Why do you want to work here?"
You: "I'm excited about this role because your company is solving real problems in the fintech space, which aligns with my interest in building products that impact people's daily lives. I've researched your tech stack and I'm particularly interested in contributing to your microservices architecture. Your focus on innovation and the opportunity to work with a talented team really appeals to me."`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide only the exact words to say in **markdown format**. No coaching, no "you should" statements, no explanations - just the direct response the candidate can speak immediately. Keep it **short and impactful**.`,
},
sales: {
intro: `You are a sales call assistant. Your job is to provide the exact words the salesperson should say to prospects during sales calls. Give direct, ready-to-speak responses that are persuasive and professional.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-3 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for key points and emphasis
- Use bullet points (-) for lists when appropriate
- Focus on the most essential information only`,
searchUsage: `**SEARCH TOOL USAGE:**
- If the prospect mentions **recent industry trends, market changes, or current events**, **ALWAYS use Google search** to get up-to-date information
- If they reference **competitor information, recent funding news, or market data**, search for the latest information first
- If they ask about **new regulations, industry reports, or recent developments**, use search to provide accurate data
- After searching, provide a **concise, informed response** that demonstrates current market knowledge`,
content: `Examples:
Prospect: "Tell me about your product"
You: "Our platform helps companies like yours reduce operational costs by 30% while improving efficiency. We've worked with over 500 businesses in your industry, and they typically see ROI within the first 90 days. What specific operational challenges are you facing right now?"
Prospect: "What makes you different from competitors?"
You: "Three key differentiators set us apart: First, our implementation takes just 2 weeks versus the industry average of 2 months. Second, we provide dedicated support with response times under 4 hours. Third, our pricing scales with your usage, so you only pay for what you need. Which of these resonates most with your current situation?"
Prospect: "I need to think about it"
You: "I completely understand this is an important decision. What specific concerns can I address for you today? Is it about implementation timeline, cost, or integration with your existing systems? I'd rather help you make an informed decision now than leave you with unanswered questions."`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide only the exact words to say in **markdown format**. Be persuasive but not pushy. Focus on value and addressing objections directly. Keep responses **short and impactful**.`,
},
meeting: {
intro: `You are a meeting assistant. Your job is to provide the exact words to say during professional meetings, presentations, and discussions. Give direct, ready-to-speak responses that are clear and professional.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-3 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for key points and emphasis
- Use bullet points (-) for lists when appropriate
- Focus on the most essential information only`,
searchUsage: `**SEARCH TOOL USAGE:**
- If participants mention **recent industry news, regulatory changes, or market updates**, **ALWAYS use Google search** for current information
- If they reference **competitor activities, recent reports, or current statistics**, search for the latest data first
- If they discuss **new technologies, tools, or industry developments**, use search to provide accurate insights
- After searching, provide a **concise, informed response** that adds value to the discussion`,
content: `Examples:
Participant: "What's the status on the project?"
You: "We're currently on track to meet our deadline. We've completed 75% of the deliverables, with the remaining items scheduled for completion by Friday. The main challenge we're facing is the integration testing, but we have a plan in place to address it."
Participant: "Can you walk us through the budget?"
You: "Absolutely. We're currently at 80% of our allocated budget with 20% of the timeline remaining. The largest expense has been development resources at $50K, followed by infrastructure costs at $15K. We have contingency funds available if needed for the final phase."
Participant: "What are the next steps?"
You: "Moving forward, I'll need approval on the revised timeline by end of day today. Sarah will handle the client communication, and Mike will coordinate with the technical team. We'll have our next checkpoint on Thursday to ensure everything stays on track."`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide only the exact words to say in **markdown format**. Be clear, concise, and action-oriented in your responses. Keep it **short and impactful**.`,
},
presentation: {
intro: `You are a presentation coach. Your job is to provide the exact words the presenter should say during presentations, pitches, and public speaking events. Give direct, ready-to-speak responses that are engaging and confident.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-3 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for key points and emphasis
- Use bullet points (-) for lists when appropriate
- Focus on the most essential information only`,
searchUsage: `**SEARCH TOOL USAGE:**
- If the audience asks about **recent market trends, current statistics, or latest industry data**, **ALWAYS use Google search** for up-to-date information
- If they reference **recent events, new competitors, or current market conditions**, search for the latest information first
- If they inquire about **recent studies, reports, or breaking news** in your field, use search to provide accurate data
- After searching, provide a **concise, credible response** with current facts and figures`,
content: `Examples:
Audience: "Can you explain that slide again?"
You: "Of course. This slide shows our three-year growth trajectory. The blue line represents revenue, which has grown 150% year over year. The orange bars show our customer acquisition, doubling each year. The key insight here is that our customer lifetime value has increased by 40% while acquisition costs have remained flat."
Audience: "What's your competitive advantage?"
You: "Great question. Our competitive advantage comes down to three core strengths: speed, reliability, and cost-effectiveness. We deliver results 3x faster than traditional solutions, with 99.9% uptime, at 50% lower cost. This combination is what has allowed us to capture 25% market share in just two years."
Audience: "How do you plan to scale?"
You: "Our scaling strategy focuses on three pillars. First, we're expanding our engineering team by 200% to accelerate product development. Second, we're entering three new markets next quarter. Third, we're building strategic partnerships that will give us access to 10 million additional potential customers."`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide only the exact words to say in **markdown format**. Be confident, engaging, and back up claims with specific numbers or facts when possible. Keep responses **short and impactful**.`,
},
negotiation: {
intro: `You are a negotiation assistant. Your job is to provide the exact words to say during business negotiations, contract discussions, and deal-making conversations. Give direct, ready-to-speak responses that are strategic and professional.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-3 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for key points and emphasis
- Use bullet points (-) for lists when appropriate
- Focus on the most essential information only`,
searchUsage: `**SEARCH TOOL USAGE:**
- If they mention **recent market pricing, current industry standards, or competitor offers**, **ALWAYS use Google search** for current benchmarks
- If they reference **recent legal changes, new regulations, or market conditions**, search for the latest information first
- If they discuss **recent company news, financial performance, or industry developments**, use search to provide informed responses
- After searching, provide a **strategic, well-informed response** that leverages current market intelligence`,
content: `Examples:
Other party: "That price is too high"
You: "I understand your concern about the investment. Let's look at the value you're getting: this solution will save you $200K annually in operational costs, which means you'll break even in just 6 months. Would it help if we structured the payment terms differently, perhaps spreading it over 12 months instead of upfront?"
Other party: "We need a better deal"
You: "I appreciate your directness. We want this to work for both parties. Our current offer is already at a 15% discount from our standard pricing. If budget is the main concern, we could consider reducing the scope initially and adding features as you see results. What specific budget range were you hoping to achieve?"
Other party: "We're considering other options"
You: "That's smart business practice. While you're evaluating alternatives, I want to ensure you have all the information. Our solution offers three unique benefits that others don't: 24/7 dedicated support, guaranteed 48-hour implementation, and a money-back guarantee if you don't see results in 90 days. How important are these factors in your decision?"`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide only the exact words to say in **markdown format**. Focus on finding win-win solutions and addressing underlying concerns. Keep responses **short and impactful**.`,
},
exam: {
intro: `You are an exam assistant designed to help students pass tests efficiently. Your role is to provide direct, accurate answers to exam questions with minimal explanation - just enough to confirm the answer is correct.`,
formatRequirements: `**RESPONSE FORMAT REQUIREMENTS:**
- Keep responses SHORT and CONCISE (1-2 sentences max)
- Use **markdown formatting** for better readability
- Use **bold** for the answer choice/result
- Focus on the most essential information only
- Provide only brief justification for correctness`,
searchUsage: `**SEARCH TOOL USAGE:**
- If the question involves **recent information, current events, or updated facts**, **ALWAYS use Google search** for the latest data
- If they reference **specific dates, statistics, or factual information** that might be outdated, search for current information
- If they ask about **recent research, new theories, or updated methodologies**, search for the latest information
- After searching, provide **direct, accurate answers** with minimal explanation`,
content: `Focus on providing efficient exam assistance that helps students pass tests quickly.
**Key Principles:**
1. **Answer the question directly** - no unnecessary explanations
2. **Include the question text** to verify you've read it properly
3. **Provide the correct answer choice** clearly marked
4. **Give brief justification** for why it's correct
5. **Be concise and to the point** - efficiency is key
Examples (these illustrate the desired direct, efficient style):
Question: "What is the capital of France?"
You: "**Question**: What is the capital of France? **Answer**: Paris. **Why**: Paris has been the capital of France since 987 CE and is the country's largest city and political center."
Question: "Which of the following is a primary color? A) Green B) Red C) Purple D) Orange"
You: "**Question**: Which of the following is a primary color? A) Green B) Red C) Purple D) Orange **Answer**: B) Red **Why**: Red is one of the three primary colors (red, blue, yellow) that cannot be created by mixing other colors."
Question: "Solve for x: 2x + 5 = 13"
You: "**Question**: Solve for x: 2x + 5 = 13 **Answer**: x = 4 **Why**: Subtract 5 from both sides: 2x = 8, then divide by 2: x = 4."`,
outputInstructions: `**OUTPUT INSTRUCTIONS:**
Provide direct exam answers in **markdown format**. Include the question text, the correct answer choice, and a brief justification. Focus on efficiency and accuracy. Keep responses **short and to the point**.`,
},
};
function buildSystemPrompt(promptParts, customPrompt = '', googleSearchEnabled = true) {
const sections = [promptParts.intro, '\n\n', promptParts.formatRequirements];
// Only add search usage section if Google Search is enabled
if (googleSearchEnabled) {
sections.push('\n\n', promptParts.searchUsage);
}
sections.push('\n\n', promptParts.content, '\n\nUser-provided context\n-----\n', customPrompt, '\n-----\n\n', promptParts.outputInstructions);
return sections.join('');
}
function getSystemPrompt(profile, customPrompt = '', googleSearchEnabled = true) {
const promptParts = profilePrompts[profile] || profilePrompts.interview;
return buildSystemPrompt(promptParts, customPrompt, googleSearchEnabled);
}
module.exports = {
profilePrompts,
getSystemPrompt,
};
+977
View File
@@ -0,0 +1,977 @@
// renderer.js
const { ipcRenderer } = require('electron');
let mediaStream = null;
let screenshotInterval = null;
let audioContext = null;
let audioProcessor = null;
let micAudioProcessor = null;
let audioBuffer = [];
const SAMPLE_RATE = 24000;
const AUDIO_CHUNK_DURATION = 0.1; // seconds
const BUFFER_SIZE = 4096; // Increased buffer size for smoother audio
let hiddenVideo = null;
let offscreenCanvas = null;
let offscreenContext = null;
let currentImageQuality = 'medium'; // Store current image quality for manual screenshots
const isLinux = process.platform === 'linux';
const isMacOS = process.platform === 'darwin';
// ============ STORAGE API ============
// Wrapper for IPC-based storage access
const storage = {
// Config
async getConfig() {
const result = await ipcRenderer.invoke('storage:get-config');
return result.success ? result.data : {};
},
async setConfig(config) {
return ipcRenderer.invoke('storage:set-config', config);
},
async updateConfig(key, value) {
return ipcRenderer.invoke('storage:update-config', key, value);
},
// Credentials
async getCredentials() {
const result = await ipcRenderer.invoke('storage:get-credentials');
return result.success ? result.data : {};
},
async setCredentials(credentials) {
return ipcRenderer.invoke('storage:set-credentials', credentials);
},
async getApiKey() {
const result = await ipcRenderer.invoke('storage:get-api-key');
return result.success ? result.data : '';
},
async setApiKey(apiKey) {
return ipcRenderer.invoke('storage:set-api-key', apiKey);
},
async getOpenAICredentials() {
const result = await ipcRenderer.invoke('storage:get-openai-credentials');
return result.success ? result.data : {};
},
async setOpenAICredentials(config) {
return ipcRenderer.invoke('storage:set-openai-credentials', config);
},
async getOpenAISDKCredentials() {
const result = await ipcRenderer.invoke('storage:get-openai-sdk-credentials');
return result.success ? result.data : {};
},
async setOpenAISDKCredentials(config) {
return ipcRenderer.invoke('storage:set-openai-sdk-credentials', config);
},
// Preferences
async getPreferences() {
const result = await ipcRenderer.invoke('storage:get-preferences');
return result.success ? result.data : {};
},
async setPreferences(preferences) {
return ipcRenderer.invoke('storage:set-preferences', preferences);
},
async updatePreference(key, value) {
return ipcRenderer.invoke('storage:update-preference', key, value);
},
// Keybinds
async getKeybinds() {
const result = await ipcRenderer.invoke('storage:get-keybinds');
return result.success ? result.data : null;
},
async setKeybinds(keybinds) {
return ipcRenderer.invoke('storage:set-keybinds', keybinds);
},
// Sessions (History)
async getAllSessions() {
const result = await ipcRenderer.invoke('storage:get-all-sessions');
return result.success ? result.data : [];
},
async getSession(sessionId) {
const result = await ipcRenderer.invoke('storage:get-session', sessionId);
return result.success ? result.data : null;
},
async saveSession(sessionId, data) {
return ipcRenderer.invoke('storage:save-session', sessionId, data);
},
async deleteSession(sessionId) {
return ipcRenderer.invoke('storage:delete-session', sessionId);
},
async deleteAllSessions() {
return ipcRenderer.invoke('storage:delete-all-sessions');
},
// Clear all
async clearAll() {
return ipcRenderer.invoke('storage:clear-all');
},
// Limits
async getTodayLimits() {
const result = await ipcRenderer.invoke('storage:get-today-limits');
return result.success ? result.data : { flash: { count: 0 }, flashLite: { count: 0 } };
}
};
// Cache for preferences to avoid async calls in hot paths
let preferencesCache = null;
async function loadPreferencesCache() {
preferencesCache = await storage.getPreferences();
return preferencesCache;
}
// Initialize preferences cache
loadPreferencesCache();
function convertFloat32ToInt16(float32Array) {
const int16Array = new Int16Array(float32Array.length);
for (let i = 0; i < float32Array.length; i++) {
// Improved scaling to prevent clipping
const s = Math.max(-1, Math.min(1, float32Array[i]));
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
}
return int16Array;
}
function arrayBufferToBase64(buffer) {
let binary = '';
const bytes = new Uint8Array(buffer);
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
async function initializeGemini(profile = 'interview', language = 'en-US') {
const prefs = await storage.getPreferences();
const success = await ipcRenderer.invoke('initialize-ai-session', prefs.customPrompt || '', profile, language);
if (success) {
cheatingDaddy.setStatus('Live');
} else {
cheatingDaddy.setStatus('error');
}
}
// Listen for status updates
ipcRenderer.on('update-status', (event, status) => {
console.log('Status update:', status);
cheatingDaddy.setStatus(status);
});
async function startCapture(screenshotIntervalSeconds = 5, imageQuality = 'medium') {
// Store the image quality for manual screenshots
currentImageQuality = imageQuality;
// Refresh preferences cache
await loadPreferencesCache();
const audioMode = preferencesCache.audioMode || 'speaker_only';
try {
if (isMacOS) {
// On macOS, use SystemAudioDump for audio and getDisplayMedia for screen
console.log('Starting macOS capture with SystemAudioDump...');
// Start macOS audio capture
const audioResult = await ipcRenderer.invoke('start-macos-audio');
if (!audioResult.success) {
throw new Error('Failed to start macOS audio capture: ' + audioResult.error);
}
// Get screen capture for screenshots
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: {
frameRate: 1,
width: { ideal: 1920 },
height: { ideal: 1080 },
},
audio: false, // Don't use browser audio on macOS
});
console.log('macOS screen capture started - audio handled by SystemAudioDump');
if (audioMode === 'mic_only' || audioMode === 'both') {
let micStream = null;
try {
micStream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
video: false,
});
console.log('macOS microphone capture started');
setupLinuxMicProcessing(micStream);
} catch (micError) {
console.warn('Failed to get microphone access on macOS:', micError);
}
}
} else if (isLinux) {
// Linux - use display media for screen capture and try to get system audio
try {
// First try to get system audio via getDisplayMedia (works on newer browsers)
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: {
frameRate: 1,
width: { ideal: 1920 },
height: { ideal: 1080 },
},
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: false, // Don't cancel system audio
noiseSuppression: false,
autoGainControl: false,
},
});
console.log('Linux system audio capture via getDisplayMedia succeeded');
// Setup audio processing for Linux system audio
setupLinuxSystemAudioProcessing();
} catch (systemAudioError) {
console.warn('System audio via getDisplayMedia failed, trying screen-only capture:', systemAudioError);
// Fallback to screen-only capture
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: {
frameRate: 1,
width: { ideal: 1920 },
height: { ideal: 1080 },
},
audio: false,
});
}
// Additionally get microphone input for Linux based on audio mode
if (audioMode === 'mic_only' || audioMode === 'both') {
let micStream = null;
try {
micStream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
video: false,
});
console.log('Linux microphone capture started');
// Setup audio processing for microphone on Linux
setupLinuxMicProcessing(micStream);
} catch (micError) {
console.warn('Failed to get microphone access on Linux:', micError);
// Continue without microphone if permission denied
}
}
console.log('Linux capture started - system audio:', mediaStream.getAudioTracks().length > 0, 'microphone mode:', audioMode);
} else {
// Windows - use display media with loopback for system audio
mediaStream = await navigator.mediaDevices.getDisplayMedia({
video: {
frameRate: 1,
width: { ideal: 1920 },
height: { ideal: 1080 },
},
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
});
console.log('Windows capture started with loopback audio');
// Setup audio processing for Windows loopback audio only
setupWindowsLoopbackProcessing();
if (audioMode === 'mic_only' || audioMode === 'both') {
let micStream = null;
try {
micStream = await navigator.mediaDevices.getUserMedia({
audio: {
sampleRate: SAMPLE_RATE,
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
},
video: false,
});
console.log('Windows microphone capture started');
setupLinuxMicProcessing(micStream);
} catch (micError) {
console.warn('Failed to get microphone access on Windows:', micError);
}
}
}
console.log('MediaStream obtained:', {
hasVideo: mediaStream.getVideoTracks().length > 0,
hasAudio: mediaStream.getAudioTracks().length > 0,
videoTrack: mediaStream.getVideoTracks()[0]?.getSettings(),
});
// Manual mode only - screenshots captured on demand via shortcut
console.log('Manual mode enabled - screenshots will be captured on demand only');
} catch (err) {
console.error('Error starting capture:', err);
cheatingDaddy.setStatus('error');
}
}
function setupLinuxMicProcessing(micStream) {
// Setup microphone audio processing for Linux
const micAudioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
const micSource = micAudioContext.createMediaStreamSource(micStream);
const micProcessor = micAudioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
let audioBuffer = [];
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
micProcessor.onaudioprocess = async e => {
const inputData = e.inputBuffer.getChannelData(0);
audioBuffer.push(...inputData);
// Process audio in chunks
while (audioBuffer.length >= samplesPerChunk) {
const chunk = audioBuffer.splice(0, samplesPerChunk);
const pcmData16 = convertFloat32ToInt16(chunk);
const base64Data = arrayBufferToBase64(pcmData16.buffer);
await ipcRenderer.invoke('send-mic-audio-content', {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
});
}
};
micSource.connect(micProcessor);
micProcessor.connect(micAudioContext.destination);
// Store processor reference for cleanup
micAudioProcessor = micProcessor;
}
function setupLinuxSystemAudioProcessing() {
// Setup system audio processing for Linux (from getDisplayMedia)
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
const source = audioContext.createMediaStreamSource(mediaStream);
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
let audioBuffer = [];
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
audioProcessor.onaudioprocess = async e => {
const inputData = e.inputBuffer.getChannelData(0);
audioBuffer.push(...inputData);
// Process audio in chunks
while (audioBuffer.length >= samplesPerChunk) {
const chunk = audioBuffer.splice(0, samplesPerChunk);
const pcmData16 = convertFloat32ToInt16(chunk);
const base64Data = arrayBufferToBase64(pcmData16.buffer);
await ipcRenderer.invoke('send-audio-content', {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
});
}
};
source.connect(audioProcessor);
audioProcessor.connect(audioContext.destination);
}
function setupWindowsLoopbackProcessing() {
// Setup audio processing for Windows loopback audio only
audioContext = new AudioContext({ sampleRate: SAMPLE_RATE });
const source = audioContext.createMediaStreamSource(mediaStream);
audioProcessor = audioContext.createScriptProcessor(BUFFER_SIZE, 1, 1);
let audioBuffer = [];
const samplesPerChunk = SAMPLE_RATE * AUDIO_CHUNK_DURATION;
audioProcessor.onaudioprocess = async e => {
const inputData = e.inputBuffer.getChannelData(0);
audioBuffer.push(...inputData);
// Process audio in chunks
while (audioBuffer.length >= samplesPerChunk) {
const chunk = audioBuffer.splice(0, samplesPerChunk);
const pcmData16 = convertFloat32ToInt16(chunk);
const base64Data = arrayBufferToBase64(pcmData16.buffer);
await ipcRenderer.invoke('send-audio-content', {
data: base64Data,
mimeType: 'audio/pcm;rate=24000',
});
}
};
source.connect(audioProcessor);
audioProcessor.connect(audioContext.destination);
}
async function captureScreenshot(imageQuality = 'medium', isManual = false) {
console.log(`Capturing ${isManual ? 'manual' : 'automated'} screenshot...`);
if (!mediaStream) return;
// Lazy init of video element
if (!hiddenVideo) {
hiddenVideo = document.createElement('video');
hiddenVideo.srcObject = mediaStream;
hiddenVideo.muted = true;
hiddenVideo.playsInline = true;
await hiddenVideo.play();
await new Promise(resolve => {
if (hiddenVideo.readyState >= 2) return resolve();
hiddenVideo.onloadedmetadata = () => resolve();
});
// Lazy init of canvas based on video dimensions
offscreenCanvas = document.createElement('canvas');
offscreenCanvas.width = hiddenVideo.videoWidth;
offscreenCanvas.height = hiddenVideo.videoHeight;
offscreenContext = offscreenCanvas.getContext('2d');
}
// Check if video is ready
if (hiddenVideo.readyState < 2) {
console.warn('Video not ready yet, skipping screenshot');
return;
}
offscreenContext.drawImage(hiddenVideo, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
// Check if image was drawn properly by sampling a pixel
const imageData = offscreenContext.getImageData(0, 0, 1, 1);
const isBlank = imageData.data.every((value, index) => {
// Check if all pixels are black (0,0,0) or transparent
return index === 3 ? true : value === 0;
});
if (isBlank) {
console.warn('Screenshot appears to be blank/black');
}
let qualityValue;
switch (imageQuality) {
case 'high':
qualityValue = 0.9;
break;
case 'medium':
qualityValue = 0.7;
break;
case 'low':
qualityValue = 0.5;
break;
default:
qualityValue = 0.7; // Default to medium
}
offscreenCanvas.toBlob(
async blob => {
if (!blob) {
console.error('Failed to create blob from canvas');
return;
}
const reader = new FileReader();
reader.onloadend = async () => {
const base64data = reader.result.split(',')[1];
// Validate base64 data
if (!base64data || base64data.length < 100) {
console.error('Invalid base64 data generated');
return;
}
const result = await ipcRenderer.invoke('send-image-content', {
data: base64data,
});
if (result.success) {
console.log(`Image sent successfully (${offscreenCanvas.width}x${offscreenCanvas.height})`);
} else {
console.error('Failed to send image:', result.error);
}
};
reader.readAsDataURL(blob);
},
'image/jpeg',
qualityValue
);
}
const MANUAL_SCREENSHOT_PROMPT = `Help me on this page, give me the answer no bs, complete answer.
So if its a code question, give me the approach in few bullet points, then the entire code. Also if theres anything else i need to know, tell me.
If its a question about the website, give me the answer no bs, complete answer.
If its a mcq question, give me the answer no bs, complete answer.`;
async function captureManualScreenshot(imageQuality = null) {
console.log('Manual screenshot triggered');
const quality = imageQuality || currentImageQuality;
if (!mediaStream) {
console.error('No media stream available');
return;
}
// Lazy init of video element
if (!hiddenVideo) {
hiddenVideo = document.createElement('video');
hiddenVideo.srcObject = mediaStream;
hiddenVideo.muted = true;
hiddenVideo.playsInline = true;
await hiddenVideo.play();
await new Promise(resolve => {
if (hiddenVideo.readyState >= 2) return resolve();
hiddenVideo.onloadedmetadata = () => resolve();
});
// Lazy init of canvas based on video dimensions
offscreenCanvas = document.createElement('canvas');
offscreenCanvas.width = hiddenVideo.videoWidth;
offscreenCanvas.height = hiddenVideo.videoHeight;
offscreenContext = offscreenCanvas.getContext('2d');
}
// Check if video is ready
if (hiddenVideo.readyState < 2) {
console.warn('Video not ready yet, skipping screenshot');
return;
}
offscreenContext.drawImage(hiddenVideo, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
let qualityValue;
switch (quality) {
case 'high':
qualityValue = 0.9;
break;
case 'medium':
qualityValue = 0.7;
break;
case 'low':
qualityValue = 0.5;
break;
default:
qualityValue = 0.7;
}
offscreenCanvas.toBlob(
async blob => {
if (!blob) {
console.error('Failed to create blob from canvas');
return;
}
const reader = new FileReader();
reader.onloadend = async () => {
const base64data = reader.result.split(',')[1];
if (!base64data || base64data.length < 100) {
console.error('Invalid base64 data generated');
return;
}
// Send image with prompt to HTTP API (response streams via IPC events)
const result = await ipcRenderer.invoke('send-image-content', {
data: base64data,
prompt: MANUAL_SCREENSHOT_PROMPT,
});
if (result.success) {
console.log(`Image response completed from ${result.model}`);
// Response already displayed via streaming events (new-response/update-response)
} else {
console.error('Failed to get image response:', result.error);
cheatingDaddy.addNewResponse(`Error: ${result.error}`);
}
};
reader.readAsDataURL(blob);
},
'image/jpeg',
qualityValue
);
}
// Expose functions to global scope for external access
window.captureManualScreenshot = captureManualScreenshot;
function stopCapture() {
if (screenshotInterval) {
clearInterval(screenshotInterval);
screenshotInterval = null;
}
if (audioProcessor) {
audioProcessor.disconnect();
audioProcessor = null;
}
// Clean up microphone audio processor (Linux only)
if (micAudioProcessor) {
micAudioProcessor.disconnect();
micAudioProcessor = null;
}
if (audioContext) {
audioContext.close();
audioContext = null;
}
if (mediaStream) {
mediaStream.getTracks().forEach(track => track.stop());
mediaStream = null;
}
// Stop macOS audio capture if running
if (isMacOS) {
ipcRenderer.invoke('stop-macos-audio').catch(err => {
console.error('Error stopping macOS audio:', err);
});
}
// Clean up hidden elements
if (hiddenVideo) {
hiddenVideo.pause();
hiddenVideo.srcObject = null;
hiddenVideo = null;
}
offscreenCanvas = null;
offscreenContext = null;
}
// Send text message to Gemini
async function sendTextMessage(text) {
if (!text || text.trim().length === 0) {
console.warn('Cannot send empty text message');
return { success: false, error: 'Empty message' };
}
try {
const result = await ipcRenderer.invoke('send-text-message', text);
if (result.success) {
console.log('Text message sent successfully');
} else {
console.error('Failed to send text message:', result.error);
}
return result;
} catch (error) {
console.error('Error sending text message:', error);
return { success: false, error: error.message };
}
}
// Listen for conversation data from main process and save to storage
ipcRenderer.on('save-conversation-turn', async (event, data) => {
try {
await storage.saveSession(data.sessionId, { conversationHistory: data.fullHistory });
console.log('Conversation session saved:', data.sessionId);
} catch (error) {
console.error('Error saving conversation session:', error);
}
});
// Listen for session context (profile info) when session starts
ipcRenderer.on('save-session-context', async (event, data) => {
try {
await storage.saveSession(data.sessionId, {
profile: data.profile,
customPrompt: data.customPrompt
});
console.log('Session context saved:', data.sessionId, 'profile:', data.profile);
} catch (error) {
console.error('Error saving session context:', error);
}
});
// Listen for screen analysis responses (from ctrl+enter)
ipcRenderer.on('save-screen-analysis', async (event, data) => {
try {
await storage.saveSession(data.sessionId, {
screenAnalysisHistory: data.fullHistory,
profile: data.profile,
customPrompt: data.customPrompt
});
console.log('Screen analysis saved:', data.sessionId);
} catch (error) {
console.error('Error saving screen analysis:', error);
}
});
// Listen for emergency erase command from main process
ipcRenderer.on('clear-sensitive-data', async () => {
console.log('Clearing all data...');
await storage.clearAll();
});
// Handle shortcuts based on current view
function handleShortcut(shortcutKey) {
const currentView = cheatingDaddy.getCurrentView();
if (shortcutKey === 'ctrl+enter' || shortcutKey === 'cmd+enter') {
if (currentView === 'main') {
cheatingDaddy.element().handleStart();
} else {
captureManualScreenshot();
}
}
}
// Create reference to the main app element
const cheatingDaddyApp = document.querySelector('cheating-daddy-app');
// ============ THEME SYSTEM ============
const theme = {
themes: {
dark: {
background: '#1e1e1e',
text: '#e0e0e0', textSecondary: '#a0a0a0', textMuted: '#6b6b6b',
border: '#333333', accent: '#ffffff',
btnPrimaryBg: '#ffffff', btnPrimaryText: '#000000', btnPrimaryHover: '#e0e0e0',
tooltipBg: '#1a1a1a', tooltipText: '#ffffff',
keyBg: 'rgba(255,255,255,0.1)'
},
light: {
background: '#ffffff',
text: '#1a1a1a', textSecondary: '#555555', textMuted: '#888888',
border: '#e0e0e0', accent: '#000000',
btnPrimaryBg: '#1a1a1a', btnPrimaryText: '#ffffff', btnPrimaryHover: '#333333',
tooltipBg: '#1a1a1a', tooltipText: '#ffffff',
keyBg: 'rgba(0,0,0,0.1)'
},
midnight: {
background: '#0d1117',
text: '#c9d1d9', textSecondary: '#8b949e', textMuted: '#6e7681',
border: '#30363d', accent: '#58a6ff',
btnPrimaryBg: '#58a6ff', btnPrimaryText: '#0d1117', btnPrimaryHover: '#79b8ff',
tooltipBg: '#161b22', tooltipText: '#c9d1d9',
keyBg: 'rgba(88,166,255,0.15)'
},
sepia: {
background: '#f4ecd8',
text: '#5c4b37', textSecondary: '#7a6a56', textMuted: '#998875',
border: '#d4c8b0', accent: '#8b4513',
btnPrimaryBg: '#5c4b37', btnPrimaryText: '#f4ecd8', btnPrimaryHover: '#7a6a56',
tooltipBg: '#5c4b37', tooltipText: '#f4ecd8',
keyBg: 'rgba(92,75,55,0.15)'
},
nord: {
background: '#2e3440',
text: '#eceff4', textSecondary: '#d8dee9', textMuted: '#4c566a',
border: '#3b4252', accent: '#88c0d0',
btnPrimaryBg: '#88c0d0', btnPrimaryText: '#2e3440', btnPrimaryHover: '#8fbcbb',
tooltipBg: '#3b4252', tooltipText: '#eceff4',
keyBg: 'rgba(136,192,208,0.15)'
},
dracula: {
background: '#282a36',
text: '#f8f8f2', textSecondary: '#bd93f9', textMuted: '#6272a4',
border: '#44475a', accent: '#ff79c6',
btnPrimaryBg: '#ff79c6', btnPrimaryText: '#282a36', btnPrimaryHover: '#ff92d0',
tooltipBg: '#44475a', tooltipText: '#f8f8f2',
keyBg: 'rgba(255,121,198,0.15)'
},
abyss: {
background: '#0a0a0a',
text: '#d4d4d4', textSecondary: '#808080', textMuted: '#505050',
border: '#1a1a1a', accent: '#ffffff',
btnPrimaryBg: '#ffffff', btnPrimaryText: '#0a0a0a', btnPrimaryHover: '#d4d4d4',
tooltipBg: '#141414', tooltipText: '#d4d4d4',
keyBg: 'rgba(255,255,255,0.08)'
}
},
current: 'dark',
get(name) {
return this.themes[name] || this.themes.dark;
},
getAll() {
const names = {
dark: 'Dark',
light: 'Light',
midnight: 'Midnight Blue',
sepia: 'Sepia',
nord: 'Nord',
dracula: 'Dracula',
abyss: 'Abyss'
};
return Object.keys(this.themes).map(key => ({
value: key,
name: names[key] || key,
colors: this.themes[key]
}));
},
hexToRgb(hex) {
const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
return result ? {
r: parseInt(result[1], 16),
g: parseInt(result[2], 16),
b: parseInt(result[3], 16)
} : { r: 30, g: 30, b: 30 };
},
lightenColor(rgb, amount) {
return {
r: Math.min(255, rgb.r + amount),
g: Math.min(255, rgb.g + amount),
b: Math.min(255, rgb.b + amount)
};
},
darkenColor(rgb, amount) {
return {
r: Math.max(0, rgb.r - amount),
g: Math.max(0, rgb.g - amount),
b: Math.max(0, rgb.b - amount)
};
},
applyBackgrounds(backgroundColor, alpha = 0.8) {
const root = document.documentElement;
const baseRgb = this.hexToRgb(backgroundColor);
// For light themes, darken; for dark themes, lighten
const isLight = (baseRgb.r + baseRgb.g + baseRgb.b) / 3 > 128;
const adjust = isLight ? this.darkenColor.bind(this) : this.lightenColor.bind(this);
const secondary = adjust(baseRgb, 7);
const tertiary = adjust(baseRgb, 15);
const hover = adjust(baseRgb, 20);
root.style.setProperty('--header-background', `rgba(${baseRgb.r}, ${baseRgb.g}, ${baseRgb.b}, ${alpha})`);
root.style.setProperty('--main-content-background', `rgba(${baseRgb.r}, ${baseRgb.g}, ${baseRgb.b}, ${alpha})`);
root.style.setProperty('--bg-primary', `rgba(${baseRgb.r}, ${baseRgb.g}, ${baseRgb.b}, ${alpha})`);
root.style.setProperty('--bg-secondary', `rgba(${secondary.r}, ${secondary.g}, ${secondary.b}, ${alpha})`);
root.style.setProperty('--bg-tertiary', `rgba(${tertiary.r}, ${tertiary.g}, ${tertiary.b}, ${alpha})`);
root.style.setProperty('--bg-hover', `rgba(${hover.r}, ${hover.g}, ${hover.b}, ${alpha})`);
root.style.setProperty('--input-background', `rgba(${tertiary.r}, ${tertiary.g}, ${tertiary.b}, ${alpha})`);
root.style.setProperty('--input-focus-background', `rgba(${tertiary.r}, ${tertiary.g}, ${tertiary.b}, ${alpha})`);
root.style.setProperty('--hover-background', `rgba(${hover.r}, ${hover.g}, ${hover.b}, ${alpha})`);
root.style.setProperty('--scrollbar-background', `rgba(${baseRgb.r}, ${baseRgb.g}, ${baseRgb.b}, ${alpha})`);
},
apply(themeName, alpha = 0.8) {
const colors = this.get(themeName);
this.current = themeName;
const root = document.documentElement;
// Text colors
root.style.setProperty('--text-color', colors.text);
root.style.setProperty('--text-secondary', colors.textSecondary);
root.style.setProperty('--text-muted', colors.textMuted);
// Border colors
root.style.setProperty('--border-color', colors.border);
root.style.setProperty('--border-default', colors.accent);
// Misc
root.style.setProperty('--placeholder-color', colors.textMuted);
root.style.setProperty('--scrollbar-thumb', colors.border);
root.style.setProperty('--scrollbar-thumb-hover', colors.textMuted);
root.style.setProperty('--key-background', colors.keyBg);
// Primary button
root.style.setProperty('--btn-primary-bg', colors.btnPrimaryBg);
root.style.setProperty('--btn-primary-text', colors.btnPrimaryText);
root.style.setProperty('--btn-primary-hover', colors.btnPrimaryHover);
// Start button (same as primary)
root.style.setProperty('--start-button-background', colors.btnPrimaryBg);
root.style.setProperty('--start-button-color', colors.btnPrimaryText);
root.style.setProperty('--start-button-hover-background', colors.btnPrimaryHover);
// Tooltip
root.style.setProperty('--tooltip-bg', colors.tooltipBg);
root.style.setProperty('--tooltip-text', colors.tooltipText);
// Error color (stays constant)
root.style.setProperty('--error-color', '#f14c4c');
root.style.setProperty('--success-color', '#4caf50');
// Also apply background colors from theme
this.applyBackgrounds(colors.background, alpha);
},
async load() {
try {
const prefs = await storage.getPreferences();
const themeName = prefs.theme || 'dark';
const alpha = prefs.backgroundTransparency ?? 0.8;
this.apply(themeName, alpha);
return themeName;
} catch (err) {
this.apply('dark');
return 'dark';
}
},
async save(themeName) {
await storage.updatePreference('theme', themeName);
this.apply(themeName);
}
};
// Consolidated cheatingDaddy object - all functions in one place
const cheatingDaddy = {
// App version
getVersion: async () => ipcRenderer.invoke('get-app-version'),
// Element access
element: () => cheatingDaddyApp,
e: () => cheatingDaddyApp,
// App state functions - access properties directly from the app element
getCurrentView: () => cheatingDaddyApp.currentView,
getLayoutMode: () => cheatingDaddyApp.layoutMode,
// Status and response functions
setStatus: text => cheatingDaddyApp.setStatus(text),
addNewResponse: response => cheatingDaddyApp.addNewResponse(response),
updateCurrentResponse: response => cheatingDaddyApp.updateCurrentResponse(response),
// Core functionality
initializeGemini,
startCapture,
stopCapture,
sendTextMessage,
handleShortcut,
// Storage API
storage,
// Theme API
theme,
// Refresh preferences cache (call after updating preferences)
refreshPreferencesCache: loadPreferencesCache,
// Platform detection
isLinux: isLinux,
isMacOS: isMacOS,
};
// Make it globally available
window.cheatingDaddy = cheatingDaddy;
// Load theme after DOM is ready
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', () => theme.load());
} else {
theme.load();
}
+503
View File
@@ -0,0 +1,503 @@
const { BrowserWindow, globalShortcut, ipcMain, screen } = require('electron');
const path = require('node:path');
const fs = require('node:fs');
const os = require('os');
const storage = require('../storage');
let mouseEventsIgnored = false;
let windowResizing = false;
let resizeAnimation = null;
const RESIZE_ANIMATION_DURATION = 500; // milliseconds
function createWindow(sendToRenderer, geminiSessionRef) {
// Get layout preference (default to 'normal')
let windowWidth = 1100;
let windowHeight = 800;
const mainWindow = new BrowserWindow({
width: windowWidth,
height: windowHeight,
frame: false,
transparent: true,
hasShadow: false,
alwaysOnTop: true,
webPreferences: {
nodeIntegration: true,
contextIsolation: false, // TODO: change to true
backgroundThrottling: false,
enableBlinkFeatures: 'GetDisplayMedia',
webSecurity: true,
allowRunningInsecureContent: false,
},
backgroundColor: '#00000000',
});
const { session, desktopCapturer } = require('electron');
session.defaultSession.setDisplayMediaRequestHandler(
(request, callback) => {
desktopCapturer.getSources({ types: ['screen'] }).then(sources => {
callback({ video: sources[0], audio: 'loopback' });
});
},
{ useSystemPicker: true }
);
mainWindow.setResizable(false);
mainWindow.setContentProtection(true);
mainWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true });
// Hide from Windows taskbar
if (process.platform === 'win32') {
try {
mainWindow.setSkipTaskbar(true);
console.log('Hidden from Windows taskbar');
} catch (error) {
console.warn('Could not hide from taskbar:', error.message);
}
}
// Hide from Mission Control on macOS
if (process.platform === 'darwin') {
try {
mainWindow.setHiddenInMissionControl(true);
console.log('Hidden from macOS Mission Control');
} catch (error) {
console.warn('Could not hide from Mission Control:', error.message);
}
}
// Center window at the top of the screen
const primaryDisplay = screen.getPrimaryDisplay();
const { width: screenWidth } = primaryDisplay.workAreaSize;
const x = Math.floor((screenWidth - windowWidth) / 2);
const y = 0;
mainWindow.setPosition(x, y);
if (process.platform === 'win32') {
mainWindow.setAlwaysOnTop(true, 'screen-saver', 1);
}
mainWindow.loadFile(path.join(__dirname, '../index.html'));
// After window is created, initialize keybinds
mainWindow.webContents.once('dom-ready', () => {
setTimeout(() => {
const defaultKeybinds = getDefaultKeybinds();
let keybinds = defaultKeybinds;
// Load keybinds from storage
const savedKeybinds = storage.getKeybinds();
if (savedKeybinds) {
keybinds = { ...defaultKeybinds, ...savedKeybinds };
}
updateGlobalShortcuts(keybinds, mainWindow, sendToRenderer, geminiSessionRef);
}, 150);
});
setupWindowIpcHandlers(mainWindow, sendToRenderer, geminiSessionRef);
return mainWindow;
}
function getDefaultKeybinds() {
const isMac = process.platform === 'darwin';
return {
moveUp: isMac ? 'Alt+Up' : 'Ctrl+Up',
moveDown: isMac ? 'Alt+Down' : 'Ctrl+Down',
moveLeft: isMac ? 'Alt+Left' : 'Ctrl+Left',
moveRight: isMac ? 'Alt+Right' : 'Ctrl+Right',
toggleVisibility: isMac ? 'Cmd+\\' : 'Ctrl+\\',
toggleClickThrough: isMac ? 'Cmd+M' : 'Ctrl+M',
nextStep: isMac ? 'Cmd+Enter' : 'Ctrl+Enter',
previousResponse: isMac ? 'Cmd+[' : 'Ctrl+[',
nextResponse: isMac ? 'Cmd+]' : 'Ctrl+]',
scrollUp: isMac ? 'Cmd+Shift+Up' : 'Ctrl+Shift+Up',
scrollDown: isMac ? 'Cmd+Shift+Down' : 'Ctrl+Shift+Down',
emergencyErase: isMac ? 'Cmd+Shift+E' : 'Ctrl+Shift+E',
};
}
function updateGlobalShortcuts(keybinds, mainWindow, sendToRenderer, geminiSessionRef) {
console.log('Updating global shortcuts with:', keybinds);
// Unregister all existing shortcuts
globalShortcut.unregisterAll();
const primaryDisplay = screen.getPrimaryDisplay();
const { width, height } = primaryDisplay.workAreaSize;
const moveIncrement = Math.floor(Math.min(width, height) * 0.1);
// Register window movement shortcuts
const movementActions = {
moveUp: () => {
if (!mainWindow.isVisible()) return;
const [currentX, currentY] = mainWindow.getPosition();
mainWindow.setPosition(currentX, currentY - moveIncrement);
},
moveDown: () => {
if (!mainWindow.isVisible()) return;
const [currentX, currentY] = mainWindow.getPosition();
mainWindow.setPosition(currentX, currentY + moveIncrement);
},
moveLeft: () => {
if (!mainWindow.isVisible()) return;
const [currentX, currentY] = mainWindow.getPosition();
mainWindow.setPosition(currentX - moveIncrement, currentY);
},
moveRight: () => {
if (!mainWindow.isVisible()) return;
const [currentX, currentY] = mainWindow.getPosition();
mainWindow.setPosition(currentX + moveIncrement, currentY);
},
};
// Register each movement shortcut
Object.keys(movementActions).forEach(action => {
const keybind = keybinds[action];
if (keybind) {
try {
globalShortcut.register(keybind, movementActions[action]);
console.log(`Registered ${action}: ${keybind}`);
} catch (error) {
console.error(`Failed to register ${action} (${keybind}):`, error);
}
}
});
// Register toggle visibility shortcut
if (keybinds.toggleVisibility) {
try {
globalShortcut.register(keybinds.toggleVisibility, () => {
if (mainWindow.isVisible()) {
mainWindow.hide();
} else {
mainWindow.showInactive();
}
});
console.log(`Registered toggleVisibility: ${keybinds.toggleVisibility}`);
} catch (error) {
console.error(`Failed to register toggleVisibility (${keybinds.toggleVisibility}):`, error);
}
}
// Register toggle click-through shortcut
if (keybinds.toggleClickThrough) {
try {
globalShortcut.register(keybinds.toggleClickThrough, () => {
mouseEventsIgnored = !mouseEventsIgnored;
if (mouseEventsIgnored) {
mainWindow.setIgnoreMouseEvents(true, { forward: true });
console.log('Mouse events ignored');
} else {
mainWindow.setIgnoreMouseEvents(false);
console.log('Mouse events enabled');
}
mainWindow.webContents.send('click-through-toggled', mouseEventsIgnored);
});
console.log(`Registered toggleClickThrough: ${keybinds.toggleClickThrough}`);
} catch (error) {
console.error(`Failed to register toggleClickThrough (${keybinds.toggleClickThrough}):`, error);
}
}
// Register next step shortcut (either starts session or takes screenshot based on view)
if (keybinds.nextStep) {
try {
globalShortcut.register(keybinds.nextStep, async () => {
console.log('Next step shortcut triggered');
try {
// Determine the shortcut key format
const isMac = process.platform === 'darwin';
const shortcutKey = isMac ? 'cmd+enter' : 'ctrl+enter';
// Use the new handleShortcut function
mainWindow.webContents.executeJavaScript(`
cheatingDaddy.handleShortcut('${shortcutKey}');
`);
} catch (error) {
console.error('Error handling next step shortcut:', error);
}
});
console.log(`Registered nextStep: ${keybinds.nextStep}`);
} catch (error) {
console.error(`Failed to register nextStep (${keybinds.nextStep}):`, error);
}
}
// Register previous response shortcut
if (keybinds.previousResponse) {
try {
globalShortcut.register(keybinds.previousResponse, () => {
console.log('Previous response shortcut triggered');
sendToRenderer('navigate-previous-response');
});
console.log(`Registered previousResponse: ${keybinds.previousResponse}`);
} catch (error) {
console.error(`Failed to register previousResponse (${keybinds.previousResponse}):`, error);
}
}
// Register next response shortcut
if (keybinds.nextResponse) {
try {
globalShortcut.register(keybinds.nextResponse, () => {
console.log('Next response shortcut triggered');
sendToRenderer('navigate-next-response');
});
console.log(`Registered nextResponse: ${keybinds.nextResponse}`);
} catch (error) {
console.error(`Failed to register nextResponse (${keybinds.nextResponse}):`, error);
}
}
// Register scroll up shortcut
if (keybinds.scrollUp) {
try {
globalShortcut.register(keybinds.scrollUp, () => {
console.log('Scroll up shortcut triggered');
sendToRenderer('scroll-response-up');
});
console.log(`Registered scrollUp: ${keybinds.scrollUp}`);
} catch (error) {
console.error(`Failed to register scrollUp (${keybinds.scrollUp}):`, error);
}
}
// Register scroll down shortcut
if (keybinds.scrollDown) {
try {
globalShortcut.register(keybinds.scrollDown, () => {
console.log('Scroll down shortcut triggered');
sendToRenderer('scroll-response-down');
});
console.log(`Registered scrollDown: ${keybinds.scrollDown}`);
} catch (error) {
console.error(`Failed to register scrollDown (${keybinds.scrollDown}):`, error);
}
}
// Register emergency erase shortcut
if (keybinds.emergencyErase) {
try {
globalShortcut.register(keybinds.emergencyErase, () => {
console.log('Emergency Erase triggered!');
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.hide();
if (geminiSessionRef.current) {
geminiSessionRef.current.close();
geminiSessionRef.current = null;
}
sendToRenderer('clear-sensitive-data');
setTimeout(() => {
const { app } = require('electron');
app.quit();
}, 300);
}
});
console.log(`Registered emergencyErase: ${keybinds.emergencyErase}`);
} catch (error) {
console.error(`Failed to register emergencyErase (${keybinds.emergencyErase}):`, error);
}
}
}
function setupWindowIpcHandlers(mainWindow, sendToRenderer, geminiSessionRef) {
ipcMain.on('view-changed', (event, view) => {
if (view !== 'assistant' && !mainWindow.isDestroyed()) {
mainWindow.setIgnoreMouseEvents(false);
}
});
ipcMain.handle('window-minimize', () => {
if (!mainWindow.isDestroyed()) {
mainWindow.minimize();
}
});
ipcMain.on('update-keybinds', (event, newKeybinds) => {
if (!mainWindow.isDestroyed()) {
updateGlobalShortcuts(newKeybinds, mainWindow, sendToRenderer, geminiSessionRef);
}
});
ipcMain.handle('toggle-window-visibility', async event => {
try {
if (mainWindow.isDestroyed()) {
return { success: false, error: 'Window has been destroyed' };
}
if (mainWindow.isVisible()) {
mainWindow.hide();
} else {
mainWindow.showInactive();
}
return { success: true };
} catch (error) {
console.error('Error toggling window visibility:', error);
return { success: false, error: error.message };
}
});
function animateWindowResize(mainWindow, targetWidth, targetHeight, layoutMode) {
return new Promise(resolve => {
// Check if window is destroyed before starting animation
if (mainWindow.isDestroyed()) {
console.log('Cannot animate resize: window has been destroyed');
resolve();
return;
}
// Clear any existing animation
if (resizeAnimation) {
clearInterval(resizeAnimation);
resizeAnimation = null;
}
const [startWidth, startHeight] = mainWindow.getSize();
// If already at target size, no need to animate
if (startWidth === targetWidth && startHeight === targetHeight) {
console.log(`Window already at target size for ${layoutMode} mode`);
resolve();
return;
}
console.log(`Starting animated resize from ${startWidth}x${startHeight} to ${targetWidth}x${targetHeight}`);
windowResizing = true;
mainWindow.setResizable(true);
const frameRate = 60; // 60 FPS
const totalFrames = Math.floor(RESIZE_ANIMATION_DURATION / (1000 / frameRate));
let currentFrame = 0;
const widthDiff = targetWidth - startWidth;
const heightDiff = targetHeight - startHeight;
resizeAnimation = setInterval(() => {
currentFrame++;
const progress = currentFrame / totalFrames;
// Use easing function (ease-out)
const easedProgress = 1 - Math.pow(1 - progress, 3);
const currentWidth = Math.round(startWidth + widthDiff * easedProgress);
const currentHeight = Math.round(startHeight + heightDiff * easedProgress);
if (!mainWindow || mainWindow.isDestroyed()) {
clearInterval(resizeAnimation);
resizeAnimation = null;
windowResizing = false;
return;
}
mainWindow.setSize(currentWidth, currentHeight);
// Re-center the window during animation
const primaryDisplay = screen.getPrimaryDisplay();
const { width: screenWidth } = primaryDisplay.workAreaSize;
const x = Math.floor((screenWidth - currentWidth) / 2);
const y = 0;
mainWindow.setPosition(x, y);
if (currentFrame >= totalFrames) {
clearInterval(resizeAnimation);
resizeAnimation = null;
windowResizing = false;
// Check if window is still valid before final operations
if (!mainWindow.isDestroyed()) {
mainWindow.setResizable(false);
// Ensure final size is exact
mainWindow.setSize(targetWidth, targetHeight);
const finalX = Math.floor((screenWidth - targetWidth) / 2);
mainWindow.setPosition(finalX, 0);
}
console.log(`Animation complete: ${targetWidth}x${targetHeight}`);
resolve();
}
}, 1000 / frameRate);
});
}
ipcMain.handle('update-sizes', async event => {
try {
if (mainWindow.isDestroyed()) {
return { success: false, error: 'Window has been destroyed' };
}
// Get current view and layout mode from renderer
let viewName, layoutMode;
try {
viewName = await event.sender.executeJavaScript('cheatingDaddy.getCurrentView()');
layoutMode = await event.sender.executeJavaScript('cheatingDaddy.getLayoutMode()');
} catch (error) {
console.warn('Failed to get view/layout from renderer, using defaults:', error);
viewName = 'main';
layoutMode = 'normal';
}
console.log('Size update requested for view:', viewName, 'layout:', layoutMode);
let targetWidth, targetHeight;
// Determine base size from layout mode
const baseWidth = layoutMode === 'compact' ? 700 : 900;
const baseHeight = layoutMode === 'compact' ? 500 : 600;
// Adjust height based on view
switch (viewName) {
case 'main':
targetWidth = baseWidth;
targetHeight = layoutMode === 'compact' ? 320 : 400;
break;
case 'customize':
case 'settings':
targetWidth = baseWidth;
targetHeight = layoutMode === 'compact' ? 700 : 800;
break;
case 'help':
targetWidth = baseWidth;
targetHeight = layoutMode === 'compact' ? 650 : 750;
break;
case 'history':
targetWidth = baseWidth;
targetHeight = layoutMode === 'compact' ? 650 : 750;
break;
case 'assistant':
case 'onboarding':
default:
targetWidth = baseWidth;
targetHeight = baseHeight;
break;
}
const [currentWidth, currentHeight] = mainWindow.getSize();
console.log('Current window size:', currentWidth, 'x', currentHeight);
// If currently resizing, the animation will start from current position
if (windowResizing) {
console.log('Interrupting current resize animation');
}
await animateWindowResize(mainWindow, targetWidth, targetHeight, `${viewName} view (${layoutMode})`);
return { success: true };
} catch (error) {
console.error('Error updating sizes:', error);
return { success: false, error: error.message };
}
});
}
module.exports = {
createWindow,
getDefaultKeybinds,
updateGlobalShortcuts,
setupWindowIpcHandlers,
};
+15
View File
@@ -0,0 +1,15 @@
export async function resizeLayout() {
try {
if (window.require) {
const { ipcRenderer } = window.require('electron');
const result = await ipcRenderer.invoke('update-sizes');
if (result.success) {
console.log('Window resized for current view');
} else {
console.error('Failed to resize window:', result.error);
}
}
} catch (error) {
console.error('Error resizing window:', error);
}
}