From 8beccdf1019f02ee5fc4ddf7331711ba8af4e030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=20=D0=93=D0=BB=D0=B0=D0=B7=D1=83?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2?= Date: Sat, 5 Sep 2026 02:20:11 +0300 Subject: [PATCH] native first --- .vscode/launch.json | 24 + docs/local-sidecar-protocol.md | 60 ++ docs/migration/native-poc-results.md | 44 + ...acos-native-ai-companion-reconstruction.md | 825 ++++++++++++++++++ native/MastermindPOC/.gitignore | 2 + native/MastermindPOC/Package.swift | 34 + .../Sources/MastermindPOC/AppDelegate.swift | 183 ++++ .../MastermindPOC/CaptureCoordinator.swift | 100 +++ .../MastermindPOC/MenuBarController.swift | 121 +++ .../MicrophoneCaptureEngine.swift | 54 ++ .../MicrophonePCMConverter.swift | 57 ++ .../MastermindPOC/OverlaySettingsStore.swift | 33 + .../Sources/MastermindPOC/OverlayView.swift | 164 ++++ .../OverlayWindowController.swift | 58 ++ .../SampleBufferPCMExtractor.swift | 88 ++ .../SettingsWindowController.swift | 64 ++ .../Sources/MastermindPOC/main.swift | 8 + .../AudioPipelineStats.swift | 24 + .../MastermindPOCCore/OverlayActions.swift | 11 + .../MastermindPOCCore/OverlaySettings.swift | 15 + .../Sources/MastermindPOCCore/PCM16LE.swift | 51 ++ .../Sources/MastermindPOCCore/PCMFrame.swift | 22 + .../MastermindPOCCore/TrustStatus.swift | 40 + .../AudioModelTests.swift | 51 ++ .../OverlayActionsTests.swift | 24 + .../OverlaySettingsTests.swift | 26 + .../TrustStatusTests.swift | 21 + native/MastermindPOC/scripts/build-app.sh | 58 ++ package.json | 2 + scripts/mock-nemotron-sidecar.js | 66 ++ src/components/app/CheatingDaddyApp.js | 2 +- src/components/views/AICustomizeView.js | 285 +++--- src/components/views/CustomizeView.js | 6 +- src/components/views/MainView.js | 303 ++++--- src/storage.js | 8 + src/utils/gemini.js | 4 +- src/utils/localProviders.js | 304 +++++++ src/utils/localai.js | 397 ++++++++- src/utils/renderer.js | 32 +- test/localProviders.test.js | 151 ++++ 40 files changed, 3495 insertions(+), 327 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 docs/local-sidecar-protocol.md create mode 100644 docs/migration/native-poc-results.md create mode 100644 docs/superpowers/plans/2026-06-26-macos-native-ai-companion-reconstruction.md create mode 100644 native/MastermindPOC/.gitignore create mode 100644 native/MastermindPOC/Package.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/AppDelegate.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/CaptureCoordinator.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/MenuBarController.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/MicrophoneCaptureEngine.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/MicrophonePCMConverter.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/OverlaySettingsStore.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/OverlayView.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/OverlayWindowController.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/SampleBufferPCMExtractor.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/SettingsWindowController.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOC/main.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/AudioPipelineStats.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/OverlayActions.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/OverlaySettings.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/PCM16LE.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/PCMFrame.swift create mode 100644 native/MastermindPOC/Sources/MastermindPOCCore/TrustStatus.swift create mode 100644 native/MastermindPOC/Tests/MastermindPOCCoreTests/AudioModelTests.swift create mode 100644 native/MastermindPOC/Tests/MastermindPOCCoreTests/OverlayActionsTests.swift create mode 100644 native/MastermindPOC/Tests/MastermindPOCCoreTests/OverlaySettingsTests.swift create mode 100644 native/MastermindPOC/Tests/MastermindPOCCoreTests/TrustStatusTests.swift create mode 100755 native/MastermindPOC/scripts/build-app.sh create mode 100644 scripts/mock-nemotron-sidecar.js create mode 100644 src/utils/localProviders.js create mode 100644 test/localProviders.test.js diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..2356a3c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,24 @@ +{ + "configurations": [ + { + "type": "swift", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:cheating-daddy}/native/MastermindPOC", + "name": "Debug MastermindPOC (native/MastermindPOC)", + "target": "MastermindPOC", + "configuration": "debug", + "preLaunchTask": "swift: Build Debug MastermindPOC (native/MastermindPOC)" + }, + { + "type": "swift", + "request": "launch", + "args": [], + "cwd": "${workspaceFolder:cheating-daddy}/native/MastermindPOC", + "name": "Release MastermindPOC (native/MastermindPOC)", + "target": "MastermindPOC", + "configuration": "release", + "preLaunchTask": "swift: Build Release MastermindPOC (native/MastermindPOC)" + } + ] +} \ No newline at end of file diff --git a/docs/local-sidecar-protocol.md b/docs/local-sidecar-protocol.md new file mode 100644 index 0000000..0f28db6 --- /dev/null +++ b/docs/local-sidecar-protocol.md @@ -0,0 +1,60 @@ +# Local ASR Sidecar Protocol + +Mastermind local-first mode expects speech-to-text to run as an external local +streaming service. The app connects to the service over WebSocket and sends +16 kHz mono PCM audio. + +## Default Endpoint + +```text +ws://127.0.0.1:8765/v1/asr/stream +``` + +The endpoint is configurable in the Local AI settings. + +## Client Start Message + +After the WebSocket opens, the app sends a JSON start frame: + +```json +{ + "type": "start", + "sampleRate": 16000, + "channels": 1, + "encoding": "pcm_s16le", + "language": "en-US" +} +``` + +After that, the app sends binary frames containing raw little-endian signed +16-bit PCM audio at 16 kHz. + +## Sidecar Events + +The sidecar should send JSON text frames: + +```json +{ "type": "ready" } +``` + +```json +{ "type": "partial", "text": "intermediate transcript" } +``` + +```json +{ "type": "final", "text": "final transcript" } +``` + +```json +{ "type": "error", "error": "human-readable error" } +``` + +Only `final` transcript events are sent to the local LLM. `partial` events are +shown as status text. + +## v1 Scope + +- STT target: English, `en-US`. +- Sidecar implementation is external to this repository. +- The app does not require a specific Nemotron, NeMo, Riva, or ONNX runtime as + long as the WebSocket protocol above is implemented. diff --git a/docs/migration/native-poc-results.md b/docs/migration/native-poc-results.md new file mode 100644 index 0000000..20fa374 --- /dev/null +++ b/docs/migration/native-poc-results.md @@ -0,0 +1,44 @@ +# Native macOS POC Results + +## Implemented + +- SwiftPM-based macOS POC under `native/MastermindPOC`. +- AppKit lifecycle with a menu bar status item. +- Transparent non-activating overlay window with hide/show and click-through controls. +- Overlay window uses `NSWindowSharingType.none` and ScreenCaptureKit filtering excludes the current process windows from app-owned capture. +- ScreenCaptureKit screen stream proof with frame counters. +- ScreenCaptureKit system audio proof with best-effort PCM extraction to 16 kHz mono signed 16-bit little-endian frames. +- AVAudioEngine microphone proof with 16 kHz mono signed 16-bit little-endian frames. +- Separate microphone and system audio counters in the overlay. +- Compact HUD controls for Settings, Hide, and Quit. +- Draggable overlay title area. +- Settings panel with persisted overlay background opacity. +- Swift unit tests for status labels, PCM frame metadata, PCM S16LE encoding, and separate audio counters. +- App bundle build script at `native/MastermindPOC/scripts/build-app.sh`. + +## Verified Automatically + +- `swift test` passes for `MastermindPOCCore`. +- `swift build --product MastermindPOC` compiles the native executable. +- `native/MastermindPOC/scripts/build-app.sh` produces `native/MastermindPOC/build/MastermindPOC.app`. +- Existing Electron local provider tests pass when run outside the managed sandbox restrictions. + +## Manual Verification Needed + +- Launch `native/MastermindPOC/build/MastermindPOC.app`. +- Confirm the menu bar item is visible while the app runs. +- Confirm the overlay appears locally and can be hidden. +- Confirm the overlay can be dragged by its top title bar. +- Confirm the overlay Settings button opens the settings panel. +- Confirm the opacity slider changes only the HUD background opacity and persists across relaunches. +- Confirm the overlay Quit button exits the app. +- Confirm click-through lets pointer events pass to apps underneath. +- Grant Screen Recording permission when macOS prompts, then restart the POC if needed. +- Start `Screen + System Audio` and confirm screen frame and system PCM counters increase. +- Start `Microphone` and confirm microphone PCM counters increase while speaking. +- Confirm `Pause All Capture` stops screen, system audio, and microphone updates. +- Confirm app-owned screen capture does not include the Mastermind overlay. + +## Boundary + +This POC is not a stealth or anti-detection system. It shows a menu bar item while running, requests normal macOS permissions, and only excludes its own overlay from the context it captures for itself. diff --git a/docs/superpowers/plans/2026-06-26-macos-native-ai-companion-reconstruction.md b/docs/superpowers/plans/2026-06-26-macos-native-ai-companion-reconstruction.md new file mode 100644 index 0000000..3428a1a --- /dev/null +++ b/docs/superpowers/plans/2026-06-26-macos-native-ai-companion-reconstruction.md @@ -0,0 +1,825 @@ +# macOS Native AI Companion Reconstruction Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reconstruct Mastermind from an Electron interview-helper-shaped app into a native macOS AI companion with reliable overlay, screen context, system audio, voice interaction, and explicit trust indicators. + +**Architecture:** Build a Swift/AppKit-first macOS host that owns windows, capture, audio, permissions, menu bar status, and privacy boundaries. Keep AI providers and local sidecars behind protocol boundaries so the rewrite can progress incrementally instead of becoming a risky full replacement. + +**Tech Stack:** Swift, AppKit, SwiftUI where appropriate, ScreenCaptureKit, AVAudioEngine, CoreAudio, Keychain, Application Support storage, URLSession streaming, WebSocket, local ASR sidecar protocol, OpenAI-compatible HTTP/SSE, optional Electron bridge during migration. + +--- + +## Product Thesis + +Mastermind should become an **AI companion for your Mac**, not a hidden interview helper. + +The assistant is a personal, visible-to-the-user HUD and agent system. It can help during calls, presentations, coding, writing, research, and everyday computer use. It may stay out of the content the user is presenting or sharing, just like presenter notes, a timer, captions, or a local command palette. + +The product should not implement stealth, anti-detection, process hiding, monitoring bypass, proctoring bypass, or policy evasion. If a workplace, school, exam, or managed system prohibits AI assistants, Mastermind should not help users hide that it is running. + +## Core Boundary + +**Allowed:** The assistant window does not appear in the user's own screen context, screenshots, screen-share content, or presentation output when technically possible. + +**Not allowed:** Hiding the app process, bundle identifier, permissions, network use, Accessibility use, Screen Recording use, microphone use, or audio capture use from operating-system tools or managed environments. + +Practical phrasing: + +> Mastermind does not hide from the user or the system. It only avoids contaminating the content the user intentionally shares or asks the assistant to analyze. + +## Target User Experience + +### Normal Companion Mode + +The assistant is available through voice, keyboard, and a small native overlay. It can answer questions, remember context, summarize active work, and trigger actions. + +### Meeting Mode + +The assistant can listen to microphone and system audio with clear status indicators. It can summarize discussion, draft follow-ups, keep agenda state, and help the user stay oriented. + +### Presentation Mode + +The user can share slides, a browser, or an app window while seeing local prompts, timing, plan notes, likely objections, and speaker guidance in Mastermind's HUD. The shared audience does not need to see the HUD. + +### Screen Context Mode + +Screen context is opt-in. The user explicitly asks the assistant to look at the screen, capture a snapshot, or follow a bounded live stream. Mastermind excludes its own UI from that context. + +### Idle Mode + +When not actively listening, viewing, or processing, Mastermind stays quiet and visibly idle in the menu bar. + +## Trust Model + +Trust is a first-class feature, not a settings afterthought. + +Required visible signals: + +- Menu bar icon is always present while the app runs. +- Status reflects actual state: idle, listening, reading screen, processing, paused, permission problem. +- Mic capture, system audio capture, and screen context have separate indicators. +- The user can pause capture immediately. +- The app exposes a local activity log showing recent context use: microphone, system audio, screen snapshot, screen stream, provider request. +- The app does not provide stealth labels, anti-detection switches, or hidden-running modes. + +Recommended status vocabulary: + +- `Idle` +- `Listening` +- `System Audio` +- `Screen Snapshot` +- `Screen Stream` +- `Agent Working` +- `Paused` +- `Permission Needed` +- `Error` + +## Current Project Context + +Current repository shape: + +- Electron Forge app with JavaScript entry point at `src/index.js`. +- Window management and global shortcuts in `src/utils/window.js`. +- AI provider/session logic in `src/utils/gemini.js`, `src/utils/localai.js`, and `src/utils/localProviders.js`. +- Local ASR sidecar protocol already documented in `docs/local-sidecar-protocol.md`. +- JSON storage in `src/storage.js`. +- Lit-based UI under `src/components`. + +Important existing behavior to preserve: + +- Always-on-top assistant window. +- Global keyboard shortcuts. +- Click-through toggle. +- Hide/show assistant. +- Session history. +- Local mode with ASR sidecar. +- BYOK provider support. +- OpenAI-compatible provider support. +- Groq/Gemini provider support. +- Configurable prompt/profile/language. + +Important behavior to replace: + +- Electron-owned transparent window quirks. +- Chromium `getDisplayMedia` screen capture dependency. +- Fragile loopback/system-audio capture. +- Renderer-local storage access from main process. +- `nodeIntegration: true` and `contextIsolation: false`. +- Hidden coupling between capture, transcription, provider routing, and renderer events. + +## Reconstruction Strategy + +Recommended path: **Swift-native host first, full app migration second**. + +Do not start with a full SwiftUI rewrite of every screen. The riskiest parts are windowing, screen capture, audio capture, permissions, and trust indicators. Prove those first in a native macOS host, then migrate provider and UI surfaces incrementally. + +Migration shape: + +```text +Phase 1: Native capability proof + Swift/AppKit overlay + menu bar status item + ScreenCaptureKit screen context + ScreenCaptureKit system audio + AVAudioEngine microphone audio + +Phase 2: Native companion shell + state machine + permissions + trust indicators + local sidecar bridge + provider boundary + +Phase 3: Agentic assistant + modes + memory + tools + activity log + meeting and presentation flows + +Phase 4: Retire Electron + migrate settings/history + package/notarize Swift app + deprecate Electron runtime +``` + +## Proposed Native Architecture + +```text +Mastermind.app + AppCoordinator + owns lifecycle, mode, permissions, menu bar status + + OverlayWindowController + owns transparent HUD, click-through, Spaces behavior, focus behavior + + CaptureCoordinator + owns screen context and system audio through ScreenCaptureKit + + MicrophoneCaptureEngine + owns microphone stream through AVAudioEngine + + AudioPipeline + owns channel separation, resampling, VAD, PCM frame output + + AssistantSession + owns active conversation, transcript, memory references, provider routing + + ProviderClients + Gemini, Groq, OpenAI-compatible, local LLM, local ASR sidecar + + TrustStatusStore + owns visible state, activity log, pause/resume, capture indicators + + Storage + Application Support for non-secret app data + Keychain for secrets +``` + +## Agentic System Direction + +The future assistant should be built around modes and tools, not a single chat box. + +Core modes: + +- General Companion +- Meeting Assistant +- Presentation Coach +- Coding Assistant +- Research Assistant +- Focus Assistant + +Core agent abilities: + +- Listen and summarize. +- Answer conversationally. +- See screen only when requested. +- Track agenda or presentation plan. +- Draft follow-up notes. +- Remember user preferences. +- Use local tools after explicit permission. +- Explain what context it used. + +Agent boundaries: + +- No autonomous destructive actions. +- No hidden capture. +- No silent screen streaming. +- No policy bypass tooling. +- No stealth process behavior. + +## File Structure Target + +Future native app structure: + +```text +native/Mastermind/ + Mastermind.xcodeproj + Mastermind/ + App/ + MastermindApp.swift + AppCoordinator.swift + AppMode.swift + PermissionState.swift + Status/ + MenuBarController.swift + TrustStatus.swift + ActivityLog.swift + Overlay/ + OverlayWindowController.swift + OverlayRootView.swift + OverlayViewModel.swift + Capture/ + CaptureCoordinator.swift + ScreenContextCapture.swift + SystemAudioCapture.swift + CaptureExclusionPolicy.swift + Audio/ + MicrophoneCaptureEngine.swift + AudioPipeline.swift + PCMFrame.swift + VoiceActivityDetector.swift + Assistant/ + AssistantSession.swift + AssistantMode.swift + AssistantEvent.swift + AssistantMemory.swift + Providers/ + ProviderClient.swift + LocalAsrSidecarClient.swift + OpenAICompatibleClient.swift + GeminiClient.swift + GroqClient.swift + Storage/ + AppStorageStore.swift + KeychainStore.swift + MigrationStore.swift + Settings/ + SettingsWindowController.swift + SettingsRootView.swift + Presentation/ + PresentationPlan.swift + PresentationCoach.swift +``` + +Existing Electron files remain read-only during the proof phase except for bridge points explicitly required by a migration task. + +## Task 1: Write Product Reconstruction Charter + +**Files:** + +- Create: `docs/product/macos-native-ai-companion-charter.md` + +- [ ] **Step 1: Create the product charter** + +Write a concise charter with these sections: + +```markdown +# macOS Native AI Companion Charter + +## Positioning + +Mastermind is a native macOS AI companion for everyday computer work, meetings, presentations, research, and focused execution. + +## What It Is + +- A visible personal assistant for the user. +- A voice-first and context-aware companion. +- A local HUD for notes, guidance, and agent status. +- A privacy-conscious screen and audio context tool. + +## What It Is Not + +- A hidden interview helper. +- A proctoring bypass tool. +- An anti-detection tool. +- A process-hiding tool. +- A tool for hiding AI use from managed systems that prohibit it. + +## Core Promise + +Mastermind does not hide from the user or the system. It only avoids contaminating the content the user intentionally shares or asks the assistant to analyze. + +## Trust Requirements + +- Show a menu bar status item whenever running. +- Show when microphone, system audio, or screen context is active. +- Allow immediate pause. +- Keep a local activity log of context use. +- Store secrets in Keychain. +``` + +- [ ] **Step 2: Review the charter language** + +Confirm the document includes the phrases `AI companion for your Mac`, `does not hide from the user or the system`, and `visible personal assistant`. + +- [ ] **Step 3: Commit** + +```bash +git add docs/product/macos-native-ai-companion-charter.md +git commit -m "docs: define macos ai companion direction" +``` + +## Task 2: Build Native Capability Proof Project + +**Files:** + +- Create: `native/Mastermind/Mastermind.xcodeproj` +- Create: `native/Mastermind/Mastermind/App/MastermindApp.swift` +- Create: `native/Mastermind/Mastermind/App/AppCoordinator.swift` +- Create: `native/Mastermind/Mastermind/Status/MenuBarController.swift` +- Create: `native/Mastermind/Mastermind/Overlay/OverlayWindowController.swift` +- Create: `native/Mastermind/Mastermind/Capture/CaptureCoordinator.swift` +- Create: `native/Mastermind/Mastermind/Audio/MicrophoneCaptureEngine.swift` +- Create: `native/Mastermind/Mastermind/Audio/AudioPipeline.swift` + +- [ ] **Step 1: Create a minimal native macOS app** + +Create a macOS app target named `Mastermind`. Use Swift, AppKit lifecycle, and SwiftUI only for simple views. + +- [ ] **Step 2: Add a menu bar status item** + +Implement a menu bar item that always appears while the app runs. The menu must include: + +```text +Mastermind: Idle +Pause All Capture +Show Assistant +Hide Assistant +Settings +Quit Mastermind +``` + +- [ ] **Step 3: Add a transparent overlay window** + +Create an AppKit-controlled floating overlay window with these properties: + +```text +borderless +transparent background +always on top +visible across Spaces +does not steal focus when shown +can become click-through +can be hidden and restored +``` + +- [ ] **Step 4: Add screen context proof** + +Use ScreenCaptureKit to capture the main display. The proof succeeds only when captured frames exclude the assistant overlay. + +- [ ] **Step 5: Add system audio proof** + +Use ScreenCaptureKit audio output to receive system audio buffers. + +- [ ] **Step 6: Add microphone proof** + +Use AVAudioEngine to receive microphone PCM buffers. + +- [ ] **Step 7: Add 16 kHz PCM output proof** + +Convert microphone and system audio streams into 16 kHz signed 16-bit PCM frames. + +- [ ] **Step 8: Manual verification** + +Run the native app and verify: + +```text +menu bar item is visible +overlay is visible locally +overlay can become click-through +overlay does not appear in app-owned captured frames +system audio buffers arrive when another app plays audio +microphone buffers arrive when speaking +capture can be paused immediately +``` + +- [ ] **Step 9: Commit** + +```bash +git add native/Mastermind +git commit -m "feat: add native macos capture proof" +``` + +## Task 3: Define Trust State Machine + +**Files:** + +- Create: `native/Mastermind/Mastermind/Status/TrustStatus.swift` +- Create: `native/Mastermind/Mastermind/Status/ActivityLog.swift` +- Modify: `native/Mastermind/Mastermind/Status/MenuBarController.swift` +- Modify: `native/Mastermind/Mastermind/App/AppCoordinator.swift` + +- [ ] **Step 1: Define trust states** + +Use these states exactly: + +```swift +enum TrustStatus: Equatable { + case idle + case listening + case systemAudio + case screenSnapshot + case screenStream + case agentWorking + case paused + case permissionNeeded(String) + case error(String) +} +``` + +- [ ] **Step 2: Define context activity events** + +Use these events exactly: + +```swift +enum ActivityEventKind: String, Codable { + case microphoneStarted + case microphoneStopped + case systemAudioStarted + case systemAudioStopped + case screenSnapshotCaptured + case screenStreamStarted + case screenStreamStopped + case providerRequestStarted + case providerRequestFinished + case capturePaused +} +``` + +- [ ] **Step 3: Wire status to menu bar copy** + +Map trust states to visible labels: + +```text +idle -> Mastermind: Idle +listening -> Mastermind: Listening +systemAudio -> Mastermind: System Audio +screenSnapshot -> Mastermind: Screen Snapshot +screenStream -> Mastermind: Screen Stream +agentWorking -> Mastermind: Agent Working +paused -> Mastermind: Paused +permissionNeeded -> Mastermind: Permission Needed +error -> Mastermind: Error +``` + +- [ ] **Step 4: Verify state changes manually** + +Trigger each capture path and confirm the menu bar label changes before any capture data leaves the machine. + +- [ ] **Step 5: Commit** + +```bash +git add native/Mastermind/Mastermind/Status native/Mastermind/Mastermind/App +git commit -m "feat: add native trust status model" +``` + +## Task 4: Preserve Local ASR Sidecar Boundary + +**Files:** + +- Create: `native/Mastermind/Mastermind/Providers/ProviderClient.swift` +- Create: `native/Mastermind/Mastermind/Providers/LocalAsrSidecarClient.swift` +- Modify: `docs/local-sidecar-protocol.md` + +- [ ] **Step 1: Define provider boundary** + +Use this protocol as the native client boundary: + +```swift +protocol ProviderClient { + associatedtype Event + + func start() async throws + func stop() async + var events: AsyncStream { get } +} +``` + +- [ ] **Step 2: Mirror the current ASR sidecar protocol** + +Implement the native client against the existing WebSocket flow: + +```json +{ + "type": "start", + "sampleRate": 16000, + "channels": 1, + "encoding": "pcm_s16le", + "language": "en-US" +} +``` + +- [ ] **Step 3: Preserve event semantics** + +Support these sidecar events: + +```text +ready +partial +final +error +``` + +Only `final` transcript events should enter the assistant response pipeline by default. + +- [ ] **Step 4: Document native compatibility** + +Add a section to `docs/local-sidecar-protocol.md`: + +```markdown +## Native macOS Client Compatibility + +The Swift-native app uses the same WebSocket protocol as the Electron app. Audio frames are sent as raw 16 kHz mono signed 16-bit little-endian PCM. The sidecar does not need to know whether the client is Electron or Swift. +``` + +- [ ] **Step 5: Commit** + +```bash +git add native/Mastermind/Mastermind/Providers docs/local-sidecar-protocol.md +git commit -m "feat: add native local asr sidecar client" +``` + +## Task 5: Design Native Audio Pipeline + +**Files:** + +- Create: `native/Mastermind/Mastermind/Audio/PCMFrame.swift` +- Create: `native/Mastermind/Mastermind/Audio/AudioPipeline.swift` +- Create: `native/Mastermind/Mastermind/Audio/VoiceActivityDetector.swift` +- Modify: `native/Mastermind/Mastermind/Audio/MicrophoneCaptureEngine.swift` +- Modify: `native/Mastermind/Mastermind/Capture/CaptureCoordinator.swift` + +- [ ] **Step 1: Define PCM frame format** + +Use this data model: + +```swift +struct PCMFrame: Equatable { + let source: AudioSource + let sampleRate: Int + let channels: Int + let pcmS16LE: Data + let timestamp: Date +} + +enum AudioSource: String { + case microphone + case system +} +``` + +- [ ] **Step 2: Keep microphone and system audio separate** + +The pipeline should not mix microphone and system audio before transcription. Separate streams preserve future diarization and meeting-context quality. + +- [ ] **Step 3: Resample to 16 kHz** + +Every frame sent to local ASR must be: + +```text +sample rate: 16000 +channels: 1 +encoding: signed 16-bit little-endian PCM +``` + +- [ ] **Step 4: Add VAD boundary** + +Voice activity detection should emit speech segments instead of forcing every buffer into transcription. + +- [ ] **Step 5: Manual verification** + +Verify: + +```text +mic frames continue when system audio is silent +system frames continue when mic is silent +both streams can be paused together +either stream can be disabled independently +sidecar receives valid 16 kHz PCM frames +``` + +- [ ] **Step 6: Commit** + +```bash +git add native/Mastermind/Mastermind/Audio native/Mastermind/Mastermind/Capture +git commit -m "feat: add native audio pipeline model" +``` + +## Task 6: Migrate Secrets and Storage Boundaries + +**Files:** + +- Create: `native/Mastermind/Mastermind/Storage/KeychainStore.swift` +- Create: `native/Mastermind/Mastermind/Storage/AppStorageStore.swift` +- Create: `native/Mastermind/Mastermind/Storage/MigrationStore.swift` +- Reference: `src/storage.js` + +- [ ] **Step 1: Map current storage** + +Preserve these current logical groups: + +```text +config +credentials +preferences +keybinds +limits +history +``` + +- [ ] **Step 2: Move secrets to Keychain** + +Store these values in Keychain: + +```text +Gemini API key +Groq API key +OpenAI-compatible API key +local LLM API key +``` + +- [ ] **Step 3: Store non-secret data in Application Support** + +Store these values in Application Support: + +```text +preferences +profiles +keybinds +history +limits +activity log +presentation plans +assistant memory references +``` + +- [ ] **Step 4: Add one-way migration** + +Read existing Electron JSON files from: + +```text +~/Library/Application Support/cheating-daddy-config +``` + +Import values into native stores. Leave the old files untouched. + +- [ ] **Step 5: Commit** + +```bash +git add native/Mastermind/Mastermind/Storage +git commit -m "feat: add native storage migration boundary" +``` + +## Task 7: Build Presentation Coach Mode + +**Files:** + +- Create: `native/Mastermind/Mastermind/Presentation/PresentationPlan.swift` +- Create: `native/Mastermind/Mastermind/Presentation/PresentationCoach.swift` +- Modify: `native/Mastermind/Mastermind/Assistant/AssistantMode.swift` +- Modify: `native/Mastermind/Mastermind/Overlay/OverlayRootView.swift` + +- [ ] **Step 1: Define presentation plan model** + +Use this model: + +```swift +struct PresentationPlan: Codable, Equatable { + var title: String + var sections: [PresentationSection] +} + +struct PresentationSection: Codable, Equatable { + var title: String + var talkingPoints: [String] + var expectedDurationSeconds: Int +} +``` + +- [ ] **Step 2: Add presentation mode** + +Add a mode named `presentationCoach`. + +- [ ] **Step 3: Show local guidance in HUD** + +HUD should show: + +```text +current section +next talking point +elapsed time +suggested transition +likely audience question +``` + +- [ ] **Step 4: Keep guidance out of shared content** + +Use the same overlay exclusion policy as Screen Context Mode. The local HUD should be visible to the user and absent from app-owned captures. + +- [ ] **Step 5: Commit** + +```bash +git add native/Mastermind/Mastermind/Presentation native/Mastermind/Mastermind/Assistant native/Mastermind/Mastermind/Overlay +git commit -m "feat: add presentation coach mode" +``` + +## Task 8: Define Electron Retirement Gates + +**Files:** + +- Create: `docs/migration/electron-retirement-gates.md` + +- [ ] **Step 1: Create retirement gate checklist** + +Write this checklist: + +```markdown +# Electron Retirement Gates + +- [ ] Native overlay matches or exceeds Electron window behavior. +- [ ] Native menu bar status item is always visible while running. +- [ ] Native screen context excludes Mastermind UI. +- [ ] Native system audio capture works without third-party loopback drivers. +- [ ] Native microphone capture works independently from system audio. +- [ ] Native local ASR sidecar client can transcribe 16 kHz PCM. +- [ ] Native provider client can stream OpenAI-compatible responses. +- [ ] Native settings can import current Electron preferences. +- [ ] Native Keychain storage replaces JSON credential storage. +- [ ] Native package can be signed and notarized. +- [ ] Electron app remains available as fallback until native app covers core workflows. +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/migration/electron-retirement-gates.md +git commit -m "docs: add electron retirement gates" +``` + +## Task 9: Package and Distribution Direction + +**Files:** + +- Create: `docs/migration/native-distribution.md` + +- [ ] **Step 1: Document distribution requirements** + +Include: + +```text +Developer ID signing +notarization +Screen Recording permission messaging +Microphone permission messaging +optional Accessibility permission messaging +Sparkle or equivalent update path +DMG distribution +crash reporting decision +local log export +``` + +- [ ] **Step 2: Document permission copy** + +Use transparent user-facing copy: + +```text +Mastermind needs Screen Recording permission only when you ask it to use screen context. +Mastermind needs Microphone permission only when voice input or meeting listening is enabled. +Mastermind shows a menu bar status item whenever it is running. +``` + +- [ ] **Step 3: Commit** + +```bash +git add docs/migration/native-distribution.md +git commit -m "docs: define native distribution requirements" +``` + +## Verification Checklist + +Before claiming the reconstruction direction is ready for implementation: + +- [ ] The charter clearly says this is an AI companion, not a hidden helper. +- [ ] The plan preserves local ASR sidecar compatibility. +- [ ] The plan does not require a full rewrite before validating native capture. +- [ ] The plan includes a visible menu bar status item. +- [ ] The plan separates user-visible HUD behavior from stealth behavior. +- [ ] The plan includes microphone, system audio, and screen context as separate states. +- [ ] The plan includes Keychain for secrets. +- [ ] The plan includes Electron retirement gates. +- [ ] The plan keeps current Electron app functional during migration. + +## Recommended First Milestone + +The first milestone should be **Native Capability Proof**, not full product migration. + +Success definition: + +```text +A Swift/AppKit app shows a local transparent overlay, displays an always-visible menu bar status item, captures screen frames without its own overlay, receives system audio, receives microphone audio, and emits 16 kHz PCM frames compatible with the existing local ASR sidecar protocol. +``` + +If this milestone fails, keep improving the Electron app while reassessing native capture options. If it succeeds, move provider clients and agent modes into the native shell incrementally. + diff --git a/native/MastermindPOC/.gitignore b/native/MastermindPOC/.gitignore new file mode 100644 index 0000000..f0b71e2 --- /dev/null +++ b/native/MastermindPOC/.gitignore @@ -0,0 +1,2 @@ +.build/ +build/ diff --git a/native/MastermindPOC/Package.swift b/native/MastermindPOC/Package.swift new file mode 100644 index 0000000..66f6086 --- /dev/null +++ b/native/MastermindPOC/Package.swift @@ -0,0 +1,34 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "MastermindPOC", + platforms: [ + .macOS(.v14), + ], + products: [ + .executable(name: "MastermindPOC", targets: ["MastermindPOC"]), + .library(name: "MastermindPOCCore", targets: ["MastermindPOCCore"]), + ], + targets: [ + .target(name: "MastermindPOCCore"), + .executableTarget( + name: "MastermindPOC", + dependencies: ["MastermindPOCCore"], + linkerSettings: [ + .linkedFramework("AppKit"), + .linkedFramework("AVFoundation"), + .linkedFramework("CoreGraphics"), + .linkedFramework("CoreMedia"), + .linkedFramework("CoreVideo"), + .linkedFramework("ScreenCaptureKit"), + .linkedFramework("SwiftUI"), + ] + ), + .testTarget( + name: "MastermindPOCCoreTests", + dependencies: ["MastermindPOCCore"] + ), + ] +) diff --git a/native/MastermindPOC/Sources/MastermindPOC/AppDelegate.swift b/native/MastermindPOC/Sources/MastermindPOC/AppDelegate.swift new file mode 100644 index 0000000..3860491 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/AppDelegate.swift @@ -0,0 +1,183 @@ +import AppKit +import AVFoundation +import CoreGraphics +import MastermindPOCCore + +final class AppDelegate: NSObject, NSApplicationDelegate { + private let settingsStore = OverlaySettingsStore() + private lazy var overlayController = OverlayWindowController( + initialSettings: settingsStore.settings, + actions: OverlayActions( + openSettings: { [weak self] in self?.showSettings() }, + hideOverlay: { [weak self] in self?.hideOverlay() }, + quitApp: { NSApplication.shared.terminate(nil) } + ) + ) + private lazy var settingsWindowController = SettingsWindowController(settingsStore: settingsStore) + private var stats = AudioPipelineStats() + private var screenFrameCount = 0 + + private lazy var captureCoordinator = CaptureCoordinator( + onScreenFrame: { [weak self] frameCount in + Task { @MainActor in + self?.recordScreenFrame(frameCount) + } + }, + onSystemAudioFrame: { [weak self] frame in + Task { @MainActor in + self?.recordAudioFrame(frame) + } + }, + onError: { [weak self] message in + Task { @MainActor in + self?.setStatus(.error(message)) + } + } + ) + + private lazy var microphoneEngine = MicrophoneCaptureEngine( + onFrame: { [weak self] frame in + Task { @MainActor in + self?.recordAudioFrame(frame) + } + }, + onError: { [weak self] message in + Task { @MainActor in + self?.setStatus(.error(message)) + } + } + ) + + private lazy var menuBarController = MenuBarController( + actions: MenuBarController.Actions( + showOverlay: { [weak self] in self?.showOverlay() }, + hideOverlay: { [weak self] in self?.hideOverlay() }, + openSettings: { [weak self] in self?.showSettings() }, + toggleClickThrough: { [weak self] in self?.toggleClickThrough() }, + startScreenAndSystemAudio: { [weak self] in self?.startScreenAndSystemAudio() }, + startMicrophone: { [weak self] in self?.startMicrophone() }, + pauseAllCapture: { [weak self] in self?.pauseAllCapture() }, + quit: { NSApplication.shared.terminate(nil) } + ) + ) + + func applicationDidFinishLaunching(_ notification: Notification) { + settingsStore.onChange = { [weak self] settings in + self?.overlayController.viewModel.backgroundOpacity = settings.backgroundOpacity + } + + overlayController.show() + setStatus(.idle) + refreshCounters() + } + + private func showOverlay() { + overlayController.show() + menuBarController.setOverlayVisible(true) + } + + private func hideOverlay() { + overlayController.hide() + menuBarController.setOverlayVisible(false) + } + + private func showSettings() { + settingsWindowController.show() + } + + private func toggleClickThrough() { + let enabled = overlayController.toggleClickThrough() + overlayController.viewModel.clickThroughEnabled = enabled + menuBarController.setClickThrough(enabled) + } + + private func startScreenAndSystemAudio() { + Task { + do { + try await captureCoordinator.start(excludingWindowIDs: overlayController.excludedWindowIDs) + setStatus(.screenContext) + } catch { + setStatus(.error(error.localizedDescription)) + } + } + } + + private func startMicrophone() { + Task { + do { + try await requestMicrophoneAccess() + try microphoneEngine.start() + setStatus(.listening) + } catch { + setStatus(.error(error.localizedDescription)) + } + } + } + + private func pauseAllCapture() { + Task { + await captureCoordinator.stop() + microphoneEngine.stop() + setStatus(.paused) + } + } + + private func recordScreenFrame(_ frameCount: Int) { + screenFrameCount = frameCount + overlayController.viewModel.screenFrames = frameCount + setStatus(.screenContext) + } + + private func recordAudioFrame(_ frame: PCMFrame) { + stats.recordFrame(source: frame.source, byteCount: frame.pcmS16LE.count) + refreshCounters() + + switch frame.source { + case .microphone: + setStatus(.listening) + case .system: + setStatus(.systemAudio) + } + } + + private func refreshCounters() { + overlayController.viewModel.microphoneFrames = stats.microphoneFrames + overlayController.viewModel.microphoneBytes = stats.microphoneBytes + overlayController.viewModel.systemFrames = stats.systemFrames + overlayController.viewModel.systemBytes = stats.systemBytes + } + + private func setStatus(_ status: TrustStatus) { + overlayController.viewModel.apply(status: status) + menuBarController.setStatus(status) + } + + private func requestMicrophoneAccess() async throws { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .authorized: + return + case .notDetermined: + let granted = await AVCaptureDevice.requestAccess(for: .audio) + if granted { + return + } + throw POCError("Microphone permission was denied") + case .denied, .restricted: + throw POCError("Microphone permission is not available") + @unknown default: + throw POCError("Unknown microphone permission state") + } + } +} + +struct POCError: LocalizedError { + let message: String + + init(_ message: String) { + self.message = message + } + + var errorDescription: String? { + message + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/CaptureCoordinator.swift b/native/MastermindPOC/Sources/MastermindPOC/CaptureCoordinator.swift new file mode 100644 index 0000000..7f3bbc2 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/CaptureCoordinator.swift @@ -0,0 +1,100 @@ +import CoreGraphics +import CoreMedia +import CoreVideo +import Foundation +import MastermindPOCCore +import ScreenCaptureKit + +final class CaptureCoordinator: NSObject, SCStreamOutput, SCStreamDelegate { + private let outputQueue = DispatchQueue(label: "app.mastermind.poc.screencapture") + private let onScreenFrame: (Int) -> Void + private let onSystemAudioFrame: (PCMFrame) -> Void + private let onError: (String) -> Void + + private var stream: SCStream? + private var screenFrameCount = 0 + + init( + onScreenFrame: @escaping (Int) -> Void, + onSystemAudioFrame: @escaping (PCMFrame) -> Void, + onError: @escaping (String) -> Void + ) { + self.onScreenFrame = onScreenFrame + self.onSystemAudioFrame = onSystemAudioFrame + self.onError = onError + super.init() + } + + func start(excludingWindowIDs: Set) async throws { + await stop() + + let content = try await SCShareableContent.current + + guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) ?? content.displays.first else { + throw POCError("No capturable display found") + } + + let currentPID = ProcessInfo.processInfo.processIdentifier + let excludedWindows = content.windows.filter { window in + excludingWindowIDs.contains(window.windowID) || window.owningApplication?.processID == currentPID + } + + let filter = SCContentFilter(display: display, excludingWindows: excludedWindows) + let configuration = SCStreamConfiguration() + configuration.width = max(1, display.width) + configuration.height = max(1, display.height) + configuration.minimumFrameInterval = CMTime(value: 1, timescale: 2) + configuration.pixelFormat = kCVPixelFormatType_32BGRA + configuration.queueDepth = 3 + configuration.capturesAudio = true + configuration.sampleRate = 16_000 + configuration.channelCount = 1 + configuration.excludesCurrentProcessAudio = true + + let stream = SCStream(filter: filter, configuration: configuration, delegate: self) + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: outputQueue) + try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: outputQueue) + try await stream.startCapture() + + screenFrameCount = 0 + self.stream = stream + } + + func stop() async { + guard let stream else { + return + } + + do { + try await stream.stopCapture() + } catch { + onError("Screen capture stop failed: \(error.localizedDescription)") + } + + self.stream = nil + } + + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard CMSampleBufferIsValid(sampleBuffer) else { + return + } + + switch type { + case .screen: + screenFrameCount += 1 + onScreenFrame(screenFrameCount) + case .audio: + if let frame = SampleBufferPCMExtractor.extractFrame(from: sampleBuffer, source: .system) { + onSystemAudioFrame(frame) + } + case .microphone: + return + @unknown default: + return + } + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + onError("Screen capture stopped: \(error.localizedDescription)") + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/MenuBarController.swift b/native/MastermindPOC/Sources/MastermindPOC/MenuBarController.swift new file mode 100644 index 0000000..1effdbc --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/MenuBarController.swift @@ -0,0 +1,121 @@ +import AppKit +import MastermindPOCCore + +final class MenuBarController: NSObject { + struct Actions { + let showOverlay: () -> Void + let hideOverlay: () -> Void + let openSettings: () -> Void + let toggleClickThrough: () -> Void + let startScreenAndSystemAudio: () -> Void + let startMicrophone: () -> Void + let pauseAllCapture: () -> Void + let quit: () -> Void + } + + private let actions: Actions + private let statusItem = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) + private let statusMenuItem = NSMenuItem(title: "Mastermind: Idle", action: nil, keyEquivalent: "") + private let showItem = NSMenuItem(title: "Show Assistant", action: #selector(showOverlay), keyEquivalent: "") + private let hideItem = NSMenuItem(title: "Hide Assistant", action: #selector(hideOverlay), keyEquivalent: "") + private let clickThroughItem = NSMenuItem(title: "Enable Click-Through", action: #selector(toggleClickThrough), keyEquivalent: "") + + init(actions: Actions) { + self.actions = actions + super.init() + configureMenu() + setStatus(.idle) + } + + func setStatus(_ status: TrustStatus) { + let title = status.menuBarTitle + statusMenuItem.title = title + statusItem.button?.title = title + } + + func setOverlayVisible(_ visible: Bool) { + showItem.isEnabled = !visible + hideItem.isEnabled = visible + } + + func setClickThrough(_ enabled: Bool) { + clickThroughItem.title = enabled ? "Disable Click-Through" : "Enable Click-Through" + clickThroughItem.state = enabled ? .on : .off + } + + private func configureMenu() { + statusItem.button?.title = "Mastermind: Idle" + statusItem.button?.toolTip = "Mastermind AI helper status" + + let menu = NSMenu() + statusMenuItem.isEnabled = false + menu.addItem(statusMenuItem) + menu.addItem(.separator()) + + showItem.target = self + hideItem.target = self + clickThroughItem.target = self + menu.addItem(showItem) + menu.addItem(hideItem) + menu.addItem(clickThroughItem) + menu.addItem(.separator()) + + let settingsItem = NSMenuItem(title: "Settings...", action: #selector(openSettings), keyEquivalent: ",") + settingsItem.target = self + menu.addItem(settingsItem) + menu.addItem(.separator()) + + let screenItem = NSMenuItem(title: "Start Screen + System Audio", action: #selector(startScreenAndSystemAudio), keyEquivalent: "") + screenItem.target = self + menu.addItem(screenItem) + + let microphoneItem = NSMenuItem(title: "Start Microphone", action: #selector(startMicrophone), keyEquivalent: "") + microphoneItem.target = self + menu.addItem(microphoneItem) + + let pauseItem = NSMenuItem(title: "Pause All Capture", action: #selector(pauseAllCapture), keyEquivalent: "") + pauseItem.target = self + menu.addItem(pauseItem) + menu.addItem(.separator()) + + let quitItem = NSMenuItem(title: "Quit Mastermind POC", action: #selector(quit), keyEquivalent: "q") + quitItem.target = self + menu.addItem(quitItem) + + statusItem.menu = menu + setOverlayVisible(true) + setClickThrough(false) + } + + @objc private func showOverlay() { + actions.showOverlay() + } + + @objc private func hideOverlay() { + actions.hideOverlay() + } + + @objc private func openSettings() { + actions.openSettings() + } + + @objc private func toggleClickThrough() { + actions.toggleClickThrough() + } + + @objc private func startScreenAndSystemAudio() { + actions.startScreenAndSystemAudio() + } + + @objc private func startMicrophone() { + actions.startMicrophone() + } + + @objc private func pauseAllCapture() { + actions.pauseAllCapture() + } + + @objc private func quit() { + actions.quit() + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/MicrophoneCaptureEngine.swift b/native/MastermindPOC/Sources/MastermindPOC/MicrophoneCaptureEngine.swift new file mode 100644 index 0000000..4c3f7e2 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/MicrophoneCaptureEngine.swift @@ -0,0 +1,54 @@ +import AVFoundation +import Foundation +import MastermindPOCCore + +final class MicrophoneCaptureEngine { + private let engine = AVAudioEngine() + private let onFrame: (PCMFrame) -> Void + private let onError: (String) -> Void + private var isRunning = false + + init(onFrame: @escaping (PCMFrame) -> Void, onError: @escaping (String) -> Void) { + self.onFrame = onFrame + self.onError = onError + } + + func start() throws { + guard !isRunning else { + return + } + + let inputNode = engine.inputNode + let format = inputNode.outputFormat(forBus: 0) + + inputNode.removeTap(onBus: 0) + inputNode.installTap(onBus: 0, bufferSize: 1024, format: format) { [weak self] buffer, _ in + guard let self else { + return + } + + if let frame = MicrophonePCMConverter.extractFrame(from: buffer) { + self.onFrame(frame) + } + } + + do { + try engine.start() + isRunning = true + } catch { + inputNode.removeTap(onBus: 0) + onError("Microphone start failed: \(error.localizedDescription)") + throw error + } + } + + func stop() { + guard isRunning else { + return + } + + engine.inputNode.removeTap(onBus: 0) + engine.stop() + isRunning = false + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/MicrophonePCMConverter.swift b/native/MastermindPOC/Sources/MastermindPOC/MicrophonePCMConverter.swift new file mode 100644 index 0000000..f195714 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/MicrophonePCMConverter.swift @@ -0,0 +1,57 @@ +import AVFoundation +import Foundation +import MastermindPOCCore + +enum MicrophonePCMConverter { + static func extractFrame(from buffer: AVAudioPCMBuffer) -> PCMFrame? { + let frameCount = Int(buffer.frameLength) + guard frameCount > 0 else { + return nil + } + + let channelCount = max(1, Int(buffer.format.channelCount)) + let sourceRate = Int(buffer.format.sampleRate.rounded()) + let samples: [Float] + + if let floatData = buffer.floatChannelData { + samples = mixFloatChannels(floatData, channelCount: channelCount, frameCount: frameCount) + } else if let int16Data = buffer.int16ChannelData { + samples = mixInt16Channels(int16Data, channelCount: channelCount, frameCount: frameCount) + } else { + return nil + } + + let mono16k = PCM16LE.resampleLinear(samples: samples, sourceRate: sourceRate, targetRate: 16_000) + let data = PCM16LE.encode(samples: mono16k) + + return PCMFrame(source: .microphone, sampleRate: 16_000, channels: 1, pcmS16LE: data) + } + + private static func mixFloatChannels(_ channelData: UnsafePointer>, channelCount: Int, frameCount: Int) -> [Float] { + var mono = [Float](repeating: 0, count: frameCount) + let divisor = Float(channelCount) + + for channel in 0..>, channelCount: Int, frameCount: Int) -> [Float] { + var mono = [Float](repeating: 0, count: frameCount) + let divisor = Float(channelCount) + + for channel in 0.. Void)? + + private let defaults: UserDefaults + + init(defaults: UserDefaults = .standard) { + self.defaults = defaults + + if defaults.object(forKey: Self.backgroundOpacityKey) == nil { + settings = .default + } else { + settings = OverlaySettings(backgroundOpacity: defaults.double(forKey: Self.backgroundOpacityKey)) + } + } + + var backgroundOpacity: Double { + settings.backgroundOpacity + } + + func updateBackgroundOpacity(_ opacity: Double) { + let nextSettings = OverlaySettings(backgroundOpacity: opacity) + settings = nextSettings + defaults.set(nextSettings.backgroundOpacity, forKey: Self.backgroundOpacityKey) + onChange?(nextSettings) + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/OverlayView.swift b/native/MastermindPOC/Sources/MastermindPOC/OverlayView.swift new file mode 100644 index 0000000..9f8f143 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/OverlayView.swift @@ -0,0 +1,164 @@ +import MastermindPOCCore +import SwiftUI + +final class OverlayViewModel: ObservableObject { + @Published var statusTitle = "Mastermind: Idle" + @Published var statusDetail = "Native macOS companion proof" + @Published var clickThroughEnabled = false + @Published var backgroundOpacity = OverlaySettings.default.backgroundOpacity + @Published var screenFrames = 0 + @Published var microphoneFrames = 0 + @Published var microphoneBytes = 0 + @Published var systemFrames = 0 + @Published var systemBytes = 0 + + func apply(status: TrustStatus) { + statusTitle = status.menuBarTitle + + switch status { + case .idle: + statusDetail = "Ready. Capture is off." + case .listening: + statusDetail = "Microphone capture is active." + case .screenContext: + statusDetail = "Screen context proof is active." + case .systemAudio: + statusDetail = "System audio proof is active." + case .agentWorking: + statusDetail = "Agent work placeholder." + case .paused: + statusDetail = "All capture paused." + case .permissionNeeded(let message): + statusDetail = message + case .error(let message): + statusDetail = message + } + } +} + +struct OverlayView: View { + @ObservedObject var viewModel: OverlayViewModel + let actions: OverlayActions + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HudTitleBar(viewModel: viewModel, actions: actions) + + VStack(alignment: .leading, spacing: 4) { + Text(viewModel.statusTitle) + .font(.system(.title3, design: .rounded, weight: .semibold)) + .foregroundStyle(.white) + Text(viewModel.statusDetail) + .font(.callout) + .foregroundStyle(.white.opacity(0.78)) + .lineLimit(2) + } + + Divider() + .overlay(.white.opacity(0.2)) + + HStack(spacing: 14) { + CounterView(label: "Screen", value: viewModel.screenFrames) + CounterView(label: "Mic", value: viewModel.microphoneFrames) + CounterView(label: "System", value: viewModel.systemFrames) + } + + Text("PCM bytes mic \(viewModel.microphoneBytes) | system \(viewModel.systemBytes)") + .font(.caption2.monospacedDigit()) + .foregroundStyle(.white.opacity(0.62)) + } + .padding(18) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + .background( + RoundedRectangle(cornerRadius: 18, style: .continuous) + .fill(.black.opacity(viewModel.backgroundOpacity)) + .stroke(.white.opacity(0.16), lineWidth: 1) + ) + } +} + +private struct HudTitleBar: View { + @ObservedObject var viewModel: OverlayViewModel + let actions: OverlayActions + + var body: some View { + HStack(spacing: 10) { + ZStack(alignment: .leading) { + WindowDragRegion() + HStack(spacing: 8) { + Image(systemName: "sparkles") + .font(.caption.weight(.semibold)) + Text("Mastermind") + .font(.headline) + Text(viewModel.clickThroughEnabled ? "Click-through" : "Interactive") + .font(.caption) + .foregroundStyle(.white.opacity(0.66)) + } + .foregroundStyle(.white) + .allowsHitTesting(false) + } + .frame(height: 28) + + HStack(spacing: 6) { + HudIconButton(systemName: "gearshape", help: "Settings", action: actions.openSettings) + HudIconButton(systemName: "eye.slash", help: "Hide assistant", action: actions.hideOverlay) + HudIconButton(systemName: "xmark", help: "Quit Mastermind POC", role: .destructive, action: actions.quitApp) + } + } + } +} + +private struct HudIconButton: View { + let systemName: String + let help: String + var role: ButtonRole? + let action: () -> Void + + var body: some View { + Button(role: role, action: action) { + Image(systemName: systemName) + .font(.caption.weight(.semibold)) + .frame(width: 24, height: 24) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .foregroundStyle(.white.opacity(0.82)) + .background(.white.opacity(0.08), in: RoundedRectangle(cornerRadius: 6, style: .continuous)) + .help(help) + } +} + +private struct WindowDragRegion: NSViewRepresentable { + func makeNSView(context: Context) -> DragHandleView { + DragHandleView() + } + + func updateNSView(_ nsView: DragHandleView, context: Context) {} +} + +private final class DragHandleView: NSView { + override var mouseDownCanMoveWindow: Bool { + true + } + + override func mouseDown(with event: NSEvent) { + window?.performDrag(with: event) + } +} + +private struct CounterView: View { + let label: String + let value: Int + + var body: some View { + VStack(alignment: .leading, spacing: 3) { + Text(label) + .font(.caption2) + .foregroundStyle(.white.opacity(0.58)) + Text("\(value)") + .font(.caption.monospacedDigit().weight(.semibold)) + .foregroundStyle(.white) + } + .frame(maxWidth: .infinity, alignment: .leading) + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/OverlayWindowController.swift b/native/MastermindPOC/Sources/MastermindPOC/OverlayWindowController.swift new file mode 100644 index 0000000..0903447 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/OverlayWindowController.swift @@ -0,0 +1,58 @@ +import AppKit +import CoreGraphics +import MastermindPOCCore +import SwiftUI + +final class OverlayWindowController { + let viewModel = OverlayViewModel() + + private let panel: NSPanel + private var clickThroughEnabled = false + + init(initialSettings: OverlaySettings, actions: OverlayActions) { + let screenFrame = NSScreen.main?.visibleFrame ?? NSRect(x: 0, y: 0, width: 1280, height: 800) + let size = NSSize(width: 440, height: 210) + let origin = NSPoint( + x: screenFrame.maxX - size.width - 28, + y: screenFrame.maxY - size.height - 28 + ) + + panel = NSPanel( + contentRect: NSRect(origin: origin, size: size), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + + panel.isReleasedWhenClosed = false + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = true + panel.level = .floating + panel.isMovableByWindowBackground = true + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary, .ignoresCycle] + panel.sharingType = .none + panel.title = "Mastermind POC Overlay" + viewModel.backgroundOpacity = initialSettings.backgroundOpacity + panel.contentView = NSHostingView(rootView: OverlayView(viewModel: viewModel, actions: actions)) + } + + var excludedWindowIDs: Set { + [CGWindowID(panel.windowNumber)] + } + + func show() { + panel.orderFrontRegardless() + } + + func hide() { + panel.orderOut(nil) + } + + @discardableResult + func toggleClickThrough() -> Bool { + clickThroughEnabled.toggle() + panel.ignoresMouseEvents = clickThroughEnabled + return clickThroughEnabled + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOC/SampleBufferPCMExtractor.swift b/native/MastermindPOC/Sources/MastermindPOC/SampleBufferPCMExtractor.swift new file mode 100644 index 0000000..66cc85d --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOC/SampleBufferPCMExtractor.swift @@ -0,0 +1,88 @@ +import AudioToolbox +import CoreMedia +import Foundation +import MastermindPOCCore + +enum SampleBufferPCMExtractor { + static func extractFrame(from sampleBuffer: CMSampleBuffer, source: AudioSource) -> PCMFrame? { + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), + let streamDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) + else { + return nil + } + + let asbd = streamDescription.pointee + guard asbd.mFormatID == kAudioFormatLinearPCM else { + return nil + } + + var bufferListSize = 0 + var blockBuffer: CMBlockBuffer? + var status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: &bufferListSize, + bufferListOut: nil, + bufferListSize: 0, + blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, + flags: kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, + blockBufferOut: &blockBuffer + ) + + guard status == noErr, bufferListSize > 0 else { + return nil + } + + let bufferListPointer = UnsafeMutableRawPointer.allocate(byteCount: bufferListSize, alignment: MemoryLayout.alignment) + defer { + bufferListPointer.deallocate() + } + + let audioBufferList = bufferListPointer.bindMemory(to: AudioBufferList.self, capacity: 1) + status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: nil, + bufferListOut: audioBufferList, + bufferListSize: bufferListSize, + blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, + flags: kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, + blockBufferOut: &blockBuffer + ) + + guard status == noErr else { + return nil + } + + let buffers = UnsafeMutableAudioBufferListPointer(audioBufferList) + guard let firstBuffer = buffers.first, + let mData = firstBuffer.mData + else { + return nil + } + + let sourceRate = Int(asbd.mSampleRate.rounded()) + let byteCount = Int(firstBuffer.mDataByteSize) + let rawPointer = UnsafeRawPointer(mData) + + let samples: [Float] + if asbd.mBitsPerChannel == 32, asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 { + let sampleCount = byteCount / MemoryLayout.size + let pointer = rawPointer.bindMemory(to: Float.self, capacity: sampleCount) + samples = (0...size + let pointer = rawPointer.bindMemory(to: Int16.self, capacity: sampleCount) + samples = (0.. Void + public let hideOverlay: () -> Void + public let quitApp: () -> Void + + public init(openSettings: @escaping () -> Void, hideOverlay: @escaping () -> Void, quitApp: @escaping () -> Void) { + self.openSettings = openSettings + self.hideOverlay = hideOverlay + self.quitApp = quitApp + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOCCore/OverlaySettings.swift b/native/MastermindPOC/Sources/MastermindPOCCore/OverlaySettings.swift new file mode 100644 index 0000000..e01a0ae --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOCCore/OverlaySettings.swift @@ -0,0 +1,15 @@ +public struct OverlaySettings: Equatable { + public static let minimumOpacity = 0.35 + public static let maximumOpacity = 0.95 + public static let `default` = OverlaySettings(backgroundOpacity: 0.72) + + public let backgroundOpacity: Double + + public init(backgroundOpacity: Double) { + self.backgroundOpacity = Self.clamp(backgroundOpacity) + } + + public static func clamp(_ opacity: Double) -> Double { + min(maximumOpacity, max(minimumOpacity, opacity)) + } +} diff --git a/native/MastermindPOC/Sources/MastermindPOCCore/PCM16LE.swift b/native/MastermindPOC/Sources/MastermindPOCCore/PCM16LE.swift new file mode 100644 index 0000000..1f53c81 --- /dev/null +++ b/native/MastermindPOC/Sources/MastermindPOCCore/PCM16LE.swift @@ -0,0 +1,51 @@ +import Foundation + +public enum PCM16LE { + public static func encode(samples: [Float]) -> Data { + var data = Data() + data.reserveCapacity(samples.count * MemoryLayout.size) + + for sample in samples { + let clamped = max(-1.0, min(1.0, sample)) + let scaled: Int16 + + if clamped >= 1.0 { + scaled = Int16.max + } else if clamped <= -1.0 { + scaled = Int16.min + } else { + scaled = Int16((clamped * Float(Int16.max)).rounded()) + } + + var littleEndian = scaled.littleEndian + withUnsafeBytes(of: &littleEndian) { bytes in + data.append(contentsOf: bytes) + } + } + + return data + } + + public static func resampleLinear(samples: [Float], sourceRate: Int, targetRate: Int = 16_000) -> [Float] { + guard sourceRate > 0, targetRate > 0, !samples.isEmpty else { + return [] + } + + guard sourceRate != targetRate else { + return samples + } + + let ratio = Double(sourceRate) / Double(targetRate) + let outputCount = max(1, Int(Double(samples.count) / ratio)) + + return (0.. "$CONTENTS_DIR/Info.plist" <<'PLIST' + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + Mastermind POC + CFBundleExecutable + MastermindPOC + CFBundleIdentifier + app.mastermind.poc + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + Mastermind POC + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.1.0 + CFBundleVersion + 1 + LSMinimumSystemVersion + 14.0 + LSUIElement + + NSMicrophoneUsageDescription + Mastermind POC uses microphone audio only when you start microphone capture. + + +PLIST + +printf 'APPL????' > "$CONTENTS_DIR/PkgInfo" + +echo "Built $APP_DIR" diff --git a/package.json b/package.json index b5ec2ac..45367c8 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "make": "electron-forge make", "publish": "electron-forge publish", "lint": "echo \"No linting configured\"", + "test:local-providers": "node --test test/localProviders.test.js", + "mock:nemotron-sidecar": "node scripts/mock-nemotron-sidecar.js", "postinstall": "electron-rebuild -f -w onnxruntime-node" }, "keywords": [ diff --git a/scripts/mock-nemotron-sidecar.js b/scripts/mock-nemotron-sidecar.js new file mode 100644 index 0000000..2dae72e --- /dev/null +++ b/scripts/mock-nemotron-sidecar.js @@ -0,0 +1,66 @@ +#!/usr/bin/env node + +const WebSocket = require("ws"); + +const port = Number(process.env.MOCK_NEMOTRON_PORT || 8765); +const transcript = + process.env.MOCK_NEMOTRON_TRANSCRIPT || + "hello from the mock nemotron sidecar"; + +const server = new WebSocket.Server({ host: "127.0.0.1", port }); + +server.on("connection", (socket) => { + let binaryChunks = 0; + let finalSent = false; + + socket.on("message", (data, isBinary) => { + if (!isBinary) { + let message = null; + try { + message = JSON.parse(data.toString("utf8")); + } catch (_) { + socket.send(JSON.stringify({ type: "error", error: "Invalid JSON" })); + return; + } + + if (message.type === "start") { + socket.send(JSON.stringify({ type: "ready" })); + } + return; + } + + binaryChunks += 1; + + if (binaryChunks === 1) { + socket.send( + JSON.stringify({ + type: "partial", + text: transcript.split(" ").slice(0, 3).join(" "), + }), + ); + } + + if (!finalSent && binaryChunks >= 5) { + finalSent = true; + socket.send(JSON.stringify({ type: "final", text: transcript })); + } + }); +}); + +server.on("listening", () => { + console.log( + `Mock Nemotron sidecar listening on ws://127.0.0.1:${port}/v1/asr/stream`, + ); +}); + +server.on("error", (error) => { + console.error("Mock Nemotron sidecar error:", error); + process.exitCode = 1; +}); + +function shutdown() { + server.close(() => process.exit(0)); +} + +process.on("SIGINT", shutdown); +process.on("SIGTERM", shutdown); diff --git a/src/components/app/CheatingDaddyApp.js b/src/components/app/CheatingDaddyApp.js index 6fecf2d..e338e7f 100644 --- a/src/components/app/CheatingDaddyApp.js +++ b/src/components/app/CheatingDaddyApp.js @@ -612,7 +612,7 @@ export class CheatingDaddyApp extends LitElement { async handleStart() { const prefs = await cheatingDaddy.storage.getPreferences(); - const providerMode = prefs.providerMode || "byok"; + const providerMode = prefs.providerMode || "local"; if (providerMode === "local") { const success = await cheatingDaddy.initializeLocal(this.selectedProfile); diff --git a/src/components/views/AICustomizeView.js b/src/components/views/AICustomizeView.js index 095c31c..c9b81e2 100644 --- a/src/components/views/AICustomizeView.js +++ b/src/components/views/AICustomizeView.js @@ -1,143 +1,160 @@ -import { html, css, LitElement } from '../../assets/lit-core-2.7.4.min.js'; -import { unifiedPageStyles } from './sharedPageStyles.js'; +import { html, css, LitElement } from "../../assets/lit-core-2.7.4.min.js"; +import { unifiedPageStyles } from "./sharedPageStyles.js"; export class AICustomizeView extends LitElement { - static styles = [ - unifiedPageStyles, - css` - .unified-page { - height: 100%; - } - .unified-wrap { - height: 100%; - } - section.surface { - flex: 1; - display: flex; - flex-direction: column; - } - .form-grid { - flex: 1; - display: flex; - flex-direction: column; - } - .form-group.vertical { - flex: 1; - display: flex; - flex-direction: column; - } - textarea.control { - flex: 1; - resize: none; - overflow-y: auto; - min-height: 0; - } - `, + static styles = [ + unifiedPageStyles, + css` + .unified-page { + height: 100%; + } + .unified-wrap { + height: 100%; + } + section.surface { + flex: 1; + display: flex; + flex-direction: column; + } + .form-grid { + flex: 1; + display: flex; + flex-direction: column; + } + .form-group.vertical { + flex: 1; + display: flex; + flex-direction: column; + } + textarea.control { + flex: 1; + resize: none; + overflow-y: auto; + min-height: 0; + } + `, + ]; + + static properties = { + selectedProfile: { type: String }, + onProfileChange: { type: Function }, + _context: { state: true }, + _providerMode: { state: true }, + }; + + constructor() { + super(); + this.selectedProfile = "interview"; + this.onProfileChange = () => {}; + this._context = ""; + this._providerMode = "local"; + this._loadFromStorage(); + } + + async _loadFromStorage() { + try { + const prefs = await cheatingDaddy.storage.getPreferences(); + this._context = prefs.customPrompt || ""; + this._providerMode = prefs.providerMode || "local"; + this.requestUpdate(); + } catch (error) { + console.error("Error loading AI customize storage:", error); + } + } + + _handleProfileChange(e) { + this.onProfileChange(e.target.value); + } + + async _handleProviderModeChange(e) { + this._providerMode = e.target.value; + await cheatingDaddy.storage.updatePreference( + "providerMode", + this._providerMode, + ); + this.requestUpdate(); + } + + async _saveContext(val) { + this._context = val; + await cheatingDaddy.storage.updatePreference("customPrompt", val); + } + + _getProfileName(profile) { + const names = { + interview: "Job Interview", + sales: "Sales Call", + meeting: "Business Meeting", + presentation: "Presentation", + negotiation: "Negotiation", + exam: "Exam Assistant", + }; + return names[profile] || profile; + } + + render() { + const profiles = [ + { value: "interview", label: "Job Interview" }, + { value: "sales", label: "Sales Call" }, + { value: "meeting", label: "Business Meeting" }, + { value: "presentation", label: "Presentation" }, + { value: "negotiation", label: "Negotiation" }, + { value: "exam", label: "Exam Assistant" }, ]; - static properties = { - selectedProfile: { type: String }, - onProfileChange: { type: Function }, - _context: { state: true }, - _providerMode: { state: true }, - }; - - constructor() { - super(); - this.selectedProfile = 'interview'; - this.onProfileChange = () => {}; - this._context = ''; - this._providerMode = 'byok'; - this._loadFromStorage(); - } - - async _loadFromStorage() { - try { - const prefs = await cheatingDaddy.storage.getPreferences(); - this._context = prefs.customPrompt || ''; - this._providerMode = prefs.providerMode || 'byok'; - this.requestUpdate(); - } catch (error) { - console.error('Error loading AI customize storage:', error); - } - } - - _handleProfileChange(e) { - this.onProfileChange(e.target.value); - } - - async _handleProviderModeChange(e) { - this._providerMode = e.target.value; - await cheatingDaddy.storage.updatePreference('providerMode', this._providerMode); - this.requestUpdate(); - } - - async _saveContext(val) { - this._context = val; - await cheatingDaddy.storage.updatePreference('customPrompt', val); - } - - _getProfileName(profile) { - const names = { - interview: 'Job Interview', - sales: 'Sales Call', - meeting: 'Business Meeting', - presentation: 'Presentation', - negotiation: 'Negotiation', - exam: 'Exam Assistant', - }; - return names[profile] || profile; - } - - render() { - const profiles = [ - { value: 'interview', label: 'Job Interview' }, - { value: 'sales', label: 'Sales Call' }, - { value: 'meeting', label: 'Business Meeting' }, - { value: 'presentation', label: 'Presentation' }, - { value: 'negotiation', label: 'Negotiation' }, - { value: 'exam', label: 'Exam Assistant' }, - ]; - - return html` -
-
-
-
AI Context
-
- -
-
-
- - -
-
- - -
-
- - -
Sent as context at session start. Keep it short.
-
-
-
+ return html` +
+
+
+
AI Context
+
+
+
+
+ + +
+
+ + +
+
+ + +
+ Sent as context at session start. Keep it short.
+
- `; - } +
+
+
+ `; + } } -customElements.define('ai-customize-view', AICustomizeView); +customElements.define("ai-customize-view", AICustomizeView); diff --git a/src/components/views/CustomizeView.js b/src/components/views/CustomizeView.js index 68b0bf5..0d4e9ce 100644 --- a/src/components/views/CustomizeView.js +++ b/src/components/views/CustomizeView.js @@ -208,7 +208,7 @@ export class CustomizeView extends LitElement { this.onImageQualityChange = () => {}; this.onLayoutModeChange = () => {}; this.googleSearchEnabled = true; - this.providerMode = "byok"; + this.providerMode = "local"; this.isClearing = false; this.isRestoring = false; this.clearStatusMessage = ""; @@ -232,7 +232,7 @@ export class CustomizeView extends LitElement { cheatingDaddy.storage.getKeybinds(), ]); this.googleSearchEnabled = prefs.googleSearchEnabled ?? true; - this.providerMode = prefs.providerMode || "byok"; + this.providerMode = prefs.providerMode || "local"; this.backgroundTransparency = prefs.backgroundTransparency ?? 0.8; this.fontSize = prefs.fontSize ?? 20; this.audioMode = prefs.audioMode ?? "speaker_only"; @@ -664,7 +664,7 @@ export class CustomizeView extends LitElement { @change=${this.handleProviderModeChange} > - +
diff --git a/src/components/views/MainView.js b/src/components/views/MainView.js index ac82a35..0c8ca79 100644 --- a/src/components/views/MainView.js +++ b/src/components/views/MainView.js @@ -501,6 +501,11 @@ export class MainView extends LitElement { _whisperModel: { state: true }, _customWhisperModel: { state: true }, _showLocalHelp: { state: true }, + _localLlmBaseUrl: { state: true }, + _localLlmModel: { state: true }, + _localLlmApiKey: { state: true }, + _localSttUrl: { state: true }, + _localSttLanguage: { state: true }, }; constructor() { @@ -513,7 +518,7 @@ export class MainView extends LitElement { this.whisperDownloading = false; this.whisperProgress = null; - this._mode = "byok"; + this._mode = "local"; this._token = ""; this._geminiKey = ""; this._groqKey = ""; @@ -528,6 +533,11 @@ export class MainView extends LitElement { this._tokenError = false; this._keyError = false; this._showLocalHelp = false; + this._localLlmBaseUrl = "http://127.0.0.1:1234/v1"; + this._localLlmModel = ""; + this._localLlmApiKey = ""; + this._localSttUrl = "ws://127.0.0.1:8765/v1/asr/stream"; + this._localSttLanguage = "en-US"; this._ollamaHost = "http://127.0.0.1:11434"; this._ollamaModel = "llama3.1"; this._whisperModel = "Xenova/whisper-small"; @@ -549,7 +559,7 @@ export class MainView extends LitElement { cheatingDaddy.storage.getCredentials().catch(() => ({})), ]); - this._mode = prefs.providerMode || "byok"; + this._mode = prefs.providerMode || "local"; // Load keys this._token = ""; @@ -571,6 +581,13 @@ export class MainView extends LitElement { this._responseProvider = prefs.responseProvider || "gemini"; // Load local AI settings + this._localLlmBaseUrl = + prefs.localLlmBaseUrl || "http://127.0.0.1:1234/v1"; + this._localLlmModel = prefs.localLlmModel || ""; + this._localLlmApiKey = prefs.localLlmApiKey || ""; + this._localSttUrl = + prefs.localSttUrl || "ws://127.0.0.1:8765/v1/asr/stream"; + this._localSttLanguage = prefs.localSttLanguage || "en-US"; this._ollamaHost = prefs.ollamaHost || "http://127.0.0.1:11434"; this._ollamaModel = prefs.ollamaModel || "llama3.1"; this._whisperModel = prefs.whisperModel || "Xenova/whisper-small"; @@ -917,6 +934,51 @@ export class MainView extends LitElement { this.requestUpdate(); } + async _saveLocalLlmBaseUrl(val) { + this._localLlmBaseUrl = val; + await cheatingDaddy.storage.updatePreference("localLlmBaseUrl", val); + this.requestUpdate(); + } + + async _saveLocalLlmModel(val) { + this._localLlmModel = val; + await cheatingDaddy.storage.updatePreference("localLlmModel", val); + this.requestUpdate(); + } + + async _saveLocalLlmApiKey(val) { + this._localLlmApiKey = val; + await cheatingDaddy.storage.updatePreference("localLlmApiKey", val); + this.requestUpdate(); + } + + async _saveLocalSttUrl(val) { + this._localSttUrl = val; + await cheatingDaddy.storage.updatePreference("localSttUrl", val); + this.requestUpdate(); + } + + async _saveLocalSttLanguage(val) { + this._localSttLanguage = val; + await cheatingDaddy.storage.updatePreference("localSttLanguage", val); + this.requestUpdate(); + } + + _isLoopbackUrl(value) { + try { + const url = new URL(value); + const hostname = url.hostname.toLowerCase(); + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); + } catch (_) { + return false; + } + } + async _saveOllamaHost(val) { this._ollamaHost = val; await cheatingDaddy.storage.updatePreference("ollamaHost", val); @@ -1002,8 +1064,13 @@ export class MainView extends LitElement { return; } } else if (this._mode === "local") { - // Local mode doesn't need API keys, just Ollama host - if (!this._ollamaHost.trim()) { + if ( + !this._localLlmBaseUrl.trim() || + !this._localLlmModel.trim() || + !this._localSttUrl.trim() + ) { + this._keyError = true; + this.requestUpdate(); return; } } @@ -1258,105 +1325,88 @@ export class MainView extends LitElement { // ── Local AI mode ── _renderLocalMode() { + const llmIsLocal = this._isLoopbackUrl(this._localLlmBaseUrl); + const sttIsLocal = this._isLoopbackUrl(this._localSttUrl); + return html`
- + this._saveOllamaHost(e.target.value)} - /> -
Ollama must be running locally
-
- -
- - this._saveOllamaModel(e.target.value)} + placeholder="http://127.0.0.1:1234/v1" + .value=${this._localLlmBaseUrl} + @input=${(e) => this._saveLocalLlmBaseUrl(e.target.value)} + class=${this._keyError && !this._localLlmBaseUrl.trim() + ? "error" + : ""} />
- Run - ollama pull ${this._ollamaModel} - first + LM Studio local server endpoint for OpenAI-compatible chat + ${!llmIsLocal + ? html` + · not a localhost URL + ` + : ""}
-
- - ${this.whisperDownloading - ? html`` + + this._saveLocalLlmModel(e.target.value)} + class=${this._keyError && !this._localLlmModel.trim() ? "error" : ""} + /> +
+ Manual only: use the model identifier shown in LM Studio +
+
+ +
+ + this._saveLocalLlmApiKey(e.target.value)} + /> +
+ Leave blank unless your local server requires a token +
+
+ +
+ + this._saveLocalSttUrl(e.target.value)} + class=${this._keyError && !this._localSttUrl.trim() ? "error" : ""} + /> +
+ External streaming STT service that accepts 16 kHz mono PCM + ${!sttIsLocal + ? html` + · not a localhost URL + ` : ""}
- - ${this._whisperModel === "__custom__" - ? html` - this._saveCustomWhisperModel(e.target.value)} - @input=${(e) => { - this._customWhisperModel = e.target.value; - }} - style="margin-top: 6px;" - /> -
- Enter a HuggingFace model ID compatible with - @huggingface/transformers speech-to-text pipeline -
- ` - : html` -
- ${this.whisperDownloading - ? "Downloading model..." - : "Downloaded automatically on first use"} -
- `} - ${this.whisperDownloading && this.whisperProgress - ? this._renderWhisperProgress() - : ""} +
+ +
+ + this._saveLocalSttLanguage(e.target.value)} + /> +
+ First local sidecar target is English streaming ASR +
${this._renderStartButton()} @@ -1458,74 +1508,51 @@ export class MainView extends LitElement { return html`
-
What is Ollama?
+
LM Studio
- Ollama lets you run large language models locally on your machine. - Everything stays on your computer — no data leaves your device. + LM Studio runs the answer model locally and exposes an + OpenAI-compatible server for streaming responses.
-
Install Ollama
+
Start LM Studio server
- Download from - this.onExternalLink("https://ollama.com/download")} - >ollama.com/download - and install it. + Download LM Studio, load a model, then start the local server from + the Developer tab. The default endpoint is: +
+ http://127.0.0.1:1234/v1 +
+ +
+
Model ID
+
+ Enter the exact model identifier shown by LM Studio. Gemma 4 is the + reference target, but any loaded compatible model can be used.
-
Ollama must be running
+
Nemotron ASR sidecar
- Ollama needs to be running before you start a session. If it's not - running, open your terminal and type: + Speech-to-text runs as a separate local streaming service. The app + connects to:
- ollama serve + ws://127.0.0.1:8765/v1/asr/stream
-
Pull a model
+
Screenshots
- Download a model before first use: -
- ollama pull gemma3:4b -
- -
-
Recommended models
-
-
- gemma3:4b4B — fast, multimodal (images + text) -
-
- mistral-small8B — solid all-rounder, text only -
-
-
- gemma3:4b and above supports images — screenshots will work with - these models. + Manual screenshots are sent to the same local LM Studio model. Use a + vision-capable model for screen analysis.
- Avoid "thinking" models (e.g. deepseek-r1, qwq). Local inference is - already slower — a thinking model adds extra delay before - responding. -
-
- -
-
Whisper
-
- The Whisper speech-to-text model is downloaded automatically the - first time you start a session. This is a one-time download. + Non-local endpoints are allowed, but they may send audio transcripts + or screenshots outside this machine.
diff --git a/src/storage.js b/src/storage.js index 231457a..b12d7ae 100644 --- a/src/storage.js +++ b/src/storage.js @@ -30,7 +30,15 @@ const DEFAULT_PREFERENCES = { fontSize: "medium", backgroundTransparency: 0.8, googleSearchEnabled: false, + providerMode: "local", responseProvider: "gemini", + llmProvider: "lmstudio", + sttProvider: "nemotron-sidecar", + localLlmBaseUrl: "http://127.0.0.1:1234/v1", + localLlmModel: "", + localLlmApiKey: "", + localSttUrl: "ws://127.0.0.1:8765/v1/asr/stream", + localSttLanguage: "en-US", ollamaHost: "http://127.0.0.1:11434", ollamaModel: "llama3.1", whisperModel: "Xenova/whisper-small", diff --git a/src/utils/gemini.js b/src/utils/gemini.js index 7274853..44c3d3c 100644 --- a/src/utils/gemini.js +++ b/src/utils/gemini.js @@ -1073,7 +1073,7 @@ function setupGeminiIpcHandlers(geminiSessionRef) { "initialize-local", async ( event, - ollamaHost, + localConfigOrOllamaHost, ollamaModel, whisperModel, profile, @@ -1081,7 +1081,7 @@ function setupGeminiIpcHandlers(geminiSessionRef) { ) => { currentProviderMode = "local"; const success = await getLocalAi().initializeLocalSession( - ollamaHost, + localConfigOrOllamaHost, ollamaModel, whisperModel, profile, diff --git a/src/utils/localProviders.js b/src/utils/localProviders.js new file mode 100644 index 0000000..131a6fe --- /dev/null +++ b/src/utils/localProviders.js @@ -0,0 +1,304 @@ +const { EventEmitter } = require("events"); +const WebSocket = require("ws"); + +const DEFAULT_LOCAL_LLM_BASE_URL = "http://127.0.0.1:1234/v1"; +const DEFAULT_LOCAL_STT_URL = "ws://127.0.0.1:8765/v1/asr/stream"; +const DEFAULT_LOCAL_STT_LANGUAGE = "en-US"; + +function normalizeOpenAiBaseUrl(baseUrl) { + const trimmed = (baseUrl || DEFAULT_LOCAL_LLM_BASE_URL) + .trim() + .replace(/\/+$/, ""); + if (!trimmed) return DEFAULT_LOCAL_LLM_BASE_URL; + return trimmed.endsWith("/v1") ? trimmed : `${trimmed}/v1`; +} + +function isLoopbackUrl(value) { + try { + const url = new URL(value); + const hostname = url.hostname.toLowerCase(); + return ( + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname === "::1" || + hostname === "[::1]" + ); + } catch (_) { + return false; + } +} + +function buildChatMessages({ + systemPrompt, + history = [], + userText, + imageBase64, +}) { + const messages = []; + + if (systemPrompt && systemPrompt.trim()) { + messages.push({ role: "system", content: systemPrompt.trim() }); + } + + for (const message of history) { + if (!message || !message.role || !message.content) continue; + if ( + message.role !== "user" && + message.role !== "assistant" && + message.role !== "system" + ) + continue; + messages.push({ + role: message.role, + content: String(message.content), + }); + } + + const text = (userText || "").trim(); + if (imageBase64) { + messages.push({ + role: "user", + content: [ + { type: "text", text }, + { + type: "image_url", + image_url: { url: `data:image/jpeg;base64,${imageBase64}` }, + }, + ], + }); + } else if (text) { + messages.push({ role: "user", content: text }); + } + + return messages; +} + +function parseChatCompletionSseLine(line) { + if (!line.startsWith("data: ")) return null; + + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") return null; + + const parsed = JSON.parse(data); + return parsed.choices?.[0]?.delta?.content || ""; +} + +async function streamLmStudioChat({ + baseUrl, + apiKey, + model, + messages, + temperature = 0.7, + maxTokens = 2048, + onToken, +}) { + if (!model || !model.trim()) { + throw new Error("LM Studio model id is required"); + } + + const normalizedBaseUrl = normalizeOpenAiBaseUrl(baseUrl); + const response = await fetch(`${normalizedBaseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(apiKey && apiKey.trim() + ? { Authorization: `Bearer ${apiKey.trim()}` } + : {}), + }, + body: JSON.stringify({ + model: model.trim(), + messages, + stream: true, + temperature, + max_tokens: maxTokens, + }), + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => ""); + if (response.status === 400 && /image|vision|multimodal/i.test(errorText)) { + throw new Error( + "The selected local model does not appear to support image input", + ); + } + throw new Error( + `LM Studio error ${response.status}: ${errorText || response.statusText}`, + ); + } + + if (!response.body) { + throw new Error("LM Studio response did not include a stream body"); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + let fullText = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() || ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + + let token = ""; + try { + token = parseChatCompletionSseLine(trimmed) || ""; + } catch (_) { + continue; + } + + if (token) { + fullText += token; + if (onToken) onToken(token, fullText); + } + } + } + + return fullText; +} + +class NemotronSidecarClient extends EventEmitter { + constructor({ + url = DEFAULT_LOCAL_STT_URL, + language = DEFAULT_LOCAL_STT_LANGUAGE, + sampleRate = 16000, + channels = 1, + encoding = "pcm_s16le", + } = {}) { + super(); + this.url = url; + this.language = language; + this.sampleRate = sampleRate; + this.channels = channels; + this.encoding = encoding; + this.socket = null; + this.connected = false; + } + + connect() { + if (this.connected && this.socket?.readyState === WebSocket.OPEN) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const socket = new WebSocket(this.url); + this.socket = socket; + + const cleanup = () => { + socket.removeListener("open", onOpen); + socket.removeListener("error", onErrorBeforeOpen); + }; + + const onOpen = () => { + cleanup(); + this.connected = true; + this.emit("connected"); + this._sendStart(); + resolve(); + }; + + const onErrorBeforeOpen = (error) => { + cleanup(); + this.connected = false; + this.emit("error", error); + reject(error); + }; + + socket.once("open", onOpen); + socket.once("error", onErrorBeforeOpen); + + socket.on("message", (data) => this._handleMessage(data)); + socket.on("close", (code, reason) => { + this.connected = false; + this.emit("close", { code, reason: reason?.toString?.() || "" }); + }); + socket.on("error", (error) => { + this.connected = false; + this.emit("error", error); + }); + }); + } + + _sendStart() { + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) return; + this.socket.send( + JSON.stringify({ + type: "start", + sampleRate: this.sampleRate, + channels: this.channels, + encoding: this.encoding, + language: this.language, + }), + ); + } + + _handleMessage(data) { + let message; + try { + message = JSON.parse(data.toString("utf8")); + } catch (error) { + this.emit( + "error", + new Error(`Invalid ASR sidecar message: ${error.message}`), + ); + return; + } + + if (!message || !message.type) return; + + if (message.type === "partial") { + this.emit("partial", message.text || ""); + } else if (message.type === "final") { + this.emit("final", message.text || ""); + } else if (message.type === "ready") { + this.emit("ready", message); + } else if (message.type === "error") { + this.emit( + "error", + new Error(message.error || message.message || "ASR sidecar error"), + ); + } else { + this.emit(message.type, message); + } + } + + sendAudio(pcm16kBuffer) { + if (!this.socket || this.socket.readyState !== WebSocket.OPEN) return false; + if (!pcm16kBuffer || pcm16kBuffer.length === 0) return false; + this.socket.send(pcm16kBuffer, { binary: true }); + return true; + } + + close() { + if (!this.socket) return; + try { + if (this.socket.readyState === WebSocket.OPEN) { + this.socket.send(JSON.stringify({ type: "stop" })); + } + this.socket.close(); + } catch (_) { + // Best-effort close. + } + this.connected = false; + this.socket = null; + } +} + +module.exports = { + DEFAULT_LOCAL_LLM_BASE_URL, + DEFAULT_LOCAL_STT_LANGUAGE, + DEFAULT_LOCAL_STT_URL, + NemotronSidecarClient, + buildChatMessages, + isLoopbackUrl, + normalizeOpenAiBaseUrl, + parseChatCompletionSseLine, + streamLmStudioChat, +}; diff --git a/src/utils/localai.js b/src/utils/localai.js index 7f99244..1d30790 100644 --- a/src/utils/localai.js +++ b/src/utils/localai.js @@ -5,6 +5,16 @@ const { initializeNewSession, saveConversationTurn, } = require("./gemini"); +const { + DEFAULT_LOCAL_LLM_BASE_URL, + DEFAULT_LOCAL_STT_LANGUAGE, + DEFAULT_LOCAL_STT_URL, + NemotronSidecarClient, + buildChatMessages, + isLoopbackUrl, + normalizeOpenAiBaseUrl, + streamLmStudioChat, +} = require("./localProviders"); const { fork } = require("child_process"); const path = require("path"); const { getSystemNode } = require("./nodeDetect"); @@ -19,6 +29,8 @@ let whisperReady = false; let localConversationHistory = []; let currentSystemPrompt = null; let isLocalActive = false; +let localConfig = null; +let nemotronClient = null; // Set when we intentionally kill the worker to suppress crash handling let whisperShuttingDown = false; @@ -63,6 +75,58 @@ const MAX_SPEECH_BUFFER_BYTES = 16000 * 2 * 30; // 960,000 bytes // Audio resampling buffer let resampleRemainder = Buffer.alloc(0); +function resolveLocalSessionConfig( + configOrHost, + model, + whisperModel, + profile, + customPrompt, +) { + if (configOrHost && typeof configOrHost === "object") { + return { + llmProvider: configOrHost.llmProvider || "lmstudio", + sttProvider: configOrHost.sttProvider || "nemotron-sidecar", + localLlmBaseUrl: + configOrHost.localLlmBaseUrl || DEFAULT_LOCAL_LLM_BASE_URL, + localLlmModel: configOrHost.localLlmModel || "", + localLlmApiKey: configOrHost.localLlmApiKey || "", + localSttUrl: configOrHost.localSttUrl || DEFAULT_LOCAL_STT_URL, + localSttLanguage: + configOrHost.localSttLanguage || DEFAULT_LOCAL_STT_LANGUAGE, + ollamaHost: configOrHost.ollamaHost || "http://127.0.0.1:11434", + ollamaModel: configOrHost.ollamaModel || "llama3.1", + whisperModel: configOrHost.whisperModel || "Xenova/whisper-small", + profile: configOrHost.profile || profile || "interview", + customPrompt: configOrHost.customPrompt || customPrompt || "", + }; + } + + return { + llmProvider: "ollama", + sttProvider: "whisper", + localLlmBaseUrl: DEFAULT_LOCAL_LLM_BASE_URL, + localLlmModel: "", + localLlmApiKey: "", + localSttUrl: DEFAULT_LOCAL_STT_URL, + localSttLanguage: DEFAULT_LOCAL_STT_LANGUAGE, + ollamaHost: configOrHost || "http://127.0.0.1:11434", + ollamaModel: model || "llama3.1", + whisperModel: whisperModel || "Xenova/whisper-small", + profile: profile || "interview", + customPrompt: customPrompt || "", + }; +} + +function stripThinkingTags(text) { + return (text || "").replace(/[\s\S]*?<\/think>/g, "").trim(); +} + +function trimLocalHistory(maxMessages = 40) { + if (localConversationHistory.length > maxMessages) { + localConversationHistory = localConversationHistory.slice(-maxMessages); + } +} + // ── Audio Resampling (24kHz → 16kHz) ── function resample24kTo16k(inputBuffer) { @@ -550,6 +614,66 @@ async function transcribeAudio(pcm16kBuffer) { }); } +// ── Nemotron Sidecar Streaming STT ── + +async function connectNemotronSidecar(config) { + closeNemotronSidecar(); + + nemotronClient = new NemotronSidecarClient({ + url: config.localSttUrl, + language: config.localSttLanguage, + }); + + nemotronClient.on("connected", () => { + sendToRenderer("update-status", "ASR sidecar connected"); + }); + + nemotronClient.on("ready", () => { + console.log("[LocalAI] Nemotron sidecar ready"); + sendToRenderer("update-status", "ASR sidecar ready - Listening..."); + }); + + nemotronClient.on("partial", (text) => { + if (!text || !text.trim()) return; + sendToRenderer( + "update-status", + "Transcribing... " + text.trim().slice(-80), + ); + }); + + nemotronClient.on("final", (text) => { + const transcription = (text || "").trim(); + if (!transcription) return; + sendToRenderer("update-status", "Generating response..."); + handleFinalTranscription(transcription).catch((error) => { + console.error("[LocalAI] Final transcript handler error:", error); + sendToRenderer("update-status", "Local AI error: " + error.message); + }); + }); + + nemotronClient.on("close", ({ code }) => { + if (!isLocalActive) return; + console.warn("[LocalAI] Nemotron sidecar disconnected:", code); + sendToRenderer("update-status", "ASR sidecar disconnected"); + }); + + nemotronClient.on("error", (error) => { + console.error("[LocalAI] Nemotron sidecar error:", error); + sendToRenderer("update-status", "ASR sidecar error: " + error.message); + }); + + sendToRenderer("update-status", "Connecting to ASR sidecar..."); + await nemotronClient.connect(); +} + +function closeNemotronSidecar() { + if (nemotronClient) { + nemotronClient.removeAllListeners(); + nemotronClient.close(); + nemotronClient = null; + } +} + // ── Speech End Handler ── async function handleSpeechEnd(audioData) { @@ -578,7 +702,7 @@ async function handleSpeechEnd(audioData) { } sendToRenderer("update-status", "Generating response..."); - await sendToOllama(transcription); + await handleFinalTranscription(transcription); } catch (error) { console.error("[LocalAI] handleSpeechEnd error:", error); sendToRenderer( @@ -588,6 +712,144 @@ async function handleSpeechEnd(audioData) { } } +async function handleFinalTranscription(transcription) { + if (!localConfig) { + await sendToOllama(transcription); + return; + } + + if (localConfig.llmProvider === "lmstudio") { + await sendToLmStudio(transcription); + return; + } + + await sendToOllama(transcription); +} + +// ── LM Studio Chat (OpenAI-compatible) ── + +function getLmStudioConfig() { + return { + baseUrl: normalizeOpenAiBaseUrl( + localConfig?.localLlmBaseUrl || DEFAULT_LOCAL_LLM_BASE_URL, + ), + apiKey: localConfig?.localLlmApiKey || "", + model: localConfig?.localLlmModel || "", + }; +} + +async function verifyLmStudioConnection(config) { + if (!config.localLlmModel || !config.localLlmModel.trim()) { + sendToRenderer("update-status", "LM Studio model id is required"); + return false; + } + + const baseUrl = normalizeOpenAiBaseUrl(config.localLlmBaseUrl); + + if (!isLoopbackUrl(baseUrl)) { + sendToRenderer( + "update-status", + "Warning: LM Studio endpoint is not localhost", + ); + } + + try { + const response = await fetch(`${baseUrl}/models`, { + headers: { + ...(config.localLlmApiKey && config.localLlmApiKey.trim() + ? { Authorization: `Bearer ${config.localLlmApiKey.trim()}` } + : {}), + }, + }); + + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`HTTP ${response.status}${text ? ": " + text : ""}`); + } + + console.log("[LocalAI] LM Studio connection verified"); + sendToRenderer("update-status", "LM Studio reachable"); + return true; + } catch (error) { + console.error("[LocalAI] Cannot connect to LM Studio:", error); + sendToRenderer( + "update-status", + "Cannot connect to LM Studio: " + error.message, + ); + return false; + } +} + +async function sendToLmStudio(userText, imageBase64 = null) { + const config = getLmStudioConfig(); + if (!config.model || !config.model.trim()) { + sendToRenderer("update-status", "LM Studio model id is required"); + return { success: false, error: "LM Studio model id is required" }; + } + + if (!userText || !userText.trim()) { + return { success: false, error: "Empty prompt" }; + } + + const previousHistory = localConversationHistory.slice(-20); + const messages = buildChatMessages({ + systemPrompt: currentSystemPrompt || "You are a helpful assistant.", + history: previousHistory, + userText: userText.trim(), + imageBase64, + }); + + localConversationHistory.push({ + role: "user", + content: userText.trim(), + }); + trimLocalHistory(); + + try { + console.log( + `[LocalAI] Sending to LM Studio (${config.model}):`, + userText.substring(0, 100) + "...", + ); + + let isFirst = true; + const fullText = await streamLmStudioChat({ + baseUrl: config.baseUrl, + apiKey: config.apiKey, + model: config.model, + messages, + onToken: (_token, accumulated) => { + const cleaned = stripThinkingTags(accumulated); + if (!cleaned) return; + sendToRenderer(isFirst ? "new-response" : "update-response", cleaned); + isFirst = false; + }, + }); + + const cleanedResponse = stripThinkingTags(fullText); + if (cleanedResponse && cleanedResponse !== fullText) { + sendToRenderer("update-response", cleanedResponse); + } + + if (cleanedResponse) { + localConversationHistory.push({ + role: "assistant", + content: cleanedResponse, + }); + trimLocalHistory(); + + saveConversationTurn(userText, cleanedResponse); + } + + console.log("[LocalAI] LM Studio response completed"); + sendToRenderer("update-status", "Listening..."); + return { success: true, text: cleanedResponse, model: config.model }; + } catch (error) { + console.error("[LocalAI] LM Studio error:", error); + sendToRenderer("update-status", "LM Studio error: " + error.message); + return { success: false, error: error.message }; + } +} + // ── Ollama Chat ── async function sendToOllama(transcription) { @@ -658,53 +920,95 @@ async function sendToOllama(transcription) { // ── Public API ── async function initializeLocalSession( - ollamaHost, + configOrOllamaHost, model, whisperModel, profile, customPrompt, ) { - console.log("[LocalAI] Initializing local session:", { - ollamaHost, + const config = resolveLocalSessionConfig( + configOrOllamaHost, model, whisperModel, profile, + customPrompt, + ); + + console.log("[LocalAI] Initializing local session:", { + llmProvider: config.llmProvider, + sttProvider: config.sttProvider, + localLlmBaseUrl: config.localLlmBaseUrl, + localLlmModel: config.localLlmModel, + localSttUrl: config.localSttUrl, + profile: config.profile, }); sendToRenderer("session-initializing", true); try { + closeNemotronSidecar(); + isLocalActive = false; + // Setup system prompt - currentSystemPrompt = getSystemPrompt(profile, customPrompt, false); + currentSystemPrompt = getSystemPrompt( + config.profile, + config.customPrompt, + false, + ); - // Initialize Ollama client - ollamaClient = new Ollama({ host: ollamaHost }); - ollamaModel = model; + localConfig = config; + ollamaClient = null; + ollamaModel = null; - // Test Ollama connection - try { - await ollamaClient.list(); - console.log("[LocalAI] Ollama connection verified"); - } catch (error) { - console.error( - "[LocalAI] Cannot connect to Ollama at", - ollamaHost, - ":", - error.message, - ); - sendToRenderer("session-initializing", false); - sendToRenderer( - "update-status", - "Cannot connect to Ollama at " + ollamaHost, - ); - return false; + if (config.llmProvider === "lmstudio") { + const lmStudioReady = await verifyLmStudioConnection(config); + if (!lmStudioReady) { + sendToRenderer("session-initializing", false); + return false; + } + } else { + // Initialize Ollama client fallback + ollamaClient = new Ollama({ host: config.ollamaHost }); + ollamaModel = config.ollamaModel; + + try { + await ollamaClient.list(); + console.log("[LocalAI] Ollama connection verified"); + } catch (error) { + console.error( + "[LocalAI] Cannot connect to Ollama at", + config.ollamaHost, + ":", + error.message, + ); + sendToRenderer("session-initializing", false); + sendToRenderer( + "update-status", + "Cannot connect to Ollama at " + config.ollamaHost, + ); + return false; + } } - // Load Whisper model - const pipeline = await loadWhisperPipeline(whisperModel); - if (!pipeline) { - sendToRenderer("session-initializing", false); - return false; + if (config.sttProvider === "nemotron-sidecar") { + try { + await connectNemotronSidecar(config); + } catch (error) { + console.error("[LocalAI] Cannot connect to ASR sidecar:", error); + sendToRenderer("session-initializing", false); + sendToRenderer( + "update-status", + "Cannot connect to ASR sidecar: " + error.message, + ); + return false; + } + } else { + // Load Whisper model fallback + const pipeline = await loadWhisperPipeline(config.whisperModel); + if (!pipeline) { + sendToRenderer("session-initializing", false); + return false; + } } // Reset VAD state @@ -716,7 +1020,7 @@ async function initializeLocalSession( localConversationHistory = []; // Initialize conversation session - initializeNewSession(profile, customPrompt); + initializeNewSession(config.profile, config.customPrompt); isLocalActive = true; sendToRenderer("session-initializing", false); @@ -737,14 +1041,22 @@ function processLocalAudio(monoChunk24k) { // Resample from 24kHz to 16kHz const pcm16k = resample24kTo16k(monoChunk24k); - if (pcm16k.length > 0) { - processVAD(pcm16k); + if (pcm16k.length === 0) return; + + if (localConfig?.sttProvider === "nemotron-sidecar") { + if (!nemotronClient || !nemotronClient.sendAudio(pcm16k)) { + sendToRenderer("update-status", "ASR sidecar is not connected"); + } + return; } + + processVAD(pcm16k); } function closeLocalSession() { console.log("[LocalAI] Closing local session"); isLocalActive = false; + closeNemotronSidecar(); isSpeaking = false; speechBuffers = []; silenceFrameCount = 0; @@ -753,6 +1065,7 @@ function closeLocalSession() { localConversationHistory = []; ollamaClient = null; ollamaModel = null; + localConfig = null; currentSystemPrompt = null; // Note: whisperWorker is kept alive to avoid reloading model on next session // To fully clean up, call killWhisperWorker() @@ -762,14 +1075,17 @@ function isLocalSessionActive() { return isLocalActive; } -// ── Send text directly to Ollama (for manual text input) ── +// ── Send text directly to the active local LLM ── async function sendLocalText(text) { - if (!isLocalActive || !ollamaClient) { + if (!isLocalActive) { return { success: false, error: "No active local session" }; } try { + if (localConfig?.llmProvider === "lmstudio") { + return await sendToLmStudio(text); + } await sendToOllama(text); return { success: true }; } catch (error) { @@ -778,10 +1094,19 @@ async function sendLocalText(text) { } async function sendLocalImage(base64Data, prompt) { - if (!isLocalActive || !ollamaClient) { + if (!isLocalActive) { return { success: false, error: "No active local session" }; } + if (localConfig?.llmProvider === "lmstudio") { + sendToRenderer("update-status", "Analyzing image locally..."); + return await sendToLmStudio(prompt, base64Data); + } + + if (!ollamaClient) { + return { success: false, error: "No active Ollama session" }; + } + try { console.log("[LocalAI] Sending image to Ollama"); sendToRenderer("update-status", "Analyzing image..."); diff --git a/src/utils/renderer.js b/src/utils/renderer.js index 8336c24..c00182c 100644 --- a/src/utils/renderer.js +++ b/src/utils/renderer.js @@ -177,19 +177,22 @@ async function initializeGemini(profile = "interview", language = "en-US") { async function initializeLocal(profile = "interview") { const prefs = await storage.getPreferences(); - const ollamaHost = prefs.ollamaHost || "http://127.0.0.1:11434"; - const ollamaModel = prefs.ollamaModel || "llama3.1"; - const whisperModel = prefs.whisperModel || "Xenova/whisper-small"; - const customPrompt = prefs.customPrompt || ""; - - const success = await ipcRenderer.invoke( - "initialize-local", - ollamaHost, - ollamaModel, - whisperModel, + const localConfig = { + llmProvider: prefs.llmProvider || "lmstudio", + sttProvider: prefs.sttProvider || "nemotron-sidecar", + localLlmBaseUrl: prefs.localLlmBaseUrl || "http://127.0.0.1:1234/v1", + localLlmModel: prefs.localLlmModel || "", + localLlmApiKey: prefs.localLlmApiKey || "", + localSttUrl: prefs.localSttUrl || "ws://127.0.0.1:8765/v1/asr/stream", + localSttLanguage: prefs.localSttLanguage || "en-US", + ollamaHost: prefs.ollamaHost || "http://127.0.0.1:11434", + ollamaModel: prefs.ollamaModel || "llama3.1", + whisperModel: prefs.whisperModel || "Xenova/whisper-small", profile, - customPrompt, - ); + customPrompt: prefs.customPrompt || "", + }; + + const success = await ipcRenderer.invoke("initialize-local", localConfig); if (success) { cheatingDaddy.setStatus("Local AI Live"); return true; @@ -1090,7 +1093,10 @@ const theme = { // Determine if theme is light or dark const lightThemes = ["light", "sepia"]; const isLightTheme = lightThemes.includes(themeName); - document.body.setAttribute("data-theme-type", isLightTheme ? "light" : "dark"); + document.body.setAttribute( + "data-theme-type", + isLightTheme ? "light" : "dark", + ); // New design tokens (used by components) root.style.setProperty("--text-primary", colors.text); diff --git a/test/localProviders.test.js b/test/localProviders.test.js new file mode 100644 index 0000000..12881ae --- /dev/null +++ b/test/localProviders.test.js @@ -0,0 +1,151 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const http = require("node:http"); +const WebSocket = require("ws"); + +const { + buildChatMessages, + isLoopbackUrl, + NemotronSidecarClient, + normalizeOpenAiBaseUrl, + streamLmStudioChat, +} = require("../src/utils/localProviders"); + +test("normalizes LM Studio OpenAI-compatible base URLs without selecting models automatically", () => { + assert.equal( + normalizeOpenAiBaseUrl("http://127.0.0.1:1234"), + "http://127.0.0.1:1234/v1", + ); + assert.equal( + normalizeOpenAiBaseUrl("http://127.0.0.1:1234/v1/"), + "http://127.0.0.1:1234/v1", + ); +}); + +test("detects loopback URLs but does not reject non-local URLs", () => { + assert.equal(isLoopbackUrl("http://localhost:1234/v1"), true); + assert.equal(isLoopbackUrl("http://127.0.0.1:1234/v1"), true); + assert.equal(isLoopbackUrl("http://[::1]:1234/v1"), true); + assert.equal(isLoopbackUrl("http://192.168.1.40:1234/v1"), false); + assert.equal(isLoopbackUrl("https://example.com/v1"), false); +}); + +test("builds OpenAI-compatible image messages for local screenshot analysis", () => { + const messages = buildChatMessages({ + systemPrompt: "Be useful.", + history: [{ role: "assistant", content: "Previous answer" }], + userText: "Analyze this screen", + imageBase64: "abc123", + }); + + assert.deepEqual(messages[0], { role: "system", content: "Be useful." }); + assert.equal(messages[1].role, "assistant"); + assert.equal(messages[2].role, "user"); + assert.equal(messages[2].content[0].type, "text"); + assert.equal(messages[2].content[1].type, "image_url"); + assert.equal( + messages[2].content[1].image_url.url, + "data:image/jpeg;base64,abc123", + ); +}); + +test("requires a manually configured LM Studio model id", async () => { + await assert.rejects( + () => + streamLmStudioChat({ + baseUrl: "http://127.0.0.1:1234/v1", + model: "", + messages: [{ role: "user", content: "hello" }], + }), + /model id is required/, + ); +}); + +test("streams LM Studio chat completion tokens from an OpenAI-compatible endpoint", async () => { + let receivedBody = null; + const server = http.createServer((req, res) => { + assert.equal(req.method, "POST"); + assert.equal(req.url, "/v1/chat/completions"); + + let raw = ""; + req.setEncoding("utf8"); + req.on("data", (chunk) => { + raw += chunk; + }); + req.on("end", () => { + receivedBody = JSON.parse(raw); + res.writeHead(200, { + "Content-Type": "text/event-stream", + }); + res.write('data: {"choices":[{"delta":{"content":"hel"}}]}\n\n'); + res.write('data: {"choices":[{"delta":{"content":"lo"}}]}\n\n'); + res.end("data: [DONE]\n\n"); + }); + }); + + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address(); + const accumulations = []; + + const fullText = await streamLmStudioChat({ + baseUrl: `http://127.0.0.1:${port}/v1`, + model: "manual-gemma4", + messages: [{ role: "user", content: "hello" }], + onToken: (_token, accumulated) => accumulations.push(accumulated), + }); + + assert.equal(fullText, "hello"); + assert.deepEqual(accumulations, ["hel", "hello"]); + assert.equal(receivedBody.model, "manual-gemma4"); + assert.equal(receivedBody.stream, true); + assert.deepEqual(receivedBody.messages, [{ role: "user", content: "hello" }]); + + await new Promise((resolve) => server.close(resolve)); +}); + +test("Nemotron sidecar client sends start JSON, binary PCM, and emits final transcripts", async () => { + const server = new WebSocket.Server({ host: "127.0.0.1", port: 0 }); + await new Promise((resolve) => server.once("listening", resolve)); + + const { port } = server.address(); + const received = []; + + server.on("connection", (socket) => { + socket.on("message", (data, isBinary) => { + if (isBinary) { + received.push({ isBinary, data: Buffer.from(data) }); + socket.send(JSON.stringify({ type: "final", text: "hello world" })); + return; + } + + received.push({ isBinary, data: JSON.parse(data.toString("utf8")) }); + socket.send(JSON.stringify({ type: "ready" })); + }); + }); + + const client = new NemotronSidecarClient({ + url: `ws://127.0.0.1:${port}/v1/asr/stream`, + language: "en-US", + }); + + const finalPromise = new Promise((resolve) => client.once("final", resolve)); + await client.connect(); + client.sendAudio(Buffer.from([1, 2, 3, 4])); + + assert.equal(await finalPromise, "hello world"); + assert.deepEqual(received[0], { + isBinary: false, + data: { + type: "start", + sampleRate: 16000, + channels: 1, + encoding: "pcm_s16le", + language: "en-US", + }, + }); + assert.equal(received[1].isBinary, true); + assert.deepEqual([...received[1].data], [1, 2, 3, 4]); + + client.close(); + await new Promise((resolve) => server.close(resolve)); +});