From 5b2bdaae759ef5dc36b90d404fec991eb5b011ac Mon Sep 17 00:00:00 2001 From: esrad Date: Thu, 8 Jan 2026 14:05:57 +0100 Subject: [PATCH 1/2] Local changes to voice device --- packages/voice-device/public/app.js | 60 +++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/packages/voice-device/public/app.js b/packages/voice-device/public/app.js index 82be74a..295986c 100644 --- a/packages/voice-device/public/app.js +++ b/packages/voice-device/public/app.js @@ -23,6 +23,14 @@ let lastAutoListenAttempt = 0; let userGrantedMic = false; let unloadHandlerRegistered = false; let currentPromptSuggestions = []; +let hasInteracted = false; +let lastSpokenInstruction = ''; +let isAwaitingFirstInteraction = false; + +let resolveFirstUiUpdate; +const firstUiUpdatePromise = new Promise((resolve) => { + resolveFirstUiUpdate = resolve; +}); const AUTO_LISTEN_COOLDOWN_MS = 8000; const RECONNECT_DELAY_MS = 4000; @@ -37,6 +45,23 @@ const dockerInternalHostnames = new Set([ '0.0.0.0', ]); +const speak = (text) => { + if (!('speechSynthesis' in window)) { + console.warn('Speech synthesis not supported in this browser.'); + return; + } + if (!text) { + return; + } + try { + const utterance = new SpeechSynthesisUtterance(text); + utterance.lang = config?.recognitionLanguage || 'en-US'; + window.speechSynthesis.speak(utterance); + } catch (error) { + console.error('Error occured during speech synthesis:', error); + } +}; + const appendLog = (entry) => { const li = document.createElement('li'); const header = document.createElement('strong'); @@ -46,9 +71,11 @@ const appendLog = (entry) => { li.appendChild(header); li.appendChild(document.createElement('br')); li.appendChild(detail); - logList.prepend(li); + logList.prepend(li); }; + + const setListeningState = (listening) => { isListening = listening; listeningIndicator.textContent = listening ? 'active' : 'inactive'; @@ -142,6 +169,10 @@ const initialiseRecognition = () => { instance.onend = () => setListeningState(false); instance.onerror = (event) => { setListeningState(false); + if (event.error === 'no-speech') { + speak('Please try again.'); + return; + } if (event.error === 'not-allowed' && !userGrantedMic) { appendLog({ header: 'Microphone permission blocked', @@ -170,8 +201,20 @@ const initialiseRecognition = () => { }; const attachEventListeners = () => { - startBtn.addEventListener('click', () => { + startBtn.addEventListener('click', async () => { if (recognition && !isListening) { + if (!hasInteracted) { + isAwaitingFirstInteraction = true; + startBtn.disabled = true; + startBtn.textContent = 'Receiving instructions...'; + await firstUiUpdatePromise; + startBtn.textContent = 'Start Listening'; + + const initialGreeting = lastSpokenInstruction || `You can say things like: ${config.sampleCommands.join(', or ')}.`; + speak(initialGreeting); + hasInteracted = true; + isAwaitingFirstInteraction = false; + } recognition.start(); } }); @@ -267,7 +310,7 @@ const resolveWebsocketUrl = () => { }; const maybeAutoStartListening = (reason) => { - if (!recognition || isListening) { + if (!recognition || isListening || isAwaitingFirstInteraction) { return; } @@ -317,6 +360,17 @@ const handleUiUpdate = (payload) => { const suggestions = extractPromptSuggestions(payload.ui); renderPromptSuggestions(suggestions); + const components = payload.ui?.props?.children || []; + const textComponent = components.find((c) => c.component === 'text'); + if (textComponent?.props?.content) { + lastSpokenInstruction = textComponent.props.content; + } + + if (resolveFirstUiUpdate) { + resolveFirstUiUpdate(); + resolveFirstUiUpdate = null; + } + maybeAutoStartListening('Core system routed the latest UI to this voice device.'); }; From 18c2f699620a9bce7af433fd8dd2fa8d56ec33ce Mon Sep 17 00:00:00 2001 From: esrad Date: Thu, 8 Jan 2026 14:07:49 +0100 Subject: [PATCH 2/2] Save local changes before rebase --- docker-compose.yml | 10 +- packages/knowledge-base/index.js | 942 ++++++++++++++++++++++++- packages/knowledge-base/kb-data.json | 488 +++++++++++++ wireguard-config/coredns/Corefile | 6 + wireguard-config/templates/peer.conf | 11 + wireguard-config/templates/server.conf | 6 + 6 files changed, 1453 insertions(+), 10 deletions(-) create mode 100644 wireguard-config/coredns/Corefile create mode 100644 wireguard-config/templates/peer.conf create mode 100644 wireguard-config/templates/server.conf diff --git a/docker-compose.yml b/docker-compose.yml index 9aa7bc5..184c8af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: CORE_SYSTEM_PUBLIC_URL: http://core-system:3001 SERVICE_REGISTRY_PUBLIC_URL: http://core-system:3000 SERVICE_REGISTRY_URL: http://core-system:3000 - KNOWLEDGE_BASE_URL: http://knowledge-base:3005 + KNOWLEDGE_BASE_URL: http://knowledge-base:3010 BIND_ADDRESS: 0.0.0.0 ports: - "3001:3001" @@ -27,8 +27,8 @@ services: context: . dockerfile: packages/knowledge-base/Dockerfile environment: - KNOWLEDGE_BASE_PORT: 3005 - KNOWLEDGE_BASE_PUBLIC_URL: http://knowledge-base:3005 + KNOWLEDGE_BASE_PORT: 3010 + KNOWLEDGE_BASE_PUBLIC_URL: http://knowledge-base:3010 SERVICE_REGISTRY_URL: http://core-system:3000 LLM_ENDPOINT: ${LLM_ENDPOINT:-http://192.168.1.73:1234/v1/chat/completions} LLM_MODEL: ${LLM_MODEL:-gemma 3b} @@ -41,9 +41,9 @@ services: KNOWLEDGE_BASE_DATA_FILE: /data/kb-data.json BIND_ADDRESS: 0.0.0.0 ports: - - "3005:3005" + - "3010:3010" healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:3005/health"] + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:3010/health"] interval: 10s timeout: 5s retries: 5 diff --git a/packages/knowledge-base/index.js b/packages/knowledge-base/index.js index 40535c7..dd62429 100644 --- a/packages/knowledge-base/index.js +++ b/packages/knowledge-base/index.js @@ -10,11 +10,943 @@ const app = express(); app.use(express.json()); app.use(cors()); -app.use('/', router); +app.get('/health', (_req, res) => { + res.json({ status: 'ok', documents: documents.length }); +}); +const port = Number.parseInt(process.env.KNOWLEDGE_BASE_PORT || '3005', 10); +const listenAddress = process.env.BIND_ADDRESS || '0.0.0.0'; +const serviceRegistryUrl = process.env.SERVICE_REGISTRY_URL || 'http://localhost:3000'; +const knowledgeBasePublicUrl = process.env.KNOWLEDGE_BASE_PUBLIC_URL || `http://localhost:${port}`; +const llmEndpoint = process.env.LLM_ENDPOINT || 'http://localhost:1234/v1/chat/completions'; +const llmDefaultModel = process.env.LLM_MODEL || 'gemma 3b'; + +const openRouterApiKey = process.env.OPENROUTER_API_KEY || null; +const openRouterApiUrl = process.env.OPENROUTER_API_URL || 'https://openrouter.ai/api/v1/chat/completions'; +const openRouterModel = process.env.OPENROUTER_MODEL || null; +const openRouterReferer = process.env.OPENROUTER_APP_URL || process.env.OPENROUTER_REFERER || null; +const openRouterTitle = process.env.OPENROUTER_APP_NAME || process.env.OPENROUTER_TITLE || 'IMP Requirements KB'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +const outputSchema = JSON.parse(fs.readFileSync(path.join(__dirname, '../core-system/output.schema.json'), 'utf-8')); + +const deviceSelectionJsonSchema = { + $schema: 'http://json-schema.org/draft-07/schema#', + title: 'DeviceSelectionResponse', + type: 'object', + properties: { + targetDeviceId: { + type: 'string', + description: 'Identifier of the device that should receive the generated UI.', + }, + reason: { + type: 'string', + description: 'Natural language rationale explaining why the device was selected.', + }, + confidence: { + type: 'string', + enum: ['low', 'medium', 'high'], + description: 'Self-reported confidence in the selection.', + }, + alternateDeviceIds: { + type: 'array', + description: 'Optional list of fallback device identifiers in preference order.', + items: { type: 'string' }, + }, + requestedCapabilities: { + type: 'array', + items: { type: 'string' }, + description: 'Echo of the capability list considered for the selection.', + }, + considerations: { + type: 'array', + description: 'Bullet-point style list capturing the key criteria used when deciding.', + items: { type: 'string' }, + }, + }, + required: ['targetDeviceId', 'reason'], +}; + +const DATA_FILE = process.env.KNOWLEDGE_BASE_DATA_FILE + ? path.resolve(process.env.KNOWLEDGE_BASE_DATA_FILE) + : path.join(__dirname, 'kb-data.json'); + +const nowIsoString = () => new Date().toISOString(); + +const ensureDataFile = () => { + const dataDir = path.dirname(DATA_FILE); + if (!fs.existsSync(dataDir)) { + fs.mkdirSync(dataDir, { recursive: true }); + } + if (!fs.existsSync(DATA_FILE)) { + fs.writeFileSync(DATA_FILE, JSON.stringify({ documents: [] }, null, 2), 'utf-8'); + } +}; + +const registerWithServiceRegistry = async () => { + try { + await fetch(`${serviceRegistryUrl}/register`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: 'knowledge-base', + url: knowledgeBasePublicUrl, + type: 'generic', + metadata: { + service: 'knowledge-base', + description: 'RAG-powered requirement knowledge base with device selection support.', + }, + }), + }); + console.log('[KB] Registered with service registry.'); + } catch (error) { + console.error('[KB] Failed to register with service registry:', error.message); + } +}; + +const tokenize = (text = '') => + text + .toLowerCase() + .replace(/[^a-z0-9\s]/g, ' ') + .split(/\s+/) + .filter(Boolean); + +const buildTermFrequency = (tokens = []) => + tokens.reduce((acc, token) => { + acc[token] = (acc[token] || 0) + 1; + return acc; + }, {}); + +const cosineSimilarity = (vectorA, vectorB) => { + const uniqueTokens = new Set([...Object.keys(vectorA), ...Object.keys(vectorB)]); + let dotProduct = 0; + let magnitudeA = 0; + let magnitudeB = 0; + + uniqueTokens.forEach((token) => { + const a = vectorA[token] || 0; + const b = vectorB[token] || 0; + dotProduct += a * b; + magnitudeA += a * a; + magnitudeB += b * b; + }); + + if (magnitudeA === 0 || magnitudeB === 0) { + return 0; + } + + return dotProduct / (Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB)); +}; + +const loadDocuments = () => { + ensureDataFile(); + const content = fs.readFileSync(DATA_FILE, 'utf-8'); + const parsed = JSON.parse(content); + return Array.isArray(parsed.documents) ? parsed.documents : []; +}; + +const persistDocuments = (docs) => { + fs.writeFileSync(DATA_FILE, JSON.stringify({ documents: docs }, null, 2), 'utf-8'); +}; + +const invokeChatCompletion = async (requestBody, { contextLabel = 'llm-request' } = {}) => { + const effectiveModel = requestBody.model || openRouterModel || llmDefaultModel; + const basePayload = { ...requestBody, model: effectiveModel }; + + const callOpenRouter = async () => { + if (!openRouterApiKey) { + throw new Error('OpenRouter API key not configured.'); + } + + const headers = { + 'Content-Type': 'application/json', + Authorization: `Bearer ${openRouterApiKey}`, + }; + + if (openRouterReferer) { + headers['HTTP-Referer'] = openRouterReferer; + } + + if (openRouterTitle) { + headers['X-Title'] = openRouterTitle; + } + + console.log(`[KB] Invoking OpenRouter (${contextLabel}) with model ${basePayload.model}.`); + const response = await fetch(openRouterApiUrl, { + method: 'POST', + headers, + body: JSON.stringify(basePayload), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`OpenRouter responded with status ${response.status}: ${errorText}`); + } + + return response.json(); + }; + + const callLocalEndpoint = async () => { + if (!llmEndpoint) { + throw new Error('Local LLM endpoint not configured.'); + } + + console.log(`[KB] Invoking local LLM (${contextLabel}) at ${llmEndpoint} with model ${basePayload.model}.`); + const response = await fetch(llmEndpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(basePayload), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Local LLM responded with status ${response.status}: ${errorText}`); + } + + return response.json(); + }; + + if (openRouterApiKey) { + try { + const data = await callOpenRouter(); + return { data, provider: 'openrouter', model: basePayload.model }; + } catch (error) { + console.error(`[KB] OpenRouter request failed for ${contextLabel}:`, error.message); + if (!llmEndpoint) { + throw error; + } + } + } + + if (!llmEndpoint) { + throw new Error('No LLM endpoint available. Configure OPENROUTER_API_KEY or LLM_ENDPOINT.'); + } + + const data = await callLocalEndpoint(); + return { data, provider: 'local', model: basePayload.model }; +}; + +const documents = loadDocuments(); +let lastDeviceSelection = null; + +const addDocument = ({ id, content, metadata = {}, tags = [] }) => { + if (!content || typeof content !== 'string') { + throw new Error('Document `content` must be a non-empty string.'); + } + + const docId = id || `doc-${Date.now()}-${Math.round(Math.random() * 1e6)}`; + const tokens = tokenize(content); + const termFrequency = buildTermFrequency(tokens); + + const record = { + id: docId, + content, + metadata, + tags, + tokens, + termFrequency, + updatedAt: nowIsoString(), + createdAt: + documents.find((doc) => doc.id === docId)?.createdAt || nowIsoString(), + }; + + const existingIndex = documents.findIndex((doc) => doc.id === docId); + if (existingIndex > -1) { + documents[existingIndex] = record; + } else { + documents.push(record); + } + + persistDocuments(documents); + return record; +}; + +const seedDocuments = [ + { + id: 'modality-guideline-hands-occupied', + content: + 'When the user activity sensor reports the state "hands-occupied", prefer audio-first guidance. Provide spoken prompts and minimize the need for direct touch input. When the sensor reports "hands-free", present tactile controls such as buttons or toggles for the light switch.', + metadata: { + source: 'safety-guidelines', + version: '1.0.0', + }, + tags: ['modality', 'hands-occupied', 'audio', 'light-switch'], + }, + { + id: 'user-preference-primary-color', + content: + 'The primary household preference for interface accents is the color "#1F6FEB" (a vivid cobalt). Whenever possible, set the UI theme primary color to this value so buttons, toggles, and other interactive highlights align with the user preference. Ensure sufficient contrast by using light text on dark backgrounds.', + metadata: { + source: 'user-profile', + version: '2025.10', + }, + tags: ['preference', 'theme', 'primary-color', 'personalization'], + }, + { + id: 'component-selection-and-layout-guidelines', + content: + "UI Component Selection and Layout Guidelines:\n1. Control Selection Hierarchy: For any single binary state (e.g., a light being on/off), you MUST choose ONLY ONE of the following control types:\n a) A single 'toggle' or 'checkbox' component.\n b) A pair of 'button' components for explicit 'On' and 'Off' actions.\n NEVER generate both a toggle/checkbox and buttons for the same state control.\n2. Layout and Spacing: All interactive elements like buttons MUST have visible space between them. They must not touch. Group related controls inside a 'container' to provide structure.\n3. Button Labeling: Button labels must be concise and describe a single action (e.g., 'Turn On'). Avoid generic or redundant labels.", + metadata: { + source: 'design-system-best-practices', + version: '1.1', + }, + tags: ['guideline', 'ui-design', 'layout', 'component-selection'], + }, + { + id: 'redundant-controls-rule', + content: + 'UI Generation Rule: If a toggle or checkbox is present for a binary state, do not also generate separate "On" and "Off" buttons for that same state. This creates a redundant and confusing user interface.', + metadata: { + source: 'system-rules', + version: '1.0', + }, + tags: ['guideline', 'component-selection', 'redundancy'], + } +]; + +const seedKnowledgeBase = () => { + seedDocuments.forEach((doc) => { + if (!documents.some((entry) => entry.id === doc.id)) { + addDocument(doc); + console.log(`Seeded knowledge base document: ${doc.id}`); + } + }); +}; + +seedKnowledgeBase(); + +const retrieveRelevantDocuments = ({ prompt, thingDescription, capabilityData, capabilities, missingCapabilities, device, uiContext }) => { + if (!documents.length) return []; + + const querySegments = []; + + if (prompt) querySegments.push(prompt); + if (thingDescription) { + querySegments.push( + typeof thingDescription === 'string' ? thingDescription : JSON.stringify(thingDescription) + ); + } + if (capabilityData && Object.keys(capabilityData).length > 0) { + querySegments.push(JSON.stringify(capabilityData)); + } + if (Array.isArray(capabilities) && capabilities.length > 0) { + querySegments.push(`capabilities: ${capabilities.join(', ')}`); + } + if (Array.isArray(missingCapabilities) && missingCapabilities.length > 0) { + querySegments.push(`missing: ${missingCapabilities.join(', ')}`); + } + if (device) { + querySegments.push(JSON.stringify({ device })); + } + if (uiContext) { + querySegments.push(JSON.stringify(uiContext)); + } + + const query = querySegments.filter(Boolean).join('\n'); + const queryTokens = tokenize(query); + const queryVector = buildTermFrequency(queryTokens); + + if (Object.keys(queryVector).length === 0) { + return []; + } + + const scoredDocuments = documents + .map((doc) => ({ + score: cosineSimilarity(queryVector, doc.termFrequency || {}), + document: doc, + })) + .filter((entry) => entry.score > 0) + .sort((a, b) => b.score - a.score) + .slice(0, 5) + .map(({ score, document }) => ({ + ...document, + score, + })); + + return scoredDocuments; +}; + +const runDeviceSelection = async ({ + prompt, + fallbackPrompt, + desiredCapabilities = [], + thingDescription, + candidates = [], + model, + }) => { + if (!Array.isArray(candidates) || candidates.length === 0) { + throw new Error('No device candidates provided for selection.'); + } + + const candidateSummaries = candidates.map((candidate, index) => { + const supportedComponents = Array.isArray(candidate.metadata?.supportedUiComponents) + ? candidate.metadata.supportedUiComponents.join(', ') + : 'unspecified'; + + const capabilityScore = candidate.score || {}; + const matchSummary = capabilityScore.supportsAll + ? 'supports all requested capabilities' + : capabilityScore.matches || capabilityScore.missing + ? `matches ${capabilityScore.matches || 0}, missing ${(capabilityScore.missing || []).join(', ') || 'none'}` + : 'no score available'; + + const features = [ + `Capabilities: ${(candidate.capabilities || []).join(', ') || 'none'}`, + `Supported components: ${supportedComponents}`, + `Supports audio: ${candidate.metadata?.supportsAudio ? 'yes' : 'no'}`, + `Supports dictation: ${candidate.metadata?.supportsDictation ? 'yes' : 'no'}`, + `Supports theming: ${Array.isArray(candidate.metadata?.supportsTheming) ? candidate.metadata.supportsTheming.join(', ') : 'no'}`, + `Modality preference: ${candidate.metadata?.modalityPreference || 'unspecified'}`, + `Capability match: ${matchSummary}`, + ]; + + return `Candidate ${index + 1}: ${candidate.name} (${candidate.id})\n${features.join('\n')}`; + }).join('\n\n'); + + const retrievedDocuments = retrieveRelevantDocuments({ + prompt, + thingDescription, + capabilities: desiredCapabilities, + capabilityData: null, + missingCapabilities: null, + device: null, + uiContext: null, + }); + + const knowledgeContext = retrievedDocuments.length + ? retrievedDocuments.map((doc, index) => `Doc ${index + 1} (score ${doc.score.toFixed(3)}): ${doc.content}`).join('\n\n') + : null; + + const selectionMessages = [ + { + role: 'system', + content: 'You are a device orchestration planner. Choose the best device from the provided candidates to render the requested UI. Consider capability coverage, modality support, and any documented requirements. Respond strictly using the provided JSON schema.', + }, + { + role: 'system', + content: `Device candidates:\n${candidateSummaries}`, + }, + ]; + + if (knowledgeContext) { + selectionMessages.push({ + role: 'system', + content: `Supporting requirement documents:\n${knowledgeContext}`, + }); + } + + const selectionPayload = { + prompt, + fallbackPrompt, + desiredCapabilities, + thingDescription, + candidates, + }; + + selectionMessages.push({ + role: 'user', + content: JSON.stringify(selectionPayload, null, 2), + }); + + const resolvedModel = model || openRouterModel || llmDefaultModel; + const requestBody = { + model: resolvedModel, + messages: selectionMessages, + temperature: 0.3, + response_format: { + type: 'json_schema', + json_schema: { schema: deviceSelectionJsonSchema }, + }, + }; + + const { data } = await invokeChatCompletion(requestBody, { contextLabel: 'device-selection' }); + const selectionMessage = data?.choices?.[0]?.message; + if (!selectionMessage?.content) { + throw new Error('Device selection LLM returned an empty response.'); + } + + let parsed; + try { + parsed = typeof selectionMessage.content === 'string' + ? JSON.parse(selectionMessage.content) + : Array.isArray(selectionMessage.content) + ? selectionMessage.content[0] + : selectionMessage.content; + } catch (error) { + console.error('[KB] Failed to parse device selection content:', error); + throw new Error('Unable to parse device selection response.'); + } + + console.log(`[KB] Device selection chose '${parsed?.targetDeviceId || 'unknown'}' with confidence ${parsed?.confidence || 'unspecified'}'. Reason: ${parsed?.reason || 'n/a'}`); + + lastDeviceSelection = { + timestamp: nowIsoString(), + request: { + prompt, + fallbackPrompt, + desiredCapabilities, + thingDescription, + candidates, + model, + candidateSummaries, + knowledgeContext, + }, + response: parsed, + }; + + return parsed; + }; + +async function runAgent({ prompt, thingDescription, capabilities = [], uiSchema = {}, capabilityData, missingCapabilities, device, deviceId, selection }) { + const availableComponents = uiSchema.components || {}; + const availableComponentNames = Object.keys(availableComponents); + const availableTools = uiSchema.tools || {}; + const availableToolNames = Object.keys(availableTools); + + const filteredSchema = JSON.parse(JSON.stringify(outputSchema)); + + const retrievedDocuments = retrieveRelevantDocuments({ + prompt, + thingDescription, + capabilityData, + capabilities, + missingCapabilities, + device, + uiContext: uiSchema.context, + }); + + if (retrievedDocuments.length) { + console.log('Retrieved documents for context:', retrievedDocuments.map((doc) => `${doc.id} (score ${doc.score.toFixed(3)})`)); + } + + for (const key in filteredSchema.definitions) { + if (key.endsWith('Component') && !availableComponentNames.includes(key.replace('Component', ''))) { + delete filteredSchema.definitions[key]; + } + } + + if (filteredSchema.definitions.component?.oneOf) { + filteredSchema.definitions.component.oneOf = filteredSchema.definitions.component.oneOf.filter((ref) => { + const componentName = ref.$ref.split('/').pop().replace('Component', ''); + return availableComponentNames.includes(componentName) || componentName === 'toolCall'; + }); + } + + if (filteredSchema.definitions.toolCall) { + if (availableToolNames.length > 0) { + filteredSchema.definitions.toolCall.properties.tool.enum = availableToolNames; + } else { + delete filteredSchema.definitions.toolCall; + if (filteredSchema.definitions.component?.oneOf) { + filteredSchema.definitions.component.oneOf = filteredSchema.definitions.component.oneOf.filter((ref) => ref.$ref !== '#/definitions/toolCall'); + } + } + } + + const toolDefinitions = Object.entries(availableTools).map(([name, tool]) => { + const parameterSchema = tool && typeof tool.parameters === 'object' + ? tool.parameters + : { type: 'object', properties: {}, additionalProperties: false }; + + return { + type: 'function', + function: { + name, + description: tool?.description || `Invoke tool '${name}'`, + parameters: parameterSchema, + }, + }; + }); + + const requirementContext = retrievedDocuments + .map((doc, index) => `Document ${index + 1} (score: ${doc.score.toFixed(3)}):\nSource: ${doc.metadata?.source || 'unspecified'}\nTags: ${(doc.tags || []).join(', ') || 'none'}\n${doc.content}`) + .join('\n\n'); + + const messages = [ + { + role: 'system', + content: `You are a UI generator. Your goal is to create a context-aware UI based on the user's prompt, the provided thing description, the requirement documents, and the available UI components.\n\nYou must generate a UI that conforms to the provided JSON schema.\n\nAvailable components: ${JSON.stringify(availableComponents)}.`, + }, + ]; + + if (requirementContext) { + messages.push({ + role: 'system', + content: `Use the following requirement knowledge when crafting the UI:\n\n${requirementContext}`, + }); + } + + const activityDetails = capabilityData?.userActivity || {}; + const activitySample = activityDetails.data || activityDetails.cachedSample || null; + const activityState = activitySample?.id || activitySample?.state || null; + const selectionHint = selection?.reason ? selection.reason.toLowerCase() : ''; + const selectionContext = selection?.raw ? JSON.stringify(selection.raw).toLowerCase() : ''; + + const impliesHandsFree = selectionHint.includes('hands-free') + || selectionHint.includes('touch') + || selectionContext.includes('hands-free') + || selectionContext.includes('touch'); + + if (impliesHandsFree || activityState === 'hands-free') { + messages.push({ + role: 'system', + content: 'Current context indicates the user is hands-free. Avoid presenting warnings about hands being occupied or forcing voice interactions. Provide touch-friendly controls and only mention voice input as an optional enhancement.', + }); + } + + if (activityState === 'hands-occupied') { + messages.push({ + role: 'system', + content: 'Current activity is hands-occupied. Offer voice-first guidance and minimize the need for touch input.', + }); + } + if (uiSchema.theming?.supportsPrimaryColor) { + messages.push({ + role: 'system', + content: 'The device supports theming through the root `theme.primaryColor` field. When requirements or preferences mention a specific color (hex value), set `theme.primaryColor` accordingly to personalize the interface, while keeping sufficient contrast for readability.', + }); + } + + if (availableToolNames.length > 0) { + messages.push({ + role: 'system', + content: `Tools available: ${availableToolNames.join(', ')}. Call the appropriate tool to retrieve real-time capability data before finalizing the UI response.`, + }); + } + + if (selection?.reason && device?.name) { + messages.push({ + role: 'system', + content: `The core system selected device "${device.name}" (${device.id}) for this UI because: ${selection.reason}. Respect any device-specific limitations when building the UI.`, + }); + } + + const userContext = { + prompt, + thingDescription, + deviceId, + device, + capabilityData, + missingCapabilities, + selection, + }; + + messages.push({ + role: 'user', + content: JSON.stringify(userContext, null, 2), + }); + + const composeToolUrl = (base, path = '') => { + if (!path || path === '/') { + return base; + } + return `${base}${path.startsWith('/') ? path : `/${path}`}`; + }; + + const parseAssistantContent = (content) => { + if (!content || typeof content !== 'string') { + return null; + } + try { + return JSON.parse(content); + } catch (error) { + console.error('[KB] Failed to parse assistant content as JSON:', error); + return null; + } + }; + + let uiDefinition; + let toolInteractionOccurred = false; + let schemaReminderAdded = false; + let attemptsWithoutTool = 0; + const maxAttemptsWithoutTool = 2; + let enforceSchema = availableToolNames.length === 0; + + while (true) { + const requestPayload = { + model: uiSchema.model || openRouterModel || llmDefaultModel, + messages, + temperature: 0.7, + }; + + if (toolDefinitions.length > 0) { + requestPayload.tools = toolDefinitions; + requestPayload.tool_choice = 'auto'; + } + + if (enforceSchema) { + requestPayload.response_format = { + type: 'json_schema', + json_schema: { schema: filteredSchema }, + }; + } + + const { data: llmData, provider } = await invokeChatCompletion(requestPayload, { contextLabel: 'ui-generation' }); + console.log(`[KB] LLM provider ${provider} returned data for ui-generation (schema enforced? ${enforceSchema}).`); + console.log('LLM Data:', llmData); + + const responseMessage = llmData?.choices?.[0]?.message; + if (!responseMessage) { + console.error('[KB] LLM response missing message payload.'); + uiDefinition = { + type: 'container', + children: [{ type: 'text', content: 'Error: LLM returned an empty response.' }], + }; + break; + } + + const parsedContent = parseAssistantContent(responseMessage.content); + const toolCalls = Array.isArray(responseMessage.tool_calls) && responseMessage.tool_calls.length > 0 + ? responseMessage.tool_calls + : Array.isArray(parsedContent?.tool_calls) ? parsedContent.tool_calls : []; + + if (toolCalls.length > 0) { + console.log('[KB] Tool calls requested:', toolCalls.map((call) => call.function?.name).filter(Boolean)); + messages.push({ + role: 'assistant', + content: responseMessage.content || '', + tool_calls: toolCalls, + }); + + for (const toolCall of toolCalls) { + const toolName = toolCall?.function?.name; + if (!toolName) { + console.warn('[KB] Tool call missing function name, skipping.'); + continue; + } + + const toolDefinition = availableTools[toolName]; + if (!toolDefinition) { + console.error(`[KB] Tool '${toolName}' not defined in schema.`); + messages.push({ + role: 'tool', + tool_call_id: toolCall.id, + name: toolName, + content: JSON.stringify({ error: `Tool '${toolName}' unavailable.` }), + }); + continue; + } + + let args = {}; + const rawArguments = toolCall?.function?.arguments; + if (typeof rawArguments === 'string' && rawArguments.trim().length > 0) { + try { + args = JSON.parse(rawArguments); + } catch (error) { + console.error(`[KB] Failed to parse arguments for tool '${toolName}':`, error); + } + } + + const hasExplicitUrl = Boolean(toolDefinition.url); + let serviceUrl = toolDefinition.url; + if (!serviceUrl && toolDefinition.service) { + try { + const serviceResponse = await fetch(`${serviceRegistryUrl}/services/${toolDefinition.service}`); + if (!serviceResponse.ok) { + throw new Error(`Registry responded with status ${serviceResponse.status}`); + } + const serviceRecord = await serviceResponse.json(); + serviceUrl = serviceRecord.url; + } catch (error) { + console.error(`[KB] Failed to resolve service '${toolDefinition.service}' for tool '${toolName}':`, error); + messages.push({ + role: 'tool', + tool_call_id: toolCall.id, + name: toolName, + content: JSON.stringify({ error: `Unable to resolve service '${toolDefinition.service}': ${error.message}` }), + }); + continue; + } + } + + if (!serviceUrl) { + console.error(`[KB] No service URL configured for tool '${toolName}'.`); + messages.push({ + role: 'tool', + tool_call_id: toolCall.id, + name: toolName, + content: JSON.stringify({ error: 'Tool endpoint not configured.' }), + }); + continue; + } + + const endpointPath = toolDefinition.path || toolDefinition.endpoint || ''; + const requestUrl = !hasExplicitUrl && endpointPath + ? composeToolUrl(serviceUrl, endpointPath) + : serviceUrl; + const method = (toolDefinition.method || 'GET').toUpperCase(); + const headers = { ...(toolDefinition.headers || {}) }; + const requestInit = { method, headers }; + + if (method !== 'GET') { + headers['Content-Type'] = headers['Content-Type'] || 'application/json'; + requestInit.body = JSON.stringify(args || {}); + } + + console.log(`[KB] Invoking tool '${toolName}' via ${method} ${requestUrl}`); + + let toolResult; + try { + const toolResponse = await fetch(requestUrl, requestInit); + if (!toolResponse.ok) { + const errorBody = await toolResponse.text(); + toolResult = { error: `Tool responded with status ${toolResponse.status}`, details: errorBody }; + } else { + const responseText = await toolResponse.text(); + try { + toolResult = JSON.parse(responseText); + } catch { + toolResult = { data: responseText }; + } + } + } catch (error) { + console.error(`[KB] Error invoking tool '${toolName}':`, error); + toolResult = { error: error.message }; + } + + messages.push({ + role: 'tool', + tool_call_id: toolCall.id, + name: toolName, + content: JSON.stringify(toolResult), + }); + } + + toolInteractionOccurred = true; + enforceSchema = true; + + if (!schemaReminderAdded) { + messages.push({ + role: 'system', + content: 'Tool responses received. Using these fresh results, respond next with a UI object that strictly matches the provided JSON schema.', + }); + schemaReminderAdded = true; + } + + continue; + } + + if (availableToolNames.length > 0 && !toolInteractionOccurred) { + attemptsWithoutTool += 1; + console.warn(`[KB] No tool call detected despite available tools (attempt ${attemptsWithoutTool}).`); + + if (attemptsWithoutTool <= maxAttemptsWithoutTool) { + messages.push({ + role: 'system', + content: 'You must call at least one available tool (for example, getUserActivity) to fetch real-time data before finalizing the UI. Do not guess values—call a tool now.', + }); + enforceSchema = false; + continue; + } + + console.warn('[KB] Proceeding without tool execution after maximum retries.'); + enforceSchema = true; + } + + if (!parsedContent || Object.keys(parsedContent).length === 0) { + console.error('[KB] Parsed assistant content is empty.'); + uiDefinition = { + type: 'container', + children: [{ type: 'text', content: 'Error: LLM response was empty after tool usage.' }], + }; + } else { + uiDefinition = parsedContent; + } + break; + } + + return uiDefinition; +} + +app.get('/documents', (req, res) => { + res.json({ count: documents.length, documents }); +}); + +app.post('/documents', (req, res) => { + try { + const { id, content, metadata, tags } = req.body; + const record = addDocument({ id, content, metadata, tags }); + res.status(201).json({ status: 'stored', document: record }); + } catch (error) { + console.error('Error storing document:', error); + res.status(400).json({ error: error.message }); + } +}); + +app.get('/debug/last-device-selection', (req, res) => { + if (!lastDeviceSelection) { + return res.status(404).json({ error: 'No device selection has been recorded yet.' }); + } + + res.json(lastDeviceSelection); +}); + +app.post('/select-device', async (req, res) => { + const { + prompt, + fallbackPrompt, + desiredCapabilities, + thingDescription, + candidates, + model, + } = req.body || {}; + + try { + const selection = await runDeviceSelection({ + prompt, + fallbackPrompt, + desiredCapabilities, + thingDescription, + candidates, + model, + }); + res.json(selection); + } catch (error) { + console.error('[KB] Device selection failed:', error); + res.status(500).json({ error: 'Device selection failed', details: error.message }); + } +}); + +app.post('/query', async (req, res) => { + const { prompt, thingDescription, capabilities, schema, capabilityData, missingCapabilities, device, deviceId, selection } = req.body; + console.log('[KB] /query invoked', { + promptPreview: typeof prompt === 'string' ? `${prompt.slice(0, 60)}${prompt.length > 60 ? '…' : ''}` : null, + capabilities, + deviceId: deviceId || null, + }); + + try { + let generatedUi = await runAgent({ + prompt, + thingDescription, + capabilities, + uiSchema: schema || {}, + capabilityData, + missingCapabilities, + device, + deviceId, + selection, + }); + + if (!generatedUi || Object.keys(generatedUi).length === 0) { + generatedUi = { + type: 'container', + children: [ + { type: 'text', content: 'Error: UI generation failed. The generated UI is empty.' }, + ], + }; + } + console.log('[KB] Returning generated UI payload.'); + res.json(generatedUi); + } catch (error) { + console.error('Error communicating with LLM:', error); + res.status(500).json({ error: 'Failed to generate UI with LLM', details: error.message }); + } +}); -app.listen(PORT, LISTEN_ADDRESS, () => { - console.log(`Requirement Knowledge Base listening at ${LISTEN_ADDRESS}:${PORT} (public URL: ${PUBLIC_URL})`); - loadDocuments(); - seedKnowledgeBase(); +app.listen(port, listenAddress, () => { + console.log(`Requirement Knowledge Base listening at ${listenAddress}:${port} (public URL: ${knowledgeBasePublicUrl})`); + ensureDataFile(); registerWithServiceRegistry(); }); diff --git a/packages/knowledge-base/kb-data.json b/packages/knowledge-base/kb-data.json index e69de29..2f37189 100644 --- a/packages/knowledge-base/kb-data.json +++ b/packages/knowledge-base/kb-data.json @@ -0,0 +1,488 @@ +{ + "documents": [ + { + "id": "modality-guideline-hands-occupied", + "content": "When the user activity sensor reports the state \"hands-occupied\", prefer audio-first guidance. Provide spoken prompts and minimize the need for direct touch input. When the sensor reports \"hands-free\", present tactile controls such as buttons or toggles for the light switch.", + "metadata": { + "source": "safety-guidelines", + "version": "1.0.0" + }, + "tags": [ + "modality", + "hands-occupied", + "audio", + "light-switch" + ], + "tokens": [ + "when", + "the", + "user", + "activity", + "sensor", + "reports", + "the", + "state", + "hands", + "occupied", + "prefer", + "audio", + "first", + "guidance", + "provide", + "spoken", + "prompts", + "and", + "minimize", + "the", + "need", + "for", + "direct", + "touch", + "input", + "when", + "the", + "sensor", + "reports", + "hands", + "free", + "present", + "tactile", + "controls", + "such", + "as", + "buttons", + "or", + "toggles", + "for", + "the", + "light", + "switch" + ], + "termFrequency": { + "when": 2, + "the": 5, + "user": 1, + "activity": 1, + "sensor": 2, + "reports": 2, + "state": 1, + "hands": 2, + "occupied": 1, + "prefer": 1, + "audio": 1, + "first": 1, + "guidance": 1, + "provide": 1, + "spoken": 1, + "prompts": 1, + "and": 1, + "minimize": 1, + "need": 1, + "for": 2, + "direct": 1, + "touch": 1, + "input": 1, + "free": 1, + "present": 1, + "tactile": 1, + "controls": 1, + "such": 1, + "as": 1, + "buttons": 1, + "or": 1, + "toggles": 1, + "light": 1, + "switch": 1 + }, + "updatedAt": "2025-10-25T08:47:59.761Z", + "createdAt": "2025-10-25T08:47:59.762Z" + }, + { + "id": "user-preference-primary-color", + "content": "The primary household preference for interface accents is the color \"#1F6FEB\" (a vivid cobalt). Whenever possible, set the UI theme primary color to this value so buttons, toggles, and other interactive highlights align with the user preference. Ensure sufficient contrast by using light text on dark backgrounds.", + "metadata": { + "source": "user-profile", + "version": "2025.10" + }, + "tags": [ + "preference", + "theme", + "primary-color", + "personalization" + ], + "tokens": [ + "the", + "primary", + "household", + "preference", + "for", + "interface", + "accents", + "is", + "the", + "color", + "1f6feb", + "a", + "vivid", + "cobalt", + "whenever", + "possible", + "set", + "the", + "ui", + "theme", + "primary", + "color", + "to", + "this", + "value", + "so", + "buttons", + "toggles", + "and", + "other", + "interactive", + "highlights", + "align", + "with", + "the", + "user", + "preference", + "ensure", + "sufficient", + "contrast", + "by", + "using", + "light", + "text", + "on", + "dark", + "backgrounds" + ], + "termFrequency": { + "the": 4, + "primary": 2, + "household": 1, + "preference": 2, + "for": 1, + "interface": 1, + "accents": 1, + "is": 1, + "color": 2, + "1f6feb": 1, + "a": 1, + "vivid": 1, + "cobalt": 1, + "whenever": 1, + "possible": 1, + "set": 1, + "ui": 1, + "theme": 1, + "to": 1, + "this": 1, + "value": 1, + "so": 1, + "buttons": 1, + "toggles": 1, + "and": 1, + "other": 1, + "interactive": 1, + "highlights": 1, + "align": 1, + "with": 1, + "user": 1, + "ensure": 1, + "sufficient": 1, + "contrast": 1, + "by": 1, + "using": 1, + "light": 1, + "text": 1, + "on": 1, + "dark": 1, + "backgrounds": 1 + }, + "updatedAt": "2025-10-27T00:00:00.000Z", + "createdAt": "2025-10-27T00:00:00.000Z" + }, + { + "id": "component-selection-and-layout-guidelines", + "content": "UI Component Selection and Layout Guidelines:\n1. Control Selection Hierarchy: For any single binary state (e.g., a light being on/off), you MUST choose ONLY ONE of the following control types:\n a) A single 'toggle' or 'checkbox' component.\n b) A pair of 'button' components for explicit 'On' and 'Off' actions.\n NEVER generate both a toggle/checkbox and buttons for the same state control.\n2. Layout and Spacing: All interactive elements like buttons MUST have visible space between them. They must not touch. Group related controls inside a 'container' to provide structure.\n3. Button Labeling: Button labels must be concise and describe a single action (e.g., 'Turn On'). Avoid generic or redundant labels.", + "metadata": { + "source": "design-system-best-practices", + "version": "1.1" + }, + "tags": [ + "guideline", + "ui-design", + "layout", + "component-selection" + ], + "tokens": [ + "ui", + "component", + "selection", + "and", + "layout", + "guidelines", + "1", + "control", + "selection", + "hierarchy", + "for", + "any", + "single", + "binary", + "state", + "eg", + "a", + "light", + "being", + "on", + "off", + "you", + "must", + "choose", + "only", + "one", + "of", + "the", + "following", + "control", + "types", + "a", + "single", + "toggle", + "or", + "checkbox", + "component", + "b", + "pair", + "of", + "button", + "components", + "for", + "explicit", + "on", + "and", + "off", + "actions", + "never", + "generate", + "both", + "a", + "toggle", + "checkbox", + "and", + "buttons", + "for", + "the", + "same", + "state", + "control", + "2", + "layout", + "and", + "spacing", + "all", + "interactive", + "elements", + "like", + "buttons", + "must", + "have", + "visible", + "space", + "between", + "them", + "they", + "must", + "not", + "touch", + "group", + "related", + "'controls'", + "'inside'", + "'a'", + "'container'", + "to", + "provide", + "structure", + "3", + "button", + "labeling", + "button", + "labels", + "must", + "be", + "concise", + "and", + "describe", + "a", + "single", + "action", + "eg", + "turn", + "on", + "avoid", + "generic", + "or", + "redundant", + "labels" + ], + "termFrequency": { + "ui": 1, + "selection": 2, + "and": 5, + "layout": 2, + "guidelines": 1, + "1": 1, + "control": 3, + "hierarchy": 1, + "for": 3, + "any": 1, + "single": 2, + "binary": 1, + "eg": 2, + "a": 5, + "light": 1, + "being": 1, + "on": 3, + "off": 3, + "you": 1, + "must": 3, + "choose": 1, + "only": 1, + "one": 1, + "of": 2, + "the": 3, + "following": 1, + "types": 1, + "toggle": 2, + "or": 3, + "checkbox": 2, + "component": 2, + "b": 1, + "pair": 1, + "button": 3, + "components": 1, + "explicit": 1, + "actions": 1, + "never": 1, + "generate": 1, + "both": 1, + "buttons": 2, + "same": 1, + "state": 1, + "2": 1, + "spacing": 1, + "all": 1, + "interactive": 1, + "elements": 1, + "like": 1, + "have": 1, + "visible": 1, + "space": 1, + "between": 1, + "them": 1, + "they": 1, + "not": 1, + "touch": 1, + "group": 1, + "'controls'": 1, + "'inside'": 1, + "'a'": 1, + "'container'": 1, + "to": 1, + "provide": 1, + "structure": 1, + "3": 1, + "labeling": 1, + "labels": 2, + "redundant": 1 + } + }, + { + "id": "redundant-controls-rule", + "content": "UI Generation Rule: If the UI includes separate 'Turn On' and 'Turn Off' buttons for a device, you MUST NOT also include a 'Toggle' button for that same device. The 'On' and 'Off' buttons make a 'Toggle' button redundant and confusing.", + "metadata": { + "source": "design-system-best-practices", + "version": "1.0" + }, + "tags": [ + "guideline", + "ui-design", + "component-selection", + "redundancy" + ], + "tokens":[ + "ui", + "generation", + "rule", + "if", + "the", + "includes", + "separate", + "turn", + "on", + "and", + "off", + "buttons", + "for", + "a", + "device", + "you", + "must", + "not", + "also", + "include", + "toggle", + "button", + "that", + "same", + "device", + "the", + "on", + "and", + "off", + "buttons", + "make", + "a", + "toggle", + "button", + "redundant", + "and", + "confusing" + ], + "termFrequency":{ + "ui":1, + "generation":1, + "rule":1, + "if":1, + "the":2, + "includes":1, + "separate":1, + "turn":1, + "on":2, + "and":3, + "off":2, + "buttons":2, + "for":1, + "a":2, + "device":2, + "you":1, + "must":1, + "not":1, + "also":1, + "include":1, + "toggle":2, + "button":2, + "that":1, + "same":1, + "make":1, + "redundant":1, + "confusing":1 + } + } + ] +} \ No newline at end of file diff --git a/wireguard-config/coredns/Corefile b/wireguard-config/coredns/Corefile new file mode 100644 index 0000000..e26fbe6 --- /dev/null +++ b/wireguard-config/coredns/Corefile @@ -0,0 +1,6 @@ +. { + loop + errors + health + forward . /etc/resolv.conf +} diff --git a/wireguard-config/templates/peer.conf b/wireguard-config/templates/peer.conf new file mode 100644 index 0000000..d987dba --- /dev/null +++ b/wireguard-config/templates/peer.conf @@ -0,0 +1,11 @@ +[Interface] +Address = ${CLIENT_IP} +PrivateKey = $(cat /config/${PEER_ID}/privatekey-${PEER_ID}) +ListenPort = 51820 +DNS = ${PEERDNS} + +[Peer] +PublicKey = $(cat /config/server/publickey-server) +PresharedKey = $(cat /config/${PEER_ID}/presharedkey-${PEER_ID}) +Endpoint = ${SERVERURL}:${SERVERPORT} +AllowedIPs = ${ALLOWEDIPS} diff --git a/wireguard-config/templates/server.conf b/wireguard-config/templates/server.conf new file mode 100644 index 0000000..757682d --- /dev/null +++ b/wireguard-config/templates/server.conf @@ -0,0 +1,6 @@ +[Interface] +Address = ${INTERFACE}.1 +ListenPort = 51820 +PrivateKey = $(cat /config/server/privatekey-server) +PostUp = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT; iptables -t nat -A POSTROUTING -o eth+ -j MASQUERADE +PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT; iptables -t nat -D POSTROUTING -o eth+ -j MASQUERADE