diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d2a5ef4 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +AIRBYTE_SUPABASE_PROXY_URL = "https://xxxxxx/functions/v1/xxxxx/airbyte"; +AIRBYTE_LOCAL_PROXY_URL = "/api/airbyte"; +AIRBYTE_API_BASE_URL = "https://api.airbyte.com/v1"; +GROQ_SUPABASE_PROXY_URL = "https://xxxxxxx/functions/v1/xxxxx/groq"; +GROQ_LOCAL_PROXY_URL = "/api/groq"; +GROQ_API_BASE_URL = "https://api.groq.com/v1"; diff --git a/README.md b/README.md index f19ff4c..3526677 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,34 @@ Transform your business with Analysr -## ⚡ One Liner +## ⚡ Speedy Summary This is my submission for Airbyte-Motherduck Hackathon - December 2024 - January 2025 -For all you speedy folks out there, here’s the summary: +Here’s a speedy summary: - **1.0.0** - With your customer reviews in Motherduck, along with your chosen business stack and areas of interest, Analysr is ready to dish out some insightful analytics. To sweeten the deal, Groq is also integrated to help you navigate all your growth phases. - Your analytics lineup features Aspect Analysis, a Word Sentiment Heatmap (for those feelings), Advanced Text Analysis, Groq Business Analytics, Keyphrase Analysis, and a handy Competitor Comparison. + - Check it out at: + - [https://growwithanalysr.web.app/](https://growwithanalysr.web.app/) - Production + - [https://growwithanalysr.vercel.app/](https://growwithanalysr-staging.vercel.app/) - Experimental, for new features ## 🏗️ Architecture ![image](https://github.com/user-attachments/assets/0abe96f6-414a-42d2-aa0d-d0950a7da194) +## ❓ Why Analysr + +- **Scale Beyond Regular AI Capabilities:** Traditional AI systems, like ChatGPT, struggle to handle extensive datasets (e.g., 65,000+ records) effectively. Analysr bridges this gap. +- **Seamless Motherduck, Airbyte, Groq Integration:** Thanks to Motherduck wasm client, Airbyte's API and Groq SDK. +- **Data-Driven Insights:** By combining AI with visualization tools, Analysr allows users to uncover trends, anomalies, and actionable insights quickly and intuitively. +- **User-Friendly Visualization:** Visual AI integration transforms raw data into understandable and compelling graphics, enabling better decision-making. +- **Streamlined Process**: Reduces reliance on multiple tools by offering an all-in-one platform for schema analysis and visualization. + + ## 🚶Walkthrough -1) To obtain customer review insights, sync your data to Motherduck with the schema: { "review_text": "string", "stars": "number" } (More schemas will be supported soon). We recommend using Airbyte due to its extensive list of sources and seamless data movement. ![image](https://github.com/user-attachments/assets/415aece5-6594-4649-8d84-ec2fa1707988) +1) To obtain customer review insights, sync your data to Motherduck with the schema: { "review_text": "string", "stars": "number" } (More schemas support are in the future roadmap). We recommend using Airbyte due to its extensive list of sources and seamless data movement. ![image](https://github.com/user-attachments/assets/415aece5-6594-4649-8d84-ec2fa1707988) ![image](https://github.com/user-attachments/assets/00bf63f5-952f-491a-9ffd-0241d2e2bfd2) 2) Visit the Analysr website at (growwithanalysr.web.app) and click on the "Get Started Now" button for onboarding. ![image](https://github.com/user-attachments/assets/95da4b69-29bb-4c88-9433-19865bc72093) @@ -37,14 +49,18 @@ For all you speedy folks out there, here’s the summary: 8) Finally, input your area of interest for insights, such as customer satisfaction, and click "Continue to Dashboard."![image](https://github.com/user-attachments/assets/3c938fa2-a862-4ba6-b06e-b67bb139e71f) 9) Wait a few seconds until all queries are executed and visualized. ![image](https://github.com/user-attachments/assets/cf22aa51-cdb2-4e3f-99d6-ef93bf8f8c45) -10) Voilà! Your dashboard will be ready, featuring all Analysr's capabilities to support your next big step! +10) Voilà! Your dashboard will be ready, featuring all of Analysr's capabilities to support your next big step! ![image](https://github.com/user-attachments/assets/1ae1427d-c315-4e02-ac75-158e3cb14d61) -Need dataset and example method to test? -1. Hugging face dataset URL which I used, https://huggingface.co/datasets/Yelp/yelp_review_full -2. Import it to motherduck via airbyte (Set huggingface as source and motherduck as destination) -3. Get Groq token at, https://console.groq.com/keys -4. Click on continue to dashboard! That's it. Please try yourself, its fun! +**Need a dataset and one example method to test?** +1. Hugging face dataset URL which I used - https://huggingface.co/datasets/Yelp/yelp_review_full +2. Import it to Motherduck via Airbyte (Set huggingface as source and Motherduck as destination) OR attach using my share link +```bash +-- Run this snippet to attach the database +ATTACH 'md:_share/my_db/de60469b-3a05-4d74-bf63-4c1549dd55b6'; +``` +3. Get a Groq token at, https://console.groq.com/keys +4. Click on Continue to the dashboard! That's it. Please try it yourself, it's fun! ## ✨ Features @@ -57,7 +73,7 @@ Need dataset and example method to test? ## 🛠️ Technology Stack -- **Frontend**: React, TypeScript, Tailwind CSS +- **Frontend**: React, TypeScript, Tailwind CSS, Vite - **Analytics**: MotherDuck (DuckDB), GROQ AI - **Data Integration**: Airbyte - **Visualization**: Recharts @@ -67,13 +83,15 @@ Need dataset and example method to test? - **Proxy**: Supabase edge functions - **CI/CD**: GitHub Actions for automated deployment -## Future roadmap +**Declarations:** For development, the VSCode code editor, Codeium AI helper extension, and suggestions from ChatGPT were used. + +## 🔮 Future roadmap -- **Microservice for generating queries**: Currently all queries for analytics are highly coupled with code, seperation of concerns to microservice - - [x] Create mock express server and deployed as supabase functions - - [ ] Separate DuckDB queries for as an api call +- **Microservice for generating queries**: Currently all queries for analytics are highly coupled with code, separation of concerns to microservice + - [x] Create express server proxy and deploy as superbase functions + - [ ] Separate DuckDB queries as an API call response - [ ] Enhance microservice with GPT Wrapper - - [ ] Enhance business insights from Groq: Currently it hallucinates as the mixtral model is not powerful (Requires funding) + - [ ] Improve business insights from Groq: At present, it produces some inaccuracies due to the limitations of the open-source mixtral model, which lacks the necessary funding to enhance its capabilities. ## 🚀 Getting Started diff --git a/package.json b/package.json index 93022f8..b27956c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "analysr", "private": true, - "version": "1.0.10", + "version": "1.0.13", "type": "module", "scripts": { "dev": "vite", diff --git a/src/components/dashboard/Analytics/StatGrid.tsx b/src/components/dashboard/Analytics/StatGrid.tsx index 4ad0f43..8d44eee 100644 --- a/src/components/dashboard/Analytics/StatGrid.tsx +++ b/src/components/dashboard/Analytics/StatGrid.tsx @@ -31,7 +31,7 @@ export default function StatGrid({ analyticsData }: StatGridProps) { color: "purple", }, { - title: "Competitor Comparison", + title: "Competitive Advantage", value: `${ analyticsData.competitorComparison > 0 ? "+" : "" }${analyticsData.competitorComparison.toFixed(1)}%`, diff --git a/src/components/dashboard/DashboardView/DashboardContent.tsx b/src/components/dashboard/DashboardView/DashboardContent.tsx index e5a2d4b..956ccb0 100644 --- a/src/components/dashboard/DashboardView/DashboardContent.tsx +++ b/src/components/dashboard/DashboardView/DashboardContent.tsx @@ -15,6 +15,7 @@ interface DashboardContentProps { stack?: string; substack?: string; groqToken?: string; + interests?: string; } export default function DashboardContent({ @@ -22,6 +23,7 @@ export default function DashboardContent({ stack, substack, groqToken, + interests, }: DashboardContentProps) { if (!analyticsData || !analyticsData.totalReviews || !stack || !substack) { return ; @@ -46,6 +48,7 @@ export default function DashboardContent({ stack={stack} substack={substack} groqToken={groqToken} + interests={interests} /> diff --git a/src/components/dashboard/GPTInsights/BusinessInsights.tsx b/src/components/dashboard/GPTInsights/BusinessInsights.tsx index 1877eed..15e9544 100644 --- a/src/components/dashboard/GPTInsights/BusinessInsights.tsx +++ b/src/components/dashboard/GPTInsights/BusinessInsights.tsx @@ -12,6 +12,7 @@ interface BusinessInsightsProps { data: ProcessedAnalytics; stack: string; substack: string; + interests?: string; groqToken?: string; } @@ -19,6 +20,7 @@ export default function BusinessInsights({ data, stack, substack, + interests, groqToken, }: BusinessInsightsProps) { const [selectedModel, setSelectedModel] = useState("mixtral-8x7b-32768"); @@ -27,7 +29,8 @@ export default function BusinessInsights({ substack, data, groqToken, - selectedModel + selectedModel, + interests ); const handleModelChange = (modelId: string) => { diff --git a/src/components/onboarding/DataSelectionStep.tsx b/src/components/onboarding/DataSelectionStep.tsx index 706a9fc..d241e0d 100644 --- a/src/components/onboarding/DataSelectionStep.tsx +++ b/src/components/onboarding/DataSelectionStep.tsx @@ -59,7 +59,6 @@ export default function DataSelectionStep({ setDatabases(dbList); } catch (err) { - console.error("Error fetching databases:", err); setError("Failed to fetch databases"); } finally { setLoading(false); @@ -87,7 +86,6 @@ export default function DataSelectionStep({ .sort(); setTables(tableList); } catch (err) { - console.error("Error fetching tables:", err); setError(err instanceof Error ? err.message : "Failed to fetch tables"); } finally { setLoading(false); @@ -136,7 +134,6 @@ export default function DataSelectionStep({ } } } catch (err) { - console.error("Error fetching row count:", err); setError( err instanceof Error ? err.message diff --git a/src/components/onboarding/OnboardingForm.tsx b/src/components/onboarding/OnboardingForm.tsx index 0b3d7d4..695836a 100644 --- a/src/components/onboarding/OnboardingForm.tsx +++ b/src/components/onboarding/OnboardingForm.tsx @@ -91,7 +91,6 @@ export default function OnboardingForm() { navigate("/dashboard", { state: finalData }); } catch (error) { - console.error("Form submission error:", error); setValidationError( error instanceof Error ? error.message : "An error occurred" ); diff --git a/src/components/pages/Dashboard.tsx b/src/components/pages/Dashboard.tsx index da9a470..04661b0 100644 --- a/src/components/pages/Dashboard.tsx +++ b/src/components/pages/Dashboard.tsx @@ -77,8 +77,10 @@ export default function Dashboard() { return (
+
{isMockData && } +
); diff --git a/src/components/welcome/HeroContent.tsx b/src/components/welcome/HeroContent.tsx index 4be85c6..2a356a5 100644 --- a/src/components/welcome/HeroContent.tsx +++ b/src/components/welcome/HeroContent.tsx @@ -24,4 +24,4 @@ export default function HeroContent() { ); -} \ No newline at end of file +} diff --git a/src/components/welcome/WelcomeBackground.tsx b/src/components/welcome/WelcomeBackground.tsx index 2a64a1d..829cc39 100644 --- a/src/components/welcome/WelcomeBackground.tsx +++ b/src/components/welcome/WelcomeBackground.tsx @@ -40,4 +40,4 @@ export default function WelcomeBackground() {
); -} \ No newline at end of file +} diff --git a/src/components/welcome/WelcomeHero.tsx b/src/components/welcome/WelcomeHero.tsx index ce0a339..ddf813a 100644 --- a/src/components/welcome/WelcomeHero.tsx +++ b/src/components/welcome/WelcomeHero.tsx @@ -1,8 +1,8 @@ -import { motion } from 'framer-motion'; -import { BarChart3 } from 'lucide-react'; -import HeroContent from './HeroContent'; -import FeatureGrid from './FeatureGrid'; -import { welcomeScreenData } from './welcomeScreenData'; +import { motion } from "framer-motion"; +import { BarChart3 } from "lucide-react"; +import HeroContent from "./HeroContent"; +import FeatureGrid from "./FeatureGrid"; +import { welcomeScreenData } from "./welcomeScreenData"; export default function WelcomeHero() { return (
@@ -40,18 +40,12 @@ export default function WelcomeHero() { initial={{ opacity: 0 }} animate={{ opacity: 1 }} transition={{ delay: 0.8 }} - style={{ - display: "flex", - justifyContent: "center", - flexDirection: "row", - alignItems: "center", - }} - className="sm:hidden w-20 mt-8 p-0 md:p-2 bg-gradient-to-r from-blue-500/5 via-purple-500/5 to-blue-500/5 backdrop-blur-sm border border-white/5 rounded-xl" + className="hidden lg:flex items-center justify-center w-20 mt-8 p-0 md:p-2 bg-gradient-to-r from-blue-500/5 via-purple-500/5 to-blue-500/5 backdrop-blur-sm border border-white/5 rounded-xl" > -

+

{welcomeScreenData.welcomeSectionVersionBottom}

); -} \ No newline at end of file +} diff --git a/src/hooks/useAnalytics.ts b/src/hooks/useAnalytics.ts index ab5c7b3..8c07261 100644 --- a/src/hooks/useAnalytics.ts +++ b/src/hooks/useAnalytics.ts @@ -1,9 +1,8 @@ import { useState, useEffect } from 'react'; import { fetchAnalytics } from '../lib/motherduck/queries'; -import { initializeConnection } from '../lib/motherduck/connectionManager'; +import { initializeConnection } from '../lib/motherduck/analyticsConnectionManager'; import type { Analytics, LoadingStageId } from '../types/analytics'; import type { DataLimit } from '../components/onboarding/DataSelectionStep'; -import { fetchSentimentTrends } from '../lib/motherduck/queries/sentimentTrends'; export interface LoadingStage { id: LoadingStageId; @@ -102,6 +101,16 @@ export function useAnalytics( groqStatus: groqToken ? 'Initializing GROQ...' : 'GROQ token not provided', })); + const updateQueryStats = (query: string) => { + setResult((prev) => ({ + ...prev, + queryStats: { + count: prev.queryStats.count + 1, + lastQuery: query, + }, + })); + }; + const updateLoadingStage = ( stageId: LoadingStageId, status: LoadingStage['status'], @@ -119,7 +128,6 @@ export function useAnalytics( }; const handleError = (error: unknown, failedStage: LoadingStageId) => { - console.error('Analytics error:', error); const errorMessage = error instanceof Error ? error.message @@ -141,6 +149,21 @@ export function useAnalytics( })); }; + const handleProgress = (_stage: string, progress: number, currentQuery?: string) => { + if (currentQuery) { + updateQueryStats(currentQuery); + } + + if (progress <= 20) { + updateLoadingStage(LOADING_STAGES.DATA, 'loading', progress * 2); + } else if (progress <= 60) { + updateLoadingStage(LOADING_STAGES.DATA, 'loading', 80); + updateLoadingStage(LOADING_STAGES.PROCESSING, 'loading', (progress - 20) * 2.5); + } else { + updateLoadingStage(LOADING_STAGES.VISUALIZATION, 'loading', (progress - 60) * 2.5); + } + }; + useEffect(() => { let isSubscribed = true; @@ -164,29 +187,22 @@ export function useAnalytics( await initializeConnection(); if (!isSubscribed) return; updateLoadingStage(LOADING_STAGES.CONNECTION, 'complete', 100); - - updateLoadingStage(LOADING_STAGES.DATA, 'loading', 0); - const [analyticsData, sentimentTrends] = await Promise.all([ - fetchAnalytics(database, tableName, limit), - fetchSentimentTrends(database, tableName, limit), - ]); - if (!isSubscribed) return; - updateLoadingStage(LOADING_STAGES.DATA, 'complete', 100); + updateLoadingStage(LOADING_STAGES.DATA, 'loading', 5); + setResult(prev => ({ + ...prev, + queryStats: { count: 0, lastQuery: '' } + })); - updateLoadingStage(LOADING_STAGES.PROCESSING, 'loading', 50); + const [analyticsData] = await Promise.all([ + fetchAnalytics(database, tableName, limit, handleProgress), + ]); + updateLoadingStage(LOADING_STAGES.DATA, 'complete', 100); + if (!isSubscribed) return; const processedData: Analytics = { ...analyticsData, - sentimentTrends, }; - if (!isSubscribed) return; - updateLoadingStage(LOADING_STAGES.PROCESSING, 'complete', 100); - - updateLoadingStage(LOADING_STAGES.VISUALIZATION, 'loading', 50); - if (!isSubscribed) return; - updateLoadingStage(LOADING_STAGES.VISUALIZATION, 'complete', 100); - if (isSubscribed) { setResult((prev) => ({ ...prev, diff --git a/src/hooks/useGroqInsights.ts b/src/hooks/useGroqInsights.ts index eee4540..0e28a99 100644 --- a/src/hooks/useGroqInsights.ts +++ b/src/hooks/useGroqInsights.ts @@ -7,7 +7,8 @@ export function useGroqInsights( substack?: string, analyticsData?: any, groqToken?: string, - model?: string + model?: string, + interests?: string, ) { const [insights, setInsights] = useState(null); const [status, setStatus] = useState({ @@ -47,6 +48,7 @@ export function useGroqInsights( token: groqToken, stack, substack, + interests, positiveInsights: analyticsData.positiveInsights, negativeInsights: analyticsData.negativeInsights, emojiStats: analyticsData.textAnalysis.emojiStats, diff --git a/src/lib/airbyte/service.ts b/src/lib/airbyte/service.ts index 1be237a..1bdb2c1 100644 --- a/src/lib/airbyte/service.ts +++ b/src/lib/airbyte/service.ts @@ -32,7 +32,6 @@ export async function checkConnectionStatus( await response.json(); return response.ok; } catch (error) { - console.error("Connection check error:", error); return false; } } @@ -79,7 +78,6 @@ export async function triggerJob( error: data.error, }; } catch (error) { - console.error("Airbyte job error:", error); throw error; } } diff --git a/src/lib/groq/client.ts b/src/lib/groq/client.ts index cc79a60..6b84ef8 100644 --- a/src/lib/groq/client.ts +++ b/src/lib/groq/client.ts @@ -16,6 +16,7 @@ export async function generateBusinessInsights({ token, stack, substack, + interests="", positiveInsights, negativeInsights, emojiStats, @@ -28,6 +29,7 @@ export async function generateBusinessInsights({ token: string; stack: string; substack: string; + interests?: string; positiveInsights: Array<{ category: string; rating: number; @@ -55,7 +57,7 @@ export async function generateBusinessInsights({ }) { const client = getGroqClient(token); const selectedModel = GROQ_MODELS.find(m => m.id === model) || GROQ_MODELS[0]; - + const extraQuery = interests.length ? `Return in favour for these categorys: ${interests}` : ''; const prompt = `As a business analytics expert for ${stack}, specifically in ${substack}, analyze this data: Key Metrics: @@ -89,13 +91,16 @@ Provide three focused sections with exactly 2 actionable points each: - [Innovation opportunity from customer sentiment] - [Competitive advantage from top performing areas] -Keep each point specific, actionable, and directly tied to the data. Focus on the ${substack} sector specifically.`; +Keep each point specific, actionable, and directly tied to the data. Focus on the ${substack} sector specifically. ${extraQuery}`; const completion = await client.chat.completions.create({ messages: [{ role: 'user', content: prompt }], model: selectedModel.id, temperature: 0.7, - max_tokens: Math.min(selectedModel.maxTokens, 768) + max_tokens: Math.min(selectedModel.maxTokens, 768), + // TODO: For next release + // top_p: 0.8, //Focus finetuning + // presence_penalty: 0.3 // Diverse insights finetuning }); return completion.choices[0]?.message?.content || ''; diff --git a/src/lib/groq/models.ts b/src/lib/groq/models.ts index 1b10472..41a3cf9 100644 --- a/src/lib/groq/models.ts +++ b/src/lib/groq/models.ts @@ -38,7 +38,6 @@ export async function fetchAvailableModels(token: string): Promise maxTokens: model.context_window || 8192, })); } catch (error) { - console.error('Failed to fetch GROQ models:', error); return GROQ_MODELS; } } diff --git a/src/lib/motherduck/connectionManager.ts b/src/lib/motherduck/analyticsConnectionManager.ts similarity index 82% rename from src/lib/motherduck/connectionManager.ts rename to src/lib/motherduck/analyticsConnectionManager.ts index 52e2ba4..9910814 100644 --- a/src/lib/motherduck/connectionManager.ts +++ b/src/lib/motherduck/analyticsConnectionManager.ts @@ -4,6 +4,9 @@ import { getMotherDuckConfig } from './config'; let connection: ReturnType | null = null; let connectionPromise: Promise> | null = null; +//Adds lazy implementation to the connection initialization using promise, rather than initializing it immediately. +//Useful in scenarios where multiple components might simultaneously attempt to use the connection. + export const initializeConnection = async () => { const config = getMotherDuckConfig(); diff --git a/src/lib/motherduck/connection.ts b/src/lib/motherduck/connection.ts index 5f187d6..f58e2f1 100644 --- a/src/lib/motherduck/connection.ts +++ b/src/lib/motherduck/connection.ts @@ -3,6 +3,9 @@ import { getMotherDuckConfig } from './config'; let connection: ReturnType | null = null; +// Initializes the connection to MotherDuck immediately +// Useful for performing connection checks + export const getConnection = async () => { const config = getMotherDuckConfig(); diff --git a/src/lib/motherduck/queries.ts b/src/lib/motherduck/queries.ts index 2600419..892f7c2 100644 --- a/src/lib/motherduck/queries.ts +++ b/src/lib/motherduck/queries.ts @@ -1 +1 @@ -export { fetchAnalytics } from './queries/analytics'; \ No newline at end of file +export { fetchAnalytics } from './queries/analytics'; diff --git a/src/lib/motherduck/queries/analytics.ts b/src/lib/motherduck/queries/analytics.ts index fe2cdac..65cd8e2 100644 --- a/src/lib/motherduck/queries/analytics.ts +++ b/src/lib/motherduck/queries/analytics.ts @@ -1,4 +1,4 @@ -import { getConnection } from '../connectionManager'; +import { getConnection } from '../analyticsConnectionManager'; import { getTableRef, buildSampleClause } from './utils'; import { fetchAspectAnalysis } from './aspectAnalysis'; import { fetchNegativeInsights } from './negativeInsights'; @@ -12,6 +12,7 @@ export async function fetchAnalytics( database: string, tableName: string, limit: DataLimit, + onProgress?: (stage: string, progress: number, currentQuery?: string) => void ): Promise { const connection = await getConnection(); if (!connection) { @@ -23,39 +24,60 @@ export async function fetchAnalytics( try { const basicStatsQuery = ` - ${sampleClause} + ${sampleClause}, + rating_metrics AS ( SELECT COUNT(*) as total_reviews, AVG(CAST(stars as DOUBLE)) as avg_rating, - (CAST(COUNT(CASE WHEN stars >= 4 THEN 1 END) as DOUBLE) * 100.0 / NULLIF(COUNT(*), 0)) as sentiment_score + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY stars) as median_rating, + (CAST(COUNT(CASE WHEN stars >= 4 THEN 1 END) as DOUBLE) * 100.0 / NULLIF(COUNT(*), 0)) as sentiment_score, + -- Getting recent average (last 20% of reviews) + (SELECT AVG(CAST(stars as DOUBLE)) + FROM ( + SELECT stars + FROM sample_data + LIMIT (SELECT COUNT(*) * 0.2 FROM sample_data) + )) as recent_avg_rating FROM sample_data - `; + ) + SELECT + total_reviews, + avg_rating, + sentiment_score, + -- Calculating relative performance using recent vs overall trend + ((recent_avg_rating / NULLIF(avg_rating, 0)) - 1) * 100 as market_position + FROM rating_metrics + `; - const [ - basicStats, - aspectAnalysis, - negativeInsights, - positiveInsights, - textAnalysis, - sentimentData - ] = await Promise.all([ - connection.evaluateQuery(basicStatsQuery), - fetchAspectAnalysis(database, tableName, limit), - fetchNegativeInsights(database, tableName, limit), - fetchPositiveInsights(database, tableName, limit), + onProgress?.('Basic Statistics', 10, basicStatsQuery); + const basicStats = await connection.evaluateQuery(basicStatsQuery); + onProgress?.('Basic Statistics', 20); + + onProgress?.('Aspect Analysis', 25, 'Fetching aspect analysis...'); + const aspectAnalysis = await fetchAspectAnalysis(database, tableName, limit); + onProgress?.('Aspect Analysis', 40); + + onProgress?.('Negative Insights', 45, 'Analyzing negative feedback...'); + const negativeInsights = await fetchNegativeInsights(database, tableName, limit); + onProgress?.('Negative Insights', 60); + + onProgress?.('Positive Insights', 65, 'Analyzing positive feedback...'); + const positiveInsights = await fetchPositiveInsights(database, tableName, limit); + onProgress?.('Positive Insights', 80); + + onProgress?.('Text Analysis', 85, 'Processing text patterns...'); + const [textAnalysis, sentimentData] = await Promise.all([ fetchTextAnalysis(database, tableName, limit), fetchSentimentInsights(database, tableName, limit), ]); + onProgress?.('Text Analysis', 100); const stats = basicStats.data.toRows()[0]; - const industryAverage = 3.5; - const competitorComparison = ((Number(stats.avg_rating) / industryAverage) - 1) * 100; - return { totalReviews: Number(stats.total_reviews), averageRating: Number(stats.avg_rating) || 0, sentimentScore: Number(stats.sentiment_score) || 0, - competitorComparison, + competitorComparison: Number(stats.market_position) || 0, aspectAnalysis, negativeInsights, positiveInsights, @@ -63,7 +85,6 @@ export async function fetchAnalytics( sentimentInsights: sentimentData, }; } catch (error) { - console.error('Analytics query execution failed:', error); throw error; } } diff --git a/src/lib/motherduck/queries/sentimentInsights.ts b/src/lib/motherduck/queries/sentimentInsights.ts index 240e921..a8db440 100644 --- a/src/lib/motherduck/queries/sentimentInsights.ts +++ b/src/lib/motherduck/queries/sentimentInsights.ts @@ -50,7 +50,7 @@ export async function fetchSentimentInsights( 'could', 'should', 'about', 'which', 'thing', 'some', 'these' ) GROUP BY word - HAVING COUNT(*) >= 5 -- Reduced minimum frequency for testing + HAVING COUNT(*) >= 5 ) SELECT word as name, @@ -73,7 +73,7 @@ export async function fetchSentimentInsights( sentiment: Number(row.sentiment) })); } catch (error) { - console.error('Sentiment analysis error:', error); return []; } - } \ No newline at end of file + } + \ No newline at end of file diff --git a/src/lib/motherduck/queries/sentimentTrends.ts b/src/lib/motherduck/queries/sentimentTrends.ts deleted file mode 100644 index a0ec72f..0000000 --- a/src/lib/motherduck/queries/sentimentTrends.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { getConnection } from '../connection'; -import { getTableRef, buildLimitClause } from './utils'; -import type { SentimentTrend } from '../../../types/analytics'; - -export async function fetchSentimentTrends( - database: string, - tableName: string, - limit: number | 'All' -): Promise { - const connection = await getConnection(); - const tableRef = getTableRef(database, tableName); - const limitClause = buildLimitClause(limit); - - const query = ` - WITH rating_groups AS ( - SELECT - stars as avg_rating, - COUNT(*) as review_count - FROM ${tableRef} - GROUP BY stars - ORDER BY stars DESC - ${limitClause} - ) - SELECT * FROM rating_groups - `; - - const result = await connection.evaluateQuery(query); - const rows = result.data.toRows(); - - return rows.map((row, index) => { - const date = new Date(); - date.setMonth(date.getMonth() - (rows.length - index - 1)); - - return { - month: date.toISOString().split('T')[0], - avgRating: Number(row.avg_rating), - reviewCount: Number(row.review_count) - }; - }); -} diff --git a/src/lib/motherduck/queries/textAnalysis.ts b/src/lib/motherduck/queries/textAnalysis.ts index c570b89..98bf914 100644 --- a/src/lib/motherduck/queries/textAnalysis.ts +++ b/src/lib/motherduck/queries/textAnalysis.ts @@ -1,6 +1,6 @@ -import { getConnection } from '../connection'; -import { getTableRef, buildSampleClause } from './utils'; -import type { DataLimit } from '../../../components/onboarding/DataSelectionStep'; +import { getConnection } from "../connection"; +import { getTableRef, buildSampleClause } from "./utils"; +import type { DataLimit } from "../../../components/onboarding/DataSelectionStep"; export async function fetchTextAnalysis( database: string, @@ -11,41 +11,40 @@ export async function fetchTextAnalysis( const tableRef = getTableRef(database, tableName); const sampleClause = buildSampleClause(tableRef, limit); - // Mock emoji query since DuckDB doesn't support emoji analysis directly - const emojiQuery = ` - ${sampleClause} - SELECT - '⭐' as emoji, - COUNT(*) as count, - AVG(CAST(stars as DOUBLE)) as avg_rating - FROM sample_data - WHERE stars >= 4 - UNION ALL - SELECT - '👍' as emoji, - COUNT(*) as count, - AVG(CAST(stars as DOUBLE)) as avg_rating - FROM sample_data - WHERE stars >= 3 - UNION ALL - SELECT - '😊' as emoji, - COUNT(*) as count, - AVG(CAST(stars as DOUBLE)) as avg_rating - FROM sample_data - WHERE review_text LIKE '%happy%' OR review_text LIKE '%great%' - UNION ALL - SELECT - '🎉' as emoji, - COUNT(*) as count, - AVG(CAST(stars as DOUBLE)) as avg_rating - FROM sample_data - WHERE stars = 5 + const punctuationQuery = ` + ${sampleClause}, + punctuation_stats AS ( + SELECT + stars, + LENGTH(regexp_replace(review_text, '[^!]', '', 'g')) as exclamation_count, + LENGTH(regexp_replace(review_text, '[^?]', '', 'g')) as question_count + FROM sample_data + WHERE LENGTH(review_text) > 0 + ), + exclamation_stats AS ( + SELECT + COUNT(*) as total_count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM punctuation_stats + WHERE exclamation_count > 0 + ), + question_stats AS ( + SELECT + COUNT(*) as total_count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM punctuation_stats + WHERE question_count > 0 + ) + SELECT + (SELECT total_count FROM exclamation_stats) as exclamation_marks, + (SELECT avg_rating FROM exclamation_stats) as exclamation_avg_rating, + (SELECT total_count FROM question_stats) as question_marks, + (SELECT avg_rating FROM question_stats) as question_avg_rating `; const keyPhrasesQuery = ` ${sampleClause}, - word_stats AS ( + action_words AS ( SELECT word as text, COUNT(*) as occurrences, @@ -60,13 +59,20 @@ export async function fetchTextAnalysis( '\\s+' )) as t(word) WHERE LENGTH(word) > 3 + AND word IN ( + 'excellent', 'amazing', 'outstanding', 'improved', 'recommended', + 'efficient', 'effective', 'innovative', 'reliable', 'consistent', + 'performed', 'delivered', 'enhanced', 'optimized', 'streamlined', + 'accelerated', 'transformed', 'scaled', 'grew', 'expanded', + 'profitable', 'productive', 'successful', 'valuable', 'beneficial' + ) GROUP BY word HAVING COUNT(*) >= 5 ) SELECT * - FROM word_stats + FROM action_words ORDER BY occurrences DESC - LIMIT 20 + LIMIT 15 `; const capsAnalysisQuery = ` @@ -87,32 +93,69 @@ export async function fetchTextAnalysis( ORDER BY stars DESC `; - const [emojiResult, keyPhrasesResult, capsResult] = await Promise.all([ - connection.evaluateQuery(emojiQuery), - connection.evaluateQuery(keyPhrasesQuery), - connection.evaluateQuery(capsAnalysisQuery) - ]); + // Mock emoji query since DuckDB doesn't support emoji analysis directly + const emojiQuery = ` + ${sampleClause} + SELECT + '⭐' as emoji, + COUNT(*) as count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM sample_data + WHERE stars >= 4 + UNION ALL + SELECT + '👍' as emoji, + COUNT(*) as count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM sample_data + WHERE stars >= 3 + UNION ALL + SELECT + '😊' as emoji, + COUNT(*) as count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM sample_data + WHERE review_text LIKE '%happy%' OR review_text LIKE '%great%' + UNION ALL + SELECT + '🎉' as emoji, + COUNT(*) as count, + AVG(CAST(stars as DOUBLE)) as avg_rating + FROM sample_data + WHERE stars = 5 + `; + + const [emojiResult, punctuationResult, keyPhrasesResult, capsResult] = + await Promise.all([ + connection.evaluateQuery(emojiQuery), + connection.evaluateQuery(punctuationQuery), + connection.evaluateQuery(keyPhrasesQuery), + connection.evaluateQuery(capsAnalysisQuery), + ]); + + const punctuationStats = punctuationResult.data.toRows()[0]; return { - emojiStats: emojiResult.data.toRows().map(row => ({ + emojiStats: emojiResult.data.toRows().map((row) => ({ emoji: String(row.emoji), count: Number(row.count), - avgRating: Number(row.avg_rating) + avgRating: Number(row.avg_rating), })), punctuationStats: { - questionMarks: 0, - questionAvgRating: 0, - exclamationMarks: 0, - exclamationAvgRating: 0 + questionMarks: Number(punctuationStats.question_marks) || 0, + questionAvgRating: Number(punctuationStats.question_avg_rating) || 0, + exclamationMarks: Number(punctuationStats.exclamation_marks) || 0, + exclamationAvgRating: + Number(punctuationStats.exclamation_avg_rating) || 0, }, - capsAnalysis: capsResult.data.toRows().map(row => ({ + capsAnalysis: capsResult.data.toRows().map((row) => ({ stars: Number(row.stars), - capsPercentage: Number(row.caps_percentage) + capsPercentage: Number(row.caps_percentage), })), - keyPhrases: keyPhrasesResult.data.toRows().map(row => ({ + keyPhrases: keyPhrasesResult.data.toRows().map((row) => ({ text: String(row.text), occurrences: Number(row.occurrences), - sentiment: Number(row.sentiment) - })) + sentiment: Number(row.sentiment), + })), }; } diff --git a/src/lib/motherduck/types.ts b/src/lib/motherduck/types.ts index b51f350..b5559f2 100644 --- a/src/lib/motherduck/types.ts +++ b/src/lib/motherduck/types.ts @@ -35,3 +35,4 @@ export interface Dataset { database: string; tableName: string; } + diff --git a/src/types/analytics.ts b/src/types/analytics.ts index afc40ad..fb648d5 100644 --- a/src/types/analytics.ts +++ b/src/types/analytics.ts @@ -75,7 +75,6 @@ export interface Analytics { negativeInsights: NegativeInsight[]; positiveInsights: PositiveInsight[]; textAnalysis: TextAnalysis; - sentimentTrends: SentimentTrend[]; } -export type LoadingStageId = (typeof LOADING_STAGES)[keyof typeof LOADING_STAGES]; \ No newline at end of file +export type LoadingStageId = (typeof LOADING_STAGES)[keyof typeof LOADING_STAGES]; diff --git a/tailwind.config.js b/tailwind.config.js index d21f1cd..9fe3597 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -3,6 +3,12 @@ export default { content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], theme: { extend: {}, + screens: { + sm: '640px', + md: '768px', + lg: '1024px', + xl: '1280px', + }, }, plugins: [], };