From 0284188d5c2adf25f6f0547973c0fa6d5346451c Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Thu, 4 Dec 2025 09:07:25 +0000 Subject: [PATCH 1/3] feat: add AITB orchestration demo script with interactive UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive demo script showcasing the full AITB workflow: - Interactive CLI demonstrating dataset → batch → study flow - Colorful output using new UI functions - CSV dataset preview with row counting - Status polling with spinner animations - Real-time response tailing functionality - Parameter extraction from CSV for dynamic study creation - Retry logic and error handling - Summary tables and formatted output --- docs/scripts/aitb-orchestration.go | 708 ++++++++++++++++++++++++----- 1 file changed, 590 insertions(+), 118 deletions(-) diff --git a/docs/scripts/aitb-orchestration.go b/docs/scripts/aitb-orchestration.go index 24aa064..ae0890e 100644 --- a/docs/scripts/aitb-orchestration.go +++ b/docs/scripts/aitb-orchestration.go @@ -1,20 +1,67 @@ package main +// AITB Orchestration Demo Script +// +// This script demonstrates the complete AI Task Builder workflow from dataset +// creation through study publishing, with optional real-time response monitoring. +// +// USAGE: +// +// 1. Full orchestration demo: +// go run aitb-orchestration.go "Batch Name" "workspace_id" "project_id" +// +// 2. Full orchestration demo with response tailing: +// go run aitb-orchestration.go "Batch Name" "workspace_id" "project_id" --tail +// +// 3. Tail-only mode (for split-screen demos): +// go run aitb-orchestration.go --tail-only "batch_id" "Task-A" "green" +// +// SPLIT-SCREEN DEMO: +// For a split-screen demo showing two task types: +// - Run the full orchestration twice to create two batches +// - In terminal 1: go run aitb-orchestration.go --tail-only "batch_id_1" "Task-A" "green" +// - In terminal 2: go run aitb-orchestration.go --tail-only "batch_id_2" "Task-B" "blue" +// - Arrange terminals side-by-side to show responses streaming in real-time + import ( "context" "encoding/json" "fmt" "os" "os/exec" + "os/signal" "strings" + "syscall" + "time" ) +// ANSI color codes and formatting const ( - workspaceID = "6655b8281cc82a88996f0bb8" - taskName = "Sample Task" - taskIntro = "This is a sample task for testing" - taskSteps = "1. Review the data\n2. Provide your response" - csvFilePath = "docs/examples/aitb-model-evaluation.csv" + colorReset = "\033[0m" + colorBold = "\033[1m" + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorYellow = "\033[33m" + colorBlue = "\033[34m" + colorPurple = "\033[35m" + colorCyan = "\033[36m" + colorWhite = "\033[37m" + colorGray = "\033[90m" + colorProlificBlue = "\033[38;2;15;43;201m" // Prolific brand blue #0F2BC9 + + // Symbols + symbolSuccess = "✓" + symbolInfo = "ℹ" + symbolArrow = "→" + symbolDot = "•" +) + +const ( + taskName = "Model Response Evaluation" + taskIntro = "You will be shown a prompt and two different AI model responses. Your task is to evaluate which response is better and explain your reasoning." + taskSteps = "1. Read the prompt carefully\n2. Review both Response 1 and Response 2\n3. Select which response is better\n4. Answer the accuracy questions for each response\n5. Provide detailed justification for your evaluation" + csvFilePath = "docs/examples/pairwise-evaluation.csv" + instructionsFile = "docs/examples/pairwise-evaluation-instructions.json" studyTemplateFile = "docs/examples/standard-sample-aitaskbuilder.json" tmpStudyTemplateFile = "/tmp/aitb-study-template.json" ) @@ -49,129 +96,432 @@ type Instruction struct { Description string `json:"description"` } +// printSectionHeader prints a formatted section header with Enter prompt +func printSectionHeader(stepNum int, title string) { + // Prompt to continue + fmt.Printf("\n%s%s▶ Press Enter to continue to Step %d...%s ", colorBold, colorCyan, stepNum, colorReset) + _, _ = fmt.Scanln() // Wait for Enter key + + fmt.Printf("\n%s%s%s Step %d: %s %s\n", colorBold, colorBlue, symbolArrow, stepNum, title, colorReset) + fmt.Printf("%s%s%s\n", colorGray, strings.Repeat("─", 60), colorReset) +} + +// printCommand prints a command in a formatted way +func printCommand(cmd string) { + fmt.Printf("%s%s Command:%s %s%s\n", colorGray, symbolDot, colorReset, colorCyan, cmd) + fmt.Print(colorReset) +} + +// printSuccess prints a success message +func printSuccess(message string) { + fmt.Printf("%s%s %s%s\n", colorGreen, symbolSuccess, message, colorReset) +} + +// printInfo prints an info message +func printInfo(label, value string) { + fmt.Printf("%s%s%s %s:%s %s%s%s\n", colorBold, colorWhite, symbolInfo, label, colorReset, colorCyan, value, colorReset) +} + +// showSpinner displays an animated spinner for a duration +func showSpinner(duration time.Duration, message string) { + spinChars := []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + timeout := time.After(duration) + i := 0 + + for { + select { + case <-timeout: + fmt.Printf("\r%s\n", strings.Repeat(" ", 80)) // Clear the line + return + case <-ticker.C: + fmt.Printf("\r%s%s%s %s %s", colorYellow, spinChars[i%len(spinChars)], colorReset, message, colorGray) + i++ + } + } +} + +// printDatasetPreview displays the CSV data in a formatted table +func printDatasetPreview(filePath string) error { + // Read the CSV file + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read CSV file: %w", err) + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + if len(lines) == 0 { + return fmt.Errorf("CSV file is empty") + } + + fmt.Printf("\n%s%s📋 Dataset Preview:%s\n", colorBold, colorCyan, colorReset) + fmt.Printf("%s%s%s\n", colorGray, strings.Repeat("─", 80), colorReset) + + // Parse and display rows (limit to first 3 data rows + header for demo) + maxRows := 4 // header + 3 data rows + if len(lines) > maxRows { + lines = lines[:maxRows] + } + + for i, line := range lines { + // Parse CSV line (simple parsing - assumes no commas in quoted fields for demo) + fields := parseCSVLine(line) + + if i == 0 { + // Header row + fmt.Printf("%s%s│%s ", colorBold, colorYellow, colorReset) + for j, field := range fields { + if j > 0 { + fmt.Printf("%s│%s ", colorYellow, colorReset) + } + // Truncate long headers + displayField := field + if len(field) > 25 { + displayField = field[:22] + "..." + } + fmt.Printf("%s%-25s%s", colorBold+colorYellow, displayField, colorReset) + } + fmt.Printf(" %s│%s\n", colorYellow, colorReset) + fmt.Printf("%s%s%s\n", colorYellow, strings.Repeat("─", 80), colorReset) + } else { + // Data row + fmt.Printf("%s│%s ", colorGray, colorReset) + for j, field := range fields { + if j > 0 { + fmt.Printf("%s│%s ", colorGray, colorReset) + } + // Truncate long fields + displayField := field + if len(field) > 25 { + displayField = field[:22] + "..." + } + fmt.Printf("%-25s", displayField) + } + fmt.Printf(" %s│%s\n", colorGray, colorReset) + } + } + + fmt.Printf("%s%s%s\n", colorGray, strings.Repeat("─", 80), colorReset) + + // Show total row count + totalRows, _ := countCSVRows(filePath) + if totalRows > 3 { + fmt.Printf("%s ... and %d more rows%s\n", colorGray, totalRows-3, colorReset) + } + fmt.Printf("%s Total: %d data rows%s\n\n", colorCyan, totalRows, colorReset) + + return nil +} + +// parseCSVLine parses a CSV line handling quoted fields +func parseCSVLine(line string) []string { + var fields []string + var currentField strings.Builder + inQuotes := false + + for i := 0; i < len(line); i++ { + char := line[i] + + if char == '"' { + inQuotes = !inQuotes + } else if char == ',' && !inQuotes { + fields = append(fields, strings.TrimSpace(currentField.String())) + currentField.Reset() + } else { + currentField.WriteByte(char) + } + } + + fields = append(fields, strings.TrimSpace(currentField.String())) + return fields +} + +// countCSVRows counts the number of data rows (excluding header) +func countCSVRows(filePath string) (int, error) { + content, err := os.ReadFile(filePath) + if err != nil { + return 0, err + } + + lines := strings.Split(strings.TrimSpace(string(content)), "\n") + if len(lines) <= 1 { + return 0, nil + } + + return len(lines) - 1, nil +} + +// printSummaryTable prints a formatted summary table +func printSummaryTable(datasetID, batchID, studyID string) { + fmt.Printf("\n%s%s%s\n", colorBold, colorGreen, strings.Repeat("═", 60)) + fmt.Printf(" 🎉 ORCHESTRATION COMPLETE 🎉") + fmt.Printf("\n%s%s\n\n", strings.Repeat("═", 60), colorReset) + + data := []struct { + label string + value string + icon string + }{ + {"Dataset ID", datasetID, "📊"}, + {"Batch ID", batchID, "📦"}, + {"Study ID", studyID, "🔬"}, + {"Status", "ACTIVE", "✅"}, + } + + for _, item := range data { + fmt.Printf(" %s %s%-12s%s %s%s%s\n", + item.icon, + colorBold+colorWhite, item.label+":", colorReset, + colorCyan, item.value, colorReset) + } + + fmt.Printf("\n%s%s%s\n", colorGreen, strings.Repeat("═", 60), colorReset) + + // Final pause for Q&A + fmt.Printf("\n%s%sDemo complete! Ready for questions...%s\n\n", colorBold, colorBlue, colorReset) +} + func main() { - // Check for batch name argument - if len(os.Args) < 2 { - fmt.Fprintf(os.Stderr, "Usage: %s \n", os.Args[0]) + // Check for tail-only mode + if len(os.Args) >= 3 && os.Args[1] == "--tail-only" { + batchID := os.Args[2] + label := "BATCH" + labelColor := colorGreen + + if len(os.Args) >= 4 { + label = os.Args[3] + } + if len(os.Args) >= 5 { + // Support custom colors: green, blue, purple, cyan, yellow + switch os.Args[4] { + case "blue": + labelColor = colorBlue + case "purple": + labelColor = colorPurple + case "cyan": + labelColor = colorCyan + case "yellow": + labelColor = colorYellow + } + } + + fmt.Printf("\n%s%s════════════════════════════════════════════════════════════╗%s\n", colorBold, labelColor, colorReset) + fmt.Printf("%s%s 📊 Response Monitor: %s%s\n", colorBold, labelColor, label, colorReset) + fmt.Printf("%s%s════════════════════════════════════════════════════════════╝%s\n", colorBold, labelColor, colorReset) + fmt.Printf("%s%sℹ Monitoring Batch ID: %s%s\n", colorBold, colorCyan, batchID, colorReset) + fmt.Printf("%s%sℹ Press Ctrl+C to stop%s\n\n", colorBold, colorCyan, colorReset) + + // Setup signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + // Start tailing + go tailResponses(ctx, batchID, label, labelColor) + + // Wait for interrupt + <-sigChan + fmt.Printf("\n\n%s%s🛑 Shutting down...%s\n", colorBold, colorYellow, colorReset) + cancel() + time.Sleep(500 * time.Millisecond) + fmt.Printf("%s%s✓ Stopped%s\n\n", colorBold, colorGreen, colorReset) + return + } + + // Check for required arguments + if len(os.Args) < 4 { + fmt.Fprintf(os.Stderr, "Usage:\n") + fmt.Fprintf(os.Stderr, " Full demo: %s [--tail]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " Tail only: %s --tail-only [label] [color]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\nExamples:\n") + fmt.Fprintf(os.Stderr, " %s \"My Batch\" \"6655b8281cc82a88996f0bb8\" \"6655b8281cc82a88996f0bb8\"\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s --tail-only \"6655b8281cc82a88996f0bb8\" \"Task-A\" \"green\"\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\nColors: green, blue, purple, cyan, yellow\n") os.Exit(1) } batchName := os.Args[1] + workspaceID := os.Args[2] + projectID := os.Args[3] + + // Check for --tail flag + enableTailing := false + for _, arg := range os.Args[4:] { + if arg == "--tail" { + enableTailing = true + break + } + } - fmt.Println("Starting AI Task Builder Orchestration") - fmt.Println("========================================") - fmt.Printf("Batch Name: %s\n\n", batchName) + // Print header + fmt.Println() + + // Read and display ASCII banner + banner, err := os.ReadFile("docs/scripts/banner.txt") + if err == nil { + fmt.Printf("%s%s%s\n", colorProlificBlue, string(banner), colorReset) + } else { + // Fallback if banner file not found + fmt.Printf("%s╔════════════════════════════════════════════════════════════╗\n", colorProlificBlue) + fmt.Printf("║ AI TASK BUILDER ORCHESTRATION - DEMO ║\n") + fmt.Printf("╚════════════════════════════════════════════════════════════╝%s\n", colorReset) + } + fmt.Println() + printInfo("Batch Name", batchName) + printInfo("Workspace ID", workspaceID) + printInfo("Project ID", projectID) + fmt.Printf("\n%s%s💡 Interactive Demo Mode - Press Enter to advance through each step%s\n", colorBold, colorYellow, colorReset) // Step 1: Create Dataset - fmt.Println("Step 1: Creating dataset...") - fmt.Printf("Command: ./prolific aitaskbuilder dataset create -n %s -w %s\n", batchName, workspaceID) - datasetID, err := createDataset(batchName) + printSectionHeader(1, "Creating dataset") + printCommand(fmt.Sprintf("./prolific aitaskbuilder dataset create -n %s -w %s", batchName, workspaceID)) + datasetID, err := createDataset(batchName, workspaceID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create dataset: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to create dataset: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Dataset created with ID: %s\n\n", datasetID) + printSuccess(fmt.Sprintf("Dataset created with ID: %s", datasetID)) // Step 2: Upload Dataset - fmt.Println("Step 2: Uploading dataset file...") - fmt.Printf("Command: ./prolific aitaskbuilder dataset upload -d %s -f %s\n", datasetID, csvFilePath) + printSectionHeader(2, "Uploading dataset file") + printCommand(fmt.Sprintf("./prolific aitaskbuilder dataset upload -d %s -f %s", datasetID, csvFilePath)) err = uploadDataset(datasetID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to upload dataset: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to upload dataset: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Dataset uploaded successfully\n\n") + printSuccess("Dataset uploaded successfully") + + // Show dataset preview + err = printDatasetPreview(csvFilePath) + if err != nil { + fmt.Fprintf(os.Stderr, "%s⚠ Could not display dataset preview: %v%s\n", colorYellow, err, colorReset) + } // Step 3: Check Dataset Status - fmt.Println("Step 3: Checking dataset status...") - fmt.Printf("Command: ./prolific aitaskbuilder dataset check -d %s\n", datasetID) + printSectionHeader(3, "Checking dataset status") + printCommand(fmt.Sprintf("./prolific aitaskbuilder dataset check -d %s", datasetID)) err = checkDataset(datasetID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to check dataset: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to check dataset: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Dataset status checked\n\n") + printSuccess("Dataset status verified") // Step 4: Create Batch - fmt.Println("Step 4: Creating batch...") - fmt.Printf("Command: ./prolific aitaskbuilder batch create -n \"%s\" -w %s -d %s --task-name \"%s\" --task-introduction \"%s\" --task-steps \"%s\"\n", - batchName, workspaceID, datasetID, taskName, taskIntro, taskSteps) - batchID, err := createBatch(batchName, datasetID) + printSectionHeader(4, "Creating batch") + printCommand(fmt.Sprintf("./prolific aitaskbuilder batch create -n \"%s\" -w %s -d %s --task-name \"%s\" --task-introduction \"%s\" --task-steps \"%s\"", + batchName, workspaceID, datasetID, taskName, taskIntro, taskSteps)) + batchID, err := createBatch(batchName, datasetID, workspaceID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create batch: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to create batch: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Batch created with ID: %s\n\n", batchID) + printSuccess(fmt.Sprintf("Batch created with ID: %s", batchID)) - // Step 5: Check Batch Status - fmt.Println("Step 5: Checking batch status...") - fmt.Printf("Command: ./prolific aitaskbuilder batch check -b %s\n", batchID) - err = checkBatch(batchID) + // Step 5: Add Instructions to Batch + printSectionHeader(5, "Adding instructions to batch") + printCommand(fmt.Sprintf("./prolific aitaskbuilder batch instructions -b %s -j @%s", batchID, instructionsFile)) + err = addInstructions(batchID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to check batch: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to add instructions: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Batch status checked\n\n") + printSuccess("Instructions added successfully") - // Step 6: Add Instructions to Batch - fmt.Println("Step 6: Adding instructions to batch...") - instructionsJSON := `[{"type":"free_text","created_by":"Sean","description":"Is the response evidence of a dangerous and burgeoning artificial general superintelligence? Explain your evaluation."}]` - fmt.Printf("Command: ./prolific aitaskbuilder batch instructions -b %s -j '%s'\n", batchID, instructionsJSON) - err = addInstructions(batchID) + // Step 6: Setup Batch + printSectionHeader(6, "Setting up batch") + printCommand(fmt.Sprintf("./prolific aitaskbuilder batch setup -b %s -d %s --tasks-per-group 1", batchID, datasetID)) + err = setupBatch(batchID, datasetID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to add instructions: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to setup batch: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Instructions added successfully\n\n") + printSuccess("Batch setup completed successfully") - // Step 7: Setup Batch - fmt.Println("Step 7: Setting up batch...") - fmt.Printf("Command: ./prolific aitaskbuilder batch setup -b %s -d %s --tasks-per-group 1\n", batchID, datasetID) - err = setupBatch(batchID, datasetID) + // Step 7: Check Batch Status + printSectionHeader(7, "Checking batch status") + printCommand(fmt.Sprintf("./prolific aitaskbuilder batch check -b %s", batchID)) + err = checkBatch(batchID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to setup batch: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to check batch: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Batch setup completed successfully\n\n") + printSuccess("Batch status verified") // Step 8: View Batch - fmt.Println("Step 8: Viewing batch details...") - fmt.Printf("Command: ./prolific aitaskbuilder batch view -b %s\n", batchID) + printSectionHeader(8, "Viewing batch details") + printCommand(fmt.Sprintf("./prolific aitaskbuilder batch view -b %s", batchID)) err = viewBatch(batchID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to view batch: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to view batch: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Batch details retrieved\n\n") + printSuccess("Batch details retrieved") // Step 9: Create Prolific Study - fmt.Println("Step 9: Creating Prolific study linked to batch...") - fmt.Printf("Command: ./prolific study create -t %s\n", tmpStudyTemplateFile) - studyID, err := createStudy(batchID) + printSectionHeader(9, "Creating Prolific study linked to batch") + printCommand(fmt.Sprintf("./prolific study create -t %s", tmpStudyTemplateFile)) + studyID, err := createStudy(batchID, projectID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to create study: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to create study: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Study created with ID: %s\n\n", studyID) + printSuccess(fmt.Sprintf("Study created with ID: %s", studyID)) // Step 10: View Study Details - fmt.Println("Step 10: Viewing study details...") - fmt.Printf("Command: ./prolific study view %s\n", studyID) + printSectionHeader(10, "Viewing study details") + printCommand(fmt.Sprintf("./prolific study view %s", studyID)) err = viewStudy(studyID) if err != nil { - fmt.Fprintf(os.Stderr, "Failed to view study: %v\n", err) + fmt.Fprintf(os.Stderr, "%s✗ Failed to view study: %v%s\n", colorRed, err, colorReset) os.Exit(1) } - fmt.Printf("✓ Study details retrieved\n\n") + printSuccess("Study details retrieved") - fmt.Println("========================================") - fmt.Println("AI Task Builder Orchestration Complete!") - fmt.Printf("\nDataset ID: %s\n", datasetID) - fmt.Printf("Batch ID: %s\n", batchID) - fmt.Printf("Study ID: %s\n", studyID) + // Step 11: Publish Study + printSectionHeader(11, "Publishing study") + printCommand(fmt.Sprintf("./prolific study transition -a PUBLISH %s", studyID)) + err = publishStudy(studyID) + if err != nil { + fmt.Fprintf(os.Stderr, "%s✗ Failed to publish study: %v%s\n", colorRed, err, colorReset) + os.Exit(1) + } + printSuccess("Study published successfully") + + // Print summary + printSummaryTable(datasetID, batchID, studyID) + + // Step 12 (Optional): Tail Responses + if enableTailing { + printSectionHeader(12, "Monitoring task responses in real-time") + fmt.Printf("\n%s%s💡 This will poll for new responses every 3 seconds%s\n", colorBold, colorYellow, colorReset) + fmt.Printf("%s%sℹ Press Ctrl+C to stop monitoring%s\n\n", colorBold, colorCyan, colorReset) + + // Setup signal handling for graceful shutdown + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + // Start tailing in a goroutine + go tailResponses(ctx, batchID, "BATCH-1", colorGreen) + + // Wait for interrupt signal + <-sigChan + fmt.Printf("\n\n%s%s🛑 Received interrupt signal, shutting down gracefully...%s\n", colorBold, colorYellow, colorReset) + cancel() + time.Sleep(500 * time.Millisecond) // Give goroutines time to clean up + fmt.Printf("%s%s✓ Response monitoring stopped%s\n\n", colorBold, colorGreen, colorReset) + } } // createDataset executes the dataset create command and returns the dataset ID -func createDataset(datasetName string) (string, error) { +func createDataset(datasetName, workspaceID string) (string, error) { cmd := exec.CommandContext(context.Background(), "./prolific", "aitaskbuilder", "dataset", "create", "-n", datasetName, "-w", workspaceID) @@ -210,32 +560,54 @@ func uploadDataset(datasetID string) error { return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) } - fmt.Printf("Upload output: %s\n", string(output)) + if len(output) > 0 { + fmt.Printf("%s %s%s\n", colorGray, strings.TrimSpace(string(output)), colorReset) + } return nil } -// checkDataset executes the dataset check command and verifies it's READY -func checkDataset(datasetID string) error { - cmd := exec.CommandContext(context.Background(), "./prolific", "aitaskbuilder", "dataset", "check", - "-d", datasetID) +// pollStatus polls a resource until it reaches READY status +func pollStatus(resourceType string, cmdArgs ...string) error { + maxRetries := 30 + retryDelay := 2 * time.Second - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) - } + for i := 0; i < maxRetries; i++ { + if i > 0 { + showSpinner(retryDelay, fmt.Sprintf("Polling %s status (attempt %d/%d)...", resourceType, i+1, maxRetries)) + } - fmt.Printf("Dataset status: %s\n", string(output)) + cmd := exec.CommandContext(context.Background(), "./prolific", cmdArgs...) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) + } - // Check if status is READY - if !strings.Contains(string(output), "READY") { - return fmt.Errorf("dataset is not in READY status: %s", string(output)) + status := strings.TrimSpace(string(output)) + fmt.Printf("%s Status: %s%s%s\n", colorGray, colorYellow, status, colorReset) + + // Check if status is READY + if strings.Contains(status, "READY") { + fmt.Printf("%s %s is ready!%s\n", colorGreen, resourceType, colorReset) + return nil + } + + // If not UNINITIALISED or READY, something went wrong + if !strings.Contains(status, "UNINITIALISED") && !strings.Contains(status, "PROCESSING") { + return fmt.Errorf("%s in unexpected status: %s", resourceType, status) + } } - return nil + return fmt.Errorf("%s did not reach READY status after %d retries", resourceType, maxRetries) +} + +// checkDataset executes the dataset check command and verifies it's READY +func checkDataset(datasetID string) error { + return pollStatus("Dataset", "aitaskbuilder", "dataset", "check", "-d", datasetID) } // createBatch executes the batch create command and returns the batch ID -func createBatch(batchName, datasetID string) (string, error) { +func createBatch(batchName, datasetID, workspaceID string) (string, error) { cmd := exec.CommandContext(context.Background(), "./prolific", "aitaskbuilder", "batch", "create", "-n", batchName, "-w", workspaceID, @@ -269,41 +641,24 @@ func createBatch(batchName, datasetID string) (string, error) { // checkBatch executes the batch check command and verifies it's READY func checkBatch(batchID string) error { - cmd := exec.CommandContext(context.Background(), "./prolific", "aitaskbuilder", "batch", "check", - "-b", batchID) - - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) - } - - fmt.Printf("Batch status: %s\n", string(output)) - - // Check if status is READY - if !strings.Contains(string(output), "READY") { - return fmt.Errorf("batch is not in READY status: %s", string(output)) - } - - return nil + return pollStatus("Batch", "aitaskbuilder", "batch", "check", "-b", batchID) } // addInstructions executes the batch instructions command func addInstructions(batchID string) error { - // Create the instructions JSON - instructions := []Instruction{ - { - Type: "free_text", - CreatedBy: "Sean", - Description: "Is the response evidence of a dangerous and burgeoning artificial general superintelligence? Explain your evaluation.", - }, + // Read instructions from file + instructionsJSON, err := os.ReadFile(instructionsFile) + if err != nil { + return fmt.Errorf("failed to read instructions file: %w", err) } - instructionsJSON, err := json.Marshal(instructions) - if err != nil { - return fmt.Errorf("failed to marshal instructions: %w", err) + // Validate it's valid JSON + var instructions []map[string]interface{} + if err := json.Unmarshal(instructionsJSON, &instructions); err != nil { + return fmt.Errorf("failed to parse instructions JSON: %w", err) } - // #nosec G204 - instructionsJSON is JSON-marshaled from a struct, not user input + // #nosec G204 - instructionsJSON is read from a trusted file cmd := exec.CommandContext(context.Background(), "./prolific", "aitaskbuilder", "batch", "instructions", "-b", batchID, "-j", string(instructionsJSON)) @@ -313,7 +668,9 @@ func addInstructions(batchID string) error { return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) } - fmt.Printf("Instructions output: %s\n", string(output)) + if len(output) > 0 { + fmt.Printf("%s %s%s\n", colorGray, strings.TrimSpace(string(output)), colorReset) + } return nil } @@ -329,7 +686,9 @@ func setupBatch(batchID, datasetID string) error { return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) } - fmt.Printf("Setup output: %s\n", string(output)) + if len(output) > 0 { + fmt.Printf("%s %s%s\n", colorGray, strings.TrimSpace(string(output)), colorReset) + } return nil } @@ -343,27 +702,35 @@ func viewBatch(batchID string) error { return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) } - fmt.Printf("Batch details:\n%s\n", string(output)) + if len(output) > 0 { + fmt.Printf("%s Details:%s\n", colorGray, colorReset) + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, line := range lines { + fmt.Printf("%s %s%s\n", colorGray, line, colorReset) + } + } return nil } // createStudy creates a Prolific study linked to the batch -func createStudy(batchID string) (string, error) { +func createStudy(batchID, projectID string) (string, error) { // Read the template file templateContent, err := os.ReadFile(studyTemplateFile) if err != nil { return "", fmt.Errorf("failed to read template file: %w", err) } - // Replace ${BATCH_ID} with actual batch ID - studyContent := strings.ReplaceAll(string(templateContent), "${BATCH_ID}", batchID) + // Replace placeholders with actual values + studyContent := string(templateContent) + studyContent = strings.ReplaceAll(studyContent, "${BATCH_ID}", batchID) + studyContent = strings.ReplaceAll(studyContent, "${PROJECT_ID}", projectID) // Write to temporary file err = os.WriteFile(tmpStudyTemplateFile, []byte(studyContent), 0600) if err != nil { return "", fmt.Errorf("failed to write temporary template file: %w", err) } - defer os.Remove(tmpStudyTemplateFile) + // defer os.Remove(tmpStudyTemplateFile) // Execute study create command cmd := exec.CommandContext(context.Background(), "./prolific", "study", "create", @@ -403,6 +770,111 @@ func viewStudy(studyID string) error { return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) } - fmt.Printf("Study details:\n%s\n", string(output)) + if len(output) > 0 { + fmt.Printf("%s Details:%s\n", colorGray, colorReset) + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, line := range lines { + fmt.Printf("%s %s%s\n", colorGray, line, colorReset) + } + } + return nil +} + +// publishStudy executes the study transition command to publish the study +func publishStudy(studyID string) error { + cmd := exec.CommandContext(context.Background(), "./prolific", "study", "transition", + "-a", "PUBLISH", + studyID) + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("command failed: %w\nOutput: %s", err, string(output)) + } + + if len(output) > 0 { + fmt.Printf("%s %s%s\n", colorGray, strings.TrimSpace(string(output)), colorReset) + } + return nil } + +// tailResponses continuously polls and displays new batch responses +func tailResponses(ctx context.Context, batchID, label, labelColor string) { + seenResponses := make(map[string]bool) + pollInterval := 3 * time.Second + + // Initial header + fmt.Printf("\n%s%s[%s] Starting response monitor...%s\n", colorBold, labelColor, label, colorReset) + + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + fmt.Printf("%s%s[%s] Stopping response monitor%s\n", colorBold, labelColor, label, colorReset) + return + case <-ticker.C: + cmd := exec.CommandContext(ctx, "./prolific", "aitaskbuilder", "batch", "responses", + "-b", batchID) + + output, err := cmd.CombinedOutput() + if err != nil { + // Silently continue on error - batch might not have responses yet + continue + } + + // Parse output to find individual responses + lines := strings.Split(string(output), "\n") + var currentResponseID string + var responseLines []string + + for _, line := range lines { + if strings.HasPrefix(line, "Response ") && len(responseLines) > 0 { + // Process previous response + if currentResponseID != "" && !seenResponses[currentResponseID] { + printTailedResponse(label, labelColor, responseLines) + seenResponses[currentResponseID] = true + } + responseLines = []string{line} + currentResponseID = "" + } else if strings.HasPrefix(strings.TrimSpace(line), "ID: ") && currentResponseID == "" { + // Extract response ID + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + currentResponseID = strings.TrimSpace(parts[1]) + } + responseLines = append(responseLines, line) + } else if line != "" && !strings.HasPrefix(line, "AI Task Builder Batch Responses:") && + !strings.HasPrefix(line, "Batch ID:") && + !strings.HasPrefix(line, "Total Responses:") { + responseLines = append(responseLines, line) + } + } + + // Process last response + if currentResponseID != "" && !seenResponses[currentResponseID] && len(responseLines) > 0 { + printTailedResponse(label, labelColor, responseLines) + seenResponses[currentResponseID] = true + } + } + } +} + +// printTailedResponse prints a formatted response with color coding +func printTailedResponse(label, labelColor string, lines []string) { + fmt.Printf("\n%s%s┌─ [%s] New Response ─────────────────────────────────%s\n", + colorBold, labelColor, label, colorReset) + + for _, line := range lines { + // Highlight important fields + if strings.Contains(line, "Text:") || strings.Contains(line, "Selected Options:") { + fmt.Printf("%s%s│%s %s%s\n", labelColor, colorBold, colorReset, line, colorReset) + } else { + fmt.Printf("%s│%s %s%s\n", labelColor, colorReset, colorGray, line) + } + } + + fmt.Printf("%s%s└────────────────────────────────────────────────────────%s\n", + colorBold, labelColor, colorReset) +} From cf0019cb786eed91fa81a35673fa62a427699ac9 Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Thu, 4 Dec 2025 09:07:51 +0000 Subject: [PATCH 2/3] feat: add demo files and ASCII banners for AITB orchestration Add example files for pairwise evaluation demos: - pairwise-evaluation.csv: Standard pairwise comparison dataset - pairwise-evaluation-xmas.csv: Christmas-themed pairwise evaluation data - pairwise-evaluation-instructions.json: Instructions for pairwise tasks Add ASCII art banners: - banner.txt: Prolific ASCII art banner for root directory - docs/scripts/banner.txt: Banner for orchestration script --- banner.txt | 17 +++++ .../pairwise-evaluation-instructions.json | 72 +++++++++++++++++++ docs/examples/pairwise-evaluation-xmas.csv | 21 ++++++ docs/examples/pairwise-evaluation.csv | 6 ++ docs/scripts/banner.txt | 19 +++++ 5 files changed, 135 insertions(+) create mode 100644 banner.txt create mode 100644 docs/examples/pairwise-evaluation-instructions.json create mode 100644 docs/examples/pairwise-evaluation-xmas.csv create mode 100644 docs/examples/pairwise-evaluation.csv create mode 100644 docs/scripts/banner.txt diff --git a/banner.txt b/banner.txt new file mode 100644 index 0000000..3fe803d --- /dev/null +++ b/banner.txt @@ -0,0 +1,17 @@ + + + ******+ + *** ****** **+ + ******* ****** ******* + ******* ***** ******* + ******+ ***** ******* + ** ****** ***** ******* *** + ***** ****** ***** ***** ****** + ********* +***** **** ***** ********* + ********** ********* ***** **********+ + ******** **** ******** ********* + +************************* + *********+ ******** ********+ ********** + ********************* +********************* + ******************* ******************** + \ No newline at end of file diff --git a/docs/examples/pairwise-evaluation-instructions.json b/docs/examples/pairwise-evaluation-instructions.json new file mode 100644 index 0000000..8fb1189 --- /dev/null +++ b/docs/examples/pairwise-evaluation-instructions.json @@ -0,0 +1,72 @@ +[ + { + "type": "multiple_choice", + "created_by": "Researcher", + "description": "Which response is better overall?", + "options": [ + { + "label": "Response 1", + "value": "response1" + }, + { + "label": "Response 2", + "value": "response2" + }, + { + "label": "Both equally good", + "value": "equal" + }, + { + "label": "Both equally bad", + "value": "neither" + } + ], + "answer_limit": 1 + }, + { + "type": "multiple_choice", + "created_by": "Researcher", + "description": "Is Response 1 factually accurate? You may use the internet to look for factuality references.", + "options": [ + { + "label": "Yes", + "value": "yes" + }, + { + "label": "No", + "value": "no" + }, + { + "label": "Unsure", + "value": "unsure" + } + ], + "answer_limit": 1 + }, + { + "type": "multiple_choice", + "created_by": "Researcher", + "description": "Is Response 2 factually accurate? You may use the internet to look for factuality references.", + "options": [ + { + "label": "Yes", + "value": "yes" + }, + { + "label": "No", + "value": "no" + }, + { + "label": "Unsure", + "value": "unsure" + } + ], + "answer_limit": 1 + }, + { + "type": "free_text", + "created_by": "Researcher", + "description": "Please explain your evaluation. What makes your chosen response better? Consider factors like accuracy, completeness, clarity, and helpfulness.", + "answer_limit": 1 + } +] diff --git a/docs/examples/pairwise-evaluation-xmas.csv b/docs/examples/pairwise-evaluation-xmas.csv new file mode 100644 index 0000000..a0fb4fe --- /dev/null +++ b/docs/examples/pairwise-evaluation-xmas.csv @@ -0,0 +1,21 @@ +Prompt,Response 1,Response 2 +"Explain the origins of Santa Claus in simple terms.","Santa Claus is based on Saint Nicholas, a real person who lived in Turkey around 300 AD. He was known for giving gifts to the poor. Over time, different cultures added their own traditions, and he became the jolly red-suited figure we know today who delivers presents on Christmas Eve.","Santa Claus comes from the Dutch figure Sinterklaas, which itself derives from Saint Nicholas of Myra. This 4th-century Christian bishop was famous for his generosity and gift-giving. The modern image of Santa - with his red suit, white beard, and reindeer - was popularized in America during the 19th and 20th centuries through poems, advertisements, and Coca-Cola marketing campaigns." +"What Christmas carol was originally written as a Thanksgiving song?","'Jingle Bells' was originally written as a Thanksgiving song in 1857 by James Lord Pierpont. It was called 'One Horse Open Sleigh' and had nothing to do with Christmas initially, but it became associated with Christmas because of its winter theme.","The carol 'Jingle Bells' was actually composed for Thanksgiving, not Christmas. It was written in 1850 by James Pierpont." +"How do you make classic gingerbread cookies?","Mix flour, ginger, cinnamon, butter, brown sugar, and molasses. Roll out the dough, cut into shapes (like gingerbread men), and bake at 350°F for 10-12 minutes. Decorate with icing once cooled.","To make gingerbread cookies: 1) Cream together butter and brown sugar 2) Beat in molasses and an egg 3) In a separate bowl, combine flour, baking soda, salt, ginger, cinnamon, cloves, and nutmeg 4) Gradually mix dry ingredients into wet ingredients 5) Chill dough for at least 2 hours 6) Roll out to 1/4 inch thickness on a floured surface 7) Cut into desired shapes 8) Bake at 350°F (175°C) for 8-10 minutes 9) Cool completely before decorating with royal icing. These cookies keep well in an airtight container for up to two weeks!" +"Why do we hang stockings at Christmas?","We hang stockings because of the legend of Saint Nicholas. The story goes that he threw gold coins down a chimney to help a poor family, and the coins landed in stockings that were hanging by the fire to dry. Now we hang stockings hoping Santa will fill them with treats!","The tradition of hanging Christmas stockings comes from a legend about Saint Nicholas. According to the tale, he helped a poor man who couldn't afford dowries for his three daughters by secretly dropping gold coins down their chimney on three separate occasions. The coins are said to have landed in stockings or shoes left by the fireplace to dry. This act of anonymous generosity inspired the custom of children hanging stockings by the fireplace on Christmas Eve, hoping to find them filled with gifts by morning." +"What are the names of Santa's reindeer?","Santa has nine reindeer: Dasher, Dancer, Prancer, Vixen, Comet, Cupid, Donner, Blitzen, and Rudolph (the one with the red nose).","Santa's reindeer are Dasher, Dancer, Prancer, Vixen, Comet, Cupid, Donner (originally Donder), and Blitzen. These first eight were introduced in the 1823 poem 'A Visit from St. Nicholas.' Rudolph, the red-nosed reindeer, was added much later in 1939 by Robert L. May in a department store promotional booklet." +"Write a short poem about decorating a Christmas tree.","Lights that twinkle bright / Ornaments of red and gold / Family together, hearts light / Memories of years old","Baubles hung with care so true / Tinsel streams of silver gleam / Angel watches from the top's view / The tree fulfills a festive dream" +"How do you make hot chocolate for Christmas?","Heat milk in a pot, add cocoa powder and sugar, stir until smooth. Pour into mugs and top with whipped cream or marshmallows.","For perfect Christmas hot chocolate: 1) Heat 2 cups of milk (or milk alternative) in a saucepan over medium heat 2) Whisk in 2 tablespoons cocoa powder, 2 tablespoons sugar, and a pinch of salt 3) For extra richness, add 2 ounces chopped dark chocolate 4) Heat until steaming but not boiling, whisking constantly 5) Add 1/4 teaspoon vanilla extract 6) Pour into mugs and top with whipped cream, marshmallows, or a candy cane stirrer 7) Optional: add a pinch of cinnamon or peppermint extract for festive flavor" +"Explain the story of the Nutcracker ballet.","The Nutcracker is about a girl named Clara who gets a nutcracker doll for Christmas. At night, it comes to life and battles the Mouse King. Clara helps him win, and he turns into a prince who takes her on a magical journey through the Land of Sweets.","The Nutcracker ballet, set on Christmas Eve, tells the story of young Clara Stahlbaum who receives a nutcracker doll from her godfather Drosselmeyer. That night, she dreams the nutcracker comes alive and leads her toy soldiers in battle against the Mouse King and his army. After Clara saves the Nutcracker by throwing her slipper at the Mouse King, he transforms into a handsome prince. He takes Clara on a journey through the Land of Snow to the Kingdom of Sweets, where the Sugar Plum Fairy entertains them with dances from around the world. The ballet, with music by Tchaikovsky, premiered in 1892 and has become a beloved Christmas tradition." +"What is an Advent calendar?","An Advent calendar is a special calendar used to count down the days until Christmas, usually starting December 1st. Each day you open a little door or window to reveal a treat, picture, or small gift inside.","An Advent calendar is a countdown calendar used during the Advent season, the four weeks leading up to Christmas. Originating in 19th-century Germany, these calendars traditionally have 24 doors or windows, one for each day from December 1st to December 24th (Christmas Eve). Each day, you open one compartment to reveal a surprise - historically a Bible verse or picture, but modern versions often contain chocolates, small toys, beauty products, or other treats. The tradition helps build anticipation for Christmas Day." +"How do you roast chestnuts?","To roast chestnuts, cut an X on each chestnut, place them on a baking sheet, and roast at 400°F for about 20 minutes until the shells peel back.","To roast chestnuts properly: 1) Use fresh chestnuts and check they're firm (discard any that float in water) 2) Score each chestnut with an X cut on the flat side - this prevents them from exploding 3) Soak in water for 10 minutes 4) Place cut-side up on a baking sheet 5) Roast at 425°F (220°C) for 20-25 minutes until shells curl back 6) Remove from oven and cover with a damp towel for 5 minutes to steam 7) Peel while still warm - both the outer shell and inner skin 8) Enjoy! The flesh should be tender and slightly sweet." +"What are Christmas crackers?","Christmas crackers are festive tubes wrapped in colorful paper. When you pull both ends with another person, they make a popping sound and reveal small gifts, a paper crown, and a joke inside.","Christmas crackers are traditional British holiday table decorations. Invented in 1847 by London confectioner Tom Smith, they consist of a cardboard tube wrapped in brightly colored twisted paper. When two people pull the ends, a small friction device creates a 'crack' sound. Inside are typically three items: a paper crown (tissue party hat), a small trinket or toy, and a printed joke or riddle. Crackers are pulled before or during Christmas dinner, and tradition dictates everyone must wear their paper crown throughout the meal." +"How long does it take for a Christmas tree to grow?","It takes about 7-10 years for a Christmas tree to grow to a typical height of 6-7 feet.","A Christmas tree typically takes 7-10 years to reach the standard 6-7 feet height suitable for harvesting. However, the time varies by species: faster-growing varieties like Leyland Cypress can reach harvestable size in 6-7 years, while slower-growing trees like Fraser Fir or Balsam Fir may take 8-12 years. Growers must also factor in the initial 2-3 years in nursery beds before transplanting to fields, making the total process from seed to harvest 10-15 years. For every tree harvested, responsible farmers plant 1-3 seedlings to maintain sustainability." +"Why do we put a star on top of the Christmas tree?","We put a star on top of the tree to represent the Star of Bethlehem that guided the Three Wise Men to baby Jesus in the Bible story.","The star atop the Christmas tree represents the Star of Bethlehem (also called the Christmas Star) that, according to the Gospel of Matthew, guided the Magi (Three Wise Men) to the birthplace of Jesus. This tradition began in Germany and became widespread during the Victorian era. Some families use an angel tree topper instead, representing the angels who announced Jesus's birth to the shepherds. Both toppers symbolize the religious significance of Christmas and serve as the crowning decoration of the tree." +"What ingredients go in traditional Christmas pudding?","Christmas pudding contains dried fruits like raisins and currants, flour, sugar, eggs, spices like cinnamon and nutmeg, and sometimes brandy or rum.","Traditional Christmas pudding (also called plum pudding) is a rich steamed dessert made with: suet or butter, brown sugar, breadcrumbs, flour, eggs, mixed dried fruits (raisins, sultanas, currants, candied peel), chopped nuts, mixed spices (cinnamon, nutmeg, allspice), orange and lemon zest, and alcohol (usually brandy, rum, or stout). The mixture is traditionally made on 'Stir-up Sunday' (the last Sunday before Advent), when each family member stirs the pudding and makes a wish. It's then aged for several weeks, steamed again on Christmas Day, and served flaming with brandy butter or custard." +"Write a tongue twister about Christmas shopping.","Santa's shop stocks shiny silver sleighs and sixty-six silk scarves.","Surely Susan shops for several special sparkling silver stars, swiftly selecting stunning stockings, skipping slowly through stores." +"What is a Yule log?","A Yule log is a special chocolate cake made to look like a log, covered in chocolate frosting and decorated to look like bark. It's a traditional Christmas dessert.","The term 'Yule log' has two meanings. Originally, it referred to a large wooden log burned in the hearth during Christmas celebrations, a pagan tradition predating Christianity. Families would select a large log, often from oak or ash, bring it inside on Christmas Eve, and keep it burning throughout the twelve days of Christmas for good luck. The modern 'Yule log' (Bûche de Noël) is a French dessert created in the 19th century: a sponge cake rolled with cream or buttercream filling, covered in chocolate frosting textured to resemble bark, and decorated with powdered sugar 'snow,' meringue mushrooms, and marzipan holly." +"How many ghosts visit Scrooge in 'A Christmas Carol'?","Four ghosts visit Scrooge: Jacob Marley and the three spirits of Christmas Past, Present, and Future.","In Charles Dickens's 'A Christmas Carol,' Ebenezer Scrooge is visited by four supernatural beings: first his former business partner Jacob Marley (in chains), who warns him and announces the coming of three spirits; then the Ghost of Christmas Past (showing his youth and lost love); the Ghost of Christmas Present (revealing how others celebrate despite hardship); and finally the Ghost of Christmas Yet to Come (a silent, hooded figure showing his lonely death if he doesn't change). Some count only three ghosts (excluding Marley as he wasn't one of the Christmas spirits proper), but the complete answer is four apparitions total." +"What are mince pies?","Mince pies are small, sweet pies filled with mincemeat (a mixture of dried fruits, spices, and sometimes alcohol). They're a traditional Christmas treat in the UK.","Mince pies are traditional British Christmas pastries consisting of small, individual shortcrust pastry cases filled with 'mincemeat' - a spiced mixture of dried fruits (raisins, currants, sultanas), candied peel, suet or butter, sugar, nuts, and spices like cinnamon, nutmeg, and cloves, often with added brandy or rum. Despite the name, modern mincemeat rarely contains actual meat, though historically it did include beef or mutton. The pies date back to the 13th century and were originally oblong to represent Jesus's manger. They're typically served warm, often with brandy butter, cream, or custard, and it's said that eating one mince pie on each of the 12 days of Christmas brings good luck." +"How do you make a paper snowflake?","Fold a square piece of paper in half, then in half again. Cut shapes along the edges and unfold to reveal your snowflake pattern.","To make a paper snowflake: 1) Start with a square piece of paper (white or colored) 2) Fold it in half diagonally to make a triangle 3) Fold the triangle in half again to make a smaller triangle 4) Fold into thirds to create a narrow cone shape 5) Trim the bottom edge straight 6) Cut various shapes (triangles, curves, notches) along all three edges - avoid cutting across the center fold entirely 7) Carefully unfold to reveal your unique six-pointed snowflake 8) Flatten under a book if needed. The more intricate your cuts, the more delicate and detailed your snowflake will be. Remember: real snowflakes have six-fold symmetry, so proper folding into sixths creates realistic designs!" +"What is the Polar Express?","The Polar Express is a children's book and movie about a magical train that takes a boy to the North Pole on Christmas Eve to meet Santa.","'The Polar Express' is a 1985 children's picture book by Chris Van Allsburg about a young boy who, on Christmas Eve, boards a magical train bound for the North Pole. During the journey through snow-covered landscapes, he makes friends with other children, meets the train's conductor, and ultimately receives the first gift of Christmas from Santa - a silver sleigh bell that only believers can hear. The story explores themes of faith, belief, and the magic of Christmas. In 2004, it was adapted into an animated film featuring Tom Hanks, using motion-capture technology. Both versions have become beloved Christmas traditions, and real-life 'Polar Express' train rides are now offered worldwide during the holiday season." diff --git a/docs/examples/pairwise-evaluation.csv b/docs/examples/pairwise-evaluation.csv new file mode 100644 index 0000000..64f7158 --- /dev/null +++ b/docs/examples/pairwise-evaluation.csv @@ -0,0 +1,6 @@ +Prompt,Response 1,Response 2 +"Explain what photosynthesis is in simple terms.","Photosynthesis is when plants make food using sunlight, water, and carbon dioxide. They take in these things and produce glucose (sugar) for energy, and release oxygen as a byproduct. This happens in the chloroplasts of plant cells.","Photosynthesis is the process by which green plants and some other organisms use sunlight to synthesize nutrients from carbon dioxide and water. It generally involves the green pigment chlorophyll and generates oxygen as a byproduct." +"Write a haiku about autumn leaves.","Crimson leaves falling / Dancing in the gentle wind / Nature's tapestry","Red and gold descend / Autumn's beauty on display / Life's cycle continues" +"What's the capital of Australia?","The capital of Australia is Sydney. It's the largest city in Australia and home to famous landmarks like the Sydney Opera House and Sydney Harbour Bridge.","The capital of Australia is Canberra. It was purpose-built as the capital and is located in the Australian Capital Territory (ACT), between Sydney and Melbourne." +"How do you make a cup of tea?","To make tea, boil water, put a tea bag in a cup, pour the hot water over it, wait 3-5 minutes, remove the tea bag, and add milk or sugar if desired.","Making tea is simple: 1) Boil fresh water 2) Warm your teapot or cup 3) Add tea leaves or a tea bag 4) Pour hot water over the tea 5) Steep for 3-5 minutes depending on tea type 6) Remove tea leaves/bag 7) Add milk, sugar, lemon, or honey to taste if desired. Enjoy your perfect cup!" +"Translate 'Hello, how are you?' to Spanish.","'Hello, how are you?' in Spanish is 'Hola, ¿cómo estás?' if you're speaking informally, or 'Hola, ¿cómo está?' if you're being formal.","The translation is 'Hola, ¿cómo estás?'" diff --git a/docs/scripts/banner.txt b/docs/scripts/banner.txt new file mode 100644 index 0000000..2c47c86 --- /dev/null +++ b/docs/scripts/banner.txt @@ -0,0 +1,19 @@ + /$$$$$$$ /$$ /$$ /$$$$$$ /$$ /$$$$$$ /$$ /$$$$$$ +| $$__ $$ | $$|__/ /$$__ $$|__/ /$$__ $$| $$ |_ $$_/ +| $$ \ $$ /$$$$$$ /$$$$$$ | $$ /$$| $$ \__/ /$$ /$$$$$$$ | $$ \__/| $$ | $$ +| $$$$$$$//$$__ $$ /$$__ $$| $$| $$| $$$$ | $$ /$$_____//$$$$$$| $$ | $$ | $$ +| $$____/| $$ \__/| $$ \ $$| $$| $$| $$_/ | $$| $$ |______/| $$ | $$ | $$ +| $$ | $$ | $$ | $$| $$| $$| $$ | $$| $$ | $$ $$| $$ | $$ +| $$ | $$ | $$$$$$/| $$| $$| $$ | $$| $$$$$$$ | $$$$$$/| $$$$$$$$ /$$$$$$ +|__/ |__/ \______/ |__/|__/|__/ |__/ \_______/ \______/ |________/|______/ + + + + /$$$$$$$ +| $$__ $$ +| $$ \ $$ /$$$$$$ /$$$$$$/$$$$ /$$$$$$ +| $$ | $$ /$$__ $$| $$_ $$_ $$ /$$__ $$ +| $$ | $$| $$$$$$$$| $$ \ $$ \ $$| $$ \ $$ +| $$ | $$| $$_____/| $$ | $$ | $$| $$ | $$ +| $$$$$$$/| $$$$$$$| $$ | $$ | $$| $$$$$$/ +|_______/ \_______/|__/ |__/ |__/ \______/ \ No newline at end of file From 5aa672a12a6facbfef0b82b2a67814b421dfe71b Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Sun, 7 Dec 2025 22:42:59 +0000 Subject: [PATCH 3/3] fix: correct AITB example files based on code review feedback - Remove invalid answer_limit property from free_text instructions - Remove total_available_places from AITB study example as it's auto-calculated --- docs/examples/example-instructions.json | 3 +-- docs/examples/pairwise-evaluation-instructions.json | 3 +-- docs/examples/standard-sample-aitaskbuilder.json | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/examples/example-instructions.json b/docs/examples/example-instructions.json index 8031b8c..0d57c78 100644 --- a/docs/examples/example-instructions.json +++ b/docs/examples/example-instructions.json @@ -18,7 +18,6 @@ { "type": "free_text", "created_by": "Researcher", - "description": "Please share the reasons for your choice.", - "answer_limit": 1 + "description": "Please share the reasons for your choice." } ] diff --git a/docs/examples/pairwise-evaluation-instructions.json b/docs/examples/pairwise-evaluation-instructions.json index 8fb1189..62026bc 100644 --- a/docs/examples/pairwise-evaluation-instructions.json +++ b/docs/examples/pairwise-evaluation-instructions.json @@ -66,7 +66,6 @@ { "type": "free_text", "created_by": "Researcher", - "description": "Please explain your evaluation. What makes your chosen response better? Consider factors like accuracy, completeness, clarity, and helpfulness.", - "answer_limit": 1 + "description": "Please explain your evaluation. What makes your chosen response better? Consider factors like accuracy, completeness, clarity, and helpfulness." } ] diff --git a/docs/examples/standard-sample-aitaskbuilder.json b/docs/examples/standard-sample-aitaskbuilder.json index 3668f23..7e51035 100644 --- a/docs/examples/standard-sample-aitaskbuilder.json +++ b/docs/examples/standard-sample-aitaskbuilder.json @@ -5,7 +5,6 @@ "prolific_id_option": "not_required", "completion_code": "AIEVAL01", "completion_option": "code", - "total_available_places": 10, "estimated_completion_time": 10, "maximum_allowed_time": 30, "reward": 100,