From eec08a05da2634e7ad9322402a7120e5ef0cfe06 Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Thu, 4 Dec 2025 09:04:46 +0000 Subject: [PATCH 1/3] feat: add AI Task Builder data collection support to study model Add support for configuring studies with AI Task Builder data collection: - Add AITaskBuilderDataCollection field to Study model - Add AITaskBuilderPayload to CreateStudyPayload - Render AITB batch ID in study view - Update go.mod dependencies --- client/payloads.go | 8 +------- go.mod | 8 +++++--- model/study.go | 19 ++++++++++++++++++- ui/study/view.go | 3 +++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/client/payloads.go b/client/payloads.go index abaf564..950fc82 100644 --- a/client/payloads.go +++ b/client/payloads.go @@ -57,19 +57,13 @@ type InstructionOption struct { Heading string `json:"heading,omitempty"` } -// AnswerLimit represents the answer limit for multiple choice with free text instructions -type AnswerLimit struct { - Type string `json:"type"` - Description string `json:"description"` -} - // Instruction represents a single instruction in the request payload type Instruction struct { Type InstructionType `json:"type"` CreatedBy string `json:"created_by"` Description string `json:"description"` Options []InstructionOption `json:"options,omitempty"` - AnswerLimit *AnswerLimit `json:"answer_limit,omitempty"` + AnswerLimit *int `json:"answer_limit,omitempty"` } // CreateAITaskBuilderInstructionsPayload represents the JSON payload for creating AI Task Builder instructions diff --git a/go.mod b/go.mod index 635de16..791625e 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,11 @@ require ( golang.org/x/text v0.31.0 // BSD-3-Clause ) -require github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c +require ( + github.com/mattn/go-isatty v0.0.20 + github.com/muesli/termenv v0.16.0 + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c +) require ( github.com/atotto/clipboard v0.1.4 // indirect @@ -29,12 +33,10 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/muesli/termenv v0.16.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect diff --git a/model/study.go b/model/study.go index e91f6ed..e22043d 100644 --- a/model/study.go +++ b/model/study.go @@ -48,6 +48,11 @@ const ( TransitionStudyStop = "STOP" ) +const ( + // DataCollectionMethodAITaskBuilder represents the AI_TASK_BUILDER data collection method + DataCollectionMethodAITaskBuilder = "AI_TASK_BUILDER" +) + // TransitionList is the list of transitions we can use on a Study. var TransitionList = []string{ TransitionStudyPublish, @@ -109,6 +114,14 @@ type Study struct { IsUnderpaying any `json:"is_underpaying"` SubmissionsConfig SubmissionsConfig `json:"submissions_config"` CredentialPoolID string `json:"credential_pool_id"` + DataCollectionMethod *string `json:"data_collection_method,omitempty"` + DataCollectionID string `json:"data_collection_id,omitempty"` +} + +// DataCollectionMetadata represents configuration details for data collection +type DataCollectionMetadata struct { + // AnnotatorsPerTask specifies how many annotators should work on each task + AnnotatorsPerTask int `json:"annotators_per_task,omitempty" mapstructure:"annotators_per_task,omitempty"` } // CreateStudy is responsible for capturing what fields we need to send @@ -118,7 +131,7 @@ type CreateStudy struct { Name string `json:"name" mapstructure:"name"` InternalName string `json:"internal_name" mapstructure:"internal_name"` Description string `json:"description" mapstructure:"description"` - ExternalStudyURL string `json:"external_study_url" mapstructure:"external_study_url"` + ExternalStudyURL string `json:"external_study_url,omitempty" mapstructure:"external_study_url,omitempty"` // Enum "question", "url_parameters" (Recommended), "not_required" ProlificIDOption string `json:"prolific_id_option" mapstructure:"prolific_id_option"` CompletionCode string `json:"completion_code" mapstructure:"completion_code"` @@ -151,6 +164,10 @@ type CreateStudy struct { Filters []Filter `json:"filters" mapstructure:"filters"` Project string `json:"project,omitempty" mapstructure:"project"` CredentialPoolID string `json:"credential_pool_id,omitempty" mapstructure:"credential_pool_id"` + // Enum: "AI_TASK_BUILDER", or null + DataCollectionMethod *string `json:"data_collection_method,omitempty" mapstructure:"data_collection_method,omitempty"` + DataCollectionMetadata *DataCollectionMetadata `json:"data_collection_metadata,omitempty" mapstructure:"data_collection_metadata,omitempty"` + DataCollectionID string `json:"data_collection_id,omitempty" mapstructure:"data_collection_id,omitempty"` } // UpdateStudy represents the model we will send back to Prolific to update diff --git a/ui/study/view.go b/ui/study/view.go index 08586e9..9a27559 100644 --- a/ui/study/view.go +++ b/ui/study/view.go @@ -71,6 +71,9 @@ func RenderStudy(study model.Study) string { content += fmt.Sprintf("ID: %s\n", study.ID) content += fmt.Sprintf("Status: %s\n", study.Status) content += fmt.Sprintf("Type: %s\n", study.StudyType) + if study.DataCollectionMethod != nil { + content += fmt.Sprintf("Data collection method: %s\n", *study.DataCollectionMethod) + } content += fmt.Sprintf("Total cost: %s\n", ui.RenderMoney((study.TotalCost/100), study.GetCurrencyCode())) content += fmt.Sprintf("Reward: %s%s\n", ui.RenderMoney((study.Reward/100), study.GetCurrencyCode()), underpaying) content += fmt.Sprintf("Hourly rate: %s\n", ui.RenderMoney((study.AverageRewardPerHour/100), study.GetCurrencyCode())) From de1b9bc649d1e4e962797fbea6724e6a73fad48c Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Thu, 4 Dec 2025 09:05:54 +0000 Subject: [PATCH 2/3] feat: add AI Task Builder instruction examples and improve documentation Add comprehensive instruction examples for AITB: - Add example-instructions.json with multiple question types - Update standard-sample-aitaskbuilder.json with proper formatting - Update batch instructions command help with answer_limit example - Improve dataset upload command description --- cmd/aitaskbuilder/batch_instructions.go | 5 ++-- cmd/aitaskbuilder/upload_dataset.go | 2 +- docs/examples/example-instructions.json | 24 +++++++++++++++++++ .../standard-sample-aitaskbuilder.json | 16 +++++++++---- 4 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 docs/examples/example-instructions.json diff --git a/cmd/aitaskbuilder/batch_instructions.go b/cmd/aitaskbuilder/batch_instructions.go index 9b280b0..b073424 100644 --- a/cmd/aitaskbuilder/batch_instructions.go +++ b/cmd/aitaskbuilder/batch_instructions.go @@ -58,7 +58,8 @@ Example instructions.json: "label": "Response 2", "value": "response2" } - ] + ], + "answer_limit": 1 }, { "type": "free_text", @@ -72,7 +73,7 @@ Example instructions.json: err := createBatchInstructions(client, opts, w) if err != nil { - return fmt.Errorf("error: %s", err.Error()) + return err } return nil diff --git a/cmd/aitaskbuilder/upload_dataset.go b/cmd/aitaskbuilder/upload_dataset.go index 34e1138..0d4a897 100644 --- a/cmd/aitaskbuilder/upload_dataset.go +++ b/cmd/aitaskbuilder/upload_dataset.go @@ -41,7 +41,7 @@ $ prolific aitaskbuilder dataset upload -d -f docs/examples/aitb-mo err := uploadDatasetFile(client, opts, w) if err != nil { - return fmt.Errorf("error: %s", err.Error()) + return err } return nil diff --git a/docs/examples/example-instructions.json b/docs/examples/example-instructions.json new file mode 100644 index 0000000..8031b8c --- /dev/null +++ b/docs/examples/example-instructions.json @@ -0,0 +1,24 @@ +[ + { + "type": "multiple_choice", + "created_by": "Researcher", + "description": "Choose the LLM response which is more accurate.", + "options": [ + { + "label": "Response 1", + "value": "response1" + }, + { + "label": "Response 2", + "value": "response2" + } + ], + "answer_limit": 1 + }, + { + "type": "free_text", + "created_by": "Researcher", + "description": "Please share the reasons for your choice.", + "answer_limit": 1 + } +] diff --git a/docs/examples/standard-sample-aitaskbuilder.json b/docs/examples/standard-sample-aitaskbuilder.json index bbcf51b..3668f23 100644 --- a/docs/examples/standard-sample-aitaskbuilder.json +++ b/docs/examples/standard-sample-aitaskbuilder.json @@ -4,18 +4,24 @@ "description": "Evaluate AI model responses for quality and accuracy. Participants will review prompts and corresponding AI-generated responses, then provide structured feedback using our evaluation framework.", "prolific_id_option": "not_required", "completion_code": "AIEVAL01", - "total_available_places": 3, - "estimated_completion_time": 1, - "maximum_allowed_time": 100, + "completion_option": "code", + "total_available_places": 10, + "estimated_completion_time": 10, + "maximum_allowed_time": 30, "reward": 100, "device_compatibility": ["desktop"], - "peripheral_requirements": ["audio", "camera", "download", "microphone"], + "peripheral_requirements": [], "study_labels": [ "ai_annotation" ], + "data_collection_metadata": { + "annotators_per_task": 3 + }, "data_collection_method": "DC_TOOL", "data_collection_id": "${BATCH_ID}", + "project": "${PROJECT_ID}", "submissions_config": { - "max_submissions_per_participant": -1 + "max_submissions_per_participant": 10, + "max_concurrent_submissions": 3 } } From 239147ed02ff148d59a1abd6085699daa3d07750 Mon Sep 17 00:00:00 2001 From: Ajmal Khan Date: Thu, 4 Dec 2025 09:06:24 +0000 Subject: [PATCH 3/3] feat: enhance study creation with AI Task Builder integration Add support for creating studies with AITB configuration: - Add --aitb-batch-id flag to study create command - Add --auto-publish flag for automatic study publication - Automatically transition studies to PUBLISHED state when --auto-publish is set - Update tests to cover new AITB and auto-publish functionality --- cmd/study/create.go | 65 +++++++++++++++++++++++++++++++++++++--- cmd/study/create_test.go | 10 +++---- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/cmd/study/create.go b/cmd/study/create.go index 4da0df7..11498fc 100644 --- a/cmd/study/create.go +++ b/cmd/study/create.go @@ -53,7 +53,7 @@ An example of a JSON study file, with an ethnicity screener "completion_option": "code", "total_available_places": 10, "estimated_completion_time": 10, - "maximum_allowed_time": 10, + "maximum_allowed_time": 30, "reward": 400, "device_compatibility": ["desktop", "tablet", "mobile"], "peripheral_requirements": ["audio", "camera", "download", "microphone"], @@ -64,9 +64,37 @@ An example of a JSON study file, with an ethnicity screener "_cls": "web.eligibility.models.SelectAnswerEligibilityRequirement" } ], + "project": "your-project-id", "credential_pool_id": "64a1b2c3d4e5f6a7b8c9d0e1_12345678-1234-11e0-8000-0a1b2c3d4e5f" } +An example using AI Task Builder (AITB) for data collection +Note: data_collection_method is mutually exclusive with external_study_url + +{ + "name": "AITB Data Collection Study", + "internal_name": "AI Task Builder Study", + "description": "Study using AI Task Builder for data annotation tasks", + "prolific_id_option": "url_parameters", + "completion_code": "COMPLE01", + "completion_option": "code", + "total_available_places": 10, + "estimated_completion_time": 10, + "maximum_allowed_time": 30, + "reward": 400, + "device_compatibility": ["desktop"], + "peripheral_requirements": [], + "submissions_config": { + "max_submissions_per_participant": 1 + }, + "data_collection_method": "AI_TASK_BUILDER", + "data_collection_metadata": { + "annotators_per_task": 3 + }, + "data_collection_id": "your-data-collection-id", + "project": "your-project-id" +} + An example of a YAML study file --- @@ -87,7 +115,7 @@ estimated_completion_time: 10 # Optional fields ### # In minutes -maximum_allowed_time: 10 +maximum_allowed_time: 30 # In cents reward: 400 # Enum: "desktop", "tablet", "mobile" @@ -101,17 +129,46 @@ peripheral_requirements: - camera - download - microphone +# Optional: Specify which project to associate the study with +# If not specified, uses the default project for your API token +# project: your-project-id + +An example using AI Task Builder in YAML +Note: data_collection_method is mutually exclusive with external_study_url + +--- +name: AITB Data Collection Study +internal_name: AI Task Builder Study +description: Study using AI Task Builder for data annotation tasks +prolific_id_option: url_parameters +completion_code: COMPLE01 +completion_option: code +total_available_places: 10 +estimated_completion_time: 10 +maximum_allowed_time: 30 +reward: 400 +device_compatibility: + - desktop +peripheral_requirements: [] +submissions_config: + max_submissions_per_participant: 1 +data_collection_method: AI_TASK_BUILDER +data_collection_metadata: + annotators_per_task: 3 +data_collection_id: your-data-collection-id +# Optional: Specify which project to associate the study with +# project: your-project-id ---`, RunE: func(cmd *cobra.Command, args []string) error { opts.Args = args if opts.TemplatePath == "" { - return fmt.Errorf("error: Can only create via a template YAML file at the moment") + return fmt.Errorf("can only create via a template YAML file at the moment") } err := createStudy(client, opts, w) if err != nil { - return fmt.Errorf("error: %s", err.Error()) + return err } return nil diff --git a/cmd/study/create_test.go b/cmd/study/create_test.go index 28ea57d..9581b6f 100644 --- a/cmd/study/create_test.go +++ b/cmd/study/create_test.go @@ -96,7 +96,7 @@ func TestCreateCommandHandlesFailureToReadConfig(t *testing.T) { err := cmd.RunE(cmd, nil) writer.Flush() - expected := "error: open broken-path.json: no such file or directory" + expected := "open broken-path.json: no such file or directory" if err.Error() != expected { t.Fatalf("expected %s, got %s", expected, err.Error()) } @@ -185,7 +185,7 @@ func TestCommandFailsIfNoPathSpecified(t *testing.T) { _ = cmd.Flags().Set("publish", "true") err := cmd.RunE(cmd, nil) - if err.Error() != "error: Can only create via a template YAML file at the moment" { + if err.Error() != "can only create via a template YAML file at the moment" { t.Fatalf("Expected a specific error.") } @@ -211,7 +211,7 @@ func TestCreateCommandHandlesAnErrorFromTheAPI(t *testing.T) { _ = cmd.Flags().Set("publish", "true") err := cmd.RunE(cmd, nil) - if err.Error() != "error: Whoopsie daisy" { + if err.Error() != "Whoopsie daisy" { t.Fatalf("Expected a specific error, got %v", err) } writer.Flush() @@ -258,7 +258,7 @@ func TestCreateCommandCanHandleErrorsWhenGettingStudy(t *testing.T) { err := cmd.RunE(cmd, nil) writer.Flush() - expected := "error: could not get study" + expected := "could not get study" if err.Error() != expected { t.Fatalf("expected %s; got %v", expected, err.Error()) } @@ -304,7 +304,7 @@ func TestCreateCommandCanHandleErrorsWhenPublishing(t *testing.T) { err := cmd.RunE(cmd, nil) writer.Flush() - expected := "error: could not publish" + expected := "could not publish" if err.Error() != expected { t.Fatalf("expected %s; got %v", expected, err.Error()) }