diff --git a/.rubocop.yml b/.rubocop.yml index e3462a7..ac57ae2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -10,4 +10,28 @@ Style/StringLiteralsInInterpolation: EnforcedStyle: double_quotes Layout/LineLength: - Max: 120 + Enabled: false + +Metrics/MethodLength: + Enabled: false + +Metrics/BlockLength: + Enabled: false + +Naming/MethodParameterName: + Enabled: false + +Metrics/AbcSize: + Enabled: false + +Metrics/ClassLength: + Enabled: false + +Metrics/ParameterLists: + Enabled: false + +Metrics/CyclomaticComplexity: + Enabled: false + +Metrics/PerceivedComplexity: + Enabled: false diff --git a/irt_ruby.gemspec b/irt_ruby.gemspec index 3cdc63e..1a6253f 100644 --- a/irt_ruby.gemspec +++ b/irt_ruby.gemspec @@ -6,8 +6,8 @@ Gem::Specification.new do |spec| spec.authors = ["Alex Kholodniak"] spec.email = ["alexandrkholodniak@gmail.com"] - spec.summary = %q{A Ruby gem that provides implementations of Rasch, Two-Parameter, and Three-Parameter models for Item Response Theory (IRT).} - spec.description = %q{IrtRuby is a Ruby gem that provides implementations of the Rasch model, Two-Parameter model, and Three-Parameter model for Item Response Theory (IRT). It allows you to estimate the abilities of individuals and the difficulties, discriminations, and guessing parameters of items based on their responses to a set of items.} + spec.summary = "A Ruby gem that provides implementations of Rasch, Two-Parameter, and Three-Parameter models for Item Response Theory (IRT)." + spec.description = "IrtRuby is a Ruby gem that provides implementations of the Rasch model, Two-Parameter model, and Three-Parameter model for Item Response Theory (IRT). It allows you to estimate the abilities of individuals and the difficulties, discriminations, and guessing parameters of items based on their responses to a set of items." spec.homepage = "https://github.com/SyntaxSpirits/irt_ruby" spec.license = "MIT" diff --git a/lib/irt_ruby/rasch_model.rb b/lib/irt_ruby/rasch_model.rb index 1095af7..c6ce120 100644 --- a/lib/irt_ruby/rasch_model.rb +++ b/lib/irt_ruby/rasch_model.rb @@ -3,56 +3,123 @@ require "matrix" module IrtRuby - # A class representing the Rasch model for Item Response Theory. + # A class representing the Rasch model for Item Response Theory (ability - difficulty). + # Incorporates: + # - Adaptive learning rate + # - Missing data handling (skip nil) + # - Multiple convergence checks (log-likelihood + parameter updates) class RaschModel - def initialize(data, max_iter: 1000, tolerance: 1e-6, learning_rate: 0.01) + def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, + learning_rate: 0.01, decay_factor: 0.5) + # data: A Matrix or array-of-arrays of responses (0/1 or nil for missing). + # Rows = respondents, Columns = items. + @data = data - @abilities = Array.new(data.row_count) { rand } - @difficulties = Array.new(data.column_count) { rand } + @data_array = data.to_a + num_rows = @data_array.size + num_cols = @data_array.first.size + + # Initialize parameters near zero + @abilities = Array.new(num_rows) { rand(-0.25..0.25) } + @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } + @max_iter = max_iter @tolerance = tolerance + @param_tolerance = param_tolerance @learning_rate = learning_rate + @decay_factor = decay_factor end - # Sigmoid function to calculate probability def sigmoid(x) 1.0 / (1.0 + Math.exp(-x)) end - # Calculate the log-likelihood of the data given the current parameters - def likelihood - likelihood = 0 - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| + def log_likelihood + total_ll = 0.0 + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + prob = sigmoid(@abilities[i] - @difficulties[j]) - likelihood += response == 1 ? Math.log(prob) : Math.log(1 - prob) + total_ll += if resp == 1 + Math.log(prob + 1e-15) + else + Math.log((1 - prob) + 1e-15) + end end end - likelihood + total_ll end - # Update parameters using gradient ascent - def update_parameters - last_likelihood = likelihood - @max_iter.times do |_iter| - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| - prob = sigmoid(@abilities[i] - @difficulties[j]) - error = response - prob - @abilities[i] += @learning_rate * error - @difficulties[j] -= @learning_rate * error - end + def compute_gradient + grad_abilities = Array.new(@abilities.size, 0.0) + grad_difficulties = Array.new(@difficulties.size, 0.0) + + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + + prob = sigmoid(@abilities[i] - @difficulties[j]) + error = resp - prob + + grad_abilities[i] += error + grad_difficulties[j] -= error end - current_likelihood = likelihood - break if (last_likelihood - current_likelihood).abs < @tolerance + end + + [grad_abilities, grad_difficulties] + end + + def apply_gradient_update(grad_abilities, grad_difficulties) + old_abilities = @abilities.dup + old_difficulties = @difficulties.dup + + @abilities.each_index do |i| + @abilities[i] += @learning_rate * grad_abilities[i] + end - last_likelihood = current_likelihood + @difficulties.each_index do |j| + @difficulties[j] += @learning_rate * grad_difficulties[j] end + + [old_abilities, old_difficulties] + end + + def average_param_update(old_abilities, old_difficulties) + deltas = [] + @abilities.each_with_index do |a, i| + deltas << (a - old_abilities[i]).abs + end + @difficulties.each_with_index do |d, j| + deltas << (d - old_difficulties[j]).abs + end + deltas.sum / deltas.size end - # Fit the model to the data def fit - update_parameters + prev_ll = log_likelihood + + @max_iter.times do + grad_abilities, grad_difficulties = compute_gradient + + old_abilities, old_difficulties = apply_gradient_update(grad_abilities, grad_difficulties) + + current_ll = log_likelihood + param_delta = average_param_update(old_abilities, old_difficulties) + + if current_ll < prev_ll + @abilities = old_abilities + @difficulties = old_difficulties + @learning_rate *= @decay_factor + else + ll_diff = (current_ll - prev_ll).abs + + break if ll_diff < @tolerance && param_delta < @param_tolerance + + prev_ll = current_ll + end + end + { abilities: @abilities, difficulties: @difficulties } end end diff --git a/lib/irt_ruby/three_parameter_model.rb b/lib/irt_ruby/three_parameter_model.rb index 73eb299..d8ddd4d 100644 --- a/lib/irt_ruby/three_parameter_model.rb +++ b/lib/irt_ruby/three_parameter_model.rb @@ -3,66 +3,161 @@ require "matrix" module IrtRuby - # A class representing the Three-Parameter model for Item Response Theory. + # A class representing the Three-Parameter model (3PL) for Item Response Theory. + # Incorporates: + # - Adaptive learning rate + # - Missing data handling + # - Parameter clamping for discrimination, guessing + # - Multiple convergence checks + # - Separate gradient calculation & updates class ThreeParameterModel - def initialize(data, max_iter: 1000, tolerance: 1e-6, learning_rate: 0.01) + def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, + learning_rate: 0.01, decay_factor: 0.5) @data = data - @abilities = Array.new(data.row_count) { rand } - @difficulties = Array.new(data.column_count) { rand } - @discriminations = Array.new(data.column_count) { rand } - @guessings = Array.new(data.column_count) { rand * 0.3 } - @max_iter = max_iter - @tolerance = tolerance - @learning_rate = learning_rate + @data_array = data.to_a + num_rows = @data_array.size + num_cols = @data_array.first.size + + # Typical initialization for 3PL + @abilities = Array.new(num_rows) { rand(-0.25..0.25) } + @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } + @discriminations = Array.new(num_cols) { rand(0.5..1.5) } + @guessings = Array.new(num_cols) { rand(0.0..0.3) } + + @max_iter = max_iter + @tolerance = tolerance + @param_tolerance = param_tolerance + @learning_rate = learning_rate + @decay_factor = decay_factor end - # Sigmoid function to calculate probability def sigmoid(x) 1.0 / (1.0 + Math.exp(-x)) end - # Probability function for the 3PL model + # Probability for the 3PL model: c + (1-c)*sigmoid(a*(θ - b)) def probability(theta, a, b, c) - c + (1 - c) * sigmoid(a * (theta - b)) + c + (1.0 - c) * sigmoid(a * (theta - b)) end - # Calculate the log-likelihood of the data given the current parameters - def likelihood - likelihood = 0 - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| - prob = probability(@abilities[i], @discriminations[j], @difficulties[j], @guessings[j]) - likelihood += response == 1 ? Math.log(prob) : Math.log(1 - prob) + def log_likelihood + ll = 0.0 + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + + prob = probability(@abilities[i], @discriminations[j], + @difficulties[j], @guessings[j]) + ll += if resp == 1 + Math.log(prob + 1e-15) + else + Math.log((1 - prob) + 1e-15) + end end end - likelihood + ll end - # Update parameters using gradient ascent - def update_parameters - last_likelihood = likelihood - @max_iter.times do |_iter| - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| - prob = probability(@abilities[i], @discriminations[j], @difficulties[j], @guessings[j]) - error = response - prob - @abilities[i] += @learning_rate * error * @discriminations[j] - @difficulties[j] -= @learning_rate * error * @discriminations[j] - @discriminations[j] += @learning_rate * error * (@abilities[i] - @difficulties[j]) - @guessings[j] += @learning_rate * error * (1 - prob) - @guessings[j] = [[@guessings[j], 0].max, 1].min # Keep guessings within [0, 1] - end + def compute_gradient + grad_abilities = Array.new(@abilities.size, 0.0) + grad_difficulties = Array.new(@difficulties.size, 0.0) + grad_discriminations = Array.new(@discriminations.size, 0.0) + grad_guessings = Array.new(@guessings.size, 0.0) + + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + + theta = @abilities[i] + a = @discriminations[j] + b = @difficulties[j] + c = @guessings[j] + + prob = probability(theta, a, b, c) + error = resp - prob + + grad_abilities[i] += error * a * (1 - c) + grad_difficulties[j] -= error * a * (1 - c) + grad_discriminations[j] += error * (theta - b) * (1 - c) + + grad_guessings[j] += error * 1.0 end - current_likelihood = likelihood - break if (last_likelihood - current_likelihood).abs < @tolerance + end - last_likelihood = current_likelihood + [grad_abilities, grad_difficulties, grad_discriminations, grad_guessings] + end + + def apply_gradient_update(ga, gd, gdisc, gc) + old_abilities = @abilities.dup + old_difficulties = @difficulties.dup + old_discriminations = @discriminations.dup + old_guessings = @guessings.dup + + @abilities.each_index do |i| + @abilities[i] += @learning_rate * ga[i] + end + + @difficulties.each_index do |j| + @difficulties[j] += @learning_rate * gd[j] + end + + @discriminations.each_index do |j| + @discriminations[j] += @learning_rate * gdisc[j] + @discriminations[j] = 0.01 if @discriminations[j] < 0.01 + @discriminations[j] = 5.0 if @discriminations[j] > 5.0 end + + @guessings.each_index do |j| + @guessings[j] += @learning_rate * gc[j] + @guessings[j] = 0.0 if @guessings[j] < 0.0 + @guessings[j] = 0.35 if @guessings[j] > 0.35 + end + + [old_abilities, old_difficulties, old_discriminations, old_guessings] + end + + def average_param_update(old_a, old_d, old_disc, old_c) + deltas = [] + @abilities.each_with_index do |x, i| + deltas << (x - old_a[i]).abs + end + @difficulties.each_with_index do |x, j| + deltas << (x - old_d[j]).abs + end + @discriminations.each_with_index do |x, j| + deltas << (x - old_disc[j]).abs + end + @guessings.each_with_index do |x, j| + deltas << (x - old_c[j]).abs + end + deltas.sum / deltas.size end - # Fit the model to the data def fit - update_parameters + prev_ll = log_likelihood + + @max_iter.times do + ga, gd, gdisc, gc = compute_gradient + old_a, old_d, old_disc, old_c = apply_gradient_update(ga, gd, gdisc, gc) + + curr_ll = log_likelihood + param_delta = average_param_update(old_a, old_d, old_disc, old_c) + + if curr_ll < prev_ll + @abilities = old_a + @difficulties = old_d + @discriminations = old_disc + @guessings = old_c + + @learning_rate *= @decay_factor + else + ll_diff = (curr_ll - prev_ll).abs + break if ll_diff < @tolerance && param_delta < @param_tolerance + + prev_ll = curr_ll + end + end + { abilities: @abilities, difficulties: @difficulties, diff --git a/lib/irt_ruby/two_parameter_model.rb b/lib/irt_ruby/two_parameter_model.rb index 61d6306..5004f69 100644 --- a/lib/irt_ruby/two_parameter_model.rb +++ b/lib/irt_ruby/two_parameter_model.rb @@ -3,63 +3,140 @@ require "matrix" module IrtRuby - # A class representing the Two-Parameter model for Item Response Theory. + # A class representing the Two-Parameter model (2PL) for IRT. + # Incorporates: + # - Adaptive learning rate + # - Missing data handling + # - Parameter clamping for discrimination + # - Multiple convergence checks + # - Separate gradient calculation & parameter update class TwoParameterModel - def initialize(data, max_iter: 1000, tolerance: 1e-6, learning_rate: 0.01) + def initialize(data, max_iter: 1000, tolerance: 1e-6, param_tolerance: 1e-6, + learning_rate: 0.01, decay_factor: 0.5) @data = data - @abilities = Array.new(data.row_count) { rand } - @difficulties = Array.new(data.column_count) { rand } - @discriminations = Array.new(data.column_count) { rand } - @max_iter = max_iter - @tolerance = tolerance - @learning_rate = learning_rate + @data_array = data.to_a + num_rows = @data_array.size + num_cols = @data_array.first.size + + # Initialize parameters + # Typically: ability ~ 0, difficulty ~ 0, discrimination ~ 1 + @abilities = Array.new(num_rows) { rand(-0.25..0.25) } + @difficulties = Array.new(num_cols) { rand(-0.25..0.25) } + @discriminations = Array.new(num_cols) { rand(0.5..1.5) } # Start around 1.0 + + @max_iter = max_iter + @tolerance = tolerance + @param_tolerance = param_tolerance + @learning_rate = learning_rate + @decay_factor = decay_factor end - # Sigmoid function def sigmoid(x) 1.0 / (1.0 + Math.exp(-x)) end - # Calculate the log-likelihood of the data given the current parameters - def likelihood - likelihood = 0 - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| + def log_likelihood + ll = 0.0 + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) - if response == 1 - likelihood += Math.log(prob) - elsif response.zero? - likelihood += Math.log(1 - prob) - end + ll += if resp == 1 + Math.log(prob + 1e-15) + else + Math.log((1 - prob) + 1e-15) + end end end - likelihood + ll end - # Update parameters using gradient ascent - def update_parameters - last_likelihood = likelihood - @max_iter.times do |_iter| - @data.row_vectors.each_with_index do |row, i| - row.to_a.each_with_index do |response, j| - prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) - error = response - prob - @abilities[i] += @learning_rate * error * @discriminations[j] - @difficulties[j] -= @learning_rate * error * @discriminations[j] - @discriminations[j] += @learning_rate * error * (@abilities[i] - @difficulties[j]) - end + def compute_gradient + grad_abilities = Array.new(@abilities.size, 0.0) + grad_difficulties = Array.new(@difficulties.size, 0.0) + grad_discriminations = Array.new(@discriminations.size, 0.0) + + @data_array.each_with_index do |row, i| + row.each_with_index do |resp, j| + next if resp.nil? + + prob = sigmoid(@discriminations[j] * (@abilities[i] - @difficulties[j])) + error = resp - prob + + grad_abilities[i] += error * @discriminations[j] + grad_difficulties[j] -= error * @discriminations[j] + grad_discriminations[j] += error * (@abilities[i] - @difficulties[j]) end - current_likelihood = likelihood - break if (last_likelihood - current_likelihood).abs < @tolerance + end + + [grad_abilities, grad_difficulties, grad_discriminations] + end + + def apply_gradient_update(ga, gd, gdisc) + old_abilities = @abilities.dup + old_difficulties = @difficulties.dup + old_discriminations = @discriminations.dup + + @abilities.each_index do |i| + @abilities[i] += @learning_rate * ga[i] + end + + @difficulties.each_index do |j| + @difficulties[j] += @learning_rate * gd[j] + end + + @discriminations.each_index do |j| + @discriminations[j] += @learning_rate * gdisc[j] + @discriminations[j] = 0.01 if @discriminations[j] < 0.01 + @discriminations[j] = 5.0 if @discriminations[j] > 5.0 + end - last_likelihood = current_likelihood + [old_abilities, old_difficulties, old_discriminations] + end + + def average_param_update(old_a, old_d, old_disc) + deltas = [] + @abilities.each_with_index do |x, i| + deltas << (x - old_a[i]).abs + end + @difficulties.each_with_index do |x, j| + deltas << (x - old_d[j]).abs + end + @discriminations.each_with_index do |x, j| + deltas << (x - old_disc[j]).abs end + deltas.sum / deltas.size end - # Fit the model to the data def fit - update_parameters - { abilities: @abilities, difficulties: @difficulties, discriminations: @discriminations } + prev_ll = log_likelihood + + @max_iter.times do + ga, gd, gdisc = compute_gradient + old_a, old_d, old_disc = apply_gradient_update(ga, gd, gdisc) + + curr_ll = log_likelihood + param_delta = average_param_update(old_a, old_d, old_disc) + + if curr_ll < prev_ll + @abilities = old_a + @difficulties = old_d + @discriminations = old_disc + @learning_rate *= @decay_factor + else + ll_diff = (curr_ll - prev_ll).abs + break if ll_diff < @tolerance && param_delta < @param_tolerance + + prev_ll = curr_ll + end + end + + { + abilities: @abilities, + difficulties: @difficulties, + discriminations: @discriminations + } end end end diff --git a/spec/irt_ruby/rasch_model_spec.rb b/spec/irt_ruby/rasch_model_spec.rb index 323636d..d618ae9 100644 --- a/spec/irt_ruby/rasch_model_spec.rb +++ b/spec/irt_ruby/rasch_model_spec.rb @@ -3,32 +3,137 @@ require "spec_helper" RSpec.describe IrtRuby::RaschModel do - let(:data) { Matrix[[1, 0, 1], [0, 1, 0], [1, 1, 1]] } - let(:model) { IrtRuby::RaschModel.new(data, max_iter: 2000) } + let(:data_array) do + [ + [1, 1, 0], + [1, 0, 0], + [0, 1, 1] + ] + end + + let(:data_matrix) { Matrix[*data_array] } + + describe "Basic fitting and improvement" do + it "fits the model on an array-of-arrays dataset and improves log-likelihood" do + model = described_class.new(data_array, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + result = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll + expect(result[:abilities].size).to eq(3) + expect(result[:difficulties].size).to eq(3) + end + + it "fits the model on a Matrix dataset and improves log-likelihood" do + model = described_class.new(data_matrix, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + result = model.fit + final_ll = model.log_likelihood - describe "#initialize" do - it "initializes with data" do - expect(model.instance_variable_get(:@data)).to eq(data) + expect(final_ll).to be > initial_ll + expect(result[:abilities].size).to eq(3) + expect(result[:difficulties].size).to eq(3) end end - describe "#sigmoid" do - it "calculates the sigmoid function" do - expect(model.sigmoid(0)).to eq(0.5) + describe "Missing data" do + it "handles missing entries (nil) gracefully" do + missing_data = [ + [1, nil, 0], + [nil, 0, 0], + [0, 1, 1] + ] + model = described_class.new(missing_data, max_iter: 200, learning_rate: 0.05) + + expect { model.fit }.not_to raise_error + result = model.fit + expect(result[:abilities]).not_to be_empty + expect(result[:difficulties]).not_to be_empty end end - describe "#likelihood" do - it "calculates the likelihood of the data" do - expect(model.likelihood).to be_a(Float) + describe "Edge cases" do + it "works with a single examinee and single item" do + data = [[1]] + model = described_class.new(data, max_iter: 100) + expect { model.fit }.not_to raise_error + + result = model.fit + expect(result[:abilities].size).to eq(1) + expect(result[:difficulties].size).to eq(1) + end + + it "works with all responses correct" do + data = [ + [1, 1], + [1, 1] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + result = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(result[:abilities].size).to eq(2) + expect(result[:difficulties].size).to eq(2) + end + + it "works with all responses incorrect" do + data = [ + [0, 0], + [0, 0] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + result = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(result[:abilities].size).to eq(2) + expect(result[:difficulties].size).to eq(2) + end + + it "handles an entire row missing" do + data = [ + [1, 0, 1], + [nil, nil, nil] + ] + model = described_class.new(data) + expect { model.fit }.not_to raise_error + expect(model.fit[:abilities].size).to eq(2) + expect(model.fit[:difficulties].size).to eq(3) + end + + it "handles an entire column missing" do + data = [ + [1, nil, 0], + [1, nil, 0] + ] + model = described_class.new(data) + expect { model.fit }.not_to raise_error + expect(model.fit[:abilities].size).to eq(2) + expect(model.fit[:difficulties].size).to eq(3) end end - describe "#fit" do - it "fits the model and returns abilities and difficulties" do + describe "Hyperparameter extremes" do + it "does not diverge with a large learning rate (but may revert updates)" do + model = described_class.new(data_array, max_iter: 200, learning_rate: 5.0) + expect { model.fit }.not_to raise_error + result = model.fit - expect(result[:abilities].size).to eq(data.row_count) - expect(result[:difficulties].size).to eq(data.column_count) + expect(result[:abilities]).not_to be_empty + expect(result[:difficulties]).not_to be_empty + end + + it "still improves log-likelihood with a very small learning rate" do + model = described_class.new(data_array, max_iter: 2000, learning_rate: 1e-4) + initial_ll = model.log_likelihood + model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll end end end diff --git a/spec/irt_ruby/three_parameter_model_spec.rb b/spec/irt_ruby/three_parameter_model_spec.rb index abbcd15..0f92b02 100644 --- a/spec/irt_ruby/three_parameter_model_spec.rb +++ b/spec/irt_ruby/three_parameter_model_spec.rb @@ -1,42 +1,165 @@ # frozen_string_literal: true require "spec_helper" +require "matrix" RSpec.describe IrtRuby::ThreeParameterModel do - let(:data) { Matrix[[1, 0, 1], [0, 1, 0], [1, 1, 1]] } - let(:model) { IrtRuby::ThreeParameterModel.new(data, max_iter: 1500) } + let(:data_array) do + [ + [1, 1, 0], + [1, 0, 1], + [0, 1, 1], + [1, 1, 1] + ] + end + + let(:data_matrix) { Matrix[*data_array] } + + describe "Basic fitting and improvement" do + it "fits the 3PL model with an array-of-arrays and improves log-likelihood" do + model = described_class.new(data_array, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood - describe "#initialize" do - it "initializes with data" do - expect(model.instance_variable_get(:@data)).to eq(data) + expect(final_ll).to be > initial_ll + expect(results[:abilities].size).to eq(4) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) + expect(results[:guessings].size).to eq(3) end - end - describe "#sigmoid" do - it "calculates the sigmoid function" do - expect(model.sigmoid(0)).to eq(0.5) + it "fits the 3PL model with a Matrix and improves log-likelihood" do + model = described_class.new(data_matrix, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll + expect(results[:abilities].size).to eq(4) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) + expect(results[:guessings].size).to eq(3) end end - describe "#probability" do - it "calculates the probability with guessing parameter" do - expect(model.probability(0, 1, 0, 0.2)).to be_within(0.01).of(0.6) + describe "Missing data handling" do + it "handles nil entries gracefully without raising errors" do + missing_data = [ + [1, nil, 0], + [1, 0, 1], + [0, 1, nil], + [1, 1, 1] + ] + model = described_class.new(missing_data, max_iter: 200, learning_rate: 0.05) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities]).not_to be_empty + expect(results[:difficulties]).not_to be_empty + expect(results[:discriminations]).not_to be_empty + expect(results[:guessings]).not_to be_empty end end - describe "#likelihood" do - it "calculates the likelihood of the data" do - expect(model.likelihood).to be_a(Float) + describe "Edge cases" do + it "works with a single examinee and single item" do + data = [[0]] + model = described_class.new(data, max_iter: 100) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(1) + expect(results[:difficulties].size).to eq(1) + expect(results[:discriminations].size).to eq(1) + expect(results[:guessings].size).to eq(1) + end + + it "handles all responses correct" do + data = [ + [1, 1], + [1, 1] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(2) + expect(results[:discriminations].size).to eq(2) + expect(results[:guessings].size).to eq(2) + end + + it "handles all responses incorrect" do + data = [ + [0, 0], + [0, 0] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(2) + expect(results[:discriminations].size).to eq(2) + expect(results[:guessings].size).to eq(2) + end + + it "handles an entire row missing" do + data = [ + [1, 0], + [nil, nil] + ] + model = described_class.new(data, max_iter: 200) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(2) + expect(results[:discriminations].size).to eq(2) + expect(results[:guessings].size).to eq(2) + end + + it "handles an entire column missing" do + data = [ + [1, nil, 0], + [1, nil, 1], + [0, nil, 1] + ] + model = described_class.new(data, max_iter: 200) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(3) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) + expect(results[:guessings].size).to eq(3) end end - describe "#fit" do - it "fits the model and returns abilities, difficulties, discriminations, and guessings" do - result = model.fit - expect(result[:abilities].size).to eq(data.row_count) - expect(result[:difficulties].size).to eq(data.column_count) - expect(result[:discriminations].size).to eq(data.column_count) - expect(result[:guessings].size).to eq(data.column_count) + describe "Hyperparameter extremes" do + it "does not diverge with a large learning rate (but may revert updates)" do + model = described_class.new(data_array, max_iter: 200, learning_rate: 5.0) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities]).not_to be_empty + expect(results[:difficulties]).not_to be_empty + expect(results[:discriminations]).not_to be_empty + expect(results[:guessings]).not_to be_empty + end + + it "shows improvement with a very small learning rate" do + model = described_class.new(data_array, max_iter: 2000, learning_rate: 1e-4) + initial_ll = model.log_likelihood + model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll end end end diff --git a/spec/irt_ruby/two_parameter_model_spec.rb b/spec/irt_ruby/two_parameter_model_spec.rb index 92b6a18..bb1bb89 100644 --- a/spec/irt_ruby/two_parameter_model_spec.rb +++ b/spec/irt_ruby/two_parameter_model_spec.rb @@ -1,35 +1,155 @@ # frozen_string_literal: true require "spec_helper" +require "matrix" RSpec.describe IrtRuby::TwoParameterModel do - let(:data) { Matrix[[1, 0, 1], [0, 1, 0], [1, 1, 1]] } - let(:model) { IrtRuby::TwoParameterModel.new(data, max_iter: 3000) } + let(:data_array) do + [ + [1, 1, 0], + [1, 0, 1], + [0, 1, 1], + [1, 1, 1] + ] + end + + let(:data_matrix) { Matrix[*data_array] } + + describe "Basic fitting and improvement" do + it "fits the 2PL model with an array-of-arrays and improves log-likelihood" do + model = described_class.new(data_array, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll + expect(results[:abilities].size).to eq(4) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) + end + + it "fits the 2PL model with a Matrix and improves log-likelihood" do + model = described_class.new(data_matrix, max_iter: 300, learning_rate: 0.1) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood - describe "#initialize" do - it "initializes with data" do - expect(model.instance_variable_get(:@data)).to eq(data) + expect(final_ll).to be > initial_ll + expect(results[:abilities].size).to eq(4) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) end end - describe "#sigmoid" do - it "calculates the sigmoid function" do - expect(model.sigmoid(0)).to eq(0.5) + describe "Missing data handling" do + it "does not raise an error with missing data (nil) in 2PL" do + missing_data = [ + [1, nil, 0], + [1, 0, 1], + [0, 1, nil] + ] + model = described_class.new(missing_data, max_iter: 200, learning_rate: 0.05) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities]).not_to be_empty + expect(results[:difficulties]).not_to be_empty + expect(results[:discriminations]).not_to be_empty end end - describe "#likelihood" do - it "calculates the likelihood of the data" do - expect(model.likelihood).to be_a(Float) + describe "Edge cases" do + it "works with a single examinee and single item" do + data = [[1]] + model = described_class.new(data, max_iter: 100) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(1) + expect(results[:difficulties].size).to eq(1) + expect(results[:discriminations].size).to eq(1) + end + + it "handles all responses correct" do + data = [ + [1, 1], + [1, 1] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(2) + expect(results[:discriminations].size).to eq(2) + end + + it "handles all responses incorrect" do + data = [ + [0, 0], + [0, 0] + ] + model = described_class.new(data, max_iter: 100) + initial_ll = model.log_likelihood + results = model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be >= initial_ll + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(2) + expect(results[:discriminations].size).to eq(2) + end + + it "handles an entire row missing" do + data = [ + [1, 0, 1], + [nil, nil, nil] + ] + model = described_class.new(data) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(2) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) + end + + it "handles an entire column missing" do + data = [ + [1, nil, 0], + [1, nil, 1], + [0, nil, 1] + ] + model = described_class.new(data) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities].size).to eq(3) + expect(results[:difficulties].size).to eq(3) + expect(results[:discriminations].size).to eq(3) end end - describe "#fit" do - it "fits the model and returns abilities, difficulties, and discriminations" do - result = model.fit - expect(result[:abilities].size).to eq(data.row_count) - expect(result[:difficulties].size).to eq(data.column_count) - expect(result[:discriminations].size).to eq(data.column_count) + describe "Hyperparameter extremes" do + it "does not diverge with a large learning rate (but may revert updates)" do + model = described_class.new(data_array, max_iter: 200, learning_rate: 5.0) + expect { model.fit }.not_to raise_error + + results = model.fit + expect(results[:abilities]).not_to be_empty + expect(results[:difficulties]).not_to be_empty + expect(results[:discriminations]).not_to be_empty + end + + it "improves log-likelihood with a very small learning rate, though slowly" do + model = described_class.new(data_array, max_iter: 2000, learning_rate: 1e-4) + initial_ll = model.log_likelihood + model.fit + final_ll = model.log_likelihood + + expect(final_ll).to be > initial_ll end end end