From 6f66e63469f98512fc8db0f93c95acf19e51bd89 Mon Sep 17 00:00:00 2001 From: Mathieu Mitchell Date: Tue, 28 Aug 2018 23:29:52 -0400 Subject: [PATCH 1/3] Always verify mock arguments Currently, mock arguments were verified inside the mock itself. Using a mock in a script not running in errexit mode (set -e) or hiding the failure meant silent failures and successful tests. Signal the invocation failure through the mocks workspace and verify success during the verify_mocks phase. Additionally, the build script has been adjusted to avoid having to add new lines at the end of every source file. Finally, a "contains" assert has been added. --- src/aggregate.sh | 17 +++++++---- src/asserts.sh | 4 +++ src/mocking.sh | 9 ++++-- test-fixtures/mocking-args-strict/src/code.sh | 8 +++++ .../test/test_verify_arguments_order.sh | 9 ++++++ test/test_assert.sh | 30 ++++++++++++++++++- test/test_mocking_args_matching.sh | 18 ++++++----- 7 files changed, 78 insertions(+), 17 deletions(-) create mode 100755 test-fixtures/mocking-args-strict/src/code.sh create mode 100644 test-fixtures/mocking-args-strict/test/test_verify_arguments_order.sh diff --git a/src/aggregate.sh b/src/aggregate.sh index 24f2639..12fb2fd 100644 --- a/src/aggregate.sh +++ b/src/aggregate.sh @@ -3,9 +3,14 @@ set -eu src=$(dirname $0) -cat ${src}/header.sh -cat ${src}/utils.sh -cat ${src}/asserts.sh -cat ${src}/mocking.sh -cat ${src}/cli.sh -cat ${src}/test-runner.sh +include() { + cat $1 + printf '\n' +} + +include ${src}/header.sh +include ${src}/utils.sh +include ${src}/asserts.sh +include ${src}/mocking.sh +include ${src}/cli.sh +include ${src}/test-runner.sh diff --git a/src/asserts.sh b/src/asserts.sh index 82c3729..a02123f 100644 --- a/src/asserts.sh +++ b/src/asserts.sh @@ -18,3 +18,7 @@ _assert_succeeded() { _assert_failed() { test ${1} -ne 0 || assertion_failed "Expected failure exit code\nGot: <$1>" } + +_assert_contains() { + echo "$1" | grep -q "$2" || assertion_failed "Expected: <$1>\nTo match pattern: <$2>" +} \ No newline at end of file diff --git a/src/mocking.sh b/src/mocking.sh index 61eb52c..ca2a996 100644 --- a/src/mocking.sh +++ b/src/mocking.sh @@ -39,13 +39,12 @@ validate-args() { args="\$@" if [ "\${args}" != "${expected}" ]; then - - cat < \$0_error < Expected : <"${expected}"> OUT - exit 1 + exit 1 # Kept for historical reasons. Proper usage would be mock xyz --with-args "arg1 arg2" --and exit-code 1. fi EOF } @@ -159,5 +158,9 @@ _verify_mocks() { assertion_failed "Command '${command}' was expected to be called $(_format_count ${expected_calls} "time")\nCalled : $(_format_count ${call_count} "time")" fi done + + for invocation in $(find ${mock} -maxdepth 1 -name "invocation_*_error" 2>/dev/null || true); do + assertion_failed "$(cat $invocation)" + done done } diff --git a/test-fixtures/mocking-args-strict/src/code.sh b/test-fixtures/mocking-args-strict/src/code.sh new file mode 100755 index 0000000..6b8ef97 --- /dev/null +++ b/test-fixtures/mocking-args-strict/src/code.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# N.B. no `set -e` here! + +some-command one un +some-command three trois || true # Failure here doesnt matter to the actual code. +some-command two deux + +exit 0 \ No newline at end of file diff --git a/test-fixtures/mocking-args-strict/test/test_verify_arguments_order.sh b/test-fixtures/mocking-args-strict/test/test_verify_arguments_order.sh new file mode 100644 index 0000000..e374c8e --- /dev/null +++ b/test-fixtures/mocking-args-strict/test/test_verify_arguments_order.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +test_fails_when_verifying_arguments_even_if_exit_code_is_ignored() { + mock some-command --with-args "one un" --once + mock some-command --with-args "two deux" --once + mock some-command --with-args "three trois" --once + + ./code.sh +} \ No newline at end of file diff --git a/test/test_assert.sh b/test/test_assert.sh index 38e7125..b2ebfb4 100644 --- a/test/test_assert.sh +++ b/test/test_assert.sh @@ -31,7 +31,7 @@ test_assert_equals_with_strings_passing() { test_assert_equals_with_strings_failing() { assert "no" equals "yes" > assertion_output result=${?} - rm .assertion_error # The test runner would think the test failed + _hide_assertion_failure_from_test_runner assertion_error=$(cat assertion_output) @@ -49,3 +49,31 @@ test_assert_multiworks_string_works() { assert ${?} succeeded } + +test_assert_contains() { + assert "hello world" contains "^hello world$" + assert "hello world" contains "^hello" + assert "hello world" contains "world$" + assert "hello world" contains "w" +} + +test_assert_contains_fails_with_proper_error_message() { + assert "abc" contains "z$" > assertion_output + result=${?} + _hide_assertion_failure_from_test_runner + + assertion_error=$(cat assertion_output) + + expected_error=$(cat <<-EXP +Expected: +To match pattern: +EXP +) + assert ${result} failed + assert "${assertion_error}" equals "${expected_error}" +} + +_hide_assertion_failure_from_test_runner() { + # The test runner would think the test failed + rm .assertion_error +} \ No newline at end of file diff --git a/test/test_mocking_args_matching.sh b/test/test_mocking_args_matching.sh index e3cfeb0..de1ba09 100644 --- a/test/test_mocking_args_matching.sh +++ b/test/test_mocking_args_matching.sh @@ -1,21 +1,25 @@ #!/bin/bash -test_fails_if_given_arguments_isnt_right() { - mock some-command --with-args "one two three" +test_fails_when_verifying_arguments_even_when_exit_code_is_ignored() { + cp -aR ${TEST_ROOT_DIR}/../test-fixtures/mocking-args-strict/* . - some-command three two one > assertion_output + unset RUN_SINGLE_TEST + actual=$(${TEST_ROOT_DIR}/../target/sbtest.sh verify_arguments_order) assert ${?} failed expected_error=$(cat <<-EXP Unexpected invocation for command 'some-command': -Got : <"three two one"> -Expected : <"one two three"> +Got : <"three trois"> +Expected : <"two deux"> +Unexpected invocation for command 'some-command': +Got : <"two deux"> +Expected : <"three trois"> EXP ) - assert "$(cat assertion_output)" equals "${expected_error}" + assert "${actual}" contains "${expected_error}" } -test_fails_if_2_with_args_argments_are_given() { +test_fails_if_2_with_args_arguments_are_given() { mock some-command --with-args "a" --with-args "b" > error assert ${?} failed From 3b27c79ab0b6a7d7c327966dfdd7b59648da499b Mon Sep 17 00:00:00 2001 From: Mathieu Mitchell Date: Thu, 18 Oct 2018 13:30:36 -0400 Subject: [PATCH 2/3] Improve assert contains matcher message --- src/asserts.sh | 4 ++-- test/test_assert.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/asserts.sh b/src/asserts.sh index a02123f..1f1b8a9 100644 --- a/src/asserts.sh +++ b/src/asserts.sh @@ -20,5 +20,5 @@ _assert_failed() { } _assert_contains() { - echo "$1" | grep -q "$2" || assertion_failed "Expected: <$1>\nTo match pattern: <$2>" -} \ No newline at end of file + echo "$1" | grep -q "$2" || assertion_failed "Expected: <$1>\nTo contain pattern: <$2>" +} diff --git a/test/test_assert.sh b/test/test_assert.sh index b2ebfb4..5ae5622 100644 --- a/test/test_assert.sh +++ b/test/test_assert.sh @@ -65,8 +65,8 @@ test_assert_contains_fails_with_proper_error_message() { assertion_error=$(cat assertion_output) expected_error=$(cat <<-EXP -Expected: -To match pattern: +Expected: +To contain pattern: EXP ) assert ${result} failed @@ -76,4 +76,4 @@ EXP _hide_assertion_failure_from_test_runner() { # The test runner would think the test failed rm .assertion_error -} \ No newline at end of file +} From 4ebe71c6c86b0fa8335257004317693499ad229f Mon Sep 17 00:00:00 2001 From: Mathieu Mitchell Date: Tue, 27 Nov 2018 11:10:54 -0500 Subject: [PATCH 3/3] Bring back basic "fails on wrong arguments" case --- test/test_mocking_args_matching.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_mocking_args_matching.sh b/test/test_mocking_args_matching.sh index de1ba09..eae385d 100644 --- a/test/test_mocking_args_matching.sh +++ b/test/test_mocking_args_matching.sh @@ -1,5 +1,24 @@ #!/bin/bash +test_fails_if_given_arguments_isnt_right() { + mock some-command --with-args "one two three" + + some-command three two one + assert ${?} failed + + expected_error=$(cat <<-EXP +Unexpected invocation for command 'some-command': +Got : <"three two one"> +Expected : <"one two three"> +EXP +) + error_filename=$(find $mocks -name 'invocation_*_error') + assert "$(cat $error_filename)" equals "${expected_error}" + + # Hide failure from runner. + rm $error_filename +} + test_fails_when_verifying_arguments_even_when_exit_code_is_ignored() { cp -aR ${TEST_ROOT_DIR}/../test-fixtures/mocking-args-strict/* .