From 465bc4ada582a1f485fcb06b0642e1497ffe024a Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 17 Sep 2025 09:25:42 -0400 Subject: [PATCH 01/14] denis: split only first two underscores of tag Signed-off-by: Joel Savitz --- denis/utilities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/denis/utilities.py b/denis/utilities.py index c8b41103..1afe2ecc 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -82,7 +82,7 @@ def update_tags(assignment, component): def check_corrupt_or_missing(repo, tag, username_to_subs): - [assignment, component, user] = tag.split('_') + [assignment, component, user] = tag.split('_', 2) gradable = username_to_subs[user] @@ -104,7 +104,7 @@ def check_corrupt_or_missing(repo, tag, username_to_subs): def check_signed_off_by(repo, tag): - [_, _, user] = tag.split('_') + [_, _, user] = tag.split('_', 2) hostname = os.getenv("SINGULARITY_HOSTNAME") msg = 'signed off by check' @@ -132,7 +132,7 @@ def check_signed_off_by(repo, tag): def check_subject_tag(repo, tag): - [assignment, component, user] = tag.split('_') + [assignment, component, user] = tag.split('_', 2) msg = 'subject tag check' msg += '\n' From a9ed7623e00ccb980174ad4a2fa9c6585f77a608 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 10:40:40 -0400 Subject: [PATCH 02/14] test-sub.sh: a new testing script Setup the submissions repo and go through the entire patchset submission process, exercising all return paths of mailman/patchset.py Includes intitial, peer review, and final submissions Signed-off-by: Joel Savitz --- script-lint.sh | 1 + test-sub.sh | 503 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100755 test-sub.sh diff --git a/script-lint.sh b/script-lint.sh index 10cf94fa..455ca2a9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -7,6 +7,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh +shellcheck test-sub.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-sub.sh b/test-sub.sh new file mode 100755 index 00000000..d95648ec --- /dev/null +++ b/test-sub.sh @@ -0,0 +1,503 @@ +#!/bin/bash + +set -exuo pipefail + +SCRIPT_DIR=$(dirname "$0") + +cd "$SCRIPT_DIR" + +get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# nuke grading repo inside container +# shellcheck disable=SC2016 +podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + +# setup submissions repo +rm -rf repos/submissions +git/admin.sh submissions "course submissions repository" +pushd repos +git init --bare submissions +echo "course submissions repository" > submissions/description +git init submissions_init +pushd submissions_init +echo "# submissions" > README.md +git add README.md +git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' +git push ../submissions master +popd +rm -rf submissions_init +pushd submissions +git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions +popd +popd + +# create bob,bib,bab users or change their passwords to builder +orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m + +WORKDIR=$(mktemp -d) + +cat << EOF > "$WORKDIR"/setup_rubric +[{('--- /dev/null', '+++ b/bob/setup/work'): 0}, +{('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] +EOF + +# create or recreate setup assignment +if denis/configure.sh dump | grep -q "^setup:"; then + denis/configure.sh remove -a setup +fi +denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric +denis/configure.sh reload + +# setup assignment environment +pushd "$WORKDIR" + +mkdir certs +pushd certs +podman volume export singularity_ssl-certs > certs.tar +tar xf certs.tar +popd +git clone http://localhost:"$(get_git_port)"/submissions + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_intial_bob patchsets + +# good patchset +pushd submissions +git checkout --orphan setup_initial_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# corrupt patchset +pushd submissions +git checkout --orphan setup_initial_bob_corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc --cover-letter -v1 -3 +rm ./*-0002-*.patch + +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with whitespace errors +pushd submissions +git checkout --orphan setup_initial_bob_whitespace +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def " >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter +pushd submissions +git checkout --orphan setup_initial_bob_nocover +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter and corrupt first patch +pushd submissions +git checkout --orphan setup_initial_bob_nocover-corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd + +# end of setup_initial_bob patchsets + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_initial_bab submissions + +# bab good patchset +pushd submissions +git checkout --orphan setup_initial_bab_good +git rm -rf ./* +mkdir -p bab/setup + +echo "abc" > bab/setup/work +git add bab/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bab/setup/work +git add bab/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm -rf ./*.patch +popd + +# end of bab initial submission + +# bib env setup +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib initial submissions + +# bib good patchset +pushd submissions +git checkout --orphan setup_initial_bib_good +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib initial submission + +# investigate grading repo +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading +git fetch --tag +git tag + + +STATUSES=( +'patchset applies.' +'patch 2 failed to apply!' +'whitespace error patch 2?' +'missing cover letter!' +'missing cover letter and first patch failed to apply!' +'patchset applies.' +'patchset applies.') + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + if [[ $i == 5 ]]; then + PEER1_ID=$t + elif [[ $i == 6 ]]; then + PEER2_ID=$t + fi + i=$((i+1)) +done + +popd + +echo "wait for initial submission deadline (10 secs)" +sleep 10 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# setup bob peer review +pushd submissions +git checkout --orphan setup_reviews_bob +git rm -rf ./* +# mkdir -p bob/setup (must be able to omit if missing) +echo ORPHAN > ORPHAN +git add ORPHAN +git commit -m 'orphan base' + +# submit peer review 1 + +cat < review1 +Subject: setup review 1 for bab +In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW1 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 +rm review1 + +# end peer review 1 + +# submit peer review 2 + +cat < review2 +Subject: setup review 2 for bib +In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 +rm review2 + +# end peer review 2 + +# end bob peer review +popd +sleep 1 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bob make final submission + +# good final patchset +pushd submissions +git checkout --orphan setup_final_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --cover-letter -v2 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end bob final submission + +# bib env setup + +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib final submission labeled RFC +pushd submissions +git checkout --orphan setup_final_bib_rfc +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib final submission labeled RFC + + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# bab final submission in wrong directory +pushd submissions +git checkout --orphan setup_final_bab_illegal +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > work +git add work +git commit -sm 'add abc to work' + +echo "def" >> work +git add work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bab final submission in wrong directory + + +echo "wait for final submission deadline (10 secs)" +sleep 10 + +pushd grading +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git +git notes --ref=grade add setup_final_bob -m '66' +git notes --ref=grade add setup_review1_bob -m '22' +git notes --ref=grade add setup_review2_bob -m '99' +git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' + +git notes --ref=grade add setup_final_bib -m '100' +git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' + +git notes --ref=feedback add setup_final_bab -m 'Please review git' + +git push origin refs/notes/*:refs/notes/* + +echo "$WORKDIR" From 5bf849bd58314745caf77ab5baac1d9a2e4d8a97 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 19:24:16 -0400 Subject: [PATCH 03/14] CI: enhance PINPing by running test-sub.sh This script the project in a more integration-test-like manner. Signed-off-by: Joel Savitz --- Containerfile | 5 ++++- start.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Containerfile b/Containerfile index ca3c8a18..de7b9f6e 100644 --- a/Containerfile +++ b/Containerfile @@ -11,7 +11,10 @@ RUN dnf update -y && \ python-flake8 \ python-virtualenv \ python-pip \ - git + gawk \ + socat \ + git \ + git-email # needed because for some reason newuidmap and newgidmap programs # lose their xattrs giving them caps when the container image for diff --git a/start.sh b/start.sh index 9394b8f4..99f1a3b7 100755 --- a/start.sh +++ b/start.sh @@ -18,6 +18,18 @@ podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port if [ -f test.sh ] then ./test.sh + if [ -f test-sub.sh ] + then + podman-compose down + yes | podman volume prune + podman-compose up -d + podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' + ./dev_sockets.sh & + git config --global user.name PINP + git config --global user.email podman@podman + ./test-sub.sh + + fi else virtualenv . pip install -r requirements.txt From 14e5438eda2ecf5e7fbf41caf486efe5182349c7 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 20:08:36 -0400 Subject: [PATCH 04/14] test-sub-check.sh: add script to check results of test-sub.sh test-sub.sh is good for interactive testing, but lacks validation when automated. Add test-sub-check.sh to run some basic checks that make sure test-sub.sh ran smoothly. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 1 + test-sub-check.sh | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100755 test-sub-check.sh diff --git a/script-lint.sh b/script-lint.sh index 455ca2a9..3e259437 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -8,6 +8,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh shellcheck test-sub.sh +shellcheck test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index 99f1a3b7..9df6a678 100755 --- a/start.sh +++ b/start.sh @@ -28,6 +28,7 @@ then git config --global user.name PINP git config --global user.email podman@podman ./test-sub.sh + ./test-sub-check.sh fi else diff --git a/test-sub-check.sh b/test-sub-check.sh new file mode 100755 index 00000000..2e38b9a6 --- /dev/null +++ b/test-sub-check.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + +set -exuo pipefail + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +DOCKER=${DOCKER:-podman} + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +# Create test dir if it does not exist yet +mkdir -p test + +# Reset the test directory +rm -f test/* + +${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "setup" + +grep "Total Score: 64.9" test/dashboard + +grep "missing cover letter and first patch failed to apply!" test/dashboard + +grep "bib Peer Review" test/dashboard + +grep "bab Peer Review" test/dashboard + +grep "patchset applies." test/dashboard + +grep "Good effort but try harder next time." test/dashboard + +# login as bab +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bab \ + --data "username=bab&password=builder" \ + | tee test/login_success_bab \ + | grep "msg = bab authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bab \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bab \ + | grep "setup" + +grep "Total Score: 0.0" test/dashboard_bab + +grep "bib Peer Review" test/dashboard_bab + +grep "bob Peer Review" test/dashboard_bab + +grep "patchset applies." test/dashboard_bab + +grep "illegal patch 1: permission denied for path work!" test/dashboard_bab + +grep 'Please review git' test/dashboard_bab + +# login as bib +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bib \ + --data "username=bib&password=builder" \ + | tee test/login_success_bib \ + | grep "msg = bib authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bib \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bib \ + | grep "setup" + +grep "Total Score: 80.0" test/dashboard_bib + +grep "bob Peer Review" test/dashboard_bib + +grep "bab Peer Review" test/dashboard_bib + +grep "patchset applies." test/dashboard_bib + +grep 'Perfect. But no peer review!' test/dashboard_bib + +echo "ALL SUBMISSION TESTS PASS" From 0889a62fd76fd3feed25a370b1d1203a8a666499 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:01:18 -0400 Subject: [PATCH 05/14] treewide: s/DOCKER/PODMAN/g We use podman around these parts Signed-off-by: Joel Savitz --- git/admin.sh | 4 ++-- orbit/warpdrive.sh | 4 ++-- test-sub-check.sh | 4 ++-- test.sh | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/git/admin.sh b/git/admin.sh index f3a9e46e..bbd5189a 100755 --- a/git/admin.sh +++ b/git/admin.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec git ./create-repo.sh "$@" +$PODMAN_COMPOSE exec git ./create-repo.sh "$@" diff --git a/orbit/warpdrive.sh b/orbit/warpdrive.sh index 8d92572d..c78e8158 100755 --- a/orbit/warpdrive.sh +++ b/orbit/warpdrive.sh @@ -4,6 +4,6 @@ set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec orbit ./hyperspace.py "$@" +$PODMAN_COMPOSE exec orbit ./hyperspace.py "$@" diff --git a/test-sub-check.sh b/test-sub-check.sh index 2e38b9a6..894de144 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -15,7 +15,7 @@ set -o allexport exec jq -r -n "env.SINGULARITY_HOSTNAME" ')" -DOCKER=${DOCKER:-podman} +PODMAN=${PODMAN:-podman} SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} @@ -25,7 +25,7 @@ mkdir -p test # Reset the test directory rm -f test/* -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 7e13f23f..2b5d9430 100755 --- a/test.sh +++ b/test.sh @@ -10,15 +10,15 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -DOCKER=${DOCKER:-podman} -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl require jq require flake8 -require "${DOCKER}" -require "${DOCKER_COMPOSE}" +require "${PODMAN}" +require "${PODMAN_COMPOSE}" # Check for shell script style compliance with shellcheck ./script-lint.sh @@ -40,7 +40,7 @@ exec jq -r -n "env.SINGULARITY_HOSTNAME" SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem CURL_OPTS=( \ --verbose \ @@ -150,10 +150,10 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ orbit/warpdrive.sh -u resu -p ssap -n # Limit `resu`'s access to the empty inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu # Update list of email to include new message -${DOCKER_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail +${PODMAN_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail # Check that the user can get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ @@ -173,7 +173,7 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ # Remove limit on `resu`'s access to the inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu # Check that `resu` can now get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ From 5db952a7cbc3b7e9f5e2d84a478649a20e59f924 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:28:42 -0400 Subject: [PATCH 06/14] test: consolidate repetitive setup in test-lib test.sh and test-sub-check.sh now source test-lib script-lint.sh modified to follow source commands for those two files. Signed-off-by: Joel Savitz --- script-lint.sh | 4 ++-- test-lib | 29 +++++++++++++++++++++++++++++ test-sub-check.sh | 25 ++----------------------- test.sh | 26 ++------------------------ 4 files changed, 35 insertions(+), 49 deletions(-) create mode 100644 test-lib diff --git a/script-lint.sh b/script-lint.sh index 3e259437..0f68fbe6 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -6,9 +6,9 @@ require shellcheck set -ex shellcheck script-lint.sh -shellcheck test.sh +shellcheck -x test.sh shellcheck test-sub.sh -shellcheck test-sub-check.sh +shellcheck -x test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-lib b/test-lib new file mode 100644 index 00000000..a29d3963 --- /dev/null +++ b/test-lib @@ -0,0 +1,29 @@ +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +setup_testdir() { + # Create test dir if it does not exist yet + mkdir -p test + + # Reset the test directory + rm -f test/* + + # put the cert in there + ${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +} + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + diff --git a/test-sub-check.sh b/test-sub-check.sh index 894de144..ae42ac92 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,31 +1,10 @@ #!/usr/bin/env bash -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) - set -exuo pipefail -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -PODMAN=${PODMAN:-podman} - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* +source test-lib -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +setup_testdir # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 2b5d9430..c28d13da 100755 --- a/test.sh +++ b/test.sh @@ -10,8 +10,7 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -PODMAN=${PODMAN:-podman} -PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl @@ -26,28 +25,7 @@ require "${PODMAN_COMPOSE}" # Check python style compliance flake8 -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* - -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem - -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) +setup_testdir # Check that registration fails before user creation curl --url "https://$SINGULARITY_HOSTNAME/register" \ From 50b8912d26f21dcf1325ab9721cbbd5e2a513aa4 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 1 Jun 2025 22:21:32 -0400 Subject: [PATCH 07/14] test: add more rubric tests cleanly by consolidating common testing code Add test-sub2.sh which tests out 12 invalid and 1 valid submission on a rubric for an example coding assignment and validate the resulting status values added to the email ID tags in the grading repo. Simplify the new tests by abstracting the setup section and repetitive commands, enabling quicker development of shorter future testing scripts the same flavor as the rest of test-sub*.sh. This is important as we want to make sure our automated grading features actually do what we think they will do to the fullest extent possible before this code goes into production with real students. Signed-off-by: Joel Savitz --- script-lint.sh | 5 +- start.sh | 1 + test-lib | 166 +++++++++++++++++ test-sub-check.sh | 2 - test-sub.sh | 457 ++++++++-------------------------------------- test-sub2.sh | 154 ++++++++++++++++ test.sh | 8 - 7 files changed, 400 insertions(+), 393 deletions(-) create mode 100755 test-sub2.sh diff --git a/script-lint.sh b/script-lint.sh index 0f68fbe6..2731cad9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -5,10 +5,12 @@ require shellcheck set -ex +# -x needed to make shellcheck follow `source` command shellcheck script-lint.sh shellcheck -x test.sh -shellcheck test-sub.sh +shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh +shellcheck -x test-sub2.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh @@ -18,6 +20,5 @@ shellcheck git/setup-repo.sh shellcheck git/cgi-bin/git-receive-pack shellcheck git/hooks/post-update -# -x needed to make shellcheck follow `source` command shellcheck -x backup/backup.sh shellcheck -x backup/restore.sh diff --git a/start.sh b/start.sh index 9df6a678..07b38725 100755 --- a/start.sh +++ b/start.sh @@ -29,6 +29,7 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + ./test-sub2.sh fi else diff --git a/test-lib b/test-lib index a29d3963..1b9607fb 100644 --- a/test-lib +++ b/test-lib @@ -1,5 +1,15 @@ +# This line: +# - aborts the script after any pipeline returns nonzero (e) +# - shows all commands as they are run (x) +# - sets any dereference of an unset variable to trigger an error (u) +# - causes the return value of a pipeline to be the nonzero return value +# of the furthest right failing command or zero if no command failed (o pipefail) +set -exuo pipefail + PODMAN=${PODMAN:-podman} PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +WORKDIR=$(mktemp -d) HOSTNAME_FROM_DOTENV="$(sh -c ' set -o allexport @@ -27,3 +37,159 @@ CURL_OPTS=( \ --no-progress-meter \ ) + +get_git_port() { ${PODMAN} port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# preconditions: +# - SCRIPT_DIR defined (singularity repo root) +# - WORKDIR defined (arbitrary) temp directory +setup_submissions_and_grading_repo() { + pushd "$SCRIPT_DIR" + # nuke grading repo inside container + # shellcheck disable=SC2016 + ${PODMAN_COMPOSE} exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + # crete and push fresh submissions repo + rm -rf repos/submissions + git/admin.sh submissions "course submissions repository" + pushd repos + git init --bare submissions + echo "course submissions repository" > submissions/description + # create a temporary workdir to push an initial commit to the submission repo + git init submissions_init + pushd submissions_init + echo "# submissions" > README.md + git add README.md + git status + git -c user.name=singularity -c user.email=singularity@singularity commit -sm 'init submissions repo' + git push ../submissions master + popd + rm -rf submissions_init + pushd submissions + git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions + popd + popd + popd + + pushd "$WORKDIR" + mkdir certs + pushd certs + ${PODMAN} volume export singularity_ssl-certs > certs.tar + tar xf certs.tar + popd + git clone http://localhost:"$(get_git_port)"/submissions + popd +} + +create_lighting_assignment() { + test -n "$1" + test -n "$2" + test -n "$3" + test -n "$4" + + local asn="$1" + local i_s="$2" + local p_s="$3" + local f_s="$4" + set +u + local rub="$5" + set -u + + RUBRIC= + if [ -n "$rub" ] + then + RUBRIC="-r $rub" + fi + + pushd "$SCRIPT_DIR" + # create or recreate setup assignment + if denis/configure.sh dump | grep -q "^$asn:"; then + denis/configure.sh remove -a "$asn" + fi + denis/configure.sh create -a "$asn" -i "$(date -d "$i_s secs" +%s)" -p "$(date -d "$p_s secs" +%s)" -f "$(date -d "$f_s secs" +%s)" ${RUBRIC} + denis/configure.sh reload + popd +} + +# preconditions: +# - called after setup_submissions_and_grading_repo +setup_submissions_for() { + test -n "$1" + local user="$1" + + pushd "$SCRIPT_DIR" + orbit/warpdrive.sh -u "$user" -p builder -n || orbit/warpdrive.sh -u "$user" -p builder -m + popd + + pushd "$WORKDIR"/submissions + git config user.name "$user" + git config user.email "$user"@localhost.localdomain + git config sendemail.smtpUser "$user" + git config sendemail.smtpPass builder + git config sendemail.smtpserver localhost.localdomain + git config sendemail.smtpserverport 465 + git config sendemail.smtpencryption ssl + popd +} + +# preconditions: +# - no non-tracked files in submissions repo +# - called after setup_submissions_for +enter_and_checkout() { + test -n "$1" + local branch="$1" + + pushd "$WORKDIR"/submissions + git checkout --orphan "$1" + if [ -n "$(ls)" ] + then + git rm -rf ./* + fi +} + +# preconditions: +# - called after a call to enter_and_checkout +exit_after_sending() { + test -n "$1" + local asn="$1" + + git send-email \ + --confirm=never \ + --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem \ + --to "$asn"@localhost.localdomain \ + ./*.patch + rm ./*.patch + popd + sleep 1 +} + +# preconditions: +# - called after a call to enter_and_checkout +write_commit_to() { + test -n "$1" && test -n "$2" + local content="$1" + local file="$2" + set +u # necessary since $3 is unbound in 2 arg case + local opt="$3" + set -x + + mkdir -p $(dirname "$file") + + if [ "$opt" == "append" ]; then + echo "$content" >> "$file" + else + echo "$content" > "$file" + fi + + git add "$file" + git commit -sm "add $content to $file" +} + +# preconditions: +# - called after a call to write_commit_to +fixup_cover() { + test -n "$1" + sed -i "s/\*\*\* SUBJECT HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "s/\*\*\* BLURB HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "\$a\\Signed-off-by: $(git config user.name) <$(git config user.email)>" *0000-cover-letter.patch +} + diff --git a/test-sub-check.sh b/test-sub-check.sh index ae42ac92..4f49ae3d 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -exuo pipefail - source test-lib setup_testdir diff --git a/test-sub.sh b/test-sub.sh index d95648ec..b8ae94b0 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -1,283 +1,90 @@ #!/bin/bash -set -exuo pipefail - -SCRIPT_DIR=$(dirname "$0") - -cd "$SCRIPT_DIR" - -get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; - -# nuke grading repo inside container -# shellcheck disable=SC2016 -podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' - -# setup submissions repo -rm -rf repos/submissions -git/admin.sh submissions "course submissions repository" -pushd repos -git init --bare submissions -echo "course submissions repository" > submissions/description -git init submissions_init -pushd submissions_init -echo "# submissions" > README.md -git add README.md -git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' -git push ../submissions master -popd -rm -rf submissions_init -pushd submissions -git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions -popd -popd +# assumes singularity is running when invoked +# script is written to be as idempotent as posible -# create bob,bib,bab users or change their passwords to builder -orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +source test-lib -WORKDIR=$(mktemp -d) +setup_submissions_and_grading_repo cat << EOF > "$WORKDIR"/setup_rubric [{('--- /dev/null', '+++ b/bob/setup/work'): 0}, {('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] EOF -# create or recreate setup assignment -if denis/configure.sh dump | grep -q "^setup:"; then - denis/configure.sh remove -a setup -fi -denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric -denis/configure.sh reload - -# setup assignment environment -pushd "$WORKDIR" - -mkdir certs -pushd certs -podman volume export singularity_ssl-certs > certs.tar -tar xf certs.tar -popd -git clone http://localhost:"$(get_git_port)"/submissions - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +create_lighting_assignment setup 15 25 30 "$WORKDIR"/setup_rubric # setup_intial_bob patchsets +setup_submissions_for bob # good patchset -pushd submissions -git checkout --orphan setup_initial_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Good patchset" +exit_after_sending setup # corrupt patchset -pushd submissions -git checkout --orphan setup_initial_bob_corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -3 rm ./*-0002-*.patch - -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Corrupt patchset" +exit_after_sending setup # patchset with whitespace errors -pushd submissions -git checkout --orphan setup_initial_bob_whitespace -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def " >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_whitespace +write_commit_to "abc" bob/setup/work +write_commit_to "def " bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Patchset with whitespace errors" +exit_after_sending setup # patchset with no cover letter -pushd submissions -git checkout --orphan setup_initial_bob_nocover -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_nocover +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +exit_after_sending setup # patchset with no cover letter and corrupt first patch -pushd submissions -git checkout --orphan setup_initial_bob_nocover-corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_nocover-corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd - -# end of setup_initial_bob patchsets - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +exit_after_sending setup # setup_initial_bab submissions +setup_submissions_for bab # bab good patchset -pushd submissions -git checkout --orphan setup_initial_bab_good -git rm -rf ./* -mkdir -p bab/setup - -echo "abc" > bab/setup/work -git add bab/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bab/setup/work -git add bab/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bab_good +write_commit_to "abc" bab/setup/work +write_commit_to "def" bab/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +fixup_cover "Good patchset" +exit_after_sending setup -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm -rf ./*.patch -popd - -# end of bab initial submission - -# bib env setup -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib initial submissions +# setup_initial_bib submissions +setup_submissions_for bib # bib good patchset -pushd submissions -git checkout --orphan setup_initial_bib_good -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bib_good +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib initial submission +fixup_cover "Good patchset" +exit_after_sending setup # investigate grading repo +pushd "$WORKDIR" git clone http://localhost:"$(get_git_port)"/grading.git pushd grading git fetch --tag -git tag - STATUSES=( 'patchset applies.' @@ -300,36 +107,17 @@ for t in $(git tag); do fi i=$((i+1)) done - popd -echo "wait for initial submission deadline (10 secs)" -sleep 10 - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +echo "wait for initial submission deadline (5 secs)" +sleep 5 +setup_submissions_for bob # setup bob peer review -pushd submissions -git checkout --orphan setup_reviews_bob -git rm -rf ./* -# mkdir -p bob/setup (must be able to omit if missing) -echo ORPHAN > ORPHAN -git add ORPHAN -git commit -m 'orphan base' +enter_and_checkout setup_reviews_bob # submit peer review 1 - cat < review1 Subject: setup review 1 for bab In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> @@ -338,14 +126,10 @@ Looks good to me Acked-by: bob PEER_REVIEW1 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 rm review1 -# end peer review 1 - # submit peer review 2 - cat < review2 Subject: setup review 2 for bib In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> @@ -354,131 +138,39 @@ Looks good to me Acked-by: bob PEER_REVIEW2 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 rm review2 -# end peer review 2 - -# end bob peer review popd sleep 1 +# end bob peer review -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bob make final submission - -# good final patchset -pushd submissions -git checkout --orphan setup_final_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +# setup_final_bob good patchset +setup_submissions_for bob +enter_and_checkout setup_final_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --cover-letter -v2 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end bob final submission - -# bib env setup - -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib final submission labeled RFC -pushd submissions -git checkout --orphan setup_final_bib_rfc -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +fixup_cover "Good final patchset" +exit_after_sending setup + +# setup_final_bib final submission incorrectly labeled RFC +setup_submissions_for bib +enter_and_checkout setup_final_bib_rfc +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib final submission labeled RFC - - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - - -# bab final submission in wrong directory -pushd submissions -git checkout --orphan setup_final_bab_illegal -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > work -git add work -git commit -sm 'add abc to work' - -echo "def" >> work -git add work -git commit -sm 'add def to work' - +fixup_cover "RFC tagged final patchset" +exit_after_sending setup + +# setup_final_bab submission in wrong directory +setup_submissions_for bab +enter_and_checkout setup_final_bab_illegal +write_commit_to "abc" work +write_commit_to "def" work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bab final submission in wrong directory - +fixup_cover "Illegal patchset" +exit_after_sending setup echo "wait for final submission deadline (10 secs)" sleep 10 @@ -488,14 +180,17 @@ git fetch --tag sleep 2 git fetch -f origin refs/notes/*:refs/notes/* git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + git notes --ref=grade add setup_final_bob -m '66' git notes --ref=grade add setup_review1_bob -m '22' git notes --ref=grade add setup_review2_bob -m '99' git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' +# automatic 0 for peer reviews git notes --ref=grade add setup_final_bib -m '100' git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' +# automatic 0 for peer reviews and final submission git notes --ref=feedback add setup_final_bab -m 'Please review git' git push origin refs/notes/*:refs/notes/* diff --git a/test-sub2.sh b/test-sub2.sh new file mode 100755 index 00000000..80e5e733 --- /dev/null +++ b/test-sub2.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# test a bunch of rubric violations + +source test-lib + +setup_submissions_and_grading_repo + +cat << EOF > "$WORKDIR"/coding_rubric +[{('--- /dev/null', '+++ b/user/coding/program.c'): 0}, +{('--- a/user/coding/program.c', '+++ b/user/coding/program.c'): 0}, +{('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] +EOF + +create_lighting_assignment coding 25 28 30 "$WORKDIR"/coding_rubric + +setup_submissions_for bob + +enter_and_checkout coding_initial_bob_toomany +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Yet more C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -4 +fixup_cover "Too many patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_toofew +write_commit_to "C program" bob/coding/program.c +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -2 +fixup_cover "Too few patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-first +write_commit_to "C program" program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-first +write_commit_to "C program" bob/program.c +write_commit_to "More C program" bob/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-first +write_commit_to "C program" bob/coding/program/program.c +write_commit_to "More C program" bob/coding/program/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/program/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-first +write_commit_to "C program" bob/setup/program.c +write_commit_to "More C program" bob/setup/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/setup/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-first +write_commit_to "C program" bob/coding/hello.c +write_commit_to "More C program" bob/coding/hello.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Snakefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_good +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Good patches" +exit_after_sending coding + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +STATUSES=( +"patch count 4 violates expected rubric patch count of 3!" +"patch count 2 violates expected rubric patch count of 3!" +"illegal patch 1: permission denied for path program.c!" +"illegal patch 3: permission denied for path Makefile!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patchset applies." +) + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + i=$((i+1)) +done +popd +popd + +echo "RUBRIC CHECKS PASS" +echo "$WORKDIR" diff --git a/test.sh b/test.sh index c28d13da..f87528c9 100755 --- a/test.sh +++ b/test.sh @@ -2,14 +2,6 @@ # Testing script for singularity and orbit -# This line: -# - aborts the script after any pipeline returns nonzero (e) -# - shows all commands as they are run (x) -# - sets any dereference of an unset variable to trigger an error (u) -# - causes the return value of a pipeline to be the nonzero return value -# of the furthest right failing command or zero if no command failed (o pipefail) -set -exuo pipefail - source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } From 04549c112c3e647d4ba7ce2eecde2178614abc2a Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 2 Jun 2025 15:02:54 -0400 Subject: [PATCH 08/14] test: add test for whether only initial submission gets autozero If a student makes an inital submission but not a final submission, they should automatically receive a zero for the assignment. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 7 +++++++ test-sub3.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100755 test-sub3.sh diff --git a/script-lint.sh b/script-lint.sh index 2731cad9..d9823ae4 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -11,6 +11,7 @@ shellcheck -x test.sh shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh shellcheck -x test-sub2.sh +shellcheck -x test-sub3.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index 07b38725..b428e225 100755 --- a/start.sh +++ b/start.sh @@ -29,7 +29,14 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + podman-compose down + yes | podman volume prune + podman-compose up -d ./test-sub2.sh + podman-compose down + yes | podman volume prune + podman-compose up -d + ./test-sub3.sh fi else diff --git a/test-sub3.sh b/test-sub3.sh new file mode 100755 index 00000000..55b28c30 --- /dev/null +++ b/test-sub3.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# test only initial submission results in automatic 0 + +source test-lib + +setup_submissions_and_grading_repo + +create_lighting_assignment fun 5 7 10 + +setup_submissions_for bob + +enter_and_checkout fun_initial_bob_only-initial +write_commit_to "C program" bob/fun/program.c +write_commit_to "More C program" bob/fun/program.c "append" +write_commit_to "Yet more C program" bob/fun/program.c "append" +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Only initial sub patchset" +exit_after_sending fun + +# wait for initial submission deadline +sleep 9 + +setup_testdir + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for fun assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "fun" + +grep "patchset applies." test/dashboard +grep "No submission" test/dashboard +grep "Total Score: 0.0" test/dashboard + +echo "INITIAL SUBMISSION ONLY GETS AUTO ZERO CONFIRMED" From 8bf9692dde50d8aeed3a2296000f8bf34443850b Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 9 Jul 2025 18:08:16 -0400 Subject: [PATCH 09/14] test: add test-sub4 to check peer review corner cases Check that total score is calculated correctly when only two students submit and only one review is required or no students submit so no reviews are required. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 4 ++ test-sub4.sh | 116 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+) create mode 100755 test-sub4.sh diff --git a/script-lint.sh b/script-lint.sh index d9823ae4..ef2d49ad 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -12,6 +12,7 @@ shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh shellcheck -x test-sub2.sh shellcheck -x test-sub3.sh +shellcheck -x test-sub4.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index b428e225..9649b7bc 100755 --- a/start.sh +++ b/start.sh @@ -37,6 +37,10 @@ then yes | podman volume prune podman-compose up -d ./test-sub3.sh + podman-compose down + yes | podman volume prune + podman-compose up -d + ./test-sub4.sh fi else diff --git a/test-sub4.sh b/test-sub4.sh new file mode 100755 index 00000000..76c3f65c --- /dev/null +++ b/test-sub4.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# test total score calculation when assigned only 0 or 1 peer reviews legitimately + +source test-lib + +setup_submissions_and_grading_repo + +# single peer review case + +cat << EOF > "$WORKDIR"/first_rubric +[{('--- /dev/null', '+++ b/bob/first/work'): 0}, +{('--- a/bob/first/work', '+++ b/bob/first/work'): 0}] +EOF + +create_lighting_assignment first 10 11 20 "$WORKDIR"/first_rubric + +setup_submissions_for bob + +enter_and_checkout first_initial_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +setup_submissions_for bab + +enter_and_checkout first_initial_bab_good +write_commit_to "abc" bab/first/work +write_commit_to "def" bab/first/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending first + +sleep 8 + +setup_submissions_for bob +enter_and_checkout first_final_bob_good +write_commit_to "abc" bob/first/work +write_commit_to "def" bob/first/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending first + +sleep 9 + +# no peer review case + +cat << EOF > "$WORKDIR"/second_rubric +[{('--- /dev/null', '+++ b/bob/second/work'): 0}, +{('--- a/bob/second/work', '+++ b/bob/second/work'): 0}] +EOF + +create_lighting_assignment second 5 7 10 "$WORKDIR"/second_rubric + +setup_submissions_for bob + +enter_and_checkout second_initial_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --rfc --cover-letter -v1 -2 +fixup_cover "Good patchset" +exit_after_sending second + +sleep 4 + +enter_and_checkout second_final_bob_good +write_commit_to "abc" bob/second/work +write_commit_to "def" bob/second/work "append" +git format-patch --cover-letter -v2 -2 +fixup_cover "Good final patchset" +exit_after_sending second + +sleep 4 + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + +git notes --ref=grade add first_final_bob -m '100' +git notes --ref=grade add first_review1_bob -fm '100' +git notes --ref=grade add second_final_bob -m '50' +git push origin refs/notes/*:refs/notes/* + +popd +popd + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# get bob dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" > test/dashboard + +# first assginment should have full score since the single required peer review was completed perfectly +grep "Total Score: 100.0" test/dashboard +# assginment should have half score since no peer review was needed and final sub got half score +grep "Total Score: 50.0" test/dashboard + +echo "PEER REVIEW CORNER CASES PASS" +echo "$WORKDIR" From 9917f19d745ae0c370150005bc88d4025ddd0239 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Tue, 26 Aug 2025 07:50:27 -0400 Subject: [PATCH 10/14] test-lib: use unpriveleged port CI now runs the containers as a non-root user and exposes 1465 for SMTP instead of 465 Signed-off-by: Joel Savitz --- test-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-lib b/test-lib index 1b9607fb..57a49e60 100644 --- a/test-lib +++ b/test-lib @@ -126,7 +126,7 @@ setup_submissions_for() { git config sendemail.smtpUser "$user" git config sendemail.smtpPass builder git config sendemail.smtpserver localhost.localdomain - git config sendemail.smtpserverport 465 + git config sendemail.smtpserverport 1465 git config sendemail.smtpencryption ssl popd } From 1bbdd60c518ab3d54eeb71a585be519c4df7132a Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Tue, 26 Aug 2025 07:52:29 -0400 Subject: [PATCH 11/14] test-sub: correct bob version numbers Signed-off-by: Joel Savitz --- test-sub.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-sub.sh b/test-sub.sh index b8ae94b0..47667481 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -30,7 +30,7 @@ enter_and_checkout setup_initial_bob_corrupt write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" write_commit_to "ghi" bob/setup/work "append" -git format-patch --rfc --cover-letter -v1 -3 +git format-patch --rfc --cover-letter -v2 -3 rm ./*-0002-*.patch fixup_cover "Corrupt patchset" exit_after_sending setup @@ -39,7 +39,7 @@ exit_after_sending setup enter_and_checkout setup_initial_bob_whitespace write_commit_to "abc" bob/setup/work write_commit_to "def " bob/setup/work "append" -git format-patch --rfc --cover-letter -v1 -2 +git format-patch --rfc --cover-letter -v3 -2 fixup_cover "Patchset with whitespace errors" exit_after_sending setup @@ -47,7 +47,7 @@ exit_after_sending setup enter_and_checkout setup_initial_bob_nocover write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" -git format-patch --rfc -v1 -2 +git format-patch --rfc -v4 -2 exit_after_sending setup # patchset with no cover letter and corrupt first patch @@ -55,7 +55,7 @@ enter_and_checkout setup_initial_bob_nocover-corrupt write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" write_commit_to "ghi" bob/setup/work "append" -git format-patch --rfc -v1 -2 +git format-patch --rfc -v5 -2 exit_after_sending setup # setup_initial_bab submissions @@ -150,7 +150,7 @@ setup_submissions_for bob enter_and_checkout setup_final_bob_good write_commit_to "abc" bob/setup/work write_commit_to "def" bob/setup/work "append" -git format-patch --cover-letter -v2 -2 +git format-patch --cover-letter -v6 -2 fixup_cover "Good final patchset" exit_after_sending setup From d809b224b3f0223eb25dd665fec0e6a82b4405c5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 7 Sep 2025 16:39:05 -0400 Subject: [PATCH 12/14] test: remove rubrics Remove assignment rubric specificiation when invoking create_lighting_assignment since the feature is deferred to 0.8 Signed-off-by: Joel Savitz --- test-sub.sh | 2 +- test-sub2.sh | 2 +- test-sub4.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-sub.sh b/test-sub.sh index 47667481..1a5dbc6d 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -12,7 +12,7 @@ cat << EOF > "$WORKDIR"/setup_rubric {('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] EOF -create_lighting_assignment setup 15 25 30 "$WORKDIR"/setup_rubric +create_lighting_assignment setup 15 25 30 # setup_intial_bob patchsets setup_submissions_for bob diff --git a/test-sub2.sh b/test-sub2.sh index 80e5e733..d9b6893b 100755 --- a/test-sub2.sh +++ b/test-sub2.sh @@ -12,7 +12,7 @@ cat << EOF > "$WORKDIR"/coding_rubric {('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] EOF -create_lighting_assignment coding 25 28 30 "$WORKDIR"/coding_rubric +create_lighting_assignment coding 25 28 30 setup_submissions_for bob diff --git a/test-sub4.sh b/test-sub4.sh index 76c3f65c..77350786 100755 --- a/test-sub4.sh +++ b/test-sub4.sh @@ -13,7 +13,7 @@ cat << EOF > "$WORKDIR"/first_rubric {('--- a/bob/first/work', '+++ b/bob/first/work'): 0}] EOF -create_lighting_assignment first 10 11 20 "$WORKDIR"/first_rubric +create_lighting_assignment first 10 11 20 setup_submissions_for bob @@ -52,7 +52,7 @@ cat << EOF > "$WORKDIR"/second_rubric {('--- a/bob/second/work', '+++ b/bob/second/work'): 0}] EOF -create_lighting_assignment second 5 7 10 "$WORKDIR"/second_rubric +create_lighting_assignment second 5 7 10 setup_submissions_for bob From 97ea2ea5ae0b7ac5c2d20ce698922dd168b854e8 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 7 Sep 2025 16:47:22 -0400 Subject: [PATCH 13/14] test: remove rubric status checks Signed-off-by: Joel Savitz --- test-sub2.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test-sub2.sh b/test-sub2.sh index d9b6893b..cb97aa99 100755 --- a/test-sub2.sh +++ b/test-sub2.sh @@ -125,18 +125,18 @@ git clone http://localhost:"$(get_git_port)"/grading.git pushd grading STATUSES=( -"patch count 4 violates expected rubric patch count of 3!" -"patch count 2 violates expected rubric patch count of 3!" +"patchset applies." +"patchset applies." "illegal patch 1: permission denied for path program.c!" "illegal patch 3: permission denied for path Makefile!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" -"patch 1 violates the assignment rubric!" -"patch 3 violates the assignment rubric!" +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." +"patchset applies." "patchset applies." ) From da68058a51bd91dbd523317e97e12650f737c671 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 8 Sep 2025 13:54:22 -0400 Subject: [PATCH 14/14] test: start.sh: simplify volume deletion Signed-off-by: Joel Savitz --- start.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/start.sh b/start.sh index 9649b7bc..da5ea7e3 100755 --- a/start.sh +++ b/start.sh @@ -20,8 +20,7 @@ then ./test.sh if [ -f test-sub.sh ] then - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' ./dev_sockets.sh & @@ -29,16 +28,13 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub2.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub3.sh - podman-compose down - yes | podman volume prune + podman-compose down -v podman-compose up -d ./test-sub4.sh