launch

#!/bin/bash

SERVERLIST='Hackgrab-a Hackgrab-b Hackgrab-c Hackgrab-d'
BASE='/srv/hackgrab'

THIS="${BASE}/bin/launch"
RETRIES=10
INTERVAL=1

HOSTNAME=`hostname`
LOG="${BASE}/logs/${HOSTNAME}.log"


ARRAY=(${SERVERLIST})
SIZE=${#ARRAY[@]}

#Check if we have been outed
if [ -e ${BASE}/metadata/outed_${HOSTNAME} ]
then
    exit 2
fi

#Check to see if we're one of the worknodes.
if echo ${SERVERLIST} | grep -qw "${HOSTNAME}"
then

    #See if anyone else has been outed.
    if ls ${BASE}/metadata/outed_*
    then
	#Well, then we are, too.
	touch ${BASE}/metadata/outed_${HOSTNAME}
	echo "Outed" > ${BASE}/metadata/status_${HOSTNAME}
	exit 2
    fi
    
    #Check to see if we're busy.  If so, reject the job.
    if [ -e /dev/shm/Hackgrab_PID ]
    then
	OLDPID=`cat /dev/shm/Hackgrab_PID`
	if ps -p ${OLDPID}
	then
            exit 2
	fi
    else
	#If we're not busy, then run the job.
	/srv/hackgrab/bin/grab_one "${1}" >> ${LOG} 2>&1 &
    fi
else
    #If we're not a worknode, then pick a worknode at random and dispatch the job.
    RETRY=true
    while ${RETRY}
    do
	  INDEX=$(($RANDOM % $SIZE))
	  SERVER=${ARRAY[$INDEX]}
	  echo "Launching on ${SERVER}"
	  echo "${THIS} \"${1}\"" | ssh ${SERVER}
	  RESULT=$?
	  if [ ${RESULT} != 2 ] && [ ${RESULT} != 255 ]
	  then
	      RETRY=false
	  else
	      RETRIES=$((${RETRIES} - 1))
	      if [ ${RETRIES} = 0 ]
	      then
		  echo "Out of retries.  Exiting."
		  RETRY=false
	      else
		  echo "${SERVER} is busy.  Trying again in ${INTERVAL} seconds."
		  sleep ${INTERVAL}
	      fi
	  fi
    done
fi