diff --git a/Ruby/.gitrun b/Ruby/.gitrun new file mode 100644 index 00000000..018a9de4 --- /dev/null +++ b/Ruby/.gitrun @@ -0,0 +1,12 @@ + +$XAS_CI = true; + +$runner = GitRestart::Runner.new do |r| + r.name = "XasStart CI"; + r.repo = "XasWorks/XasCode"; + + r.mqtt = "mqtt://xasin.hopto.org"; + r.octokit = Octokit::Client.new(netrc: true); + + r.start_on = r.current_branch(); +end diff --git a/Ruby/GitRestart/.gitignore b/Ruby/GitRestart/.gitignore new file mode 100644 index 00000000..d98a4bfe --- /dev/null +++ b/Ruby/GitRestart/.gitignore @@ -0,0 +1,2 @@ +.yardoc +doc diff --git a/Ruby/GitRestart/.gittask b/Ruby/GitRestart/.gittask new file mode 100644 index 00000000..9e6f45d6 --- /dev/null +++ b/Ruby/GitRestart/.gittask @@ -0,0 +1,11 @@ + +GitRestart::Task.new do |t| + t.ci_task = true; + t.name = "GitRestart Task" + + t.report_status = true; + + t.active = true; + + t.targets << "ruby tests/tc_task.rb" +end diff --git a/Ruby/GitRestart/Guardfile b/Ruby/GitRestart/Guardfile new file mode 100644 index 00000000..73140a36 --- /dev/null +++ b/Ruby/GitRestart/Guardfile @@ -0,0 +1,9 @@ + +# with Minitest::Unit +guard :minitest, + test_folders: ['tests'], + test_file_patterns: ["tc_*.rb"] do + + watch(%r{^lib/git-restart/([^/]+)\.rb}) { |m| "tests/tc_#{m[1]}.rb"} + watch(%r{^tests/tc_(.*).rb}) +end diff --git a/Ruby/GitRestart/bin/git-restart b/Ruby/GitRestart/bin/git-restart new file mode 100755 index 00000000..b9aa7e47 --- /dev/null +++ b/Ruby/GitRestart/bin/git-restart @@ -0,0 +1,22 @@ +#!/usr/bin/ruby + +require 'mqtt/sub_handler.rb' +require 'git-restart/runner.rb' + +puts "Starting runner ..." + +$taskfiles = Array.new(); +target = ".gitrun" +ARGV.each do |t| + if(t =~ /\.gitrun$/) + target = t; + else + $taskfiles << t; + end +end + +raise ArgumentError, "No valid runner file specified!" unless File.exist? target + +load target + +GitRestart::Task.runner.mqtt.lockAndListen diff --git a/Ruby/GitRestart/git-restart.gemspec b/Ruby/GitRestart/git-restart.gemspec new file mode 100644 index 00000000..c574bcd0 --- /dev/null +++ b/Ruby/GitRestart/git-restart.gemspec @@ -0,0 +1,27 @@ +Gem::Specification.new do |s| + s.name = 'git-restart' + s.version = '0.1.0' + s.summary = '(Re)start scripts and monitor them on a GitHub push' + s.description = 'This gem can be used to (re)start scripts whenever a GitHub push event is recorded. +The exit status of scripts can be monitored, and a failure can be sent back, making this capable of running simple tests too!' + s.authors = ['Xasin'] + + s.files = [ 'bin/git-restart', + 'lib/git-restart/runner.rb', + 'lib/git-restart/task.rb'] + + s.executables << 'git-restart' + + + s.homepage = + 'https://github.com/XasWorks/XasCode/tree/master/Ruby/GitRestart' + s.license = 'GPL-3.0' + + s.add_runtime_dependency "mqtt-sub_handler", "~> 0.1" + s.add_runtime_dependency "git", "~> 1.4" + s.add_runtime_dependency "octokit", "~> 4.0" + + s.add_development_dependency "minitest" + s.add_development_dependency "guard" + s.add_development_dependency "guard-minitest" +end diff --git a/Ruby/GitRestart/lib/git-restart/runner.rb b/Ruby/GitRestart/lib/git-restart/runner.rb new file mode 100644 index 00000000..dbdc7ddc --- /dev/null +++ b/Ruby/GitRestart/lib/git-restart/runner.rb @@ -0,0 +1,237 @@ + +require 'mqtt/sub_handler' + +require 'git' +require 'octokit' + +require_relative "task.rb" + +module GitRestart + class Runner + # Sets a name for this Runner, used for reporting to GitHub + attr_accessor :name + + # Which repository to listen to. Uses "Owner/Repo" syntax. + attr_accessor :repo + # A white- and blacklist of branches. If neither are specified, all are used. + # If the whitelist is used, only it is considered. + attr_accessor :branches, :exclude_branches + # Which branch to start on. + # This not only makes the system switch branch, but it will also execute + # ALL active tasks. Very useful for auto-updating servers. + attr_accessor :start_on + # A list of tasks that this Runner will actually look at. + # If nil, all tasks are allowed. + attr_accessor :allowed_tasks + + attr_reader :next_tasks + attr_reader :current_task_file + + # The MQTT::SubHandler to use to listen to GitHub updates. + # Can be specified as either MQTT::SubHandler class or String, the latter + # will be interpreted as URI + attr_reader :mqtt + # Octokit to use for optional status reporting + attr_accessor :octokit + + # @return [String] Full SHA of the current commit + def current_commit() + @git.object("HEAD").sha; + end + # @return [String] Name of the current branch + def current_branch() + @git.current_branch(); + end + # @return [Array] A list of all files that were modified in the commit we are checking + def current_modified() + @current_modified; + end + + def initialize(fileList = nil) + raise ArgumentError, "File list needs to be nil or an Array!" unless (fileList.is_a? Array or fileList.nil?) + + GitRestart::Task.runner = self; + + @allowed_tasks = Array.new(); + @allowed_tasks << fileList if(fileList); + @allowed_tasks << $taskfiles unless($taskfiles.empty?) + + @current_tasks = Hash.new(); + @next_tasks = Hash.new(); + + @branches = Array.new(); + @exclude_branches = Array.new(); + + @branchQueue = Queue.new(); + + @git = Git.open("."); + + yield(self); + + @allowed_tasks.flatten! + + @listenedSub = @mqtt.subscribe_to "GitHub/#{@repo}" do |data| + begin + data = JSON.parse(data, symbolize_names: true); + rescue + next; + end + + next unless data[:branch]; + if(not @branches.empty?) + next unless @branches.include? data[:branch]; + elsif(not @exclude_branches.empty?) + next if @exclude_branches.include? data[:branch]; + end + + @branchQueue << data; + end + + autostart(); + _start_task_thread(); + + at_exit { + _stop_all_tasks(); + } + end + + # Update the GitHub status for the task of given name, with optional status message + # Only prints a line if no octokit is specified + def update_status(name, newStatus, message = nil) + puts "Task #{name} assumed a new status: #{newStatus}#{message ? " MSG:#{message}" : ""}" + + return unless @octokit; + + begin + @octokit.create_status(@repo, current_commit(), newStatus, { + context: "#{name}/#{name}".gsub(" ", "_"), + description: message, + }) + rescue + end + end + + # Start the task responsible for queueing and executing the individual + # task stop, branch switch, task start cycles + def _start_task_thread() + @taskThread = Thread.new do + loop do + newData = @branchQueue.pop; + + @current_modified = newData[:touched]; + _switch_to(newData[:branch], newData[:head_commit]); + end + end.abort_on_exception = true; + end + private :_start_task_thread + + # Stop all tasks of the given hash, not list. Waits for them to stop. + def _stop_tasks(taskList) + taskList.each do |name, t| + t.stop(); + end + taskList.each do |name, t| + t.join(); + @current_tasks.delete(name); + end + end + private :_stop_tasks + def _stop_all_tasks() + _stop_tasks(@current_tasks); + end + private :_stop_all_tasks + # Stop all tasks that have marked themselves as affected by the + # current set of file-changes. This way, applications are only + # restarted when their own files have been altered. + def _stop_triggered_tasks() + _stop_tasks(@current_tasks.select {|k,v| v.triggered?}); + end + private :_stop_triggered_tasks + + # Scan through the file-tree for .gittask files, or use the @allowed_tasks + # list of tasks. + def _generate_next_tasks() + puts "Generating new tasks..." + @next_tasks = Hash.new(); + + taskFiles = `find ./ -nowarn -iname "*.gittask"` + [taskFiles.split("\n"), @allowed_tasks].flatten.each do |t| + puts "Looking at: #{t}" + t.gsub!(/^\.\//,""); + @current_task_file = t; + + unless(@allowed_tasks.empty?) + next unless @allowed_tasks.include? @current_task_file + end + + begin + load(t); + rescue ScriptError, StandardError + update_status("File #{t}", :failure, "File could not be parsed!") + puts("File #{t} could not be loaded!"); + rescue GitRestart::TaskValidityError + update_status("File #{t}", :failure, "Task-file not configured properly!") + puts("Task-File #{t} is not configured properly!"); + end + end + + puts "Finished loading! Next tasks: #{@next_tasks.keys}" + end + private :_generate_next_tasks + + # Start all new tasks that have marked themselves as affected by the current + # set of filechanges + def _start_next_tasks() + _generate_next_tasks(); + + puts "\nStarting next tasks!" + @next_tasks.each do |name, t| + next unless t.active; + next unless t.triggered? + + t.start(); + @current_tasks[name] = t; + end + end + private :_start_next_tasks + + # Perform an entire cycle of git fetch & checkout, stop tasks, pull, restart. + # *CAUTION* A HARD RESET IS USED HERE + def _switch_to(branch, commit = nil) + puts "\n\nSwitching to branch: #{branch}#{commit ? ",commit: #{commit}" : ""}" + + begin + @git.fetch(); + rescue + end + + if(branch != current_branch()) + _stop_all_tasks(); + else + _stop_triggered_tasks(); + end + @git.reset_hard(); + @git.checkout(branch); + @git.merge("origin/#{branch}"); + + @git.reset_hard(commit); + + _start_next_tasks(); + end + private :_switch_to + + def autostart() + return unless @start_on; + @branchQueue << {branch: @start_on}; + end + private :autostart + + def mqtt=(mqtt) + if(mqtt.is_a? String) + @mqtt = MQTT::SubHandler.new(mqtt); + else + @mqtt = mqtt; + end + end + end +end diff --git a/Ruby/GitRestart/lib/git-restart/task.rb b/Ruby/GitRestart/lib/git-restart/task.rb new file mode 100644 index 00000000..b6e3f978 --- /dev/null +++ b/Ruby/GitRestart/lib/git-restart/task.rb @@ -0,0 +1,269 @@ + +# @author Xasin +module GitRestart + # The Error-Class used to signal when a Task is set up wrong + class TaskValidityError < StandardError + end + + # This class is used to define "Tasks". Each task represents + # a set of commands, which it executes in chronological order, or until + # a task errors. + # Additionally, it will kill execution of tasks with a specified kill-signal + # when an update was detected from GitHub. + # The failure-status of the tasks can also be reported via Octokit, allowing this + # to be used as a simple CI or Test system for various languages. + class Task + # The array of tasks to execute. Each target will be executed in the given + # order via `Process.spawn`. + # @return [Array] + attr_reader :targets + + # The signal (as String, like Signal.list) to use to kill the process. + # Can be nil to disable killing + attr_accessor :signal + # Whether or not to report failure if the currently running target + # has a non-zero exit status after having been killed. Only makes sense + # together with report_status + attr_accessor :expect_clean_exit + # Whether or not to report failure/success status to GitHub using Octokit + attr_accessor :report_status + # Defines this as a "CI_Task". Such a task will always run on an update, + # regardless what files changed. Useful if you always want a status report + # on GitHub. + attr_accessor :ci_task + # Name of the Task. *Required*. Used as *unique* ID, and to report status to GitHub + attr_accessor :name + # The file to use to retrieve a single-line status info for the "description" + # string of the GitHub status. Only the last *non-indented* line is used, + # which allows the output of Minitest to be used directly. + attr_accessor :status_file + + # Whether or not this task is active. Usually set via #on_branches, + # but can be used to manually disable or enable this task based on + # config files, ENV variables etc. + attr_accessor :active + + # The last status-code of this Task. Used internally. + attr_reader :lastStatus + # The last status-message of this task. Used internally. + attr_reader :status_message + + # @api private + def self.runner=(runner) + @runner = runner; + end + # @api private + def self.runner() + return @runner; + end + # @return [GitRestart::Runner] Responsible Runner class + def runner() + return self.class.runner(); + end + + # @return [String] Name of the current branch + def branch() + runner().current_branch(); + end + # @return [String] Full SHA of the current commit + def current_commit() + runner().current_commit(); + end + + # @return [Array] A list of all files that were modified in the commit we are checking + def modified() + runner().current_modified(); + end + + # Use this function to specify which files trigger a restart for this Task + # Files can be specified as a RegEx, and can be "local/like.this" or "/reference/from/project.root" + def watch(regEx) + if(regEx.is_a? String) + regEx = Regexp.quote(regEx); + end + + @watched << Regexp.new(regEx); + end + + # Specify which branches to run on. Not needed if "active" is just set to true + def on_branches(branches) + [branches].flatten.each do |b| + @active |= (b == branch()); + end + end + + # Create a new Task. This function does not take any input values, instead, + # one has to set the class up inside the block! + # A validity check will be run directly after yield(), as such, at the very least + # the name and a valid signal must have been specified! + def initialize() + @statuschange_mutex = Mutex.new(); + + @targets = Array.new(); + @watched = Array.new(); + + @signal = "INT" + @expect_clean_exit = true; + @exiting = false; + + @lastStatus = 0; + @chdir = File.dirname(runner().current_task_file); + + watch(File.basename(runner().current_task_file)); + + yield(self); + + valid? + + @status_file ||= "/tmp/TaskLog_#{@name}_#{current_commit()}"; + + if(runner().next_tasks[@name]) + raise TaskValidityError, "A task of name #{@name} already exists!" + else + runner().next_tasks[@name] = self; + end + end + + # Checks whether or not the current set of modified files would require this + # task to be (re)started. Always returns true if @ci_task is set, or if + # the runner just has been started using @start_on + # @api private + def triggered? + return true if modified().nil? + return true if @ci_task + + @watched.each do |regEx| + modified().each do |f| + if regEx.to_s =~ /^\(\?\-mix:\\\/(.*)\)$/ then + return true if f =~ Regexp.new($1); + else + next unless f =~ /#{Regexp.quote(@chdir)}(.*)/ + return true if $1 =~ regEx; + end + end + end + + return false; + end + + # Checks whether or not this task has been set up properly. Currently only + # checks the name and abort signal. + # @api private + def valid?() + unless Signal.list[@signal] or @signal.nil? + raise TaskValidityError, "The specified kill-signal is not valid!" + end + + unless @name + raise TaskValidityError, "A name needs to be set for identification!" + end + end + + def _rm_logfile() + if File.exist?("/tmp/TaskLog_#{@name}_#{current_commit()}") then + File.delete("/tmp/TaskLog_#{@name}_#{current_commit()}"); + end + end + private :_rm_logfile + def _get_statusline() + return "No status specified" unless File.exist? @status_file + + sMsg = "" + File.open(@status_file, "r") do |sFile| + sFile.each_line do |l| + l.chomp! + next if l == ""; + next if l =~ /^\s+/; + + sMsg = l; + end + end + + return sMsg; + end + private :_rm_logfile + + def _report_status(status, message = nil) + message ||= _get_statusline(); + @status_message = message; + + return unless @report_status + + runner().update_status(@name, status, message); + end + private :_report_status + + # Starts this task. + # Once the task has been started it will run each given + # target one after another, waiting for each one to finish. If @report_status + # is set, it will also do just that. + # Task execution is handled within a thread, meaning that the function + # itself does not block. + # @api private + def start() + puts "Starting Task: #{@name}" + + if @targets.empty? + _report_status(:success, "No tasks to run!"); + return + end + + @executionThread = Thread.new do + _report_status(:pending); + + _rm_logfile(); + @targets.each do |target| + # Mutex to ensure there either is no task running or a PID given + @statuschange_mutex.synchronize { + break if @exiting + options = { + [:out, :err] => "/tmp/TaskLog_#{@name}_#{current_commit()}" + } + options[:chdir] = @chdir if @chdir + + @currentPID = Process.spawn(target, options); + } + + status = Process.wait2(@currentPID)[1]; + @currentPID = nil; + @lastStatus = status.exitstatus(); + + break unless @lastStatus == 0; + end + + if(@lastStatus == 0) + _report_status(:success); + _rm_logfile(); + elsif(!@exiting || @expect_clean_exit) + _report_status(:failure); + end + end + @executionThread.abort_on_exception = true; + + sleep 0.01 + end + + # Stop this task. + # Stopping it means immediately killing the currently running target with + # the specified signal, and not running any further targets. + # *Except* when nil is specified as signal, in which case the stop will be ignored! + # @api private + def stop() + puts "Stopping Task: #{@name}" + return if @signal.nil? + + @statuschange_mutex.synchronize { + @exiting = true; + if(p = @currentPID) + Process.kill(@signal, p); + end + } + end + + # Wait for this task to finish execution. + # Either by naturally ending, or by being killed. + def join() + @executionThread.join(); + end + end +end diff --git a/Ruby/GitRestart/tests/tc_task.rb b/Ruby/GitRestart/tests/tc_task.rb new file mode 100644 index 00000000..23c4378c --- /dev/null +++ b/Ruby/GitRestart/tests/tc_task.rb @@ -0,0 +1,188 @@ + +require_relative "../lib/git-restart/task.rb" +require 'minitest/autorun' + +class DummyRunner + attr_accessor :current_branch, :current_commit, :current_modified; + attr_accessor :current_task_file; + attr_accessor :next_tasks; + + def initialize() + @current_branch = ""; + @current_commit = "wkdfaosdo2988a9sd"; + @current_modified = Array.new(); + @current_task_file = ""; + + @next_tasks = Hash.new(); + end + + def update_status(*args) + end +end + +class Test_Task < Minitest::Test + def setup() + `rm /tmp/TEST_FILE_* 2>/dev/null` + + @runner = DummyRunner.new(); + GitRestart::Task.runner = @runner; + end + + def teardown() + `rm /tmp/TEST_FILE_* 2>/dev/null` + end + + def test_init + # Test if the whole code starts normally + @task = GitRestart::Task.new() do |t| + t.name = "Test-Task" + end + @task.valid? + + # Check if signals are checked properly + @task.signal = nil; + @task.valid? + + @task.signal = "NonexistantSignal" + assert_raises { @task.valid? } + @task.signal = "INT" + end + + def test_abort_on_error + @task = GitRestart::Task.new() do |t| + t.name = "TestTask" + + t.targets << "touch /tmp/TEST_FILE_1"; + t.targets << "exit 1"; + t.targets << "touch /tmp/TEST_FILE_2"; + end + + @task.start(); + @task.join(); + + assert_equal 1, @task.lastStatus; + assert File.exist?("/tmp/TEST_FILE_1"); + refute File.exist?("/tmp/TEST_FILE_2"); + end + + def test_stop_execution + @task = GitRestart::Task.new() do |t| + t.name = "TestTask" + + t.targets << "touch /tmp/TEST_FILE_1"; + t.targets << "sleep 10"; + t.targets << "touch /tmp/TEST_FILE_2"; + end + + @task.start(); + sleep(0.5); + @task.stop(); + @task.join(); + + assert File.exist?("/tmp/TEST_FILE_1"); + refute File.exist?("/tmp/TEST_FILE_2"); + end + + def test_complete_execution + @task = GitRestart::Task.new() do |t| + t.name = "TestTask" + + t.targets << "touch /tmp/TEST_FILE_1"; + t.targets << "sleep 1"; + t.targets << "touch /tmp/TEST_FILE_2"; + + t.signal = nil; + end + + @task.start(); + sleep 0.5; + @task.stop(); + @task.join(); + + assert File.exist?("/tmp/TEST_FILE_1"); + assert File.exist?("/tmp/TEST_FILE_2"); + end + + def test_chdir + @runner.current_task_file = "/tmp/.gittask"; + + @task = GitRestart::Task.new() do |t| + t.name = "TestTask"; + + t.targets << "touch TEST_FILE_1"; + end + + @task.start(); + @task.join(); + + assert File.exist?("/tmp/TEST_FILE_1"); + end + + def test_status_msg() + @task = GitRestart::Task.new do |t| + t.name = "TestTask" + t.ci_task = true; + + t.targets << 'echo "Status A!"' + end + + @task.start(); + @task.join(); + + refute File.exist? "/tmp/TaskLog_TestTask_#{@runner.current_commit()}" + assert_equal "Status A!", @task.status_message + + @task = GitRestart::Task.new do |t| + t.name = "TestTask2" + t.ci_task = true; + + t.targets << 'echo "Status A!" && exit 1' + end + + @task.start(); + @task.join(); + + assert File.exist? "/tmp/TaskLog_TestTask2_#{@runner.current_commit()}" + assert_equal "Status A!", @task.status_message + end + + def test_triggers() + @runner.current_branch = "master"; + @runner.current_modified = ["Tests/Test1/Test.rb"]; + + @runner.current_task_file = "Tests/Test1/.gittask"; + + @task = GitRestart::Task.new() do |t| + t.name = "TestTask" + + t.watch(%r{.*\.rb}); + assert t.triggered? + + t.watch("Test.rb"); + assert t.triggered? + + @runner.current_modified = ["Tests/Test1/NotTested.txt"]; + refute t.triggered? + + @runner.current_modified = ["OutOfCHDir/Test.rb"]; + refute t.triggered? + + t.watch(%r{/OutOfCHDir/.*}); + assert t.triggered? + + t.on_branches ["master", "dev"]; + assert t.active; + t.active = false; + + t.on_branches "dev"; + refute t.active; + + @runner.current_modified = ["Tests/Test1/.gittask"]; + assert t.triggered? + + @runner.current_modified = nil; + assert t.triggered? + end + + end +end diff --git a/Ruby/MQTT/.gittask b/Ruby/MQTT/.gittask new file mode 100644 index 00000000..54a6b1d5 --- /dev/null +++ b/Ruby/MQTT/.gittask @@ -0,0 +1,11 @@ + +GitRestart::Task.new do |t| + t.ci_task = true; + t.name = "MQTT CI" + + t.report_status = true; + + t.active = true; + + t.targets << "ruby tests/test_all.rb" +end diff --git a/Ruby/MQTT/tests/test_all.rb b/Ruby/MQTT/tests/test_all.rb new file mode 100644 index 00000000..cc5eeefb --- /dev/null +++ b/Ruby/MQTT/tests/test_all.rb @@ -0,0 +1,4 @@ + +Dir.chdir(File.dirname(__FILE__)) { + Dir["*.rb"].each { |f| require_relative f } +}