diff --git a/Gemfile b/Gemfile index 824bd11..bc60558 100644 --- a/Gemfile +++ b/Gemfile @@ -47,11 +47,6 @@ gem 'hirb' # gem 'hirb-unicode' # incompatible with new rubocop gem 'sequel', '~> 5.0' -# Asynchronicity -gem 'aws-sdk-sqs', '~> 1.0' -gem 'concurrent-ruby', '~> 1.0' -gem 'shoryuken', '~> 6.0' - group :development, :test do gem 'sqlite3', '~> 1.0' end @@ -60,6 +55,14 @@ group :production do gem 'pg', '~> 1.0' end +# Asynchronicity +gem 'aws-sdk-sqs', '~> 1.0' +gem 'concurrent-ruby', '~> 1.0' + +# WORKER +gem 'faye', '~> 1.0' +gem 'shoryuken', '~> 6.0' + # TESTING group :test do # API Unit/Integration/Acceptance Tests diff --git a/Gemfile.lock b/Gemfile.lock index aebf117..55494e4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -23,6 +23,7 @@ GEM bigdecimal (3.3.1) coderay (1.1.3) concurrent-ruby (1.3.5) + cookiejar (0.3.4) crack (1.0.1) bigdecimal rexml @@ -83,6 +84,27 @@ GEM dry-initializer (~> 3.2) dry-schema (~> 1.14) zeitwerk (~> 2.6) + em-http-request (1.1.7) + addressable (>= 2.3.4) + cookiejar (!= 0.3.1) + em-socksify (>= 0.3) + eventmachine (>= 1.0.3) + http_parser.rb (>= 0.6.0) + em-socksify (0.3.3) + base64 + eventmachine (>= 1.0.0.beta.4) + eventmachine (1.2.7) + faye (1.4.1) + cookiejar (>= 0.3.0) + em-http-request (>= 1.1.6) + eventmachine (>= 0.12.0) + faye-websocket (>= 0.11.0) + multi_json (>= 1.0.0) + rack (>= 1.0.0) + websocket-driver (>= 0.5.1) + faye-websocket (0.12.0) + eventmachine (>= 0.12.0) + websocket-driver (>= 0.8.0) ffi (1.17.2) ffi (1.17.2-arm64-darwin) ffi (1.17.2-x86_64-linux-gnu) @@ -105,6 +127,7 @@ GEM http-cookie (1.1.0) domain_name (~> 0.5) http-form_data (2.3.0) + http_parser.rb (0.8.0) ice_nine (0.11.2) jmespath (1.6.2) json (2.17.1) @@ -233,6 +256,10 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) + websocket-driver (0.8.0) + base64 + websocket-extensions (>= 0.1.0) + websocket-extensions (0.1.5) zeitwerk (2.7.3) PLATFORMS @@ -249,6 +276,7 @@ DEPENDENCIES dry-transaction (~> 0) dry-types (~> 1.0) dry-validation (~> 1.0) + faye (~> 1.0) figaro (~> 1.0) flog hirb diff --git a/Rakefile b/Rakefile index 6f4f61c..e87656a 100644 --- a/Rakefile +++ b/Rakefile @@ -9,7 +9,6 @@ end desc 'Run unit and integration tests' Rake::TestTask.new(:spec) do |t| - puts 'Make sure worker is running in separate process' t.pattern = 'spec/tests/**/*_spec.rb' t.warning = false end diff --git a/app/application/controllers/app.rb b/app/application/controllers/app.rb index fe2e01f..90f1862 100644 --- a/app/application/controllers/app.rb +++ b/app/application/controllers/app.rb @@ -36,11 +36,17 @@ class App < Roda response.cache_control public: true, max_age: 300 end + request_id = [request.env, request.path, Time.now.to_f].hash + path_request = Request::ProjectPath.new( owner_name, project_name, request ) - result = Service::AppraiseProject.new.call(requested: path_request) + result = Service::AppraiseProject.new.call( + requested: path_request, + request_id: request_id, + config: App.config + ) if result.failure? failed = Representer::HttpResponse.new(result.failure) diff --git a/app/application/services/appraise_project.rb b/app/application/services/appraise_project.rb index 9dfc583..2582504 100644 --- a/app/application/services/appraise_project.rb +++ b/app/application/services/appraise_project.rb @@ -21,9 +21,10 @@ class AppraiseProject CLONE_ERR = 'Could not clone this project' TOO_LARGE_ERR = 'Project is too large to analyze' NO_FOLDER_ERR = 'Could not find that folder' - PROCESSING_MSG = 'Processing the appraisal request; please check back later' + PROCESSING_MSG = 'Processing the appraisal request' # rubocop:enable Lint/UselessConstantScoping + # input hash keys expected: :project, :requested, :config def find_project_details(input) input[:project] = Repository::For.klass(Entity::Project).find_full_name( input[:requested].owner_name, input[:requested].project_name @@ -42,7 +43,7 @@ def check_project_eligibility(input) if input[:project].too_large? Failure(Response::ApiResult.new(status: :forbidden, message: TOO_LARGE_ERR)) else - input[:gitrepo] = GitRepo.new(input[:project]) + input[:gitrepo] = GitRepo.new(input[:project], input[:config]) Success(input) end end @@ -50,11 +51,13 @@ def check_project_eligibility(input) def request_cloning_worker(input) return Success(input) if input[:gitrepo].exists_locally? - Messaging::Queue - .new(App.config.CLONE_QUEUE_URL, App.config) - .send(Representer::Project.new(input[:project]).to_json) + Messaging::Queue.new(App.config.CLONE_QUEUE_URL, App.config) + .send(clone_request_json(input)) - Failure(Response::ApiResult.new(status: :processing, message: PROCESSING_MSG)) + Failure(Response::ApiResult.new( + status: :processing, + message: { request_id: input[:request_id], msg: PROCESSING_MSG } + )) rescue StandardError => e log_error(e) Failure(Response::ApiResult.new(status: :internal_error, message: CLONE_ERR)) @@ -82,6 +85,12 @@ def full_request_path(input) def log_error(error) App.logger.error [error.inspect, error.backtrace].flatten.join("\n") end + + def clone_request_json(input) + Response::CloneRequest.new(input[:project], input[:request_id]) + .then { Representer::CloneRequest.new(_1) } + .then(&:to_json) + end end end end diff --git a/app/infrastructure/git/repositories/git_repo.rb b/app/infrastructure/git/repositories/git_repo.rb index 6406d3e..c9c0210 100644 --- a/app/infrastructure/git/repositories/git_repo.rb +++ b/app/infrastructure/git/repositories/git_repo.rb @@ -9,10 +9,10 @@ class Errors CannotOverwriteLocalGitRepo = Class.new(StandardError) end - def initialize(project, config = CodePraise::App.config) + def initialize(project, config) @project = project remote = Git::RemoteGitRepo.new(@project.http_url) - @local = Git::LocalGitRepo.new(remote) + @local = Git::LocalGitRepo.new(remote, config.REPOSTORE_PATH) end def local @@ -27,7 +27,7 @@ def exists_locally? @local.exists? end - def clone + def clone_locally raise Errors::TooLargeToClone if @project.too_large? raise Errors::CannotOverwriteLocalGitRepo if exists_locally? diff --git a/app/infrastructure/git/repositories/local_repo.rb b/app/infrastructure/git/repositories/local_repo.rb index 48af88a..fe1fb58 100644 --- a/app/infrastructure/git/repositories/local_repo.rb +++ b/app/infrastructure/git/repositories/local_repo.rb @@ -16,9 +16,9 @@ class LocalGitRepo attr_reader :git_repo_path - def initialize(remote) + def initialize(remote, repostore_path) @remote = remote - @git_repo_path = [ENV['REPOSTORE_PATH'], @remote.unique_id].join('/') + @git_repo_path = [repostore_path, @remote.unique_id].join('/') end def clone_remote diff --git a/app/presentation/representers/clone_request_representer.rb b/app/presentation/representers/clone_request_representer.rb new file mode 100644 index 0000000..0055626 --- /dev/null +++ b/app/presentation/representers/clone_request_representer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'roar/decorator' +require 'roar/json' +require_relative 'project_representer' + +# Represents essential Repo information for API output +module CodePraise + module Representer + # Representer object for project clone requests + class CloneRequest < Roar::Decorator + include Roar::JSON + + property :project, extend: Representer::Project, class: OpenStruct + property :id + end + end +end diff --git a/app/presentation/representers/project_representer.rb b/app/presentation/representers/project_representer.rb index 97d00d0..5c0428f 100644 --- a/app/presentation/representers/project_representer.rb +++ b/app/presentation/representers/project_representer.rb @@ -25,7 +25,7 @@ class Project < Roar::Decorator collection :contributors, extend: Representer::Member, class: OpenStruct link :self do - "#{App.config.API_HOST}/projects/#{project_name}/#{owner_name}" + "#{ENV.fetch('API_HOST')}/api/v1/projects/#{project_name}/#{owner_name}" end private diff --git a/app/presentation/responses/clone_request.rb b/app/presentation/responses/clone_request.rb new file mode 100644 index 0000000..a61b8a3 --- /dev/null +++ b/app/presentation/responses/clone_request.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module CodePraise + module Response + CloneRequest = Struct.new :project, :id + end +end diff --git a/config.ru b/config.ru index d970161..51c852d 100644 --- a/config.ru +++ b/config.ru @@ -1,6 +1,8 @@ # frozen_string_literal: true +require 'faye' require_relative 'require_app' require_app +use Faye::RackAdapter, mount: '/faye', timeout: 25 run CodePraise::App.freeze.app diff --git a/config/environment.rb b/config/environment.rb index f959df0..1a3ee2b 100644 --- a/config/environment.rb +++ b/config/environment.rb @@ -34,6 +34,9 @@ def self.config = Figaro.env end configure :production do + puts 'RUNNING IN PRODUCTION MODE' + # Set DATABASE_URL environment variable on production platform + use Rack::Cache, verbose: true, metastore: "#{config.REDISCLOUD_URL}/0/metastore", diff --git a/config/secrets_example.yml b/config/secrets_example.yml index 2b3725a..c80db59 100644 --- a/config/secrets_example.yml +++ b/config/secrets_example.yml @@ -11,8 +11,10 @@ development: AWS_ACCESS_KEY_ID: AWS_SECRET_ACCESS_KEY: AWS_REGION: - CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE: soa-codepraise-clone-dev CLONE_QUEUE_URL: + REPORT_QUEUE: codepraise-report-development + REPORT_QUEUE_URL: AWS_SECRET_ACCESS_KEY: AWS_REGION: - CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE: codepraise-clone-test CLONE_QUEUE_URL: + REPORT_QUEUE: codepraise-report-test + REPORT_QUEUE_URL: AWS_SECRET_ACCESS_KEY: AWS_REGION: - CLONE_QUEUE: codepraise-clone-development + CLONE_QUEUE: codepraise-clone-test CLONE_QUEUE_URL: + REPORT_QUEUE: codepraise-report-test + REPORT_QUEUE_URL: 15, + 'Cloning' => 30, + 'remote' => 70, + 'Receiving' => 85, + 'Resolving' => 95, + 'Checking' => 100, + 'FINISHED' => 100 + }.freeze + + def self.starting_percent + CLONE_PROGRESS['STARTED'].to_s + end + + def self.finished_percent + CLONE_PROGRESS['FINISHED'].to_s + end + + def self.progress(line) + CLONE_PROGRESS[first_word_of(line)].to_s + end + + def self.percent(stage) + CLONE_PROGRESS[stage].to_s + end + + def self.first_word_of(line) + line.match(/^[A-Za-z]+/).to_s + end + end +end diff --git a/workers/git_clone_worker.rb b/workers/git_clone_worker.rb index bd895e5..e63270d 100644 --- a/workers/git_clone_worker.rb +++ b/workers/git_clone_worker.rb @@ -1,36 +1,48 @@ # frozen_string_literal: true require_relative '../require_app' +require_relative 'clone_monitor' +require_relative 'job_reporter' require_app require 'figaro' require 'shoryuken' -# Shoryuken worker class to clone repos in parallel -class GitCloneWorker - # Environment variables setup - Figaro.application = Figaro::Application.new( - environment: ENV['RACK_ENV'] || 'development', - path: File.expand_path('config/secrets.yml') - ) - Figaro.load - def self.config = Figaro.env - - Shoryuken.sqs_client = Aws::SQS::Client.new( - access_key_id: config.AWS_ACCESS_KEY_ID, - secret_access_key: config.AWS_SECRET_ACCESS_KEY, - region: config.AWS_REGION - ) - - include Shoryuken::Worker - - shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true - - def perform(_sqs_msg, request) - project = CodePraise::Representer::Project - .new(OpenStruct.new).from_json(request) - CodePraise::GitRepo.new(project).clone - rescue CodePraise::GitRepo::Errors::CannotOverwriteLocalGitRepo - puts 'CLONE EXISTS -- ignoring request' +module GitClone + # Shoryuken worker class to clone repos in parallel + class Worker + # Environment variables setup + Figaro.application = Figaro::Application.new( + environment: ENV['RACK_ENV'] || 'development', + path: File.expand_path('config/secrets.yml') + ) + Figaro.load + def self.config = Figaro.env + + Shoryuken.sqs_client = Aws::SQS::Client.new( + access_key_id: config.AWS_ACCESS_KEY_ID, + secret_access_key: config.AWS_SECRET_ACCESS_KEY, + region: config.AWS_REGION + ) + + include Shoryuken::Worker + + Shoryuken.sqs_client_receive_message_opts = { wait_time_seconds: 20 } + shoryuken_options queue: config.CLONE_QUEUE_URL, auto_delete: true + + def perform(_sqs_msg, request) + job = JobReporter.new(request, Worker.config) + + job.report(CloneMonitor.starting_percent) + CodePraise::GitRepo.new(job.project, Worker.config).clone_locally do |line| + job.report CloneMonitor.progress(line) + end + + # Keep sending finished status to any latecoming subscribers + job.report_each_second(5) { CloneMonitor.finished_percent } + rescue CodePraise::GitRepo::Errors::CannotOverwriteLocalGitRepo + # worker should crash fail early - only catch errors we expect! + puts 'CLONE EXISTS -- ignoring request' + end end end diff --git a/workers/job_reporter.rb b/workers/job_reporter.rb new file mode 100644 index 0000000..0398582 --- /dev/null +++ b/workers/job_reporter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative 'progress_publisher' + +module GitClone + # Reports job progress to client + class JobReporter + attr_accessor :project + + def initialize(request_json, config) + clone_request = CodePraise::Representer::CloneRequest + .new(OpenStruct.new) + .from_json(request_json) + + @project = clone_request.project + @publisher = ProgressPublisher.new(config, clone_request.id) + end + + def report(msg) + @publisher.publish msg + end + + def report_each_second(seconds, &operation) + seconds.times do + sleep(1) + report(operation.call) + end + end + end +end diff --git a/workers/progress_publisher.rb b/workers/progress_publisher.rb new file mode 100644 index 0000000..5dd2d5a --- /dev/null +++ b/workers/progress_publisher.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'http' + +module GitClone + # Publishes progress as percent to Faye endpoint + class ProgressPublisher + def initialize(config, channel_id) + @config = config + @channel_id = channel_id + end + + def publish(message) + print "Progress: #{message} " + print "[post: #{@config.API_HOST}/faye] " + response = HTTP.headers(content_type: 'application/json') + .post( + "#{@config.API_HOST}/faye", + body: message_body(message) + ) + puts "(#{response.status})" + rescue HTTP::ConnectionError + puts '(Faye server not found - progress not sent)' + end + + private + + def message_body(message) + { channel: "/#{@channel_id}", + data: message }.to_json + end + end +end