Merge pull request #468 from dsander/job-management

Simple job management page
This commit is contained in:
Dominik Sander 2014-09-07 14:16:50 +02:00
commit 3fa68bd1b3
22 changed files with 467 additions and 95 deletions

View file

@ -124,4 +124,7 @@ ENABLE_INSECURE_AGENTS=false
#USE_GRAPHVIZ_DOT=dot
# Timezone. Use `rake time:zones:local` or `rake time:zones:all` to get your zone name
TIMEZONE="Pacific Time (US & Canada)"
TIMEZONE="Pacific Time (US & Canada)"
# Number of failed jobs to keep in the database
FAILED_JOBS_TO_KEEP=100

View file

@ -45,9 +45,6 @@ gem 'delayed_job', '~> 4.0.0'
gem 'delayed_job_active_record', '~> 4.0.0'
gem 'daemons', '~> 1.1.9'
# To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
# gem 'delayed_job_web'
gem 'foreman', '~> 0.63.0'
gem 'sass-rails', '~> 4.0.0'

View file

@ -85,12 +85,6 @@ See [private development instructions](https://github.com/cantino/huginn/wiki/Pr
In order to use the WeatherAgent you need an [API key with Wunderground](http://www.wunderground.com/weather/api/). Signup for one and then change the value of `api_key: your-key` in your seeded WeatherAgent.
#### Enable DelayedJobWeb for handy delayed\_job monitoring and control
* Edit `config.ru`, uncomment the DelayedJobWeb section, and change the DelayedJobWeb username and password.
* Uncomment `match "/delayed_job" => DelayedJobWeb, :anchor => false` in `config/routes.rb`.
* Uncomment `gem "delayed_job_web"` in Gemfile and run `bundle`.
#### Disable SSL
We assume your deployment will run over SSL. This is a very good idea! However, if you wish to turn this off, you'll probably need to edit `config/initializers/devise.rb` and modify the line containing `config.rememberable_options = { :secure => true }`. You will also need to edit `config/environments/production.rb` and modify the value of `config.force_ssl`.

View file

@ -1,22 +1,29 @@
$ ->
firstEventCount = null
previousJobs = null
if $("#job-indicator").length
if $(".job-indicator").length
check = ->
$.getJSON "/worker_status", (json) ->
for method in ['pending', 'awaiting_retry', 'recent_failures']
count = json[method]
elem = $(".job-indicator[role=#{method}]")
if count > 0
tooltipOptions = {
title: "#{count} jobs #{method.split('_').join(' ')}"
delay: 0
placement: "bottom"
trigger: "hover"
}
if elem.is(":visible")
elem.tooltip('destroy').tooltip(tooltipOptions).find(".number").text(count)
else
elem.tooltip('destroy').tooltip(tooltipOptions).fadeIn().find(".number").text(count)
else
if elem.is(":visible")
elem.tooltip('destroy').fadeOut()
firstEventCount = json.event_count unless firstEventCount?
if json.pending? && json.pending > 0
tooltipOptions = {
title: "#{json.pending} jobs pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
delay: 0
placement: "bottom"
trigger: "hover"
}
$("#job-indicator").tooltip('destroy').tooltip(tooltipOptions).fadeIn().find(".number").text(json.pending)
else
$("#job-indicator:visible").tooltip('destroy').fadeOut()
if firstEventCount? && json.event_count > firstEventCount
$("#event-indicator").tooltip('destroy').
tooltip(title: "Click to reload", delay: 0, placement: "bottom", trigger: "hover").
@ -26,6 +33,12 @@ $ ->
else
$("#event-indicator").tooltip('destroy').fadeOut()
currentJobs = [json.pending, json.awaiting_retry, json.recent_failures]
if document.location.pathname == '/jobs' && $(".modal[aria-hidden=false]").length == 0 && previousJobs? && previousJobs.join(',') != currentJobs.join(',')
$.get '/jobs', (data) =>
$("#main-content").html(data)
previousJobs = currentJobs
window.workerCheckTimeout = setTimeout check, 2000
check()

View file

@ -88,9 +88,10 @@ span.not-applicable:after {
}
// Navbar
#job-indicator, #event-indicator {
display: none;
.nav > li {
&.job-indicator, &#event-indicator {
display: none;
}
}
.navbar-search > .spinner {

View file

@ -0,0 +1,3 @@
.big-modal-dialog {
width: 90% !important;
}

View file

@ -14,6 +14,10 @@ class ApplicationController < ActionController::Base
devise_parameter_sanitizer.for(:account_update) { |u| u.permit(:username, :email, :password, :password_confirmation, :current_password) }
end
def authenticate_admin!
redirect_to(root_path, alert: 'Admin access required to view that page.') unless current_user && current_user.admin
end
def upgrade_warning
return unless current_user
twitter_oauth_check

View file

@ -0,0 +1,55 @@
class JobsController < ApplicationController
before_filter :authenticate_admin!
def index
@jobs = Delayed::Job.order("coalesce(failed_at,'1000-01-01'), run_at asc").page(params[:page])
respond_to do |format|
format.html { render layout: !request.xhr? }
format.json { render json: @jobs }
end
end
def destroy
@job = Delayed::Job.find(params[:id])
respond_to do |format|
if !running? && @job.destroy
format.html { redirect_to jobs_path, notice: "Job deleted." }
format.json { render json: "", status: :ok }
else
format.html { redirect_to jobs_path, alert: 'Can not delete a running job.' }
format.json { render json: "", status: :unprocessable_entity }
end
end
end
def run
@job = Delayed::Job.find(params[:id])
@job.last_error = nil
respond_to do |format|
if !running? && @job.update_attributes!(run_at: Time.now, failed_at: nil)
format.html { redirect_to jobs_path, notice: "Job enqueued." }
format.json { render json: @job, status: :ok }
else
format.html { redirect_to jobs_path, alert: 'Can not enqueue a running job.' }
format.json { render json: "", status: :unprocessable_entity }
end
end
end
def destroy_failed
Delayed::Job.where.not(failed_at: nil).delete_all
respond_to do |format|
format.html { redirect_to jobs_path, notice: "Failed jobs removed." }
format.json { render json: '', status: :ok }
end
end
private
def running?
(@job.locked_at || @job.locked_by) && @job.failed_at.nil?
end
end

View file

@ -38,4 +38,8 @@ module ApplicationHelper
link_to 'No', agent_path(agent, tab: (agent.recent_error_logs? ? 'logs' : 'details')), class: 'label label-danger'
end
end
def user_is_admin?
current_user && current_user.admin == true
end
end

View file

@ -0,0 +1,21 @@
module JobsHelper
def status(job)
case
when job.failed_at
content_tag :span, 'failed', class: 'label label-danger'
when job.locked_at && job.locked_by
content_tag :span, 'running', class: 'label label-info'
else
content_tag :span, 'queued', class: 'label label-warning'
end
end
def relative_distance_of_time_in_words(time)
if time < (now = Time.now)
time_ago_in_words(time) + ' ago'
else
'in ' + distance_of_time_in_words(time, now)
end
end
end

View file

@ -0,0 +1,75 @@
<div class='container'>
<div class='row'>
<div class='col-md-12'>
<div class="page-header">
<h2>
Background Jobs
</h2>
</div>
<div class='table-responsive'>
<table class='table table-striped events'>
<tr>
<th>Status</th>
<th>Created</th>
<th>Next Run</th>
<th>Attempts</th>
<th>Last Error</th>
<th></th>
</tr>
<% @jobs.each do |job| %>
<tr>
<td><%= status(job) %></td>
<td title='<%= job.created_at %>'><%= time_ago_in_words job.created_at %> ago</td>
<td title='<%= job.run_at %>'>
<% if !job.failed_at %>
<%= relative_distance_of_time_in_words job.run_at %>
<% end %>
</td>
<td><%= job.attempts %></td>
<td>
<a data-toggle="modal" data-target="#error<%= job.id %>"><%= truncate job.last_error, :length => 90, :omission => "", :separator => "\n" %></a>
<div class="modal fade" id="error<%= job.id %>" tabindex="-1" role="dialog" aria-labelledby="#<%= "error#{job.id}" %>" aria-hidden="true">
<div class="modal-dialog big-modal-dialog">
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
<h4 class="modal-title" id="myModalLabel">Error Backtrace</h4>
</div>
<div class="modal-body">
<pre>
<%= raw html_escape(job.last_error).split("\n").join('<br/>') %>
</pre>
</div>
</div>
</div>
</div>
</td>
<td>
<% if (!job.locked_at && !job.locked_by) || job.failed_at.present? %>
<div class="btn-group btn-group-xs" style="float: right">
<% if (job.run_at > Time.now) || job.failed_at.present? %>
<%= link_to 'Run now', run_job_path(job), class: "btn btn-default", method: :put %>
<% end %>
<%= link_to 'Delete', job_path(job), class: "btn btn-danger", method: :delete, data: { confirm: 'Really delete this job?' } %>
</div>
<% end %>
</td>
</tr>
<% end %>
</table>
</div>
<%= paginate @jobs, :theme => 'twitter-bootstrap-3' %>
<br />
<div class="btn-group">
<%= link_to destroy_failed_jobs_path, class: "btn btn-default", method: :delete do %>
<span class="glyphicon glyphicon-trash"></span> Remove failed jobs
<% end %>
</div>
</div>
</div>
</div>

View file

@ -35,15 +35,19 @@
</div>
</form>
<li id='job-indicator'>
<% if defined?(DelayedJobWeb) %>
<a href="/delayed_job">
<span class="badge"><span class="glyphicon glyphicon-refresh icon-white"></span> <span class='number'>0</span></span>
</a>
<% else %>
<a href="#" onclick='return false;'>
<span class="badge"><span class="glyphicon glyphicon-refresh icon-white"></span> <span class='number'>0</span></span>
</a>
<li class='job-indicator' role='pending'>
<%= link_to current_user.admin? ? jobs_path : '#' do %>
<span class="badge"><span class="glyphicon glyphicon-refresh icon-white"></span> <span class='number'>0</span></span>
<% end %>
</li>
<li class='job-indicator' role='awaiting_retry'>
<%= link_to current_user.admin? ? jobs_path : '#' do %>
<span class="badge"><span class="glyphicon glyphicon-question-sign icon-yellow"></span> <span class='number'>0</span></span>
<% end %>
</li>
<li class='job-indicator' role='recent_failures'>
<%= link_to current_user.admin? ? jobs_path : '#' do %>
<span class="badge"><span class="glyphicon glyphicon-exclamation-sign icon-white"></span> <span class='number'>0</span></span>
<% end %>
</li>
<li id='event-indicator'>
@ -66,7 +70,11 @@
<%= link_to 'Sign up', new_user_registration_path, :tabindex => "-1" %>
<% end %>
</li>
<% if user_signed_in? && current_user.admin %>
<li>
<%= link_to 'Job Management', jobs_path, :tabindex => '-1' %>
</li>
<% end %>
<li>
<%= link_to 'About', 'https://github.com/cantino/huginn', :tabindex => "-1" %>
</li>

View file

@ -28,7 +28,9 @@
<%= render "upgrade_warning" %>
<% end %>
<%= yield %>
<div id="main-content">
<%= yield %>
</div>
</div>

View file

@ -2,12 +2,4 @@
require ::File.expand_path('../config/environment', __FILE__)
# To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
# if Rails.env.production?
# DelayedJobWeb.use Rack::Auth::Basic do |username, password|
# username == 'admin' && password == 'password'
# end
# end
run Huginn::Application

View file

@ -1,4 +1,4 @@
Delayed::Worker.destroy_failed_jobs = true
Delayed::Worker.destroy_failed_jobs = false
Delayed::Worker.max_attempts = 5
Delayed::Worker.max_run_time = 20.minutes
Delayed::Worker.read_ahead = 5

View file

@ -51,6 +51,15 @@ Huginn::Application.routes.draw do
end
end
resources :jobs, :only => [:index, :destroy] do
member do
put :run
end
collection do
delete :destroy_failed
end
end
get "/worker_status" => "worker_status#show"
post "/users/:user_id/update_location/:secret" => "user_location_updates#create"
@ -58,9 +67,6 @@ Huginn::Application.routes.draw do
match "/users/:user_id/web_requests/:agent_id/:secret" => "web_requests#handle_request", :as => :web_requests, :via => [:get, :post, :put, :delete]
post "/users/:user_id/webhooks/:agent_id/:secret" => "web_requests#handle_request" # legacy
# To enable DelayedJobWeb, see the 'Enable DelayedJobWeb' section of the README.
# get "/delayed_job" => DelayedJobWeb, :anchor => false
devise_for :users, :sign_out_via => [ :post, :delete ]
get '/auth/:provider/callback', to: 'services#callback'

View file

@ -1,16 +1,54 @@
require 'rufus/scheduler'
class HuginnScheduler
FAILED_JOBS_TO_KEEP = 100
attr_accessor :mutex
def initialize
@rufus_scheduler = Rufus::Scheduler.new
self.mutex = Mutex.new
end
def stop
@rufus_scheduler.stop
end
def run!
tzinfo_friendly_timezone = ActiveSupport::TimeZone::MAPPING[ENV['TIMEZONE'].present? ? ENV['TIMEZONE'] : "Pacific Time (US & Canada)"]
# Schedule event propagation.
@rufus_scheduler.every '1m' do
propagate!
end
# Schedule event cleanup.
@rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do
cleanup_expired_events!
end
# Schedule failed job cleanup.
@rufus_scheduler.every '1h' do
cleanup_failed_jobs!
end
# Schedule repeating events.
%w[1m 2m 5m 10m 30m 1h 2h 5h 12h 1d 2d 7d].each do |schedule|
@rufus_scheduler.every schedule do
run_schedule "every_#{schedule}"
end
end
# Schedule events for specific times.
24.times do |hour|
@rufus_scheduler.cron "0 #{hour} * * * " + tzinfo_friendly_timezone do
run_schedule hour_to_schedule_name(hour)
end
end
@rufus_scheduler.join
end
private
def run_schedule(time)
with_mutex do
puts "Queuing schedule for #{time}"
@ -32,6 +70,24 @@ class HuginnScheduler
end
end
def cleanup_failed_jobs!
num_to_keep = (ENV['FAILED_JOBS_TO_KEEP'].presence || FAILED_JOBS_TO_KEEP).to_i
first_to_delete = Delayed::Job.where.not(failed_at: nil).order("failed_at DESC").offset(num_to_keep).limit(1).pluck(:failed_at).first
Delayed::Job.where(["failed_at <= ?", first_to_delete]).delete_all if first_to_delete.present?
end
def hour_to_schedule_name(hour)
if hour == 0
"midnight"
elsif hour < 12
"#{hour}am"
elsif hour == 12
"noon"
else
"#{hour - 12}pm"
end
end
def with_mutex
ActiveRecord::Base.connection_pool.with_connection do
mutex.synchronize do
@ -39,49 +95,4 @@ class HuginnScheduler
end
end
end
def run!
self.mutex = Mutex.new
tzinfo_friendly_timezone = ActiveSupport::TimeZone::MAPPING[ENV['TIMEZONE'].present? ? ENV['TIMEZONE'] : "Pacific Time (US & Canada)"]
# Schedule event propagation.
@rufus_scheduler.every '1m' do
propagate!
end
# Schedule event cleanup.
@rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do
cleanup_expired_events!
end
# Schedule repeating events.
%w[1m 2m 5m 10m 30m 1h 2h 5h 12h 1d 2d 7d].each do |schedule|
@rufus_scheduler.every schedule do
run_schedule "every_#{schedule}"
end
end
# Schedule events for specific times.
# Times are assumed to be in PST for now. Can store a user#timezone later.
24.times do |hour|
@rufus_scheduler.cron "0 #{hour} * * * " + tzinfo_friendly_timezone do
if hour == 0
run_schedule "midnight"
elsif hour < 12
run_schedule "#{hour}am"
elsif hour == 12
run_schedule "noon"
else
run_schedule "#{hour - 12}pm"
end
end
end
@rufus_scheduler.join
end
end

View file

@ -0,0 +1,72 @@
require 'spec_helper'
describe JobsController do
describe "GET index" do
before do
Delayed::Job.create!
Delayed::Job.create!
Delayed::Job.count.should > 0
end
it "does not allow normal users" do
sign_in users(:bob)
get(:index).should redirect_to(root_path)
end
it "returns all jobs", focus: true do
sign_in users(:jane)
get :index
assigns(:jobs).length.should == 2
end
end
describe "DELETE destroy" do
before do
@not_running = Delayed::Job.create
@running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test')
sign_in users(:jane)
end
it "destroy a job which is not running" do
expect { delete :destroy, id: @not_running.id }.to change(Delayed::Job, :count).by(-1)
end
it "does not destroy a running job" do
expect { delete :destroy, id: @running.id }.to change(Delayed::Job, :count).by(0)
end
end
describe "PUT run" do
before do
@not_running = Delayed::Job.create(run_at: Time.now - 1.hour)
@running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test')
@failed = Delayed::Job.create(run_at: Time.now - 1.hour, locked_at: Time.now, failed_at: Time.now)
sign_in users(:jane)
end
it "queue a job which is not running" do
expect { put :run, id: @not_running.id }.to change { @not_running.reload.run_at }
end
it "queue a job that failed" do
expect { put :run, id: @failed.id }.to change { @failed.reload.run_at }
end
it "not queue a running job" do
expect { put :run, id: @running.id }.not_to change { @not_running.reload.run_at }
end
end
describe "DELETE destroy_failed" do
before do
@failed = Delayed::Job.create(failed_at: Time.now - 1.minute)
@running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test')
sign_in users(:jane)
end
it "just destroy failed jobs" do
expect { delete :destroy_failed, id: @failed.id }.to change(Delayed::Job, :count).by(-1)
expect { delete :destroy_failed, id: @running.id }.to change(Delayed::Job, :count).by(0)
end
end
end

View file

@ -3,3 +3,4 @@ TWITTER_OAUTH_KEY=twitteroauthkey
TWITTER_OAUTH_SECRET=twitteroauthsecret
THIRTY_SEVEN_SIGNALS_OAUTH_KEY=TESTKEY
THIRTY_SEVEN_SIGNALS_OAUTH_SECRET=TESTSECRET
FAILED_JOBS_TO_KEEP=2

View file

@ -10,4 +10,5 @@ jane:
email: "jane@example.com"
username: jane
invitation_code: <%= User::INVITATION_CODES.last %>
scenario_count: 1
scenario_count: 1
admin: true

View file

@ -0,0 +1,32 @@
require 'spec_helper'
describe JobsHelper do
let(:job) { Delayed::Job.new }
describe '#status' do
it "works for failed jobs" do
job.failed_at = Time.now
status(job).should == '<span class="label label-danger">failed</span>'
end
it "works for running jobs" do
job.locked_at = Time.now
job.locked_by = 'test'
status(job).should == '<span class="label label-info">running</span>'
end
it "works for queued jobs" do
status(job).should == '<span class="label label-warning">queued</span>'
end
end
describe '#relative_distance_of_time_in_words' do
it "in the past" do
relative_distance_of_time_in_words(Time.now-5.minutes).should == '5m ago'
end
it "in the future" do
relative_distance_of_time_in_words(Time.now+5.minutes).should == 'in 5m'
end
end
end

View file

@ -0,0 +1,77 @@
require 'spec_helper'
describe HuginnScheduler do
before(:each) do
@scheduler = HuginnScheduler.new
stub
end
it "should stop the scheduler" do
mock.instance_of(Rufus::Scheduler).stop
@scheduler.stop
end
it "schould register the schedules with the rufus scheduler and run" do
mock.instance_of(Rufus::Scheduler).join
@scheduler.run!
end
it "should run scheduled agents" do
mock(Agent).run_schedule('every_1h')
mock.instance_of(IO).puts('Queuing schedule for every_1h')
@scheduler.send(:run_schedule, 'every_1h')
end
it "should propagate events" do
mock(Agent).receive!
stub.instance_of(IO).puts
@scheduler.send(:propagate!)
end
it "schould clean up expired events" do
mock(Event).cleanup_expired!
stub.instance_of(IO).puts
@scheduler.send(:cleanup_expired_events!)
end
describe "#hour_to_schedule_name" do
it "for 0h" do
@scheduler.send(:hour_to_schedule_name, 0).should == 'midnight'
end
it "for the forenoon" do
@scheduler.send(:hour_to_schedule_name, 6).should == '6am'
end
it "for 12h" do
@scheduler.send(:hour_to_schedule_name, 12).should == 'noon'
end
it "for the afternoon" do
@scheduler.send(:hour_to_schedule_name, 17).should == '5pm'
end
end
describe "cleanup_failed_jobs!" do
before do
3.times do |i|
Delayed::Job.create(failed_at: Time.now - i.minutes)
end
@keep = Delayed::Job.order(:failed_at)[1]
end
it "work with set FAILED_JOBS_TO_KEEP env variable", focus: true do
expect { @scheduler.send(:cleanup_failed_jobs!) }.to change(Delayed::Job, :count).by(-1)
expect { @scheduler.send(:cleanup_failed_jobs!) }.to change(Delayed::Job, :count).by(0)
@keep.id.should == Delayed::Job.order(:failed_at)[0].id
end
it "work without the FAILED_JOBS_TO_KEEP env variable" do
old = ENV['FAILED_JOBS_TO_KEEP']
ENV['FAILED_JOBS_TO_KEEP'] = nil
expect { @scheduler.send(:cleanup_failed_jobs!) }.to change(Delayed::Job, :count).by(0)
ENV['FAILED_JOBS_TO_KEEP'] = old
end
end
end