add Agent Logs; add logging to WebsiteAgent; refactor flash notices and add event notices

This commit is contained in:
Andrew Cantino 2013-08-24 22:46:06 -06:00
parent 3efaed7e2a
commit 00727fbd4d
28 changed files with 421 additions and 42 deletions

View file

@ -40,3 +40,6 @@ EMAIL_FROM_ADDRESS=from_address@gmail.com
# This invitation code will be required for users to signup with your Huginn installation.
# You can see its use in user.rb.
INVITATION_CODE=try-huginn
# Number of lines of log messages to keep per Agent
AGENT_LOG_LENGTH=100

View file

@ -8,10 +8,6 @@
#= require ./worker-checker
#= require_self
# Place all the behaviors and hooks related to the matching controller here.
# All this logic will automatically be available in application.js.
# You can use CoffeeScript in this file: http://jashkenas.github.com/coffee-script/
setupJsonEditor = ->
JSONEditor.prototype.ADD_IMG = '<%= image_path 'json-editor/add.png' %>'
JSONEditor.prototype.DELETE_IMG = '<%= image_path 'json-editor/delete.png' %>'
@ -48,12 +44,43 @@ showEventDescriptions = ->
$(".event-descriptions").html("").hide()
$(document).ready ->
# JSON Editor
setupJsonEditor()
# Select2 Selects
$(".select2").select2(width: 'resolve')
if $(".top-flash").length
setTimeout((-> $(".top-flash").slideUp(-> $(".top-flash").remove())), 5000)
# Flash
if $(".flash").length
setTimeout((-> $(".flash").slideUp(-> $(".flash").remove())), 5000)
# Agent Show
fetchLogs = (e) ->
agentId = $(e.target).closest("[data-agent-id]").data("agent-id")
e.preventDefault()
$("#logs .spinner").show()
$("#logs .refresh, #logs .clear").hide()
$.get "/agents/#{agentId}/logs", (html) =>
$("#logs .logs").html html
$("#logs .spinner").stop(true, true).fadeOut ->
$("#logs .refresh, #logs .clear").show()
clearLogs = (e) ->
if confirm("Are you sure you want to clear all logs for this Agent?")
agentId = $(e.target).closest("[data-agent-id]").data("agent-id")
e.preventDefault()
$("#logs .spinner").show()
$("#logs .refresh, #logs .clear").hide()
$.post "/agents/#{agentId}/logs/clear", { "_method": "DELETE" }, (html) =>
$("#logs .logs").html html
$("#logs .spinner").stop(true, true).fadeOut ->
$("#logs .refresh, #logs .clear").show()
$(".agent-show #show-tabs a[href='#logs']").on "click", fetchLogs
$("#logs .refresh").on "click", fetchLogs
$("#logs .clear").on "click", clearLogs
# Editing Agents
$("#agent_source_ids").on "change", showEventDescriptions
$("#agent_type").on "change", ->

View file

@ -1,7 +1,11 @@
$ ->
firstEventCount = null
if $("#job-indicator").length
check = ->
$.getJSON "/worker_status", (json) ->
firstEventCount = json.event_count unless firstEventCount?
if json.pending? && json.pending > 0
tooltipOptions = {
title: "#{json.pending} pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
@ -12,5 +16,20 @@ $ ->
$("#job-indicator").tooltip('destroy').tooltip(tooltipOptions).fadeIn().find(".number").text(json.pending)
else
$("#job-indicator:visible").tooltip('destroy').fadeOut()
if firstEventCount? && json.event_count > firstEventCount
$("#event-indicator").tooltip('destroy').
tooltip(title: "Click to reload", delay: 0, placement: "bottom", trigger: "hover").
fadeIn().
find(".number").
text(json.event_count - firstEventCount)
else
$("#event-indicator").tooltip('destroy').fadeOut()
window.workerCheckTimeout = setTimeout check, 2000
check()
$("#event-indicator a").on "click", (e) ->
e.preventDefault()
window.location.reload()

View file

@ -51,7 +51,7 @@ table.events {
margin-left: 0 !important;
}
#job-indicator {
#job-indicator, #event-indicator {
display: none;
}
@ -85,3 +85,28 @@ img.spinner {
.show-view {
overflow: hidden;
}
// Flash
.flash {
position: fixed;
width: 500px;
z-index: 99999;
top: 1px;
margin-left: 250px;
.alert {
}
}
// Logs
#logs .action-icon {
height: 16px;
display: inline-block;
vertical-align: inherit;
&.refresh {
margin: 0 10px;
}
}

View file

@ -9,8 +9,13 @@ class AgentsController < ApplicationController
end
def run
Agent.async_check(current_user.agents.find(params[:id]).id)
redirect_to agents_path, notice: "Agent run queued"
agent = current_user.agents.find(params[:id])
Agent.async_check(agent.id)
if params[:return] == "show"
redirect_to agent_path(agent), notice: "Agent run queued"
else
redirect_to agents_path, notice: "Agent run queued"
end
end
def type_details

View file

@ -0,0 +1,19 @@
class LogsController < ApplicationController
before_filter :load_agent
def index
@logs = @agent.logs.all
render :action => :index, :layout => false
end
def clear
@agent.logs.delete_all
index
end
protected
def load_agent
@agent = current_user.agents.find(params[:agent_id])
end
end

View file

@ -1,12 +1,11 @@
class WorkerStatusController < ApplicationController
skip_before_filter :authenticate_user!
def show
start = Time.now.to_f
render :json => {
:pending => Delayed::Job.where("run_at <= ? AND locked_at IS NULL AND attempts = 0", Time.now).count,
:awaiting_retry => Delayed::Job.where("failed_at IS NULL AND attempts > 0").count,
:recent_failures => Delayed::Job.where("failed_at IS NOT NULL AND failed_at > ?", 5.days.ago).count,
:event_count => current_user.events.count,
:compute_time => Time.now.to_f - start
}
end

View file

@ -0,0 +1,2 @@
module LogsHelper
end

View file

@ -30,6 +30,7 @@ class Agent < ActiveRecord::Base
belongs_to :user, :inverse_of => :agents
has_many :events, :dependent => :delete_all, :inverse_of => :agent, :order => "events.id desc"
has_many :logs, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog", :order => "agent_logs.id desc"
has_many :received_events, :through => :sources, :class_name => "Event", :source => :events, :order => "events.id desc"
has_many :links_as_source, :dependent => :delete_all, :foreign_key => "source_id", :class_name => "Link", :inverse_of => :source
has_many :links_as_receiver, :dependent => :delete_all, :foreign_key => "receiver_id", :class_name => "Link", :inverse_of => :receiver
@ -139,6 +140,10 @@ class Agent < ActiveRecord::Base
end
end
def log(message, options = {})
AgentLog.log_for_agent(self, message, options)
end
# Class Methods
class << self
def cannot_be_scheduled!

23
app/models/agent_log.rb Normal file
View file

@ -0,0 +1,23 @@
class AgentLog < ActiveRecord::Base
attr_accessible :agent, :inbound_event, :level, :message, :outbound_event
belongs_to :agent
belongs_to :inbound_event, :class_name => "Event"
belongs_to :outbound_event, :class_name => "Event"
validates_presence_of :agent, :message
validates_numericality_of :level, :only_integer => true, :greater_than_or_equal_to => 0, :less_than => 5
def self.log_for_agent(agent, message, options = {})
log = agent.logs.create! options.merge(:message => message)
if agent.logs.count > log_length
oldest_id_to_keep = agent.logs.limit(1).offset(log_length - 1).pluck("agent_logs.id")
agent.logs.where("agent_logs.id < ?", oldest_id_to_keep).delete_all
end
log
end
def self.log_length
ENV['AGENT_LOG_LENGTH'].present? ? ENV['AGENT_LOG_LENGTH'].to_i : 100
end
end

View file

@ -66,29 +66,38 @@ module Agents
def check
hydra = Typhoeus::Hydra.new
log "Fetching #{options[:url]}"
request = Typhoeus::Request.new(options[:url], :followlocation => true)
request.on_complete do |response|
request.on_failure do |response|
log "Failed: #{response.inspect}"
end
request.on_success do |response|
doc = parse(response.body)
output = {}
options[:extract].each do |name, extraction_details|
if extraction_type == "json"
output[name] = Utils.values_at(doc, extraction_details[:path])
else
output[name] = doc.css(extraction_details[:css]).map { |node|
if extraction_details[:attr]
node.attr(extraction_details[:attr])
elsif extraction_details[:text]
node.text()
else
raise StandardError, ":attr or :text is required on HTML or XML extraction patterns"
end
}
end
result = if extraction_type == "json"
output[name] = Utils.values_at(doc, extraction_details[:path])
else
output[name] = doc.css(extraction_details[:css]).map { |node|
if extraction_details[:attr]
node.attr(extraction_details[:attr])
elsif extraction_details[:text]
node.text()
else
log ":attr or :text is required on HTML or XML extraction patterns"
return
end
}
end
log "Extracting #{extraction_type} at #{extraction_details[:path] || extraction_details[:css]}: #{result}"
end
num_unique_lengths = options[:extract].keys.map { |name| output[name].length }.uniq
raise StandardError, "Got an uneven number of matches for #{options[:name]}: #{options[:extract].inspect}" unless num_unique_lengths.length == 1
if num_unique_lengths.length != 1
log "Got an uneven number of matches for #{options[:name]}: #{options[:extract].inspect}", :level => 4
return
end
previous_payloads = events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change"
num_unique_lengths.first.times do |index|
@ -101,7 +110,7 @@ module Agents
end
if !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json))
Rails.logger.info "Storing new result for '#{name}': #{result.inspect}"
log "Storing new result for '#{name}': #{result.inspect}"
create_event :payload => result
end
end

View file

@ -23,6 +23,7 @@ class User < ActiveRecord::Base
has_many :events, :order => "events.created_at desc", :dependent => :delete_all, :inverse_of => :user
has_many :agents, :order => "agents.created_at desc", :dependent => :destroy, :inverse_of => :user
has_many :logs, :through => :agents, :class_name => "AgentLog"
# Allow users to login via either email or username.
def self.find_first_by_auth_conditions(warden_conditions)

View file

@ -48,7 +48,7 @@
<%= link_to 'Edit', edit_agent_path(agent), class: "btn btn-mini" %>
<%= link_to 'Delete', agent_path(agent), method: :delete, data: {confirm: 'Are you sure?'}, class: "btn btn-mini" %>
<% if agent.can_be_scheduled? %>
<%= link_to 'Run', run_agent_path(agent), method: :post, class: "btn btn-mini" %>
<%= link_to 'Run', run_agent_path(agent, :return => "index"), method: :post, class: "btn btn-mini" %>
<% else %>
<%= link_to 'Run', "#", class: "btn btn-mini disabled" %>
<% end %>

View file

@ -1,4 +1,4 @@
<div class='container'>
<div class='container agent-show'>
<div class='row'>
<div class='span12'>
@ -11,6 +11,7 @@
<li class='disabled'><a><i class='icon-picture'></i> Summary</a></li>
<li class='active'><a href="#details" data-toggle="tab"><i class='icon-indent-left'></i> Details</a></li>
<% end %>
<li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>"><i class='icon-list-alt'></i> Logs</a></li>
<% if @agent.events.count > 0 %>
<li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
@ -18,17 +19,24 @@
<li><%= link_to '<i class="icon-chevron-left"></i> Back'.html_safe, agents_path %></li>
<li><%= link_to '<i class="icon-pencil"></i> Edit'.html_safe, edit_agent_path(@agent) %></li>
<% if @agent.events.count > 0 %>
<% if @agent.can_be_scheduled? || @agent.events.count > 0 %>
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Actions <b class="caret"></b></a>
<ul class="dropdown-menu" role="menu" aria-labelledby="dLabel">
<% if @agent.can_be_scheduled? %>
<li>
<%= link_to '<i class="icon-refresh"></i> Run'.html_safe, run_agent_path(@agent, :return => "show"), method: :post, :tabindex => "-1" %>
</li>
<% end %>
<% if @agent.events.count > 0 %>
<li>
<%= link_to '<i class="icon-trash"></i> Delete all events'.html_safe, remove_events_agent_path(@agent), method: :delete, data: {confirm: 'Are you sure you want to delete ALL events for this Agent?'}, :tabindex => "-1" %>
</li>
<% end %>
</ul>
</li>
<% end %>
</ul>
<div class="tab-content">
@ -42,6 +50,18 @@
<% end %>
</div>
<div class="tab-pane" id="logs" data-agent-id="<%= @agent.id %>">
<h2>
<%= @agent.name %> Logs
<%= image_tag "spinner-arrows.gif", :class => "spinner" %>
<i class="icon-refresh action-icon refresh"></i>
<i class="icon-trash action-icon clear"></i>
</h2>
<div class='logs'>
Just a moment...
</div>
</div>
<div class="tab-pane <%= agent_show_view(@agent).present? ? "" : "active" %>" id="details">
<h2><%= @agent.name %> Details</h2>

View file

@ -1,6 +1,10 @@
<% flash.each do |name, msg| %>
<div class="top-flash alert alert-<%= name == :notice ? "success" : "error" %>">
<a class="close" data-dismiss="alert">&#215;</a>
<%= content_tag :div, msg, :id => "flash_#{name}" if msg.is_a?(String) %>
<% if flash.keys.length > 0 %>
<div class="flash">
<% flash.each do |name, msg| %>
<div class="alert alert-<%= name == :notice ? "success" : "error" %>">
<a class="close" data-dismiss="alert">&#215;</a>
<%= content_tag :div, msg, :id => "flash_#{name}" if msg.is_a?(String) %>
</div>
<% end %>
</div>
<% end %>

View file

@ -14,6 +14,11 @@
<span class="badge"><i class="icon-refresh icon-white"></i> <span class='number'>0</span></span>
</a>
</li>
<li id='event-indicator'>
<a href="#">
<span class="badge"><i class="icon-random icon-white"></i> <span class='number'>0</span> new events</span>
</a>
</li>
<% end %>
<li class="dropdown">

View file

@ -0,0 +1,30 @@
<table class='table table-striped logs'>
<tr>
<th>Message</th>
<th>When</th>
<th></th>
</tr>
<% @logs.each do |log| %>
<tr>
<td><%= truncate log.message, :length => 200, :omission => "..." %></td>
<td><%= time_ago_in_words log.created_at %> ago</td>
<td>
<div class="btn-group">
<% if log.inbound_event_id.present? %>
<%= link_to 'Event In', event_path(log.inbound_event), class: "btn btn-mini" %>
<% else %>
<%= link_to 'Event In', '#', class: "btn btn-mini disabled" %>
<% end %>
<% if log.outbound_event_id.present? %>
<%= link_to 'Event Out', event_path(log.outbound_event), class: "btn btn-mini" %>
<% else %>
<%= link_to 'Event Out', '#', class: "btn btn-mini disabled" %>
<% end %>
</div>
</td>
</tr>
<% end %>
</table>

View file

@ -11,6 +11,12 @@ Huginn::Application.routes.draw do
get :event_descriptions
get :diagram
end
resources :logs, :only => [:index] do
collection do
delete :clear
end
end
end
resources :events, :only => [:index, :show, :destroy]
match "/worker_status" => "worker_status#show"

View file

@ -0,0 +1,13 @@
class CreateAgentLogs < ActiveRecord::Migration
def change
create_table :agent_logs do |t|
t.integer :agent_id, :null => false
t.text :message, :null => false
t.integer :level, :default => 3, :null => false
t.integer :inbound_event_id
t.integer :outbound_event_id
t.timestamps
end
end
end

View file

@ -11,7 +11,17 @@
#
# It's strongly recommended to check this file into your version control system.
ActiveRecord::Schema.define(:version => 20130509053743) do
ActiveRecord::Schema.define(:version => 20130819160603) do
create_table "agent_logs", :force => true do |t|
t.integer "agent_id", :null => false
t.text "message", :null => false
t.integer "level", :default => 3, :null => false
t.integer "inbound_event_id"
t.integer "outbound_event_id"
t.datetime "created_at", :null => false
t.datetime "updated_at", :null => false
end
create_table "agents", :force => true do |t|
t.integer "user_id"

View file

@ -12,10 +12,21 @@ describe EventsController do
get :index
assigns(:events).all? {|i| i.user.should == users(:bob) }.should be_true
end
it "can filter by Agent" do
sign_in users(:bob)
get :index, :agent => agents(:bob_website_agent)
assigns(:events).length.should == agents(:bob_website_agent).events.length
assigns(:events).all? {|i| i.agent.should == agents(:bob_website_agent) }.should be_true
lambda {
get :index, :agent => agents(:jane_website_agent)
}.should raise_error(ActiveRecord::RecordNotFound)
end
end
describe "GET show" do
it "only shows Agents for the current user" do
it "only shows Events for the current user" do
sign_in users(:bob)
get :show, :id => events(:bob_website_agent_event).to_param
assigns(:event).should eq(events(:bob_website_agent_event))

View file

@ -0,0 +1,37 @@
require 'spec_helper'
describe LogsController do
describe "GET index" do
it "can filter by Agent" do
sign_in users(:bob)
get :index, :agent_id => agents(:bob_weather_agent).id
assigns(:logs).length.should == agents(:bob_weather_agent).logs.length
assigns(:logs).all? {|i| i.agent.should == agents(:bob_weather_agent) }.should be_true
end
it "only loads Agents owned by the current user" do
sign_in users(:bob)
lambda {
get :index, :agent_id => agents(:jane_weather_agent).id
}.should raise_error(ActiveRecord::RecordNotFound)
end
end
describe "DELETE clear" do
it "deletes all logs for a specific Agent" do
sign_in users(:bob)
lambda {
delete :clear, :agent_id => agents(:bob_weather_agent).id
}.should change { AgentLog.count }.by(-1 * agents(:bob_weather_agent).logs.count)
assigns(:logs).length.should == 0
agents(:bob_weather_agent).logs.count.should == 0
end
it "only deletes logs for an Agent owned by the current user" do
sign_in users(:bob)
lambda {
delete :clear, :agent_id => agents(:jane_weather_agent).id
}.should raise_error(ActiveRecord::RecordNotFound)
end
end
end

15
spec/fixtures/agent_logs.yml vendored Normal file
View file

@ -0,0 +1,15 @@
log_for_jane_website_agent:
agent: jane_website_agent
message: "fetching some website data"
log_for_bob_website_agent:
agent: bob_website_agent
message: "fetching some other website data"
first_log_for_bob_weather_agent:
agent: bob_weather_agent
message: "checking the weather"
second_log_for_bob_weather_agent:
agent: bob_weather_agent
message: "checking the weather again"

View file

@ -0,0 +1,15 @@
require 'spec_helper'
# Specs in this file have access to a helper object that includes
# the AgentLogsHelper. For example:
#
# describe AgentLogsHelper do
# describe "string concat" do
# it "concats two strings with spaces" do
# expect(helper.concat_strings("this","that")).to eq("this that")
# end
# end
# end
describe LogsHelper do
pending "add some examples to (or delete) #{__FILE__}"
end

View file

@ -0,0 +1,77 @@
require 'spec_helper'
describe AgentLog do
describe "validations" do
before do
@log = AgentLog.new(:agent => agents(:jane_website_agent), :message => "The agent did something", :level => 3)
@log.should be_valid
end
it "requires an agent" do
@log.agent = nil
@log.should_not be_valid
@log.should have(1).error_on(:agent)
end
it "requires a message" do
@log.message = ""
@log.should_not be_valid
@log.message = nil
@log.should_not be_valid
@log.should have(1).error_on(:message)
end
it "requires a valid log level" do
@log.level = nil
@log.should_not be_valid
@log.should have(1).error_on(:level)
@log.level = -1
@log.should_not be_valid
@log.should have(1).error_on(:level)
@log.level = 5
@log.should_not be_valid
@log.should have(1).error_on(:level)
@log.level = 4
@log.should be_valid
@log.level = 0
@log.should be_valid
end
end
describe "#log_for_agent" do
it "creates AgentLogs" do
log = AgentLog.log_for_agent(agents(:jane_website_agent), "some message", :level => 4, :outbound_event => events(:jane_website_agent_event))
log.should_not be_new_record
log.agent.should == agents(:jane_website_agent)
log.outbound_event.should == events(:jane_website_agent_event)
log.message.should == "some message"
log.level.should == 4
end
it "cleans up old logs when there are more than log_length" do
stub(AgentLog).log_length { 4 }
AgentLog.log_for_agent(agents(:jane_website_agent), "message 1")
AgentLog.log_for_agent(agents(:jane_website_agent), "message 2")
AgentLog.log_for_agent(agents(:jane_website_agent), "message 3")
AgentLog.log_for_agent(agents(:jane_website_agent), "message 4")
agents(:jane_website_agent).logs.order("agent_logs.id desc").first.message.should == "message 4"
agents(:jane_website_agent).logs.order("agent_logs.id desc").last.message.should == "message 1"
AgentLog.log_for_agent(agents(:jane_website_agent), "message 5")
agents(:jane_website_agent).logs.order("agent_logs.id desc").first.message.should == "message 5"
agents(:jane_website_agent).logs.order("agent_logs.id desc").last.message.should == "message 2"
AgentLog.log_for_agent(agents(:jane_website_agent), "message 6")
agents(:jane_website_agent).logs.order("agent_logs.id desc").first.message.should == "message 6"
agents(:jane_website_agent).logs.order("agent_logs.id desc").last.message.should == "message 3"
end
end
describe "#log_length" do
it "defaults to 100" do
AgentLog.log_length.should == 100
end
end
end

View file

@ -35,11 +35,10 @@ describe Agents::WebsiteAgent do
end
it "should log an error if the number of results for a set of extraction patterns differs" do
lambda {
@site[:extract][:url][:css] = "div"
@checker.options = @site
@checker.check
}.should raise_error(StandardError, /Got an uneven number of matches/)
@site[:extract][:url][:css] = "div"
@checker.options = @site
@checker.check
@checker.logs.first.message.should =~ /Got an uneven number of matches/
end
end

View file

View file