Merge remote-tracking branch 'huginn/master' into chef-solo

This commit is contained in:
Dominik Sander 2014-04-23 21:52:47 +02:00
commit 53ebf6b6aa
7 changed files with 171 additions and 46 deletions

View file

@ -11,6 +11,8 @@ module Agents
The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.
The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches.
All rules must match for the Agent to match. The resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
@ -49,25 +51,30 @@ module Agents
incoming_events.each do |event|
match = options['rules'].all? do |rule|
value_at_path = Utils.value_at(event['payload'], rule['path'])
case rule['type']
rule_values = rule['value']
rule_values = [rule_values] unless rule_values.is_a?(Array)
match_found = rule_values.any? do |rule_value|
case rule['type']
when "regex"
value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
when "!regex"
value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
value_at_path.to_s !~ Regexp.new(rule_value, Regexp::IGNORECASE)
when "field>value"
value_at_path.to_f > rule['value'].to_f
value_at_path.to_f > rule_value.to_f
when "field>=value"
value_at_path.to_f >= rule['value'].to_f
value_at_path.to_f >= rule_value.to_f
when "field<value"
value_at_path.to_f < rule['value'].to_f
value_at_path.to_f < rule_value.to_f
when "field<=value"
value_at_path.to_f <= rule['value'].to_f
value_at_path.to_f <= rule_value.to_f
when "field==value"
value_at_path.to_s == rule['value'].to_s
value_at_path.to_s == rule_value.to_s
when "field!=value"
value_at_path.to_s != rule['value'].to_s
value_at_path.to_s != rule_value.to_s
else
raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
end
end
end

View file

@ -4,7 +4,6 @@ require 'date'
module Agents
class WebsiteAgent < Agent
cannot_receive_events!
default_schedule "every_12h"
@ -46,6 +45,8 @@ module Agents
Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance). This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
MD
event_description do
@ -105,19 +106,23 @@ module Agents
end
def check
hydra = Typhoeus::Hydra.new
log "Fetching #{options['url']}"
check_url options['url']
end
def check_url(in_url)
hydra = Typhoeus::Hydra.new
request_opts = { :followlocation => true }
request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
requests = []
if options['url'].kind_of?(Array)
options['url'].each do |url|
if in_url.kind_of?(Array)
in_url.each do |url|
requests.push(Typhoeus::Request.new(url, request_opts))
end
else
requests.push(Typhoeus::Request.new(options['url'], request_opts))
requests.push(Typhoeus::Request.new(in_url, request_opts))
end
requests.each do |request|
@ -185,7 +190,7 @@ module Agents
options['extract'].keys.each do |name|
result[name] = output[name][index]
if name.to_s == 'url'
result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
result[name] = URI.join(request.base_url, result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
end
end
@ -202,6 +207,13 @@ module Agents
end
end
def receive(incoming_events)
incoming_events.each do |event|
url_to_scrape = event.payload['url']
check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
end
end
private
# This method returns true if the result should be stored as a new event.
@ -275,5 +287,7 @@ module Agents
false
end
end
end
end

View file

@ -3,37 +3,34 @@
Vagrant.configure("2") do |config|
config.omnibus.chef_version = :latest
config.vm.define :vb do |vb|
vb.vm.box = "precise32"
vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
vb.vm.network :forwarded_port, host: 3000, guest: 3000
vb.vm.provision :chef_solo do |chef|
chef.roles_path = "roles"
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
chef.add_role("huginn_development")
end
config.vm.provision :chef_solo do |chef|
chef.roles_path = "roles"
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
chef.add_role("huginn_development")
# chef.add_role("huginn_production")
end
config.vm.define :ec2 do |ec2|
ec2.vm.box = "dummy"
ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
config.vm.provider :virtualbox do |vb, override|
override.vm.box = "hashicorp/precise64"
override.vm.network :forwarded_port, host: 3000, guest: 3000
end
ec2.vm.provider :aws do |aws, override|
aws.access_key_id = ""
aws.secret_access_key = ""
aws.keypair_name = ""
aws.region = "us-east-1"
aws.ami = "ami-d0f89fb9"
config.vm.provider :parallels do |prl, override|
override.vm.box = "parallels/ubuntu-12.04"
end
override.ssh.username = "ubuntu"
override.ssh.private_key_path = ""
end
ec2.vm.provision :chef_solo do |chef|
chef.roles_path = "roles"
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
chef.add_role("huginn_production")
end
config.vm.provider :aws do |aws, override|
override.vm.box = "dummy"
override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
aws.access_key_id = ""
aws.secret_access_key = ""
aws.keypair_name = ""
aws.region = "us-east-1"
aws.ami = "ami-d0f89fb9"
override.ssh.username = "ubuntu"
override.ssh.private_key_path = ""
end
end

View file

@ -23,6 +23,7 @@
"recipe[git]",
"recipe[apt]",
"recipe[mysql::server]",
"recipe[mysql::client]",
"recipe[nodejs::install_from_binary]",
"recipe[huginn_development]"
]

View file

@ -16,12 +16,19 @@ group "huginn" do
action :create
end
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev").each do |pkg|
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
package pkg do
action :install
end
end
bash "Setting default ruby version to 1.9" do
code <<-EOH
update-alternatives --set ruby /usr/bin/ruby1.9.1
update-alternatives --set gem /usr/bin/gem1.9.1
EOH
end
git "/home/huginn/huginn" do
repository 'git://github.com/cantino/huginn.git'
reference 'master'
@ -48,7 +55,7 @@ bash "huginn dependencies" do
export LANG="en_US.UTF-8"
export LC_ALL="en_US.UTF-8"
sudo bundle install
sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
sudo bundle exec rake db:create
sudo bundle exec rake db:migrate
sudo bundle exec rake db:seed
@ -59,6 +66,6 @@ bash "huginn has been installed and will start in a minute" do
user "huginn"
cwd "/home/huginn/huginn"
code <<-EOH
sudo foreman start
sudo nohup foreman start &
EOH
end

View file

@ -71,6 +71,28 @@ describe Agents::TriggerAgent do
}.should change { Event.count }.by(1)
end
it "handles array of regex" do
@event.payload['foo']['bar']['baz'] = "a222b"
@checker.options['rules'][0] = {
'type' => "regex",
'value' => ["a\\db", "a\\Wb"],
'path' => "foo.bar.baz",
}
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@event.payload['foo']['bar']['baz'] = "a2b"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@event.payload['foo']['bar']['baz'] = "a b"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "handles negated regex" do
@event.payload['foo']['bar']['baz'] = "a2b"
@checker.options['rules'][0] = {
@ -89,6 +111,24 @@ describe Agents::TriggerAgent do
}.should change { Event.count }.by(1)
end
it "handles array of negated regex" do
@event.payload['foo']['bar']['baz'] = "a2b"
@checker.options['rules'][0] = {
'type' => "!regex",
'value' => ["a\\db", "a2b"],
'path' => "foo.bar.baz",
}
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@event.payload['foo']['bar']['baz'] = "a3b"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "puts can extract values into the message based on paths" do
@checker.receive([@event])
Event.last.payload['message'].should == "I saw 'a2b' from Joe"
@ -109,6 +149,21 @@ describe Agents::TriggerAgent do
}.should_not change { Event.count }
end
it "handles array of numerical comparisons" do
@event.payload['foo']['bar']['baz'] = "5"
@checker.options['rules'].first['value'] = [6, 3]
@checker.options['rules'].first['type'] = "field<value"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@checker.options['rules'].first['value'] = [4, 3]
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
end
it "handles exact comparisons" do
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field==value"
@ -124,6 +179,21 @@ describe Agents::TriggerAgent do
}.should change { Event.count }.by(1)
end
it "handles array of exact comparisons" do
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field==value"
@checker.options['rules'].first['value'] = ["hello there", "hello universe"]
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@checker.options['rules'].first['value'] = ["hello world", "hello universe"]
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "handles negated comparisons" do
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field!=value"
@ -140,6 +210,22 @@ describe Agents::TriggerAgent do
}.should change { Event.count }.by(1)
end
it "handles array of negated comparisons" do
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field!=value"
@checker.options['rules'].first['value'] = ["hello world", "hello world"]
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@checker.options['rules'].first['value'] = ["hello there", "hello world"]
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "does fine without dots in the path" do
@event.payload = { 'hello' => "world" }
@checker.options['rules'].first['type'] = "field==value"

View file

@ -331,6 +331,19 @@ describe Agents::WebsiteAgent do
end
end
end
describe "#receive" do
it "should scrape from the url element in incoming event payload" do
@event = Event.new
@event.agent = agents(:bob_rain_notifier_agent)
@event.payload = { 'url' => "http://xkcd.com" }
lambda {
@checker.options = @site
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
end
end
describe "checking with http basic auth" do
@ -361,4 +374,4 @@ describe Agents::WebsiteAgent do
end
end
end
end
end