mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Merge remote-tracking branch 'huginn/master' into chef-solo
This commit is contained in:
commit
53ebf6b6aa
7 changed files with 171 additions and 46 deletions
|
@ -11,6 +11,8 @@ module Agents
|
|||
|
||||
The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.
|
||||
|
||||
The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches.
|
||||
|
||||
All rules must match for the Agent to match. The resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
|
||||
|
||||
Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
|
||||
|
@ -49,25 +51,30 @@ module Agents
|
|||
incoming_events.each do |event|
|
||||
match = options['rules'].all? do |rule|
|
||||
value_at_path = Utils.value_at(event['payload'], rule['path'])
|
||||
case rule['type']
|
||||
rule_values = rule['value']
|
||||
rule_values = [rule_values] unless rule_values.is_a?(Array)
|
||||
|
||||
match_found = rule_values.any? do |rule_value|
|
||||
case rule['type']
|
||||
when "regex"
|
||||
value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
|
||||
value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
|
||||
when "!regex"
|
||||
value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
|
||||
value_at_path.to_s !~ Regexp.new(rule_value, Regexp::IGNORECASE)
|
||||
when "field>value"
|
||||
value_at_path.to_f > rule['value'].to_f
|
||||
value_at_path.to_f > rule_value.to_f
|
||||
when "field>=value"
|
||||
value_at_path.to_f >= rule['value'].to_f
|
||||
value_at_path.to_f >= rule_value.to_f
|
||||
when "field<value"
|
||||
value_at_path.to_f < rule['value'].to_f
|
||||
value_at_path.to_f < rule_value.to_f
|
||||
when "field<=value"
|
||||
value_at_path.to_f <= rule['value'].to_f
|
||||
value_at_path.to_f <= rule_value.to_f
|
||||
when "field==value"
|
||||
value_at_path.to_s == rule['value'].to_s
|
||||
value_at_path.to_s == rule_value.to_s
|
||||
when "field!=value"
|
||||
value_at_path.to_s != rule['value'].to_s
|
||||
value_at_path.to_s != rule_value.to_s
|
||||
else
|
||||
raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -4,7 +4,6 @@ require 'date'
|
|||
|
||||
module Agents
|
||||
class WebsiteAgent < Agent
|
||||
cannot_receive_events!
|
||||
|
||||
default_schedule "every_12h"
|
||||
|
||||
|
@ -46,6 +45,8 @@ module Agents
|
|||
Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance). This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
|
||||
|
||||
Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
|
||||
|
||||
The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
|
||||
MD
|
||||
|
||||
event_description do
|
||||
|
@ -105,19 +106,23 @@ module Agents
|
|||
end
|
||||
|
||||
def check
|
||||
hydra = Typhoeus::Hydra.new
|
||||
log "Fetching #{options['url']}"
|
||||
check_url options['url']
|
||||
end
|
||||
|
||||
def check_url(in_url)
|
||||
hydra = Typhoeus::Hydra.new
|
||||
request_opts = { :followlocation => true }
|
||||
request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
|
||||
|
||||
requests = []
|
||||
|
||||
if options['url'].kind_of?(Array)
|
||||
options['url'].each do |url|
|
||||
if in_url.kind_of?(Array)
|
||||
in_url.each do |url|
|
||||
requests.push(Typhoeus::Request.new(url, request_opts))
|
||||
end
|
||||
else
|
||||
requests.push(Typhoeus::Request.new(options['url'], request_opts))
|
||||
requests.push(Typhoeus::Request.new(in_url, request_opts))
|
||||
end
|
||||
|
||||
requests.each do |request|
|
||||
|
@ -185,7 +190,7 @@ module Agents
|
|||
options['extract'].keys.each do |name|
|
||||
result[name] = output[name][index]
|
||||
if name.to_s == 'url'
|
||||
result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
|
||||
result[name] = URI.join(request.base_url, result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
|
||||
end
|
||||
end
|
||||
|
||||
|
@ -202,6 +207,13 @@ module Agents
|
|||
end
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
incoming_events.each do |event|
|
||||
url_to_scrape = event.payload['url']
|
||||
check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# This method returns true if the result should be stored as a new event.
|
||||
|
@ -275,5 +287,7 @@ module Agents
|
|||
false
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
|
51
deployment/Vagrantfile
vendored
51
deployment/Vagrantfile
vendored
|
@ -3,37 +3,34 @@
|
|||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.omnibus.chef_version = :latest
|
||||
config.vm.define :vb do |vb|
|
||||
vb.vm.box = "precise32"
|
||||
vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
|
||||
vb.vm.network :forwarded_port, host: 3000, guest: 3000
|
||||
|
||||
vb.vm.provision :chef_solo do |chef|
|
||||
chef.roles_path = "roles"
|
||||
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
|
||||
chef.add_role("huginn_development")
|
||||
end
|
||||
config.vm.provision :chef_solo do |chef|
|
||||
chef.roles_path = "roles"
|
||||
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
|
||||
chef.add_role("huginn_development")
|
||||
# chef.add_role("huginn_production")
|
||||
end
|
||||
|
||||
config.vm.define :ec2 do |ec2|
|
||||
ec2.vm.box = "dummy"
|
||||
ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
|
||||
config.vm.provider :virtualbox do |vb, override|
|
||||
override.vm.box = "hashicorp/precise64"
|
||||
override.vm.network :forwarded_port, host: 3000, guest: 3000
|
||||
end
|
||||
|
||||
ec2.vm.provider :aws do |aws, override|
|
||||
aws.access_key_id = ""
|
||||
aws.secret_access_key = ""
|
||||
aws.keypair_name = ""
|
||||
aws.region = "us-east-1"
|
||||
aws.ami = "ami-d0f89fb9"
|
||||
config.vm.provider :parallels do |prl, override|
|
||||
override.vm.box = "parallels/ubuntu-12.04"
|
||||
end
|
||||
|
||||
override.ssh.username = "ubuntu"
|
||||
override.ssh.private_key_path = ""
|
||||
end
|
||||
ec2.vm.provision :chef_solo do |chef|
|
||||
chef.roles_path = "roles"
|
||||
chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
|
||||
chef.add_role("huginn_production")
|
||||
|
||||
end
|
||||
config.vm.provider :aws do |aws, override|
|
||||
override.vm.box = "dummy"
|
||||
override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
|
||||
|
||||
aws.access_key_id = ""
|
||||
aws.secret_access_key = ""
|
||||
aws.keypair_name = ""
|
||||
aws.region = "us-east-1"
|
||||
aws.ami = "ami-d0f89fb9"
|
||||
|
||||
override.ssh.username = "ubuntu"
|
||||
override.ssh.private_key_path = ""
|
||||
end
|
||||
end
|
||||
|
|
|
@ -23,6 +23,7 @@
|
|||
"recipe[git]",
|
||||
"recipe[apt]",
|
||||
"recipe[mysql::server]",
|
||||
"recipe[mysql::client]",
|
||||
"recipe[nodejs::install_from_binary]",
|
||||
"recipe[huginn_development]"
|
||||
]
|
||||
|
|
|
@ -16,12 +16,19 @@ group "huginn" do
|
|||
action :create
|
||||
end
|
||||
|
||||
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev").each do |pkg|
|
||||
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
|
||||
package pkg do
|
||||
action :install
|
||||
end
|
||||
end
|
||||
|
||||
bash "Setting default ruby version to 1.9" do
|
||||
code <<-EOH
|
||||
update-alternatives --set ruby /usr/bin/ruby1.9.1
|
||||
update-alternatives --set gem /usr/bin/gem1.9.1
|
||||
EOH
|
||||
end
|
||||
|
||||
git "/home/huginn/huginn" do
|
||||
repository 'git://github.com/cantino/huginn.git'
|
||||
reference 'master'
|
||||
|
@ -48,7 +55,7 @@ bash "huginn dependencies" do
|
|||
export LANG="en_US.UTF-8"
|
||||
export LC_ALL="en_US.UTF-8"
|
||||
sudo bundle install
|
||||
sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
|
||||
sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
|
||||
sudo bundle exec rake db:create
|
||||
sudo bundle exec rake db:migrate
|
||||
sudo bundle exec rake db:seed
|
||||
|
@ -59,6 +66,6 @@ bash "huginn has been installed and will start in a minute" do
|
|||
user "huginn"
|
||||
cwd "/home/huginn/huginn"
|
||||
code <<-EOH
|
||||
sudo foreman start
|
||||
sudo nohup foreman start &
|
||||
EOH
|
||||
end
|
||||
|
|
|
@ -71,6 +71,28 @@ describe Agents::TriggerAgent do
|
|||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles array of regex" do
|
||||
@event.payload['foo']['bar']['baz'] = "a222b"
|
||||
@checker.options['rules'][0] = {
|
||||
'type' => "regex",
|
||||
'value' => ["a\\db", "a\\Wb"],
|
||||
'path' => "foo.bar.baz",
|
||||
}
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should_not change { Event.count }
|
||||
|
||||
@event.payload['foo']['bar']['baz'] = "a2b"
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
|
||||
@event.payload['foo']['bar']['baz'] = "a b"
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles negated regex" do
|
||||
@event.payload['foo']['bar']['baz'] = "a2b"
|
||||
@checker.options['rules'][0] = {
|
||||
|
@ -89,6 +111,24 @@ describe Agents::TriggerAgent do
|
|||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles array of negated regex" do
|
||||
@event.payload['foo']['bar']['baz'] = "a2b"
|
||||
@checker.options['rules'][0] = {
|
||||
'type' => "!regex",
|
||||
'value' => ["a\\db", "a2b"],
|
||||
'path' => "foo.bar.baz",
|
||||
}
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should_not change { Event.count }
|
||||
|
||||
@event.payload['foo']['bar']['baz'] = "a3b"
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "puts can extract values into the message based on paths" do
|
||||
@checker.receive([@event])
|
||||
Event.last.payload['message'].should == "I saw 'a2b' from Joe"
|
||||
|
@ -109,6 +149,21 @@ describe Agents::TriggerAgent do
|
|||
}.should_not change { Event.count }
|
||||
end
|
||||
|
||||
it "handles array of numerical comparisons" do
|
||||
@event.payload['foo']['bar']['baz'] = "5"
|
||||
@checker.options['rules'].first['value'] = [6, 3]
|
||||
@checker.options['rules'].first['type'] = "field<value"
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
|
||||
@checker.options['rules'].first['value'] = [4, 3]
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should_not change { Event.count }
|
||||
end
|
||||
|
||||
it "handles exact comparisons" do
|
||||
@event.payload['foo']['bar']['baz'] = "hello world"
|
||||
@checker.options['rules'].first['type'] = "field==value"
|
||||
|
@ -124,6 +179,21 @@ describe Agents::TriggerAgent do
|
|||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles array of exact comparisons" do
|
||||
@event.payload['foo']['bar']['baz'] = "hello world"
|
||||
@checker.options['rules'].first['type'] = "field==value"
|
||||
|
||||
@checker.options['rules'].first['value'] = ["hello there", "hello universe"]
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should_not change { Event.count }
|
||||
|
||||
@checker.options['rules'].first['value'] = ["hello world", "hello universe"]
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles negated comparisons" do
|
||||
@event.payload['foo']['bar']['baz'] = "hello world"
|
||||
@checker.options['rules'].first['type'] = "field!=value"
|
||||
|
@ -140,6 +210,22 @@ describe Agents::TriggerAgent do
|
|||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "handles array of negated comparisons" do
|
||||
@event.payload['foo']['bar']['baz'] = "hello world"
|
||||
@checker.options['rules'].first['type'] = "field!=value"
|
||||
@checker.options['rules'].first['value'] = ["hello world", "hello world"]
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should_not change { Event.count }
|
||||
|
||||
@checker.options['rules'].first['value'] = ["hello there", "hello world"]
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
|
||||
it "does fine without dots in the path" do
|
||||
@event.payload = { 'hello' => "world" }
|
||||
@checker.options['rules'].first['type'] = "field==value"
|
||||
|
|
|
@ -331,6 +331,19 @@ describe Agents::WebsiteAgent do
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "#receive" do
|
||||
it "should scrape from the url element in incoming event payload" do
|
||||
@event = Event.new
|
||||
@event.agent = agents(:bob_rain_notifier_agent)
|
||||
@event.payload = { 'url' => "http://xkcd.com" }
|
||||
|
||||
lambda {
|
||||
@checker.options = @site
|
||||
@checker.receive([@event])
|
||||
}.should change { Event.count }.by(1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "checking with http basic auth" do
|
||||
|
@ -361,4 +374,4 @@ describe Agents::WebsiteAgent do
|
|||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Add table
Reference in a new issue