Merge remote-tracking branch 'huginn/master' into chef-solo

app/models/agents/trigger_agent.rb

@@ -11,6 +11,8 @@ module Agents
 
       The `type` can be one of #{VALID_COMPARISON_TYPES.map { |t| "`#{t}`" }.to_sentence} and compares with the `value`.
 
+      The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches. 
+
       All rules must match for the Agent to match.  The resulting Event will have a payload message of `message`.  You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
@@ -49,25 +51,30 @@ module Agents
       incoming_events.each do |event|
         match = options['rules'].all? do |rule|
           value_at_path = Utils.value_at(event['payload'], rule['path'])
-          case rule['type']
+          rule_values = rule['value']
+          rule_values = [rule_values] unless rule_values.is_a?(Array)
+
+          match_found = rule_values.any? do |rule_value|
+            case rule['type']
             when "regex"
-              value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
+              value_at_path.to_s =~ Regexp.new(rule_value, Regexp::IGNORECASE)
             when "!regex"
-              value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
+              value_at_path.to_s !~ Regexp.new(rule_value, Regexp::IGNORECASE)
             when "field>value"
-              value_at_path.to_f > rule['value'].to_f
+              value_at_path.to_f > rule_value.to_f
             when "field>=value"
-              value_at_path.to_f >= rule['value'].to_f
+              value_at_path.to_f >= rule_value.to_f
             when "field<value"
-              value_at_path.to_f < rule['value'].to_f
+              value_at_path.to_f < rule_value.to_f
             when "field<=value"
-              value_at_path.to_f <= rule['value'].to_f
+              value_at_path.to_f <= rule_value.to_f
             when "field==value"
-              value_at_path.to_s == rule['value'].to_s
+              value_at_path.to_s == rule_value.to_s
             when "field!=value"
-              value_at_path.to_s != rule['value'].to_s
+              value_at_path.to_s != rule_value.to_s
             else
               raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
+            end
           end
         end
 

app/models/agents/website_agent.rb

@@ -4,7 +4,6 @@ require 'date'
 
 module Agents
   class WebsiteAgent < Agent
-    cannot_receive_events!
 
     default_schedule "every_12h"
 
@@ -46,6 +45,8 @@ module Agents
       Set `uniqueness_look_back` to limit the number of events checked for uniqueness (typically for performance).  This defaults to the larger of #{UNIQUENESS_LOOK_BACK} or #{UNIQUENESS_FACTOR}x the number of detected received results.
 
       Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset.
+
+      The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
     MD
 
     event_description do
@@ -105,19 +106,23 @@ module Agents
     end
 
     def check
-      hydra = Typhoeus::Hydra.new
       log "Fetching #{options['url']}"
+      check_url options['url']
+    end
+
+    def check_url(in_url)
+      hydra = Typhoeus::Hydra.new
       request_opts = { :followlocation => true }
       request_opts[:userpwd] = options['basic_auth'] if options['basic_auth'].present?
 
       requests = []
 
-      if options['url'].kind_of?(Array)
-        options['url'].each do |url|
+      if in_url.kind_of?(Array)
+        in_url.each do |url|
            requests.push(Typhoeus::Request.new(url, request_opts))
         end
       else
-        requests.push(Typhoeus::Request.new(options['url'], request_opts))
+        requests.push(Typhoeus::Request.new(in_url, request_opts))
       end
 
       requests.each do |request|
@@ -185,7 +190,7 @@ module Agents
               options['extract'].keys.each do |name|
                 result[name] = output[name][index]
                 if name.to_s == 'url'
-                  result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
+                  result[name] = URI.join(request.base_url, result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
                 end
               end
 
@@ -202,6 +207,13 @@ module Agents
       end
     end
 
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        url_to_scrape = event.payload['url']
+        check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
+      end
+    end
+
     private
 
     # This method returns true if the result should be stored as a new event.
@@ -275,5 +287,7 @@ module Agents
         false
       end
     end
+
   end
+
 end

deployment/Vagrantfile

@@ -3,37 +3,34 @@
 
 Vagrant.configure("2") do |config|
   config.omnibus.chef_version = :latest
-  config.vm.define :vb do |vb|
-    vb.vm.box = "precise32"
-    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
-    vb.vm.network :forwarded_port, host: 3000, guest: 3000
 
-    vb.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_development")
-    end
+  config.vm.provision :chef_solo do |chef|
+    chef.roles_path = "roles"
+    chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
+    chef.add_role("huginn_development")
+    # chef.add_role("huginn_production")
   end
 
-  config.vm.define :ec2 do |ec2|
-    ec2.vm.box = "dummy"
-    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
+  config.vm.provider :virtualbox do |vb, override|
+    override.vm.box = "hashicorp/precise64"
+    override.vm.network :forwarded_port, host: 3000, guest: 3000
+  end
 
-    ec2.vm.provider :aws do |aws, override|
-      aws.access_key_id = ""
-      aws.secret_access_key = ""
-      aws.keypair_name = ""
-      aws.region = "us-east-1"
-      aws.ami = "ami-d0f89fb9"
+  config.vm.provider :parallels do |prl, override|
+    override.vm.box = "parallels/ubuntu-12.04"
+  end
 
-      override.ssh.username = "ubuntu"
-      override.ssh.private_key_path = ""
-    end
-    ec2.vm.provision :chef_solo do |chef|
-      chef.roles_path = "roles"
-      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
-      chef.add_role("huginn_production")
-    
-    end
+  config.vm.provider :aws do |aws, override| 
+    override.vm.box = "dummy"
+    override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
+
+    aws.access_key_id = ""
+    aws.secret_access_key = ""
+    aws.keypair_name = ""
+    aws.region = "us-east-1"
+    aws.ami = "ami-d0f89fb9"
+
+    override.ssh.username = "ubuntu"
+    override.ssh.private_key_path = ""
   end
 end

deployment/roles/huginn_development.json

@@ -23,6 +23,7 @@
              "recipe[git]",
              "recipe[apt]",
              "recipe[mysql::server]",
+             "recipe[mysql::client]",
              "recipe[nodejs::install_from_binary]",
              "recipe[huginn_development]"
            ]

deployment/site-cookbooks/huginn_development/recipes/default.rb

@@ -16,12 +16,19 @@ group "huginn" do
   action :create
 end
 
-%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev").each do |pkg|
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
   package pkg do
     action :install
   end
 end
 
+bash "Setting default ruby version to 1.9" do
+  code <<-EOH
+    update-alternatives --set ruby /usr/bin/ruby1.9.1
+    update-alternatives --set gem /usr/bin/gem1.9.1
+  EOH
+end
+
 git "/home/huginn/huginn" do
   repository 'git://github.com/cantino/huginn.git'
   reference 'master'
@@ -48,7 +55,7 @@ bash "huginn dependencies" do
     export LANG="en_US.UTF-8"
     export LC_ALL="en_US.UTF-8"
     sudo bundle install
-    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
+    sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env
     sudo bundle exec rake db:create
     sudo bundle exec rake db:migrate
     sudo bundle exec rake db:seed
@@ -59,6 +66,6 @@ bash "huginn has been installed and will start in a minute" do
   user "huginn"
   cwd "/home/huginn/huginn"
   code <<-EOH
-    sudo foreman start
+    sudo nohup foreman start &
     EOH
 end

spec/models/agents/trigger_agent_spec.rb

@@ -71,6 +71,28 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of regex" do
+      @event.payload['foo']['bar']['baz'] = "a222b"
+      @checker.options['rules'][0] = {
+        'type' => "regex",
+        'value' => ["a\\db", "a\\Wb"],
+        'path' => "foo.bar.baz",
+      }
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @event.payload['foo']['bar']['baz'] = "a2b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+
+      @event.payload['foo']['bar']['baz'] = "a b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "handles negated regex" do
       @event.payload['foo']['bar']['baz'] = "a2b"
       @checker.options['rules'][0] = {
@@ -89,6 +111,24 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of negated regex" do
+      @event.payload['foo']['bar']['baz'] = "a2b"
+      @checker.options['rules'][0] = {
+        'type' => "!regex",
+        'value' => ["a\\db", "a2b"],
+        'path' => "foo.bar.baz",
+      }
+
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @event.payload['foo']['bar']['baz'] = "a3b"
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "puts can extract values into the message based on paths" do
       @checker.receive([@event])
       Event.last.payload['message'].should == "I saw 'a2b' from Joe"
@@ -109,6 +149,21 @@ describe Agents::TriggerAgent do
       }.should_not change { Event.count }
     end
 
+    it "handles array of numerical comparisons" do
+      @event.payload['foo']['bar']['baz'] = "5"
+      @checker.options['rules'].first['value'] = [6, 3]
+      @checker.options['rules'].first['type'] = "field<value"
+
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+
+      @checker.options['rules'].first['value'] = [4, 3]
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+    end
+
     it "handles exact comparisons" do
       @event.payload['foo']['bar']['baz'] = "hello world"
       @checker.options['rules'].first['type'] = "field==value"
@@ -124,6 +179,21 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of exact comparisons" do
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field==value"
+
+      @checker.options['rules'].first['value'] = ["hello there", "hello universe"]
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @checker.options['rules'].first['value'] = ["hello world", "hello universe"]
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "handles negated comparisons" do
       @event.payload['foo']['bar']['baz'] = "hello world"
       @checker.options['rules'].first['type'] = "field!=value"
@@ -140,6 +210,22 @@ describe Agents::TriggerAgent do
       }.should change { Event.count }.by(1)
     end
 
+    it "handles array of negated comparisons" do
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field!=value"
+      @checker.options['rules'].first['value'] = ["hello world", "hello world"]
+
+      lambda {
+        @checker.receive([@event])
+      }.should_not change { Event.count }
+
+      @checker.options['rules'].first['value'] = ["hello there", "hello world"]
+
+      lambda {
+        @checker.receive([@event])
+      }.should change { Event.count }.by(1)
+    end
+
     it "does fine without dots in the path" do
       @event.payload = { 'hello' => "world" }
       @checker.options['rules'].first['type'] = "field==value"

spec/models/agents/website_agent_spec.rb

@@ -331,6 +331,19 @@ describe Agents::WebsiteAgent do
         end
       end
     end
+
+    describe "#receive" do
+      it "should scrape from the url element in incoming event payload" do
+        @event = Event.new
+        @event.agent = agents(:bob_rain_notifier_agent)
+        @event.payload = { 'url' => "http://xkcd.com" }
+
+        lambda {
+          @checker.options = @site
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+      end
+    end
   end
 
   describe "checking with http basic auth" do
@@ -361,4 +374,4 @@ describe Agents::WebsiteAgent do
       end
     end
   end
-end
+end