WebsiteAgent: Provide a variable _response_ for interpolation.

* `_response_.headers.Header-Name` is expanded to the value of header
  named _Header-Name_.

* `_response_.status` is expanded to the HTTP status as integer, which
  is almost always 200, since currently the WebsiteAgent will only
  create an event when a request succeeds, and the request method is
  limited to GET.

app/models/agents/website_agent.rb

@@ -75,6 +75,14 @@ module Agents
       The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
 
       The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
+
+      In Liquid templating, the following variable is available:
+
+      * `_response_`: A response object with the following keys:
+
+          * `status`: HTTP status as integer. (Almost always 200)
+
+          * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insentitive to cases and -/_.
     MD
 
     event_description do
@@ -149,7 +157,10 @@ module Agents
       Array(in_url).each do |url|
         log "Fetching #{url}"
         response = faraday.get(url)
-        if response.success?
+        raise "Failed: #{response.inspect}" unless response.success?
+
+        interpolation_context.stack {
+          interpolation_context['_response_'] = ResponseDrop.new(response)
           body = response.body
           if (encoding = interpolated['force_encoding']).present?
             body = body.encode(Encoding::UTF_8, encoding)
@@ -195,9 +206,7 @@ module Agents
               create_event :payload => result
             end
           end
-        else
-          raise "Failed: #{response.inspect}"
-        end
+        }
       end
     rescue => e
       error e.message
@@ -344,5 +353,24 @@ module Agents
     rescue
       false
     end
+
+    # Wraps Faraday::Response
+    class ResponseDrop < LiquidDroppable::Drop
+      def headers
+        HeaderDrop.new(@object.headers)
+      end
+
+      # Integer value of HTTP status
+      def status
+        @object.status
+      end
+    end
+
+    # Wraps Faraday::Utilsa::Headers
+    class HeaderDrop < LiquidDroppable::Drop
+      def before_method(name)
+        @object[name.tr('_', '-')]
+      end
+    end
   end
 end

spec/models/agents/website_agent_spec.rb

@@ -3,7 +3,11 @@ require 'spec_helper'
 describe Agents::WebsiteAgent do
   describe "checking without basic auth" do
     before do
-      stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")),
+                                           status: 200,
+                                           headers: {
+                                             'X-Status-Message' => 'OK'
+                                           })
       @valid_options = {
         'name' => "XKCD",
         'expected_update_period_in_days' => "2",
@@ -305,6 +309,17 @@ describe Agents::WebsiteAgent do
         event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language."
       end
 
+      it "should interpolate _response_" do
+        @valid_options['extract']['response_info'] =
+          @valid_options['extract']['url'].merge(
+            'value' => '"{{ "The reponse was " | append:_response_.status | append:" " | append:_response_.headers.X-Status-Message | append:"." }}"'
+          )
+        @checker.options = @valid_options
+        @checker.check
+        event = Event.last
+        event.payload['response_info'].should == 'The reponse was 200 OK.'
+      end
+
       describe "JSON" do
         it "works with paths" do
           json = {
@@ -490,6 +505,22 @@ fire: hot
           'to' => 'http://dynamic.xkcd.com/random/comic/',
         }
       end
+
+      it "should interpolate values from incoming event payload and _response_" do
+        @event.payload['title'] = 'XKCD'
+
+        lambda {
+          @valid_options['extract'] = {
+            'response_info' => @valid_options['extract']['url'].merge(
+              'value' => '{% capture sentence %}The reponse from {{title}} was {{_response_.status}} {{_response_.headers.X-Status-Message}}.{% endcapture %}{{sentence | to_xpath}}'
+            )
+          }
+          @checker.options = @valid_options
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+
+        Event.last.payload['response_info'].should == 'The reponse from XKCD was 200 OK.'
+      end
     end
   end