From c21bada22611663a2bc0fa0d33e1c7c0d87e8f1a Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Mon, 1 Sep 2014 23:17:19 +0900
Subject: [PATCH] WebsiteAgent: Provide a variable _response_ for
 interpolation.

* `_response_.headers.Header-Name` is expanded to the value of header
  named _Header-Name_.

* `_response_.status` is expanded to the HTTP status as integer, which
  is almost always 200, since currently the WebsiteAgent will only
  create an event when a request succeeds, and the request method is
  limited to GET.
---
 app/models/agents/website_agent.rb       | 36 +++++++++++++++++++++---
 spec/models/agents/website_agent_spec.rb | 33 +++++++++++++++++++++-
 2 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index fd4f48e3..cabab6ec 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -75,6 +75,14 @@ module Agents
       The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
 
       The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
+
+      In Liquid templating, the following variable is available:
+
+      * `_response_`: A response object with the following keys:
+
+          * `status`: HTTP status as integer. (Almost always 200)
+
+          * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insentitive to cases and -/_.
     MD
 
     event_description do
@@ -149,7 +157,10 @@ module Agents
       Array(in_url).each do |url|
         log "Fetching #{url}"
         response = faraday.get(url)
-        if response.success?
+        raise "Failed: #{response.inspect}" unless response.success?
+
+        interpolation_context.stack {
+          interpolation_context['_response_'] = ResponseDrop.new(response)
           body = response.body
           if (encoding = interpolated['force_encoding']).present?
             body = body.encode(Encoding::UTF_8, encoding)
@@ -195,9 +206,7 @@ module Agents
               create_event :payload => result
             end
           end
-        else
-          raise "Failed: #{response.inspect}"
-        end
+        }
       end
     rescue => e
       error e.message
@@ -344,5 +353,24 @@ module Agents
     rescue
       false
     end
+
+    # Wraps Faraday::Response
+    class ResponseDrop < LiquidDroppable::Drop
+      def headers
+        HeaderDrop.new(@object.headers)
+      end
+
+      # Integer value of HTTP status
+      def status
+        @object.status
+      end
+    end
+
+    # Wraps Faraday::Utilsa::Headers
+    class HeaderDrop < LiquidDroppable::Drop
+      def before_method(name)
+        @object[name.tr('_', '-')]
+      end
+    end
   end
 end
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 98bcafc7..045dcbe7 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -3,7 +3,11 @@ require 'spec_helper'
 describe Agents::WebsiteAgent do
   describe "checking without basic auth" do
     before do
-      stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
+      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")),
+                                           status: 200,
+                                           headers: {
+                                             'X-Status-Message' => 'OK'
+                                           })
       @valid_options = {
         'name' => "XKCD",
         'expected_update_period_in_days' => "2",
@@ -305,6 +309,17 @@ describe Agents::WebsiteAgent do
         event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language."
       end
 
+      it "should interpolate _response_" do
+        @valid_options['extract']['response_info'] =
+          @valid_options['extract']['url'].merge(
+            'value' => '"{{ "The reponse was " | append:_response_.status | append:" " | append:_response_.headers.X-Status-Message | append:"." }}"'
+          )
+        @checker.options = @valid_options
+        @checker.check
+        event = Event.last
+        event.payload['response_info'].should == 'The reponse was 200 OK.'
+      end
+
       describe "JSON" do
         it "works with paths" do
           json = {
@@ -490,6 +505,22 @@ fire: hot
           'to' => 'http://dynamic.xkcd.com/random/comic/',
         }
       end
+
+      it "should interpolate values from incoming event payload and _response_" do
+        @event.payload['title'] = 'XKCD'
+
+        lambda {
+          @valid_options['extract'] = {
+            'response_info' => @valid_options['extract']['url'].merge(
+              'value' => '{% capture sentence %}The reponse from {{title}} was {{_response_.status}} {{_response_.headers.X-Status-Message}}.{% endcapture %}{{sentence | to_xpath}}'
+            )
+          }
+          @checker.options = @valid_options
+          @checker.receive([@event])
+        }.should change { Event.count }.by(1)
+
+        Event.last.payload['response_info'].should == 'The reponse from XKCD was 200 OK.'
+      end
     end
   end