From c21bada22611663a2bc0fa0d33e1c7c0d87e8f1a Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 1 Sep 2014 23:17:19 +0900 Subject: [PATCH] WebsiteAgent: Provide a variable _response_ for interpolation. * `_response_.headers.Header-Name` is expanded to the value of header named _Header-Name_. * `_response_.status` is expanded to the HTTP status as integer, which is almost always 200, since currently the WebsiteAgent will only create an event when a request succeeds, and the request method is limited to GET. --- app/models/agents/website_agent.rb | 36 +++++++++++++++++++++--- spec/models/agents/website_agent_spec.rb | 33 +++++++++++++++++++++- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index fd4f48e3..cabab6ec 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -75,6 +75,14 @@ module Agents The `headers` field is optional. When present, it should be a hash of headers to send with the request. The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload. + + In Liquid templating, the following variable is available: + + * `_response_`: A response object with the following keys: + + * `status`: HTTP status as integer. (Almost always 200) + + * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insentitive to cases and -/_. MD event_description do @@ -149,7 +157,10 @@ module Agents Array(in_url).each do |url| log "Fetching #{url}" response = faraday.get(url) - if response.success? + raise "Failed: #{response.inspect}" unless response.success? + + interpolation_context.stack { + interpolation_context['_response_'] = ResponseDrop.new(response) body = response.body if (encoding = interpolated['force_encoding']).present? body = body.encode(Encoding::UTF_8, encoding) @@ -195,9 +206,7 @@ module Agents create_event :payload => result end end - else - raise "Failed: #{response.inspect}" - end + } end rescue => e error e.message @@ -344,5 +353,24 @@ module Agents rescue false end + + # Wraps Faraday::Response + class ResponseDrop < LiquidDroppable::Drop + def headers + HeaderDrop.new(@object.headers) + end + + # Integer value of HTTP status + def status + @object.status + end + end + + # Wraps Faraday::Utilsa::Headers + class HeaderDrop < LiquidDroppable::Drop + def before_method(name) + @object[name.tr('_', '-')] + end + end end end diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 98bcafc7..045dcbe7 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -3,7 +3,11 @@ require 'spec_helper' describe Agents::WebsiteAgent do describe "checking without basic auth" do before do - stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200) + stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), + status: 200, + headers: { + 'X-Status-Message' => 'OK' + }) @valid_options = { 'name' => "XKCD", 'expected_update_period_in_days' => "2", @@ -305,6 +309,17 @@ describe Agents::WebsiteAgent do event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language." end + it "should interpolate _response_" do + @valid_options['extract']['response_info'] = + @valid_options['extract']['url'].merge( + 'value' => '"{{ "The reponse was " | append:_response_.status | append:" " | append:_response_.headers.X-Status-Message | append:"." }}"' + ) + @checker.options = @valid_options + @checker.check + event = Event.last + event.payload['response_info'].should == 'The reponse was 200 OK.' + end + describe "JSON" do it "works with paths" do json = { @@ -490,6 +505,22 @@ fire: hot 'to' => 'http://dynamic.xkcd.com/random/comic/', } end + + it "should interpolate values from incoming event payload and _response_" do + @event.payload['title'] = 'XKCD' + + lambda { + @valid_options['extract'] = { + 'response_info' => @valid_options['extract']['url'].merge( + 'value' => '{% capture sentence %}The reponse from {{title}} was {{_response_.status}} {{_response_.headers.X-Status-Message}}.{% endcapture %}{{sentence | to_xpath}}' + ) + } + @checker.options = @valid_options + @checker.receive([@event]) + }.should change { Event.count }.by(1) + + Event.last.payload['response_info'].should == 'The reponse from XKCD was 200 OK.' + end end end