From b959d53f4180fba20d923cadb43b39c976919e22 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 14 Jan 2016 14:32:28 -0800 Subject: [PATCH] PostAgent can emit events --- app/models/agents/post_agent.rb | 27 ++++++++++++-- app/models/agents/website_agent.rb | 2 + spec/models/agents/post_agent_spec.rb | 53 ++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index 65082840..2f5388c5 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -2,8 +2,6 @@ module Agents class PostAgent < Agent include WebRequestConcern - cannot_create_events! - default_schedule "never" description <<-MD @@ -15,6 +13,9 @@ module Agents By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`). Change `content_type` to `json` to send JSON instead. Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`. + If `emit_events` is set to `true`, the server response will be emitted as an Event and can be fed to a WebsiteAgent for parsing (using its `data_from_event` and `type` options). No data processing + will be attempted by this Agent, so the "body" value will always be raw text. + Other Options: * `headers` - When present, it should be a hash of headers to send with the request. @@ -23,7 +24,17 @@ module Agents * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}"). MD - event_description "Does not produce events." + event_description <<-MD + Events look like this: + { + "status": 200, + "response_headers": { + "Content-Type": "text/html", + ... + }, + "body": "Some data..." + } + MD def default_options { @@ -56,6 +67,10 @@ module Agents errors.add(:base, "if provided, payload must be a hash") end + if options['emit_events'].present? && ![true, false, 'true', 'false'].include?(options['emit_events']) + errors.add(:base, "if provided, emit_events must be true or false") + end + unless %w[post get put delete patch].include?(method) errors.add(:base, "method must be 'post', 'get', 'put', 'delete', or 'patch'") end @@ -112,9 +127,13 @@ module Agents error "Invalid method '#{method}'" end - faraday.run_request(method.to_sym, url, body, headers) { |request| + response = faraday.run_request(method.to_sym, url, body, headers) { |request| request.params.update(params) if params } + + if boolify(interpolated['emit_events']) + create_event payload: { body: response.body, response_headers: response.headers, status: response.status } + end end end end diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 3e641287..74b7f623 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -59,6 +59,8 @@ module Agents "description": { "path": "results.data[*].description" } } + The `extract` option can be skipped for the JSON type, causing the full JSON response to be returned. + # Scraping Text When parsing text, each sub-hash should contain a `regexp` and `index`. Output text is matched against the regular expression repeatedly from the beginning through to the end, collecting a captured group specified by `index` in each match. Each index should be either an integer or a string name which corresponds to (?<name>...). For example, to parse lines of word: definition, the following should work: diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb index b5a4786d..80120560 100644 --- a/spec/models/agents/post_agent_spec.rb +++ b/spec/models/agents/post_agent_spec.rb @@ -50,7 +50,7 @@ describe Agents::PostAgent do raise "unexpected Content-Type: #{content_type}" end end - { status: 200, body: "ok" } + { status: 200, body: "a webpage!", headers: { 'Content-Type' => 'text/html' } } } end @@ -186,6 +186,40 @@ describe Agents::PostAgent do expect(@sent_requests[:get][0].data).to eq(@checker.options['payload'].to_query) end + + describe "emitting events" do + context "when emit_events is not set to true" do + it "does not emit events" do + expect { + @checker.check + }.not_to change { @checker.events.count } + end + end + + context "when emit_events is set to true" do + before do + @checker.options['emit_events'] = 'true' + @checker.save! + end + + it "emits the response status" do + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + expect(@checker.events.last.payload['status']).to eq 200 + end + + it "emits the body" do + @checker.check + expect(@checker.events.last.payload['body']).to eq 'a webpage!' + end + + it "emits the response headers" do + @checker.check + expect(@checker.events.last.payload['response_headers']).to eq({ 'Content-Type' => 'text/html' }) + end + end + end end describe "#working?" do @@ -286,5 +320,22 @@ describe Agents::PostAgent do @checker.options['headers'] = { "Authorization" => "foo bar" } expect(@checker).to be_valid end + + it "requires emit_events to be true or false, if present" do + @checker.options['emit_events'] = 'what?' + expect(@checker).not_to be_valid + + @checker.options['emit_events'] = '' + expect(@checker).to be_valid + + @checker.options['emit_events'] = 'true' + expect(@checker).to be_valid + + @checker.options['emit_events'] = 'false' + expect(@checker).to be_valid + + @checker.options['emit_events'] = true + expect(@checker).to be_valid + end end end