GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-17 12:21:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into fix_http_status_agent

Browse source
This commit is contained in:
Akinori MUSHA 2017-01-01 02:54:59 +09:00
commit fe3b43ba56
26 changed files with 1062 additions and 428 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.travis.yml
View file

@ -3,6 +3,7 @@ sudo: required
language: ruby
services:
- docker
- mysql
- postgresql
env:
global:
@ -37,9 +38,6 @@ rvm:
- 2.3.1
cache: bundler
bundler_args: --without development production
before_install:
- sudo apt-get -qq update
- sudo apt-get install -y mysql-server
script:
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi

18
Gemfile
View file

@ -3,6 +3,12 @@ source 'https://rubygems.org'
# Ruby 2.2.2 is the minimum requirement
ruby ['2.2.2', RUBY_VERSION].max
# Ensure github repositories are fetched using HTTPS
git_source(:github) do |repo_name|
repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
"https://github.com/#{repo_name}.git"
end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2')
# Load vendored dotenv gem and .env file
require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb')
GemfileHelper.load_dotenv do |dotenv_dir|
@ -50,7 +56,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn'
gem 'omniauth-twitter', '~> 1.2.1'
# Tumblr Agents
gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master' # '>= 0.8.5'
gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master', ref: '0c59b04e49f2a8c89860613b18cf4e8f978d8dc7' # '>= 0.8.5'
gem 'omniauth-tumblr', '~> 1.2'
# Dropbox Agents
@ -90,9 +96,9 @@ gem 'delayed_job', '~> 4.1.0'
gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails5'
gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
gem 'faraday', '~> 0.9'
gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
gem 'feedjira', '~> 2.0'
gem 'feedjira', '~> 2.1'
gem 'font-awesome-sass', '~> 4.3.2'
gem 'foreman', '~> 0.63.0'
gem 'geokit', '~> 1.8.4'
@ -103,7 +109,7 @@ gem 'jquery-rails', '~> 4.2.1'
gem 'huginn_agent', '~> 0.4.0'
gem 'json', '~> 1.8.1'
gem 'jsonpathv2', '~> 0.0.8'
gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable'
gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable', ref: 'abbf93d557208ee1d0b612c612cd079f86ed54f4'
gem 'kramdown', '~> 1.3.3'
gem 'liquid', '~> 3.0.3'
gem 'loofah', '~> 2.0'
@ -111,7 +117,7 @@ gem 'mini_magick'
gem 'multi_xml'
gem 'nokogiri'
gem 'omniauth', '~> 1.3.1'
gem 'rails', '~> 5.0.0.1'
gem 'rails', '~> 5.0.1'
gem 'rufus-scheduler', '~> 3.0.8', require: false
gem 'sass-rails', '~> 5.0.6'
gem 'select2-rails', '~> 3.5.4'
@ -128,7 +134,7 @@ group :development do
gem 'guard-rspec', '~> 4.6.4'
gem 'rack-livereload', '~> 0.3.16'
gem 'letter_opener_web', '~> 1.3.0'
gem 'web-console'
gem 'web-console', '>= 3.3.0'
gem 'capistrano', '~> 3.4.0'
gem 'capistrano-rails', '~> 1.1'

116
Gemfile.lock
View file

@ -1,6 +1,7 @@
GIT
remote: git://github.com/amatsuda/kaminari.git
remote: https://github.com/amatsuda/kaminari.git
revision: abbf93d557208ee1d0b612c612cd079f86ed54f4
ref: abbf93d557208ee1d0b612c612cd079f86ed54f4
branch: 0-17-stable
specs:
kaminari (0.17.0)
@ -8,7 +9,7 @@ GIT
activesupport (>= 3.0.0)
GIT
remote: git://github.com/cantino/twitter-stream.git
remote: https://github.com/cantino/twitter-stream.git
revision: f7e7edb0bae013bffabf3598e7147773d9fd370f
branch: huginn
specs:
@ -18,7 +19,7 @@ GIT
simple_oauth (~> 0.3.0)
GIT
remote: git://github.com/dsander/delayed_job_active_record.git
remote: https://github.com/dsander/delayed_job_active_record.git
revision: b314972ccc92e0e8b03b1589174d8fb9a82b3cd0
branch: rails5
specs:
@ -27,7 +28,7 @@ GIT
delayed_job (>= 3.0, < 5)
GIT
remote: git://github.com/dsander/weibo_2.git
remote: https://github.com/dsander/weibo_2.git
revision: e5b77f21a7e9a666b582c48e16b1e96fca198cf8
branch: master
specs:
@ -38,16 +39,17 @@ GIT
rest-client (~> 1.8)
GIT
remote: git://github.com/lostisland/faraday_middleware.git
revision: c5836ae55857272732b33eb0e0a98d60e995a376
remote: https://github.com/lostisland/faraday_middleware.git
revision: 59088da02940d0ee2010b2e3156343346767c31e
branch: master
specs:
faraday_middleware (0.10.0)
faraday (>= 0.7.4, < 0.10)
faraday (>= 0.7.4, < 1.0)
GIT
remote: git://github.com/tumblr/tumblr_client.git
remote: https://github.com/tumblr/tumblr_client.git
revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
ref: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
branch: master
specs:
tumblr_client (0.8.5)
@ -69,45 +71,45 @@ GEM
remote: https://rubygems.org/
specs:
ace-rails-ap (2.0.1)
actioncable (5.0.0.1)
actionpack (= 5.0.0.1)
actioncable (5.0.1)
actionpack (= 5.0.1)
nio4r (~> 1.2)
websocket-driver (~> 0.6.1)
actionmailer (5.0.0.1)
actionpack (= 5.0.0.1)
actionview (= 5.0.0.1)
activejob (= 5.0.0.1)
actionmailer (5.0.1)
actionpack (= 5.0.1)
actionview (= 5.0.1)
activejob (= 5.0.1)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
actionpack (5.0.0.1)
actionview (= 5.0.0.1)
activesupport (= 5.0.0.1)
actionpack (5.0.1)
actionview (= 5.0.1)
activesupport (= 5.0.1)
rack (~> 2.0)
rack-test (~> 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
actionview (5.0.0.1)
activesupport (= 5.0.0.1)
actionview (5.0.1)
activesupport (= 5.0.1)
builder (~> 3.1)
erubis (~> 2.7.0)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
activejob (5.0.0.1)
activesupport (= 5.0.0.1)
activejob (5.0.1)
activesupport (= 5.0.1)
globalid (>= 0.3.6)
activemodel (5.0.0.1)
activesupport (= 5.0.0.1)
activerecord (5.0.0.1)
activemodel (= 5.0.0.1)
activesupport (= 5.0.0.1)
activemodel (5.0.1)
activesupport (= 5.0.1)
activerecord (5.0.1)
activemodel (= 5.0.1)
activesupport (= 5.0.1)
arel (~> 7.0)
activesupport (5.0.0.1)
activesupport (5.0.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.3.8)
arel (7.1.1)
arel (7.1.4)
autoparse (0.3.3)
addressable (>= 2.3.1)
extlib (>= 0.9.15)
@ -157,7 +159,7 @@ GEM
execjs
coffee-script-source (1.10.0)
colorize (0.7.7)
concurrent-ruby (1.0.2)
concurrent-ruby (1.0.3)
cookiejar (0.3.2)
coveralls (0.7.12)
multi_json (~> 1.10)
@ -214,11 +216,11 @@ GEM
extlib (0.9.16)
faraday (0.9.2)
multipart-post (>= 1.2, < 3)
feedjira (2.0.0)
faraday (~> 0.9)
faraday_middleware (~> 0.9)
loofah (~> 2.0)
sax-machine (~> 1.0)
feedjira (2.1.0)
faraday (>= 0.9)
faraday_middleware (>= 0.9)
loofah (>= 2.0)
sax-machine (>= 1.0)
ffi (1.9.10)
font-awesome-sass (4.3.2.1)
sass (~> 3.2)
@ -327,7 +329,7 @@ GEM
mimemagic (0.3.1)
mini_magick (4.2.3)
mini_portile2 (2.1.0)
minitest (5.9.0)
minitest (5.10.1)
mqtt (0.3.1)
multi_json (1.12.1)
multi_xml (0.5.5)
@ -402,17 +404,17 @@ GEM
rack
rack-test (0.6.3)
rack (>= 1.0)
rails (5.0.0.1)
actioncable (= 5.0.0.1)
actionmailer (= 5.0.0.1)
actionpack (= 5.0.0.1)
actionview (= 5.0.0.1)
activejob (= 5.0.0.1)
activemodel (= 5.0.0.1)
activerecord (= 5.0.0.1)
activesupport (= 5.0.0.1)
rails (5.0.1)
actioncable (= 5.0.1)
actionmailer (= 5.0.1)
actionpack (= 5.0.1)
actionview (= 5.0.1)
activejob (= 5.0.1)
activemodel (= 5.0.1)
activerecord (= 5.0.1)
activesupport (= 5.0.1)
bundler (>= 1.3.0, < 2.0)
railties (= 5.0.0.1)
railties (= 5.0.1)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.1)
actionpack (~> 5.x)
@ -423,14 +425,14 @@ GEM
nokogiri (~> 1.6.0)
rails-html-sanitizer (1.0.3)
loofah (~> 2.0)
railties (5.0.0.1)
actionpack (= 5.0.0.1)
activesupport (= 5.0.0.1)
railties (5.0.1)
actionpack (= 5.0.1)
activesupport (= 5.0.1)
method_source
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
raindrops (0.17.0)
rake (11.2.2)
rake (12.0.0)
rb-fsevent (0.9.7)
rb-inotify (0.9.5)
ffi (>= 0.5.0)
@ -514,7 +516,7 @@ GEM
spring-watcher-listen (2.0.0)
listen (>= 2.7, < 4.0)
spring (~> 1.2)
sprockets (3.7.0)
sprockets (3.7.1)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.0)
@ -531,7 +533,7 @@ GEM
therubyracer (0.12.2)
libv8 (~> 3.16.14.0)
ref
thor (0.19.1)
thor (0.19.4)
thread_safe (0.3.5)
tilt (2.0.5)
tins (1.10.1)
@ -617,9 +619,9 @@ DEPENDENCIES
dropbox-api
em-http-request (~> 1.1.2)
evernote_oauth
faraday (~> 0.9.0)
faraday (~> 0.9)
faraday_middleware!
feedjira (~> 2.0)
feedjira (~> 2.1)
ffi (>= 1.9.4)
font-awesome-sass (~> 4.3.2)
forecast_io (~> 2.0.0)
@ -663,7 +665,7 @@ DEPENDENCIES
pry-byebug
pry-rails
rack-livereload (~> 0.3.16)
rails (~> 5.0.0.1)
rails (~> 5.0.1)
rails-controller-testing
rb-kqueue (>= 0.2)
rr
@ -693,14 +695,14 @@ DEPENDENCIES
uglifier (~> 2.7.2)
unicorn (~> 5.1.0)
vcr
web-console
web-console (>= 3.3.0)
webmock (~> 1.17.4)
weibo_2!
wunderground (~> 1.2.0)
xmpp4r (~> 0.5.6)
RUBY VERSION
ruby 2.3.1p112
ruby 2.3.3p222
BUNDLED WITH
1.13.6
1.13.7

7
app/concerns/liquid_interpolatable.rb
View file

@ -129,10 +129,11 @@ module LiquidInterpolatable
# userinfo, host, port, registry, path, opaque, query, and
# fragment.
def to_uri(uri, base_uri = nil)
if base_uri
Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
else
case base_uri
when nil, ''
Utils.normalize_uri(uri.to_s)
else
Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
end
rescue URI::Error
nil

8
app/models/agent.rb
View file

@ -370,7 +370,7 @@ class Agent < ActiveRecord::Base
def receive!(options={})
Agent.transaction do
scope = Agent.
select("agents.id AS receiver_agent_id, events.id AS event_id").
select("agents.id AS receiver_agent_id, sources.type AS source_agent_type, agents.type AS receiver_agent_type, events.id AS event_id").
joins("JOIN links ON (links.receiver_id = agents.id)").
joins("JOIN agents AS sources ON (links.source_id = sources.id)").
joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)").
@ -379,10 +379,11 @@ class Agent < ActiveRecord::Base
scope = scope.where("agents.id in (?)", options[:only_receivers])
end
sql = scope.to_sql()
sql = scope.to_sql
agents_to_events = {}
Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id|
Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_type, receiver_agent_type, event_id|
next unless const_defined?(source_agent_type) && const_defined?(receiver_agent_type)
agents_to_events[receiver_agent_id.to_i] ||= []
agents_to_events[receiver_agent_id.to_i] << event_id
end
@ -417,6 +418,7 @@ class Agent < ActiveRecord::Base
return if schedule == 'never'
types = where(:schedule => schedule).group(:type).pluck(:type)
types.each do |type|
next unless const_defined?(type)
type.constantize.bulk_check(schedule)
end
end

129
app/models/agents/beeper_agent.rb
View file

@ -1,129 +0,0 @@
module Agents
class BeeperAgent < Agent
cannot_be_scheduled!
cannot_create_events!
no_bulk_receive!
description <<-MD
Beeper agent sends messages to Beeper app on your mobile device via Push notifications.
You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io)
You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID
Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`.
Depending on message type you have to provide additional fields:
##### Message
* `text` – **required**
##### Image
* `image` – **required** (Image URL or Base64-encoded image)
* `text` – optional
##### Event
* `text` – **required**
* `start_time` – **required** (Corresponding to ISO 8601)
* `end_time` – optional (Corresponding to ISO 8601)
##### Location
* `latitude` – **required**
* `longitude` – **required**
* `text` – optional
##### Task
* `text` – **required**
You can see additional documentation at [Beeper website](https://beeper.io/docs)
MD
BASE_URL = 'https://api.beeper.io/api'
TYPE_ATTRIBUTES = {
'message' => %w(text),
'image' => %w(text image),
'event' => %w(text start_time end_time),
'location' => %w(text latitude longitude),
'task' => %w(text)
}
MESSAGE_TYPES = TYPE_ATTRIBUTES.keys
TYPE_REQUIRED_ATTRIBUTES = {
'message' => %w(text),
'image' => %w(image),
'event' => %w(text start_time),
'location' => %w(latitude longitude),
'task' => %w(text)
}
def default_options
{
'type' => 'message',
'app_id' => '',
'api_key' => '',
'sender_id' => '',
'phone' => '',
'text' => '{{title}}'
}
end
def validate_options
%w(app_id api_key sender_id type).each do |attr|
errors.add(:base, "you need to specify a #{attr}") if options[attr].blank?
end
if options['type'].in?(MESSAGE_TYPES)
required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']]
if required_attributes.any? { |attr| options[attr].blank? }
errors.add(:base, "you need to specify a #{required_attributes.join(', ')}")
end
else
errors.add(:base, 'you need to specify a valid message type')
end
unless options['group_id'].blank? ^ options['phone'].blank?
errors.add(:base, 'you need to specify a phone or group_id')
end
end
def working?
received_event_without_error? && !recent_error_logs?
end
def receive(incoming_events)
incoming_events.each do |event|
send_message(event)
end
end
def send_message(event)
mo = interpolated(event)
begin
response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers)
error(response.body) if response.code != 201
rescue HTTParty::Error => e
error(e.message)
end
end
private
def headers
{
'X-Beeper-Application-Id' => options['app_id'],
'X-Beeper-REST-API-Key' => options['api_key'],
'Content-Type' => 'application/json'
}
end
def payload_for(mo)
mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json
end
def endpoint_for(type)
"#{BASE_URL}/#{type}s.json"
end
end
end

6
app/models/agents/google_flights_agent.rb
View file

@ -60,6 +60,7 @@ module Agents
'seniorCount'=> 0,
'return_date' => '2016-04-18',
'roundtrip' => true,
'preferredCabin' => 'COACH',
'solutions'=> 3
}
end
@ -69,6 +70,7 @@ module Agents
form_configurable :origin, type: :string
form_configurable :destination, type: :string
form_configurable :date, type: :string
form_configurable :preferredCabin, type: :array, values: %w(COACH PREMIUM_COACH BUSINESS FIRST)
form_configurable :childCount
form_configurable :infantInSeatCount
form_configurable :infantInLapCount
@ -101,9 +103,9 @@ module Agents
def post_params
if round_trip?
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s } ], :solutions=> interpolated["solutions"]}}
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s} ], :solutions=> interpolated["solutions"]}}
else
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }], :solutions=> interpolated["solutions"]}}
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }], :solutions=> interpolated["solutions"]}}
end
end

161
app/models/agents/phantom_js_cloud_agent.rb Normal file
View file

@ -0,0 +1,161 @@
require 'json'
require 'uri'
module Agents
class PhantomJsCloudAgent < Agent
include ERB::Util
include FormConfigurable
include WebRequestConcern
can_dry_run!
default_schedule 'every_12h'
description <<-MD
This Agent generates [PhantomJs Cloud](https://phantomjscloud.com/) URLs that can be used to render JavaScript-heavy webpages for content extraction.
URLs generated by this Agent are formulated in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
The generated URLs can then be supplied to a Website Agent to fetch and parse the content.
[Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
Please see the [Huginn Wiki for more info](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud).
Options:
* `Api key` - PhantomJs Cloud API Key credential stored in Huginn
* `Url` - The url to render
* `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`)
* `Render type` - Render as html or plain text without html tags (default: `html`)
* `Output as json` - Return the page conents and metadata as a JSON object (default: `false`)
* `Ignore images` - Skip loading of inlined images (default: `false`)
* `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`)
* `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading.
This is useful in case there are any AJAX requests or animations that need to finish up.
This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
MD
event_description <<-MD
Events look like this:
{
"url": "..."
}
MD
def default_options
{
'mode' => 'clean',
'url' => 'http://xkcd.com',
'render_type' => 'html',
'output_as_json' => false,
'ignore_images' => false,
'user_agent' => self.class.default_user_agent,
'wait_interval' => '1000'
}
end
form_configurable :mode, type: :array, values: ['clean', 'merge']
form_configurable :api_key, roles: :completable
form_configurable :url
form_configurable :render_type, type: :array, values: ['html', 'plainText']
form_configurable :output_as_json, type: :boolean
form_configurable :ignore_images, type: :boolean
form_configurable :user_agent, type: :text
form_configurable :wait_interval
def mode
interpolated['mode'].presence || default_options['mode']
end
def render_type
interpolated['render_type'].presence || default_options['render_type']
end
def output_as_json
boolify(interpolated['output_as_json'].presence ||
default_options['output_as_json'])
end
def ignore_images
boolify(interpolated['ignore_images'].presence ||
default_options['ignore_images'])
end
def user_agent
interpolated['user_agent'].presence || self.class.default_user_agent
end
def wait_interval
interpolated['wait_interval'].presence || default_options['wait_interval']
end
def page_request_settings
prs = {}
prs[:ignoreImages] = ignore_images if ignore_images
prs[:userAgent] = user_agent if user_agent.present?
if wait_interval != default_options['wait_interval']
prs[:wait_interval] = wait_interval
end
prs
end
def build_phantom_url(interpolated)
api_key = interpolated[:api_key]
page_request_hash = {
url: interpolated[:url],
renderType: render_type
}
page_request_hash[:outputAsJson] = output_as_json if output_as_json
page_request_settings_hash = page_request_settings
if page_request_settings_hash.any?
page_request_hash[:requestSettings] = page_request_settings_hash
end
request = page_request_hash.to_json
log "Generated request: #{request}"
encoded = url_encode(request)
"https://phantomjscloud.com/api/browser/v2/#{api_key}/?request=#{encoded}"
end
def check
phantom_url = build_phantom_url(interpolated)
create_event payload: { 'url' => phantom_url }
end
def receive(incoming_events)
incoming_events.each do |event|
interpolate_with(event) do
existing_payload = interpolated['mode'].to_s == 'merge' ? event.payload : {}
phantom_url = build_phantom_url(interpolated)
result = { 'url' => phantom_url }
create_event payload: existing_payload.merge(result)
end
end
end
def complete_api_key
user.user_credentials.map { |c| { text: c.credential_name, id: "{% credential #{c.credential_name} %}" } }
end
def working?
!recent_error_logs? || received_event_without_error?
end
def validate_options
# Check for required fields
errors.add(:base, 'Url is required') unless options['url'].present?
errors.add(:base, 'API key (credential) is required') unless options['api_key'].present?
end
end
end

85
app/models/agents/rss_agent.rb
View file

@ -66,6 +66,22 @@ module Agents
"copyright": "...",
"icon": "http://example.com/icon.png",
"authors": [ "..." ],
"itunes_block": "no",
"itunes_categories": [
"Technology", "Gadgets",
"TV & Film",
"Arts", "Food"
],
"itunes_complete": "yes",
"itunes_explicit": "yes",
"itunes_image": "http://...",
"itunes_new_feed_url": "http://...",
"itunes_owners": [ "John Doe <john.doe@example.com>" ],
"itunes_subtitle": "...",
"itunes_summary": "...",
"language": "en-US",
"date_published": "2014-09-11T01:30:00-07:00",
"last_updated": "2014-09-11T01:30:00-07:00"
},
@ -84,6 +100,16 @@ module Agents
"enclosure": {
"url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789"
},
"itunes_block": "no",
"itunes_closed_captioned": "yes",
"itunes_duration": "04:34",
"itunes_explicit": "yes",
"itunes_image": "http://...",
"itunes_order": "1",
"itunes_subtitle": "...",
"itunes_summary": "...",
"date_published": "2014-09-11T01:30:00-0700",
"last_updated": "2014-09-11T01:30:00-0700"
}
@ -91,7 +117,8 @@ module Agents
Some notes:
- The `feed` key is present only if `include_feed_info` is set to true.
- Each element in `authors` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
- The keys starting with `itunes_`, and `language` are only present when the feed is a podcast. See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details.
- Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
- Timestamps are converted to the ISO 8601 format.
MD
@ -179,7 +206,7 @@ module Agents
else
# Encoding is already known, so do not let the parser detect
# it from the XML declaration in the content.
body.sub!(/(\A\u{FEFF}?\s*<\?xml(?:\s+\w+\s*=\s*(['"]).*?\2)*)\s+encoding\s*=\s*(['"]).*?\3/, '\\1')
body.sub!(/(?<noenc>\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?<av>\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g<av>/, '\\k<noenc>')
end
body
end
@ -206,9 +233,40 @@ module Agents
authors: feed.authors,
date_published: feed.date_published,
last_updated: feed.last_updated,
**itunes_feed_data(feed)
}
end
def itunes_feed_data(feed)
data = {}
case feed
when Feedjira::Parser::ITunesRSS
%i[
itunes_block
itunes_categories
itunes_complete
itunes_explicit
itunes_image
itunes_new_feed_url
itunes_owners
itunes_subtitle
itunes_summary
language
].each { |attr|
if value = feed.try(attr).presence
data[attr] =
case attr
when :itunes_summary
clean_fragment(value)
else
value
end
end
}
end
data
end
def entry_data(entry)
{
id: entry.id,
@ -224,9 +282,32 @@ module Agents
categories: Array(entry.try(:categories)),
date_published: entry.date_published,
last_updated: entry.last_updated,
**itunes_entry_data(entry)
}
end
def itunes_entry_data(entry)
data = {}
case entry
when Feedjira::Parser::ITunesRSSItem
%i[
itunes_block
itunes_closed_captioned
itunes_duration
itunes_explicit
itunes_image
itunes_order
itunes_subtitle
itunes_summary
].each { |attr|
if value = entry.try(attr).presence
data[attr] = value
end
}
end
data
end
def feed_to_events(feed)
payload_base = {}

127
app/models/agents/website_agent.rb
View file

@ -27,8 +27,6 @@ module Agents
* Alternatively, set `data_from_event` to a Liquid template to use data directly without fetching any URL. (For example, set it to `{{ html }}` to use HTML contained in the `html` key of the incoming Event.)
* If you specify `merge` for the `mode` option, Huginn will retain the old payload and update it with new values.
If a created Event has a key named `url` containing a relative URL, it is automatically resolved using the request URL as base.
# Supported Document Types
The `type` value can be `xml`, `html`, `json`, or `text`.
@ -37,6 +35,8 @@ module Agents
Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor except when it has `repeat` set to true. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
For extractors with `hidden` set to true, they will be excluded from the payloads of events created by the Agent, but can be used and interpolated in the `template` option explained below.
For extractors with `repeat` set to true, their first matches will be included in all extracts. This is useful such as when you want to include the title of a page in all events created from the page.
# Scraping HTML and XML
@ -116,11 +116,10 @@ module Agents
Set `http_success_codes` to an array of status codes (e.g., `[404, 422]`) to treat HTTP response codes beyond 200 as successes.
If a `template` option is given, it is used as a Liquid template for each event created by this Agent, instead of directly emitting the results of extraction as events. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
"template": {
"url": "{{ url }}",
"title": "{{ title }}",
"url": "{{ url | to_uri: _request_.url }}",
"description": "{{ body_text }}",
"last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
}
@ -129,17 +128,17 @@ module Agents
# Liquid Templating
In Liquid templating, the following variables are available except when invoked by `data_from_event`:
In Liquid templating, the following variables are available:
* `_url_`: The URL specified to fetch the content from.
* `_url_`: The URL specified to fetch the content from. When parsing `data_from_event`, this is not set.
* `_response_`: A response object with the following keys:
* `status`: HTTP status as integer. (Almost always 200)
* `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_.
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event, if it is a hash.
* `url`: The final URL of the fetched page, following redirects. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
* `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
# Ordering Events
@ -159,7 +158,11 @@ module Agents
end
def event_keys
(options['template'].presence || options['extract']).try(:keys)
extract = options['extract'] or return nil
extract.each_with_object([]) { |(key, value), keys|
keys << key unless boolify(value['hidden'])
} | (options['template'].presence.try!(:keys) || [])
end
def working?
@ -362,6 +365,8 @@ module Agents
end
def handle_data(body, url, existing_payload)
# Beware, url may be a URI object, string or nil
doc = parse(body)
if extract_full_json?
@ -382,41 +387,18 @@ module Agents
extract_xml(doc)
end
num_tuples = output.each_value.inject(nil) { |num, value|
case size = value.size
when Float::INFINITY
num
when Integer
if num && num != size
raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
end
size
end
} or raise "At least one non-repeat key is required"
num_tuples = output.size or
raise "At least one non-repeat key is required"
old_events = previous_payloads num_tuples
template = options['template'].presence
num_tuples.times.zip(*output.values) do |index, *values|
extracted = output.each_key.lazy.zip(values).to_h
output.each do |extracted|
result = extracted.except(*output.hidden_keys)
result =
if template
interpolate_with(extracted) do
interpolate_options(template)
end
else
extracted
end
# url may be URI, string or nil
if (payload_url = result['url'].presence) && (url = url.presence)
begin
result['url'] = (Utils.normalize_uri(url) + Utils.normalize_uri(payload_url)).to_s
rescue URI::Error
error "Cannot resolve url: <#{payload_url}> on <#{url}>"
end
if template
result.update(interpolate_options(template, extracted))
end
if store_payload!(old_events, result)
@ -460,7 +442,10 @@ module Agents
end
def handle_event_data(data, event, existing_payload)
handle_data(data, event.payload['url'], existing_payload)
interpolation_context.stack {
interpolation_context['_response_'] = ResponseFromEventDrop.new(event)
handle_data(data, event.payload['url'].presence, existing_payload)
}
rescue => e
error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event
end
@ -528,7 +513,7 @@ module Agents
end
def extract_each(&block)
interpolated['extract'].each_with_object({}) { |(name, extraction_details), output|
interpolated['extract'].each_with_object(Output.new) { |(name, extraction_details), output|
if boolify(extraction_details['repeat'])
values = Repeater.new { |repeater|
block.call(extraction_details, repeater)
@ -538,7 +523,13 @@ module Agents
block.call(extraction_details, values)
end
log "Values extracted: #{values}"
output[name] = values
begin
output[name] = values
rescue UnevenSizeError
raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
else
output.hidden_keys << name if boolify(extraction_details['hidden'])
end
}
end
@ -617,6 +608,38 @@ module Agents
false
end
class UnevenSizeError < ArgumentError
end
class Output
def initialize
@hash = {}
@size = nil
@hidden_keys = []
end
attr_reader :size
attr_reader :hidden_keys
def []=(key, value)
case size = value.size
when Integer
if @size && @size != size
raise UnevenSizeError, 'got an uneven size'
end
@size = size
end
@hash[key] = value
end
def each
@size.times.zip(*@hash.values) do |index, *values|
yield @hash.each_key.lazy.zip(values).to_h
end
end
end
class Repeater < Enumerator
# Repeater.new { |y|
# # ...
@ -659,6 +682,24 @@ module Agents
end
end
class ResponseFromEventDrop < LiquidDroppable::Drop
def headers
headers = Faraday::Utils::Headers.from(@object.payload[:headers]) rescue {}
HeaderDrop.new(headers)
end
# Integer value of HTTP status
def status
Integer(@object.payload[:status]) rescue nil
end
# The URL
def url
@object.payload[:url]
end
end
# Wraps Faraday::Utils::Headers
class HeaderDrop < LiquidDroppable::Drop
def before_method(name)

8
app/views/application/undefined_agents.html.erb
View file

@ -19,20 +19,20 @@
</ul>
<br/>
<p>
The issue most probably occurred because of one or more of the following reasons:
This issue probably occurred for one or more of the following reasons:
</p>
<ul>
<li>If the respective Agent is distributed as part of the Huginn application codebase, it may have been removed or moved to an Agent gem. Please see <a href="https://github.com/cantino/huginn/wiki/Dealing-with-Deleted-Agent-Types" target="_blank">this wiki page for more information</a>.</li>
<li>If the respective Agent is distributed as a Ruby gem, it might have been removed from the <code>ADDITIONAL_GEMS</code> environment setting.</li>
<li>If the respective Agent is distributed as part of the Huginn application codebase, it might have been removed from that either on purpose (because the Agent has been deprecated or been moved to an Agent gem) or accidentally. Please check if the Agent(s) in question are available in your Huginn codebase under the path <code>app/models/agents/</code>.</li>
</ul>
<br/>
<p>
You can fix the issue by adding the Agent(s) back to the application codebase by
You can fix this issue by:
</p>
<ul>
<li>deleting the respective Agent(s) from the database using the button below.</li>
<li>adding the respective Agent(s) to the the <code>ADDITIONAL_GEMS</code> environment setting. Please see <a href="https://github.com/cantino/huginn_agent" target="_blank">https://github.com/cantino/huginn_agent</a> for documentation on how to properly set it.</li>
<li>adding the respective Agent(s) code to the Huginn application codebase (in case it was deleted accidentally).</li>
<li>deleting the respective Agent(s) from the database using the button below.</li>
</ul>
<br/>
<div class="btn-group">

2
app/views/layouts/application.html.erb
View file

@ -52,7 +52,7 @@
window.agentPaths = {};
window.agentNames = [];
<% if current_user.present? -%>
var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| m[a.first] = agent_path(a.last); m }) %>;
var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| next if a.last.nil?; m[a.first] = agent_path(a.last); m }) %>;
var myScenarios = <%= Utils.jsonify(current_user.scenarios.pluck(:name, :id).inject({}) {|m, s| m[s.first + " Scenario"] = scenario_path(s.last); m }) %>;
$.extend(window.agentPaths, myAgents);
$.extend(window.agentPaths, myScenarios);

36
db/migrate/20161124061256_convert_website_agent_template_for_merge.rb Normal file
View file

@ -0,0 +1,36 @@
class ConvertWebsiteAgentTemplateForMerge < ActiveRecord::Migration[5.0]
def up
Agents::WebsiteAgent.find_each do |agent|
extract = agent.options['extract'].presence
template = agent.options['template'].presence
next unless extract.is_a?(Hash) && template.is_a?(Hash)
(extract.keys - template.keys).each do |key|
extract[key]['hidden'] = true
end
template.delete_if { |key, value|
extract.key?(key) &&
value.match(/\A\{\{\s*#{Regexp.quote(key)}\s*\}\}\z/)
}
agent.save!(validate: false)
end
end
def down
Agents::WebsiteAgent.find_each do |agent|
extract = agent.options['extract'].presence
template = agent.options['template'].presence
next unless extract.is_a?(Hash) && template.is_a?(Hash)
(extract.keys - template.keys).each do |key|
unless extract[key].delete('hidden').in?([true, 'true'])
template[key] = "{{ #{key} }}"
end
end
agent.save!(validate: false)
end
end
end

16
db/migrate/20161124065838_add_templates_to_resolve_url.rb Normal file
View file

@ -0,0 +1,16 @@
class AddTemplatesToResolveUrl < ActiveRecord::Migration[5.0]
def up
Agents::WebsiteAgent.find_each do |agent|
if agent.event_keys.try!(:include?, 'url')
agent.options['template'] = (agent.options['template'] || {}).tap { |template|
template['url'] ||= '{{ url | to_uri: _response_.url }}'
}
agent.save!(validate: false)
end
end
end
def down
# No need to revert
end
end

17
lib/feedjira_extension.rb
View file

@ -57,6 +57,13 @@ module FeedjiraExtension
value :content
end
class ITunesRssOwner < Author
include SAXMachine
element :'itunes:name', as: :name
element :'itunes:email', as: :email
end
class Enclosure
include SAXMachine
@ -290,6 +297,16 @@ module FeedjiraExtension
def copyright
@copyright || super
end
if /ITunes/ === name
sax_config.collection_elements['itunes:owner'].clear
elements :"itunes:owner", as: :_itunes_owners, class: ITunesRssOwner
private :_itunes_owners
def itunes_owners
_itunes_owners.reject(&:empty?)
end
end
end
sax_config.collection_elements.each_value do |collection_elements|

9
spec/concerns/liquid_interpolatable_spec.rb
View file

@ -119,6 +119,15 @@ describe LiquidInterpolatable::Filters do
@agent.interpolation_context['s'] = 'foo/index.html'
expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
end
it 'should normalize a URI value if an empty base URI is given' do
@agent.options['foo'] = '{{ u | to_uri: b }}'
@agent.interpolation_context['u'] = "\u{3042}"
@agent.interpolation_context['b'] = ""
expect(@agent.interpolated['foo']).to eq('%E3%81%82')
@agent.interpolation_context['b'] = nil
expect(@agent.interpolated['foo']).to eq('%E3%81%82')
end
end
describe 'uri_expand' do

76
spec/data_fixtures/podcast.rss Normal file
View file

@ -0,0 +1,76 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>All About Everything</title>
<link>http://www.example.com/podcasts/everything/index.html</link>
<language>en-us</language>
<copyright>&#x2117; &amp; &#xA9; 2014 John Doe &amp; Family</copyright>
<itunes:subtitle>A show about everything</itunes:subtitle>
<itunes:author>John Doe</itunes:author>
<itunes:summary>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</itunes:summary>
<description>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</description>
<itunes:owner>
<itunes:name>John Doe</itunes:name>
<itunes:email>john.doe@example.com</itunes:email>
</itunes:owner>
<itunes:complete>yes</itunes:complete>
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg"/>
<itunes:category text="Technology">
<itunes:category text="Gadgets"/>
</itunes:category>
<itunes:category text="TV &amp; Film"/>
<itunes:category text="Arts">
<itunes:category text="Food"/>
</itunes:category>
<itunes:explicit>no</itunes:explicit>
<item>
<title>Shake Shake Shake Your Spices</title>
<itunes:author>John Doe</itunes:author>
<itunes:subtitle>A short primer on table spices</itunes:subtitle>
<itunes:summary><![CDATA[This week we talk about <a href="https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!]]></itunes:summary>
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg"/>
<enclosure length="8727310" type="audio/x-m4a" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a"/>
<guid>http://example.com/podcasts/archive/aae20140615.m4a</guid>
<pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
<itunes:duration>07:04</itunes:duration>
<itunes:explicit>no</itunes:explicit>
</item>
<item>
<title>Socket Wrench Shootout</title>
<itunes:author>Jane Doe</itunes:author>
<itunes:subtitle>Comparing socket wrenches is fun!</itunes:subtitle>
<itunes:summary>This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here.</itunes:summary>
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg"/>
<enclosure length="5650889" type="video/mp4" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4"/>
<guid>http://example.com/podcasts/archive/aae20140608.mp4</guid>
<pubDate>Wed, 09 Mar 2016 13:00:00 EST</pubDate>
<itunes:duration>04:34</itunes:duration>
<itunes:explicit>no</itunes:explicit>
</item>
<item>
<title>The Best Chili</title>
<itunes:author>Jane Doe</itunes:author>
<itunes:subtitle>Jane and Eric</itunes:subtitle>
<itunes:summary>This week we talk about the best Chili in the world. Which chili is better?</itunes:summary>
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg"/>
<enclosure length="5650889" type="video/x-m4v" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v"/>
<guid>http://example.com/podcasts/archive/aae20140697.m4v</guid>
<pubDate>Thu, 10 Mar 2016 02:00:00 -0700</pubDate>
<itunes:duration>04:34</itunes:duration>
<itunes:explicit>no</itunes:explicit>
<itunes:isClosedCaptioned>Yes</itunes:isClosedCaptioned>
</item>
<item>
<title>Red,Whine, &amp; Blue</title>
<itunes:author>Various</itunes:author>
<itunes:subtitle>Red + Blue != Purple</itunes:subtitle>
<itunes:summary>This week we talk about surviving in a Red state if you are a Blue person. Or vice versa.</itunes:summary>
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg"/>
<enclosure length="498537" type="audio/mpeg" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3"/>
<guid>http://example.com/podcasts/archive/aae20140601.mp3</guid>
<pubDate>Fri, 11 Mar 2016 01:15:00 +3000</pubDate>
<itunes:duration>03:59</itunes:duration>
<itunes:explicit>no</itunes:explicit>
</item>
</channel>
</rss>

9
spec/features/create_an_agent_spec.rb
View file

@ -38,7 +38,7 @@ describe "Creating a new agent", js: true do
visit new_agent_path
end
it "shows all options for agents that can be scheduled, create and receive events" do
select2("Website Agent", from: "Type")
select2("Website Agent scrapes", from: "Type")
expect(page).not_to have_content('This type of Agent cannot create events.')
end
@ -49,9 +49,8 @@ describe "Creating a new agent", js: true do
end
it "allows to click on on the agent name in select2 tags" do
agent = agents(:bob_weather_agent)
visit new_agent_path
select2("Website Agent", from: "Type")
select2("Website Agent scrapes", from: "Type")
select2("SF Weather", from: 'Sources')
click_on "SF Weather"
expect(page).to have_content "Editing your WeatherAgent"
@ -63,7 +62,7 @@ describe "Creating a new agent", js: true do
end
it "does not send previously configured sources when the current agent does not support them" do
select2("Website Agent", from: "Type")
select2("Website Agent scrapes", from: "Type")
select2("SF Weather", from: 'Sources')
select2("Webhook Agent", from: "Type")
fill_in(:agent_name, with: "No sources")
@ -85,7 +84,7 @@ describe "Creating a new agent", js: true do
end
it "does not send previously configured receivers when the current agent does not support them" do
select2("Website Agent", from: "Type")
select2("Website Agent scrapes", from: "Type")
select2("ZKCD", from: 'Receivers')
select2("Email Agent", from: "Type")
fill_in(:agent_name, with: "No receivers")

92
spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb Normal file
View file

@ -0,0 +1,92 @@
load 'spec/rails_helper.rb'
load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
describe ConvertWebsiteAgentTemplateForMerge do
let :old_extract do
{
'url' => { 'css' => "#comic img", 'value' => "@src" },
'title' => { 'css' => "#comic img", 'value' => "@alt" },
'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
}
end
let :new_extract do
{
'url' => { 'css' => "#comic img", 'value' => "@src" },
'title' => { 'css' => "#comic img", 'value' => "@alt" },
'hovertext' => { 'css' => "#comic img", 'value' => "@title", 'hidden' => true }
}
end
let :reverted_extract do
old_extract
end
let :old_template do
{
'url' => '{{url}}',
'title' => '{{ title }}',
'description' => '{{ hovertext }}',
'comment' => '{{ comment }}'
}
end
let :new_template do
{
'description' => '{{ hovertext }}',
'comment' => '{{ comment }}'
}
end
let :reverted_template do
old_template.merge('url' => '{{ url }}')
end
let :valid_options do
{
'name' => "XKCD",
'expected_update_period_in_days' => "2",
'type' => "html",
'url' => "{{ url | default: 'http://xkcd.com/' }}",
'mode' => 'on_change',
'extract' => old_extract,
'template' => old_template
}
end
let :agent do
Agents::WebsiteAgent.create!(
user: users(:bob),
name: "xkcd",
options: valid_options,
keep_events_for: 2.days
)
end
describe 'up' do
it 'should update extract and template options for an existing WebsiteAgent' do
expect(agent.options).to include('extract' => old_extract,
'template' => old_template)
ConvertWebsiteAgentTemplateForMerge.new.up
agent.reload
expect(agent.options).to include('extract' => new_extract,
'template' => new_template)
end
end
describe 'down' do
let :valid_options do
super().merge('extract' => new_extract,
'template' => new_template)
end
it 'should revert extract and template options for an updated WebsiteAgent' do
expect(agent.options).to include('extract' => new_extract,
'template' => new_template)
ConvertWebsiteAgentTemplateForMerge.new.down
agent.reload
expect(agent.options).to include('extract' => reverted_extract,
'template' => reverted_template)
end
end
end

39
spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb Normal file
View file

@ -0,0 +1,39 @@
load 'spec/rails_helper.rb'
load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
describe AddTemplatesToResolveUrl do
let :valid_options do
{
'name' => "XKCD",
'expected_update_period_in_days' => "2",
'type' => "html",
'url' => "http://xkcd.com",
'mode' => 'on_change',
'extract' => {
'url' => { 'css' => "#comic img", 'value' => "@src" },
'title' => { 'css' => "#comic img", 'value' => "@alt" },
'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
}
}
end
let :agent do
Agents::WebsiteAgent.create!(
user: users(:bob),
name: "xkcd",
options: valid_options,
keep_events_for: 2.days
)
end
it 'should add a template for an existing WebsiteAgent with `url`' do
expect(agent.options).not_to include('template')
AddTemplatesToResolveUrl.new.up
agent.reload
expect(agent.options).to include(
'template' => {
'url' => '{{ url | to_uri: _response_.url }}'
}
)
end
end

33
spec/models/agent_spec.rb
View file

@ -74,6 +74,13 @@ describe Agent do
Agent.run_schedule("midnight")
end
it "ignores unknown types" do
Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
mock(Agents::WeatherAgent).bulk_check("midnight").once
mock(Agents::WebsiteAgent).bulk_check("midnight").once
Agent.run_schedule("midnight")
end
it "only runs agents with the given schedule" do
do_not_allow(Agents::WebsiteAgent).async_check
Agent.run_schedule("blah")
@ -283,13 +290,37 @@ describe Agent do
Agent.receive!
end
it "should not propogate to disabled Agents" do
it "should not propagate to disabled Agents" do
Agent.async_check(agents(:bob_weather_agent).id)
agents(:bob_rain_notifier_agent).update_attribute :disabled, true
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
Agent.receive!
end
it "should not propagate to Agents with unknown types" do
Agent.async_check(agents(:jane_weather_agent).id)
Agent.async_check(agents(:bob_weather_agent).id)
Agent.where(id: agents(:bob_rain_notifier_agent).id).update_all type: 'UnknownTypeAgent'
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
Agent.receive!
end
it "should not propagate from Agents with unknown types" do
Agent.async_check(agents(:jane_weather_agent).id)
Agent.async_check(agents(:bob_weather_agent).id)
Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
Agent.receive!
end
it "should log exceptions" do
mock.any_instance_of(Agents::TriggerAgent).receive(anything).once {
raise "foo"

145
spec/models/agents/beeper_agent_spec.rb
View file

@ -1,145 +0,0 @@
require 'rails_helper'
describe Agents::BeeperAgent do
let(:base_params) {
{
'type' => 'message',
'app_id' => 'some-app-id',
'api_key' => 'some-api-key',
'sender_id' => 'sender-id',
'phone' => '+111111111111',
'text' => 'Some text'
}
}
subject {
agent = described_class.new(name: 'beeper-agent', options: base_params)
agent.user = users(:jane)
agent.save! and return agent
}
context 'validation' do
it 'valid' do
expect(subject).to be_valid
end
[:type, :app_id, :api_key, :sender_id].each do |attr|
it "invalid without #{attr}" do
subject.options[attr] = nil
expect(subject).not_to be_valid
end
end
it 'invalid with group_id and phone' do
subject.options['group_id'] ='some-group-id'
expect(subject).not_to be_valid
end
context '#message' do
it 'requires text' do
subject.options[:text] = nil
expect(subject).not_to be_valid
end
end
context '#image' do
before(:each) do
subject.options[:type] = 'image'
end
it 'invalid without image' do
expect(subject).not_to be_valid
end
it 'valid with image' do
subject.options[:image] = 'some-url'
expect(subject).to be_valid
end
end
context '#event' do
before(:each) do
subject.options[:type] = 'event'
end
it 'invalid without start_time' do
expect(subject).not_to be_valid
end
it 'valid with start_time' do
subject.options[:start_time] = Time.now
expect(subject).to be_valid
end
end
context '#location' do
before(:each) do
subject.options[:type] = 'location'
end
it 'invalid without latitude and longitude' do
expect(subject).not_to be_valid
end
it 'valid with latitude and longitude' do
subject.options[:latitude] = 15.0
subject.options[:longitude] = 16.0
expect(subject).to be_valid
end
end
context '#task' do
before(:each) do
subject.options[:type] = 'task'
end
it 'valid with text' do
expect(subject).to be_valid
end
end
end
context 'payload_for' do
it 'removes unwanted attributes' do
result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text',
'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'})
expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}')
end
end
context 'headers' do
it 'sets X-Beeper-Application-Id header with app_id' do
expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id'])
end
it 'sets X-Beeper-REST-API-Key header with api_key' do
expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key'])
end
it 'sets Content-Type' do
expect(subject.send(:headers)['Content-Type']).to eq('application/json')
end
end
context 'endpoint_for' do
it 'returns valid URL for message' do
expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json')
end
it 'returns valid URL for image' do
expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json')
end
it 'returns valid URL for event' do
expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json')
end
it 'returns valid URL for location' do
expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json')
end
it 'returns valid URL for task' do
expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json')
end
end
end

1
spec/models/agents/google_flights_spec.rb
View file

@ -15,6 +15,7 @@ describe Agents::GoogleFlightsAgent do
'origin' => 'BOS',
'destination' => 'SFO',
'date' => '2016-04-11',
'preferredCabin' => 'COACH',
'childCount' => 0,
'infantInSeatCount' => 0,
'infantInLapCount'=> 0,

117
spec/models/agents/phantom_js_cloud_agent_spec.rb Normal file
View file

@ -0,0 +1,117 @@
require 'rails_helper'
describe Agents::PhantomJsCloudAgent do
before do
@valid_options = {
'name' => "XKCD",
'render_type' => "html",
'url' => "http://xkcd.com",
'mode' => 'clean',
'api_key' => '1234567890'
}
@checker = Agents::PhantomJsCloudAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
@checker.user = users(:jane)
@checker.save!
end
describe "validations" do
before do
expect(@checker).to be_valid
end
it "should validate the presence of url" do
@checker.options['url'] = "http://google.com"
expect(@checker).to be_valid
@checker.options['url'] = ""
expect(@checker).not_to be_valid
@checker.options['url'] = nil
expect(@checker).not_to be_valid
end
end
describe "emitting event" do
it "should emit url as event" do
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
end
it "should set render type as plain text" do
@checker.options['render_type'] = 'plainText'
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22plainText%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
end
it "should set output as json" do
@checker.options['output_as_json'] = true
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22outputAsJson%22%3Atrue%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
end
it "should not set ignore images" do
@checker.options['ignore_images'] = false
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
end
it "should set ignore images" do
@checker.options['ignore_images'] = true
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22ignoreImages%22%3Atrue%2C%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
end
it "should set wait interval to zero" do
@checker.options['wait_interval'] = '0'
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%2C%22wait_interval%22%3A%220%22%7D%7D")
end
it "should set user agent to BlackBerry" do
@checker.options['user_agent'] = 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+'
expect {
@checker.check
}.to change { @checker.events.count }.by(1)
item,* = @checker.events.last(1)
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Mozilla%2F5.0%20%28BlackBerry%3B%20U%3B%20BlackBerry%209900%3B%20en%29%20AppleWebKit%2F534.11%2B%20%28KHTML%2C%20like%20Gecko%29%20Version%2F7.1.0.346%20Mobile%20Safari%2F534.11%2B%22%7D%7D")
end
end
end

156
spec/models/agents/rss_agent_spec.rb
View file

@ -13,6 +13,7 @@ describe Agents::RssAgent do
stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
end
let(:agent) do
@ -295,6 +296,161 @@ describe Agents::RssAgent do
event = agent.events.first
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
end
it "decodes the content properly with force_encoding specified" do
@valid_options['force_encoding'] = 'iso-8859-1'
agent.check
event = agent.events.first
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
end
end
context 'with podcast elements' do
before do
@valid_options['url'] = 'http://example.com/podcast.rss'
@valid_options['include_feed_info'] = true
end
let :feed_info do
{
"id" => nil,
"type" => "rss",
"url" => "http://www.example.com/podcasts/everything/index.html",
"links" => [ { "href" => "http://www.example.com/podcasts/everything/index.html" } ],
"title" => "All About Everything",
"description" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
"copyright" => "℗ & © 2014 John Doe & Family",
"generator" => nil,
"icon" => nil,
"authors" => [
"John Doe"
],
"date_published" => nil,
"last_updated" => nil,
"itunes_categories" => [
"Technology", "Gadgets",
"TV & Film",
"Arts", "Food"
],
"itunes_complete" => "yes",
"itunes_explicit" => "no",
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything.jpg",
"itunes_owners" => ["John Doe <john.doe@example.com>"],
"itunes_subtitle" => "A show about everything",
"itunes_summary" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
"language" => "en-us"
}
end
it "is parsed correctly" do
expect {
agent.check
}.to change { agent.events.count }.by(4)
expect(agent.events.map(&:payload)).to match([
{
"feed" => feed_info,
"id" => "http://example.com/podcasts/archive/aae20140601.mp3",
"url" => nil,
"urls" => [],
"links" => [],
"title" => "Red,Whine, & Blue",
"description" => nil,
"content" => nil,
"image" => nil,
"enclosure" => {
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3",
"type" => "audio/mpeg",
"length" => "498537"
},
"authors" => ["<Various>"],
"categories" => [],
"date_published" => "2016-03-11T01:15:00+00:00",
"last_updated" => "2016-03-11T01:15:00+00:00",
"itunes_duration" => "03:59",
"itunes_explicit" => "no",
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg",
"itunes_subtitle" => "Red + Blue != Purple",
"itunes_summary" => "This week we talk about surviving in a Red state if you are a Blue person. Or vice versa."
},
{
"feed" => feed_info,
"id" => "http://example.com/podcasts/archive/aae20140697.m4v",
"url" => nil,
"urls" => [],
"links" => [],
"title" => "The Best Chili",
"description" => nil,
"content" => nil,
"image" => nil,
"enclosure" => {
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v",
"type" => "video/x-m4v",
"length" => "5650889"
},
"authors" => ["Jane Doe"],
"categories" => [],
"date_published" => "2016-03-10T02:00:00-07:00",
"last_updated" => "2016-03-10T02:00:00-07:00",
"itunes_closed_captioned" => "Yes",
"itunes_duration" => "04:34",
"itunes_explicit" => "no",
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg",
"itunes_subtitle" => "Jane and Eric",
"itunes_summary" => "This week we talk about the best Chili in the world. Which chili is better?"
},
{
"feed" => feed_info,
"id" => "http://example.com/podcasts/archive/aae20140608.mp4",
"url" => nil,
"urls" => [],
"links" => [],
"title" => "Socket Wrench Shootout",
"description" => nil,
"content" => nil,
"image" => nil,
"enclosure" => {
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4",
"type" => "video/mp4",
"length" => "5650889"
},
"authors" => ["Jane Doe"],
"categories" => [],
"date_published" => "2016-03-09T13:00:00-05:00",
"last_updated" => "2016-03-09T13:00:00-05:00",
"itunes_duration" => "04:34",
"itunes_explicit" => "no",
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg",
"itunes_subtitle" => "Comparing socket wrenches is fun!",
"itunes_summary" => "This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here."
},
{
"feed" => feed_info,
"id" => "http://example.com/podcasts/archive/aae20140615.m4a",
"url" => nil,
"urls" => [],
"links" => [],
"title" => "Shake Shake Shake Your Spices",
"description" => nil,
"content" => nil,
"image" => nil,
"enclosure" => {
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a",
"type" => "audio/x-m4a",
"length" => "8727310"
},
"authors" => ["John Doe"],
"categories" => [],
"date_published" => "2016-03-08T12:00:00+00:00",
"last_updated" => "2016-03-08T12:00:00+00:00",
"itunes_duration" => "07:04",
"itunes_explicit" => "no",
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg",
"itunes_subtitle" => "A short primer on table spices",
"itunes_summary" => "This week we talk about <a href=\"https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11\">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!"
}
])
end
end
end

73
spec/models/agents/website_agent_spec.rb
View file

@ -651,28 +651,22 @@ describe Agents::WebsiteAgent do
@checker.options = @valid_options
@checker.check
event = Event.last
expect(event.payload['url']).to eq("http://imgs.xkcd.com/comics/evolving.png")
expect(event.payload['title']).to eq("Evolving")
expect(event.payload['hovertext']).to match(/^Biologists play reverse/)
expect(event.payload).to match(
'url' => 'http://imgs.xkcd.com/comics/evolving.png',
'title' => 'Evolving',
'hovertext' => /^Biologists play reverse/
)
end
it "should turn relative urls to absolute" do
rel_site = {
'name' => "XKCD",
'expected_update_period_in_days' => "2",
'type' => "html",
'url' => "http://xkcd.com",
'mode' => "on_change",
'extract' => {
'url' => {'css' => "#topLeft a", 'value' => "@href"},
}
}
rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
rel.user = users(:bob)
rel.save!
rel.check
it "should exclude hidden keys" do
@valid_options['extract']['hovertext']['hidden'] = true
@checker.options = @valid_options
@checker.check
event = Event.last
expect(event.payload['url']).to eq("http://xkcd.com/about")
expect(event.payload).to match(
'url' => 'http://imgs.xkcd.com/comics/evolving.png',
'title' => 'Evolving'
)
end
it "should return an integer value if XPath evaluates to one" do
@ -749,9 +743,9 @@ describe Agents::WebsiteAgent do
expect(event.payload['original_url']).to eq('http://xkcd.com/index')
end
it "should be formatted by template after extraction" do
it "should format and merge values in template after extraction" do
@valid_options['extract']['hovertext']['hidden'] = true
@valid_options['template'] = {
'url' => '{{url}}',
'title' => '{{title | upcase}}',
'summary' => '{{title}}: {{hovertext | truncate: 20}}',
}
@ -1185,7 +1179,11 @@ fire: hot
'some_object' => {
'some_data' => { hello: 'world', href: '/world' }.to_json
},
url: 'http://example.com/'
url: 'http://example.com/',
'headers' => {
'Content-Type' => 'application/json'
},
'status' => 200
}
@event.save!
@ -1195,6 +1193,12 @@ fire: hot
'extract' => {
'value' => { 'path' => 'hello' },
'url' => { 'path' => 'href' },
},
'template' => {
'value' => '{{ value }}',
'url' => '{{ url | to_uri: _response_.url }}',
'type' => '{{ _response_.headers.content_type }}',
'status' => '{{ _response_.status | as_object }}'
}
)
end
@ -1203,7 +1207,7 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world' })
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200 })
end
it "should support merge mode" do
@ -1212,7 +1216,25 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world'))
expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
end
it "should convert headers and status in the event data properly" do
@event.payload[:status] = '201'
@event.payload[:headers] = [['Content-Type', 'application/rss+xml']]
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/rss+xml', 'status' => 201 })
end
it "should ignore inconvertible headers and status in the event data" do
@event.payload[:status] = 'ok'
@event.payload[:headers] = ['Content-Type', 'Content-Length']
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => '', 'status' => nil })
end
it "should output an error when nothing can be found at the path" do
@ -1349,6 +1371,9 @@ fire: hot
'mode' => 'all',
'extract' => {
'url' => { 'css' => "a", 'value' => "@href" },
},
'template' => {
'url' => '{{ url | to_uri }}',
}
}
@checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options)