mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Merge branch 'master' into fix_http_status_agent
This commit is contained in:
commit
fe3b43ba56
26 changed files with 1062 additions and 428 deletions
|
@ -3,6 +3,7 @@ sudo: required
|
|||
language: ruby
|
||||
services:
|
||||
- docker
|
||||
- mysql
|
||||
- postgresql
|
||||
env:
|
||||
global:
|
||||
|
@ -37,9 +38,6 @@ rvm:
|
|||
- 2.3.1
|
||||
cache: bundler
|
||||
bundler_args: --without development production
|
||||
before_install:
|
||||
- sudo apt-get -qq update
|
||||
- sudo apt-get install -y mysql-server
|
||||
script:
|
||||
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi
|
||||
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi
|
||||
|
|
18
Gemfile
18
Gemfile
|
@ -3,6 +3,12 @@ source 'https://rubygems.org'
|
|||
# Ruby 2.2.2 is the minimum requirement
|
||||
ruby ['2.2.2', RUBY_VERSION].max
|
||||
|
||||
# Ensure github repositories are fetched using HTTPS
|
||||
git_source(:github) do |repo_name|
|
||||
repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
|
||||
"https://github.com/#{repo_name}.git"
|
||||
end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2')
|
||||
|
||||
# Load vendored dotenv gem and .env file
|
||||
require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb')
|
||||
GemfileHelper.load_dotenv do |dotenv_dir|
|
||||
|
@ -50,7 +56,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn'
|
|||
gem 'omniauth-twitter', '~> 1.2.1'
|
||||
|
||||
# Tumblr Agents
|
||||
gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master' # '>= 0.8.5'
|
||||
gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master', ref: '0c59b04e49f2a8c89860613b18cf4e8f978d8dc7' # '>= 0.8.5'
|
||||
gem 'omniauth-tumblr', '~> 1.2'
|
||||
|
||||
# Dropbox Agents
|
||||
|
@ -90,9 +96,9 @@ gem 'delayed_job', '~> 4.1.0'
|
|||
gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails5'
|
||||
gem 'devise','~> 4.2.0'
|
||||
gem 'em-http-request', '~> 1.1.2'
|
||||
gem 'faraday', '~> 0.9.0'
|
||||
gem 'faraday', '~> 0.9'
|
||||
gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
|
||||
gem 'feedjira', '~> 2.0'
|
||||
gem 'feedjira', '~> 2.1'
|
||||
gem 'font-awesome-sass', '~> 4.3.2'
|
||||
gem 'foreman', '~> 0.63.0'
|
||||
gem 'geokit', '~> 1.8.4'
|
||||
|
@ -103,7 +109,7 @@ gem 'jquery-rails', '~> 4.2.1'
|
|||
gem 'huginn_agent', '~> 0.4.0'
|
||||
gem 'json', '~> 1.8.1'
|
||||
gem 'jsonpathv2', '~> 0.0.8'
|
||||
gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable'
|
||||
gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable', ref: 'abbf93d557208ee1d0b612c612cd079f86ed54f4'
|
||||
gem 'kramdown', '~> 1.3.3'
|
||||
gem 'liquid', '~> 3.0.3'
|
||||
gem 'loofah', '~> 2.0'
|
||||
|
@ -111,7 +117,7 @@ gem 'mini_magick'
|
|||
gem 'multi_xml'
|
||||
gem 'nokogiri'
|
||||
gem 'omniauth', '~> 1.3.1'
|
||||
gem 'rails', '~> 5.0.0.1'
|
||||
gem 'rails', '~> 5.0.1'
|
||||
gem 'rufus-scheduler', '~> 3.0.8', require: false
|
||||
gem 'sass-rails', '~> 5.0.6'
|
||||
gem 'select2-rails', '~> 3.5.4'
|
||||
|
@ -128,7 +134,7 @@ group :development do
|
|||
gem 'guard-rspec', '~> 4.6.4'
|
||||
gem 'rack-livereload', '~> 0.3.16'
|
||||
gem 'letter_opener_web', '~> 1.3.0'
|
||||
gem 'web-console'
|
||||
gem 'web-console', '>= 3.3.0'
|
||||
|
||||
gem 'capistrano', '~> 3.4.0'
|
||||
gem 'capistrano-rails', '~> 1.1'
|
||||
|
|
116
Gemfile.lock
116
Gemfile.lock
|
@ -1,6 +1,7 @@
|
|||
GIT
|
||||
remote: git://github.com/amatsuda/kaminari.git
|
||||
remote: https://github.com/amatsuda/kaminari.git
|
||||
revision: abbf93d557208ee1d0b612c612cd079f86ed54f4
|
||||
ref: abbf93d557208ee1d0b612c612cd079f86ed54f4
|
||||
branch: 0-17-stable
|
||||
specs:
|
||||
kaminari (0.17.0)
|
||||
|
@ -8,7 +9,7 @@ GIT
|
|||
activesupport (>= 3.0.0)
|
||||
|
||||
GIT
|
||||
remote: git://github.com/cantino/twitter-stream.git
|
||||
remote: https://github.com/cantino/twitter-stream.git
|
||||
revision: f7e7edb0bae013bffabf3598e7147773d9fd370f
|
||||
branch: huginn
|
||||
specs:
|
||||
|
@ -18,7 +19,7 @@ GIT
|
|||
simple_oauth (~> 0.3.0)
|
||||
|
||||
GIT
|
||||
remote: git://github.com/dsander/delayed_job_active_record.git
|
||||
remote: https://github.com/dsander/delayed_job_active_record.git
|
||||
revision: b314972ccc92e0e8b03b1589174d8fb9a82b3cd0
|
||||
branch: rails5
|
||||
specs:
|
||||
|
@ -27,7 +28,7 @@ GIT
|
|||
delayed_job (>= 3.0, < 5)
|
||||
|
||||
GIT
|
||||
remote: git://github.com/dsander/weibo_2.git
|
||||
remote: https://github.com/dsander/weibo_2.git
|
||||
revision: e5b77f21a7e9a666b582c48e16b1e96fca198cf8
|
||||
branch: master
|
||||
specs:
|
||||
|
@ -38,16 +39,17 @@ GIT
|
|||
rest-client (~> 1.8)
|
||||
|
||||
GIT
|
||||
remote: git://github.com/lostisland/faraday_middleware.git
|
||||
revision: c5836ae55857272732b33eb0e0a98d60e995a376
|
||||
remote: https://github.com/lostisland/faraday_middleware.git
|
||||
revision: 59088da02940d0ee2010b2e3156343346767c31e
|
||||
branch: master
|
||||
specs:
|
||||
faraday_middleware (0.10.0)
|
||||
faraday (>= 0.7.4, < 0.10)
|
||||
faraday (>= 0.7.4, < 1.0)
|
||||
|
||||
GIT
|
||||
remote: git://github.com/tumblr/tumblr_client.git
|
||||
remote: https://github.com/tumblr/tumblr_client.git
|
||||
revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
|
||||
ref: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
|
||||
branch: master
|
||||
specs:
|
||||
tumblr_client (0.8.5)
|
||||
|
@ -69,45 +71,45 @@ GEM
|
|||
remote: https://rubygems.org/
|
||||
specs:
|
||||
ace-rails-ap (2.0.1)
|
||||
actioncable (5.0.0.1)
|
||||
actionpack (= 5.0.0.1)
|
||||
actioncable (5.0.1)
|
||||
actionpack (= 5.0.1)
|
||||
nio4r (~> 1.2)
|
||||
websocket-driver (~> 0.6.1)
|
||||
actionmailer (5.0.0.1)
|
||||
actionpack (= 5.0.0.1)
|
||||
actionview (= 5.0.0.1)
|
||||
activejob (= 5.0.0.1)
|
||||
actionmailer (5.0.1)
|
||||
actionpack (= 5.0.1)
|
||||
actionview (= 5.0.1)
|
||||
activejob (= 5.0.1)
|
||||
mail (~> 2.5, >= 2.5.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
actionpack (5.0.0.1)
|
||||
actionview (= 5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
actionpack (5.0.1)
|
||||
actionview (= 5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
rack (~> 2.0)
|
||||
rack-test (~> 0.6.3)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
||||
actionview (5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
actionview (5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
builder (~> 3.1)
|
||||
erubis (~> 2.7.0)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
||||
activejob (5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
activejob (5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
globalid (>= 0.3.6)
|
||||
activemodel (5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
activerecord (5.0.0.1)
|
||||
activemodel (= 5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
activemodel (5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
activerecord (5.0.1)
|
||||
activemodel (= 5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
arel (~> 7.0)
|
||||
activesupport (5.0.0.1)
|
||||
activesupport (5.0.1)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
i18n (~> 0.7)
|
||||
minitest (~> 5.1)
|
||||
tzinfo (~> 1.1)
|
||||
addressable (2.3.8)
|
||||
arel (7.1.1)
|
||||
arel (7.1.4)
|
||||
autoparse (0.3.3)
|
||||
addressable (>= 2.3.1)
|
||||
extlib (>= 0.9.15)
|
||||
|
@ -157,7 +159,7 @@ GEM
|
|||
execjs
|
||||
coffee-script-source (1.10.0)
|
||||
colorize (0.7.7)
|
||||
concurrent-ruby (1.0.2)
|
||||
concurrent-ruby (1.0.3)
|
||||
cookiejar (0.3.2)
|
||||
coveralls (0.7.12)
|
||||
multi_json (~> 1.10)
|
||||
|
@ -214,11 +216,11 @@ GEM
|
|||
extlib (0.9.16)
|
||||
faraday (0.9.2)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
feedjira (2.0.0)
|
||||
faraday (~> 0.9)
|
||||
faraday_middleware (~> 0.9)
|
||||
loofah (~> 2.0)
|
||||
sax-machine (~> 1.0)
|
||||
feedjira (2.1.0)
|
||||
faraday (>= 0.9)
|
||||
faraday_middleware (>= 0.9)
|
||||
loofah (>= 2.0)
|
||||
sax-machine (>= 1.0)
|
||||
ffi (1.9.10)
|
||||
font-awesome-sass (4.3.2.1)
|
||||
sass (~> 3.2)
|
||||
|
@ -327,7 +329,7 @@ GEM
|
|||
mimemagic (0.3.1)
|
||||
mini_magick (4.2.3)
|
||||
mini_portile2 (2.1.0)
|
||||
minitest (5.9.0)
|
||||
minitest (5.10.1)
|
||||
mqtt (0.3.1)
|
||||
multi_json (1.12.1)
|
||||
multi_xml (0.5.5)
|
||||
|
@ -402,17 +404,17 @@ GEM
|
|||
rack
|
||||
rack-test (0.6.3)
|
||||
rack (>= 1.0)
|
||||
rails (5.0.0.1)
|
||||
actioncable (= 5.0.0.1)
|
||||
actionmailer (= 5.0.0.1)
|
||||
actionpack (= 5.0.0.1)
|
||||
actionview (= 5.0.0.1)
|
||||
activejob (= 5.0.0.1)
|
||||
activemodel (= 5.0.0.1)
|
||||
activerecord (= 5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
rails (5.0.1)
|
||||
actioncable (= 5.0.1)
|
||||
actionmailer (= 5.0.1)
|
||||
actionpack (= 5.0.1)
|
||||
actionview (= 5.0.1)
|
||||
activejob (= 5.0.1)
|
||||
activemodel (= 5.0.1)
|
||||
activerecord (= 5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
bundler (>= 1.3.0, < 2.0)
|
||||
railties (= 5.0.0.1)
|
||||
railties (= 5.0.1)
|
||||
sprockets-rails (>= 2.0.0)
|
||||
rails-controller-testing (1.0.1)
|
||||
actionpack (~> 5.x)
|
||||
|
@ -423,14 +425,14 @@ GEM
|
|||
nokogiri (~> 1.6.0)
|
||||
rails-html-sanitizer (1.0.3)
|
||||
loofah (~> 2.0)
|
||||
railties (5.0.0.1)
|
||||
actionpack (= 5.0.0.1)
|
||||
activesupport (= 5.0.0.1)
|
||||
railties (5.0.1)
|
||||
actionpack (= 5.0.1)
|
||||
activesupport (= 5.0.1)
|
||||
method_source
|
||||
rake (>= 0.8.7)
|
||||
thor (>= 0.18.1, < 2.0)
|
||||
raindrops (0.17.0)
|
||||
rake (11.2.2)
|
||||
rake (12.0.0)
|
||||
rb-fsevent (0.9.7)
|
||||
rb-inotify (0.9.5)
|
||||
ffi (>= 0.5.0)
|
||||
|
@ -514,7 +516,7 @@ GEM
|
|||
spring-watcher-listen (2.0.0)
|
||||
listen (>= 2.7, < 4.0)
|
||||
spring (~> 1.2)
|
||||
sprockets (3.7.0)
|
||||
sprockets (3.7.1)
|
||||
concurrent-ruby (~> 1.0)
|
||||
rack (> 1, < 3)
|
||||
sprockets-rails (3.2.0)
|
||||
|
@ -531,7 +533,7 @@ GEM
|
|||
therubyracer (0.12.2)
|
||||
libv8 (~> 3.16.14.0)
|
||||
ref
|
||||
thor (0.19.1)
|
||||
thor (0.19.4)
|
||||
thread_safe (0.3.5)
|
||||
tilt (2.0.5)
|
||||
tins (1.10.1)
|
||||
|
@ -617,9 +619,9 @@ DEPENDENCIES
|
|||
dropbox-api
|
||||
em-http-request (~> 1.1.2)
|
||||
evernote_oauth
|
||||
faraday (~> 0.9.0)
|
||||
faraday (~> 0.9)
|
||||
faraday_middleware!
|
||||
feedjira (~> 2.0)
|
||||
feedjira (~> 2.1)
|
||||
ffi (>= 1.9.4)
|
||||
font-awesome-sass (~> 4.3.2)
|
||||
forecast_io (~> 2.0.0)
|
||||
|
@ -663,7 +665,7 @@ DEPENDENCIES
|
|||
pry-byebug
|
||||
pry-rails
|
||||
rack-livereload (~> 0.3.16)
|
||||
rails (~> 5.0.0.1)
|
||||
rails (~> 5.0.1)
|
||||
rails-controller-testing
|
||||
rb-kqueue (>= 0.2)
|
||||
rr
|
||||
|
@ -693,14 +695,14 @@ DEPENDENCIES
|
|||
uglifier (~> 2.7.2)
|
||||
unicorn (~> 5.1.0)
|
||||
vcr
|
||||
web-console
|
||||
web-console (>= 3.3.0)
|
||||
webmock (~> 1.17.4)
|
||||
weibo_2!
|
||||
wunderground (~> 1.2.0)
|
||||
xmpp4r (~> 0.5.6)
|
||||
|
||||
RUBY VERSION
|
||||
ruby 2.3.1p112
|
||||
ruby 2.3.3p222
|
||||
|
||||
BUNDLED WITH
|
||||
1.13.6
|
||||
1.13.7
|
||||
|
|
|
@ -129,10 +129,11 @@ module LiquidInterpolatable
|
|||
# userinfo, host, port, registry, path, opaque, query, and
|
||||
# fragment.
|
||||
def to_uri(uri, base_uri = nil)
|
||||
if base_uri
|
||||
Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
|
||||
else
|
||||
case base_uri
|
||||
when nil, ''
|
||||
Utils.normalize_uri(uri.to_s)
|
||||
else
|
||||
Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
|
||||
end
|
||||
rescue URI::Error
|
||||
nil
|
||||
|
|
|
@ -370,7 +370,7 @@ class Agent < ActiveRecord::Base
|
|||
def receive!(options={})
|
||||
Agent.transaction do
|
||||
scope = Agent.
|
||||
select("agents.id AS receiver_agent_id, events.id AS event_id").
|
||||
select("agents.id AS receiver_agent_id, sources.type AS source_agent_type, agents.type AS receiver_agent_type, events.id AS event_id").
|
||||
joins("JOIN links ON (links.receiver_id = agents.id)").
|
||||
joins("JOIN agents AS sources ON (links.source_id = sources.id)").
|
||||
joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)").
|
||||
|
@ -379,10 +379,11 @@ class Agent < ActiveRecord::Base
|
|||
scope = scope.where("agents.id in (?)", options[:only_receivers])
|
||||
end
|
||||
|
||||
sql = scope.to_sql()
|
||||
sql = scope.to_sql
|
||||
|
||||
agents_to_events = {}
|
||||
Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id|
|
||||
Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_type, receiver_agent_type, event_id|
|
||||
next unless const_defined?(source_agent_type) && const_defined?(receiver_agent_type)
|
||||
agents_to_events[receiver_agent_id.to_i] ||= []
|
||||
agents_to_events[receiver_agent_id.to_i] << event_id
|
||||
end
|
||||
|
@ -417,6 +418,7 @@ class Agent < ActiveRecord::Base
|
|||
return if schedule == 'never'
|
||||
types = where(:schedule => schedule).group(:type).pluck(:type)
|
||||
types.each do |type|
|
||||
next unless const_defined?(type)
|
||||
type.constantize.bulk_check(schedule)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,129 +0,0 @@
|
|||
module Agents
|
||||
class BeeperAgent < Agent
|
||||
cannot_be_scheduled!
|
||||
cannot_create_events!
|
||||
no_bulk_receive!
|
||||
|
||||
description <<-MD
|
||||
Beeper agent sends messages to Beeper app on your mobile device via Push notifications.
|
||||
|
||||
You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io)
|
||||
|
||||
You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID
|
||||
|
||||
Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`.
|
||||
|
||||
Depending on message type you have to provide additional fields:
|
||||
|
||||
##### Message
|
||||
* `text` – **required**
|
||||
|
||||
##### Image
|
||||
* `image` – **required** (Image URL or Base64-encoded image)
|
||||
* `text` – optional
|
||||
|
||||
##### Event
|
||||
* `text` – **required**
|
||||
* `start_time` – **required** (Corresponding to ISO 8601)
|
||||
* `end_time` – optional (Corresponding to ISO 8601)
|
||||
|
||||
##### Location
|
||||
* `latitude` – **required**
|
||||
* `longitude` – **required**
|
||||
* `text` – optional
|
||||
|
||||
##### Task
|
||||
* `text` – **required**
|
||||
|
||||
You can see additional documentation at [Beeper website](https://beeper.io/docs)
|
||||
MD
|
||||
|
||||
BASE_URL = 'https://api.beeper.io/api'
|
||||
|
||||
TYPE_ATTRIBUTES = {
|
||||
'message' => %w(text),
|
||||
'image' => %w(text image),
|
||||
'event' => %w(text start_time end_time),
|
||||
'location' => %w(text latitude longitude),
|
||||
'task' => %w(text)
|
||||
}
|
||||
|
||||
MESSAGE_TYPES = TYPE_ATTRIBUTES.keys
|
||||
|
||||
TYPE_REQUIRED_ATTRIBUTES = {
|
||||
'message' => %w(text),
|
||||
'image' => %w(image),
|
||||
'event' => %w(text start_time),
|
||||
'location' => %w(latitude longitude),
|
||||
'task' => %w(text)
|
||||
}
|
||||
|
||||
def default_options
|
||||
{
|
||||
'type' => 'message',
|
||||
'app_id' => '',
|
||||
'api_key' => '',
|
||||
'sender_id' => '',
|
||||
'phone' => '',
|
||||
'text' => '{{title}}'
|
||||
}
|
||||
end
|
||||
|
||||
def validate_options
|
||||
%w(app_id api_key sender_id type).each do |attr|
|
||||
errors.add(:base, "you need to specify a #{attr}") if options[attr].blank?
|
||||
end
|
||||
|
||||
if options['type'].in?(MESSAGE_TYPES)
|
||||
required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']]
|
||||
if required_attributes.any? { |attr| options[attr].blank? }
|
||||
errors.add(:base, "you need to specify a #{required_attributes.join(', ')}")
|
||||
end
|
||||
else
|
||||
errors.add(:base, 'you need to specify a valid message type')
|
||||
end
|
||||
|
||||
unless options['group_id'].blank? ^ options['phone'].blank?
|
||||
errors.add(:base, 'you need to specify a phone or group_id')
|
||||
end
|
||||
end
|
||||
|
||||
def working?
|
||||
received_event_without_error? && !recent_error_logs?
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
incoming_events.each do |event|
|
||||
send_message(event)
|
||||
end
|
||||
end
|
||||
|
||||
def send_message(event)
|
||||
mo = interpolated(event)
|
||||
begin
|
||||
response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers)
|
||||
error(response.body) if response.code != 201
|
||||
rescue HTTParty::Error => e
|
||||
error(e.message)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def headers
|
||||
{
|
||||
'X-Beeper-Application-Id' => options['app_id'],
|
||||
'X-Beeper-REST-API-Key' => options['api_key'],
|
||||
'Content-Type' => 'application/json'
|
||||
}
|
||||
end
|
||||
|
||||
def payload_for(mo)
|
||||
mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json
|
||||
end
|
||||
|
||||
def endpoint_for(type)
|
||||
"#{BASE_URL}/#{type}s.json"
|
||||
end
|
||||
end
|
||||
end
|
|
@ -60,6 +60,7 @@ module Agents
|
|||
'seniorCount'=> 0,
|
||||
'return_date' => '2016-04-18',
|
||||
'roundtrip' => true,
|
||||
'preferredCabin' => 'COACH',
|
||||
'solutions'=> 3
|
||||
}
|
||||
end
|
||||
|
@ -69,6 +70,7 @@ module Agents
|
|||
form_configurable :origin, type: :string
|
||||
form_configurable :destination, type: :string
|
||||
form_configurable :date, type: :string
|
||||
form_configurable :preferredCabin, type: :array, values: %w(COACH PREMIUM_COACH BUSINESS FIRST)
|
||||
form_configurable :childCount
|
||||
form_configurable :infantInSeatCount
|
||||
form_configurable :infantInLapCount
|
||||
|
@ -101,9 +103,9 @@ module Agents
|
|||
|
||||
def post_params
|
||||
if round_trip?
|
||||
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s } ], :solutions=> interpolated["solutions"]}}
|
||||
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s} ], :solutions=> interpolated["solutions"]}}
|
||||
else
|
||||
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }], :solutions=> interpolated["solutions"]}}
|
||||
post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }], :solutions=> interpolated["solutions"]}}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
161
app/models/agents/phantom_js_cloud_agent.rb
Normal file
161
app/models/agents/phantom_js_cloud_agent.rb
Normal file
|
@ -0,0 +1,161 @@
|
|||
require 'json'
|
||||
require 'uri'
|
||||
|
||||
module Agents
|
||||
class PhantomJsCloudAgent < Agent
|
||||
include ERB::Util
|
||||
include FormConfigurable
|
||||
include WebRequestConcern
|
||||
|
||||
can_dry_run!
|
||||
|
||||
default_schedule 'every_12h'
|
||||
|
||||
description <<-MD
|
||||
This Agent generates [PhantomJs Cloud](https://phantomjscloud.com/) URLs that can be used to render JavaScript-heavy webpages for content extraction.
|
||||
|
||||
URLs generated by this Agent are formulated in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
|
||||
The generated URLs can then be supplied to a Website Agent to fetch and parse the content.
|
||||
|
||||
[Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
|
||||
|
||||
Please see the [Huginn Wiki for more info](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud).
|
||||
|
||||
Options:
|
||||
|
||||
* `Api key` - PhantomJs Cloud API Key credential stored in Huginn
|
||||
* `Url` - The url to render
|
||||
* `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`)
|
||||
* `Render type` - Render as html or plain text without html tags (default: `html`)
|
||||
* `Output as json` - Return the page conents and metadata as a JSON object (default: `false`)
|
||||
* `Ignore images` - Skip loading of inlined images (default: `false`)
|
||||
* `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`)
|
||||
* `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading.
|
||||
This is useful in case there are any AJAX requests or animations that need to finish up.
|
||||
This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
|
||||
|
||||
As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
|
||||
MD
|
||||
|
||||
event_description <<-MD
|
||||
Events look like this:
|
||||
{
|
||||
"url": "..."
|
||||
}
|
||||
MD
|
||||
|
||||
def default_options
|
||||
{
|
||||
'mode' => 'clean',
|
||||
'url' => 'http://xkcd.com',
|
||||
'render_type' => 'html',
|
||||
'output_as_json' => false,
|
||||
'ignore_images' => false,
|
||||
'user_agent' => self.class.default_user_agent,
|
||||
'wait_interval' => '1000'
|
||||
}
|
||||
end
|
||||
|
||||
form_configurable :mode, type: :array, values: ['clean', 'merge']
|
||||
form_configurable :api_key, roles: :completable
|
||||
form_configurable :url
|
||||
form_configurable :render_type, type: :array, values: ['html', 'plainText']
|
||||
form_configurable :output_as_json, type: :boolean
|
||||
form_configurable :ignore_images, type: :boolean
|
||||
form_configurable :user_agent, type: :text
|
||||
form_configurable :wait_interval
|
||||
|
||||
def mode
|
||||
interpolated['mode'].presence || default_options['mode']
|
||||
end
|
||||
|
||||
def render_type
|
||||
interpolated['render_type'].presence || default_options['render_type']
|
||||
end
|
||||
|
||||
def output_as_json
|
||||
boolify(interpolated['output_as_json'].presence ||
|
||||
default_options['output_as_json'])
|
||||
end
|
||||
|
||||
def ignore_images
|
||||
boolify(interpolated['ignore_images'].presence ||
|
||||
default_options['ignore_images'])
|
||||
end
|
||||
|
||||
def user_agent
|
||||
interpolated['user_agent'].presence || self.class.default_user_agent
|
||||
end
|
||||
|
||||
def wait_interval
|
||||
interpolated['wait_interval'].presence || default_options['wait_interval']
|
||||
end
|
||||
|
||||
def page_request_settings
|
||||
prs = {}
|
||||
|
||||
prs[:ignoreImages] = ignore_images if ignore_images
|
||||
prs[:userAgent] = user_agent if user_agent.present?
|
||||
|
||||
if wait_interval != default_options['wait_interval']
|
||||
prs[:wait_interval] = wait_interval
|
||||
end
|
||||
|
||||
prs
|
||||
end
|
||||
|
||||
def build_phantom_url(interpolated)
|
||||
api_key = interpolated[:api_key]
|
||||
page_request_hash = {
|
||||
url: interpolated[:url],
|
||||
renderType: render_type
|
||||
}
|
||||
|
||||
page_request_hash[:outputAsJson] = output_as_json if output_as_json
|
||||
|
||||
page_request_settings_hash = page_request_settings
|
||||
|
||||
if page_request_settings_hash.any?
|
||||
page_request_hash[:requestSettings] = page_request_settings_hash
|
||||
end
|
||||
|
||||
request = page_request_hash.to_json
|
||||
log "Generated request: #{request}"
|
||||
|
||||
encoded = url_encode(request)
|
||||
"https://phantomjscloud.com/api/browser/v2/#{api_key}/?request=#{encoded}"
|
||||
end
|
||||
|
||||
def check
|
||||
phantom_url = build_phantom_url(interpolated)
|
||||
|
||||
create_event payload: { 'url' => phantom_url }
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
incoming_events.each do |event|
|
||||
interpolate_with(event) do
|
||||
existing_payload = interpolated['mode'].to_s == 'merge' ? event.payload : {}
|
||||
phantom_url = build_phantom_url(interpolated)
|
||||
|
||||
result = { 'url' => phantom_url }
|
||||
create_event payload: existing_payload.merge(result)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def complete_api_key
|
||||
user.user_credentials.map { |c| { text: c.credential_name, id: "{% credential #{c.credential_name} %}" } }
|
||||
end
|
||||
|
||||
def working?
|
||||
!recent_error_logs? || received_event_without_error?
|
||||
end
|
||||
|
||||
def validate_options
|
||||
# Check for required fields
|
||||
errors.add(:base, 'Url is required') unless options['url'].present?
|
||||
errors.add(:base, 'API key (credential) is required') unless options['api_key'].present?
|
||||
end
|
||||
end
|
||||
end
|
|
@ -66,6 +66,22 @@ module Agents
|
|||
"copyright": "...",
|
||||
"icon": "http://example.com/icon.png",
|
||||
"authors": [ "..." ],
|
||||
|
||||
"itunes_block": "no",
|
||||
"itunes_categories": [
|
||||
"Technology", "Gadgets",
|
||||
"TV & Film",
|
||||
"Arts", "Food"
|
||||
],
|
||||
"itunes_complete": "yes",
|
||||
"itunes_explicit": "yes",
|
||||
"itunes_image": "http://...",
|
||||
"itunes_new_feed_url": "http://...",
|
||||
"itunes_owners": [ "John Doe <john.doe@example.com>" ],
|
||||
"itunes_subtitle": "...",
|
||||
"itunes_summary": "...",
|
||||
"language": "en-US",
|
||||
|
||||
"date_published": "2014-09-11T01:30:00-07:00",
|
||||
"last_updated": "2014-09-11T01:30:00-07:00"
|
||||
},
|
||||
|
@ -84,6 +100,16 @@ module Agents
|
|||
"enclosure": {
|
||||
"url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789"
|
||||
},
|
||||
|
||||
"itunes_block": "no",
|
||||
"itunes_closed_captioned": "yes",
|
||||
"itunes_duration": "04:34",
|
||||
"itunes_explicit": "yes",
|
||||
"itunes_image": "http://...",
|
||||
"itunes_order": "1",
|
||||
"itunes_subtitle": "...",
|
||||
"itunes_summary": "...",
|
||||
|
||||
"date_published": "2014-09-11T01:30:00-0700",
|
||||
"last_updated": "2014-09-11T01:30:00-0700"
|
||||
}
|
||||
|
@ -91,7 +117,8 @@ module Agents
|
|||
Some notes:
|
||||
|
||||
- The `feed` key is present only if `include_feed_info` is set to true.
|
||||
- Each element in `authors` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
|
||||
- The keys starting with `itunes_`, and `language` are only present when the feed is a podcast. See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details.
|
||||
- Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
|
||||
- Timestamps are converted to the ISO 8601 format.
|
||||
MD
|
||||
|
||||
|
@ -179,7 +206,7 @@ module Agents
|
|||
else
|
||||
# Encoding is already known, so do not let the parser detect
|
||||
# it from the XML declaration in the content.
|
||||
body.sub!(/(\A\u{FEFF}?\s*<\?xml(?:\s+\w+\s*=\s*(['"]).*?\2)*)\s+encoding\s*=\s*(['"]).*?\3/, '\\1')
|
||||
body.sub!(/(?<noenc>\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?<av>\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g<av>/, '\\k<noenc>')
|
||||
end
|
||||
body
|
||||
end
|
||||
|
@ -206,9 +233,40 @@ module Agents
|
|||
authors: feed.authors,
|
||||
date_published: feed.date_published,
|
||||
last_updated: feed.last_updated,
|
||||
**itunes_feed_data(feed)
|
||||
}
|
||||
end
|
||||
|
||||
def itunes_feed_data(feed)
|
||||
data = {}
|
||||
case feed
|
||||
when Feedjira::Parser::ITunesRSS
|
||||
%i[
|
||||
itunes_block
|
||||
itunes_categories
|
||||
itunes_complete
|
||||
itunes_explicit
|
||||
itunes_image
|
||||
itunes_new_feed_url
|
||||
itunes_owners
|
||||
itunes_subtitle
|
||||
itunes_summary
|
||||
language
|
||||
].each { |attr|
|
||||
if value = feed.try(attr).presence
|
||||
data[attr] =
|
||||
case attr
|
||||
when :itunes_summary
|
||||
clean_fragment(value)
|
||||
else
|
||||
value
|
||||
end
|
||||
end
|
||||
}
|
||||
end
|
||||
data
|
||||
end
|
||||
|
||||
def entry_data(entry)
|
||||
{
|
||||
id: entry.id,
|
||||
|
@ -224,9 +282,32 @@ module Agents
|
|||
categories: Array(entry.try(:categories)),
|
||||
date_published: entry.date_published,
|
||||
last_updated: entry.last_updated,
|
||||
**itunes_entry_data(entry)
|
||||
}
|
||||
end
|
||||
|
||||
def itunes_entry_data(entry)
|
||||
data = {}
|
||||
case entry
|
||||
when Feedjira::Parser::ITunesRSSItem
|
||||
%i[
|
||||
itunes_block
|
||||
itunes_closed_captioned
|
||||
itunes_duration
|
||||
itunes_explicit
|
||||
itunes_image
|
||||
itunes_order
|
||||
itunes_subtitle
|
||||
itunes_summary
|
||||
].each { |attr|
|
||||
if value = entry.try(attr).presence
|
||||
data[attr] = value
|
||||
end
|
||||
}
|
||||
end
|
||||
data
|
||||
end
|
||||
|
||||
def feed_to_events(feed)
|
||||
payload_base = {}
|
||||
|
||||
|
|
|
@ -27,8 +27,6 @@ module Agents
|
|||
* Alternatively, set `data_from_event` to a Liquid template to use data directly without fetching any URL. (For example, set it to `{{ html }}` to use HTML contained in the `html` key of the incoming Event.)
|
||||
* If you specify `merge` for the `mode` option, Huginn will retain the old payload and update it with new values.
|
||||
|
||||
If a created Event has a key named `url` containing a relative URL, it is automatically resolved using the request URL as base.
|
||||
|
||||
# Supported Document Types
|
||||
|
||||
The `type` value can be `xml`, `html`, `json`, or `text`.
|
||||
|
@ -37,6 +35,8 @@ module Agents
|
|||
|
||||
Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor except when it has `repeat` set to true. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
|
||||
|
||||
For extractors with `hidden` set to true, they will be excluded from the payloads of events created by the Agent, but can be used and interpolated in the `template` option explained below.
|
||||
|
||||
For extractors with `repeat` set to true, their first matches will be included in all extracts. This is useful such as when you want to include the title of a page in all events created from the page.
|
||||
|
||||
# Scraping HTML and XML
|
||||
|
@ -116,11 +116,10 @@ module Agents
|
|||
|
||||
Set `http_success_codes` to an array of status codes (e.g., `[404, 422]`) to treat HTTP response codes beyond 200 as successes.
|
||||
|
||||
If a `template` option is given, it is used as a Liquid template for each event created by this Agent, instead of directly emitting the results of extraction as events. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
|
||||
If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
|
||||
|
||||
"template": {
|
||||
"url": "{{ url }}",
|
||||
"title": "{{ title }}",
|
||||
"url": "{{ url | to_uri: _request_.url }}",
|
||||
"description": "{{ body_text }}",
|
||||
"last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
|
||||
}
|
||||
|
@ -129,17 +128,17 @@ module Agents
|
|||
|
||||
# Liquid Templating
|
||||
|
||||
In Liquid templating, the following variables are available except when invoked by `data_from_event`:
|
||||
In Liquid templating, the following variables are available:
|
||||
|
||||
* `_url_`: The URL specified to fetch the content from.
|
||||
* `_url_`: The URL specified to fetch the content from. When parsing `data_from_event`, this is not set.
|
||||
|
||||
* `_response_`: A response object with the following keys:
|
||||
|
||||
* `status`: HTTP status as integer. (Almost always 200)
|
||||
* `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
|
||||
|
||||
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_.
|
||||
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event, if it is a hash.
|
||||
|
||||
* `url`: The final URL of the fetched page, following redirects. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
|
||||
* `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
|
||||
|
||||
# Ordering Events
|
||||
|
||||
|
@ -159,7 +158,11 @@ module Agents
|
|||
end
|
||||
|
||||
def event_keys
|
||||
(options['template'].presence || options['extract']).try(:keys)
|
||||
extract = options['extract'] or return nil
|
||||
|
||||
extract.each_with_object([]) { |(key, value), keys|
|
||||
keys << key unless boolify(value['hidden'])
|
||||
} | (options['template'].presence.try!(:keys) || [])
|
||||
end
|
||||
|
||||
def working?
|
||||
|
@ -362,6 +365,8 @@ module Agents
|
|||
end
|
||||
|
||||
def handle_data(body, url, existing_payload)
|
||||
# Beware, url may be a URI object, string or nil
|
||||
|
||||
doc = parse(body)
|
||||
|
||||
if extract_full_json?
|
||||
|
@ -382,41 +387,18 @@ module Agents
|
|||
extract_xml(doc)
|
||||
end
|
||||
|
||||
num_tuples = output.each_value.inject(nil) { |num, value|
|
||||
case size = value.size
|
||||
when Float::INFINITY
|
||||
num
|
||||
when Integer
|
||||
if num && num != size
|
||||
raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
|
||||
end
|
||||
size
|
||||
end
|
||||
} or raise "At least one non-repeat key is required"
|
||||
num_tuples = output.size or
|
||||
raise "At least one non-repeat key is required"
|
||||
|
||||
old_events = previous_payloads num_tuples
|
||||
|
||||
template = options['template'].presence
|
||||
|
||||
num_tuples.times.zip(*output.values) do |index, *values|
|
||||
extracted = output.each_key.lazy.zip(values).to_h
|
||||
output.each do |extracted|
|
||||
result = extracted.except(*output.hidden_keys)
|
||||
|
||||
result =
|
||||
if template
|
||||
interpolate_with(extracted) do
|
||||
interpolate_options(template)
|
||||
end
|
||||
else
|
||||
extracted
|
||||
end
|
||||
|
||||
# url may be URI, string or nil
|
||||
if (payload_url = result['url'].presence) && (url = url.presence)
|
||||
begin
|
||||
result['url'] = (Utils.normalize_uri(url) + Utils.normalize_uri(payload_url)).to_s
|
||||
rescue URI::Error
|
||||
error "Cannot resolve url: <#{payload_url}> on <#{url}>"
|
||||
end
|
||||
if template
|
||||
result.update(interpolate_options(template, extracted))
|
||||
end
|
||||
|
||||
if store_payload!(old_events, result)
|
||||
|
@ -460,7 +442,10 @@ module Agents
|
|||
end
|
||||
|
||||
def handle_event_data(data, event, existing_payload)
|
||||
handle_data(data, event.payload['url'], existing_payload)
|
||||
interpolation_context.stack {
|
||||
interpolation_context['_response_'] = ResponseFromEventDrop.new(event)
|
||||
handle_data(data, event.payload['url'].presence, existing_payload)
|
||||
}
|
||||
rescue => e
|
||||
error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event
|
||||
end
|
||||
|
@ -528,7 +513,7 @@ module Agents
|
|||
end
|
||||
|
||||
def extract_each(&block)
|
||||
interpolated['extract'].each_with_object({}) { |(name, extraction_details), output|
|
||||
interpolated['extract'].each_with_object(Output.new) { |(name, extraction_details), output|
|
||||
if boolify(extraction_details['repeat'])
|
||||
values = Repeater.new { |repeater|
|
||||
block.call(extraction_details, repeater)
|
||||
|
@ -538,7 +523,13 @@ module Agents
|
|||
block.call(extraction_details, values)
|
||||
end
|
||||
log "Values extracted: #{values}"
|
||||
output[name] = values
|
||||
begin
|
||||
output[name] = values
|
||||
rescue UnevenSizeError
|
||||
raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
|
||||
else
|
||||
output.hidden_keys << name if boolify(extraction_details['hidden'])
|
||||
end
|
||||
}
|
||||
end
|
||||
|
||||
|
@ -617,6 +608,38 @@ module Agents
|
|||
false
|
||||
end
|
||||
|
||||
class UnevenSizeError < ArgumentError
|
||||
end
|
||||
|
||||
class Output
|
||||
def initialize
|
||||
@hash = {}
|
||||
@size = nil
|
||||
@hidden_keys = []
|
||||
end
|
||||
|
||||
attr_reader :size
|
||||
attr_reader :hidden_keys
|
||||
|
||||
def []=(key, value)
|
||||
case size = value.size
|
||||
when Integer
|
||||
if @size && @size != size
|
||||
raise UnevenSizeError, 'got an uneven size'
|
||||
end
|
||||
@size = size
|
||||
end
|
||||
|
||||
@hash[key] = value
|
||||
end
|
||||
|
||||
def each
|
||||
@size.times.zip(*@hash.values) do |index, *values|
|
||||
yield @hash.each_key.lazy.zip(values).to_h
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class Repeater < Enumerator
|
||||
# Repeater.new { |y|
|
||||
# # ...
|
||||
|
@ -659,6 +682,24 @@ module Agents
|
|||
end
|
||||
end
|
||||
|
||||
class ResponseFromEventDrop < LiquidDroppable::Drop
|
||||
def headers
|
||||
headers = Faraday::Utils::Headers.from(@object.payload[:headers]) rescue {}
|
||||
|
||||
HeaderDrop.new(headers)
|
||||
end
|
||||
|
||||
# Integer value of HTTP status
|
||||
def status
|
||||
Integer(@object.payload[:status]) rescue nil
|
||||
end
|
||||
|
||||
# The URL
|
||||
def url
|
||||
@object.payload[:url]
|
||||
end
|
||||
end
|
||||
|
||||
# Wraps Faraday::Utils::Headers
|
||||
class HeaderDrop < LiquidDroppable::Drop
|
||||
def before_method(name)
|
||||
|
|
|
@ -19,20 +19,20 @@
|
|||
</ul>
|
||||
<br/>
|
||||
<p>
|
||||
The issue most probably occurred because of one or more of the following reasons:
|
||||
This issue probably occurred for one or more of the following reasons:
|
||||
</p>
|
||||
<ul>
|
||||
<li>If the respective Agent is distributed as part of the Huginn application codebase, it may have been removed or moved to an Agent gem. Please see <a href="https://github.com/cantino/huginn/wiki/Dealing-with-Deleted-Agent-Types" target="_blank">this wiki page for more information</a>.</li>
|
||||
<li>If the respective Agent is distributed as a Ruby gem, it might have been removed from the <code>ADDITIONAL_GEMS</code> environment setting.</li>
|
||||
<li>If the respective Agent is distributed as part of the Huginn application codebase, it might have been removed from that either on purpose (because the Agent has been deprecated or been moved to an Agent gem) or accidentally. Please check if the Agent(s) in question are available in your Huginn codebase under the path <code>app/models/agents/</code>.</li>
|
||||
</ul>
|
||||
<br/>
|
||||
<p>
|
||||
You can fix the issue by adding the Agent(s) back to the application codebase by
|
||||
You can fix this issue by:
|
||||
</p>
|
||||
<ul>
|
||||
<li>deleting the respective Agent(s) from the database using the button below.</li>
|
||||
<li>adding the respective Agent(s) to the the <code>ADDITIONAL_GEMS</code> environment setting. Please see <a href="https://github.com/cantino/huginn_agent" target="_blank">https://github.com/cantino/huginn_agent</a> for documentation on how to properly set it.</li>
|
||||
<li>adding the respective Agent(s) code to the Huginn application codebase (in case it was deleted accidentally).</li>
|
||||
<li>deleting the respective Agent(s) from the database using the button below.</li>
|
||||
</ul>
|
||||
<br/>
|
||||
<div class="btn-group">
|
||||
|
|
|
@ -52,7 +52,7 @@
|
|||
window.agentPaths = {};
|
||||
window.agentNames = [];
|
||||
<% if current_user.present? -%>
|
||||
var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| m[a.first] = agent_path(a.last); m }) %>;
|
||||
var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| next if a.last.nil?; m[a.first] = agent_path(a.last); m }) %>;
|
||||
var myScenarios = <%= Utils.jsonify(current_user.scenarios.pluck(:name, :id).inject({}) {|m, s| m[s.first + " Scenario"] = scenario_path(s.last); m }) %>;
|
||||
$.extend(window.agentPaths, myAgents);
|
||||
$.extend(window.agentPaths, myScenarios);
|
||||
|
|
|
@ -0,0 +1,36 @@
|
|||
class ConvertWebsiteAgentTemplateForMerge < ActiveRecord::Migration[5.0]
|
||||
def up
|
||||
Agents::WebsiteAgent.find_each do |agent|
|
||||
extract = agent.options['extract'].presence
|
||||
template = agent.options['template'].presence
|
||||
next unless extract.is_a?(Hash) && template.is_a?(Hash)
|
||||
|
||||
(extract.keys - template.keys).each do |key|
|
||||
extract[key]['hidden'] = true
|
||||
end
|
||||
|
||||
template.delete_if { |key, value|
|
||||
extract.key?(key) &&
|
||||
value.match(/\A\{\{\s*#{Regexp.quote(key)}\s*\}\}\z/)
|
||||
}
|
||||
|
||||
agent.save!(validate: false)
|
||||
end
|
||||
end
|
||||
|
||||
def down
|
||||
Agents::WebsiteAgent.find_each do |agent|
|
||||
extract = agent.options['extract'].presence
|
||||
template = agent.options['template'].presence
|
||||
next unless extract.is_a?(Hash) && template.is_a?(Hash)
|
||||
|
||||
(extract.keys - template.keys).each do |key|
|
||||
unless extract[key].delete('hidden').in?([true, 'true'])
|
||||
template[key] = "{{ #{key} }}"
|
||||
end
|
||||
end
|
||||
|
||||
agent.save!(validate: false)
|
||||
end
|
||||
end
|
||||
end
|
16
db/migrate/20161124065838_add_templates_to_resolve_url.rb
Normal file
16
db/migrate/20161124065838_add_templates_to_resolve_url.rb
Normal file
|
@ -0,0 +1,16 @@
|
|||
class AddTemplatesToResolveUrl < ActiveRecord::Migration[5.0]
|
||||
def up
|
||||
Agents::WebsiteAgent.find_each do |agent|
|
||||
if agent.event_keys.try!(:include?, 'url')
|
||||
agent.options['template'] = (agent.options['template'] || {}).tap { |template|
|
||||
template['url'] ||= '{{ url | to_uri: _response_.url }}'
|
||||
}
|
||||
agent.save!(validate: false)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def down
|
||||
# No need to revert
|
||||
end
|
||||
end
|
|
@ -57,6 +57,13 @@ module FeedjiraExtension
|
|||
value :content
|
||||
end
|
||||
|
||||
class ITunesRssOwner < Author
|
||||
include SAXMachine
|
||||
|
||||
element :'itunes:name', as: :name
|
||||
element :'itunes:email', as: :email
|
||||
end
|
||||
|
||||
class Enclosure
|
||||
include SAXMachine
|
||||
|
||||
|
@ -290,6 +297,16 @@ module FeedjiraExtension
|
|||
def copyright
|
||||
@copyright || super
|
||||
end
|
||||
|
||||
if /ITunes/ === name
|
||||
sax_config.collection_elements['itunes:owner'].clear
|
||||
elements :"itunes:owner", as: :_itunes_owners, class: ITunesRssOwner
|
||||
private :_itunes_owners
|
||||
|
||||
def itunes_owners
|
||||
_itunes_owners.reject(&:empty?)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
sax_config.collection_elements.each_value do |collection_elements|
|
||||
|
|
|
@ -119,6 +119,15 @@ describe LiquidInterpolatable::Filters do
|
|||
@agent.interpolation_context['s'] = 'foo/index.html'
|
||||
expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
|
||||
end
|
||||
|
||||
it 'should normalize a URI value if an empty base URI is given' do
|
||||
@agent.options['foo'] = '{{ u | to_uri: b }}'
|
||||
@agent.interpolation_context['u'] = "\u{3042}"
|
||||
@agent.interpolation_context['b'] = ""
|
||||
expect(@agent.interpolated['foo']).to eq('%E3%81%82')
|
||||
@agent.interpolation_context['b'] = nil
|
||||
expect(@agent.interpolated['foo']).to eq('%E3%81%82')
|
||||
end
|
||||
end
|
||||
|
||||
describe 'uri_expand' do
|
||||
|
|
76
spec/data_fixtures/podcast.rss
Normal file
76
spec/data_fixtures/podcast.rss
Normal file
|
@ -0,0 +1,76 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
|
||||
<channel>
|
||||
<title>All About Everything</title>
|
||||
<link>http://www.example.com/podcasts/everything/index.html</link>
|
||||
<language>en-us</language>
|
||||
<copyright>℗ & © 2014 John Doe & Family</copyright>
|
||||
<itunes:subtitle>A show about everything</itunes:subtitle>
|
||||
<itunes:author>John Doe</itunes:author>
|
||||
<itunes:summary>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</itunes:summary>
|
||||
<description>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</description>
|
||||
<itunes:owner>
|
||||
<itunes:name>John Doe</itunes:name>
|
||||
<itunes:email>john.doe@example.com</itunes:email>
|
||||
</itunes:owner>
|
||||
<itunes:complete>yes</itunes:complete>
|
||||
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg"/>
|
||||
<itunes:category text="Technology">
|
||||
<itunes:category text="Gadgets"/>
|
||||
</itunes:category>
|
||||
<itunes:category text="TV & Film"/>
|
||||
<itunes:category text="Arts">
|
||||
<itunes:category text="Food"/>
|
||||
</itunes:category>
|
||||
<itunes:explicit>no</itunes:explicit>
|
||||
<item>
|
||||
<title>Shake Shake Shake Your Spices</title>
|
||||
<itunes:author>John Doe</itunes:author>
|
||||
<itunes:subtitle>A short primer on table spices</itunes:subtitle>
|
||||
<itunes:summary><![CDATA[This week we talk about <a href="https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!]]></itunes:summary>
|
||||
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg"/>
|
||||
<enclosure length="8727310" type="audio/x-m4a" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a"/>
|
||||
<guid>http://example.com/podcasts/archive/aae20140615.m4a</guid>
|
||||
<pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
|
||||
<itunes:duration>07:04</itunes:duration>
|
||||
<itunes:explicit>no</itunes:explicit>
|
||||
</item>
|
||||
<item>
|
||||
<title>Socket Wrench Shootout</title>
|
||||
<itunes:author>Jane Doe</itunes:author>
|
||||
<itunes:subtitle>Comparing socket wrenches is fun!</itunes:subtitle>
|
||||
<itunes:summary>This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here.</itunes:summary>
|
||||
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg"/>
|
||||
<enclosure length="5650889" type="video/mp4" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4"/>
|
||||
<guid>http://example.com/podcasts/archive/aae20140608.mp4</guid>
|
||||
<pubDate>Wed, 09 Mar 2016 13:00:00 EST</pubDate>
|
||||
<itunes:duration>04:34</itunes:duration>
|
||||
<itunes:explicit>no</itunes:explicit>
|
||||
</item>
|
||||
<item>
|
||||
<title>The Best Chili</title>
|
||||
<itunes:author>Jane Doe</itunes:author>
|
||||
<itunes:subtitle>Jane and Eric</itunes:subtitle>
|
||||
<itunes:summary>This week we talk about the best Chili in the world. Which chili is better?</itunes:summary>
|
||||
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg"/>
|
||||
<enclosure length="5650889" type="video/x-m4v" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v"/>
|
||||
<guid>http://example.com/podcasts/archive/aae20140697.m4v</guid>
|
||||
<pubDate>Thu, 10 Mar 2016 02:00:00 -0700</pubDate>
|
||||
<itunes:duration>04:34</itunes:duration>
|
||||
<itunes:explicit>no</itunes:explicit>
|
||||
<itunes:isClosedCaptioned>Yes</itunes:isClosedCaptioned>
|
||||
</item>
|
||||
<item>
|
||||
<title>Red,Whine, & Blue</title>
|
||||
<itunes:author>Various</itunes:author>
|
||||
<itunes:subtitle>Red + Blue != Purple</itunes:subtitle>
|
||||
<itunes:summary>This week we talk about surviving in a Red state if you are a Blue person. Or vice versa.</itunes:summary>
|
||||
<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg"/>
|
||||
<enclosure length="498537" type="audio/mpeg" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3"/>
|
||||
<guid>http://example.com/podcasts/archive/aae20140601.mp3</guid>
|
||||
<pubDate>Fri, 11 Mar 2016 01:15:00 +3000</pubDate>
|
||||
<itunes:duration>03:59</itunes:duration>
|
||||
<itunes:explicit>no</itunes:explicit>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
|
@ -38,7 +38,7 @@ describe "Creating a new agent", js: true do
|
|||
visit new_agent_path
|
||||
end
|
||||
it "shows all options for agents that can be scheduled, create and receive events" do
|
||||
select2("Website Agent", from: "Type")
|
||||
select2("Website Agent scrapes", from: "Type")
|
||||
expect(page).not_to have_content('This type of Agent cannot create events.')
|
||||
end
|
||||
|
||||
|
@ -49,9 +49,8 @@ describe "Creating a new agent", js: true do
|
|||
end
|
||||
|
||||
it "allows to click on on the agent name in select2 tags" do
|
||||
agent = agents(:bob_weather_agent)
|
||||
visit new_agent_path
|
||||
select2("Website Agent", from: "Type")
|
||||
select2("Website Agent scrapes", from: "Type")
|
||||
select2("SF Weather", from: 'Sources')
|
||||
click_on "SF Weather"
|
||||
expect(page).to have_content "Editing your WeatherAgent"
|
||||
|
@ -63,7 +62,7 @@ describe "Creating a new agent", js: true do
|
|||
end
|
||||
|
||||
it "does not send previously configured sources when the current agent does not support them" do
|
||||
select2("Website Agent", from: "Type")
|
||||
select2("Website Agent scrapes", from: "Type")
|
||||
select2("SF Weather", from: 'Sources')
|
||||
select2("Webhook Agent", from: "Type")
|
||||
fill_in(:agent_name, with: "No sources")
|
||||
|
@ -85,7 +84,7 @@ describe "Creating a new agent", js: true do
|
|||
end
|
||||
|
||||
it "does not send previously configured receivers when the current agent does not support them" do
|
||||
select2("Website Agent", from: "Type")
|
||||
select2("Website Agent scrapes", from: "Type")
|
||||
select2("ZKCD", from: 'Receivers')
|
||||
select2("Email Agent", from: "Type")
|
||||
fill_in(:agent_name, with: "No receivers")
|
||||
|
|
|
@ -0,0 +1,92 @@
|
|||
load 'spec/rails_helper.rb'
|
||||
load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
|
||||
|
||||
describe ConvertWebsiteAgentTemplateForMerge do
|
||||
let :old_extract do
|
||||
{
|
||||
'url' => { 'css' => "#comic img", 'value' => "@src" },
|
||||
'title' => { 'css' => "#comic img", 'value' => "@alt" },
|
||||
'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
|
||||
}
|
||||
end
|
||||
|
||||
let :new_extract do
|
||||
{
|
||||
'url' => { 'css' => "#comic img", 'value' => "@src" },
|
||||
'title' => { 'css' => "#comic img", 'value' => "@alt" },
|
||||
'hovertext' => { 'css' => "#comic img", 'value' => "@title", 'hidden' => true }
|
||||
}
|
||||
end
|
||||
|
||||
let :reverted_extract do
|
||||
old_extract
|
||||
end
|
||||
|
||||
let :old_template do
|
||||
{
|
||||
'url' => '{{url}}',
|
||||
'title' => '{{ title }}',
|
||||
'description' => '{{ hovertext }}',
|
||||
'comment' => '{{ comment }}'
|
||||
}
|
||||
end
|
||||
|
||||
let :new_template do
|
||||
{
|
||||
'description' => '{{ hovertext }}',
|
||||
'comment' => '{{ comment }}'
|
||||
}
|
||||
end
|
||||
|
||||
let :reverted_template do
|
||||
old_template.merge('url' => '{{ url }}')
|
||||
end
|
||||
|
||||
let :valid_options do
|
||||
{
|
||||
'name' => "XKCD",
|
||||
'expected_update_period_in_days' => "2",
|
||||
'type' => "html",
|
||||
'url' => "{{ url | default: 'http://xkcd.com/' }}",
|
||||
'mode' => 'on_change',
|
||||
'extract' => old_extract,
|
||||
'template' => old_template
|
||||
}
|
||||
end
|
||||
|
||||
let :agent do
|
||||
Agents::WebsiteAgent.create!(
|
||||
user: users(:bob),
|
||||
name: "xkcd",
|
||||
options: valid_options,
|
||||
keep_events_for: 2.days
|
||||
)
|
||||
end
|
||||
|
||||
describe 'up' do
|
||||
it 'should update extract and template options for an existing WebsiteAgent' do
|
||||
expect(agent.options).to include('extract' => old_extract,
|
||||
'template' => old_template)
|
||||
ConvertWebsiteAgentTemplateForMerge.new.up
|
||||
agent.reload
|
||||
expect(agent.options).to include('extract' => new_extract,
|
||||
'template' => new_template)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'down' do
|
||||
let :valid_options do
|
||||
super().merge('extract' => new_extract,
|
||||
'template' => new_template)
|
||||
end
|
||||
|
||||
it 'should revert extract and template options for an updated WebsiteAgent' do
|
||||
expect(agent.options).to include('extract' => new_extract,
|
||||
'template' => new_template)
|
||||
ConvertWebsiteAgentTemplateForMerge.new.down
|
||||
agent.reload
|
||||
expect(agent.options).to include('extract' => reverted_extract,
|
||||
'template' => reverted_template)
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,39 @@
|
|||
load 'spec/rails_helper.rb'
|
||||
load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
|
||||
|
||||
describe AddTemplatesToResolveUrl do
|
||||
let :valid_options do
|
||||
{
|
||||
'name' => "XKCD",
|
||||
'expected_update_period_in_days' => "2",
|
||||
'type' => "html",
|
||||
'url' => "http://xkcd.com",
|
||||
'mode' => 'on_change',
|
||||
'extract' => {
|
||||
'url' => { 'css' => "#comic img", 'value' => "@src" },
|
||||
'title' => { 'css' => "#comic img", 'value' => "@alt" },
|
||||
'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
let :agent do
|
||||
Agents::WebsiteAgent.create!(
|
||||
user: users(:bob),
|
||||
name: "xkcd",
|
||||
options: valid_options,
|
||||
keep_events_for: 2.days
|
||||
)
|
||||
end
|
||||
|
||||
it 'should add a template for an existing WebsiteAgent with `url`' do
|
||||
expect(agent.options).not_to include('template')
|
||||
AddTemplatesToResolveUrl.new.up
|
||||
agent.reload
|
||||
expect(agent.options).to include(
|
||||
'template' => {
|
||||
'url' => '{{ url | to_uri: _response_.url }}'
|
||||
}
|
||||
)
|
||||
end
|
||||
end
|
|
@ -74,6 +74,13 @@ describe Agent do
|
|||
Agent.run_schedule("midnight")
|
||||
end
|
||||
|
||||
it "ignores unknown types" do
|
||||
Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
|
||||
mock(Agents::WeatherAgent).bulk_check("midnight").once
|
||||
mock(Agents::WebsiteAgent).bulk_check("midnight").once
|
||||
Agent.run_schedule("midnight")
|
||||
end
|
||||
|
||||
it "only runs agents with the given schedule" do
|
||||
do_not_allow(Agents::WebsiteAgent).async_check
|
||||
Agent.run_schedule("blah")
|
||||
|
@ -283,13 +290,37 @@ describe Agent do
|
|||
Agent.receive!
|
||||
end
|
||||
|
||||
it "should not propogate to disabled Agents" do
|
||||
it "should not propagate to disabled Agents" do
|
||||
Agent.async_check(agents(:bob_weather_agent).id)
|
||||
agents(:bob_rain_notifier_agent).update_attribute :disabled, true
|
||||
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
|
||||
Agent.receive!
|
||||
end
|
||||
|
||||
it "should not propagate to Agents with unknown types" do
|
||||
Agent.async_check(agents(:jane_weather_agent).id)
|
||||
Agent.async_check(agents(:bob_weather_agent).id)
|
||||
|
||||
Agent.where(id: agents(:bob_rain_notifier_agent).id).update_all type: 'UnknownTypeAgent'
|
||||
|
||||
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
|
||||
mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
|
||||
|
||||
Agent.receive!
|
||||
end
|
||||
|
||||
it "should not propagate from Agents with unknown types" do
|
||||
Agent.async_check(agents(:jane_weather_agent).id)
|
||||
Agent.async_check(agents(:bob_weather_agent).id)
|
||||
|
||||
Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
|
||||
|
||||
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
|
||||
mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
|
||||
|
||||
Agent.receive!
|
||||
end
|
||||
|
||||
it "should log exceptions" do
|
||||
mock.any_instance_of(Agents::TriggerAgent).receive(anything).once {
|
||||
raise "foo"
|
||||
|
|
|
@ -1,145 +0,0 @@
|
|||
require 'rails_helper'
|
||||
|
||||
|
||||
describe Agents::BeeperAgent do
|
||||
let(:base_params) {
|
||||
{
|
||||
'type' => 'message',
|
||||
'app_id' => 'some-app-id',
|
||||
'api_key' => 'some-api-key',
|
||||
'sender_id' => 'sender-id',
|
||||
'phone' => '+111111111111',
|
||||
'text' => 'Some text'
|
||||
}
|
||||
}
|
||||
|
||||
subject {
|
||||
agent = described_class.new(name: 'beeper-agent', options: base_params)
|
||||
agent.user = users(:jane)
|
||||
agent.save! and return agent
|
||||
}
|
||||
|
||||
context 'validation' do
|
||||
it 'valid' do
|
||||
expect(subject).to be_valid
|
||||
end
|
||||
|
||||
[:type, :app_id, :api_key, :sender_id].each do |attr|
|
||||
it "invalid without #{attr}" do
|
||||
subject.options[attr] = nil
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
it 'invalid with group_id and phone' do
|
||||
subject.options['group_id'] ='some-group-id'
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
|
||||
context '#message' do
|
||||
it 'requires text' do
|
||||
subject.options[:text] = nil
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
context '#image' do
|
||||
before(:each) do
|
||||
subject.options[:type] = 'image'
|
||||
end
|
||||
|
||||
it 'invalid without image' do
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
|
||||
it 'valid with image' do
|
||||
subject.options[:image] = 'some-url'
|
||||
expect(subject).to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
context '#event' do
|
||||
before(:each) do
|
||||
subject.options[:type] = 'event'
|
||||
end
|
||||
|
||||
it 'invalid without start_time' do
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
|
||||
it 'valid with start_time' do
|
||||
subject.options[:start_time] = Time.now
|
||||
expect(subject).to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
context '#location' do
|
||||
before(:each) do
|
||||
subject.options[:type] = 'location'
|
||||
end
|
||||
|
||||
it 'invalid without latitude and longitude' do
|
||||
expect(subject).not_to be_valid
|
||||
end
|
||||
|
||||
it 'valid with latitude and longitude' do
|
||||
subject.options[:latitude] = 15.0
|
||||
subject.options[:longitude] = 16.0
|
||||
expect(subject).to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
context '#task' do
|
||||
before(:each) do
|
||||
subject.options[:type] = 'task'
|
||||
end
|
||||
|
||||
it 'valid with text' do
|
||||
expect(subject).to be_valid
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'payload_for' do
|
||||
it 'removes unwanted attributes' do
|
||||
result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text',
|
||||
'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'})
|
||||
expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}')
|
||||
end
|
||||
end
|
||||
|
||||
context 'headers' do
|
||||
it 'sets X-Beeper-Application-Id header with app_id' do
|
||||
expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id'])
|
||||
end
|
||||
|
||||
it 'sets X-Beeper-REST-API-Key header with api_key' do
|
||||
expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key'])
|
||||
end
|
||||
|
||||
it 'sets Content-Type' do
|
||||
expect(subject.send(:headers)['Content-Type']).to eq('application/json')
|
||||
end
|
||||
end
|
||||
|
||||
context 'endpoint_for' do
|
||||
it 'returns valid URL for message' do
|
||||
expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json')
|
||||
end
|
||||
|
||||
it 'returns valid URL for image' do
|
||||
expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json')
|
||||
end
|
||||
|
||||
it 'returns valid URL for event' do
|
||||
expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json')
|
||||
end
|
||||
|
||||
it 'returns valid URL for location' do
|
||||
expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json')
|
||||
end
|
||||
it 'returns valid URL for task' do
|
||||
expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json')
|
||||
end
|
||||
end
|
||||
end
|
|
@ -15,6 +15,7 @@ describe Agents::GoogleFlightsAgent do
|
|||
'origin' => 'BOS',
|
||||
'destination' => 'SFO',
|
||||
'date' => '2016-04-11',
|
||||
'preferredCabin' => 'COACH',
|
||||
'childCount' => 0,
|
||||
'infantInSeatCount' => 0,
|
||||
'infantInLapCount'=> 0,
|
||||
|
|
117
spec/models/agents/phantom_js_cloud_agent_spec.rb
Normal file
117
spec/models/agents/phantom_js_cloud_agent_spec.rb
Normal file
|
@ -0,0 +1,117 @@
|
|||
require 'rails_helper'
|
||||
|
||||
describe Agents::PhantomJsCloudAgent do
|
||||
before do
|
||||
|
||||
@valid_options = {
|
||||
'name' => "XKCD",
|
||||
'render_type' => "html",
|
||||
'url' => "http://xkcd.com",
|
||||
'mode' => 'clean',
|
||||
'api_key' => '1234567890'
|
||||
}
|
||||
|
||||
@checker = Agents::PhantomJsCloudAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
|
||||
@checker.user = users(:jane)
|
||||
@checker.save!
|
||||
end
|
||||
|
||||
describe "validations" do
|
||||
before do
|
||||
expect(@checker).to be_valid
|
||||
end
|
||||
|
||||
it "should validate the presence of url" do
|
||||
@checker.options['url'] = "http://google.com"
|
||||
expect(@checker).to be_valid
|
||||
|
||||
@checker.options['url'] = ""
|
||||
expect(@checker).not_to be_valid
|
||||
|
||||
@checker.options['url'] = nil
|
||||
expect(@checker).not_to be_valid
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
describe "emitting event" do
|
||||
it "should emit url as event" do
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should set render type as plain text" do
|
||||
@checker.options['render_type'] = 'plainText'
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22plainText%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should set output as json" do
|
||||
@checker.options['output_as_json'] = true
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22outputAsJson%22%3Atrue%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should not set ignore images" do
|
||||
@checker.options['ignore_images'] = false
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should set ignore images" do
|
||||
@checker.options['ignore_images'] = true
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22ignoreImages%22%3Atrue%2C%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should set wait interval to zero" do
|
||||
@checker.options['wait_interval'] = '0'
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%2C%22wait_interval%22%3A%220%22%7D%7D")
|
||||
end
|
||||
|
||||
it "should set user agent to BlackBerry" do
|
||||
@checker.options['user_agent'] = 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+'
|
||||
|
||||
expect {
|
||||
@checker.check
|
||||
}.to change { @checker.events.count }.by(1)
|
||||
|
||||
item,* = @checker.events.last(1)
|
||||
expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Mozilla%2F5.0%20%28BlackBerry%3B%20U%3B%20BlackBerry%209900%3B%20en%29%20AppleWebKit%2F534.11%2B%20%28KHTML%2C%20like%20Gecko%29%20Version%2F7.1.0.346%20Mobile%20Safari%2F534.11%2B%22%7D%7D")
|
||||
end
|
||||
|
||||
|
||||
|
||||
end
|
||||
|
||||
end
|
|
@ -13,6 +13,7 @@ describe Agents::RssAgent do
|
|||
stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
|
||||
stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
|
||||
stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
|
||||
stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
|
||||
end
|
||||
|
||||
let(:agent) do
|
||||
|
@ -295,6 +296,161 @@ describe Agents::RssAgent do
|
|||
event = agent.events.first
|
||||
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
|
||||
end
|
||||
|
||||
it "decodes the content properly with force_encoding specified" do
|
||||
@valid_options['force_encoding'] = 'iso-8859-1'
|
||||
agent.check
|
||||
event = agent.events.first
|
||||
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
|
||||
end
|
||||
end
|
||||
|
||||
context 'with podcast elements' do
|
||||
before do
|
||||
@valid_options['url'] = 'http://example.com/podcast.rss'
|
||||
@valid_options['include_feed_info'] = true
|
||||
end
|
||||
|
||||
let :feed_info do
|
||||
{
|
||||
"id" => nil,
|
||||
"type" => "rss",
|
||||
"url" => "http://www.example.com/podcasts/everything/index.html",
|
||||
"links" => [ { "href" => "http://www.example.com/podcasts/everything/index.html" } ],
|
||||
"title" => "All About Everything",
|
||||
"description" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
|
||||
"copyright" => "℗ & © 2014 John Doe & Family",
|
||||
"generator" => nil,
|
||||
"icon" => nil,
|
||||
"authors" => [
|
||||
"John Doe"
|
||||
],
|
||||
"date_published" => nil,
|
||||
"last_updated" => nil,
|
||||
"itunes_categories" => [
|
||||
"Technology", "Gadgets",
|
||||
"TV & Film",
|
||||
"Arts", "Food"
|
||||
],
|
||||
"itunes_complete" => "yes",
|
||||
"itunes_explicit" => "no",
|
||||
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything.jpg",
|
||||
"itunes_owners" => ["John Doe <john.doe@example.com>"],
|
||||
"itunes_subtitle" => "A show about everything",
|
||||
"itunes_summary" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
|
||||
"language" => "en-us"
|
||||
}
|
||||
end
|
||||
|
||||
it "is parsed correctly" do
|
||||
expect {
|
||||
agent.check
|
||||
}.to change { agent.events.count }.by(4)
|
||||
|
||||
expect(agent.events.map(&:payload)).to match([
|
||||
{
|
||||
"feed" => feed_info,
|
||||
"id" => "http://example.com/podcasts/archive/aae20140601.mp3",
|
||||
"url" => nil,
|
||||
"urls" => [],
|
||||
"links" => [],
|
||||
"title" => "Red,Whine, & Blue",
|
||||
"description" => nil,
|
||||
"content" => nil,
|
||||
"image" => nil,
|
||||
"enclosure" => {
|
||||
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3",
|
||||
"type" => "audio/mpeg",
|
||||
"length" => "498537"
|
||||
},
|
||||
"authors" => ["<Various>"],
|
||||
"categories" => [],
|
||||
"date_published" => "2016-03-11T01:15:00+00:00",
|
||||
"last_updated" => "2016-03-11T01:15:00+00:00",
|
||||
"itunes_duration" => "03:59",
|
||||
"itunes_explicit" => "no",
|
||||
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg",
|
||||
"itunes_subtitle" => "Red + Blue != Purple",
|
||||
"itunes_summary" => "This week we talk about surviving in a Red state if you are a Blue person. Or vice versa."
|
||||
},
|
||||
{
|
||||
"feed" => feed_info,
|
||||
"id" => "http://example.com/podcasts/archive/aae20140697.m4v",
|
||||
"url" => nil,
|
||||
"urls" => [],
|
||||
"links" => [],
|
||||
"title" => "The Best Chili",
|
||||
"description" => nil,
|
||||
"content" => nil,
|
||||
"image" => nil,
|
||||
"enclosure" => {
|
||||
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v",
|
||||
"type" => "video/x-m4v",
|
||||
"length" => "5650889"
|
||||
},
|
||||
"authors" => ["Jane Doe"],
|
||||
"categories" => [],
|
||||
"date_published" => "2016-03-10T02:00:00-07:00",
|
||||
"last_updated" => "2016-03-10T02:00:00-07:00",
|
||||
"itunes_closed_captioned" => "Yes",
|
||||
"itunes_duration" => "04:34",
|
||||
"itunes_explicit" => "no",
|
||||
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg",
|
||||
"itunes_subtitle" => "Jane and Eric",
|
||||
"itunes_summary" => "This week we talk about the best Chili in the world. Which chili is better?"
|
||||
},
|
||||
{
|
||||
"feed" => feed_info,
|
||||
"id" => "http://example.com/podcasts/archive/aae20140608.mp4",
|
||||
"url" => nil,
|
||||
"urls" => [],
|
||||
"links" => [],
|
||||
"title" => "Socket Wrench Shootout",
|
||||
"description" => nil,
|
||||
"content" => nil,
|
||||
"image" => nil,
|
||||
"enclosure" => {
|
||||
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4",
|
||||
"type" => "video/mp4",
|
||||
"length" => "5650889"
|
||||
},
|
||||
"authors" => ["Jane Doe"],
|
||||
"categories" => [],
|
||||
"date_published" => "2016-03-09T13:00:00-05:00",
|
||||
"last_updated" => "2016-03-09T13:00:00-05:00",
|
||||
"itunes_duration" => "04:34",
|
||||
"itunes_explicit" => "no",
|
||||
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg",
|
||||
"itunes_subtitle" => "Comparing socket wrenches is fun!",
|
||||
"itunes_summary" => "This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here."
|
||||
},
|
||||
{
|
||||
"feed" => feed_info,
|
||||
"id" => "http://example.com/podcasts/archive/aae20140615.m4a",
|
||||
"url" => nil,
|
||||
"urls" => [],
|
||||
"links" => [],
|
||||
"title" => "Shake Shake Shake Your Spices",
|
||||
"description" => nil,
|
||||
"content" => nil,
|
||||
"image" => nil,
|
||||
"enclosure" => {
|
||||
"url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a",
|
||||
"type" => "audio/x-m4a",
|
||||
"length" => "8727310"
|
||||
},
|
||||
"authors" => ["John Doe"],
|
||||
"categories" => [],
|
||||
"date_published" => "2016-03-08T12:00:00+00:00",
|
||||
"last_updated" => "2016-03-08T12:00:00+00:00",
|
||||
"itunes_duration" => "07:04",
|
||||
"itunes_explicit" => "no",
|
||||
"itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg",
|
||||
"itunes_subtitle" => "A short primer on table spices",
|
||||
"itunes_summary" => "This week we talk about <a href=\"https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11\">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!"
|
||||
}
|
||||
])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -651,28 +651,22 @@ describe Agents::WebsiteAgent do
|
|||
@checker.options = @valid_options
|
||||
@checker.check
|
||||
event = Event.last
|
||||
expect(event.payload['url']).to eq("http://imgs.xkcd.com/comics/evolving.png")
|
||||
expect(event.payload['title']).to eq("Evolving")
|
||||
expect(event.payload['hovertext']).to match(/^Biologists play reverse/)
|
||||
expect(event.payload).to match(
|
||||
'url' => 'http://imgs.xkcd.com/comics/evolving.png',
|
||||
'title' => 'Evolving',
|
||||
'hovertext' => /^Biologists play reverse/
|
||||
)
|
||||
end
|
||||
|
||||
it "should turn relative urls to absolute" do
|
||||
rel_site = {
|
||||
'name' => "XKCD",
|
||||
'expected_update_period_in_days' => "2",
|
||||
'type' => "html",
|
||||
'url' => "http://xkcd.com",
|
||||
'mode' => "on_change",
|
||||
'extract' => {
|
||||
'url' => {'css' => "#topLeft a", 'value' => "@href"},
|
||||
}
|
||||
}
|
||||
rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
|
||||
rel.user = users(:bob)
|
||||
rel.save!
|
||||
rel.check
|
||||
it "should exclude hidden keys" do
|
||||
@valid_options['extract']['hovertext']['hidden'] = true
|
||||
@checker.options = @valid_options
|
||||
@checker.check
|
||||
event = Event.last
|
||||
expect(event.payload['url']).to eq("http://xkcd.com/about")
|
||||
expect(event.payload).to match(
|
||||
'url' => 'http://imgs.xkcd.com/comics/evolving.png',
|
||||
'title' => 'Evolving'
|
||||
)
|
||||
end
|
||||
|
||||
it "should return an integer value if XPath evaluates to one" do
|
||||
|
@ -749,9 +743,9 @@ describe Agents::WebsiteAgent do
|
|||
expect(event.payload['original_url']).to eq('http://xkcd.com/index')
|
||||
end
|
||||
|
||||
it "should be formatted by template after extraction" do
|
||||
it "should format and merge values in template after extraction" do
|
||||
@valid_options['extract']['hovertext']['hidden'] = true
|
||||
@valid_options['template'] = {
|
||||
'url' => '{{url}}',
|
||||
'title' => '{{title | upcase}}',
|
||||
'summary' => '{{title}}: {{hovertext | truncate: 20}}',
|
||||
}
|
||||
|
@ -1185,7 +1179,11 @@ fire: hot
|
|||
'some_object' => {
|
||||
'some_data' => { hello: 'world', href: '/world' }.to_json
|
||||
},
|
||||
url: 'http://example.com/'
|
||||
url: 'http://example.com/',
|
||||
'headers' => {
|
||||
'Content-Type' => 'application/json'
|
||||
},
|
||||
'status' => 200
|
||||
}
|
||||
@event.save!
|
||||
|
||||
|
@ -1195,6 +1193,12 @@ fire: hot
|
|||
'extract' => {
|
||||
'value' => { 'path' => 'hello' },
|
||||
'url' => { 'path' => 'href' },
|
||||
},
|
||||
'template' => {
|
||||
'value' => '{{ value }}',
|
||||
'url' => '{{ url | to_uri: _response_.url }}',
|
||||
'type' => '{{ _response_.headers.content_type }}',
|
||||
'status' => '{{ _response_.status | as_object }}'
|
||||
}
|
||||
)
|
||||
end
|
||||
|
@ -1203,7 +1207,7 @@ fire: hot
|
|||
expect {
|
||||
@checker.receive([@event])
|
||||
}.to change { Event.count }.by(1)
|
||||
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world' })
|
||||
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200 })
|
||||
end
|
||||
|
||||
it "should support merge mode" do
|
||||
|
@ -1212,7 +1216,25 @@ fire: hot
|
|||
expect {
|
||||
@checker.receive([@event])
|
||||
}.to change { Event.count }.by(1)
|
||||
expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world'))
|
||||
expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
|
||||
end
|
||||
|
||||
it "should convert headers and status in the event data properly" do
|
||||
@event.payload[:status] = '201'
|
||||
@event.payload[:headers] = [['Content-Type', 'application/rss+xml']]
|
||||
expect {
|
||||
@checker.receive([@event])
|
||||
}.to change { Event.count }.by(1)
|
||||
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/rss+xml', 'status' => 201 })
|
||||
end
|
||||
|
||||
it "should ignore inconvertible headers and status in the event data" do
|
||||
@event.payload[:status] = 'ok'
|
||||
@event.payload[:headers] = ['Content-Type', 'Content-Length']
|
||||
expect {
|
||||
@checker.receive([@event])
|
||||
}.to change { Event.count }.by(1)
|
||||
expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => '', 'status' => nil })
|
||||
end
|
||||
|
||||
it "should output an error when nothing can be found at the path" do
|
||||
|
@ -1349,6 +1371,9 @@ fire: hot
|
|||
'mode' => 'all',
|
||||
'extract' => {
|
||||
'url' => { 'css' => "a", 'value' => "@href" },
|
||||
},
|
||||
'template' => {
|
||||
'url' => '{{ url | to_uri }}',
|
||||
}
|
||||
}
|
||||
@checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options)
|
||||
|
|
Loading…
Add table
Reference in a new issue