GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-16 03:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into rss_agent_logs_exact_feed_on_error

Browse source
This commit is contained in:
Andrew Cantino 2015-08-01 12:13:12 -07:00
commit fe1e806368
22 changed files with 655 additions and 148 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env.example
View file

@ -167,7 +167,7 @@ TIMEZONE="Pacific Time (US & Canada)"
FAILED_JOBS_TO_KEEP=100
# Maximum runtime of background jobs in minutes
DELAYED_JOB_MAX_RUNTIME=20
DELAYED_JOB_MAX_RUNTIME=2
# Amount of seconds for delayed_job to sleep before checking for new jobs
DELAYED_JOB_SLEEP_DELAY=10

7
CHANGES.md
View file

@ -1,12 +1,19 @@
# Changes
* Jul 30, 2015 - RssAgent can configure the order of events created via `events_order`.
* Jul 29, 2015 - WebsiteAgent can configure the order of events created via `events_order`.
* Jul 29, 2015 - DataOutputAgent can configure the order of events in the output via `events_order`.
* Jul 20, 2015 - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
* Jul 20, 2015 - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
* Jul 8, 2015 - DataOutputAgent supports feed icon, and a new template variable `events`.
* Jul 1, 2015 - DeDuplicationAgent properly handles destruction of memory.
* Jun 26, 2015 - Add `max_events_per_run` to RssAgent.
* Jun 19, 2015 - Add `url_from_event` to WebsiteAgent.
* Jun 17, 2015 - RssAgent emits events for new feed items in chronological order.
* Jun 17, 2015 - Liquid filter `unescape` added.
* Jun 17, 2015 - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
* Jun 15, 2015 - Liquid filter `uri_expand` added.
* Jun 13, 2015 - Liquid templating engine is upgraded to version 3.
* Jun 12, 2015 - RSSAgent can now accept an array of URLs.
* Jun 8, 2015 - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
* May 27, 2015 - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.

3
Gemfile
View file

@ -1,5 +1,8 @@
source 'https://rubygems.org'
# Ruby 2.0 is the minimum requirement
ruby ['2.0.0', RUBY_VERSION].max
# Optional libraries. To conserve RAM, comment out any that you don't need,
# then run `bundle` and commit the updated Gemfile and Gemfile.lock.
gem 'twilio-ruby', '~> 3.11.5' # TwilioAgent

5
Gemfile.lock
View file

@ -235,7 +235,7 @@ GEM
launchy (2.4.2)
addressable (~> 2.3)
libv8 (3.16.14.7)
liquid (3.0.3)
liquid (3.0.6)
listen (2.7.9)
celluloid (>= 0.15.2)
rb-fsevent (>= 0.9.3)
@ -580,6 +580,3 @@ DEPENDENCIES
weibo_2!
wunderground (~> 1.2.0)
xmpp4r (~> 0.5.6)
BUNDLED WITH
1.10.5

6
README.md
View file

@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif
## Deployment
[![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup. Be sure to click 'View it' after launch!)
Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup. Be sure to click 'View it' after launch!)
Huginn can run on Heroku for free! Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
### Optional Setup

39
app/concerns/dry_runnable.rb
View file

@ -1,10 +1,8 @@
module DryRunnable
def dry_run!
readonly!
extend ActiveSupport::Concern
class << self
prepend Sandbox
end
def dry_run!
@dry_run = true
log = StringIO.new
@dry_run_logger = Logger.new(log)
@ -14,6 +12,7 @@ module DryRunnable
begin
raise "#{short_type} does not support dry-run" unless can_dry_run?
readonly!
check
rescue => e
error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
@ -23,28 +22,38 @@ module DryRunnable
memory: memory,
log: log.string,
)
ensure
@dry_run = false
end
def dry_run?
is_a? Sandbox
!!@dry_run
end
module Sandbox
included do
prepend Wrapper
end
module Wrapper
attr_accessor :results
def logger
return super unless dry_run?
@dry_run_logger
end
def save
valid?
def save(options = {})
return super unless dry_run?
perform_validations(options)
end
def save!
save or raise ActiveRecord::RecordNotSaved
def save!(options = {})
return super unless dry_run?
save(options) or raise_record_invalid
end
def log(message, options = {})
return super unless dry_run?
case options[:level] || 3
when 0..2
sev = Logger::DEBUG
@ -57,10 +66,12 @@ module DryRunnable
logger.log(sev, message)
end
def create_event(event_hash)
def create_event(event)
return super unless dry_run?
if can_create_events?
@dry_run_results[:events] << event_hash[:payload]
events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
event = build_event(event)
@dry_run_results[:events] << event.payload
event
else
error "This Agent cannot create events!"
end

161
app/concerns/sortable_events.rb Normal file
View file

@ -0,0 +1,161 @@
module SortableEvents
extend ActiveSupport::Concern
included do
validate :validate_events_order
end
def description_events_order(*args)
self.class.description_events_order(*args)
end
module ClassMethods
def can_order_created_events!
raise if cannot_create_events?
prepend AutomaticSorter
end
def can_order_created_events?
include? AutomaticSorter
end
def cannot_order_created_events?
!can_order_created_events?
end
def description_events_order(events = 'events created in each run')
<<-MD.lstrip
To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
* _expression_ is a Liquid template to generate a string to be used as sort key.
* _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
* _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
Sort keys listed earlier take precedence over ones listed later. For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
Sorting is done stably, so even if all events have the same set of sort key values the original order is retained. Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
MD
end
end
def can_order_created_events?
self.class.can_order_created_events?
end
def cannot_order_created_events?
self.class.cannot_order_created_events?
end
def events_order
options['events_order']
end
module AutomaticSorter
def check
return super unless events_order
sorting_events do
super
end
end
def receive(incoming_events)
return super unless events_order
# incoming events should be processed sequentially
incoming_events.each do |event|
sorting_events do
super([event])
end
end
end
def create_event(event)
if @sortable_events
event = build_event(event)
@sortable_events << event
event
else
super
end
end
private
def sorting_events(&block)
@sortable_events = []
yield
ensure
events, @sortable_events = @sortable_events, nil
sort_events(events).each do |event|
create_event(event)
end
end
end
private
EXPRESSION_PARSER = {
'string' => ->string { string },
'number' => ->string { string.to_f },
'time' => ->string { Time.zone.parse(string) },
}
EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
def validate_events_order
case order_by = events_order
when nil
when Array
# Each tuple may be either [expression, type, desc] or just
# expression.
order_by.each do |expression, type, desc|
case expression
when String
# ok
else
errors.add(:base, "first element of each events_order tuple must be a Liquid template")
break
end
case type
when nil, *EXPRESSION_TYPES
# ok
else
errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
break
end
if !desc.nil? && boolify(desc).nil?
errors.add(:base, "third element of each events_order tuple must be a boolean value")
break
end
end
else
errors.add(:base, "events_order must be an array of arrays")
end
end
# Sort given events in order specified by the "events_order" option
def sort_events(events)
order_by = events_order.presence or
return events
orders = order_by.map { |_, _, desc = false| boolify(desc) }
Utils.sort_tuples!(
events.map.with_index { |event, index|
interpolate_with(event) {
interpolation_context['_index_'] = index
order_by.map { |expression, type, _|
string = interpolate_string(expression)
begin
EXPRESSION_PARSER[type || 'string'.freeze][string]
rescue
error "Cannot parse #{string.inspect} as #{type}; treating it as string"
string
end
}
} << index << event # index is to make sorting stable
},
orders
).collect!(&:last)
end
end

1
app/helpers/application_helper.rb
View file

@ -80,6 +80,7 @@ module ApplicationHelper
end
def service_label(service)
return if service.nil?
content_tag :span, [
omniauth_provider_icon(service.provider),
service_label_text(service)

18
app/models/agent.rb
View file

@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base
include HasGuid
include LiquidDroppable
include DryRunnable
include SortableEvents
markdown_class_attributes :description, :event_description
@ -104,12 +105,19 @@ class Agent < ActiveRecord::Base
raise "Implement me in your subclass"
end
def create_event(attrs)
def build_event(event)
event = events.build(event) if event.is_a?(Hash)
event.agent = self
event.user = user
event.expires_at ||= new_event_expiration_date
event
end
def create_event(event)
if can_create_events?
events.create!({
:user => user,
:expires_at => new_event_expiration_date
}.merge(attrs))
event = build_event(event)
event.save!
event
else
error "This Agent cannot create events!"
end

8
app/models/agents/data_output_agent.rb
View file

@ -40,11 +40,15 @@ module Agents
"_contents": "tag contents (can be an object for nesting)"
}
# Ordering events in the output
#{description_events_order('events in the output')}
# Liquid Templating
In Liquid templating, the following variable is available:
* `events`: An array of events being output, sorted in descending order up to `events_to_show` in number. For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
* `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number. For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
MD
end
@ -134,7 +138,7 @@ module Agents
end
end
source_events = received_events.order(id: :desc).limit(events_to_show).to_a
source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
interpolation_context.stack do
interpolation_context['events'] = source_events

49
app/models/agents/rss_agent.rb
View file

@ -9,6 +9,8 @@ module Agents
can_dry_run!
default_schedule "every_1d"
DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
description do
<<-MD
This Agent consumes RSS feeds and emits events when they change.
@ -29,6 +31,12 @@ module Agents
* `disable_url_encoding` - Set to `true` to disable url encoding.
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
* `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
# Ordering Events
#{description_events_order}
In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
MD
end
@ -70,6 +78,11 @@ module Agents
end
validate_web_request_options!
validate_events_order
end
def events_order
super.presence || DEFAULT_EVENTS_ORDER
end
def check
@ -84,26 +97,15 @@ module Agents
response = faraday.get(url)
if response.success?
feed = FeedNormalizer::FeedNormalizer.parse(response.body)
feed.clean! if interpolated['clean'] == 'true'
feed.clean! if boolify(interpolated['clean'])
max_events = (interpolated['max_events_per_run'].presence || 0).to_i
created_event_count = 0
feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
sort_events(feed_to_events(feed)).each.with_index do |event, index|
break if max_events && max_events > 0 && index >= max_events
entry_id = get_entry_id(entry)
entry_id = event.payload[:id]
if check_and_track(entry_id)
created_event_count += 1
create_event(payload: {
id: entry_id,
date_published: entry.date_published,
last_updated: entry.last_updated,
url: entry.url,
urls: entry.urls,
description: entry.description,
content: entry.content,
title: entry.title,
authors: entry.authors,
categories: entry.categories
})
create_event(event)
end
end
log "Fetched #{url} and created #{created_event_count} event(s)."
@ -128,5 +130,22 @@ module Agents
true
end
end
def feed_to_events(feed)
feed.entries.map { |entry|
Event.new(payload: {
id: get_entry_id(entry),
date_published: entry.date_published,
last_updated: entry.last_updated,
url: entry.url,
urls: entry.urls,
description: entry.description,
content: entry.content,
title: entry.title,
authors: entry.authors,
categories: entry.categories
})
}
end
end
end

5
app/models/agents/website_agent.rb
View file

@ -6,6 +6,7 @@ module Agents
include WebRequestConcern
can_dry_run!
can_order_created_events!
default_schedule "every_12h"
@ -105,6 +106,10 @@ module Agents
* `status`: HTTP status as integer. (Almost always 200)
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_.
# Ordering Events
#{description_events_order}
MD
event_description do

15
app/models/scenario_import.rb
View file

@ -278,21 +278,6 @@ class ScenarioImport
yield 'disabled', disabled, boolean if disabled.requires_merge?
end
# Unfortunately Ruby 1.9's OpenStruct doesn't expose [] and []=.
unless instance_methods.include?(:[]=)
def [](key)
self.send(sanitize key)
end
def []=(key, val)
self.send("#{sanitize key}=", val)
end
def sanitize(key)
key.gsub(/[^a-zA-Z0-9_-]/, '')
end
end
def agent_instance
"Agents::#{self.type.updated}".constantize.new
end

2
config/initializers/delayed_job.rb
View file

@ -1,6 +1,6 @@
Delayed::Worker.destroy_failed_jobs = false
Delayed::Worker.max_attempts = 5
Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 2).to_i.minutes
Delayed::Worker.read_ahead = 5
Delayed::Worker.default_priority = 10
Delayed::Worker.delay_jobs = !Rails.env.test?

3
lib/ar_mysql_column_charset.rb
View file

@ -1,6 +1,3 @@
# Module#prepend support for Ruby 1.9
require 'prepend' unless Module.method_defined?(:prepend)
require 'active_support'
ActiveSupport.on_load :active_record do

85
lib/prepend.rb
View file

@ -1,85 +0,0 @@
# Fake implementation of prepend(), which does not support overriding
# inherited methods nor methods that are formerly overridden by
# another invocation of prepend().
#
# Here's what <Original>.prepend(<Wrapper>) does:
#
# - Create an anonymous stub module (hereinafter <Stub>) and define
# <Stub>#<method> that calls #<method>_without_<Wrapper> for each
# instance method of <Wrapper>.
#
# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
# instance method of <Wrapper>.
#
# - Include <Stub> and <Wrapper> into <Original> in that order.
#
# This way, a call of <Original>#<method> is dispatched to
# <Wrapper><method>, which may call super which is dispatched to
# <Stub>#<method>, which finally calls
# <Original>#<method>_without_<Wrapper> which is used to be called
# <Original>#<method>.
#
# Usage:
#
# class Mechanize
# # module with methods that overrides those of X
# module Y
# end
#
# unless X.respond_to?(:prepend, true)
# require 'mechanize/prependable'
# X.extend(Prependable)
# end
#
# class X
# prepend Y
# end
# end
class Module
def prepend(mod)
stub = Module.new
mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
mod.instance_methods.each { |name|
method_defined?(name) or next
original = instance_method(name)
next if original.owner != self
name = name.to_s
name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
arity = original.arity
arglist = (
if arity >= 0
(1..arity).map { |i| 'x%d' % i }
else
(1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
end << '&b'
).join(', ')
if name.end_with?('=')
stub.module_eval %{
def #{name}(#{arglist})
__send__(:#{name_without}, #{arglist})
end
}
else
stub.module_eval %{
def #{name}(#{arglist})
#{name_without}(#{arglist})
end
}
end
module_eval {
alias_method name_without, name
remove_method name
}
}
include stub
include mod
end
private :prepend
end unless Module.method_defined?(:prepend)

39
lib/utils.rb
View file

@ -79,4 +79,43 @@ module Utils
def self.pretty_jsonify(thing)
JSON.pretty_generate(thing).gsub('</', '<\/')
end
class TupleSorter
class SortableTuple
attr_reader :array
# The <=> method will call orders[n] to determine if the nth element
# should be compared in descending order.
def initialize(array, orders = [])
@array = array
@orders = orders
end
def <=> other
other = other.array
@array.each_with_index do |e, i|
o = other[i]
case cmp = e <=> o || e.to_s <=> o.to_s
when 0
next
else
return @orders[i] ? -cmp : cmp
end
end
0
end
end
class << self
def sort!(array, orders = [])
array.sort_by! do |e|
SortableTuple.new(e, orders)
end
end
end
end
def self.sort_tuples!(array, orders = [])
TupleSorter.sort!(array, orders)
end
end

264
spec/concerns/sortable_events_spec.rb Normal file
View file

@ -0,0 +1,264 @@
require 'spec_helper'
describe SortableEvents do
let(:agent_class) {
Class.new(Agent) do
include SortableEvents
default_schedule 'never'
def self.valid_type?(name)
true
end
end
}
def new_agent(events_order = nil)
options = {}
options['events_order'] = events_order if events_order
agent_class.new(name: 'test', options: options) { |agent|
agent.user = users(:bob)
}
end
describe 'validations' do
let(:agent_class) {
Class.new(Agent) do
include SortableEvents
default_schedule 'never'
def self.valid_type?(name)
true
end
end
}
def new_agent(events_order = nil)
options = {}
options['events_order'] = events_order if events_order
agent_class.new(name: 'test', options: options) { |agent|
agent.user = users(:bob)
}
end
it 'should allow events_order to be unspecified, null or an empty array' do
expect(new_agent()).to be_valid
expect(new_agent(nil)).to be_valid
expect(new_agent([])).to be_valid
end
it 'should not allow events_order to be a non-array object' do
agent = new_agent(0)
expect(agent).not_to be_valid
expect(agent.errors[:base]).to include(/events_order/)
agent = new_agent('')
expect(agent).not_to be_valid
expect(agent.errors[:base]).to include(/events_order/)
agent = new_agent({})
expect(agent).not_to be_valid
expect(agent.errors[:base]).to include(/events_order/)
end
it 'should not allow events_order to be an array containing unexpected objects' do
agent = new_agent(['{{key}}', 1])
expect(agent).not_to be_valid
expect(agent.errors[:base]).to include(/events_order/)
agent = new_agent(['{{key1}}', ['{{key2}}', 'unknown']])
expect(agent).not_to be_valid
expect(agent.errors[:base]).to include(/events_order/)
end
it 'should allow events_order to be an array containing strings and valid tuples' do
agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number']])
expect(agent).to be_valid
agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number'], ['{{key4}}', 'time', true]])
expect(agent).to be_valid
end
end
describe 'sort_events' do
let(:payloads) {
[
{ 'title' => 'TitleA', 'score' => 4, 'updated_on' => '7 Jul 2015' },
{ 'title' => 'TitleB', 'score' => 2, 'updated_on' => '25 Jun 2014' },
{ 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
{ 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
]
}
let(:events) {
payloads.map { |payload| Event.new(payload: payload) }
}
it 'should sort events by a given key' do
agent = new_agent(['{{title}}'])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleC TitleD])
agent = new_agent([['{{title}}', 'string', true]])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleD TitleC TitleB TitleA])
end
it 'should sort events by multiple keys' do
agent = new_agent([['{{score}}', 'number'], '{{title}}'])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleC TitleD])
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
end
it 'should sort events by time' do
agent = new_agent([['{{updated_on}}', 'time']])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleD TitleC TitleA])
end
it 'should sort events stably' do
agent = new_agent(['<constant>'])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
agent = new_agent([['<constant>', 'string', true]])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
end
it 'should support _index_' do
agent = new_agent([['{{_index_}}', 'number', true]])
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleC TitleD TitleB TitleA])
end
end
describe 'automatic event sorter' do
describe 'declaration' do
let(:passive_agent_class) {
Class.new(Agent) do
include SortableEvents
cannot_create_events!
end
}
let(:active_agent_class) {
Class.new(Agent) do
include SortableEvents
end
}
describe 'can_order_created_events!' do
it 'should refuse to work if called from an Agent that cannot create events' do
expect {
passive_agent_class.class_eval do
can_order_created_events!
end
}.to raise_error
end
it 'should work if called from an Agent that can create events' do
expect {
active_agent_class.class_eval do
can_order_created_events!
end
}.not_to raise_error
end
end
describe 'can_order_created_events?' do
it 'should return false unless an Agent declares can_order_created_events!' do
expect(active_agent_class.can_order_created_events?).to eq(false)
expect(active_agent_class.new.can_order_created_events?).to eq(false)
end
it 'should return true if an Agent declares can_order_created_events!' do
active_agent_class.class_eval do
can_order_created_events!
end
expect(active_agent_class.can_order_created_events?).to eq(true)
expect(active_agent_class.new.can_order_created_events?).to eq(true)
end
end
end
describe 'behavior' do
class Agents::EventOrderableAgent < Agent
include SortableEvents
default_schedule 'never'
can_order_created_events!
attr_accessor :payloads_to_emit
def self.valid_type?(name)
true
end
def check
payloads_to_emit.each do |payload|
create_event payload: payload
end
end
def receive(events)
events.each do |event|
payloads_to_emit.each do |payload|
create_event payload: payload.merge('title' => payload['title'] + event.payload['title_suffix'])
end
end
end
end
def new_agent(events_order = nil)
options = {}
options['events_order'] = events_order if events_order
Agents::EventOrderableAgent.new(name: 'test', options: options) { |agent|
agent.user = users(:bob)
agent.payloads_to_emit = payloads
}
end
let(:payloads) {
[
{ 'title' => 'TitleA', 'score' => 4, 'updated_on' => '7 Jul 2015' },
{ 'title' => 'TitleB', 'score' => 2, 'updated_on' => '25 Jun 2014' },
{ 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
{ 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
]
}
it 'should keep the order of created events unless events_order is specified' do
[[], [nil], [[]]].each do |args|
agent = new_agent(*args)
agent.save!
expect { agent.check }.to change { Event.count }.by(4)
events = agent.events.last(4).sort_by(&:id)
expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
end
end
it 'should sort events created in check() in the order specified in events_order' do
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
agent.save!
expect { agent.check }.to change { Event.count }.by(4)
events = agent.events.last(4).sort_by(&:id)
expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
end
it 'should sort events created in receive() in the order specified in events_order' do
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
agent.save!
expect {
agent.receive([Event.new(payload: { 'title_suffix' => ' [new]' }),
Event.new(payload: { 'title_suffix' => ' [popular]' })])
}.to change { Event.count }.by(8)
events = agent.events.last(8).sort_by(&:id)
expect(events.map { |event| event.payload['title'] }).to eq([
'TitleB [new]', 'TitleA [new]', 'TitleD [new]', 'TitleC [new]',
'TitleB [popular]', 'TitleA [popular]', 'TitleD [popular]', 'TitleC [popular]',
])
end
end
end
end

2
spec/controllers/agents_controller_spec.rb
View file

@ -372,7 +372,7 @@ describe AgentsController do
sign_in users(:bob)
agent = agents(:bob_weather_agent)
expect {
post :dry_run, id: agents(:bob_website_agent), agent: valid_attributes(name: 'New Name')
post :dry_run, id: agent, agent: valid_attributes(name: 'New Name')
}.not_to change {
[users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
}

58
spec/lib/utils_spec.rb
View file

@ -114,4 +114,62 @@ describe Utils do
expect(cleaned_json).to include("<\\/script>")
end
end
describe "#sort_tuples!" do
let(:tuples) {
time = Time.now
[
[2, "a", time - 1], # 0
[2, "b", time - 1], # 1
[1, "b", time - 1], # 2
[1, "b", time], # 3
[1, "a", time], # 4
[2, "a", time + 1], # 5
[2, "a", time], # 6
]
}
it "sorts tuples like arrays by default" do
expected = tuples.values_at(4, 2, 3, 0, 6, 5, 1)
Utils.sort_tuples!(tuples)
expect(tuples).to eq expected
end
it "sorts tuples in order specified: case 1" do
# order by x1 asc, x2 desc, c3 asc
orders = [false, true, false]
expected = tuples.values_at(2, 3, 4, 1, 0, 6, 5)
Utils.sort_tuples!(tuples, orders)
expect(tuples).to eq expected
end
it "sorts tuples in order specified: case 2" do
# order by x1 desc, x2 asc, c3 desc
orders = [true, false, true]
expected = tuples.values_at(5, 6, 0, 1, 4, 3, 2)
Utils.sort_tuples!(tuples, orders)
expect(tuples).to eq expected
end
it "always succeeds in sorting even if it finds pairs of incomparable objects" do
time = Time.now
tuples = [
[2, "a", time - 1], # 0
[1, "b", nil], # 1
[1, "b", time], # 2
["2", nil, time], # 3
[1, nil, time], # 4
[nil, "a", time + 1], # 5
[2, "a", time], # 6
]
orders = [true, false, true]
expected = tuples.values_at(3, 6, 0, 4, 2, 1, 5)
Utils.sort_tuples!(tuples, orders)
expect(tuples).to eq expected
end
end
end

16
spec/models/agents/data_output_agent_spec.rb
View file

@ -209,6 +209,22 @@ describe Agents::DataOutputAgent do
})
end
describe 'ordering' do
before do
agent.options['events_order'] = ['{{title}}']
end
it 'can reorder the events_to_show last events based on a Liquid expression' do
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
agent.options['events_order'] = [['{{title}}', 'string', true]]
desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
expect(desc_content['items']).to eq(asc_content['items'].reverse)
end
end
describe "interpolating \"events\"" do
before do
agent.options['template']['title'] = "XKCD comics as a feed{% if events.first.site_title %} ({{events.first.site_title}}){% endif %}"

15
spec/models/agents/rss_agent_spec.rb
View file

@ -66,6 +66,21 @@ describe Agents::RssAgent do
expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/d465158f77dcd9078697e6167b50abbfdfa8b1af"])
end
it "should emit items as events in the order specified in the events_order option" do
expect {
agent.options['events_order'] = ['{{title | replace_regex: "^[[:space:]]+", "" }}']
agent.check
}.to change { agent.events.count }.by(20)
first, *, last = agent.events.last(20)
expect(first.payload['title'].strip).to eq('upgrade rails and gems')
expect(first.payload['url']).to eq("https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01")
expect(first.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01"])
expect(last.payload['title'].strip).to eq('Dashed line in a diagram indicates propagate_immediately being false.')
expect(last.payload['url']).to eq("https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535")
expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535"])
end
it "should track ids and not re-emit the same item when seen again" do
agent.check
expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| e.payload['id'] })