mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-16 03:41:41 +00:00
Merge branch 'master' into rss_agent_logs_exact_feed_on_error
This commit is contained in:
commit
fe1e806368
22 changed files with 655 additions and 148 deletions
|
@ -167,7 +167,7 @@ TIMEZONE="Pacific Time (US & Canada)"
|
|||
FAILED_JOBS_TO_KEEP=100
|
||||
|
||||
# Maximum runtime of background jobs in minutes
|
||||
DELAYED_JOB_MAX_RUNTIME=20
|
||||
DELAYED_JOB_MAX_RUNTIME=2
|
||||
|
||||
# Amount of seconds for delayed_job to sleep before checking for new jobs
|
||||
DELAYED_JOB_SLEEP_DELAY=10
|
||||
|
|
|
@ -1,12 +1,19 @@
|
|||
# Changes
|
||||
|
||||
* Jul 30, 2015 - RssAgent can configure the order of events created via `events_order`.
|
||||
* Jul 29, 2015 - WebsiteAgent can configure the order of events created via `events_order`.
|
||||
* Jul 29, 2015 - DataOutputAgent can configure the order of events in the output via `events_order`.
|
||||
* Jul 20, 2015 - Control Links (used by the SchedularAgent) are correctly exported in Scenarios.
|
||||
* Jul 20, 2015 - keep\_events\_for was moved from days to seconds; Scenarios have a schema verison.
|
||||
* Jul 8, 2015 - DataOutputAgent supports feed icon, and a new template variable `events`.
|
||||
* Jul 1, 2015 - DeDuplicationAgent properly handles destruction of memory.
|
||||
* Jun 26, 2015 - Add `max_events_per_run` to RssAgent.
|
||||
* Jun 19, 2015 - Add `url_from_event` to WebsiteAgent.
|
||||
* Jun 17, 2015 - RssAgent emits events for new feed items in chronological order.
|
||||
* Jun 17, 2015 - Liquid filter `unescape` added.
|
||||
* Jun 17, 2015 - Liquid filter `regex_replace` and `regex_replace_first` added, with escape sequence support.
|
||||
* Jun 15, 2015 - Liquid filter `uri_expand` added.
|
||||
* Jun 13, 2015 - Liquid templating engine is upgraded to version 3.
|
||||
* Jun 12, 2015 - RSSAgent can now accept an array of URLs.
|
||||
* Jun 8, 2015 - WebsiteAgent includes a `use_namespaces` option to enable XML namespaces.
|
||||
* May 27, 2015 - Validation warns user if they have not provided a `path` when using JSONPath in WebsiteAgent.
|
||||
|
|
3
Gemfile
3
Gemfile
|
@ -1,5 +1,8 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
# Ruby 2.0 is the minimum requirement
|
||||
ruby ['2.0.0', RUBY_VERSION].max
|
||||
|
||||
# Optional libraries. To conserve RAM, comment out any that you don't need,
|
||||
# then run `bundle` and commit the updated Gemfile and Gemfile.lock.
|
||||
gem 'twilio-ruby', '~> 3.11.5' # TwilioAgent
|
||||
|
|
|
@ -235,7 +235,7 @@ GEM
|
|||
launchy (2.4.2)
|
||||
addressable (~> 2.3)
|
||||
libv8 (3.16.14.7)
|
||||
liquid (3.0.3)
|
||||
liquid (3.0.6)
|
||||
listen (2.7.9)
|
||||
celluloid (>= 0.15.2)
|
||||
rb-fsevent (>= 0.9.3)
|
||||
|
@ -580,6 +580,3 @@ DEPENDENCIES
|
|||
weibo_2!
|
||||
wunderground (~> 1.2.0)
|
||||
xmpp4r (~> 0.5.6)
|
||||
|
||||
BUNDLED WITH
|
||||
1.10.5
|
||||
|
|
|
@ -80,9 +80,11 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif
|
|||
|
||||
## Deployment
|
||||
|
||||
[](https://heroku.com/deploy) (Takes a few minutes to setup. Be sure to click 'View it' after launch!)
|
||||
Try Huginn on Heroku: [](https://heroku.com/deploy) (Takes a few minutes to setup. Be sure to click 'View it' after launch!)
|
||||
|
||||
Huginn can run on Heroku for free! Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
|
||||
Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container.
|
||||
|
||||
Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers.
|
||||
|
||||
### Optional Setup
|
||||
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
module DryRunnable
|
||||
def dry_run!
|
||||
readonly!
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
class << self
|
||||
prepend Sandbox
|
||||
end
|
||||
def dry_run!
|
||||
@dry_run = true
|
||||
|
||||
log = StringIO.new
|
||||
@dry_run_logger = Logger.new(log)
|
||||
|
@ -14,6 +12,7 @@ module DryRunnable
|
|||
|
||||
begin
|
||||
raise "#{short_type} does not support dry-run" unless can_dry_run?
|
||||
readonly!
|
||||
check
|
||||
rescue => e
|
||||
error "Exception during dry-run. #{e.message}: #{e.backtrace.join("\n")}"
|
||||
|
@ -23,28 +22,38 @@ module DryRunnable
|
|||
memory: memory,
|
||||
log: log.string,
|
||||
)
|
||||
ensure
|
||||
@dry_run = false
|
||||
end
|
||||
|
||||
def dry_run?
|
||||
is_a? Sandbox
|
||||
!!@dry_run
|
||||
end
|
||||
|
||||
module Sandbox
|
||||
included do
|
||||
prepend Wrapper
|
||||
end
|
||||
|
||||
module Wrapper
|
||||
attr_accessor :results
|
||||
|
||||
def logger
|
||||
return super unless dry_run?
|
||||
@dry_run_logger
|
||||
end
|
||||
|
||||
def save
|
||||
valid?
|
||||
def save(options = {})
|
||||
return super unless dry_run?
|
||||
perform_validations(options)
|
||||
end
|
||||
|
||||
def save!
|
||||
save or raise ActiveRecord::RecordNotSaved
|
||||
def save!(options = {})
|
||||
return super unless dry_run?
|
||||
save(options) or raise_record_invalid
|
||||
end
|
||||
|
||||
def log(message, options = {})
|
||||
return super unless dry_run?
|
||||
case options[:level] || 3
|
||||
when 0..2
|
||||
sev = Logger::DEBUG
|
||||
|
@ -57,10 +66,12 @@ module DryRunnable
|
|||
logger.log(sev, message)
|
||||
end
|
||||
|
||||
def create_event(event_hash)
|
||||
def create_event(event)
|
||||
return super unless dry_run?
|
||||
if can_create_events?
|
||||
@dry_run_results[:events] << event_hash[:payload]
|
||||
events.build({ user: user, expires_at: new_event_expiration_date }.merge(event_hash))
|
||||
event = build_event(event)
|
||||
@dry_run_results[:events] << event.payload
|
||||
event
|
||||
else
|
||||
error "This Agent cannot create events!"
|
||||
end
|
||||
|
|
161
app/concerns/sortable_events.rb
Normal file
161
app/concerns/sortable_events.rb
Normal file
|
@ -0,0 +1,161 @@
|
|||
module SortableEvents
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
included do
|
||||
validate :validate_events_order
|
||||
end
|
||||
|
||||
def description_events_order(*args)
|
||||
self.class.description_events_order(*args)
|
||||
end
|
||||
|
||||
module ClassMethods
|
||||
def can_order_created_events!
|
||||
raise if cannot_create_events?
|
||||
prepend AutomaticSorter
|
||||
end
|
||||
|
||||
def can_order_created_events?
|
||||
include? AutomaticSorter
|
||||
end
|
||||
|
||||
def cannot_order_created_events?
|
||||
!can_order_created_events?
|
||||
end
|
||||
|
||||
def description_events_order(events = 'events created in each run')
|
||||
<<-MD.lstrip
|
||||
To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
|
||||
|
||||
* _expression_ is a Liquid template to generate a string to be used as sort key.
|
||||
|
||||
* _type_ (optional) is one of `string` (default), `number` and `time`, which specifies how to evaluate _expression_ for comparison.
|
||||
|
||||
* _descending_ (optional) is a boolean value to determine if comparison should be done in descending (reverse) order, which defaults to `false`.
|
||||
|
||||
Sort keys listed earlier take precedence over ones listed later. For example, if you want to sort articles by the date and then by the author, specify `[["{{date}}", "time"], "{{author}}"]`.
|
||||
|
||||
Sorting is done stably, so even if all events have the same set of sort key values the original order is retained. Also, a special Liquid variable `_index_` is provided, which contains the zero-based index number of each event, which means you can exactly reverse the order of events by specifying `[["{{_index_}}", "number", true]]`.
|
||||
MD
|
||||
end
|
||||
end
|
||||
|
||||
def can_order_created_events?
|
||||
self.class.can_order_created_events?
|
||||
end
|
||||
|
||||
def cannot_order_created_events?
|
||||
self.class.cannot_order_created_events?
|
||||
end
|
||||
|
||||
def events_order
|
||||
options['events_order']
|
||||
end
|
||||
|
||||
module AutomaticSorter
|
||||
def check
|
||||
return super unless events_order
|
||||
sorting_events do
|
||||
super
|
||||
end
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
return super unless events_order
|
||||
# incoming events should be processed sequentially
|
||||
incoming_events.each do |event|
|
||||
sorting_events do
|
||||
super([event])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def create_event(event)
|
||||
if @sortable_events
|
||||
event = build_event(event)
|
||||
@sortable_events << event
|
||||
event
|
||||
else
|
||||
super
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def sorting_events(&block)
|
||||
@sortable_events = []
|
||||
yield
|
||||
ensure
|
||||
events, @sortable_events = @sortable_events, nil
|
||||
sort_events(events).each do |event|
|
||||
create_event(event)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
EXPRESSION_PARSER = {
|
||||
'string' => ->string { string },
|
||||
'number' => ->string { string.to_f },
|
||||
'time' => ->string { Time.zone.parse(string) },
|
||||
}
|
||||
EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
|
||||
|
||||
def validate_events_order
|
||||
case order_by = events_order
|
||||
when nil
|
||||
when Array
|
||||
# Each tuple may be either [expression, type, desc] or just
|
||||
# expression.
|
||||
order_by.each do |expression, type, desc|
|
||||
case expression
|
||||
when String
|
||||
# ok
|
||||
else
|
||||
errors.add(:base, "first element of each events_order tuple must be a Liquid template")
|
||||
break
|
||||
end
|
||||
case type
|
||||
when nil, *EXPRESSION_TYPES
|
||||
# ok
|
||||
else
|
||||
errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
|
||||
break
|
||||
end
|
||||
if !desc.nil? && boolify(desc).nil?
|
||||
errors.add(:base, "third element of each events_order tuple must be a boolean value")
|
||||
break
|
||||
end
|
||||
end
|
||||
else
|
||||
errors.add(:base, "events_order must be an array of arrays")
|
||||
end
|
||||
end
|
||||
|
||||
# Sort given events in order specified by the "events_order" option
|
||||
def sort_events(events)
|
||||
order_by = events_order.presence or
|
||||
return events
|
||||
|
||||
orders = order_by.map { |_, _, desc = false| boolify(desc) }
|
||||
|
||||
Utils.sort_tuples!(
|
||||
events.map.with_index { |event, index|
|
||||
interpolate_with(event) {
|
||||
interpolation_context['_index_'] = index
|
||||
order_by.map { |expression, type, _|
|
||||
string = interpolate_string(expression)
|
||||
begin
|
||||
EXPRESSION_PARSER[type || 'string'.freeze][string]
|
||||
rescue
|
||||
error "Cannot parse #{string.inspect} as #{type}; treating it as string"
|
||||
string
|
||||
end
|
||||
}
|
||||
} << index << event # index is to make sorting stable
|
||||
},
|
||||
orders
|
||||
).collect!(&:last)
|
||||
end
|
||||
end
|
|
@ -80,6 +80,7 @@ module ApplicationHelper
|
|||
end
|
||||
|
||||
def service_label(service)
|
||||
return if service.nil?
|
||||
content_tag :span, [
|
||||
omniauth_provider_icon(service.provider),
|
||||
service_label_text(service)
|
||||
|
|
|
@ -13,6 +13,7 @@ class Agent < ActiveRecord::Base
|
|||
include HasGuid
|
||||
include LiquidDroppable
|
||||
include DryRunnable
|
||||
include SortableEvents
|
||||
|
||||
markdown_class_attributes :description, :event_description
|
||||
|
||||
|
@ -104,12 +105,19 @@ class Agent < ActiveRecord::Base
|
|||
raise "Implement me in your subclass"
|
||||
end
|
||||
|
||||
def create_event(attrs)
|
||||
def build_event(event)
|
||||
event = events.build(event) if event.is_a?(Hash)
|
||||
event.agent = self
|
||||
event.user = user
|
||||
event.expires_at ||= new_event_expiration_date
|
||||
event
|
||||
end
|
||||
|
||||
def create_event(event)
|
||||
if can_create_events?
|
||||
events.create!({
|
||||
:user => user,
|
||||
:expires_at => new_event_expiration_date
|
||||
}.merge(attrs))
|
||||
event = build_event(event)
|
||||
event.save!
|
||||
event
|
||||
else
|
||||
error "This Agent cannot create events!"
|
||||
end
|
||||
|
|
|
@ -40,11 +40,15 @@ module Agents
|
|||
"_contents": "tag contents (can be an object for nesting)"
|
||||
}
|
||||
|
||||
# Ordering events in the output
|
||||
|
||||
#{description_events_order('events in the output')}
|
||||
|
||||
# Liquid Templating
|
||||
|
||||
In Liquid templating, the following variable is available:
|
||||
|
||||
* `events`: An array of events being output, sorted in descending order up to `events_to_show` in number. For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
|
||||
* `events`: An array of events being output, sorted in the given order, up to `events_to_show` in number. For example, if source events contain a site title in the `site_title` key, you can refer to it in `template.title` by putting `{{events.first.site_title}}`.
|
||||
|
||||
MD
|
||||
end
|
||||
|
@ -134,7 +138,7 @@ module Agents
|
|||
end
|
||||
end
|
||||
|
||||
source_events = received_events.order(id: :desc).limit(events_to_show).to_a
|
||||
source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
|
||||
|
||||
interpolation_context.stack do
|
||||
interpolation_context['events'] = source_events
|
||||
|
|
|
@ -9,6 +9,8 @@ module Agents
|
|||
can_dry_run!
|
||||
default_schedule "every_1d"
|
||||
|
||||
DEFAULT_EVENTS_ORDER = [['{{date_published}}', 'time'], ['{{last_updated}}', 'time']]
|
||||
|
||||
description do
|
||||
<<-MD
|
||||
This Agent consumes RSS feeds and emits events when they change.
|
||||
|
@ -29,6 +31,12 @@ module Agents
|
|||
* `disable_url_encoding` - Set to `true` to disable url encoding.
|
||||
* `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}").
|
||||
* `max_events_per_run` - Limit number of events created (items parsed) per run for feed.
|
||||
|
||||
# Ordering Events
|
||||
|
||||
#{description_events_order}
|
||||
|
||||
In this Agent, the default value for `events_order` is `#{DEFAULT_EVENTS_ORDER.to_json}`.
|
||||
MD
|
||||
end
|
||||
|
||||
|
@ -70,6 +78,11 @@ module Agents
|
|||
end
|
||||
|
||||
validate_web_request_options!
|
||||
validate_events_order
|
||||
end
|
||||
|
||||
def events_order
|
||||
super.presence || DEFAULT_EVENTS_ORDER
|
||||
end
|
||||
|
||||
def check
|
||||
|
@ -84,26 +97,15 @@ module Agents
|
|||
response = faraday.get(url)
|
||||
if response.success?
|
||||
feed = FeedNormalizer::FeedNormalizer.parse(response.body)
|
||||
feed.clean! if interpolated['clean'] == 'true'
|
||||
feed.clean! if boolify(interpolated['clean'])
|
||||
max_events = (interpolated['max_events_per_run'].presence || 0).to_i
|
||||
created_event_count = 0
|
||||
feed.entries.sort_by { |entry| [entry.date_published, entry.last_updated] }.each.with_index do |entry, index|
|
||||
sort_events(feed_to_events(feed)).each.with_index do |event, index|
|
||||
break if max_events && max_events > 0 && index >= max_events
|
||||
entry_id = get_entry_id(entry)
|
||||
entry_id = event.payload[:id]
|
||||
if check_and_track(entry_id)
|
||||
created_event_count += 1
|
||||
create_event(payload: {
|
||||
id: entry_id,
|
||||
date_published: entry.date_published,
|
||||
last_updated: entry.last_updated,
|
||||
url: entry.url,
|
||||
urls: entry.urls,
|
||||
description: entry.description,
|
||||
content: entry.content,
|
||||
title: entry.title,
|
||||
authors: entry.authors,
|
||||
categories: entry.categories
|
||||
})
|
||||
create_event(event)
|
||||
end
|
||||
end
|
||||
log "Fetched #{url} and created #{created_event_count} event(s)."
|
||||
|
@ -128,5 +130,22 @@ module Agents
|
|||
true
|
||||
end
|
||||
end
|
||||
|
||||
def feed_to_events(feed)
|
||||
feed.entries.map { |entry|
|
||||
Event.new(payload: {
|
||||
id: get_entry_id(entry),
|
||||
date_published: entry.date_published,
|
||||
last_updated: entry.last_updated,
|
||||
url: entry.url,
|
||||
urls: entry.urls,
|
||||
description: entry.description,
|
||||
content: entry.content,
|
||||
title: entry.title,
|
||||
authors: entry.authors,
|
||||
categories: entry.categories
|
||||
})
|
||||
}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -6,6 +6,7 @@ module Agents
|
|||
include WebRequestConcern
|
||||
|
||||
can_dry_run!
|
||||
can_order_created_events!
|
||||
|
||||
default_schedule "every_12h"
|
||||
|
||||
|
@ -105,6 +106,10 @@ module Agents
|
|||
* `status`: HTTP status as integer. (Almost always 200)
|
||||
|
||||
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_.
|
||||
|
||||
# Ordering Events
|
||||
|
||||
#{description_events_order}
|
||||
MD
|
||||
|
||||
event_description do
|
||||
|
|
|
@ -278,21 +278,6 @@ class ScenarioImport
|
|||
yield 'disabled', disabled, boolean if disabled.requires_merge?
|
||||
end
|
||||
|
||||
# Unfortunately Ruby 1.9's OpenStruct doesn't expose [] and []=.
|
||||
unless instance_methods.include?(:[]=)
|
||||
def [](key)
|
||||
self.send(sanitize key)
|
||||
end
|
||||
|
||||
def []=(key, val)
|
||||
self.send("#{sanitize key}=", val)
|
||||
end
|
||||
|
||||
def sanitize(key)
|
||||
key.gsub(/[^a-zA-Z0-9_-]/, '')
|
||||
end
|
||||
end
|
||||
|
||||
def agent_instance
|
||||
"Agents::#{self.type.updated}".constantize.new
|
||||
end
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
Delayed::Worker.destroy_failed_jobs = false
|
||||
Delayed::Worker.max_attempts = 5
|
||||
Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 20).to_i.minutes
|
||||
Delayed::Worker.max_run_time = (ENV['DELAYED_JOB_MAX_RUNTIME'].presence || 2).to_i.minutes
|
||||
Delayed::Worker.read_ahead = 5
|
||||
Delayed::Worker.default_priority = 10
|
||||
Delayed::Worker.delay_jobs = !Rails.env.test?
|
||||
|
|
|
@ -1,6 +1,3 @@
|
|||
# Module#prepend support for Ruby 1.9
|
||||
require 'prepend' unless Module.method_defined?(:prepend)
|
||||
|
||||
require 'active_support'
|
||||
|
||||
ActiveSupport.on_load :active_record do
|
||||
|
|
|
@ -1,85 +0,0 @@
|
|||
# Fake implementation of prepend(), which does not support overriding
|
||||
# inherited methods nor methods that are formerly overridden by
|
||||
# another invocation of prepend().
|
||||
#
|
||||
# Here's what <Original>.prepend(<Wrapper>) does:
|
||||
#
|
||||
# - Create an anonymous stub module (hereinafter <Stub>) and define
|
||||
# <Stub>#<method> that calls #<method>_without_<Wrapper> for each
|
||||
# instance method of <Wrapper>.
|
||||
#
|
||||
# - Rename <Original>#<method> to #<method>_without_<Wrapper> for each
|
||||
# instance method of <Wrapper>.
|
||||
#
|
||||
# - Include <Stub> and <Wrapper> into <Original> in that order.
|
||||
#
|
||||
# This way, a call of <Original>#<method> is dispatched to
|
||||
# <Wrapper><method>, which may call super which is dispatched to
|
||||
# <Stub>#<method>, which finally calls
|
||||
# <Original>#<method>_without_<Wrapper> which is used to be called
|
||||
# <Original>#<method>.
|
||||
#
|
||||
# Usage:
|
||||
#
|
||||
# class Mechanize
|
||||
# # module with methods that overrides those of X
|
||||
# module Y
|
||||
# end
|
||||
#
|
||||
# unless X.respond_to?(:prepend, true)
|
||||
# require 'mechanize/prependable'
|
||||
# X.extend(Prependable)
|
||||
# end
|
||||
#
|
||||
# class X
|
||||
# prepend Y
|
||||
# end
|
||||
# end
|
||||
class Module
|
||||
def prepend(mod)
|
||||
stub = Module.new
|
||||
|
||||
mod_id = (mod.name || 'Module__%d' % mod.object_id).gsub(/::/, '__')
|
||||
|
||||
mod.instance_methods.each { |name|
|
||||
method_defined?(name) or next
|
||||
|
||||
original = instance_method(name)
|
||||
next if original.owner != self
|
||||
|
||||
name = name.to_s
|
||||
name_without = name.sub(/(?=[?!=]?\z)/) { '_without_%s' % mod_id }
|
||||
|
||||
arity = original.arity
|
||||
arglist = (
|
||||
if arity >= 0
|
||||
(1..arity).map { |i| 'x%d' % i }
|
||||
else
|
||||
(1..(-arity - 1)).map { |i| 'x%d' % i } << '*a'
|
||||
end << '&b'
|
||||
).join(', ')
|
||||
|
||||
if name.end_with?('=')
|
||||
stub.module_eval %{
|
||||
def #{name}(#{arglist})
|
||||
__send__(:#{name_without}, #{arglist})
|
||||
end
|
||||
}
|
||||
else
|
||||
stub.module_eval %{
|
||||
def #{name}(#{arglist})
|
||||
#{name_without}(#{arglist})
|
||||
end
|
||||
}
|
||||
end
|
||||
module_eval {
|
||||
alias_method name_without, name
|
||||
remove_method name
|
||||
}
|
||||
}
|
||||
|
||||
include stub
|
||||
include mod
|
||||
end
|
||||
private :prepend
|
||||
end unless Module.method_defined?(:prepend)
|
39
lib/utils.rb
39
lib/utils.rb
|
@ -79,4 +79,43 @@ module Utils
|
|||
def self.pretty_jsonify(thing)
|
||||
JSON.pretty_generate(thing).gsub('</', '<\/')
|
||||
end
|
||||
|
||||
class TupleSorter
|
||||
class SortableTuple
|
||||
attr_reader :array
|
||||
|
||||
# The <=> method will call orders[n] to determine if the nth element
|
||||
# should be compared in descending order.
|
||||
def initialize(array, orders = [])
|
||||
@array = array
|
||||
@orders = orders
|
||||
end
|
||||
|
||||
def <=> other
|
||||
other = other.array
|
||||
@array.each_with_index do |e, i|
|
||||
o = other[i]
|
||||
case cmp = e <=> o || e.to_s <=> o.to_s
|
||||
when 0
|
||||
next
|
||||
else
|
||||
return @orders[i] ? -cmp : cmp
|
||||
end
|
||||
end
|
||||
0
|
||||
end
|
||||
end
|
||||
|
||||
class << self
|
||||
def sort!(array, orders = [])
|
||||
array.sort_by! do |e|
|
||||
SortableTuple.new(e, orders)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.sort_tuples!(array, orders = [])
|
||||
TupleSorter.sort!(array, orders)
|
||||
end
|
||||
end
|
||||
|
|
264
spec/concerns/sortable_events_spec.rb
Normal file
264
spec/concerns/sortable_events_spec.rb
Normal file
|
@ -0,0 +1,264 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe SortableEvents do
|
||||
let(:agent_class) {
|
||||
Class.new(Agent) do
|
||||
include SortableEvents
|
||||
|
||||
default_schedule 'never'
|
||||
|
||||
def self.valid_type?(name)
|
||||
true
|
||||
end
|
||||
end
|
||||
}
|
||||
|
||||
def new_agent(events_order = nil)
|
||||
options = {}
|
||||
options['events_order'] = events_order if events_order
|
||||
agent_class.new(name: 'test', options: options) { |agent|
|
||||
agent.user = users(:bob)
|
||||
}
|
||||
end
|
||||
|
||||
describe 'validations' do
|
||||
let(:agent_class) {
|
||||
Class.new(Agent) do
|
||||
include SortableEvents
|
||||
|
||||
default_schedule 'never'
|
||||
|
||||
def self.valid_type?(name)
|
||||
true
|
||||
end
|
||||
end
|
||||
}
|
||||
|
||||
def new_agent(events_order = nil)
|
||||
options = {}
|
||||
options['events_order'] = events_order if events_order
|
||||
agent_class.new(name: 'test', options: options) { |agent|
|
||||
agent.user = users(:bob)
|
||||
}
|
||||
end
|
||||
|
||||
it 'should allow events_order to be unspecified, null or an empty array' do
|
||||
expect(new_agent()).to be_valid
|
||||
expect(new_agent(nil)).to be_valid
|
||||
expect(new_agent([])).to be_valid
|
||||
end
|
||||
|
||||
it 'should not allow events_order to be a non-array object' do
|
||||
agent = new_agent(0)
|
||||
expect(agent).not_to be_valid
|
||||
expect(agent.errors[:base]).to include(/events_order/)
|
||||
|
||||
agent = new_agent('')
|
||||
expect(agent).not_to be_valid
|
||||
expect(agent.errors[:base]).to include(/events_order/)
|
||||
|
||||
agent = new_agent({})
|
||||
expect(agent).not_to be_valid
|
||||
expect(agent.errors[:base]).to include(/events_order/)
|
||||
end
|
||||
|
||||
it 'should not allow events_order to be an array containing unexpected objects' do
|
||||
agent = new_agent(['{{key}}', 1])
|
||||
expect(agent).not_to be_valid
|
||||
expect(agent.errors[:base]).to include(/events_order/)
|
||||
|
||||
agent = new_agent(['{{key1}}', ['{{key2}}', 'unknown']])
|
||||
expect(agent).not_to be_valid
|
||||
expect(agent.errors[:base]).to include(/events_order/)
|
||||
end
|
||||
|
||||
it 'should allow events_order to be an array containing strings and valid tuples' do
|
||||
agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number']])
|
||||
expect(agent).to be_valid
|
||||
|
||||
agent = new_agent(['{{key1}}', ['{{key2}}'], ['{{key3}}', 'number'], ['{{key4}}', 'time', true]])
|
||||
expect(agent).to be_valid
|
||||
end
|
||||
end
|
||||
|
||||
describe 'sort_events' do
|
||||
let(:payloads) {
|
||||
[
|
||||
{ 'title' => 'TitleA', 'score' => 4, 'updated_on' => '7 Jul 2015' },
|
||||
{ 'title' => 'TitleB', 'score' => 2, 'updated_on' => '25 Jun 2014' },
|
||||
{ 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
|
||||
{ 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
|
||||
]
|
||||
}
|
||||
|
||||
let(:events) {
|
||||
payloads.map { |payload| Event.new(payload: payload) }
|
||||
}
|
||||
|
||||
it 'should sort events by a given key' do
|
||||
agent = new_agent(['{{title}}'])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleC TitleD])
|
||||
|
||||
agent = new_agent([['{{title}}', 'string', true]])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleD TitleC TitleB TitleA])
|
||||
end
|
||||
|
||||
it 'should sort events by multiple keys' do
|
||||
agent = new_agent([['{{score}}', 'number'], '{{title}}'])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleC TitleD])
|
||||
|
||||
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
|
||||
end
|
||||
|
||||
it 'should sort events by time' do
|
||||
agent = new_agent([['{{updated_on}}', 'time']])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleB TitleD TitleC TitleA])
|
||||
end
|
||||
|
||||
it 'should sort events stably' do
|
||||
agent = new_agent(['<constant>'])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
|
||||
|
||||
agent = new_agent([['<constant>', 'string', true]])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
|
||||
end
|
||||
|
||||
it 'should support _index_' do
|
||||
agent = new_agent([['{{_index_}}', 'number', true]])
|
||||
expect(agent.__send__(:sort_events, events).map { |e| e.payload['title'] }).to eq(%w[TitleC TitleD TitleB TitleA])
|
||||
end
|
||||
end
|
||||
|
||||
describe 'automatic event sorter' do
|
||||
describe 'declaration' do
|
||||
let(:passive_agent_class) {
|
||||
Class.new(Agent) do
|
||||
include SortableEvents
|
||||
|
||||
cannot_create_events!
|
||||
end
|
||||
}
|
||||
|
||||
let(:active_agent_class) {
|
||||
Class.new(Agent) do
|
||||
include SortableEvents
|
||||
end
|
||||
}
|
||||
|
||||
describe 'can_order_created_events!' do
|
||||
it 'should refuse to work if called from an Agent that cannot create events' do
|
||||
expect {
|
||||
passive_agent_class.class_eval do
|
||||
can_order_created_events!
|
||||
end
|
||||
}.to raise_error
|
||||
end
|
||||
|
||||
it 'should work if called from an Agent that can create events' do
|
||||
expect {
|
||||
active_agent_class.class_eval do
|
||||
can_order_created_events!
|
||||
end
|
||||
}.not_to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
describe 'can_order_created_events?' do
|
||||
it 'should return false unless an Agent declares can_order_created_events!' do
|
||||
expect(active_agent_class.can_order_created_events?).to eq(false)
|
||||
expect(active_agent_class.new.can_order_created_events?).to eq(false)
|
||||
end
|
||||
|
||||
it 'should return true if an Agent declares can_order_created_events!' do
|
||||
active_agent_class.class_eval do
|
||||
can_order_created_events!
|
||||
end
|
||||
|
||||
expect(active_agent_class.can_order_created_events?).to eq(true)
|
||||
expect(active_agent_class.new.can_order_created_events?).to eq(true)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe 'behavior' do
|
||||
class Agents::EventOrderableAgent < Agent
|
||||
include SortableEvents
|
||||
|
||||
default_schedule 'never'
|
||||
|
||||
can_order_created_events!
|
||||
|
||||
attr_accessor :payloads_to_emit
|
||||
|
||||
def self.valid_type?(name)
|
||||
true
|
||||
end
|
||||
|
||||
def check
|
||||
payloads_to_emit.each do |payload|
|
||||
create_event payload: payload
|
||||
end
|
||||
end
|
||||
|
||||
def receive(events)
|
||||
events.each do |event|
|
||||
payloads_to_emit.each do |payload|
|
||||
create_event payload: payload.merge('title' => payload['title'] + event.payload['title_suffix'])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def new_agent(events_order = nil)
|
||||
options = {}
|
||||
options['events_order'] = events_order if events_order
|
||||
Agents::EventOrderableAgent.new(name: 'test', options: options) { |agent|
|
||||
agent.user = users(:bob)
|
||||
agent.payloads_to_emit = payloads
|
||||
}
|
||||
end
|
||||
|
||||
let(:payloads) {
|
||||
[
|
||||
{ 'title' => 'TitleA', 'score' => 4, 'updated_on' => '7 Jul 2015' },
|
||||
{ 'title' => 'TitleB', 'score' => 2, 'updated_on' => '25 Jun 2014' },
|
||||
{ 'title' => 'TitleD', 'score' => 10, 'updated_on' => '10 Jan 2015' },
|
||||
{ 'title' => 'TitleC', 'score' => 10, 'updated_on' => '9 Feb 2015' },
|
||||
]
|
||||
}
|
||||
|
||||
it 'should keep the order of created events unless events_order is specified' do
|
||||
[[], [nil], [[]]].each do |args|
|
||||
agent = new_agent(*args)
|
||||
agent.save!
|
||||
expect { agent.check }.to change { Event.count }.by(4)
|
||||
events = agent.events.last(4).sort_by(&:id)
|
||||
expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleA TitleB TitleD TitleC])
|
||||
end
|
||||
end
|
||||
|
||||
it 'should sort events created in check() in the order specified in events_order' do
|
||||
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
|
||||
agent.save!
|
||||
expect { agent.check }.to change { Event.count }.by(4)
|
||||
events = agent.events.last(4).sort_by(&:id)
|
||||
expect(events.map { |event| event.payload['title'] }).to eq(%w[TitleB TitleA TitleD TitleC])
|
||||
end
|
||||
|
||||
it 'should sort events created in receive() in the order specified in events_order' do
|
||||
agent = new_agent([['{{score}}', 'number'], ['{{title}}', 'string', true]])
|
||||
agent.save!
|
||||
expect {
|
||||
agent.receive([Event.new(payload: { 'title_suffix' => ' [new]' }),
|
||||
Event.new(payload: { 'title_suffix' => ' [popular]' })])
|
||||
}.to change { Event.count }.by(8)
|
||||
events = agent.events.last(8).sort_by(&:id)
|
||||
expect(events.map { |event| event.payload['title'] }).to eq([
|
||||
'TitleB [new]', 'TitleA [new]', 'TitleD [new]', 'TitleC [new]',
|
||||
'TitleB [popular]', 'TitleA [popular]', 'TitleD [popular]', 'TitleC [popular]',
|
||||
])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -372,7 +372,7 @@ describe AgentsController do
|
|||
sign_in users(:bob)
|
||||
agent = agents(:bob_weather_agent)
|
||||
expect {
|
||||
post :dry_run, id: agents(:bob_website_agent), agent: valid_attributes(name: 'New Name')
|
||||
post :dry_run, id: agent, agent: valid_attributes(name: 'New Name')
|
||||
}.not_to change {
|
||||
[users(:bob).agents.count, users(:bob).events.count, users(:bob).logs.count, agent.name, agent.updated_at]
|
||||
}
|
||||
|
|
|
@ -114,4 +114,62 @@ describe Utils do
|
|||
expect(cleaned_json).to include("<\\/script>")
|
||||
end
|
||||
end
|
||||
|
||||
describe "#sort_tuples!" do
|
||||
let(:tuples) {
|
||||
time = Time.now
|
||||
[
|
||||
[2, "a", time - 1], # 0
|
||||
[2, "b", time - 1], # 1
|
||||
[1, "b", time - 1], # 2
|
||||
[1, "b", time], # 3
|
||||
[1, "a", time], # 4
|
||||
[2, "a", time + 1], # 5
|
||||
[2, "a", time], # 6
|
||||
]
|
||||
}
|
||||
|
||||
it "sorts tuples like arrays by default" do
|
||||
expected = tuples.values_at(4, 2, 3, 0, 6, 5, 1)
|
||||
|
||||
Utils.sort_tuples!(tuples)
|
||||
expect(tuples).to eq expected
|
||||
end
|
||||
|
||||
it "sorts tuples in order specified: case 1" do
|
||||
# order by x1 asc, x2 desc, c3 asc
|
||||
orders = [false, true, false]
|
||||
expected = tuples.values_at(2, 3, 4, 1, 0, 6, 5)
|
||||
|
||||
Utils.sort_tuples!(tuples, orders)
|
||||
expect(tuples).to eq expected
|
||||
end
|
||||
|
||||
it "sorts tuples in order specified: case 2" do
|
||||
# order by x1 desc, x2 asc, c3 desc
|
||||
orders = [true, false, true]
|
||||
expected = tuples.values_at(5, 6, 0, 1, 4, 3, 2)
|
||||
|
||||
Utils.sort_tuples!(tuples, orders)
|
||||
expect(tuples).to eq expected
|
||||
end
|
||||
|
||||
it "always succeeds in sorting even if it finds pairs of incomparable objects" do
|
||||
time = Time.now
|
||||
tuples = [
|
||||
[2, "a", time - 1], # 0
|
||||
[1, "b", nil], # 1
|
||||
[1, "b", time], # 2
|
||||
["2", nil, time], # 3
|
||||
[1, nil, time], # 4
|
||||
[nil, "a", time + 1], # 5
|
||||
[2, "a", time], # 6
|
||||
]
|
||||
orders = [true, false, true]
|
||||
expected = tuples.values_at(3, 6, 0, 4, 2, 1, 5)
|
||||
|
||||
Utils.sort_tuples!(tuples, orders)
|
||||
expect(tuples).to eq expected
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -209,6 +209,22 @@ describe Agents::DataOutputAgent do
|
|||
})
|
||||
end
|
||||
|
||||
describe 'ordering' do
|
||||
before do
|
||||
agent.options['events_order'] = ['{{title}}']
|
||||
end
|
||||
|
||||
it 'can reorder the events_to_show last events based on a Liquid expression' do
|
||||
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
|
||||
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
|
||||
|
||||
agent.options['events_order'] = [['{{title}}', 'string', true]]
|
||||
|
||||
desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
|
||||
expect(desc_content['items']).to eq(asc_content['items'].reverse)
|
||||
end
|
||||
end
|
||||
|
||||
describe "interpolating \"events\"" do
|
||||
before do
|
||||
agent.options['template']['title'] = "XKCD comics as a feed{% if events.first.site_title %} ({{events.first.site_title}}){% endif %}"
|
||||
|
|
|
@ -66,6 +66,21 @@ describe Agents::RssAgent do
|
|||
expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/d465158f77dcd9078697e6167b50abbfdfa8b1af"])
|
||||
end
|
||||
|
||||
it "should emit items as events in the order specified in the events_order option" do
|
||||
expect {
|
||||
agent.options['events_order'] = ['{{title | replace_regex: "^[[:space:]]+", "" }}']
|
||||
agent.check
|
||||
}.to change { agent.events.count }.by(20)
|
||||
|
||||
first, *, last = agent.events.last(20)
|
||||
expect(first.payload['title'].strip).to eq('upgrade rails and gems')
|
||||
expect(first.payload['url']).to eq("https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01")
|
||||
expect(first.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/87a7abda23a82305d7050ac0bb400ce36c863d01"])
|
||||
expect(last.payload['title'].strip).to eq('Dashed line in a diagram indicates propagate_immediately being false.')
|
||||
expect(last.payload['url']).to eq("https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535")
|
||||
expect(last.payload['urls']).to eq(["https://github.com/cantino/huginn/commit/0e80f5341587aace2c023b06eb9265b776ac4535"])
|
||||
end
|
||||
|
||||
it "should track ids and not re-emit the same item when seen again" do
|
||||
agent.check
|
||||
expect(agent.memory['seen_ids']).to eq(agent.events.map {|e| e.payload['id'] })
|
||||
|
|
Loading…
Add table
Reference in a new issue