GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-17 12:21:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1444 from cantino/data_output_agent_limits_events_after_ordering

Browse source 
DataOutputAgent limits events after ordering
This commit is contained in:
Akinori MUSHA 2016-08-08 14:12:25 +09:00 • committed by GitHub
commit 9b23c28049
5 changed files with 120 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
app/concerns/sortable_events.rb
View file

@ -5,6 +5,8 @@ module SortableEvents
validate :validate_events_order
end
EVENTS_ORDER_KEY = 'events_order'.freeze
def description_events_order(*args)
self.class.description_events_order(*args)
end
@ -23,9 +25,9 @@ module SortableEvents
!can_order_created_events?
end
def description_events_order(events = 'events created in each run')
def description_events_order(events = 'events created in each run', events_order_key = EVENTS_ORDER_KEY)
<<-MD.lstrip
To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
To specify the order of #{events}, set `#{events_order_key}` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
* _expression_ is a Liquid template to generate a string to be used as sort key.
@ -48,8 +50,8 @@ module SortableEvents
self.class.cannot_order_created_events?
end
def events_order
options['events_order']
def events_order(key = EVENTS_ORDER_KEY)
options[key]
end
module AutomaticSorter
@ -102,8 +104,8 @@ module SortableEvents
}
EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
def validate_events_order
case order_by = events_order
def validate_events_order(events_order_key = EVENTS_ORDER_KEY)
case order_by = events_order(events_order_key)
when nil
when Array
# Each tuple may be either [expression, type, desc] or just
@ -113,29 +115,29 @@ module SortableEvents
when String
# ok
else
errors.add(:base, "first element of each events_order tuple must be a Liquid template")
errors.add(:base, "first element of each #{events_order_key} tuple must be a Liquid template")
break
end
case type
when nil, *EXPRESSION_TYPES
# ok
else
errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
errors.add(:base, "second element of each #{events_order_key} tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
break
end
if !desc.nil? && boolify(desc).nil?
errors.add(:base, "third element of each events_order tuple must be a boolean value")
errors.add(:base, "third element of each #{events_order_key} tuple must be a boolean value")
break
end
end
else
errors.add(:base, "events_order must be an array of arrays")
errors.add(:base, "#{events_order_key} must be an array of arrays")
end
end
# Sort given events in order specified by the "events_order" option
def sort_events(events)
order_by = events_order.presence or
def sort_events(events, events_order_key = EVENTS_ORDER_KEY)
order_by = events_order(events_order_key).presence or
return events
orders = order_by.map { |_, _, desc = false| boolify(desc) }

67
app/models/agents/data_output_agent.rb
View file

@ -46,9 +46,14 @@ module Agents
"_contents": "tag contents (can be an object for nesting)"
}
# Ordering events in the output
# Ordering events
#{description_events_order('events in the output')}
#{description_events_order('events')}
DataOutputAgent will select the last `events_to_show` entries of its received events sorted in the order specified by `events_order`, which is defaulted to the event creation time.
So, if you have multiple source agents that may create many events in a run, you may want to either increase `events_to_show` to have a larger "window", or specify the `events_order` option to an appropriate value (like `date_published`) so events from various sources are properly mixed in the resulted feed.
There is also an option `events_list_order` that only controls the order of events listed in the final output, without attempting to maintain a total order of received events. It has the same format as `events_order` and is defaulted to `#{Utils.jsonify(DEFAULT_EVENTS_ORDER['events_list_order'])}` so the selected events are listed in reverse order like most popular RSS feeds list their articles.
# Liquid Templating
@ -176,6 +181,59 @@ module Agents
interpolated['push_hubs'].presence || []
end
DEFAULT_EVENTS_ORDER = {
'events_order' => nil,
'events_list_order' => [["{{_index_}}", "number", true]],
}
def events_order(key = SortableEvents::EVENTS_ORDER_KEY)
super || DEFAULT_EVENTS_ORDER[key]
end
def latest_events(reload = false)
events =
if (event_ids = memory[:event_ids]) &&
memory[:events_order] == events_order &&
memory[:events_to_show] >= events_to_show
received_events.where(id: event_ids).to_a
else
memory[:last_event_id] = nil
reload = true
[]
end
if reload
memory[:events_order] = events_order
memory[:events_to_show] = events_to_show
new_events =
if last_event_id = memory[:last_event_id]
received_events.where(Event.arel_table[:id].gt(last_event_id)).
order(id: :asc).to_a
else
source_ids.flat_map { |source_id|
# dig twice as many events as the number of
# `events_to_show`
received_events.where(agent_id: source_id).
last(2 * events_to_show)
}.sort_by(&:id)
end
unless new_events.empty?
memory[:last_event_id] = new_events.last.id
events.concat(new_events)
end
end
events = sort_events(events).last(events_to_show)
if reload
memory[:event_ids] = events.map(&:id)
end
events
end
def receive_web_request(params, method, format)
unless interpolated['secrets'].include?(params['secret'])
if format =~ /json/
@ -185,7 +243,7 @@ module Agents
end
end
source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
source_events = sort_events(latest_events(), 'events_list_order')
interpolation_context.stack do
interpolation_context['events'] = source_events
@ -252,6 +310,9 @@ module Agents
def receive(incoming_events)
url = feed_url(secret: interpolated['secrets'].first, format: :xml)
# Reload new events and update cache
latest_events(true)
push_hubs.each do |hub|
push_to_hub(hub, url)
end

8
app/models/agents/rss_agent.rb
View file

@ -82,8 +82,12 @@ module Agents
validate_events_order
end
def events_order
super.presence || DEFAULT_EVENTS_ORDER
def events_order(key = SortableEvents::EVENTS_ORDER_KEY)
if key == SortableEvents::EVENTS_ORDER_KEY
super.presence || DEFAULT_EVENTS_ORDER
else
raise ArgumentError, "unsupported key: #{key}"
end
end
def check

25
db/migrate/20160607055850_change_events_order_to_events_list_order.rb Normal file
View file

@ -0,0 +1,25 @@
class ChangeEventsOrderToEventsListOrder < ActiveRecord::Migration
def up
Agents::DataOutputAgent.find_each do |agent|
if value = agent.options.delete('events_order')
agent.options['events_list_order'] = value
agent.save!(validate: false)
end
end
end
def down
Agents::DataOutputAgent.transaction do
Agents::DataOutputAgent.find_each do |agent|
if agent.options['events_order']
raise ActiveRecord::IrreversibleMigration, "Cannot revert migration because events_order is configured"
end
if value = agent.options.delete('events_list_order')
agent.options['events_order'] = value
agent.save!(validate: false)
end
end
end
end
end

16
spec/models/agents/data_output_agent_spec.rb
View file

@ -142,7 +142,7 @@ describe Agents::DataOutputAgent do
"url" => "http://imgs.xkcd.com/comics/evolving0.png",
"title" => "Evolving yet again with a past date",
"date" => '2014/05/05',
"hovertext" => "Something else"
"hovertext" => "A small text"
}
end
@ -166,7 +166,7 @@ describe Agents::DataOutputAgent do
<item>
<title>Evolving yet again with a past date</title>
<description>Secret hovertext: Something else</description>
<description>Secret hovertext: A small text</description>
<link>http://imgs.xkcd.com/comics/evolving0.png</link>
<pubDate>#{Time.zone.parse(event3.payload['date']).rfc2822}</pubDate>
<guid isPermaLink="false">#{event3.id}</guid>
@ -216,7 +216,7 @@ describe Agents::DataOutputAgent do
'items' => [
{
'title' => 'Evolving yet again with a past date',
'description' => 'Secret hovertext: Something else',
'description' => 'Secret hovertext: A small text',
'link' => 'http://imgs.xkcd.com/comics/evolving0.png',
'guid' => {"contents" => event3.id, "isPermaLink" => "false"},
'pubDate' => Time.zone.parse(event3.payload['date']).rfc2822,
@ -244,14 +244,20 @@ describe Agents::DataOutputAgent do
describe 'ordering' do
before do
agent.options['events_order'] = ['{{title}}']
agent.options['events_order'] = ['{{hovertext}}']
agent.options['events_list_order'] = ['{{title}}']
end
it 'can reorder the events_to_show last events based on a Liquid expression' do
agent.options['events_to_show'] = 2
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again"])
agent.options['events_to_show'] = 40
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
agent.options['events_order'] = [['{{title}}', 'string', true]]
agent.options['events_list_order'] = [['{{title}}', 'string', true]]
desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
expect(desc_content['items']).to eq(asc_content['items'].reverse)