mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Merge pull request #1444 from cantino/data_output_agent_limits_events_after_ordering
DataOutputAgent limits events after ordering
This commit is contained in:
commit
9b23c28049
5 changed files with 120 additions and 22 deletions
|
@ -5,6 +5,8 @@ module SortableEvents
|
|||
validate :validate_events_order
|
||||
end
|
||||
|
||||
EVENTS_ORDER_KEY = 'events_order'.freeze
|
||||
|
||||
def description_events_order(*args)
|
||||
self.class.description_events_order(*args)
|
||||
end
|
||||
|
@ -23,9 +25,9 @@ module SortableEvents
|
|||
!can_order_created_events?
|
||||
end
|
||||
|
||||
def description_events_order(events = 'events created in each run')
|
||||
def description_events_order(events = 'events created in each run', events_order_key = EVENTS_ORDER_KEY)
|
||||
<<-MD.lstrip
|
||||
To specify the order of #{events}, set `events_order` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
|
||||
To specify the order of #{events}, set `#{events_order_key}` to an array of sort keys, each of which looks like either `expression` or `[expression, type, descending]`, as described as follows:
|
||||
|
||||
* _expression_ is a Liquid template to generate a string to be used as sort key.
|
||||
|
||||
|
@ -48,8 +50,8 @@ module SortableEvents
|
|||
self.class.cannot_order_created_events?
|
||||
end
|
||||
|
||||
def events_order
|
||||
options['events_order']
|
||||
def events_order(key = EVENTS_ORDER_KEY)
|
||||
options[key]
|
||||
end
|
||||
|
||||
module AutomaticSorter
|
||||
|
@ -102,8 +104,8 @@ module SortableEvents
|
|||
}
|
||||
EXPRESSION_TYPES = EXPRESSION_PARSER.keys.freeze
|
||||
|
||||
def validate_events_order
|
||||
case order_by = events_order
|
||||
def validate_events_order(events_order_key = EVENTS_ORDER_KEY)
|
||||
case order_by = events_order(events_order_key)
|
||||
when nil
|
||||
when Array
|
||||
# Each tuple may be either [expression, type, desc] or just
|
||||
|
@ -113,29 +115,29 @@ module SortableEvents
|
|||
when String
|
||||
# ok
|
||||
else
|
||||
errors.add(:base, "first element of each events_order tuple must be a Liquid template")
|
||||
errors.add(:base, "first element of each #{events_order_key} tuple must be a Liquid template")
|
||||
break
|
||||
end
|
||||
case type
|
||||
when nil, *EXPRESSION_TYPES
|
||||
# ok
|
||||
else
|
||||
errors.add(:base, "second element of each events_order tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
|
||||
errors.add(:base, "second element of each #{events_order_key} tuple must be #{EXPRESSION_TYPES.to_sentence(last_word_connector: ' or ')}")
|
||||
break
|
||||
end
|
||||
if !desc.nil? && boolify(desc).nil?
|
||||
errors.add(:base, "third element of each events_order tuple must be a boolean value")
|
||||
errors.add(:base, "third element of each #{events_order_key} tuple must be a boolean value")
|
||||
break
|
||||
end
|
||||
end
|
||||
else
|
||||
errors.add(:base, "events_order must be an array of arrays")
|
||||
errors.add(:base, "#{events_order_key} must be an array of arrays")
|
||||
end
|
||||
end
|
||||
|
||||
# Sort given events in order specified by the "events_order" option
|
||||
def sort_events(events)
|
||||
order_by = events_order.presence or
|
||||
def sort_events(events, events_order_key = EVENTS_ORDER_KEY)
|
||||
order_by = events_order(events_order_key).presence or
|
||||
return events
|
||||
|
||||
orders = order_by.map { |_, _, desc = false| boolify(desc) }
|
||||
|
|
|
@ -46,9 +46,14 @@ module Agents
|
|||
"_contents": "tag contents (can be an object for nesting)"
|
||||
}
|
||||
|
||||
# Ordering events in the output
|
||||
# Ordering events
|
||||
|
||||
#{description_events_order('events in the output')}
|
||||
#{description_events_order('events')}
|
||||
|
||||
DataOutputAgent will select the last `events_to_show` entries of its received events sorted in the order specified by `events_order`, which is defaulted to the event creation time.
|
||||
So, if you have multiple source agents that may create many events in a run, you may want to either increase `events_to_show` to have a larger "window", or specify the `events_order` option to an appropriate value (like `date_published`) so events from various sources are properly mixed in the resulted feed.
|
||||
|
||||
There is also an option `events_list_order` that only controls the order of events listed in the final output, without attempting to maintain a total order of received events. It has the same format as `events_order` and is defaulted to `#{Utils.jsonify(DEFAULT_EVENTS_ORDER['events_list_order'])}` so the selected events are listed in reverse order like most popular RSS feeds list their articles.
|
||||
|
||||
# Liquid Templating
|
||||
|
||||
|
@ -176,6 +181,59 @@ module Agents
|
|||
interpolated['push_hubs'].presence || []
|
||||
end
|
||||
|
||||
DEFAULT_EVENTS_ORDER = {
|
||||
'events_order' => nil,
|
||||
'events_list_order' => [["{{_index_}}", "number", true]],
|
||||
}
|
||||
|
||||
def events_order(key = SortableEvents::EVENTS_ORDER_KEY)
|
||||
super || DEFAULT_EVENTS_ORDER[key]
|
||||
end
|
||||
|
||||
def latest_events(reload = false)
|
||||
events =
|
||||
if (event_ids = memory[:event_ids]) &&
|
||||
memory[:events_order] == events_order &&
|
||||
memory[:events_to_show] >= events_to_show
|
||||
received_events.where(id: event_ids).to_a
|
||||
else
|
||||
memory[:last_event_id] = nil
|
||||
reload = true
|
||||
[]
|
||||
end
|
||||
|
||||
if reload
|
||||
memory[:events_order] = events_order
|
||||
memory[:events_to_show] = events_to_show
|
||||
|
||||
new_events =
|
||||
if last_event_id = memory[:last_event_id]
|
||||
received_events.where(Event.arel_table[:id].gt(last_event_id)).
|
||||
order(id: :asc).to_a
|
||||
else
|
||||
source_ids.flat_map { |source_id|
|
||||
# dig twice as many events as the number of
|
||||
# `events_to_show`
|
||||
received_events.where(agent_id: source_id).
|
||||
last(2 * events_to_show)
|
||||
}.sort_by(&:id)
|
||||
end
|
||||
|
||||
unless new_events.empty?
|
||||
memory[:last_event_id] = new_events.last.id
|
||||
events.concat(new_events)
|
||||
end
|
||||
end
|
||||
|
||||
events = sort_events(events).last(events_to_show)
|
||||
|
||||
if reload
|
||||
memory[:event_ids] = events.map(&:id)
|
||||
end
|
||||
|
||||
events
|
||||
end
|
||||
|
||||
def receive_web_request(params, method, format)
|
||||
unless interpolated['secrets'].include?(params['secret'])
|
||||
if format =~ /json/
|
||||
|
@ -185,7 +243,7 @@ module Agents
|
|||
end
|
||||
end
|
||||
|
||||
source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a)
|
||||
source_events = sort_events(latest_events(), 'events_list_order')
|
||||
|
||||
interpolation_context.stack do
|
||||
interpolation_context['events'] = source_events
|
||||
|
@ -252,6 +310,9 @@ module Agents
|
|||
def receive(incoming_events)
|
||||
url = feed_url(secret: interpolated['secrets'].first, format: :xml)
|
||||
|
||||
# Reload new events and update cache
|
||||
latest_events(true)
|
||||
|
||||
push_hubs.each do |hub|
|
||||
push_to_hub(hub, url)
|
||||
end
|
||||
|
|
|
@ -82,8 +82,12 @@ module Agents
|
|||
validate_events_order
|
||||
end
|
||||
|
||||
def events_order
|
||||
super.presence || DEFAULT_EVENTS_ORDER
|
||||
def events_order(key = SortableEvents::EVENTS_ORDER_KEY)
|
||||
if key == SortableEvents::EVENTS_ORDER_KEY
|
||||
super.presence || DEFAULT_EVENTS_ORDER
|
||||
else
|
||||
raise ArgumentError, "unsupported key: #{key}"
|
||||
end
|
||||
end
|
||||
|
||||
def check
|
||||
|
|
|
@ -0,0 +1,25 @@
|
|||
class ChangeEventsOrderToEventsListOrder < ActiveRecord::Migration
|
||||
def up
|
||||
Agents::DataOutputAgent.find_each do |agent|
|
||||
if value = agent.options.delete('events_order')
|
||||
agent.options['events_list_order'] = value
|
||||
agent.save!(validate: false)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def down
|
||||
Agents::DataOutputAgent.transaction do
|
||||
Agents::DataOutputAgent.find_each do |agent|
|
||||
if agent.options['events_order']
|
||||
raise ActiveRecord::IrreversibleMigration, "Cannot revert migration because events_order is configured"
|
||||
end
|
||||
|
||||
if value = agent.options.delete('events_list_order')
|
||||
agent.options['events_order'] = value
|
||||
agent.save!(validate: false)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -142,7 +142,7 @@ describe Agents::DataOutputAgent do
|
|||
"url" => "http://imgs.xkcd.com/comics/evolving0.png",
|
||||
"title" => "Evolving yet again with a past date",
|
||||
"date" => '2014/05/05',
|
||||
"hovertext" => "Something else"
|
||||
"hovertext" => "A small text"
|
||||
}
|
||||
end
|
||||
|
||||
|
@ -166,7 +166,7 @@ describe Agents::DataOutputAgent do
|
|||
|
||||
<item>
|
||||
<title>Evolving yet again with a past date</title>
|
||||
<description>Secret hovertext: Something else</description>
|
||||
<description>Secret hovertext: A small text</description>
|
||||
<link>http://imgs.xkcd.com/comics/evolving0.png</link>
|
||||
<pubDate>#{Time.zone.parse(event3.payload['date']).rfc2822}</pubDate>
|
||||
<guid isPermaLink="false">#{event3.id}</guid>
|
||||
|
@ -216,7 +216,7 @@ describe Agents::DataOutputAgent do
|
|||
'items' => [
|
||||
{
|
||||
'title' => 'Evolving yet again with a past date',
|
||||
'description' => 'Secret hovertext: Something else',
|
||||
'description' => 'Secret hovertext: A small text',
|
||||
'link' => 'http://imgs.xkcd.com/comics/evolving0.png',
|
||||
'guid' => {"contents" => event3.id, "isPermaLink" => "false"},
|
||||
'pubDate' => Time.zone.parse(event3.payload['date']).rfc2822,
|
||||
|
@ -244,14 +244,20 @@ describe Agents::DataOutputAgent do
|
|||
|
||||
describe 'ordering' do
|
||||
before do
|
||||
agent.options['events_order'] = ['{{title}}']
|
||||
agent.options['events_order'] = ['{{hovertext}}']
|
||||
agent.options['events_list_order'] = ['{{title}}']
|
||||
end
|
||||
|
||||
it 'can reorder the events_to_show last events based on a Liquid expression' do
|
||||
agent.options['events_to_show'] = 2
|
||||
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
|
||||
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again"])
|
||||
|
||||
agent.options['events_to_show'] = 40
|
||||
asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
|
||||
expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])
|
||||
|
||||
agent.options['events_order'] = [['{{title}}', 'string', true]]
|
||||
agent.options['events_list_order'] = [['{{title}}', 'string', true]]
|
||||
|
||||
desc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json')
|
||||
expect(desc_content['items']).to eq(asc_content['items'].reverse)
|
||||
|
|
Loading…
Add table
Reference in a new issue