From fe50730033bb061f0ee3ad45ae9e49cea6048dd1 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 19 Apr 2016 23:25:52 +0900 Subject: [PATCH] DataOutputAgent should limit events after ordering This should fix #1044. --- app/models/agents/data_output_agent.rb | 42 +++++++++++++++++++- spec/models/agents/data_output_agent_spec.rb | 6 +++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/app/models/agents/data_output_agent.rb b/app/models/agents/data_output_agent.rb index 29ac95d8..ca2d4aaf 100644 --- a/app/models/agents/data_output_agent.rb +++ b/app/models/agents/data_output_agent.rb @@ -175,6 +175,43 @@ module Agents interpolated['push_hubs'].presence || [] end + def sorted_events(reload = false) + events = + if (event_ids = memory[:event_ids]) && + memory[:events_order] == events_order && + memory[:events_to_show] >= events_to_show + received_events.where(id: event_ids).to_a + else + memory[:last_event_id] = nil + reload = true + [] + end + + if reload + new_events = + if last_event_id = memory[:last_event_id] + received_events.order(id: :desc).where(Event.arel_table[:id].gt(last_event_id)) + else + # dig at least twice as many events as the number of + # `events_to_show` + received_events.order(id: :desc).limit([source_ids.count, 2].max * events_to_show) + end.to_a + events = new_events.concat(events) + memory[:events_order] = events_order + memory[:events_to_show] = events_to_show + memory[:last_event_id] = events.first.try!(:id) + end + + events = sort_events(events).last(events_to_show) + + if reload + memory[:event_ids] = events.map(&:id) + save + end + + events + end + def receive_web_request(params, method, format) unless interpolated['secrets'].include?(params['secret']) if format =~ /json/ @@ -184,7 +221,7 @@ module Agents end end - source_events = sort_events(received_events.order(id: :desc).limit(events_to_show).to_a) + source_events = sorted_events() interpolation_context.stack do interpolation_context['events'] = source_events @@ -251,6 +288,9 @@ module Agents def receive(incoming_events) url = feed_url(secret: interpolated['secrets'].first, format: :xml) + # Reload new events and update cache + sorted_events(true) + push_hubs.each do |hub| push_to_hub(hub, url) end diff --git a/spec/models/agents/data_output_agent_spec.rb b/spec/models/agents/data_output_agent_spec.rb index 5b9e3f5c..19487f57 100644 --- a/spec/models/agents/data_output_agent_spec.rb +++ b/spec/models/agents/data_output_agent_spec.rb @@ -248,6 +248,12 @@ describe Agents::DataOutputAgent do end it 'can reorder the events_to_show last events based on a Liquid expression' do + # Check that ordering takes place before limiting, not after + agent.options['events_to_show'] = 2 + asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json') + expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving again", "Evolving yet again with a past date"]) + + agent.options['events_to_show'] = 40 asc_content, _status, _content_type = agent.receive_web_request({ 'secret' => 'secret2' }, 'get', 'application/json') expect(asc_content['items'].map {|i| i["title"] }).to eq(["Evolving", "Evolving again", "Evolving yet again with a past date"])