calculate ids if none are available

This commit is contained in:
Andrew Cantino 2014-09-13 11:51:12 -07:00
parent f29aedd8c6
commit 34e7a52f0c
3 changed files with 31 additions and 3 deletions

View file

@ -55,10 +55,11 @@ module Agents
feed.clean! if interpolated['clean'] == 'true'
created_event_count = 0
feed.entries.each do |entry|
if check_and_track(entry.id)
entry_id = get_entry_id(entry)
if check_and_track(entry_id)
created_event_count += 1
create_event(:payload => {
:id => entry.id,
:id => entry_id,
:date_published => entry.date_published,
:last_updated => entry.last_updated,
:urls => entry.urls,
@ -78,6 +79,10 @@ module Agents
protected
def get_entry_id(entry)
entry.id.presence || Digest::MD5.hexdigest(entry.content)
end
def check_and_track(entry_id)
memory['seen_ids'] ||= []
if memory['seen_ids'].include?(entry_id)

File diff suppressed because one or more lines are too long

View file

@ -8,10 +8,11 @@ describe Agents::RssAgent do
}
stub_request(:any, /github.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/github_rss.atom")), :status => 200)
stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
end
let(:agent) do
_agent = Agents::RssAgent.new(:name => "github rss feed", :options => @valid_options)
_agent = Agents::RssAgent.new(:name => "rss feed", :options => @valid_options)
_agent.user = users(:bob)
_agent.save!
_agent
@ -78,4 +79,17 @@ describe Agents::RssAgent do
agent.memory['seen_ids'].length.should == 500
end
end
context "when no ids are available" do
before do
@valid_options['url'] = 'http://feeds.feedburner.com/SlickdealsnetFP?format=atom'
end
it "calculates content MD5 sums" do
lambda {
agent.check
}.should change { agent.events.count }.by(79)
agent.memory['seen_ids'].should == agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) }
end
end
end