mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
calculate ids if none are available
This commit is contained in:
parent
f29aedd8c6
commit
34e7a52f0c
3 changed files with 31 additions and 3 deletions
|
@ -55,10 +55,11 @@ module Agents
|
|||
feed.clean! if interpolated['clean'] == 'true'
|
||||
created_event_count = 0
|
||||
feed.entries.each do |entry|
|
||||
if check_and_track(entry.id)
|
||||
entry_id = get_entry_id(entry)
|
||||
if check_and_track(entry_id)
|
||||
created_event_count += 1
|
||||
create_event(:payload => {
|
||||
:id => entry.id,
|
||||
:id => entry_id,
|
||||
:date_published => entry.date_published,
|
||||
:last_updated => entry.last_updated,
|
||||
:urls => entry.urls,
|
||||
|
@ -78,6 +79,10 @@ module Agents
|
|||
|
||||
protected
|
||||
|
||||
def get_entry_id(entry)
|
||||
entry.id.presence || Digest::MD5.hexdigest(entry.content)
|
||||
end
|
||||
|
||||
def check_and_track(entry_id)
|
||||
memory['seen_ids'] ||= []
|
||||
if memory['seen_ids'].include?(entry_id)
|
||||
|
|
9
spec/data_fixtures/slickdeals.atom
Normal file
9
spec/data_fixtures/slickdeals.atom
Normal file
File diff suppressed because one or more lines are too long
|
@ -8,10 +8,11 @@ describe Agents::RssAgent do
|
|||
}
|
||||
|
||||
stub_request(:any, /github.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/github_rss.atom")), :status => 200)
|
||||
stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
|
||||
end
|
||||
|
||||
let(:agent) do
|
||||
_agent = Agents::RssAgent.new(:name => "github rss feed", :options => @valid_options)
|
||||
_agent = Agents::RssAgent.new(:name => "rss feed", :options => @valid_options)
|
||||
_agent.user = users(:bob)
|
||||
_agent.save!
|
||||
_agent
|
||||
|
@ -78,4 +79,17 @@ describe Agents::RssAgent do
|
|||
agent.memory['seen_ids'].length.should == 500
|
||||
end
|
||||
end
|
||||
|
||||
context "when no ids are available" do
|
||||
before do
|
||||
@valid_options['url'] = 'http://feeds.feedburner.com/SlickdealsnetFP?format=atom'
|
||||
end
|
||||
|
||||
it "calculates content MD5 sums" do
|
||||
lambda {
|
||||
agent.check
|
||||
}.should change { agent.events.count }.by(79)
|
||||
agent.memory['seen_ids'].should == agent.events.map {|e| Digest::MD5.hexdigest(e.payload['content']) }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Add table
Reference in a new issue