GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-15 19:31:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support the AtomYoutube feed format of Feedjira

Browse source 
This fixes #2054.
This commit is contained in:
Akinori MUSHA 2017-08-07 02:21:05 +09:00
parent 83a6fd091a
commit 0339a23e45
3 changed files with 409 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/feedjira_extension.rb
View file

@ -295,6 +295,8 @@ module FeedjiraExtension
_itunes_owners.reject(&:empty?) _itunes_owners.reject(&:empty?)
end end
end end
else
element :subtitle, as: :description unless method_defined?(:description)
end end
sax_config.collection_elements.each_value do |collection_elements| sax_config.collection_elements.each_value do |collection_elements|

358
spec/data_fixtures/youtube.xml Normal file
View file

@ -0,0 +1,358 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns:yt="http://www.youtube.com/xml/schemas/2015" xmlns:media="http://search.yahoo.com/mrss/" xmlns="http://www.w3.org/2005/Atom">
<link rel="self" href="http://www.youtube.com/feeds/videos.xml?channel_id=UCoTLdfNePDQzvdEgIToLIUg"/>
<id>yt:channel:UCoTLdfNePDQzvdEgIToLIUg</id>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>SecDSM</title>
<link rel="alternate" href="https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2016-07-28T18:46:21+00:00</published>
<entry>
<id>yt:video:OCs1E0vP7Oc</id>
<yt:videoId>OCs1E0vP7Oc</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>SecDSM 2017 March - Talk 01</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=OCs1E0vP7Oc"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-06-15T02:36:17+00:00</published>
<updated>2017-06-15T02:36:17+00:00</updated>
<media:group>
<media:title>SecDSM 2017 March - Talk 01</media:title>
<media:content url="https://www.youtube.com/v/OCs1E0vP7Oc?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i4.ytimg.com/vi/OCs1E0vP7Oc/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="36"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:Qky7RDF5Yk8</id>
<yt:videoId>Qky7RDF5Yk8</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>SecDSM 2017 March - Talk 02</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=Qky7RDF5Yk8"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-06-15T02:36:09+00:00</published>
<updated>2017-06-15T02:36:09+00:00</updated>
<media:group>
<media:title>SecDSM 2017 March - Talk 02</media:title>
<media:content url="https://www.youtube.com/v/Qky7RDF5Yk8?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i2.ytimg.com/vi/Qky7RDF5Yk8/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="14"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:4b6GtFgQYuY</id>
<yt:videoId>4b6GtFgQYuY</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>SecDSM March 2017 - Announcements</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=4b6GtFgQYuY"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-05-11T03:29:47+00:00</published>
<updated>2017-05-11T03:29:47+00:00</updated>
<media:group>
<media:title>SecDSM March 2017 - Announcements</media:title>
<media:content url="https://www.youtube.com/v/4b6GtFgQYuY?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i1.ytimg.com/vi/4b6GtFgQYuY/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="13"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:CrkSsvl3HRg</id>
<yt:videoId>CrkSsvl3HRg</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>SecDSM 2017 April - Principal CTF</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=CrkSsvl3HRg"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-05-01T21:24:37+00:00</published>
<updated>2017-05-04T07:08:10+00:00</updated>
<media:group>
<media:title>SecDSM 2017 April - Principal CTF</media:title>
<media:content url="https://www.youtube.com/v/CrkSsvl3HRg?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i4.ytimg.com/vi/CrkSsvl3HRg/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="66"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:Pin_Mq06fWE</id>
<yt:videoId>Pin_Mq06fWE</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Tom Pohl - SecDSM, 1 year later</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=Pin_Mq06fWE"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-03-12T04:17:12+00:00</published>
<updated>2017-03-12T04:17:12+00:00</updated>
<media:group>
<media:title>Tom Pohl - SecDSM, 1 year later</media:title>
<media:content url="https://www.youtube.com/v/Pin_Mq06fWE?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i1.ytimg.com/vi/Pin_Mq06fWE/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="28"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:vsPFzzvFdkQ</id>
<yt:videoId>vsPFzzvFdkQ</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Jared Brees - Password Cracking</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=vsPFzzvFdkQ"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-03-12T04:16:57+00:00</published>
<updated>2017-03-12T04:16:57+00:00</updated>
<media:group>
<media:title>Jared Brees - Password Cracking</media:title>
<media:content url="https://www.youtube.com/v/vsPFzzvFdkQ?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i3.ytimg.com/vi/vsPFzzvFdkQ/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="20"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:WvdoRVZsVDA</id>
<yt:videoId>WvdoRVZsVDA</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Nicholas Starke - SQLMap</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=WvdoRVZsVDA"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-03-12T04:16:37+00:00</published>
<updated>2017-03-12T04:16:37+00:00</updated>
<media:group>
<media:title>Nicholas Starke - SQLMap</media:title>
<media:content url="https://www.youtube.com/v/WvdoRVZsVDA?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i4.ytimg.com/vi/WvdoRVZsVDA/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="7"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:vI6uX5DBx6Q</id>
<yt:videoId>vI6uX5DBx6Q</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Ben Schmitt - dnscat2</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=vI6uX5DBx6Q"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-01-22T22:05:17+00:00</published>
<updated>2017-01-24T05:09:50+00:00</updated>
<media:group>
<media:title>Ben Schmitt - dnscat2</media:title>
<media:content url="https://www.youtube.com/v/vI6uX5DBx6Q?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i3.ytimg.com/vi/vI6uX5DBx6Q/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="1" average="5.00" min="1" max="5"/>
<media:statistics views="133"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:zN2WijXEY9k</id>
<yt:videoId>zN2WijXEY9k</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Nicholas Starke &amp; Tom Pohl - WeevBot - Browser Based DNS Exfiltration</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=zN2WijXEY9k"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-01-22T22:04:46+00:00</published>
<updated>2017-03-30T01:27:17+00:00</updated>
<media:group>
<media:title>Nicholas Starke &amp; Tom Pohl - WeevBot - Browser Based DNS Exfiltration</media:title>
<media:content url="https://www.youtube.com/v/zN2WijXEY9k?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i3.ytimg.com/vi/zN2WijXEY9k/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="2" average="5.00" min="1" max="5"/>
<media:statistics views="122"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:wR_O3485-Gw</id>
<yt:videoId>wR_O3485-Gw</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Aaron Tekippe - Finding a needle in a haystack: Utilizing DNS to detect threats</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=wR_O3485-Gw"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-01-22T21:49:32+00:00</published>
<updated>2017-01-22T21:49:32+00:00</updated>
<media:group>
<media:title>Aaron Tekippe - Finding a needle in a haystack: Utilizing DNS to detect threats</media:title>
<media:content url="https://www.youtube.com/v/wR_O3485-Gw?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i4.ytimg.com/vi/wR_O3485-Gw/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="33"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:hvhde2F6Jms</id>
<yt:videoId>hvhde2F6Jms</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Sean Flattery - Sudoers and Auditd!</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=hvhde2F6Jms"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2017-01-20T22:41:07+00:00</published>
<updated>2017-04-21T01:49:33+00:00</updated>
<media:group>
<media:title>Sean Flattery - Sudoers and Auditd!</media:title>
<media:content url="https://www.youtube.com/v/hvhde2F6Jms?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i1.ytimg.com/vi/hvhde2F6Jms/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="1" average="5.00" min="1" max="5"/>
<media:statistics views="43"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:6YIlFnvlMjU</id>
<yt:videoId>6YIlFnvlMjU</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>2016 Defcon Recap</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=6YIlFnvlMjU"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2016-11-28T16:21:11+00:00</published>
<updated>2016-11-29T01:58:27+00:00</updated>
<media:group>
<media:title>2016 Defcon Recap</media:title>
<media:content url="https://www.youtube.com/v/6YIlFnvlMjU?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i3.ytimg.com/vi/6YIlFnvlMjU/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="22"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:0GVO3ouyudE</id>
<yt:videoId>0GVO3ouyudE</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>James Stumme - Updating the SecDSM Website</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=0GVO3ouyudE"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2016-11-28T16:21:10+00:00</published>
<updated>2016-11-28T16:54:15+00:00</updated>
<media:group>
<media:title>James Stumme - Updating the SecDSM Website</media:title>
<media:content url="https://www.youtube.com/v/0GVO3ouyudE?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i1.ytimg.com/vi/0GVO3ouyudE/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="14"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:x0z1_crrigI</id>
<yt:videoId>x0z1_crrigI</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Michael Jackson - NextGen Vendor Risk Management: How to Raise the Bar on aaS Providers.</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=x0z1_crrigI"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2016-11-28T16:21:10+00:00</published>
<updated>2016-11-28T16:54:50+00:00</updated>
<media:group>
<media:title>Michael Jackson - NextGen Vendor Risk Management: How to Raise the Bar on aaS Providers.</media:title>
<media:content url="https://www.youtube.com/v/x0z1_crrigI?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i1.ytimg.com/vi/x0z1_crrigI/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="0" average="0.00" min="1" max="5"/>
<media:statistics views="9"/>
</media:community>
</media:group>
</entry>
<entry>
<id>yt:video:nSM2vTYyACQ</id>
<yt:videoId>nSM2vTYyACQ</yt:videoId>
<yt:channelId>UCoTLdfNePDQzvdEgIToLIUg</yt:channelId>
<title>Nicholas Starke - BinaryNinja</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=nSM2vTYyACQ"/>
<author>
<name>SecDSM</name>
<uri>https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg</uri>
</author>
<published>2016-11-28T16:21:10+00:00</published>
<updated>2017-02-03T09:11:35+00:00</updated>
<media:group>
<media:title>Nicholas Starke - BinaryNinja</media:title>
<media:content url="https://www.youtube.com/v/nSM2vTYyACQ?version=3" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://i3.ytimg.com/vi/nSM2vTYyACQ/hqdefault.jpg" width="480" height="360"/>
<media:description></media:description>
<media:community>
<media:starRating count="2" average="5.00" min="1" max="5"/>
<media:statistics views="46"/>
</media:community>
</media:group>
</entry>
</feed>

49
spec/models/agents/rss_agent_spec.rb
View file

@ -14,6 +14,7 @@ describe Agents::RssAgent do
stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200) stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200) stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200) stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
stub_request(:any, /youtube/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/youtube.xml")), status: 200)
end end
let(:agent) do let(:agent) do
@ -485,6 +486,54 @@ describe Agents::RssAgent do
]) ])
end end
end end
context 'of YouTube' do
before do
@valid_options['url'] = 'http://example.com/youtube.xml'
@valid_options['include_feed_info'] = true
end
it "is parsed correctly" do
expect {
agent.check
}.to change { agent.events.count }.by(15)
expect(agent.events.first.payload).to match({
"feed" => {
"id" => "yt:channel:UCoTLdfNePDQzvdEgIToLIUg",
"type" => "atom",
"url" => "https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg",
"links" => [
{ "href" => "http://www.youtube.com/feeds/videos.xml?channel_id=UCoTLdfNePDQzvdEgIToLIUg", "rel" => "self" },
{ "href" => "https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg", "rel" => "alternate" }
],
"title" => "SecDSM",
"description" => nil,
"copyright" => nil,
"generator" => nil,
"icon" => nil,
"authors" => ["SecDSM (https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg)"],
"date_published" => "2016-07-28T18:46:21+00:00",
"last_updated" => "2016-07-28T18:46:21+00:00"
},
"id" => "yt:video:OCs1E0vP7Oc",
"authors" => ["SecDSM (https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg)"],
"categories" => [],
"content" => nil,
"date_published" => "2017-06-15T02:36:17+00:00",
"description" => nil,
"enclosure" => nil,
"image" => nil,
"last_updated" => "2017-06-15T02:36:17+00:00",
"links" => [
{ "href"=>"https://www.youtube.com/watch?v=OCs1E0vP7Oc", "rel"=>"alternate" }
],
"title" => "SecDSM 2017 March - Talk 01",
"url" => "https://www.youtube.com/watch?v=OCs1E0vP7Oc",
"urls" => ["https://www.youtube.com/watch?v=OCs1E0vP7Oc"]
})
end
end
end end
describe 'logging errors with the feed url' do describe 'logging errors with the feed url' do