From 0339a23e455cfdec4215b146bac8fe0f9b40bb52 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 7 Aug 2017 02:21:05 +0900 Subject: [PATCH] Support the `AtomYoutube` feed format of Feedjira This fixes #2054. --- lib/feedjira_extension.rb | 2 + spec/data_fixtures/youtube.xml | 358 +++++++++++++++++++++++++++ spec/models/agents/rss_agent_spec.rb | 49 ++++ 3 files changed, 409 insertions(+) create mode 100644 spec/data_fixtures/youtube.xml diff --git a/lib/feedjira_extension.rb b/lib/feedjira_extension.rb index 2aee706b..a5058561 100644 --- a/lib/feedjira_extension.rb +++ b/lib/feedjira_extension.rb @@ -295,6 +295,8 @@ module FeedjiraExtension _itunes_owners.reject(&:empty?) end end + else + element :subtitle, as: :description unless method_defined?(:description) end sax_config.collection_elements.each_value do |collection_elements| diff --git a/spec/data_fixtures/youtube.xml b/spec/data_fixtures/youtube.xml new file mode 100644 index 00000000..148fa101 --- /dev/null +++ b/spec/data_fixtures/youtube.xml @@ -0,0 +1,358 @@ + + + + yt:channel:UCoTLdfNePDQzvdEgIToLIUg + UCoTLdfNePDQzvdEgIToLIUg + SecDSM + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2016-07-28T18:46:21+00:00 + + yt:video:OCs1E0vP7Oc + OCs1E0vP7Oc + UCoTLdfNePDQzvdEgIToLIUg + SecDSM 2017 March - Talk 01 + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-06-15T02:36:17+00:00 + 2017-06-15T02:36:17+00:00 + + SecDSM 2017 March - Talk 01 + + + + + + + + + + + yt:video:Qky7RDF5Yk8 + Qky7RDF5Yk8 + UCoTLdfNePDQzvdEgIToLIUg + SecDSM 2017 March - Talk 02 + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-06-15T02:36:09+00:00 + 2017-06-15T02:36:09+00:00 + + SecDSM 2017 March - Talk 02 + + + + + + + + + + + yt:video:4b6GtFgQYuY + 4b6GtFgQYuY + UCoTLdfNePDQzvdEgIToLIUg + SecDSM March 2017 - Announcements + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-05-11T03:29:47+00:00 + 2017-05-11T03:29:47+00:00 + + SecDSM March 2017 - Announcements + + + + + + + + + + + yt:video:CrkSsvl3HRg + CrkSsvl3HRg + UCoTLdfNePDQzvdEgIToLIUg + SecDSM 2017 April - Principal CTF + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-05-01T21:24:37+00:00 + 2017-05-04T07:08:10+00:00 + + SecDSM 2017 April - Principal CTF + + + + + + + + + + + yt:video:Pin_Mq06fWE + Pin_Mq06fWE + UCoTLdfNePDQzvdEgIToLIUg + Tom Pohl - SecDSM, 1 year later + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-03-12T04:17:12+00:00 + 2017-03-12T04:17:12+00:00 + + Tom Pohl - SecDSM, 1 year later + + + + + + + + + + + yt:video:vsPFzzvFdkQ + vsPFzzvFdkQ + UCoTLdfNePDQzvdEgIToLIUg + Jared Brees - Password Cracking + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-03-12T04:16:57+00:00 + 2017-03-12T04:16:57+00:00 + + Jared Brees - Password Cracking + + + + + + + + + + + yt:video:WvdoRVZsVDA + WvdoRVZsVDA + UCoTLdfNePDQzvdEgIToLIUg + Nicholas Starke - SQLMap + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-03-12T04:16:37+00:00 + 2017-03-12T04:16:37+00:00 + + Nicholas Starke - SQLMap + + + + + + + + + + + yt:video:vI6uX5DBx6Q + vI6uX5DBx6Q + UCoTLdfNePDQzvdEgIToLIUg + Ben Schmitt - dnscat2 + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-01-22T22:05:17+00:00 + 2017-01-24T05:09:50+00:00 + + Ben Schmitt - dnscat2 + + + + + + + + + + + yt:video:zN2WijXEY9k + zN2WijXEY9k + UCoTLdfNePDQzvdEgIToLIUg + Nicholas Starke & Tom Pohl - WeevBot - Browser Based DNS Exfiltration + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-01-22T22:04:46+00:00 + 2017-03-30T01:27:17+00:00 + + Nicholas Starke & Tom Pohl - WeevBot - Browser Based DNS Exfiltration + + + + + + + + + + + yt:video:wR_O3485-Gw + wR_O3485-Gw + UCoTLdfNePDQzvdEgIToLIUg + Aaron Tekippe - Finding a needle in a haystack: Utilizing DNS to detect threats + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-01-22T21:49:32+00:00 + 2017-01-22T21:49:32+00:00 + + Aaron Tekippe - Finding a needle in a haystack: Utilizing DNS to detect threats + + + + + + + + + + + yt:video:hvhde2F6Jms + hvhde2F6Jms + UCoTLdfNePDQzvdEgIToLIUg + Sean Flattery - Sudoers and Auditd! + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2017-01-20T22:41:07+00:00 + 2017-04-21T01:49:33+00:00 + + Sean Flattery - Sudoers and Auditd! + + + + + + + + + + + yt:video:6YIlFnvlMjU + 6YIlFnvlMjU + UCoTLdfNePDQzvdEgIToLIUg + 2016 Defcon Recap + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2016-11-28T16:21:11+00:00 + 2016-11-29T01:58:27+00:00 + + 2016 Defcon Recap + + + + + + + + + + + yt:video:0GVO3ouyudE + 0GVO3ouyudE + UCoTLdfNePDQzvdEgIToLIUg + James Stumme - Updating the SecDSM Website + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2016-11-28T16:21:10+00:00 + 2016-11-28T16:54:15+00:00 + + James Stumme - Updating the SecDSM Website + + + + + + + + + + + yt:video:x0z1_crrigI + x0z1_crrigI + UCoTLdfNePDQzvdEgIToLIUg + Michael Jackson - NextGen Vendor Risk Management: How to Raise the Bar on aaS Providers. + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2016-11-28T16:21:10+00:00 + 2016-11-28T16:54:50+00:00 + + Michael Jackson - NextGen Vendor Risk Management: How to Raise the Bar on aaS Providers. + + + + + + + + + + + yt:video:nSM2vTYyACQ + nSM2vTYyACQ + UCoTLdfNePDQzvdEgIToLIUg + Nicholas Starke - BinaryNinja + + + SecDSM + https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg + + 2016-11-28T16:21:10+00:00 + 2017-02-03T09:11:35+00:00 + + Nicholas Starke - BinaryNinja + + + + + + + + + + diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb index 5a375401..ef44d5bc 100644 --- a/spec/models/agents/rss_agent_spec.rb +++ b/spec/models/agents/rss_agent_spec.rb @@ -14,6 +14,7 @@ describe Agents::RssAgent do stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=)[^<]*/, ''), status: 200) stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200) stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200) + stub_request(:any, /youtube/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/youtube.xml")), status: 200) end let(:agent) do @@ -485,6 +486,54 @@ describe Agents::RssAgent do ]) end end + + context 'of YouTube' do + before do + @valid_options['url'] = 'http://example.com/youtube.xml' + @valid_options['include_feed_info'] = true + end + + it "is parsed correctly" do + expect { + agent.check + }.to change { agent.events.count }.by(15) + + expect(agent.events.first.payload).to match({ + "feed" => { + "id" => "yt:channel:UCoTLdfNePDQzvdEgIToLIUg", + "type" => "atom", + "url" => "https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg", + "links" => [ + { "href" => "http://www.youtube.com/feeds/videos.xml?channel_id=UCoTLdfNePDQzvdEgIToLIUg", "rel" => "self" }, + { "href" => "https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg", "rel" => "alternate" } + ], + "title" => "SecDSM", + "description" => nil, + "copyright" => nil, + "generator" => nil, + "icon" => nil, + "authors" => ["SecDSM (https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg)"], + "date_published" => "2016-07-28T18:46:21+00:00", + "last_updated" => "2016-07-28T18:46:21+00:00" + }, + "id" => "yt:video:OCs1E0vP7Oc", + "authors" => ["SecDSM (https://www.youtube.com/channel/UCoTLdfNePDQzvdEgIToLIUg)"], + "categories" => [], + "content" => nil, + "date_published" => "2017-06-15T02:36:17+00:00", + "description" => nil, + "enclosure" => nil, + "image" => nil, + "last_updated" => "2017-06-15T02:36:17+00:00", + "links" => [ + { "href"=>"https://www.youtube.com/watch?v=OCs1E0vP7Oc", "rel"=>"alternate" } + ], + "title" => "SecDSM 2017 March - Talk 01", + "url" => "https://www.youtube.com/watch?v=OCs1E0vP7Oc", + "urls" => ["https://www.youtube.com/watch?v=OCs1E0vP7Oc"] + }) + end + end end describe 'logging errors with the feed url' do