From 7ac691652b90893bc84d89f31d996edc290d5259 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sat, 26 Nov 2016 13:26:09 +0900 Subject: [PATCH 01/23] Spec that force_encoding works with encoding declaration in RssAgent --- spec/models/agents/rss_agent_spec.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb index a70565f8..31c6737e 100644 --- a/spec/models/agents/rss_agent_spec.rb +++ b/spec/models/agents/rss_agent_spec.rb @@ -295,6 +295,13 @@ describe Agents::RssAgent do event = agent.events.first expect(event.payload['title']).to eq('Mëkanïk Zaïn') end + + it "decodes the content properly with force_encoding specified" do + @valid_options['force_encoding'] = 'iso-8859-1' + agent.check + event = agent.events.first + expect(event.payload['title']).to eq('Mëkanïk Zaïn') + end end end From f530305edc39b730b9a8a607bbc61589c60da4a9 Mon Sep 17 00:00:00 2001 From: bobbysteel Date: Sat, 26 Nov 2016 20:13:00 +0000 Subject: [PATCH 02/23] Add class of service chooser for Google Flights Agent (#1778) * Add class of service chooser * Add cabin chooser test * Fix preferredCabin * Per @cantino feedback taking out check --- app/models/agents/google_flights_agent.rb | 6 ++++-- spec/models/agents/google_flights_spec.rb | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/models/agents/google_flights_agent.rb b/app/models/agents/google_flights_agent.rb index f7a82521..ccdc66b4 100644 --- a/app/models/agents/google_flights_agent.rb +++ b/app/models/agents/google_flights_agent.rb @@ -60,6 +60,7 @@ module Agents 'seniorCount'=> 0, 'return_date' => '2016-04-18', 'roundtrip' => true, + 'preferredCabin' => 'COACH', 'solutions'=> 3 } end @@ -69,6 +70,7 @@ module Agents form_configurable :origin, type: :string form_configurable :destination, type: :string form_configurable :date, type: :string + form_configurable :preferredCabin, type: :array, values: %w(COACH PREMIUM_COACH BUSINESS FIRST) form_configurable :childCount form_configurable :infantInSeatCount form_configurable :infantInLapCount @@ -101,9 +103,9 @@ module Agents def post_params if round_trip? - post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s } ], :solutions=> interpolated["solutions"]}} + post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s} ], :solutions=> interpolated["solutions"]}} else - post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }], :solutions=> interpolated["solutions"]}} + post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }], :solutions=> interpolated["solutions"]}} end end diff --git a/spec/models/agents/google_flights_spec.rb b/spec/models/agents/google_flights_spec.rb index 28f5994d..a0b970b8 100644 --- a/spec/models/agents/google_flights_spec.rb +++ b/spec/models/agents/google_flights_spec.rb @@ -15,6 +15,7 @@ describe Agents::GoogleFlightsAgent do 'origin' => 'BOS', 'destination' => 'SFO', 'date' => '2016-04-11', + 'preferredCabin' => 'COACH', 'childCount' => 0, 'infantInSeatCount' => 0, 'infantInLapCount'=> 0, From 3d91469733bf2b0c0d00815a33faf0ac688e18ee Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 27 Nov 2016 13:33:24 +0900 Subject: [PATCH 03/23] Make WebsiteAgent merge `template` with the results of `extract` (#1816) A new extraction option `hidden` is added so that keys with it gets excluded from the final payloads while they can be used in `template`. --- app/models/agents/website_agent.rb | 82 +++++++++++------ ...onvert_website_agent_template_for_merge.rb | 36 ++++++++ ...t_website_agent_template_for_merge_spec.rb | 92 +++++++++++++++++++ spec/models/agents/website_agent_spec.rb | 23 ++++- 4 files changed, 201 insertions(+), 32 deletions(-) create mode 100644 db/migrate/20161124061256_convert_website_agent_template_for_merge.rb create mode 100644 spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 437df864..f36d70e3 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -37,6 +37,8 @@ module Agents Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor except when it has `repeat` set to true. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful. + For extractors with `hidden` set to true, they will be excluded from the payloads of events created by the Agent, but can be used and interpolated in the `template` option explained below. + For extractors with `repeat` set to true, their first matches will be included in all extracts. This is useful such as when you want to include the title of a page in all events created from the page. # Scraping HTML and XML @@ -116,11 +118,9 @@ module Agents Set `http_success_codes` to an array of status codes (e.g., `[404, 422]`) to treat HTTP response codes beyond 200 as successes. - If a `template` option is given, it is used as a Liquid template for each event created by this Agent, instead of directly emitting the results of extraction as events. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example: + If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example: "template": { - "url": "{{ url }}", - "title": "{{ title }}", "description": "{{ body_text }}", "last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}" } @@ -159,7 +159,11 @@ module Agents end def event_keys - (options['template'].presence || options['extract']).try(:keys) + extract = options['extract'] or return nil + + extract.each_with_object([]) { |(key, value), keys| + keys << key unless boolify(value['hidden']) + } | (options['template'].presence.try!(:keys) || []) end def working? @@ -382,33 +386,19 @@ module Agents extract_xml(doc) end - num_tuples = output.each_value.inject(nil) { |num, value| - case size = value.size - when Float::INFINITY - num - when Integer - if num && num != size - raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}" - end - size - end - } or raise "At least one non-repeat key is required" + num_tuples = output.size or + raise "At least one non-repeat key is required" old_events = previous_payloads num_tuples template = options['template'].presence - num_tuples.times.zip(*output.values) do |index, *values| - extracted = output.each_key.lazy.zip(values).to_h + output.each do |extracted| + result = extracted.except(*output.hidden_keys) - result = - if template - interpolate_with(extracted) do - interpolate_options(template) - end - else - extracted - end + if template + result.update(interpolate_options(template, extracted)) + end # url may be URI, string or nil if (payload_url = result['url'].presence) && (url = url.presence) @@ -528,7 +518,7 @@ module Agents end def extract_each(&block) - interpolated['extract'].each_with_object({}) { |(name, extraction_details), output| + interpolated['extract'].each_with_object(Output.new) { |(name, extraction_details), output| if boolify(extraction_details['repeat']) values = Repeater.new { |repeater| block.call(extraction_details, repeater) @@ -538,7 +528,13 @@ module Agents block.call(extraction_details, values) end log "Values extracted: #{values}" - output[name] = values + begin + output[name] = values + rescue UnevenSizeError + raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}" + else + output.hidden_keys << name if boolify(extraction_details['hidden']) + end } end @@ -617,6 +613,38 @@ module Agents false end + class UnevenSizeError < ArgumentError + end + + class Output + def initialize + @hash = {} + @size = nil + @hidden_keys = [] + end + + attr_reader :size + attr_reader :hidden_keys + + def []=(key, value) + case size = value.size + when Integer + if @size && @size != size + raise UnevenSizeError, 'got an uneven size' + end + @size = size + end + + @hash[key] = value + end + + def each + @size.times.zip(*@hash.values) do |index, *values| + yield @hash.each_key.lazy.zip(values).to_h + end + end + end + class Repeater < Enumerator # Repeater.new { |y| # # ... diff --git a/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb new file mode 100644 index 00000000..65b22f4d --- /dev/null +++ b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb @@ -0,0 +1,36 @@ +class ConvertWebsiteAgentTemplateForMerge < ActiveRecord::Migration[5.0] + def up + Agents::WebsiteAgent.find_each do |agent| + extract = agent.options['extract'].presence + template = agent.options['template'].presence + next unless extract.is_a?(Hash) && template.is_a?(Hash) + + (extract.keys - template.keys).each do |key| + extract[key]['hidden'] = true + end + + template.delete_if { |key, value| + extract.key?(key) && + value.match(/\A\{\{\s*#{Regexp.quote(key)}\s*\}\}\z/) + } + + agent.save!(validate: false) + end + end + + def down + Agents::WebsiteAgent.find_each do |agent| + extract = agent.options['extract'].presence + template = agent.options['template'].presence + next unless extract.is_a?(Hash) && template.is_a?(Hash) + + (extract.keys - template.keys).each do |key| + unless extract[key].delete('hidden').in?([true, 'true']) + template[key] = "{{ #{key} }}" + end + end + + agent.save!(validate: false) + end + end +end diff --git a/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb new file mode 100644 index 00000000..af6c137f --- /dev/null +++ b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb @@ -0,0 +1,92 @@ +load 'spec/rails_helper.rb' +load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb') + +describe ConvertWebsiteAgentTemplateForMerge do + let :old_extract do + { + 'url' => { 'css' => "#comic img", 'value' => "@src" }, + 'title' => { 'css' => "#comic img", 'value' => "@alt" }, + 'hovertext' => { 'css' => "#comic img", 'value' => "@title" } + } + end + + let :new_extract do + { + 'url' => { 'css' => "#comic img", 'value' => "@src" }, + 'title' => { 'css' => "#comic img", 'value' => "@alt" }, + 'hovertext' => { 'css' => "#comic img", 'value' => "@title", 'hidden' => true } + } + end + + let :reverted_extract do + old_extract + end + + let :old_template do + { + 'url' => '{{url}}', + 'title' => '{{ title }}', + 'description' => '{{ hovertext }}', + 'comment' => '{{ comment }}' + } + end + + let :new_template do + { + 'description' => '{{ hovertext }}', + 'comment' => '{{ comment }}' + } + end + + let :reverted_template do + old_template.merge('url' => '{{ url }}') + end + + let :valid_options do + { + 'name' => "XKCD", + 'expected_update_period_in_days' => "2", + 'type' => "html", + 'url' => "{{ url | default: 'http://xkcd.com/' }}", + 'mode' => 'on_change', + 'extract' => old_extract, + 'template' => old_template + } + end + + let :agent do + Agents::WebsiteAgent.create!( + user: users(:bob), + name: "xkcd", + options: valid_options, + keep_events_for: 2.days + ) + end + + describe 'up' do + it 'should update extract and template options for an existing WebsiteAgent' do + expect(agent.options).to include('extract' => old_extract, + 'template' => old_template) + ConvertWebsiteAgentTemplateForMerge.new.up + agent.reload + expect(agent.options).to include('extract' => new_extract, + 'template' => new_template) + end + end + + describe 'down' do + let :valid_options do + super().merge('extract' => new_extract, + 'template' => new_template) + end + + it 'should revert extract and template options for an updated WebsiteAgent' do + expect(agent.options).to include('extract' => new_extract, + 'template' => new_template) + ConvertWebsiteAgentTemplateForMerge.new.down + agent.reload + expect(agent.options).to include('extract' => reverted_extract, + 'template' => reverted_template) + end + end +end diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 9f25752d..01d9f223 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -651,9 +651,22 @@ describe Agents::WebsiteAgent do @checker.options = @valid_options @checker.check event = Event.last - expect(event.payload['url']).to eq("http://imgs.xkcd.com/comics/evolving.png") - expect(event.payload['title']).to eq("Evolving") - expect(event.payload['hovertext']).to match(/^Biologists play reverse/) + expect(event.payload).to match( + 'url' => 'http://imgs.xkcd.com/comics/evolving.png', + 'title' => 'Evolving', + 'hovertext' => /^Biologists play reverse/ + ) + end + + it "should exclude hidden keys" do + @valid_options['extract']['hovertext']['hidden'] = true + @checker.options = @valid_options + @checker.check + event = Event.last + expect(event.payload).to match( + 'url' => 'http://imgs.xkcd.com/comics/evolving.png', + 'title' => 'Evolving' + ) end it "should turn relative urls to absolute" do @@ -749,9 +762,9 @@ describe Agents::WebsiteAgent do expect(event.payload['original_url']).to eq('http://xkcd.com/index') end - it "should be formatted by template after extraction" do + it "should format and merge values in template after extraction" do + @valid_options['extract']['hovertext']['hidden'] = true @valid_options['template'] = { - 'url' => '{{url}}', 'title' => '{{title | upcase}}', 'summary' => '{{title}}: {{hovertext | truncate: 20}}', } From 3a0c9e6274bf146af23aa14a32b8899b41910bc7 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Wed, 2 Nov 2016 21:03:31 +0900 Subject: [PATCH 04/23] Disable automatic URL normalization and absolutization on `url` This was discussed in #1766. For backward compatibility, existing WebsiteAgents with a key named `url` will be given a `template` to resolve `url`. --- app/models/agents/website_agent.rb | 50 ++++++++++++------- ...1124065838_add_templates_to_resolve_url.rb | 16 ++++++ ...65838_add_templates_to_resolve_url_spec.rb | 39 +++++++++++++++ spec/models/agents/website_agent_spec.rb | 38 ++++++-------- 4 files changed, 104 insertions(+), 39 deletions(-) create mode 100644 db/migrate/20161124065838_add_templates_to_resolve_url.rb create mode 100644 spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index f36d70e3..1918186e 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -27,8 +27,6 @@ module Agents * Alternatively, set `data_from_event` to a Liquid template to use data directly without fetching any URL. (For example, set it to `{{ html }}` to use HTML contained in the `html` key of the incoming Event.) * If you specify `merge` for the `mode` option, Huginn will retain the old payload and update it with new values. - If a created Event has a key named `url` containing a relative URL, it is automatically resolved using the request URL as base. - # Supported Document Types The `type` value can be `xml`, `html`, `json`, or `text`. @@ -121,6 +119,7 @@ module Agents If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example: "template": { + "url": "{{ url | to_uri: _request_.url }}", "description": "{{ body_text }}", "last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}" } @@ -129,17 +128,17 @@ module Agents # Liquid Templating - In Liquid templating, the following variables are available except when invoked by `data_from_event`: + In Liquid templating, the following variables are available: - * `_url_`: The URL specified to fetch the content from. + * `_url_`: The URL specified to fetch the content from. When parsing `data_from_event`, this is not set. * `_response_`: A response object with the following keys: - * `status`: HTTP status as integer. (Almost always 200) + * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event. - * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. + * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event. - * `url`: The final URL of the fetched page, following redirects. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`. + * `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`. # Ordering Events @@ -366,6 +365,8 @@ module Agents end def handle_data(body, url, existing_payload) + # Beware, url may be a URI object, string or nil + doc = parse(body) if extract_full_json? @@ -400,15 +401,6 @@ module Agents result.update(interpolate_options(template, extracted)) end - # url may be URI, string or nil - if (payload_url = result['url'].presence) && (url = url.presence) - begin - result['url'] = (Utils.normalize_uri(url) + Utils.normalize_uri(payload_url)).to_s - rescue URI::Error - error "Cannot resolve url: <#{payload_url}> on <#{url}>" - end - end - if store_payload!(old_events, result) log "Storing new parsed result for '#{name}': #{result.inspect}" create_event payload: existing_payload.merge(result) @@ -450,7 +442,10 @@ module Agents end def handle_event_data(data, event, existing_payload) - handle_data(data, event.payload['url'], existing_payload) + interpolation_context.stack { + interpolation_context['_response_'] = ResponseFromEventDrop.new(event) + handle_data(data, event.payload['url'].presence, existing_payload) + } rescue => e error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event end @@ -687,6 +682,27 @@ module Agents end end + class ResponseFromEventDrop < LiquidDroppable::Drop + def headers + case headers = @object.payload[:headers] + when Hash + HeaderDrop.new(Faraday::Utils::Headers.from(headers)) + else + HeaderDrop.new({}) + end + end + + # Integer value of HTTP status + def status + @object.payload[:status] + end + + # The URL + def url + @object.payload[:url] + end + end + # Wraps Faraday::Utils::Headers class HeaderDrop < LiquidDroppable::Drop def before_method(name) diff --git a/db/migrate/20161124065838_add_templates_to_resolve_url.rb b/db/migrate/20161124065838_add_templates_to_resolve_url.rb new file mode 100644 index 00000000..a1e62522 --- /dev/null +++ b/db/migrate/20161124065838_add_templates_to_resolve_url.rb @@ -0,0 +1,16 @@ +class AddTemplatesToResolveUrl < ActiveRecord::Migration[5.0] + def up + Agents::WebsiteAgent.find_each do |agent| + if agent.event_keys.try!(:include?, 'url') + agent.options['template'] = (agent.options['template'] || {}).tap { |template| + template['url'] ||= '{{ url | to_uri: _response_.url }}' + } + agent.save!(validate: false) + end + end + end + + def down + # No need to revert + end +end diff --git a/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb new file mode 100644 index 00000000..6d0bd14b --- /dev/null +++ b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb @@ -0,0 +1,39 @@ +load 'spec/rails_helper.rb' +load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb') + +describe AddTemplatesToResolveUrl do + let :valid_options do + { + 'name' => "XKCD", + 'expected_update_period_in_days' => "2", + 'type' => "html", + 'url' => "http://xkcd.com", + 'mode' => 'on_change', + 'extract' => { + 'url' => { 'css' => "#comic img", 'value' => "@src" }, + 'title' => { 'css' => "#comic img", 'value' => "@alt" }, + 'hovertext' => { 'css' => "#comic img", 'value' => "@title" } + } + } + end + + let :agent do + Agents::WebsiteAgent.create!( + user: users(:bob), + name: "xkcd", + options: valid_options, + keep_events_for: 2.days + ) + end + + it 'should add a template for an existing WebsiteAgent with `url`' do + expect(agent.options).not_to include('template') + AddTemplatesToResolveUrl.new.up + agent.reload + expect(agent.options).to include( + 'template' => { + 'url' => '{{ url | to_uri: _response_.url }}' + } + ) + end +end diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 01d9f223..d943ca77 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -669,25 +669,6 @@ describe Agents::WebsiteAgent do ) end - it "should turn relative urls to absolute" do - rel_site = { - 'name' => "XKCD", - 'expected_update_period_in_days' => "2", - 'type' => "html", - 'url' => "http://xkcd.com", - 'mode' => "on_change", - 'extract' => { - 'url' => {'css' => "#topLeft a", 'value' => "@href"}, - } - } - rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site) - rel.user = users(:bob) - rel.save! - rel.check - event = Event.last - expect(event.payload['url']).to eq("http://xkcd.com/about") - end - it "should return an integer value if XPath evaluates to one" do rel_site = { 'name' => "XKCD", @@ -1198,7 +1179,11 @@ fire: hot 'some_object' => { 'some_data' => { hello: 'world', href: '/world' }.to_json }, - url: 'http://example.com/' + url: 'http://example.com/', + 'headers' => { + 'Content-Type' => 'application/json' + }, + 'status' => 200 } @event.save! @@ -1208,6 +1193,12 @@ fire: hot 'extract' => { 'value' => { 'path' => 'hello' }, 'url' => { 'path' => 'href' }, + }, + 'template' => { + 'value' => '{{ value }}', + 'url' => '{{ url | to_uri: _response_.url }}', + 'type' => '{{ _response_.headers.content_type }}', + 'status' => '{{ _response_.status }}' } ) end @@ -1216,7 +1207,7 @@ fire: hot expect { @checker.receive([@event]) }.to change { Event.count }.by(1) - expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world' }) + expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' }) end it "should support merge mode" do @@ -1225,7 +1216,7 @@ fire: hot expect { @checker.receive([@event]) }.to change { Event.count }.by(1) - expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world')) + expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200')) end it "should output an error when nothing can be found at the path" do @@ -1362,6 +1353,9 @@ fire: hot 'mode' => 'all', 'extract' => { 'url' => { 'css' => "a", 'value' => "@href" }, + }, + 'template' => { + 'url' => '{{ url | to_uri }}', } } @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options) From a94cd7fd6d749cf37851f9db2eecc9bf99e00568 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sat, 12 Nov 2016 14:56:39 +0900 Subject: [PATCH 05/23] Allow an empty or null base URI --- app/concerns/liquid_interpolatable.rb | 7 ++++--- spec/concerns/liquid_interpolatable_spec.rb | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/concerns/liquid_interpolatable.rb b/app/concerns/liquid_interpolatable.rb index 83d93dec..4bc0d091 100644 --- a/app/concerns/liquid_interpolatable.rb +++ b/app/concerns/liquid_interpolatable.rb @@ -129,10 +129,11 @@ module LiquidInterpolatable # userinfo, host, port, registry, path, opaque, query, and # fragment. def to_uri(uri, base_uri = nil) - if base_uri - Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s) - else + case base_uri + when nil, '' Utils.normalize_uri(uri.to_s) + else + Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s) end rescue URI::Error nil diff --git a/spec/concerns/liquid_interpolatable_spec.rb b/spec/concerns/liquid_interpolatable_spec.rb index 6c7c9a71..b58b9996 100644 --- a/spec/concerns/liquid_interpolatable_spec.rb +++ b/spec/concerns/liquid_interpolatable_spec.rb @@ -119,6 +119,15 @@ describe LiquidInterpolatable::Filters do @agent.interpolation_context['s'] = 'foo/index.html' expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html') end + + it 'should normalize a URI value if an empty base URI is given' do + @agent.options['foo'] = '{{ u | to_uri: b }}' + @agent.interpolation_context['u'] = "\u{3042}" + @agent.interpolation_context['b'] = "" + expect(@agent.interpolated['foo']).to eq('%E3%81%82') + @agent.interpolation_context['b'] = nil + expect(@agent.interpolated['foo']).to eq('%E3%81%82') + end end describe 'uri_expand' do From 9074f3115ea572c2aa9be771997f48d4865daf22 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Wed, 23 Nov 2016 11:16:14 +0900 Subject: [PATCH 06/23] Make sure `status` is an integer when set --- app/models/agents/website_agent.rb | 4 ++-- spec/models/agents/website_agent_spec.rb | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 1918186e..e8b97686 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -134,7 +134,7 @@ module Agents * `_response_`: A response object with the following keys: - * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event. + * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer. * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event. @@ -694,7 +694,7 @@ module Agents # Integer value of HTTP status def status - @object.payload[:status] + Integer(@object.payload[:status]) rescue nil end # The URL diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index d943ca77..556f9c33 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -1198,7 +1198,7 @@ fire: hot 'value' => '{{ value }}', 'url' => '{{ url | to_uri: _response_.url }}', 'type' => '{{ _response_.headers.content_type }}', - 'status' => '{{ _response_.status }}' + 'status' => '{{ _response_.status | as_object }}' } ) end @@ -1207,7 +1207,7 @@ fire: hot expect { @checker.receive([@event]) }.to change { Event.count }.by(1) - expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' }) + expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200 }) end it "should support merge mode" do @@ -1216,7 +1216,7 @@ fire: hot expect { @checker.receive([@event]) }.to change { Event.count }.by(1) - expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200')) + expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200)) end it "should output an error when nothing can be found at the path" do From 5f5f3cd38f7b67bb3d33ac298cb3ebe4c17d33bf Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Wed, 23 Nov 2016 11:17:38 +0900 Subject: [PATCH 07/23] Do not err if `headers` is a valid headers hash --- app/models/agents/website_agent.rb | 11 ++++------- spec/models/agents/website_agent_spec.rb | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index e8b97686..37fd780a 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -136,7 +136,7 @@ module Agents * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer. - * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event. + * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event, if it is a hash. * `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`. @@ -684,12 +684,9 @@ module Agents class ResponseFromEventDrop < LiquidDroppable::Drop def headers - case headers = @object.payload[:headers] - when Hash - HeaderDrop.new(Faraday::Utils::Headers.from(headers)) - else - HeaderDrop.new({}) - end + headers = Faraday::Utils::Headers.from(@object.payload[:headers]) rescue {} + + HeaderDrop.new(headers) end # Integer value of HTTP status diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 556f9c33..afceb250 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -1219,6 +1219,24 @@ fire: hot expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200)) end + it "should convert headers and status in the event data properly" do + @event.payload[:status] = '201' + @event.payload[:headers] = [['Content-Type', 'application/rss+xml']] + expect { + @checker.receive([@event]) + }.to change { Event.count }.by(1) + expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/rss+xml', 'status' => 201 }) + end + + it "should ignore inconvertible headers and status in the event data" do + @event.payload[:status] = 'ok' + @event.payload[:headers] = ['Content-Type', 'Content-Length'] + expect { + @checker.receive([@event]) + }.to change { Event.count }.by(1) + expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => '', 'status' => nil }) + end + it "should output an error when nothing can be found at the path" do @checker.options = @checker.options.merge( 'data_from_event' => '{{ some_object.mistake }}' From f4308afdf70ae9d98f22c9853553ed1bf75fd8c5 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 27 Nov 2016 22:12:40 +0900 Subject: [PATCH 08/23] Use faraday_middleware 0.10.1, which was recently released --- Gemfile | 2 +- Gemfile.lock | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index 5a1005ac..c931b5db 100644 --- a/Gemfile +++ b/Gemfile @@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br gem 'devise','~> 4.2.0' gem 'em-http-request', '~> 1.1.2' gem 'faraday', '~> 0.9.0' -gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1' +gem 'faraday_middleware', '>= 0.10.1' gem 'feedjira', '~> 2.0' gem 'font-awesome-sass', '~> 4.3.2' gem 'foreman', '~> 0.63.0' diff --git a/Gemfile.lock b/Gemfile.lock index c18cf56a..99dc0984 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,14 +37,6 @@ GIT oauth2 (~> 1) rest-client (~> 1.8) -GIT - remote: git://github.com/lostisland/faraday_middleware.git - revision: c5836ae55857272732b33eb0e0a98d60e995a376 - branch: master - specs: - faraday_middleware (0.10.0) - faraday (>= 0.7.4, < 0.10) - GIT remote: git://github.com/tumblr/tumblr_client.git revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7 @@ -214,6 +206,8 @@ GEM extlib (0.9.16) faraday (0.9.2) multipart-post (>= 1.2, < 3) + faraday_middleware (0.10.1) + faraday (>= 0.7.4, < 1.0) feedjira (2.0.0) faraday (~> 0.9) faraday_middleware (~> 0.9) @@ -618,7 +612,7 @@ DEPENDENCIES em-http-request (~> 1.1.2) evernote_oauth faraday (~> 0.9.0) - faraday_middleware! + faraday_middleware (>= 0.10.1) feedjira (~> 2.0) ffi (>= 1.9.4) font-awesome-sass (~> 4.3.2) From 08b43cc8b9bacdb61ec22bdd9c8ffd985e80d2ca Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 27 Nov 2016 22:14:02 +0900 Subject: [PATCH 09/23] Update the RUBY VERSION with 2.3.3p222 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 99dc0984..aef7d8bc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -694,7 +694,7 @@ DEPENDENCIES xmpp4r (~> 0.5.6) RUBY VERSION - ruby 2.3.1p112 + ruby 2.3.3p222 BUNDLED WITH 1.13.6 From 8e36fdd60c54f32f9b440cfdcf6616050e830d60 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Sun, 27 Nov 2016 23:22:42 +0900 Subject: [PATCH 10/23] Revert "Use faraday_middleware 0.10.1, which was recently released" This reverts commit f4308afdf70ae9d98f22c9853553ed1bf75fd8c5. I realized 0.10.1 did not include my fix for raw deflate. --- Gemfile | 2 +- Gemfile.lock | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index c931b5db..5a1005ac 100644 --- a/Gemfile +++ b/Gemfile @@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br gem 'devise','~> 4.2.0' gem 'em-http-request', '~> 1.1.2' gem 'faraday', '~> 0.9.0' -gem 'faraday_middleware', '>= 0.10.1' +gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1' gem 'feedjira', '~> 2.0' gem 'font-awesome-sass', '~> 4.3.2' gem 'foreman', '~> 0.63.0' diff --git a/Gemfile.lock b/Gemfile.lock index aef7d8bc..90078534 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,6 +37,14 @@ GIT oauth2 (~> 1) rest-client (~> 1.8) +GIT + remote: git://github.com/lostisland/faraday_middleware.git + revision: c5836ae55857272732b33eb0e0a98d60e995a376 + branch: master + specs: + faraday_middleware (0.10.0) + faraday (>= 0.7.4, < 0.10) + GIT remote: git://github.com/tumblr/tumblr_client.git revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7 @@ -206,8 +214,6 @@ GEM extlib (0.9.16) faraday (0.9.2) multipart-post (>= 1.2, < 3) - faraday_middleware (0.10.1) - faraday (>= 0.7.4, < 1.0) feedjira (2.0.0) faraday (~> 0.9) faraday_middleware (~> 0.9) @@ -612,7 +618,7 @@ DEPENDENCIES em-http-request (~> 1.1.2) evernote_oauth faraday (~> 0.9.0) - faraday_middleware (>= 0.10.1) + faraday_middleware! feedjira (~> 2.0) ffi (>= 1.9.4) font-awesome-sass (~> 4.3.2) From 8a14a57e00716c12d060a036240cbfda789ab010 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 27 Nov 2016 15:30:50 -0500 Subject: [PATCH 11/23] Beeper.io is no more (#1808) * Beeper.io is no more * Avoid event propagation or scheduling of missing agents * Update undefined_agents.html.erb with a link to the wiki --- app/models/agent.rb | 8 +- app/models/agents/beeper_agent.rb | 129 ---------------- .../application/undefined_agents.html.erb | 8 +- spec/models/agent_spec.rb | 33 +++- spec/models/agents/beeper_agent_spec.rb | 145 ------------------ 5 files changed, 41 insertions(+), 282 deletions(-) delete mode 100644 app/models/agents/beeper_agent.rb delete mode 100644 spec/models/agents/beeper_agent_spec.rb diff --git a/app/models/agent.rb b/app/models/agent.rb index b8d32ee9..66672d28 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -370,7 +370,7 @@ class Agent < ActiveRecord::Base def receive!(options={}) Agent.transaction do scope = Agent. - select("agents.id AS receiver_agent_id, events.id AS event_id"). + select("agents.id AS receiver_agent_id, sources.type AS source_agent_type, agents.type AS receiver_agent_type, events.id AS event_id"). joins("JOIN links ON (links.receiver_id = agents.id)"). joins("JOIN agents AS sources ON (links.source_id = sources.id)"). joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)"). @@ -379,10 +379,11 @@ class Agent < ActiveRecord::Base scope = scope.where("agents.id in (?)", options[:only_receivers]) end - sql = scope.to_sql() + sql = scope.to_sql agents_to_events = {} - Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id| + Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_type, receiver_agent_type, event_id| + next unless const_defined?(source_agent_type) && const_defined?(receiver_agent_type) agents_to_events[receiver_agent_id.to_i] ||= [] agents_to_events[receiver_agent_id.to_i] << event_id end @@ -417,6 +418,7 @@ class Agent < ActiveRecord::Base return if schedule == 'never' types = where(:schedule => schedule).group(:type).pluck(:type) types.each do |type| + next unless const_defined?(type) type.constantize.bulk_check(schedule) end end diff --git a/app/models/agents/beeper_agent.rb b/app/models/agents/beeper_agent.rb deleted file mode 100644 index 4cd5a41e..00000000 --- a/app/models/agents/beeper_agent.rb +++ /dev/null @@ -1,129 +0,0 @@ -module Agents - class BeeperAgent < Agent - cannot_be_scheduled! - cannot_create_events! - no_bulk_receive! - - description <<-MD - Beeper agent sends messages to Beeper app on your mobile device via Push notifications. - - You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io) - - You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID - - Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`. - - Depending on message type you have to provide additional fields: - - ##### Message - * `text` – **required** - - ##### Image - * `image` – **required** (Image URL or Base64-encoded image) - * `text` – optional - - ##### Event - * `text` – **required** - * `start_time` – **required** (Corresponding to ISO 8601) - * `end_time` – optional (Corresponding to ISO 8601) - - ##### Location - * `latitude` – **required** - * `longitude` – **required** - * `text` – optional - - ##### Task - * `text` – **required** - - You can see additional documentation at [Beeper website](https://beeper.io/docs) - MD - - BASE_URL = 'https://api.beeper.io/api' - - TYPE_ATTRIBUTES = { - 'message' => %w(text), - 'image' => %w(text image), - 'event' => %w(text start_time end_time), - 'location' => %w(text latitude longitude), - 'task' => %w(text) - } - - MESSAGE_TYPES = TYPE_ATTRIBUTES.keys - - TYPE_REQUIRED_ATTRIBUTES = { - 'message' => %w(text), - 'image' => %w(image), - 'event' => %w(text start_time), - 'location' => %w(latitude longitude), - 'task' => %w(text) - } - - def default_options - { - 'type' => 'message', - 'app_id' => '', - 'api_key' => '', - 'sender_id' => '', - 'phone' => '', - 'text' => '{{title}}' - } - end - - def validate_options - %w(app_id api_key sender_id type).each do |attr| - errors.add(:base, "you need to specify a #{attr}") if options[attr].blank? - end - - if options['type'].in?(MESSAGE_TYPES) - required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']] - if required_attributes.any? { |attr| options[attr].blank? } - errors.add(:base, "you need to specify a #{required_attributes.join(', ')}") - end - else - errors.add(:base, 'you need to specify a valid message type') - end - - unless options['group_id'].blank? ^ options['phone'].blank? - errors.add(:base, 'you need to specify a phone or group_id') - end - end - - def working? - received_event_without_error? && !recent_error_logs? - end - - def receive(incoming_events) - incoming_events.each do |event| - send_message(event) - end - end - - def send_message(event) - mo = interpolated(event) - begin - response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers) - error(response.body) if response.code != 201 - rescue HTTParty::Error => e - error(e.message) - end - end - - private - - def headers - { - 'X-Beeper-Application-Id' => options['app_id'], - 'X-Beeper-REST-API-Key' => options['api_key'], - 'Content-Type' => 'application/json' - } - end - - def payload_for(mo) - mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json - end - - def endpoint_for(type) - "#{BASE_URL}/#{type}s.json" - end - end -end \ No newline at end of file diff --git a/app/views/application/undefined_agents.html.erb b/app/views/application/undefined_agents.html.erb index 3912af7d..229f3e19 100644 --- a/app/views/application/undefined_agents.html.erb +++ b/app/views/application/undefined_agents.html.erb @@ -19,20 +19,20 @@

- The issue most probably occurred because of one or more of the following reasons: + This issue probably occurred for one or more of the following reasons:

    +
  • If the respective Agent is distributed as part of the Huginn application codebase, it may have been removed or moved to an Agent gem. Please see this wiki page for more information.
  • If the respective Agent is distributed as a Ruby gem, it might have been removed from the ADDITIONAL_GEMS environment setting.
  • -
  • If the respective Agent is distributed as part of the Huginn application codebase, it might have been removed from that either on purpose (because the Agent has been deprecated or been moved to an Agent gem) or accidentally. Please check if the Agent(s) in question are available in your Huginn codebase under the path app/models/agents/.

- You can fix the issue by adding the Agent(s) back to the application codebase by + You can fix this issue by:

    +
  • deleting the respective Agent(s) from the database using the button below.
  • adding the respective Agent(s) to the the ADDITIONAL_GEMS environment setting. Please see https://github.com/cantino/huginn_agent for documentation on how to properly set it.
  • adding the respective Agent(s) code to the Huginn application codebase (in case it was deleted accidentally).
  • -
  • deleting the respective Agent(s) from the database using the button below.

diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb index fa98b45b..7a67a587 100644 --- a/spec/models/agent_spec.rb +++ b/spec/models/agent_spec.rb @@ -74,6 +74,13 @@ describe Agent do Agent.run_schedule("midnight") end + it "ignores unknown types" do + Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent' + mock(Agents::WeatherAgent).bulk_check("midnight").once + mock(Agents::WebsiteAgent).bulk_check("midnight").once + Agent.run_schedule("midnight") + end + it "only runs agents with the given schedule" do do_not_allow(Agents::WebsiteAgent).async_check Agent.run_schedule("blah") @@ -283,13 +290,37 @@ describe Agent do Agent.receive! end - it "should not propogate to disabled Agents" do + it "should not propagate to disabled Agents" do Agent.async_check(agents(:bob_weather_agent).id) agents(:bob_rain_notifier_agent).update_attribute :disabled, true mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0) Agent.receive! end + it "should not propagate to Agents with unknown types" do + Agent.async_check(agents(:jane_weather_agent).id) + Agent.async_check(agents(:bob_weather_agent).id) + + Agent.where(id: agents(:bob_rain_notifier_agent).id).update_all type: 'UnknownTypeAgent' + + mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0) + mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1) + + Agent.receive! + end + + it "should not propagate from Agents with unknown types" do + Agent.async_check(agents(:jane_weather_agent).id) + Agent.async_check(agents(:bob_weather_agent).id) + + Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent' + + mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0) + mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1) + + Agent.receive! + end + it "should log exceptions" do mock.any_instance_of(Agents::TriggerAgent).receive(anything).once { raise "foo" diff --git a/spec/models/agents/beeper_agent_spec.rb b/spec/models/agents/beeper_agent_spec.rb deleted file mode 100644 index e2c2aba5..00000000 --- a/spec/models/agents/beeper_agent_spec.rb +++ /dev/null @@ -1,145 +0,0 @@ -require 'rails_helper' - - -describe Agents::BeeperAgent do - let(:base_params) { - { - 'type' => 'message', - 'app_id' => 'some-app-id', - 'api_key' => 'some-api-key', - 'sender_id' => 'sender-id', - 'phone' => '+111111111111', - 'text' => 'Some text' - } - } - - subject { - agent = described_class.new(name: 'beeper-agent', options: base_params) - agent.user = users(:jane) - agent.save! and return agent - } - - context 'validation' do - it 'valid' do - expect(subject).to be_valid - end - - [:type, :app_id, :api_key, :sender_id].each do |attr| - it "invalid without #{attr}" do - subject.options[attr] = nil - expect(subject).not_to be_valid - end - end - - it 'invalid with group_id and phone' do - subject.options['group_id'] ='some-group-id' - expect(subject).not_to be_valid - end - - context '#message' do - it 'requires text' do - subject.options[:text] = nil - expect(subject).not_to be_valid - end - end - - context '#image' do - before(:each) do - subject.options[:type] = 'image' - end - - it 'invalid without image' do - expect(subject).not_to be_valid - end - - it 'valid with image' do - subject.options[:image] = 'some-url' - expect(subject).to be_valid - end - end - - context '#event' do - before(:each) do - subject.options[:type] = 'event' - end - - it 'invalid without start_time' do - expect(subject).not_to be_valid - end - - it 'valid with start_time' do - subject.options[:start_time] = Time.now - expect(subject).to be_valid - end - end - - context '#location' do - before(:each) do - subject.options[:type] = 'location' - end - - it 'invalid without latitude and longitude' do - expect(subject).not_to be_valid - end - - it 'valid with latitude and longitude' do - subject.options[:latitude] = 15.0 - subject.options[:longitude] = 16.0 - expect(subject).to be_valid - end - end - - context '#task' do - before(:each) do - subject.options[:type] = 'task' - end - - it 'valid with text' do - expect(subject).to be_valid - end - end - end - - context 'payload_for' do - it 'removes unwanted attributes' do - result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text', - 'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'}) - expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}') - end - end - - context 'headers' do - it 'sets X-Beeper-Application-Id header with app_id' do - expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id']) - end - - it 'sets X-Beeper-REST-API-Key header with api_key' do - expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key']) - end - - it 'sets Content-Type' do - expect(subject.send(:headers)['Content-Type']).to eq('application/json') - end - end - - context 'endpoint_for' do - it 'returns valid URL for message' do - expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json') - end - - it 'returns valid URL for image' do - expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json') - end - - it 'returns valid URL for event' do - expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json') - end - - it 'returns valid URL for location' do - expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json') - end - it 'returns valid URL for task' do - expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json') - end - end -end From 3612ba7333ba4b07bba59acbac5181eae997a81e Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 7 Nov 2016 22:34:06 +0900 Subject: [PATCH 12/23] Add podcast tags to events emitted by RssAgent The keys are only added when the feed is a podcast. --- Gemfile | 2 +- Gemfile.lock | 17 +-- app/models/agents/rss_agent.rb | 83 ++++++++++++++- lib/feedjira_extension.rb | 17 +++ spec/data_fixtures/podcast.rss | 76 ++++++++++++++ spec/models/agents/rss_agent_spec.rb | 152 +++++++++++++++++++++++++++ 6 files changed, 339 insertions(+), 8 deletions(-) create mode 100644 spec/data_fixtures/podcast.rss diff --git a/Gemfile b/Gemfile index 5a1005ac..ffddd76a 100644 --- a/Gemfile +++ b/Gemfile @@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0' gem 'em-http-request', '~> 1.1.2' gem 'faraday', '~> 0.9.0' gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1' -gem 'feedjira', '~> 2.0' +gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1'] gem 'font-awesome-sass', '~> 4.3.2' gem 'foreman', '~> 0.63.0' gem 'geokit', '~> 1.8.4' diff --git a/Gemfile.lock b/Gemfile.lock index 90078534..2b871768 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,6 +37,16 @@ GIT oauth2 (~> 1) rest-client (~> 1.8) +GIT + remote: git://github.com/feedjira/feedjira.git + revision: 849b7809455c084b06f9c4c2d368a752677882b0 + specs: + feedjira (2.0.0) + faraday (>= 0.9) + faraday_middleware (>= 0.9) + loofah (>= 2.0) + sax-machine (>= 1.0) + GIT remote: git://github.com/lostisland/faraday_middleware.git revision: c5836ae55857272732b33eb0e0a98d60e995a376 @@ -214,11 +224,6 @@ GEM extlib (0.9.16) faraday (0.9.2) multipart-post (>= 1.2, < 3) - feedjira (2.0.0) - faraday (~> 0.9) - faraday_middleware (~> 0.9) - loofah (~> 2.0) - sax-machine (~> 1.0) ffi (1.9.10) font-awesome-sass (4.3.2.1) sass (~> 3.2) @@ -619,7 +624,7 @@ DEPENDENCIES evernote_oauth faraday (~> 0.9.0) faraday_middleware! - feedjira (~> 2.0) + feedjira! ffi (>= 1.9.4) font-awesome-sass (~> 4.3.2) forecast_io (~> 2.0.0) diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb index 0ed34c42..0ab5719c 100644 --- a/app/models/agents/rss_agent.rb +++ b/app/models/agents/rss_agent.rb @@ -66,6 +66,22 @@ module Agents "copyright": "...", "icon": "http://example.com/icon.png", "authors": [ "..." ], + + "itunes_block": "no", + "itunes_categories": [ + "Technology", "Gadgets", + "TV & Film", + "Arts", "Food" + ], + "itunes_complete": "yes", + "itunes_explicit": "yes", + "itunes_image": "http://...", + "itunes_new_feed_url": "http://...", + "itunes_owners": [ "John Doe " ], + "itunes_subtitle": "...", + "itunes_summary": "...", + "language": "en-US", + "date_published": "2014-09-11T01:30:00-07:00", "last_updated": "2014-09-11T01:30:00-07:00" }, @@ -84,6 +100,16 @@ module Agents "enclosure": { "url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789" }, + + "itunes_block": "no", + "itunes_closed_captioned": "yes", + "itunes_duration": "04:34", + "itunes_explicit": "yes", + "itunes_image": "http://...", + "itunes_order": "1", + "itunes_subtitle": "...", + "itunes_summary": "...", + "date_published": "2014-09-11T01:30:00-0700", "last_updated": "2014-09-11T01:30:00-0700" } @@ -91,7 +117,8 @@ module Agents Some notes: - The `feed` key is present only if `include_feed_info` is set to true. - - Each element in `authors` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional. + - The keys starting with `itunes_`, and `language` are only present when the feed is a podcast. See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details. + - Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional. - Timestamps are converted to the ISO 8601 format. MD @@ -206,9 +233,40 @@ module Agents authors: feed.authors, date_published: feed.date_published, last_updated: feed.last_updated, + **itunes_feed_data(feed) } end + def itunes_feed_data(feed) + data = {} + case feed + when Feedjira::Parser::ITunesRSS + %i[ + itunes_block + itunes_categories + itunes_complete + itunes_explicit + itunes_image + itunes_new_feed_url + itunes_owners + itunes_subtitle + itunes_summary + language + ].each { |attr| + if value = feed.try(attr).presence + data[attr] = + case attr + when :itunes_summary + clean_fragment(value) + else + value + end + end + } + end + data + end + def entry_data(entry) { id: entry.id, @@ -224,9 +282,32 @@ module Agents categories: Array(entry.try(:categories)), date_published: entry.date_published, last_updated: entry.last_updated, + **itunes_entry_data(entry) } end + def itunes_entry_data(entry) + data = {} + case entry + when Feedjira::Parser::ITunesRSSItem + %i[ + itunes_block + itunes_closed_captioned + itunes_duration + itunes_explicit + itunes_image + itunes_order + itunes_subtitle + itunes_summary + ].each { |attr| + if value = entry.try(attr).presence + data[attr] = value + end + } + end + data + end + def feed_to_events(feed) payload_base = {} diff --git a/lib/feedjira_extension.rb b/lib/feedjira_extension.rb index a9a9ac11..5eab2981 100644 --- a/lib/feedjira_extension.rb +++ b/lib/feedjira_extension.rb @@ -57,6 +57,13 @@ module FeedjiraExtension value :content end + class ITunesRssOwner < Author + include SAXMachine + + element :'itunes:name', as: :name + element :'itunes:email', as: :email + end + class Enclosure include SAXMachine @@ -290,6 +297,16 @@ module FeedjiraExtension def copyright @copyright || super end + + if /ITunes/ === name + sax_config.collection_elements['itunes:owner'].clear + elements :"itunes:owner", as: :_itunes_owners, class: ITunesRssOwner + private :_itunes_owners + + def itunes_owners + _itunes_owners.reject(&:empty?) + end + end end sax_config.collection_elements.each_value do |collection_elements| diff --git a/spec/data_fixtures/podcast.rss b/spec/data_fixtures/podcast.rss new file mode 100644 index 00000000..bb64933e --- /dev/null +++ b/spec/data_fixtures/podcast.rss @@ -0,0 +1,76 @@ + + + + All About Everything + http://www.example.com/podcasts/everything/index.html + en-us + ℗ & © 2014 John Doe & Family + A show about everything + John Doe + All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store + All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store + + John Doe + john.doe@example.com + + yes + + + + + + + + + no + + Shake Shake Shake Your Spices + John Doe + A short primer on table spices + salt and pepper shakers, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!]]> + + + http://example.com/podcasts/archive/aae20140615.m4a + Tue, 08 Mar 2016 12:00:00 GMT + 07:04 + no + + + Socket Wrench Shootout + Jane Doe + Comparing socket wrenches is fun! + This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here. + + + http://example.com/podcasts/archive/aae20140608.mp4 + Wed, 09 Mar 2016 13:00:00 EST + 04:34 + no + + + The Best Chili + Jane Doe + Jane and Eric + This week we talk about the best Chili in the world. Which chili is better? + + + http://example.com/podcasts/archive/aae20140697.m4v + Thu, 10 Mar 2016 02:00:00 -0700 + 04:34 + no + Yes + + + Red,Whine, & Blue + Various + Red + Blue != Purple + This week we talk about surviving in a Red state if you are a Blue person. Or vice versa. + + + http://example.com/podcasts/archive/aae20140601.mp3 + Fri, 11 Mar 2016 01:15:00 +3000 + 03:59 + no + + + diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb index 31c6737e..ac651166 100644 --- a/spec/models/agents/rss_agent_spec.rb +++ b/spec/models/agents/rss_agent_spec.rb @@ -12,7 +12,11 @@ describe Agents::RssAgent do stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200) stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200) stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=)[^<]*/, ''), status: 200) +<<<<<<< HEAD stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200) +======= + stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200) +>>>>>>> Add podcast tags to events emitted by RssAgent end let(:agent) do @@ -303,6 +307,154 @@ describe Agents::RssAgent do expect(event.payload['title']).to eq('Mëkanïk Zaïn') end end + + context 'with podcast elements' do + before do + @valid_options['url'] = 'http://example.com/podcast.rss' + @valid_options['include_feed_info'] = true + end + + let :feed_info do + { + "id" => nil, + "type" => "rss", + "url" => "http://www.example.com/podcasts/everything/index.html", + "links" => [ { "href" => "http://www.example.com/podcasts/everything/index.html" } ], + "title" => "All About Everything", + "description" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store", + "copyright" => "℗ & © 2014 John Doe & Family", + "generator" => nil, + "icon" => nil, + "authors" => [ + "John Doe" + ], + "date_published" => nil, + "last_updated" => nil, + "itunes_categories" => [ + "Technology", "Gadgets", + "TV & Film", + "Arts", "Food" + ], + "itunes_complete" => "yes", + "itunes_explicit" => "no", + "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything.jpg", + "itunes_owners" => ["John Doe "], + "itunes_subtitle" => "A show about everything", + "itunes_summary" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store", + "language" => "en-us" + } + end + + it "is parsed correctly" do + expect { + agent.check + }.to change { agent.events.count }.by(4) + + expect(agent.events.map(&:payload)).to match([ + { + "feed" => feed_info, + "id" => "http://example.com/podcasts/archive/aae20140601.mp3", + "url" => nil, + "urls" => [], + "links" => [], + "title" => "Red,Whine, & Blue", + "description" => nil, + "content" => nil, + "image" => nil, + "enclosure" => { + "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3", + "type" => "audio/mpeg", + "length" => "498537" + }, + "authors" => [""], + "categories" => [], + "date_published" => "2016-03-11T01:15:00+00:00", + "last_updated" => "2016-03-11T01:15:00+00:00", + "itunes_duration" => "03:59", + "itunes_explicit" => "no", + "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg", + "itunes_subtitle" => "Red + Blue != Purple", + "itunes_summary" => "This week we talk about surviving in a Red state if you are a Blue person. Or vice versa." + }, + { + "feed" => feed_info, + "id" => "http://example.com/podcasts/archive/aae20140697.m4v", + "url" => nil, + "urls" => [], + "links" => [], + "title" => "The Best Chili", + "description" => nil, + "content" => nil, + "image" => nil, + "enclosure" => { + "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v", + "type" => "video/x-m4v", + "length" => "5650889" + }, + "authors" => ["Jane Doe"], + "categories" => [], + "date_published" => "2016-03-10T02:00:00-07:00", + "last_updated" => "2016-03-10T02:00:00-07:00", + "itunes_closed_captioned" => "Yes", + "itunes_duration" => "04:34", + "itunes_explicit" => "no", + "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg", + "itunes_subtitle" => "Jane and Eric", + "itunes_summary" => "This week we talk about the best Chili in the world. Which chili is better?" + }, + { + "feed" => feed_info, + "id" => "http://example.com/podcasts/archive/aae20140608.mp4", + "url" => nil, + "urls" => [], + "links" => [], + "title" => "Socket Wrench Shootout", + "description" => nil, + "content" => nil, + "image" => nil, + "enclosure" => { + "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4", + "type" => "video/mp4", + "length" => "5650889" + }, + "authors" => ["Jane Doe"], + "categories" => [], + "date_published" => "2016-03-09T13:00:00-05:00", + "last_updated" => "2016-03-09T13:00:00-05:00", + "itunes_duration" => "04:34", + "itunes_explicit" => "no", + "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg", + "itunes_subtitle" => "Comparing socket wrenches is fun!", + "itunes_summary" => "This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here." + }, + { + "feed" => feed_info, + "id" => "http://example.com/podcasts/archive/aae20140615.m4a", + "url" => nil, + "urls" => [], + "links" => [], + "title" => "Shake Shake Shake Your Spices", + "description" => nil, + "content" => nil, + "image" => nil, + "enclosure" => { + "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a", + "type" => "audio/x-m4a", + "length" => "8727310" + }, + "authors" => ["John Doe"], + "categories" => [], + "date_published" => "2016-03-08T12:00:00+00:00", + "last_updated" => "2016-03-08T12:00:00+00:00", + "itunes_duration" => "07:04", + "itunes_explicit" => "no", + "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg", + "itunes_subtitle" => "A short primer on table spices", + "itunes_summary" => "This week we talk about salt and pepper shakers, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!" + } + ]) + end + end end describe 'logging errors with the feed url' do From e26c07e75bd996ab27673aac67914c423204e7cf Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Wed, 30 Nov 2016 01:24:36 +0900 Subject: [PATCH 13/23] Fix a merge conflict --- spec/models/agents/rss_agent_spec.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb index ac651166..399cf819 100644 --- a/spec/models/agents/rss_agent_spec.rb +++ b/spec/models/agents/rss_agent_spec.rb @@ -12,11 +12,8 @@ describe Agents::RssAgent do stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200) stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200) stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=)[^<]*/, ''), status: 200) -<<<<<<< HEAD stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200) -======= stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200) ->>>>>>> Add podcast tags to events emitted by RssAgent end let(:agent) do From 15347c8a3ce7a6e33db98835619b161172960cc7 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 13 Dec 2016 18:08:16 +0900 Subject: [PATCH 14/23] Feedjira 2.1.0 is out, yay! --- Gemfile | 2 +- Gemfile.lock | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Gemfile b/Gemfile index ffddd76a..a83fc492 100644 --- a/Gemfile +++ b/Gemfile @@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0' gem 'em-http-request', '~> 1.1.2' gem 'faraday', '~> 0.9.0' gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1' -gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1'] +gem 'feedjira', '~> 2.1' gem 'font-awesome-sass', '~> 4.3.2' gem 'foreman', '~> 0.63.0' gem 'geokit', '~> 1.8.4' diff --git a/Gemfile.lock b/Gemfile.lock index 2b871768..ba055445 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,16 +37,6 @@ GIT oauth2 (~> 1) rest-client (~> 1.8) -GIT - remote: git://github.com/feedjira/feedjira.git - revision: 849b7809455c084b06f9c4c2d368a752677882b0 - specs: - feedjira (2.0.0) - faraday (>= 0.9) - faraday_middleware (>= 0.9) - loofah (>= 2.0) - sax-machine (>= 1.0) - GIT remote: git://github.com/lostisland/faraday_middleware.git revision: c5836ae55857272732b33eb0e0a98d60e995a376 @@ -224,6 +214,11 @@ GEM extlib (0.9.16) faraday (0.9.2) multipart-post (>= 1.2, < 3) + feedjira (2.1.0) + faraday (>= 0.9) + faraday_middleware (>= 0.9) + loofah (>= 2.0) + sax-machine (>= 1.0) ffi (1.9.10) font-awesome-sass (4.3.2.1) sass (~> 3.2) @@ -624,7 +619,7 @@ DEPENDENCIES evernote_oauth faraday (~> 0.9.0) faraday_middleware! - feedjira! + feedjira (~> 2.1) ffi (>= 1.9.4) font-awesome-sass (~> 4.3.2) forecast_io (~> 2.0.0) From fd18e1453883768e2bf848a055528fcf82740243 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 13 Dec 2016 19:31:32 +0900 Subject: [PATCH 15/23] Fast-forward faraday_middleware The version constraint on faraday is relaxed while at it, although some gems still have the strict constraint. --- Gemfile | 2 +- Gemfile.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index a83fc492..f6fec106 100644 --- a/Gemfile +++ b/Gemfile @@ -90,7 +90,7 @@ gem 'delayed_job', '~> 4.1.0' gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails5' gem 'devise','~> 4.2.0' gem 'em-http-request', '~> 1.1.2' -gem 'faraday', '~> 0.9.0' +gem 'faraday', '~> 0.9' gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1' gem 'feedjira', '~> 2.1' gem 'font-awesome-sass', '~> 4.3.2' diff --git a/Gemfile.lock b/Gemfile.lock index ba055445..ee694741 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -39,11 +39,11 @@ GIT GIT remote: git://github.com/lostisland/faraday_middleware.git - revision: c5836ae55857272732b33eb0e0a98d60e995a376 + revision: 59088da02940d0ee2010b2e3156343346767c31e branch: master specs: faraday_middleware (0.10.0) - faraday (>= 0.7.4, < 0.10) + faraday (>= 0.7.4, < 1.0) GIT remote: git://github.com/tumblr/tumblr_client.git @@ -617,7 +617,7 @@ DEPENDENCIES dropbox-api em-http-request (~> 1.1.2) evernote_oauth - faraday (~> 0.9.0) + faraday (~> 0.9) faraday_middleware! feedjira (~> 2.1) ffi (>= 1.9.4) From 3159b547c7854365455b194233baf562f9d2f8e2 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 19 Dec 2016 14:51:32 +0900 Subject: [PATCH 16/23] Use the builtin mysql on Travis CI --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d4abefa..eeb4f722 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ sudo: required language: ruby services: - docker + - mysql - postgresql env: global: @@ -37,9 +38,6 @@ rvm: - 2.3.1 cache: bundler bundler_args: --without development production -before_install: - - sudo apt-get -qq update - - sudo apt-get install -y mysql-server script: - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi From 2a524abff57d3d17f8e69d31b66e7f3997560470 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 20 Dec 2016 10:26:39 +0900 Subject: [PATCH 17/23] Fix the regex pattern to reduce backtracking This should fix #1832. --- app/models/agents/rss_agent.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb index 0ab5719c..8e4a3a83 100644 --- a/app/models/agents/rss_agent.rb +++ b/app/models/agents/rss_agent.rb @@ -206,7 +206,7 @@ module Agents else # Encoding is already known, so do not let the parser detect # it from the XML declaration in the content. - body.sub!(/(\A\u{FEFF}?\s*<\?xml(?:\s+\w+\s*=\s*(['"]).*?\2)*)\s+encoding\s*=\s*(['"]).*?\3/, '\\1') + body.sub!(/(?\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g/, '\\k') end body end From 4e2d1775a6f1fc9ae8f6c7a6fe212813b8d7b5df Mon Sep 17 00:00:00 2001 From: Irfan Charania Date: Tue, 20 Dec 2016 09:44:54 -0800 Subject: [PATCH 18/23] PhantomJs Cloud Agent (#1503) * Initial draft of PhantomJsCloudAgent Generates event with url for fetching html/plainText content * Add options * Pass in event instead of url Fix hash syntax Remove whitespace Add mode merge * Add some tests * Style changes - Add link to wiki entry for manually creating agent with full set of options --- app/models/agents/phantom_js_cloud_agent.rb | 162 ++++++++++++++++++ .../agents/phantom_js_cloud_agent_spec.rb | 117 +++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 app/models/agents/phantom_js_cloud_agent.rb create mode 100644 spec/models/agents/phantom_js_cloud_agent_spec.rb diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb new file mode 100644 index 00000000..f0c9e605 --- /dev/null +++ b/app/models/agents/phantom_js_cloud_agent.rb @@ -0,0 +1,162 @@ +require 'json' +require 'uri' + +module Agents + class PhantomJsCloudAgent < Agent + include ERB::Util + include FormConfigurable + include WebRequestConcern + + can_dry_run! + + default_schedule 'every_12h' + + description <<-MD + [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages. + + The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html). + This url can then be supplied to Website Agent to fetch and parse content. + + [Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials. + + + Options: + + * `Api key` - PhantomJs Cloud API Key credential stored in Huginn + * `Url` - The url to render + * `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`) + * `Render type` - Render as html or plain text without html tags (default: `html`) + * `Output as json` - Return the page conents and metadata as a JSON object (default: `false`) + * `Ignore images` - Skip loading of inlined images (default: `false`) + * `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`) + * `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading. + This is useful in case there are any AJAX requests or animations that need to finish up. + This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms) + + + As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides. + + MD + + event_description <<-MD + Events look like this: + { + "url": "..." + } + MD + + def default_options + { + 'mode' => 'clean', + 'url' => 'http://xkcd.com', + 'render_type' => 'html', + 'output_as_json' => false, + 'ignore_images' => false, + 'user_agent' => self.class.default_user_agent, + 'wait_interval' => '1000' + } + end + + form_configurable :mode, type: :array, values: ['clean', 'merge'] + form_configurable :api_key, roles: :completable + form_configurable :url + form_configurable :render_type, type: :array, values: ['html', 'plainText'] + form_configurable :output_as_json, type: :boolean + form_configurable :ignore_images, type: :boolean + form_configurable :user_agent, type: :text + form_configurable :wait_interval + + def mode + interpolated['mode'].presence || default_options['mode'] + end + + def render_type + interpolated['render_type'].presence || default_options['render_type'] + end + + def output_as_json + boolify(interpolated['output_as_json'].presence || + default_options['output_as_json']) + end + + def ignore_images + boolify(interpolated['ignore_images'].presence || + default_options['ignore_images']) + end + + def user_agent + interpolated['user_agent'].presence || self.class.default_user_agent + end + + def wait_interval + interpolated['wait_interval'].presence || default_options['wait_interval'] + end + + def page_request_settings + prs = {} + + prs[:ignoreImages] = ignore_images if ignore_images + prs[:userAgent] = user_agent if user_agent.present? + + if wait_interval != default_options['wait_interval'] + prs[:wait_interval] = wait_interval + end + + prs + end + + def build_phantom_url(interpolated) + api_key = interpolated[:api_key] + page_request_hash = { + url: interpolated[:url], + renderType: render_type + } + + page_request_hash[:outputAsJson] = output_as_json if output_as_json + + page_request_settings_hash = page_request_settings + + if page_request_settings_hash.any? + page_request_hash[:requestSettings] = page_request_settings_hash + end + + request = page_request_hash.to_json + log "Generated request: #{request}" + + encoded = url_encode(request) + "https://phantomjscloud.com/api/browser/v2/#{api_key}/?request=#{encoded}" + end + + def check + phantom_url = build_phantom_url(interpolated) + + create_event payload: { 'url' => phantom_url } + end + + def receive(incoming_events) + incoming_events.each do |event| + interpolate_with(event) do + existing_payload = interpolated['mode'].to_s == 'merge' ? event.payload : {} + phantom_url = build_phantom_url(interpolated) + + result = { 'url' => phantom_url } + create_event payload: existing_payload.merge(result) + end + end + end + + def complete_api_key + user.user_credentials.map { |c| { text: c.credential_name, id: "{% credential #{c.credential_name} %}" } } + end + + def working? + !recent_error_logs? || received_event_without_error? + end + + def validate_options + # Check for required fields + errors.add(:base, 'Url is required') unless options['url'].present? + errors.add(:base, 'API key (credential) is required') unless options['api_key'].present? + end + end +end diff --git a/spec/models/agents/phantom_js_cloud_agent_spec.rb b/spec/models/agents/phantom_js_cloud_agent_spec.rb new file mode 100644 index 00000000..07a79c2b --- /dev/null +++ b/spec/models/agents/phantom_js_cloud_agent_spec.rb @@ -0,0 +1,117 @@ +require 'rails_helper' + +describe Agents::PhantomJsCloudAgent do + before do + + @valid_options = { + 'name' => "XKCD", + 'render_type' => "html", + 'url' => "http://xkcd.com", + 'mode' => 'clean', + 'api_key' => '1234567890' + } + + @checker = Agents::PhantomJsCloudAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days) + @checker.user = users(:jane) + @checker.save! + end + + describe "validations" do + before do + expect(@checker).to be_valid + end + + it "should validate the presence of url" do + @checker.options['url'] = "http://google.com" + expect(@checker).to be_valid + + @checker.options['url'] = "" + expect(@checker).not_to be_valid + + @checker.options['url'] = nil + expect(@checker).not_to be_valid + end + + end + + describe "emitting event" do + it "should emit url as event" do + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D") + end + + it "should set render type as plain text" do + @checker.options['render_type'] = 'plainText' + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22plainText%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D") + end + + it "should set output as json" do + @checker.options['output_as_json'] = true + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22outputAsJson%22%3Atrue%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D") + end + + it "should not set ignore images" do + @checker.options['ignore_images'] = false + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D") + end + + it "should set ignore images" do + @checker.options['ignore_images'] = true + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22ignoreImages%22%3Atrue%2C%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D") + end + + it "should set wait interval to zero" do + @checker.options['wait_interval'] = '0' + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%2C%22wait_interval%22%3A%220%22%7D%7D") + end + + it "should set user agent to BlackBerry" do + @checker.options['user_agent'] = 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+' + + expect { + @checker.check + }.to change { @checker.events.count }.by(1) + + item,* = @checker.events.last(1) + expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Mozilla%2F5.0%20%28BlackBerry%3B%20U%3B%20BlackBerry%209900%3B%20en%29%20AppleWebKit%2F534.11%2B%20%28KHTML%2C%20like%20Gecko%29%20Version%2F7.1.0.346%20Mobile%20Safari%2F534.11%2B%22%7D%7D") + end + + + + end + +end From 65cea03062dfc4aa7d0ebdacf2edde5a78e75f82 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Tue, 20 Dec 2016 13:32:48 -0500 Subject: [PATCH 19/23] Fix spec failures --- app/models/agents/phantom_js_cloud_agent.rb | 9 ++++----- spec/features/create_an_agent_spec.rb | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb index f0c9e605..727de10e 100644 --- a/app/models/agents/phantom_js_cloud_agent.rb +++ b/app/models/agents/phantom_js_cloud_agent.rb @@ -12,13 +12,14 @@ module Agents default_schedule 'every_12h' description <<-MD - [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages. + This Agent generates [PhantomJs Cloud](https://phantomjscloud.com/) URLs that can be used to render JavaScript-heavy webpages for content extraction. - The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html). - This url can then be supplied to Website Agent to fetch and parse content. + URLs generated by this Agent are formulated in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html). + The generated URLs can then be supplied to a Website Agent to fetch and parse the content. [Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials. + Please see the [Huginn Wiki for more info](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud). Options: @@ -33,9 +34,7 @@ module Agents This is useful in case there are any AJAX requests or animations that need to finish up. This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms) - As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides. - MD event_description <<-MD diff --git a/spec/features/create_an_agent_spec.rb b/spec/features/create_an_agent_spec.rb index ac2ce295..ae725630 100644 --- a/spec/features/create_an_agent_spec.rb +++ b/spec/features/create_an_agent_spec.rb @@ -38,7 +38,7 @@ describe "Creating a new agent", js: true do visit new_agent_path end it "shows all options for agents that can be scheduled, create and receive events" do - select2("Website Agent", from: "Type") + select2("Website Agent scrapes", from: "Type") expect(page).not_to have_content('This type of Agent cannot create events.') end @@ -49,9 +49,8 @@ describe "Creating a new agent", js: true do end it "allows to click on on the agent name in select2 tags" do - agent = agents(:bob_weather_agent) visit new_agent_path - select2("Website Agent", from: "Type") + select2("Website Agent scrapes", from: "Type") select2("SF Weather", from: 'Sources') click_on "SF Weather" expect(page).to have_content "Editing your WeatherAgent" @@ -63,7 +62,7 @@ describe "Creating a new agent", js: true do end it "does not send previously configured sources when the current agent does not support them" do - select2("Website Agent", from: "Type") + select2("Website Agent scrapes", from: "Type") select2("SF Weather", from: 'Sources') select2("Webhook Agent", from: "Type") fill_in(:agent_name, with: "No sources") @@ -85,7 +84,7 @@ describe "Creating a new agent", js: true do end it "does not send previously configured receivers when the current agent does not support them" do - select2("Website Agent", from: "Type") + select2("Website Agent scrapes", from: "Type") select2("ZKCD", from: 'Receivers') select2("Email Agent", from: "Type") fill_in(:agent_name, with: "No receivers") From db4de696b4d71169ca9733ce0fd473a5cdc7c334 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 27 Nov 2016 12:30:45 +0100 Subject: [PATCH 20/23] Override `git_source` method in the Gemfile When the bundler version is below 2 override `git_source` to ensure `github` remotes always use HTTPS --- Gemfile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index f6fec106..95608b5b 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,12 @@ source 'https://rubygems.org' # Ruby 2.2.2 is the minimum requirement ruby ['2.2.2', RUBY_VERSION].max +# Ensure github repositories are fetched using HTTPS +git_source(:github) do |repo_name| + repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/") + "https://github.com/#{repo_name}.git" +end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2') + # Load vendored dotenv gem and .env file require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb') GemfileHelper.load_dotenv do |dotenv_dir| @@ -50,7 +56,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn' gem 'omniauth-twitter', '~> 1.2.1' # Tumblr Agents -gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master' # '>= 0.8.5' +gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master', ref: '0c59b04e49f2a8c89860613b18cf4e8f978d8dc7' # '>= 0.8.5' gem 'omniauth-tumblr', '~> 1.2' # Dropbox Agents @@ -103,7 +109,7 @@ gem 'jquery-rails', '~> 4.2.1' gem 'huginn_agent', '~> 0.4.0' gem 'json', '~> 1.8.1' gem 'jsonpathv2', '~> 0.0.8' -gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable' +gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable', ref: 'abbf93d557208ee1d0b612c612cd079f86ed54f4' gem 'kramdown', '~> 1.3.3' gem 'liquid', '~> 3.0.3' gem 'loofah', '~> 2.0' From f813a73dd067ee33dedd9fe46ffc6569a960ae3b Mon Sep 17 00:00:00 2001 From: Alex Jordan Date: Mon, 31 Oct 2016 18:39:34 -0700 Subject: [PATCH 21/23] Update Gemfile.lock with HTTPS URLs for git gems --- Gemfile.lock | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ee694741..1542b084 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,7 @@ GIT - remote: git://github.com/amatsuda/kaminari.git + remote: https://github.com/amatsuda/kaminari.git revision: abbf93d557208ee1d0b612c612cd079f86ed54f4 + ref: abbf93d557208ee1d0b612c612cd079f86ed54f4 branch: 0-17-stable specs: kaminari (0.17.0) @@ -8,7 +9,7 @@ GIT activesupport (>= 3.0.0) GIT - remote: git://github.com/cantino/twitter-stream.git + remote: https://github.com/cantino/twitter-stream.git revision: f7e7edb0bae013bffabf3598e7147773d9fd370f branch: huginn specs: @@ -18,7 +19,7 @@ GIT simple_oauth (~> 0.3.0) GIT - remote: git://github.com/dsander/delayed_job_active_record.git + remote: https://github.com/dsander/delayed_job_active_record.git revision: b314972ccc92e0e8b03b1589174d8fb9a82b3cd0 branch: rails5 specs: @@ -27,7 +28,7 @@ GIT delayed_job (>= 3.0, < 5) GIT - remote: git://github.com/dsander/weibo_2.git + remote: https://github.com/dsander/weibo_2.git revision: e5b77f21a7e9a666b582c48e16b1e96fca198cf8 branch: master specs: @@ -38,7 +39,7 @@ GIT rest-client (~> 1.8) GIT - remote: git://github.com/lostisland/faraday_middleware.git + remote: https://github.com/lostisland/faraday_middleware.git revision: 59088da02940d0ee2010b2e3156343346767c31e branch: master specs: @@ -46,8 +47,9 @@ GIT faraday (>= 0.7.4, < 1.0) GIT - remote: git://github.com/tumblr/tumblr_client.git + remote: https://github.com/tumblr/tumblr_client.git revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7 + ref: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7 branch: master specs: tumblr_client (0.8.5) @@ -703,4 +705,4 @@ RUBY VERSION ruby 2.3.3p222 BUNDLED WITH - 1.13.6 + 1.13.7 From 0285a1a9933db7b05ae7e0ead5c0e7660e72e731 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Tue, 27 Dec 2016 23:35:06 +0100 Subject: [PATCH 22/23] Update rails to 5.0.1 --- Gemfile | 4 +-- Gemfile.lock | 80 ++++++++++++++++++++++++++-------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Gemfile b/Gemfile index 95608b5b..016c72dd 100644 --- a/Gemfile +++ b/Gemfile @@ -117,7 +117,7 @@ gem 'mini_magick' gem 'multi_xml' gem 'nokogiri' gem 'omniauth', '~> 1.3.1' -gem 'rails', '~> 5.0.0.1' +gem 'rails', '~> 5.0.1' gem 'rufus-scheduler', '~> 3.0.8', require: false gem 'sass-rails', '~> 5.0.6' gem 'select2-rails', '~> 3.5.4' @@ -134,7 +134,7 @@ group :development do gem 'guard-rspec', '~> 4.6.4' gem 'rack-livereload', '~> 0.3.16' gem 'letter_opener_web', '~> 1.3.0' - gem 'web-console' + gem 'web-console', '>= 3.3.0' gem 'capistrano', '~> 3.4.0' gem 'capistrano-rails', '~> 1.1' diff --git a/Gemfile.lock b/Gemfile.lock index 1542b084..4bf8109d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -71,45 +71,45 @@ GEM remote: https://rubygems.org/ specs: ace-rails-ap (2.0.1) - actioncable (5.0.0.1) - actionpack (= 5.0.0.1) + actioncable (5.0.1) + actionpack (= 5.0.1) nio4r (~> 1.2) websocket-driver (~> 0.6.1) - actionmailer (5.0.0.1) - actionpack (= 5.0.0.1) - actionview (= 5.0.0.1) - activejob (= 5.0.0.1) + actionmailer (5.0.1) + actionpack (= 5.0.1) + actionview (= 5.0.1) + activejob (= 5.0.1) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.0.0.1) - actionview (= 5.0.0.1) - activesupport (= 5.0.0.1) + actionpack (5.0.1) + actionview (= 5.0.1) + activesupport (= 5.0.1) rack (~> 2.0) rack-test (~> 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.0.0.1) - activesupport (= 5.0.0.1) + actionview (5.0.1) + activesupport (= 5.0.1) builder (~> 3.1) erubis (~> 2.7.0) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - activejob (5.0.0.1) - activesupport (= 5.0.0.1) + activejob (5.0.1) + activesupport (= 5.0.1) globalid (>= 0.3.6) - activemodel (5.0.0.1) - activesupport (= 5.0.0.1) - activerecord (5.0.0.1) - activemodel (= 5.0.0.1) - activesupport (= 5.0.0.1) + activemodel (5.0.1) + activesupport (= 5.0.1) + activerecord (5.0.1) + activemodel (= 5.0.1) + activesupport (= 5.0.1) arel (~> 7.0) - activesupport (5.0.0.1) + activesupport (5.0.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (~> 0.7) minitest (~> 5.1) tzinfo (~> 1.1) addressable (2.3.8) - arel (7.1.1) + arel (7.1.4) autoparse (0.3.3) addressable (>= 2.3.1) extlib (>= 0.9.15) @@ -159,7 +159,7 @@ GEM execjs coffee-script-source (1.10.0) colorize (0.7.7) - concurrent-ruby (1.0.2) + concurrent-ruby (1.0.3) cookiejar (0.3.2) coveralls (0.7.12) multi_json (~> 1.10) @@ -329,7 +329,7 @@ GEM mimemagic (0.3.1) mini_magick (4.2.3) mini_portile2 (2.1.0) - minitest (5.9.0) + minitest (5.10.1) mqtt (0.3.1) multi_json (1.12.1) multi_xml (0.5.5) @@ -404,17 +404,17 @@ GEM rack rack-test (0.6.3) rack (>= 1.0) - rails (5.0.0.1) - actioncable (= 5.0.0.1) - actionmailer (= 5.0.0.1) - actionpack (= 5.0.0.1) - actionview (= 5.0.0.1) - activejob (= 5.0.0.1) - activemodel (= 5.0.0.1) - activerecord (= 5.0.0.1) - activesupport (= 5.0.0.1) + rails (5.0.1) + actioncable (= 5.0.1) + actionmailer (= 5.0.1) + actionpack (= 5.0.1) + actionview (= 5.0.1) + activejob (= 5.0.1) + activemodel (= 5.0.1) + activerecord (= 5.0.1) + activesupport (= 5.0.1) bundler (>= 1.3.0, < 2.0) - railties (= 5.0.0.1) + railties (= 5.0.1) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.1) actionpack (~> 5.x) @@ -425,14 +425,14 @@ GEM nokogiri (~> 1.6.0) rails-html-sanitizer (1.0.3) loofah (~> 2.0) - railties (5.0.0.1) - actionpack (= 5.0.0.1) - activesupport (= 5.0.0.1) + railties (5.0.1) + actionpack (= 5.0.1) + activesupport (= 5.0.1) method_source rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) raindrops (0.17.0) - rake (11.2.2) + rake (12.0.0) rb-fsevent (0.9.7) rb-inotify (0.9.5) ffi (>= 0.5.0) @@ -516,7 +516,7 @@ GEM spring-watcher-listen (2.0.0) listen (>= 2.7, < 4.0) spring (~> 1.2) - sprockets (3.7.0) + sprockets (3.7.1) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.2.0) @@ -533,7 +533,7 @@ GEM therubyracer (0.12.2) libv8 (~> 3.16.14.0) ref - thor (0.19.1) + thor (0.19.4) thread_safe (0.3.5) tilt (2.0.5) tins (1.10.1) @@ -665,7 +665,7 @@ DEPENDENCIES pry-byebug pry-rails rack-livereload (~> 0.3.16) - rails (~> 5.0.0.1) + rails (~> 5.0.1) rails-controller-testing rb-kqueue (>= 0.2) rr @@ -695,7 +695,7 @@ DEPENDENCIES uglifier (~> 2.7.2) unicorn (~> 5.1.0) vcr - web-console + web-console (>= 3.3.0) webmock (~> 1.17.4) weibo_2! wunderground (~> 1.2.0) From 070946e879872de609524b6434cc170207cada12 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Tue, 27 Dec 2016 23:35:29 +0100 Subject: [PATCH 23/23] Work around possible regression in rails `pluck` used to always trigger a database query. With Rails 5.0.1 it uses data from already loaded associations, this also includes new records for which we can not generate agent_paths --- app/views/layouts/application.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 561f9a94..69f81378 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -52,7 +52,7 @@ window.agentPaths = {}; window.agentNames = []; <% if current_user.present? -%> - var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| m[a.first] = agent_path(a.last); m }) %>; + var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| next if a.last.nil?; m[a.first] = agent_path(a.last); m }) %>; var myScenarios = <%= Utils.jsonify(current_user.scenarios.pluck(:name, :id).inject({}) {|m, s| m[s.first + " Scenario"] = scenario_path(s.last); m }) %>; $.extend(window.agentPaths, myAgents); $.extend(window.agentPaths, myScenarios);