From 7ac691652b90893bc84d89f31d996edc290d5259 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sat, 26 Nov 2016 13:26:09 +0900
Subject: [PATCH 01/23] Spec that force_encoding works with encoding
declaration in RssAgent
---
spec/models/agents/rss_agent_spec.rb | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index a70565f8..31c6737e 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -295,6 +295,13 @@ describe Agents::RssAgent do
event = agent.events.first
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
end
+
+ it "decodes the content properly with force_encoding specified" do
+ @valid_options['force_encoding'] = 'iso-8859-1'
+ agent.check
+ event = agent.events.first
+ expect(event.payload['title']).to eq('Mëkanïk Zaïn')
+ end
end
end
From f530305edc39b730b9a8a607bbc61589c60da4a9 Mon Sep 17 00:00:00 2001
From: bobbysteel <github@robk.com>
Date: Sat, 26 Nov 2016 20:13:00 +0000
Subject: [PATCH 02/23] Add class of service chooser for Google Flights Agent
(#1778)
* Add class of service chooser
* Add cabin chooser test
* Fix preferredCabin
* Per @cantino feedback taking out check
---
app/models/agents/google_flights_agent.rb | 6 ++++--
spec/models/agents/google_flights_spec.rb | 1 +
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/app/models/agents/google_flights_agent.rb b/app/models/agents/google_flights_agent.rb
index f7a82521..ccdc66b4 100644
--- a/app/models/agents/google_flights_agent.rb
+++ b/app/models/agents/google_flights_agent.rb
@@ -60,6 +60,7 @@ module Agents
'seniorCount'=> 0,
'return_date' => '2016-04-18',
'roundtrip' => true,
+ 'preferredCabin' => 'COACH',
'solutions'=> 3
}
end
@@ -69,6 +70,7 @@ module Agents
form_configurable :origin, type: :string
form_configurable :destination, type: :string
form_configurable :date, type: :string
+ form_configurable :preferredCabin, type: :array, values: %w(COACH PREMIUM_COACH BUSINESS FIRST)
form_configurable :childCount
form_configurable :infantInSeatCount
form_configurable :infantInLapCount
@@ -101,9 +103,9 @@ module Agents
def post_params
if round_trip?
- post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s } ], :solutions=> interpolated["solutions"]}}
+ post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s} ], :solutions=> interpolated["solutions"]}}
else
- post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }], :solutions=> interpolated["solutions"]}}
+ post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }], :solutions=> interpolated["solutions"]}}
end
end
diff --git a/spec/models/agents/google_flights_spec.rb b/spec/models/agents/google_flights_spec.rb
index 28f5994d..a0b970b8 100644
--- a/spec/models/agents/google_flights_spec.rb
+++ b/spec/models/agents/google_flights_spec.rb
@@ -15,6 +15,7 @@ describe Agents::GoogleFlightsAgent do
'origin' => 'BOS',
'destination' => 'SFO',
'date' => '2016-04-11',
+ 'preferredCabin' => 'COACH',
'childCount' => 0,
'infantInSeatCount' => 0,
'infantInLapCount'=> 0,
From 3d91469733bf2b0c0d00815a33faf0ac688e18ee Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 13:33:24 +0900
Subject: [PATCH 03/23] Make WebsiteAgent merge `template` with the results of
`extract` (#1816)
A new extraction option `hidden` is added so that keys with it gets
excluded from the final payloads while they can be used in `template`.
---
app/models/agents/website_agent.rb | 82 +++++++++++------
...onvert_website_agent_template_for_merge.rb | 36 ++++++++
...t_website_agent_template_for_merge_spec.rb | 92 +++++++++++++++++++
spec/models/agents/website_agent_spec.rb | 23 ++++-
4 files changed, 201 insertions(+), 32 deletions(-)
create mode 100644 db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
create mode 100644 spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index 437df864..f36d70e3 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -37,6 +37,8 @@ module Agents
Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor except when it has `repeat` set to true. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
+ For extractors with `hidden` set to true, they will be excluded from the payloads of events created by the Agent, but can be used and interpolated in the `template` option explained below.
+
For extractors with `repeat` set to true, their first matches will be included in all extracts. This is useful such as when you want to include the title of a page in all events created from the page.
# Scraping HTML and XML
@@ -116,11 +118,9 @@ module Agents
Set `http_success_codes` to an array of status codes (e.g., `[404, 422]`) to treat HTTP response codes beyond 200 as successes.
- If a `template` option is given, it is used as a Liquid template for each event created by this Agent, instead of directly emitting the results of extraction as events. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
+ If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
"template": {
- "url": "{{ url }}",
- "title": "{{ title }}",
"description": "{{ body_text }}",
"last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
}
@@ -159,7 +159,11 @@ module Agents
end
def event_keys
- (options['template'].presence || options['extract']).try(:keys)
+ extract = options['extract'] or return nil
+
+ extract.each_with_object([]) { |(key, value), keys|
+ keys << key unless boolify(value['hidden'])
+ } | (options['template'].presence.try!(:keys) || [])
end
def working?
@@ -382,33 +386,19 @@ module Agents
extract_xml(doc)
end
- num_tuples = output.each_value.inject(nil) { |num, value|
- case size = value.size
- when Float::INFINITY
- num
- when Integer
- if num && num != size
- raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
- end
- size
- end
- } or raise "At least one non-repeat key is required"
+ num_tuples = output.size or
+ raise "At least one non-repeat key is required"
old_events = previous_payloads num_tuples
template = options['template'].presence
- num_tuples.times.zip(*output.values) do |index, *values|
- extracted = output.each_key.lazy.zip(values).to_h
+ output.each do |extracted|
+ result = extracted.except(*output.hidden_keys)
- result =
- if template
- interpolate_with(extracted) do
- interpolate_options(template)
- end
- else
- extracted
- end
+ if template
+ result.update(interpolate_options(template, extracted))
+ end
# url may be URI, string or nil
if (payload_url = result['url'].presence) && (url = url.presence)
@@ -528,7 +518,7 @@ module Agents
end
def extract_each(&block)
- interpolated['extract'].each_with_object({}) { |(name, extraction_details), output|
+ interpolated['extract'].each_with_object(Output.new) { |(name, extraction_details), output|
if boolify(extraction_details['repeat'])
values = Repeater.new { |repeater|
block.call(extraction_details, repeater)
@@ -538,7 +528,13 @@ module Agents
block.call(extraction_details, values)
end
log "Values extracted: #{values}"
- output[name] = values
+ begin
+ output[name] = values
+ rescue UnevenSizeError
+ raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
+ else
+ output.hidden_keys << name if boolify(extraction_details['hidden'])
+ end
}
end
@@ -617,6 +613,38 @@ module Agents
false
end
+ class UnevenSizeError < ArgumentError
+ end
+
+ class Output
+ def initialize
+ @hash = {}
+ @size = nil
+ @hidden_keys = []
+ end
+
+ attr_reader :size
+ attr_reader :hidden_keys
+
+ def []=(key, value)
+ case size = value.size
+ when Integer
+ if @size && @size != size
+ raise UnevenSizeError, 'got an uneven size'
+ end
+ @size = size
+ end
+
+ @hash[key] = value
+ end
+
+ def each
+ @size.times.zip(*@hash.values) do |index, *values|
+ yield @hash.each_key.lazy.zip(values).to_h
+ end
+ end
+ end
+
class Repeater < Enumerator
# Repeater.new { |y|
# # ...
diff --git a/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
new file mode 100644
index 00000000..65b22f4d
--- /dev/null
+++ b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
@@ -0,0 +1,36 @@
+class ConvertWebsiteAgentTemplateForMerge < ActiveRecord::Migration[5.0]
+ def up
+ Agents::WebsiteAgent.find_each do |agent|
+ extract = agent.options['extract'].presence
+ template = agent.options['template'].presence
+ next unless extract.is_a?(Hash) && template.is_a?(Hash)
+
+ (extract.keys - template.keys).each do |key|
+ extract[key]['hidden'] = true
+ end
+
+ template.delete_if { |key, value|
+ extract.key?(key) &&
+ value.match(/\A\{\{\s*#{Regexp.quote(key)}\s*\}\}\z/)
+ }
+
+ agent.save!(validate: false)
+ end
+ end
+
+ def down
+ Agents::WebsiteAgent.find_each do |agent|
+ extract = agent.options['extract'].presence
+ template = agent.options['template'].presence
+ next unless extract.is_a?(Hash) && template.is_a?(Hash)
+
+ (extract.keys - template.keys).each do |key|
+ unless extract[key].delete('hidden').in?([true, 'true'])
+ template[key] = "{{ #{key} }}"
+ end
+ end
+
+ agent.save!(validate: false)
+ end
+ end
+end
diff --git a/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb
new file mode 100644
index 00000000..af6c137f
--- /dev/null
+++ b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb
@@ -0,0 +1,92 @@
+load 'spec/rails_helper.rb'
+load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
+
+describe ConvertWebsiteAgentTemplateForMerge do
+ let :old_extract do
+ {
+ 'url' => { 'css' => "#comic img", 'value' => "@src" },
+ 'title' => { 'css' => "#comic img", 'value' => "@alt" },
+ 'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
+ }
+ end
+
+ let :new_extract do
+ {
+ 'url' => { 'css' => "#comic img", 'value' => "@src" },
+ 'title' => { 'css' => "#comic img", 'value' => "@alt" },
+ 'hovertext' => { 'css' => "#comic img", 'value' => "@title", 'hidden' => true }
+ }
+ end
+
+ let :reverted_extract do
+ old_extract
+ end
+
+ let :old_template do
+ {
+ 'url' => '{{url}}',
+ 'title' => '{{ title }}',
+ 'description' => '{{ hovertext }}',
+ 'comment' => '{{ comment }}'
+ }
+ end
+
+ let :new_template do
+ {
+ 'description' => '{{ hovertext }}',
+ 'comment' => '{{ comment }}'
+ }
+ end
+
+ let :reverted_template do
+ old_template.merge('url' => '{{ url }}')
+ end
+
+ let :valid_options do
+ {
+ 'name' => "XKCD",
+ 'expected_update_period_in_days' => "2",
+ 'type' => "html",
+ 'url' => "{{ url | default: 'http://xkcd.com/' }}",
+ 'mode' => 'on_change',
+ 'extract' => old_extract,
+ 'template' => old_template
+ }
+ end
+
+ let :agent do
+ Agents::WebsiteAgent.create!(
+ user: users(:bob),
+ name: "xkcd",
+ options: valid_options,
+ keep_events_for: 2.days
+ )
+ end
+
+ describe 'up' do
+ it 'should update extract and template options for an existing WebsiteAgent' do
+ expect(agent.options).to include('extract' => old_extract,
+ 'template' => old_template)
+ ConvertWebsiteAgentTemplateForMerge.new.up
+ agent.reload
+ expect(agent.options).to include('extract' => new_extract,
+ 'template' => new_template)
+ end
+ end
+
+ describe 'down' do
+ let :valid_options do
+ super().merge('extract' => new_extract,
+ 'template' => new_template)
+ end
+
+ it 'should revert extract and template options for an updated WebsiteAgent' do
+ expect(agent.options).to include('extract' => new_extract,
+ 'template' => new_template)
+ ConvertWebsiteAgentTemplateForMerge.new.down
+ agent.reload
+ expect(agent.options).to include('extract' => reverted_extract,
+ 'template' => reverted_template)
+ end
+ end
+end
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 9f25752d..01d9f223 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -651,9 +651,22 @@ describe Agents::WebsiteAgent do
@checker.options = @valid_options
@checker.check
event = Event.last
- expect(event.payload['url']).to eq("http://imgs.xkcd.com/comics/evolving.png")
- expect(event.payload['title']).to eq("Evolving")
- expect(event.payload['hovertext']).to match(/^Biologists play reverse/)
+ expect(event.payload).to match(
+ 'url' => 'http://imgs.xkcd.com/comics/evolving.png',
+ 'title' => 'Evolving',
+ 'hovertext' => /^Biologists play reverse/
+ )
+ end
+
+ it "should exclude hidden keys" do
+ @valid_options['extract']['hovertext']['hidden'] = true
+ @checker.options = @valid_options
+ @checker.check
+ event = Event.last
+ expect(event.payload).to match(
+ 'url' => 'http://imgs.xkcd.com/comics/evolving.png',
+ 'title' => 'Evolving'
+ )
end
it "should turn relative urls to absolute" do
@@ -749,9 +762,9 @@ describe Agents::WebsiteAgent do
expect(event.payload['original_url']).to eq('http://xkcd.com/index')
end
- it "should be formatted by template after extraction" do
+ it "should format and merge values in template after extraction" do
+ @valid_options['extract']['hovertext']['hidden'] = true
@valid_options['template'] = {
- 'url' => '{{url}}',
'title' => '{{title | upcase}}',
'summary' => '{{title}}: {{hovertext | truncate: 20}}',
}
From 3a0c9e6274bf146af23aa14a32b8899b41910bc7 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 2 Nov 2016 21:03:31 +0900
Subject: [PATCH 04/23] Disable automatic URL normalization and absolutization
on `url`
This was discussed in #1766.
For backward compatibility, existing WebsiteAgents with a key named
`url` will be given a `template` to resolve `url`.
---
app/models/agents/website_agent.rb | 50 ++++++++++++-------
...1124065838_add_templates_to_resolve_url.rb | 16 ++++++
...65838_add_templates_to_resolve_url_spec.rb | 39 +++++++++++++++
spec/models/agents/website_agent_spec.rb | 38 ++++++--------
4 files changed, 104 insertions(+), 39 deletions(-)
create mode 100644 db/migrate/20161124065838_add_templates_to_resolve_url.rb
create mode 100644 spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index f36d70e3..1918186e 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -27,8 +27,6 @@ module Agents
* Alternatively, set `data_from_event` to a Liquid template to use data directly without fetching any URL. (For example, set it to `{{ html }}` to use HTML contained in the `html` key of the incoming Event.)
* If you specify `merge` for the `mode` option, Huginn will retain the old payload and update it with new values.
- If a created Event has a key named `url` containing a relative URL, it is automatically resolved using the request URL as base.
-
# Supported Document Types
The `type` value can be `xml`, `html`, `json`, or `text`.
@@ -121,6 +119,7 @@ module Agents
If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload. In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section. For example:
"template": {
+ "url": "{{ url | to_uri: _request_.url }}",
"description": "{{ body_text }}",
"last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
}
@@ -129,17 +128,17 @@ module Agents
# Liquid Templating
- In Liquid templating, the following variables are available except when invoked by `data_from_event`:
+ In Liquid templating, the following variables are available:
- * `_url_`: The URL specified to fetch the content from.
+ * `_url_`: The URL specified to fetch the content from. When parsing `data_from_event`, this is not set.
* `_response_`: A response object with the following keys:
- * `status`: HTTP status as integer. (Almost always 200)
+ * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event.
- * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_.
+ * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
- * `url`: The final URL of the fetched page, following redirects. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
+ * `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
# Ordering Events
@@ -366,6 +365,8 @@ module Agents
end
def handle_data(body, url, existing_payload)
+ # Beware, url may be a URI object, string or nil
+
doc = parse(body)
if extract_full_json?
@@ -400,15 +401,6 @@ module Agents
result.update(interpolate_options(template, extracted))
end
- # url may be URI, string or nil
- if (payload_url = result['url'].presence) && (url = url.presence)
- begin
- result['url'] = (Utils.normalize_uri(url) + Utils.normalize_uri(payload_url)).to_s
- rescue URI::Error
- error "Cannot resolve url: <#{payload_url}> on <#{url}>"
- end
- end
-
if store_payload!(old_events, result)
log "Storing new parsed result for '#{name}': #{result.inspect}"
create_event payload: existing_payload.merge(result)
@@ -450,7 +442,10 @@ module Agents
end
def handle_event_data(data, event, existing_payload)
- handle_data(data, event.payload['url'], existing_payload)
+ interpolation_context.stack {
+ interpolation_context['_response_'] = ResponseFromEventDrop.new(event)
+ handle_data(data, event.payload['url'].presence, existing_payload)
+ }
rescue => e
error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event
end
@@ -687,6 +682,27 @@ module Agents
end
end
+ class ResponseFromEventDrop < LiquidDroppable::Drop
+ def headers
+ case headers = @object.payload[:headers]
+ when Hash
+ HeaderDrop.new(Faraday::Utils::Headers.from(headers))
+ else
+ HeaderDrop.new({})
+ end
+ end
+
+ # Integer value of HTTP status
+ def status
+ @object.payload[:status]
+ end
+
+ # The URL
+ def url
+ @object.payload[:url]
+ end
+ end
+
# Wraps Faraday::Utils::Headers
class HeaderDrop < LiquidDroppable::Drop
def before_method(name)
diff --git a/db/migrate/20161124065838_add_templates_to_resolve_url.rb b/db/migrate/20161124065838_add_templates_to_resolve_url.rb
new file mode 100644
index 00000000..a1e62522
--- /dev/null
+++ b/db/migrate/20161124065838_add_templates_to_resolve_url.rb
@@ -0,0 +1,16 @@
+class AddTemplatesToResolveUrl < ActiveRecord::Migration[5.0]
+ def up
+ Agents::WebsiteAgent.find_each do |agent|
+ if agent.event_keys.try!(:include?, 'url')
+ agent.options['template'] = (agent.options['template'] || {}).tap { |template|
+ template['url'] ||= '{{ url | to_uri: _response_.url }}'
+ }
+ agent.save!(validate: false)
+ end
+ end
+ end
+
+ def down
+ # No need to revert
+ end
+end
diff --git a/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb
new file mode 100644
index 00000000..6d0bd14b
--- /dev/null
+++ b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb
@@ -0,0 +1,39 @@
+load 'spec/rails_helper.rb'
+load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
+
+describe AddTemplatesToResolveUrl do
+ let :valid_options do
+ {
+ 'name' => "XKCD",
+ 'expected_update_period_in_days' => "2",
+ 'type' => "html",
+ 'url' => "http://xkcd.com",
+ 'mode' => 'on_change',
+ 'extract' => {
+ 'url' => { 'css' => "#comic img", 'value' => "@src" },
+ 'title' => { 'css' => "#comic img", 'value' => "@alt" },
+ 'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
+ }
+ }
+ end
+
+ let :agent do
+ Agents::WebsiteAgent.create!(
+ user: users(:bob),
+ name: "xkcd",
+ options: valid_options,
+ keep_events_for: 2.days
+ )
+ end
+
+ it 'should add a template for an existing WebsiteAgent with `url`' do
+ expect(agent.options).not_to include('template')
+ AddTemplatesToResolveUrl.new.up
+ agent.reload
+ expect(agent.options).to include(
+ 'template' => {
+ 'url' => '{{ url | to_uri: _response_.url }}'
+ }
+ )
+ end
+end
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 01d9f223..d943ca77 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -669,25 +669,6 @@ describe Agents::WebsiteAgent do
)
end
- it "should turn relative urls to absolute" do
- rel_site = {
- 'name' => "XKCD",
- 'expected_update_period_in_days' => "2",
- 'type' => "html",
- 'url' => "http://xkcd.com",
- 'mode' => "on_change",
- 'extract' => {
- 'url' => {'css' => "#topLeft a", 'value' => "@href"},
- }
- }
- rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
- rel.user = users(:bob)
- rel.save!
- rel.check
- event = Event.last
- expect(event.payload['url']).to eq("http://xkcd.com/about")
- end
-
it "should return an integer value if XPath evaluates to one" do
rel_site = {
'name' => "XKCD",
@@ -1198,7 +1179,11 @@ fire: hot
'some_object' => {
'some_data' => { hello: 'world', href: '/world' }.to_json
},
- url: 'http://example.com/'
+ url: 'http://example.com/',
+ 'headers' => {
+ 'Content-Type' => 'application/json'
+ },
+ 'status' => 200
}
@event.save!
@@ -1208,6 +1193,12 @@ fire: hot
'extract' => {
'value' => { 'path' => 'hello' },
'url' => { 'path' => 'href' },
+ },
+ 'template' => {
+ 'value' => '{{ value }}',
+ 'url' => '{{ url | to_uri: _response_.url }}',
+ 'type' => '{{ _response_.headers.content_type }}',
+ 'status' => '{{ _response_.status }}'
}
)
end
@@ -1216,7 +1207,7 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
- expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world' })
+ expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' })
end
it "should support merge mode" do
@@ -1225,7 +1216,7 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
- expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world'))
+ expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200'))
end
it "should output an error when nothing can be found at the path" do
@@ -1362,6 +1353,9 @@ fire: hot
'mode' => 'all',
'extract' => {
'url' => { 'css' => "a", 'value' => "@href" },
+ },
+ 'template' => {
+ 'url' => '{{ url | to_uri }}',
}
}
@checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options)
From a94cd7fd6d749cf37851f9db2eecc9bf99e00568 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sat, 12 Nov 2016 14:56:39 +0900
Subject: [PATCH 05/23] Allow an empty or null base URI
---
app/concerns/liquid_interpolatable.rb | 7 ++++---
spec/concerns/liquid_interpolatable_spec.rb | 9 +++++++++
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/app/concerns/liquid_interpolatable.rb b/app/concerns/liquid_interpolatable.rb
index 83d93dec..4bc0d091 100644
--- a/app/concerns/liquid_interpolatable.rb
+++ b/app/concerns/liquid_interpolatable.rb
@@ -129,10 +129,11 @@ module LiquidInterpolatable
# userinfo, host, port, registry, path, opaque, query, and
# fragment.
def to_uri(uri, base_uri = nil)
- if base_uri
- Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
- else
+ case base_uri
+ when nil, ''
Utils.normalize_uri(uri.to_s)
+ else
+ Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
end
rescue URI::Error
nil
diff --git a/spec/concerns/liquid_interpolatable_spec.rb b/spec/concerns/liquid_interpolatable_spec.rb
index 6c7c9a71..b58b9996 100644
--- a/spec/concerns/liquid_interpolatable_spec.rb
+++ b/spec/concerns/liquid_interpolatable_spec.rb
@@ -119,6 +119,15 @@ describe LiquidInterpolatable::Filters do
@agent.interpolation_context['s'] = 'foo/index.html'
expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
end
+
+ it 'should normalize a URI value if an empty base URI is given' do
+ @agent.options['foo'] = '{{ u | to_uri: b }}'
+ @agent.interpolation_context['u'] = "\u{3042}"
+ @agent.interpolation_context['b'] = ""
+ expect(@agent.interpolated['foo']).to eq('%E3%81%82')
+ @agent.interpolation_context['b'] = nil
+ expect(@agent.interpolated['foo']).to eq('%E3%81%82')
+ end
end
describe 'uri_expand' do
From 9074f3115ea572c2aa9be771997f48d4865daf22 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 23 Nov 2016 11:16:14 +0900
Subject: [PATCH 06/23] Make sure `status` is an integer when set
---
app/models/agents/website_agent.rb | 4 ++--
spec/models/agents/website_agent_spec.rb | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index 1918186e..e8b97686 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -134,7 +134,7 @@ module Agents
* `_response_`: A response object with the following keys:
- * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event.
+ * `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
* `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
@@ -694,7 +694,7 @@ module Agents
# Integer value of HTTP status
def status
- @object.payload[:status]
+ Integer(@object.payload[:status]) rescue nil
end
# The URL
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index d943ca77..556f9c33 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -1198,7 +1198,7 @@ fire: hot
'value' => '{{ value }}',
'url' => '{{ url | to_uri: _response_.url }}',
'type' => '{{ _response_.headers.content_type }}',
- 'status' => '{{ _response_.status }}'
+ 'status' => '{{ _response_.status | as_object }}'
}
)
end
@@ -1207,7 +1207,7 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
- expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' })
+ expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200 })
end
it "should support merge mode" do
@@ -1216,7 +1216,7 @@ fire: hot
expect {
@checker.receive([@event])
}.to change { Event.count }.by(1)
- expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200'))
+ expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
end
it "should output an error when nothing can be found at the path" do
From 5f5f3cd38f7b67bb3d33ac298cb3ebe4c17d33bf Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 23 Nov 2016 11:17:38 +0900
Subject: [PATCH 07/23] Do not err if `headers` is a valid headers hash
---
app/models/agents/website_agent.rb | 11 ++++-------
spec/models/agents/website_agent_spec.rb | 18 ++++++++++++++++++
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index e8b97686..37fd780a 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -136,7 +136,7 @@ module Agents
* `status`: HTTP status as integer. (Almost always 200) When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
- * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
+ * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header. Keys are insensitive to cases and -/_. When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event, if it is a hash.
* `url`: The final URL of the fetched page, following redirects. When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event. Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
@@ -684,12 +684,9 @@ module Agents
class ResponseFromEventDrop < LiquidDroppable::Drop
def headers
- case headers = @object.payload[:headers]
- when Hash
- HeaderDrop.new(Faraday::Utils::Headers.from(headers))
- else
- HeaderDrop.new({})
- end
+ headers = Faraday::Utils::Headers.from(@object.payload[:headers]) rescue {}
+
+ HeaderDrop.new(headers)
end
# Integer value of HTTP status
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 556f9c33..afceb250 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -1219,6 +1219,24 @@ fire: hot
expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
end
+ it "should convert headers and status in the event data properly" do
+ @event.payload[:status] = '201'
+ @event.payload[:headers] = [['Content-Type', 'application/rss+xml']]
+ expect {
+ @checker.receive([@event])
+ }.to change { Event.count }.by(1)
+ expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/rss+xml', 'status' => 201 })
+ end
+
+ it "should ignore inconvertible headers and status in the event data" do
+ @event.payload[:status] = 'ok'
+ @event.payload[:headers] = ['Content-Type', 'Content-Length']
+ expect {
+ @checker.receive([@event])
+ }.to change { Event.count }.by(1)
+ expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => '', 'status' => nil })
+ end
+
it "should output an error when nothing can be found at the path" do
@checker.options = @checker.options.merge(
'data_from_event' => '{{ some_object.mistake }}'
From f4308afdf70ae9d98f22c9853553ed1bf75fd8c5 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 22:12:40 +0900
Subject: [PATCH 08/23] Use faraday_middleware 0.10.1, which was recently
released
---
Gemfile | 2 +-
Gemfile.lock | 12 +++---------
2 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/Gemfile b/Gemfile
index 5a1005ac..c931b5db 100644
--- a/Gemfile
+++ b/Gemfile
@@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br
gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
-gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
+gem 'faraday_middleware', '>= 0.10.1'
gem 'feedjira', '~> 2.0'
gem 'font-awesome-sass', '~> 4.3.2'
gem 'foreman', '~> 0.63.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index c18cf56a..99dc0984 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,14 +37,6 @@ GIT
oauth2 (~> 1)
rest-client (~> 1.8)
-GIT
- remote: git://github.com/lostisland/faraday_middleware.git
- revision: c5836ae55857272732b33eb0e0a98d60e995a376
- branch: master
- specs:
- faraday_middleware (0.10.0)
- faraday (>= 0.7.4, < 0.10)
-
GIT
remote: git://github.com/tumblr/tumblr_client.git
revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
@@ -214,6 +206,8 @@ GEM
extlib (0.9.16)
faraday (0.9.2)
multipart-post (>= 1.2, < 3)
+ faraday_middleware (0.10.1)
+ faraday (>= 0.7.4, < 1.0)
feedjira (2.0.0)
faraday (~> 0.9)
faraday_middleware (~> 0.9)
@@ -618,7 +612,7 @@ DEPENDENCIES
em-http-request (~> 1.1.2)
evernote_oauth
faraday (~> 0.9.0)
- faraday_middleware!
+ faraday_middleware (>= 0.10.1)
feedjira (~> 2.0)
ffi (>= 1.9.4)
font-awesome-sass (~> 4.3.2)
From 08b43cc8b9bacdb61ec22bdd9c8ffd985e80d2ca Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 22:14:02 +0900
Subject: [PATCH 09/23] Update the RUBY VERSION with 2.3.3p222
---
Gemfile.lock | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Gemfile.lock b/Gemfile.lock
index 99dc0984..aef7d8bc 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -694,7 +694,7 @@ DEPENDENCIES
xmpp4r (~> 0.5.6)
RUBY VERSION
- ruby 2.3.1p112
+ ruby 2.3.3p222
BUNDLED WITH
1.13.6
From 8e36fdd60c54f32f9b440cfdcf6616050e830d60 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 23:22:42 +0900
Subject: [PATCH 10/23] Revert "Use faraday_middleware 0.10.1, which was
recently released"
This reverts commit f4308afdf70ae9d98f22c9853553ed1bf75fd8c5.
I realized 0.10.1 did not include my fix for raw deflate.
---
Gemfile | 2 +-
Gemfile.lock | 12 +++++++++---
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/Gemfile b/Gemfile
index c931b5db..5a1005ac 100644
--- a/Gemfile
+++ b/Gemfile
@@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br
gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
-gem 'faraday_middleware', '>= 0.10.1'
+gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
gem 'feedjira', '~> 2.0'
gem 'font-awesome-sass', '~> 4.3.2'
gem 'foreman', '~> 0.63.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index aef7d8bc..90078534 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,6 +37,14 @@ GIT
oauth2 (~> 1)
rest-client (~> 1.8)
+GIT
+ remote: git://github.com/lostisland/faraday_middleware.git
+ revision: c5836ae55857272732b33eb0e0a98d60e995a376
+ branch: master
+ specs:
+ faraday_middleware (0.10.0)
+ faraday (>= 0.7.4, < 0.10)
+
GIT
remote: git://github.com/tumblr/tumblr_client.git
revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
@@ -206,8 +214,6 @@ GEM
extlib (0.9.16)
faraday (0.9.2)
multipart-post (>= 1.2, < 3)
- faraday_middleware (0.10.1)
- faraday (>= 0.7.4, < 1.0)
feedjira (2.0.0)
faraday (~> 0.9)
faraday_middleware (~> 0.9)
@@ -612,7 +618,7 @@ DEPENDENCIES
em-http-request (~> 1.1.2)
evernote_oauth
faraday (~> 0.9.0)
- faraday_middleware (>= 0.10.1)
+ faraday_middleware!
feedjira (~> 2.0)
ffi (>= 1.9.4)
font-awesome-sass (~> 4.3.2)
From 8a14a57e00716c12d060a036240cbfda789ab010 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@users.noreply.github.com>
Date: Sun, 27 Nov 2016 15:30:50 -0500
Subject: [PATCH 11/23] Beeper.io is no more (#1808)
* Beeper.io is no more
* Avoid event propagation or scheduling of missing agents
* Update undefined_agents.html.erb with a link to the wiki
---
app/models/agent.rb | 8 +-
app/models/agents/beeper_agent.rb | 129 ----------------
.../application/undefined_agents.html.erb | 8 +-
spec/models/agent_spec.rb | 33 +++-
spec/models/agents/beeper_agent_spec.rb | 145 ------------------
5 files changed, 41 insertions(+), 282 deletions(-)
delete mode 100644 app/models/agents/beeper_agent.rb
delete mode 100644 spec/models/agents/beeper_agent_spec.rb
diff --git a/app/models/agent.rb b/app/models/agent.rb
index b8d32ee9..66672d28 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -370,7 +370,7 @@ class Agent < ActiveRecord::Base
def receive!(options={})
Agent.transaction do
scope = Agent.
- select("agents.id AS receiver_agent_id, events.id AS event_id").
+ select("agents.id AS receiver_agent_id, sources.type AS source_agent_type, agents.type AS receiver_agent_type, events.id AS event_id").
joins("JOIN links ON (links.receiver_id = agents.id)").
joins("JOIN agents AS sources ON (links.source_id = sources.id)").
joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)").
@@ -379,10 +379,11 @@ class Agent < ActiveRecord::Base
scope = scope.where("agents.id in (?)", options[:only_receivers])
end
- sql = scope.to_sql()
+ sql = scope.to_sql
agents_to_events = {}
- Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id|
+ Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_type, receiver_agent_type, event_id|
+ next unless const_defined?(source_agent_type) && const_defined?(receiver_agent_type)
agents_to_events[receiver_agent_id.to_i] ||= []
agents_to_events[receiver_agent_id.to_i] << event_id
end
@@ -417,6 +418,7 @@ class Agent < ActiveRecord::Base
return if schedule == 'never'
types = where(:schedule => schedule).group(:type).pluck(:type)
types.each do |type|
+ next unless const_defined?(type)
type.constantize.bulk_check(schedule)
end
end
diff --git a/app/models/agents/beeper_agent.rb b/app/models/agents/beeper_agent.rb
deleted file mode 100644
index 4cd5a41e..00000000
--- a/app/models/agents/beeper_agent.rb
+++ /dev/null
@@ -1,129 +0,0 @@
-module Agents
- class BeeperAgent < Agent
- cannot_be_scheduled!
- cannot_create_events!
- no_bulk_receive!
-
- description <<-MD
- Beeper agent sends messages to Beeper app on your mobile device via Push notifications.
-
- You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io)
-
- You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID
-
- Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`.
-
- Depending on message type you have to provide additional fields:
-
- ##### Message
- * `text` – **required**
-
- ##### Image
- * `image` – **required** (Image URL or Base64-encoded image)
- * `text` – optional
-
- ##### Event
- * `text` – **required**
- * `start_time` – **required** (Corresponding to ISO 8601)
- * `end_time` – optional (Corresponding to ISO 8601)
-
- ##### Location
- * `latitude` – **required**
- * `longitude` – **required**
- * `text` – optional
-
- ##### Task
- * `text` – **required**
-
- You can see additional documentation at [Beeper website](https://beeper.io/docs)
- MD
-
- BASE_URL = 'https://api.beeper.io/api'
-
- TYPE_ATTRIBUTES = {
- 'message' => %w(text),
- 'image' => %w(text image),
- 'event' => %w(text start_time end_time),
- 'location' => %w(text latitude longitude),
- 'task' => %w(text)
- }
-
- MESSAGE_TYPES = TYPE_ATTRIBUTES.keys
-
- TYPE_REQUIRED_ATTRIBUTES = {
- 'message' => %w(text),
- 'image' => %w(image),
- 'event' => %w(text start_time),
- 'location' => %w(latitude longitude),
- 'task' => %w(text)
- }
-
- def default_options
- {
- 'type' => 'message',
- 'app_id' => '',
- 'api_key' => '',
- 'sender_id' => '',
- 'phone' => '',
- 'text' => '{{title}}'
- }
- end
-
- def validate_options
- %w(app_id api_key sender_id type).each do |attr|
- errors.add(:base, "you need to specify a #{attr}") if options[attr].blank?
- end
-
- if options['type'].in?(MESSAGE_TYPES)
- required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']]
- if required_attributes.any? { |attr| options[attr].blank? }
- errors.add(:base, "you need to specify a #{required_attributes.join(', ')}")
- end
- else
- errors.add(:base, 'you need to specify a valid message type')
- end
-
- unless options['group_id'].blank? ^ options['phone'].blank?
- errors.add(:base, 'you need to specify a phone or group_id')
- end
- end
-
- def working?
- received_event_without_error? && !recent_error_logs?
- end
-
- def receive(incoming_events)
- incoming_events.each do |event|
- send_message(event)
- end
- end
-
- def send_message(event)
- mo = interpolated(event)
- begin
- response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers)
- error(response.body) if response.code != 201
- rescue HTTParty::Error => e
- error(e.message)
- end
- end
-
- private
-
- def headers
- {
- 'X-Beeper-Application-Id' => options['app_id'],
- 'X-Beeper-REST-API-Key' => options['api_key'],
- 'Content-Type' => 'application/json'
- }
- end
-
- def payload_for(mo)
- mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json
- end
-
- def endpoint_for(type)
- "#{BASE_URL}/#{type}s.json"
- end
- end
-end
\ No newline at end of file
diff --git a/app/views/application/undefined_agents.html.erb b/app/views/application/undefined_agents.html.erb
index 3912af7d..229f3e19 100644
--- a/app/views/application/undefined_agents.html.erb
+++ b/app/views/application/undefined_agents.html.erb
@@ -19,20 +19,20 @@
</ul>
<br/>
<p>
- The issue most probably occurred because of one or more of the following reasons:
+ This issue probably occurred for one or more of the following reasons:
</p>
<ul>
+ <li>If the respective Agent is distributed as part of the Huginn application codebase, it may have been removed or moved to an Agent gem. Please see <a href="https://github.com/cantino/huginn/wiki/Dealing-with-Deleted-Agent-Types" target="_blank">this wiki page for more information</a>.</li>
<li>If the respective Agent is distributed as a Ruby gem, it might have been removed from the <code>ADDITIONAL_GEMS</code> environment setting.</li>
- <li>If the respective Agent is distributed as part of the Huginn application codebase, it might have been removed from that either on purpose (because the Agent has been deprecated or been moved to an Agent gem) or accidentally. Please check if the Agent(s) in question are available in your Huginn codebase under the path <code>app/models/agents/</code>.</li>
</ul>
<br/>
<p>
- You can fix the issue by adding the Agent(s) back to the application codebase by
+ You can fix this issue by:
</p>
<ul>
+ <li>deleting the respective Agent(s) from the database using the button below.</li>
<li>adding the respective Agent(s) to the the <code>ADDITIONAL_GEMS</code> environment setting. Please see <a href="https://github.com/cantino/huginn_agent" target="_blank">https://github.com/cantino/huginn_agent</a> for documentation on how to properly set it.</li>
<li>adding the respective Agent(s) code to the Huginn application codebase (in case it was deleted accidentally).</li>
- <li>deleting the respective Agent(s) from the database using the button below.</li>
</ul>
<br/>
<div class="btn-group">
diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb
index fa98b45b..7a67a587 100644
--- a/spec/models/agent_spec.rb
+++ b/spec/models/agent_spec.rb
@@ -74,6 +74,13 @@ describe Agent do
Agent.run_schedule("midnight")
end
+ it "ignores unknown types" do
+ Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
+ mock(Agents::WeatherAgent).bulk_check("midnight").once
+ mock(Agents::WebsiteAgent).bulk_check("midnight").once
+ Agent.run_schedule("midnight")
+ end
+
it "only runs agents with the given schedule" do
do_not_allow(Agents::WebsiteAgent).async_check
Agent.run_schedule("blah")
@@ -283,13 +290,37 @@ describe Agent do
Agent.receive!
end
- it "should not propogate to disabled Agents" do
+ it "should not propagate to disabled Agents" do
Agent.async_check(agents(:bob_weather_agent).id)
agents(:bob_rain_notifier_agent).update_attribute :disabled, true
mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
Agent.receive!
end
+ it "should not propagate to Agents with unknown types" do
+ Agent.async_check(agents(:jane_weather_agent).id)
+ Agent.async_check(agents(:bob_weather_agent).id)
+
+ Agent.where(id: agents(:bob_rain_notifier_agent).id).update_all type: 'UnknownTypeAgent'
+
+ mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
+ mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
+
+ Agent.receive!
+ end
+
+ it "should not propagate from Agents with unknown types" do
+ Agent.async_check(agents(:jane_weather_agent).id)
+ Agent.async_check(agents(:bob_weather_agent).id)
+
+ Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
+
+ mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
+ mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
+
+ Agent.receive!
+ end
+
it "should log exceptions" do
mock.any_instance_of(Agents::TriggerAgent).receive(anything).once {
raise "foo"
diff --git a/spec/models/agents/beeper_agent_spec.rb b/spec/models/agents/beeper_agent_spec.rb
deleted file mode 100644
index e2c2aba5..00000000
--- a/spec/models/agents/beeper_agent_spec.rb
+++ /dev/null
@@ -1,145 +0,0 @@
-require 'rails_helper'
-
-
-describe Agents::BeeperAgent do
- let(:base_params) {
- {
- 'type' => 'message',
- 'app_id' => 'some-app-id',
- 'api_key' => 'some-api-key',
- 'sender_id' => 'sender-id',
- 'phone' => '+111111111111',
- 'text' => 'Some text'
- }
- }
-
- subject {
- agent = described_class.new(name: 'beeper-agent', options: base_params)
- agent.user = users(:jane)
- agent.save! and return agent
- }
-
- context 'validation' do
- it 'valid' do
- expect(subject).to be_valid
- end
-
- [:type, :app_id, :api_key, :sender_id].each do |attr|
- it "invalid without #{attr}" do
- subject.options[attr] = nil
- expect(subject).not_to be_valid
- end
- end
-
- it 'invalid with group_id and phone' do
- subject.options['group_id'] ='some-group-id'
- expect(subject).not_to be_valid
- end
-
- context '#message' do
- it 'requires text' do
- subject.options[:text] = nil
- expect(subject).not_to be_valid
- end
- end
-
- context '#image' do
- before(:each) do
- subject.options[:type] = 'image'
- end
-
- it 'invalid without image' do
- expect(subject).not_to be_valid
- end
-
- it 'valid with image' do
- subject.options[:image] = 'some-url'
- expect(subject).to be_valid
- end
- end
-
- context '#event' do
- before(:each) do
- subject.options[:type] = 'event'
- end
-
- it 'invalid without start_time' do
- expect(subject).not_to be_valid
- end
-
- it 'valid with start_time' do
- subject.options[:start_time] = Time.now
- expect(subject).to be_valid
- end
- end
-
- context '#location' do
- before(:each) do
- subject.options[:type] = 'location'
- end
-
- it 'invalid without latitude and longitude' do
- expect(subject).not_to be_valid
- end
-
- it 'valid with latitude and longitude' do
- subject.options[:latitude] = 15.0
- subject.options[:longitude] = 16.0
- expect(subject).to be_valid
- end
- end
-
- context '#task' do
- before(:each) do
- subject.options[:type] = 'task'
- end
-
- it 'valid with text' do
- expect(subject).to be_valid
- end
- end
- end
-
- context 'payload_for' do
- it 'removes unwanted attributes' do
- result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text',
- 'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'})
- expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}')
- end
- end
-
- context 'headers' do
- it 'sets X-Beeper-Application-Id header with app_id' do
- expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id'])
- end
-
- it 'sets X-Beeper-REST-API-Key header with api_key' do
- expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key'])
- end
-
- it 'sets Content-Type' do
- expect(subject.send(:headers)['Content-Type']).to eq('application/json')
- end
- end
-
- context 'endpoint_for' do
- it 'returns valid URL for message' do
- expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json')
- end
-
- it 'returns valid URL for image' do
- expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json')
- end
-
- it 'returns valid URL for event' do
- expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json')
- end
-
- it 'returns valid URL for location' do
- expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json')
- end
- it 'returns valid URL for task' do
- expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json')
- end
- end
-end
From 3612ba7333ba4b07bba59acbac5181eae997a81e Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Mon, 7 Nov 2016 22:34:06 +0900
Subject: [PATCH 12/23] Add podcast tags to events emitted by RssAgent
The keys are only added when the feed is a podcast.
---
Gemfile | 2 +-
Gemfile.lock | 17 +--
app/models/agents/rss_agent.rb | 83 ++++++++++++++-
lib/feedjira_extension.rb | 17 +++
spec/data_fixtures/podcast.rss | 76 ++++++++++++++
spec/models/agents/rss_agent_spec.rb | 152 +++++++++++++++++++++++++++
6 files changed, 339 insertions(+), 8 deletions(-)
create mode 100644 spec/data_fixtures/podcast.rss
diff --git a/Gemfile b/Gemfile
index 5a1005ac..ffddd76a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
-gem 'feedjira', '~> 2.0'
+gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1']
gem 'font-awesome-sass', '~> 4.3.2'
gem 'foreman', '~> 0.63.0'
gem 'geokit', '~> 1.8.4'
diff --git a/Gemfile.lock b/Gemfile.lock
index 90078534..2b871768 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,6 +37,16 @@ GIT
oauth2 (~> 1)
rest-client (~> 1.8)
+GIT
+ remote: git://github.com/feedjira/feedjira.git
+ revision: 849b7809455c084b06f9c4c2d368a752677882b0
+ specs:
+ feedjira (2.0.0)
+ faraday (>= 0.9)
+ faraday_middleware (>= 0.9)
+ loofah (>= 2.0)
+ sax-machine (>= 1.0)
+
GIT
remote: git://github.com/lostisland/faraday_middleware.git
revision: c5836ae55857272732b33eb0e0a98d60e995a376
@@ -214,11 +224,6 @@ GEM
extlib (0.9.16)
faraday (0.9.2)
multipart-post (>= 1.2, < 3)
- feedjira (2.0.0)
- faraday (~> 0.9)
- faraday_middleware (~> 0.9)
- loofah (~> 2.0)
- sax-machine (~> 1.0)
ffi (1.9.10)
font-awesome-sass (4.3.2.1)
sass (~> 3.2)
@@ -619,7 +624,7 @@ DEPENDENCIES
evernote_oauth
faraday (~> 0.9.0)
faraday_middleware!
- feedjira (~> 2.0)
+ feedjira!
ffi (>= 1.9.4)
font-awesome-sass (~> 4.3.2)
forecast_io (~> 2.0.0)
diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb
index 0ed34c42..0ab5719c 100644
--- a/app/models/agents/rss_agent.rb
+++ b/app/models/agents/rss_agent.rb
@@ -66,6 +66,22 @@ module Agents
"copyright": "...",
"icon": "http://example.com/icon.png",
"authors": [ "..." ],
+
+ "itunes_block": "no",
+ "itunes_categories": [
+ "Technology", "Gadgets",
+ "TV & Film",
+ "Arts", "Food"
+ ],
+ "itunes_complete": "yes",
+ "itunes_explicit": "yes",
+ "itunes_image": "http://...",
+ "itunes_new_feed_url": "http://...",
+ "itunes_owners": [ "John Doe <john.doe@example.com>" ],
+ "itunes_subtitle": "...",
+ "itunes_summary": "...",
+ "language": "en-US",
+
"date_published": "2014-09-11T01:30:00-07:00",
"last_updated": "2014-09-11T01:30:00-07:00"
},
@@ -84,6 +100,16 @@ module Agents
"enclosure": {
"url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789"
},
+
+ "itunes_block": "no",
+ "itunes_closed_captioned": "yes",
+ "itunes_duration": "04:34",
+ "itunes_explicit": "yes",
+ "itunes_image": "http://...",
+ "itunes_order": "1",
+ "itunes_subtitle": "...",
+ "itunes_summary": "...",
+
"date_published": "2014-09-11T01:30:00-0700",
"last_updated": "2014-09-11T01:30:00-0700"
}
@@ -91,7 +117,8 @@ module Agents
Some notes:
- The `feed` key is present only if `include_feed_info` is set to true.
- - Each element in `authors` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
+ - The keys starting with `itunes_`, and `language` are only present when the feed is a podcast. See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details.
+ - Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
- Timestamps are converted to the ISO 8601 format.
MD
@@ -206,9 +233,40 @@ module Agents
authors: feed.authors,
date_published: feed.date_published,
last_updated: feed.last_updated,
+ **itunes_feed_data(feed)
}
end
+ def itunes_feed_data(feed)
+ data = {}
+ case feed
+ when Feedjira::Parser::ITunesRSS
+ %i[
+ itunes_block
+ itunes_categories
+ itunes_complete
+ itunes_explicit
+ itunes_image
+ itunes_new_feed_url
+ itunes_owners
+ itunes_subtitle
+ itunes_summary
+ language
+ ].each { |attr|
+ if value = feed.try(attr).presence
+ data[attr] =
+ case attr
+ when :itunes_summary
+ clean_fragment(value)
+ else
+ value
+ end
+ end
+ }
+ end
+ data
+ end
+
def entry_data(entry)
{
id: entry.id,
@@ -224,9 +282,32 @@ module Agents
categories: Array(entry.try(:categories)),
date_published: entry.date_published,
last_updated: entry.last_updated,
+ **itunes_entry_data(entry)
}
end
+ def itunes_entry_data(entry)
+ data = {}
+ case entry
+ when Feedjira::Parser::ITunesRSSItem
+ %i[
+ itunes_block
+ itunes_closed_captioned
+ itunes_duration
+ itunes_explicit
+ itunes_image
+ itunes_order
+ itunes_subtitle
+ itunes_summary
+ ].each { |attr|
+ if value = entry.try(attr).presence
+ data[attr] = value
+ end
+ }
+ end
+ data
+ end
+
def feed_to_events(feed)
payload_base = {}
diff --git a/lib/feedjira_extension.rb b/lib/feedjira_extension.rb
index a9a9ac11..5eab2981 100644
--- a/lib/feedjira_extension.rb
+++ b/lib/feedjira_extension.rb
@@ -57,6 +57,13 @@ module FeedjiraExtension
value :content
end
+ class ITunesRssOwner < Author
+ include SAXMachine
+
+ element :'itunes:name', as: :name
+ element :'itunes:email', as: :email
+ end
+
class Enclosure
include SAXMachine
@@ -290,6 +297,16 @@ module FeedjiraExtension
def copyright
@copyright || super
end
+
+ if /ITunes/ === name
+ sax_config.collection_elements['itunes:owner'].clear
+ elements :"itunes:owner", as: :_itunes_owners, class: ITunesRssOwner
+ private :_itunes_owners
+
+ def itunes_owners
+ _itunes_owners.reject(&:empty?)
+ end
+ end
end
sax_config.collection_elements.each_value do |collection_elements|
diff --git a/spec/data_fixtures/podcast.rss b/spec/data_fixtures/podcast.rss
new file mode 100644
index 00000000..bb64933e
--- /dev/null
+++ b/spec/data_fixtures/podcast.rss
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+ <channel>
+ <title>All About Everything</title>
+ <link>http://www.example.com/podcasts/everything/index.html</link>
+ <language>en-us</language>
+ <copyright>℗ & © 2014 John Doe & Family</copyright>
+ <itunes:subtitle>A show about everything</itunes:subtitle>
+ <itunes:author>John Doe</itunes:author>
+ <itunes:summary>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</itunes:summary>
+ <description>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</description>
+ <itunes:owner>
+ <itunes:name>John Doe</itunes:name>
+ <itunes:email>john.doe@example.com</itunes:email>
+ </itunes:owner>
+ <itunes:complete>yes</itunes:complete>
+ <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg"/>
+ <itunes:category text="Technology">
+ <itunes:category text="Gadgets"/>
+ </itunes:category>
+ <itunes:category text="TV & Film"/>
+ <itunes:category text="Arts">
+ <itunes:category text="Food"/>
+ </itunes:category>
+ <itunes:explicit>no</itunes:explicit>
+ <item>
+ <title>Shake Shake Shake Your Spices</title>
+ <itunes:author>John Doe</itunes:author>
+ <itunes:subtitle>A short primer on table spices</itunes:subtitle>
+ <itunes:summary><![CDATA[This week we talk about <a href="https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!]]></itunes:summary>
+ <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg"/>
+ <enclosure length="8727310" type="audio/x-m4a" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a"/>
+ <guid>http://example.com/podcasts/archive/aae20140615.m4a</guid>
+ <pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
+ <itunes:duration>07:04</itunes:duration>
+ <itunes:explicit>no</itunes:explicit>
+ </item>
+ <item>
+ <title>Socket Wrench Shootout</title>
+ <itunes:author>Jane Doe</itunes:author>
+ <itunes:subtitle>Comparing socket wrenches is fun!</itunes:subtitle>
+ <itunes:summary>This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here.</itunes:summary>
+ <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg"/>
+ <enclosure length="5650889" type="video/mp4" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4"/>
+ <guid>http://example.com/podcasts/archive/aae20140608.mp4</guid>
+ <pubDate>Wed, 09 Mar 2016 13:00:00 EST</pubDate>
+ <itunes:duration>04:34</itunes:duration>
+ <itunes:explicit>no</itunes:explicit>
+ </item>
+ <item>
+ <title>The Best Chili</title>
+ <itunes:author>Jane Doe</itunes:author>
+ <itunes:subtitle>Jane and Eric</itunes:subtitle>
+ <itunes:summary>This week we talk about the best Chili in the world. Which chili is better?</itunes:summary>
+ <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg"/>
+ <enclosure length="5650889" type="video/x-m4v" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v"/>
+ <guid>http://example.com/podcasts/archive/aae20140697.m4v</guid>
+ <pubDate>Thu, 10 Mar 2016 02:00:00 -0700</pubDate>
+ <itunes:duration>04:34</itunes:duration>
+ <itunes:explicit>no</itunes:explicit>
+ <itunes:isClosedCaptioned>Yes</itunes:isClosedCaptioned>
+ </item>
+ <item>
+ <title>Red,Whine, & Blue</title>
+ <itunes:author>Various</itunes:author>
+ <itunes:subtitle>Red + Blue != Purple</itunes:subtitle>
+ <itunes:summary>This week we talk about surviving in a Red state if you are a Blue person. Or vice versa.</itunes:summary>
+ <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg"/>
+ <enclosure length="498537" type="audio/mpeg" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3"/>
+ <guid>http://example.com/podcasts/archive/aae20140601.mp3</guid>
+ <pubDate>Fri, 11 Mar 2016 01:15:00 +3000</pubDate>
+ <itunes:duration>03:59</itunes:duration>
+ <itunes:explicit>no</itunes:explicit>
+ </item>
+ </channel>
+</rss>
diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index 31c6737e..ac651166 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -12,7 +12,11 @@ describe Agents::RssAgent do
stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
+<<<<<<< HEAD
stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
+=======
+ stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
+>>>>>>> Add podcast tags to events emitted by RssAgent
end
let(:agent) do
@@ -303,6 +307,154 @@ describe Agents::RssAgent do
expect(event.payload['title']).to eq('Mëkanïk Zaïn')
end
end
+
+ context 'with podcast elements' do
+ before do
+ @valid_options['url'] = 'http://example.com/podcast.rss'
+ @valid_options['include_feed_info'] = true
+ end
+
+ let :feed_info do
+ {
+ "id" => nil,
+ "type" => "rss",
+ "url" => "http://www.example.com/podcasts/everything/index.html",
+ "links" => [ { "href" => "http://www.example.com/podcasts/everything/index.html" } ],
+ "title" => "All About Everything",
+ "description" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
+ "copyright" => "℗ & © 2014 John Doe & Family",
+ "generator" => nil,
+ "icon" => nil,
+ "authors" => [
+ "John Doe"
+ ],
+ "date_published" => nil,
+ "last_updated" => nil,
+ "itunes_categories" => [
+ "Technology", "Gadgets",
+ "TV & Film",
+ "Arts", "Food"
+ ],
+ "itunes_complete" => "yes",
+ "itunes_explicit" => "no",
+ "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything.jpg",
+ "itunes_owners" => ["John Doe <john.doe@example.com>"],
+ "itunes_subtitle" => "A show about everything",
+ "itunes_summary" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
+ "language" => "en-us"
+ }
+ end
+
+ it "is parsed correctly" do
+ expect {
+ agent.check
+ }.to change { agent.events.count }.by(4)
+
+ expect(agent.events.map(&:payload)).to match([
+ {
+ "feed" => feed_info,
+ "id" => "http://example.com/podcasts/archive/aae20140601.mp3",
+ "url" => nil,
+ "urls" => [],
+ "links" => [],
+ "title" => "Red,Whine, & Blue",
+ "description" => nil,
+ "content" => nil,
+ "image" => nil,
+ "enclosure" => {
+ "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3",
+ "type" => "audio/mpeg",
+ "length" => "498537"
+ },
+ "authors" => ["<Various>"],
+ "categories" => [],
+ "date_published" => "2016-03-11T01:15:00+00:00",
+ "last_updated" => "2016-03-11T01:15:00+00:00",
+ "itunes_duration" => "03:59",
+ "itunes_explicit" => "no",
+ "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg",
+ "itunes_subtitle" => "Red + Blue != Purple",
+ "itunes_summary" => "This week we talk about surviving in a Red state if you are a Blue person. Or vice versa."
+ },
+ {
+ "feed" => feed_info,
+ "id" => "http://example.com/podcasts/archive/aae20140697.m4v",
+ "url" => nil,
+ "urls" => [],
+ "links" => [],
+ "title" => "The Best Chili",
+ "description" => nil,
+ "content" => nil,
+ "image" => nil,
+ "enclosure" => {
+ "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v",
+ "type" => "video/x-m4v",
+ "length" => "5650889"
+ },
+ "authors" => ["Jane Doe"],
+ "categories" => [],
+ "date_published" => "2016-03-10T02:00:00-07:00",
+ "last_updated" => "2016-03-10T02:00:00-07:00",
+ "itunes_closed_captioned" => "Yes",
+ "itunes_duration" => "04:34",
+ "itunes_explicit" => "no",
+ "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg",
+ "itunes_subtitle" => "Jane and Eric",
+ "itunes_summary" => "This week we talk about the best Chili in the world. Which chili is better?"
+ },
+ {
+ "feed" => feed_info,
+ "id" => "http://example.com/podcasts/archive/aae20140608.mp4",
+ "url" => nil,
+ "urls" => [],
+ "links" => [],
+ "title" => "Socket Wrench Shootout",
+ "description" => nil,
+ "content" => nil,
+ "image" => nil,
+ "enclosure" => {
+ "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4",
+ "type" => "video/mp4",
+ "length" => "5650889"
+ },
+ "authors" => ["Jane Doe"],
+ "categories" => [],
+ "date_published" => "2016-03-09T13:00:00-05:00",
+ "last_updated" => "2016-03-09T13:00:00-05:00",
+ "itunes_duration" => "04:34",
+ "itunes_explicit" => "no",
+ "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg",
+ "itunes_subtitle" => "Comparing socket wrenches is fun!",
+ "itunes_summary" => "This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here."
+ },
+ {
+ "feed" => feed_info,
+ "id" => "http://example.com/podcasts/archive/aae20140615.m4a",
+ "url" => nil,
+ "urls" => [],
+ "links" => [],
+ "title" => "Shake Shake Shake Your Spices",
+ "description" => nil,
+ "content" => nil,
+ "image" => nil,
+ "enclosure" => {
+ "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a",
+ "type" => "audio/x-m4a",
+ "length" => "8727310"
+ },
+ "authors" => ["John Doe"],
+ "categories" => [],
+ "date_published" => "2016-03-08T12:00:00+00:00",
+ "last_updated" => "2016-03-08T12:00:00+00:00",
+ "itunes_duration" => "07:04",
+ "itunes_explicit" => "no",
+ "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg",
+ "itunes_subtitle" => "A short primer on table spices",
+ "itunes_summary" => "This week we talk about <a href=\"https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11\">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!"
+ }
+ ])
+ end
+ end
end
describe 'logging errors with the feed url' do
From e26c07e75bd996ab27673aac67914c423204e7cf Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 30 Nov 2016 01:24:36 +0900
Subject: [PATCH 13/23] Fix a merge conflict
---
spec/models/agents/rss_agent_spec.rb | 3 ---
1 file changed, 3 deletions(-)
diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index ac651166..399cf819 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -12,11 +12,8 @@ describe Agents::RssAgent do
stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
-<<<<<<< HEAD
stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
-=======
stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
->>>>>>> Add podcast tags to events emitted by RssAgent
end
let(:agent) do
From 15347c8a3ce7a6e33db98835619b161172960cc7 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 13 Dec 2016 18:08:16 +0900
Subject: [PATCH 14/23] Feedjira 2.1.0 is out, yay!
---
Gemfile | 2 +-
Gemfile.lock | 17 ++++++-----------
2 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/Gemfile b/Gemfile
index ffddd76a..a83fc492 100644
--- a/Gemfile
+++ b/Gemfile
@@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
gem 'faraday', '~> 0.9.0'
gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
-gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1']
+gem 'feedjira', '~> 2.1'
gem 'font-awesome-sass', '~> 4.3.2'
gem 'foreman', '~> 0.63.0'
gem 'geokit', '~> 1.8.4'
diff --git a/Gemfile.lock b/Gemfile.lock
index 2b871768..ba055445 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,16 +37,6 @@ GIT
oauth2 (~> 1)
rest-client (~> 1.8)
-GIT
- remote: git://github.com/feedjira/feedjira.git
- revision: 849b7809455c084b06f9c4c2d368a752677882b0
- specs:
- feedjira (2.0.0)
- faraday (>= 0.9)
- faraday_middleware (>= 0.9)
- loofah (>= 2.0)
- sax-machine (>= 1.0)
-
GIT
remote: git://github.com/lostisland/faraday_middleware.git
revision: c5836ae55857272732b33eb0e0a98d60e995a376
@@ -224,6 +214,11 @@ GEM
extlib (0.9.16)
faraday (0.9.2)
multipart-post (>= 1.2, < 3)
+ feedjira (2.1.0)
+ faraday (>= 0.9)
+ faraday_middleware (>= 0.9)
+ loofah (>= 2.0)
+ sax-machine (>= 1.0)
ffi (1.9.10)
font-awesome-sass (4.3.2.1)
sass (~> 3.2)
@@ -624,7 +619,7 @@ DEPENDENCIES
evernote_oauth
faraday (~> 0.9.0)
faraday_middleware!
- feedjira!
+ feedjira (~> 2.1)
ffi (>= 1.9.4)
font-awesome-sass (~> 4.3.2)
forecast_io (~> 2.0.0)
From fd18e1453883768e2bf848a055528fcf82740243 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 13 Dec 2016 19:31:32 +0900
Subject: [PATCH 15/23] Fast-forward faraday_middleware
The version constraint on faraday is relaxed while at it, although
some gems still have the strict constraint.
---
Gemfile | 2 +-
Gemfile.lock | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/Gemfile b/Gemfile
index a83fc492..f6fec106 100644
--- a/Gemfile
+++ b/Gemfile
@@ -90,7 +90,7 @@ gem 'delayed_job', '~> 4.1.0'
gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails5'
gem 'devise','~> 4.2.0'
gem 'em-http-request', '~> 1.1.2'
-gem 'faraday', '~> 0.9.0'
+gem 'faraday', '~> 0.9'
gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master' # '>= 0.10.1'
gem 'feedjira', '~> 2.1'
gem 'font-awesome-sass', '~> 4.3.2'
diff --git a/Gemfile.lock b/Gemfile.lock
index ba055445..ee694741 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -39,11 +39,11 @@ GIT
GIT
remote: git://github.com/lostisland/faraday_middleware.git
- revision: c5836ae55857272732b33eb0e0a98d60e995a376
+ revision: 59088da02940d0ee2010b2e3156343346767c31e
branch: master
specs:
faraday_middleware (0.10.0)
- faraday (>= 0.7.4, < 0.10)
+ faraday (>= 0.7.4, < 1.0)
GIT
remote: git://github.com/tumblr/tumblr_client.git
@@ -617,7 +617,7 @@ DEPENDENCIES
dropbox-api
em-http-request (~> 1.1.2)
evernote_oauth
- faraday (~> 0.9.0)
+ faraday (~> 0.9)
faraday_middleware!
feedjira (~> 2.1)
ffi (>= 1.9.4)
From 3159b547c7854365455b194233baf562f9d2f8e2 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Mon, 19 Dec 2016 14:51:32 +0900
Subject: [PATCH 16/23] Use the builtin mysql on Travis CI
---
.travis.yml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 1d4abefa..eeb4f722 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ sudo: required
language: ruby
services:
- docker
+ - mysql
- postgresql
env:
global:
@@ -37,9 +38,6 @@ rvm:
- 2.3.1
cache: bundler
bundler_args: --without development production
-before_install:
- - sudo apt-get -qq update
- - sudo apt-get install -y mysql-server
script:
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi
- if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi
From 2a524abff57d3d17f8e69d31b66e7f3997560470 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 20 Dec 2016 10:26:39 +0900
Subject: [PATCH 17/23] Fix the regex pattern to reduce backtracking
This should fix #1832.
---
app/models/agents/rss_agent.rb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb
index 0ab5719c..8e4a3a83 100644
--- a/app/models/agents/rss_agent.rb
+++ b/app/models/agents/rss_agent.rb
@@ -206,7 +206,7 @@ module Agents
else
# Encoding is already known, so do not let the parser detect
# it from the XML declaration in the content.
- body.sub!(/(\A\u{FEFF}?\s*<\?xml(?:\s+\w+\s*=\s*(['"]).*?\2)*)\s+encoding\s*=\s*(['"]).*?\3/, '\\1')
+ body.sub!(/(?<noenc>\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?<av>\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g<av>/, '\\k<noenc>')
end
body
end
From 4e2d1775a6f1fc9ae8f6c7a6fe212813b8d7b5df Mon Sep 17 00:00:00 2001
From: Irfan Charania <irfancharania@users.noreply.github.com>
Date: Tue, 20 Dec 2016 09:44:54 -0800
Subject: [PATCH 18/23] PhantomJs Cloud Agent (#1503)
* Initial draft of PhantomJsCloudAgent
Generates event with url for fetching html/plainText content
* Add options
* Pass in event instead of url
Fix hash syntax
Remove whitespace
Add mode merge
* Add some tests
* Style changes
- Add link to wiki entry for manually creating agent with full set of
options
---
app/models/agents/phantom_js_cloud_agent.rb | 162 ++++++++++++++++++
.../agents/phantom_js_cloud_agent_spec.rb | 117 +++++++++++++
2 files changed, 279 insertions(+)
create mode 100644 app/models/agents/phantom_js_cloud_agent.rb
create mode 100644 spec/models/agents/phantom_js_cloud_agent_spec.rb
diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb
new file mode 100644
index 00000000..f0c9e605
--- /dev/null
+++ b/app/models/agents/phantom_js_cloud_agent.rb
@@ -0,0 +1,162 @@
+require 'json'
+require 'uri'
+
+module Agents
+ class PhantomJsCloudAgent < Agent
+ include ERB::Util
+ include FormConfigurable
+ include WebRequestConcern
+
+ can_dry_run!
+
+ default_schedule 'every_12h'
+
+ description <<-MD
+ [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages.
+
+ The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
+ This url can then be supplied to Website Agent to fetch and parse content.
+
+ [Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
+
+
+ Options:
+
+ * `Api key` - PhantomJs Cloud API Key credential stored in Huginn
+ * `Url` - The url to render
+ * `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`)
+ * `Render type` - Render as html or plain text without html tags (default: `html`)
+ * `Output as json` - Return the page conents and metadata as a JSON object (default: `false`)
+ * `Ignore images` - Skip loading of inlined images (default: `false`)
+ * `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`)
+ * `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading.
+ This is useful in case there are any AJAX requests or animations that need to finish up.
+ This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
+
+
+ As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
+
+ MD
+
+ event_description <<-MD
+ Events look like this:
+ {
+ "url": "..."
+ }
+ MD
+
+ def default_options
+ {
+ 'mode' => 'clean',
+ 'url' => 'http://xkcd.com',
+ 'render_type' => 'html',
+ 'output_as_json' => false,
+ 'ignore_images' => false,
+ 'user_agent' => self.class.default_user_agent,
+ 'wait_interval' => '1000'
+ }
+ end
+
+ form_configurable :mode, type: :array, values: ['clean', 'merge']
+ form_configurable :api_key, roles: :completable
+ form_configurable :url
+ form_configurable :render_type, type: :array, values: ['html', 'plainText']
+ form_configurable :output_as_json, type: :boolean
+ form_configurable :ignore_images, type: :boolean
+ form_configurable :user_agent, type: :text
+ form_configurable :wait_interval
+
+ def mode
+ interpolated['mode'].presence || default_options['mode']
+ end
+
+ def render_type
+ interpolated['render_type'].presence || default_options['render_type']
+ end
+
+ def output_as_json
+ boolify(interpolated['output_as_json'].presence ||
+ default_options['output_as_json'])
+ end
+
+ def ignore_images
+ boolify(interpolated['ignore_images'].presence ||
+ default_options['ignore_images'])
+ end
+
+ def user_agent
+ interpolated['user_agent'].presence || self.class.default_user_agent
+ end
+
+ def wait_interval
+ interpolated['wait_interval'].presence || default_options['wait_interval']
+ end
+
+ def page_request_settings
+ prs = {}
+
+ prs[:ignoreImages] = ignore_images if ignore_images
+ prs[:userAgent] = user_agent if user_agent.present?
+
+ if wait_interval != default_options['wait_interval']
+ prs[:wait_interval] = wait_interval
+ end
+
+ prs
+ end
+
+ def build_phantom_url(interpolated)
+ api_key = interpolated[:api_key]
+ page_request_hash = {
+ url: interpolated[:url],
+ renderType: render_type
+ }
+
+ page_request_hash[:outputAsJson] = output_as_json if output_as_json
+
+ page_request_settings_hash = page_request_settings
+
+ if page_request_settings_hash.any?
+ page_request_hash[:requestSettings] = page_request_settings_hash
+ end
+
+ request = page_request_hash.to_json
+ log "Generated request: #{request}"
+
+ encoded = url_encode(request)
+ "https://phantomjscloud.com/api/browser/v2/#{api_key}/?request=#{encoded}"
+ end
+
+ def check
+ phantom_url = build_phantom_url(interpolated)
+
+ create_event payload: { 'url' => phantom_url }
+ end
+
+ def receive(incoming_events)
+ incoming_events.each do |event|
+ interpolate_with(event) do
+ existing_payload = interpolated['mode'].to_s == 'merge' ? event.payload : {}
+ phantom_url = build_phantom_url(interpolated)
+
+ result = { 'url' => phantom_url }
+ create_event payload: existing_payload.merge(result)
+ end
+ end
+ end
+
+ def complete_api_key
+ user.user_credentials.map { |c| { text: c.credential_name, id: "{% credential #{c.credential_name} %}" } }
+ end
+
+ def working?
+ !recent_error_logs? || received_event_without_error?
+ end
+
+ def validate_options
+ # Check for required fields
+ errors.add(:base, 'Url is required') unless options['url'].present?
+ errors.add(:base, 'API key (credential) is required') unless options['api_key'].present?
+ end
+ end
+end
diff --git a/spec/models/agents/phantom_js_cloud_agent_spec.rb b/spec/models/agents/phantom_js_cloud_agent_spec.rb
new file mode 100644
index 00000000..07a79c2b
--- /dev/null
+++ b/spec/models/agents/phantom_js_cloud_agent_spec.rb
@@ -0,0 +1,117 @@
+require 'rails_helper'
+
+describe Agents::PhantomJsCloudAgent do
+ before do
+
+ @valid_options = {
+ 'name' => "XKCD",
+ 'render_type' => "html",
+ 'url' => "http://xkcd.com",
+ 'mode' => 'clean',
+ 'api_key' => '1234567890'
+ }
+
+ @checker = Agents::PhantomJsCloudAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
+ @checker.user = users(:jane)
+ @checker.save!
+ end
+
+ describe "validations" do
+ before do
+ expect(@checker).to be_valid
+ end
+
+ it "should validate the presence of url" do
+ @checker.options['url'] = "http://google.com"
+ expect(@checker).to be_valid
+
+ @checker.options['url'] = ""
+ expect(@checker).not_to be_valid
+
+ @checker.options['url'] = nil
+ expect(@checker).not_to be_valid
+ end
+
+ end
+
+ describe "emitting event" do
+ it "should emit url as event" do
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+ end
+
+ it "should set render type as plain text" do
+ @checker.options['render_type'] = 'plainText'
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22plainText%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+ end
+
+ it "should set output as json" do
+ @checker.options['output_as_json'] = true
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22outputAsJson%22%3Atrue%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+ end
+
+ it "should not set ignore images" do
+ @checker.options['ignore_images'] = false
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+ end
+
+ it "should set ignore images" do
+ @checker.options['ignore_images'] = true
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22ignoreImages%22%3Atrue%2C%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+ end
+
+ it "should set wait interval to zero" do
+ @checker.options['wait_interval'] = '0'
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%2C%22wait_interval%22%3A%220%22%7D%7D")
+ end
+
+ it "should set user agent to BlackBerry" do
+ @checker.options['user_agent'] = 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+'
+
+ expect {
+ @checker.check
+ }.to change { @checker.events.count }.by(1)
+
+ item,* = @checker.events.last(1)
+ expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Mozilla%2F5.0%20%28BlackBerry%3B%20U%3B%20BlackBerry%209900%3B%20en%29%20AppleWebKit%2F534.11%2B%20%28KHTML%2C%20like%20Gecko%29%20Version%2F7.1.0.346%20Mobile%20Safari%2F534.11%2B%22%7D%7D")
+ end
+
+
+
+ end
+
+end
From 65cea03062dfc4aa7d0ebdacf2edde5a78e75f82 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@users.noreply.github.com>
Date: Tue, 20 Dec 2016 13:32:48 -0500
Subject: [PATCH 19/23] Fix spec failures
---
app/models/agents/phantom_js_cloud_agent.rb | 9 ++++-----
spec/features/create_an_agent_spec.rb | 9 ++++-----
2 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb
index f0c9e605..727de10e 100644
--- a/app/models/agents/phantom_js_cloud_agent.rb
+++ b/app/models/agents/phantom_js_cloud_agent.rb
@@ -12,13 +12,14 @@ module Agents
default_schedule 'every_12h'
description <<-MD
- [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages.
+ This Agent generates [PhantomJs Cloud](https://phantomjscloud.com/) URLs that can be used to render JavaScript-heavy webpages for content extraction.
- The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
- This url can then be supplied to Website Agent to fetch and parse content.
+ URLs generated by this Agent are formulated in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
+ The generated URLs can then be supplied to a Website Agent to fetch and parse the content.
[Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
+ Please see the [Huginn Wiki for more info](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud).
Options:
@@ -33,9 +34,7 @@ module Agents
This is useful in case there are any AJAX requests or animations that need to finish up.
This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
-
As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
-
MD
event_description <<-MD
diff --git a/spec/features/create_an_agent_spec.rb b/spec/features/create_an_agent_spec.rb
index ac2ce295..ae725630 100644
--- a/spec/features/create_an_agent_spec.rb
+++ b/spec/features/create_an_agent_spec.rb
@@ -38,7 +38,7 @@ describe "Creating a new agent", js: true do
visit new_agent_path
end
it "shows all options for agents that can be scheduled, create and receive events" do
- select2("Website Agent", from: "Type")
+ select2("Website Agent scrapes", from: "Type")
expect(page).not_to have_content('This type of Agent cannot create events.')
end
@@ -49,9 +49,8 @@ describe "Creating a new agent", js: true do
end
it "allows to click on on the agent name in select2 tags" do
- agent = agents(:bob_weather_agent)
visit new_agent_path
- select2("Website Agent", from: "Type")
+ select2("Website Agent scrapes", from: "Type")
select2("SF Weather", from: 'Sources')
click_on "SF Weather"
expect(page).to have_content "Editing your WeatherAgent"
@@ -63,7 +62,7 @@ describe "Creating a new agent", js: true do
end
it "does not send previously configured sources when the current agent does not support them" do
- select2("Website Agent", from: "Type")
+ select2("Website Agent scrapes", from: "Type")
select2("SF Weather", from: 'Sources')
select2("Webhook Agent", from: "Type")
fill_in(:agent_name, with: "No sources")
@@ -85,7 +84,7 @@ describe "Creating a new agent", js: true do
end
it "does not send previously configured receivers when the current agent does not support them" do
- select2("Website Agent", from: "Type")
+ select2("Website Agent scrapes", from: "Type")
select2("ZKCD", from: 'Receivers')
select2("Email Agent", from: "Type")
fill_in(:agent_name, with: "No receivers")
From db4de696b4d71169ca9733ce0fd473a5cdc7c334 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Sun, 27 Nov 2016 12:30:45 +0100
Subject: [PATCH 20/23] Override `git_source` method in the Gemfile
When the bundler version is below 2 override `git_source` to ensure
`github` remotes always use HTTPS
---
Gemfile | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/Gemfile b/Gemfile
index f6fec106..95608b5b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -3,6 +3,12 @@ source 'https://rubygems.org'
# Ruby 2.2.2 is the minimum requirement
ruby ['2.2.2', RUBY_VERSION].max
+# Ensure github repositories are fetched using HTTPS
+git_source(:github) do |repo_name|
+ repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
+ "https://github.com/#{repo_name}.git"
+end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2')
+
# Load vendored dotenv gem and .env file
require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb')
GemfileHelper.load_dotenv do |dotenv_dir|
@@ -50,7 +56,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn'
gem 'omniauth-twitter', '~> 1.2.1'
# Tumblr Agents
-gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master' # '>= 0.8.5'
+gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master', ref: '0c59b04e49f2a8c89860613b18cf4e8f978d8dc7' # '>= 0.8.5'
gem 'omniauth-tumblr', '~> 1.2'
# Dropbox Agents
@@ -103,7 +109,7 @@ gem 'jquery-rails', '~> 4.2.1'
gem 'huginn_agent', '~> 0.4.0'
gem 'json', '~> 1.8.1'
gem 'jsonpathv2', '~> 0.0.8'
-gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable'
+gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable', ref: 'abbf93d557208ee1d0b612c612cd079f86ed54f4'
gem 'kramdown', '~> 1.3.3'
gem 'liquid', '~> 3.0.3'
gem 'loofah', '~> 2.0'
From f813a73dd067ee33dedd9fe46ffc6569a960ae3b Mon Sep 17 00:00:00 2001
From: Alex Jordan <alex@strugee.net>
Date: Mon, 31 Oct 2016 18:39:34 -0700
Subject: [PATCH 21/23] Update Gemfile.lock with HTTPS URLs for git gems
---
Gemfile.lock | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/Gemfile.lock b/Gemfile.lock
index ee694741..1542b084 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,7 @@
GIT
- remote: git://github.com/amatsuda/kaminari.git
+ remote: https://github.com/amatsuda/kaminari.git
revision: abbf93d557208ee1d0b612c612cd079f86ed54f4
+ ref: abbf93d557208ee1d0b612c612cd079f86ed54f4
branch: 0-17-stable
specs:
kaminari (0.17.0)
@@ -8,7 +9,7 @@ GIT
activesupport (>= 3.0.0)
GIT
- remote: git://github.com/cantino/twitter-stream.git
+ remote: https://github.com/cantino/twitter-stream.git
revision: f7e7edb0bae013bffabf3598e7147773d9fd370f
branch: huginn
specs:
@@ -18,7 +19,7 @@ GIT
simple_oauth (~> 0.3.0)
GIT
- remote: git://github.com/dsander/delayed_job_active_record.git
+ remote: https://github.com/dsander/delayed_job_active_record.git
revision: b314972ccc92e0e8b03b1589174d8fb9a82b3cd0
branch: rails5
specs:
@@ -27,7 +28,7 @@ GIT
delayed_job (>= 3.0, < 5)
GIT
- remote: git://github.com/dsander/weibo_2.git
+ remote: https://github.com/dsander/weibo_2.git
revision: e5b77f21a7e9a666b582c48e16b1e96fca198cf8
branch: master
specs:
@@ -38,7 +39,7 @@ GIT
rest-client (~> 1.8)
GIT
- remote: git://github.com/lostisland/faraday_middleware.git
+ remote: https://github.com/lostisland/faraday_middleware.git
revision: 59088da02940d0ee2010b2e3156343346767c31e
branch: master
specs:
@@ -46,8 +47,9 @@ GIT
faraday (>= 0.7.4, < 1.0)
GIT
- remote: git://github.com/tumblr/tumblr_client.git
+ remote: https://github.com/tumblr/tumblr_client.git
revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
+ ref: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
branch: master
specs:
tumblr_client (0.8.5)
@@ -703,4 +705,4 @@ RUBY VERSION
ruby 2.3.3p222
BUNDLED WITH
- 1.13.6
+ 1.13.7
From 0285a1a9933db7b05ae7e0ead5c0e7660e72e731 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Tue, 27 Dec 2016 23:35:06 +0100
Subject: [PATCH 22/23] Update rails to 5.0.1
---
Gemfile | 4 +--
Gemfile.lock | 80 ++++++++++++++++++++++++++--------------------------
2 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/Gemfile b/Gemfile
index 95608b5b..016c72dd 100644
--- a/Gemfile
+++ b/Gemfile
@@ -117,7 +117,7 @@ gem 'mini_magick'
gem 'multi_xml'
gem 'nokogiri'
gem 'omniauth', '~> 1.3.1'
-gem 'rails', '~> 5.0.0.1'
+gem 'rails', '~> 5.0.1'
gem 'rufus-scheduler', '~> 3.0.8', require: false
gem 'sass-rails', '~> 5.0.6'
gem 'select2-rails', '~> 3.5.4'
@@ -134,7 +134,7 @@ group :development do
gem 'guard-rspec', '~> 4.6.4'
gem 'rack-livereload', '~> 0.3.16'
gem 'letter_opener_web', '~> 1.3.0'
- gem 'web-console'
+ gem 'web-console', '>= 3.3.0'
gem 'capistrano', '~> 3.4.0'
gem 'capistrano-rails', '~> 1.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1542b084..4bf8109d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -71,45 +71,45 @@ GEM
remote: https://rubygems.org/
specs:
ace-rails-ap (2.0.1)
- actioncable (5.0.0.1)
- actionpack (= 5.0.0.1)
+ actioncable (5.0.1)
+ actionpack (= 5.0.1)
nio4r (~> 1.2)
websocket-driver (~> 0.6.1)
- actionmailer (5.0.0.1)
- actionpack (= 5.0.0.1)
- actionview (= 5.0.0.1)
- activejob (= 5.0.0.1)
+ actionmailer (5.0.1)
+ actionpack (= 5.0.1)
+ actionview (= 5.0.1)
+ activejob (= 5.0.1)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
- actionpack (5.0.0.1)
- actionview (= 5.0.0.1)
- activesupport (= 5.0.0.1)
+ actionpack (5.0.1)
+ actionview (= 5.0.1)
+ activesupport (= 5.0.1)
rack (~> 2.0)
rack-test (~> 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- actionview (5.0.0.1)
- activesupport (= 5.0.0.1)
+ actionview (5.0.1)
+ activesupport (= 5.0.1)
builder (~> 3.1)
erubis (~> 2.7.0)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.0.2)
- activejob (5.0.0.1)
- activesupport (= 5.0.0.1)
+ activejob (5.0.1)
+ activesupport (= 5.0.1)
globalid (>= 0.3.6)
- activemodel (5.0.0.1)
- activesupport (= 5.0.0.1)
- activerecord (5.0.0.1)
- activemodel (= 5.0.0.1)
- activesupport (= 5.0.0.1)
+ activemodel (5.0.1)
+ activesupport (= 5.0.1)
+ activerecord (5.0.1)
+ activemodel (= 5.0.1)
+ activesupport (= 5.0.1)
arel (~> 7.0)
- activesupport (5.0.0.1)
+ activesupport (5.0.1)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.3.8)
- arel (7.1.1)
+ arel (7.1.4)
autoparse (0.3.3)
addressable (>= 2.3.1)
extlib (>= 0.9.15)
@@ -159,7 +159,7 @@ GEM
execjs
coffee-script-source (1.10.0)
colorize (0.7.7)
- concurrent-ruby (1.0.2)
+ concurrent-ruby (1.0.3)
cookiejar (0.3.2)
coveralls (0.7.12)
multi_json (~> 1.10)
@@ -329,7 +329,7 @@ GEM
mimemagic (0.3.1)
mini_magick (4.2.3)
mini_portile2 (2.1.0)
- minitest (5.9.0)
+ minitest (5.10.1)
mqtt (0.3.1)
multi_json (1.12.1)
multi_xml (0.5.5)
@@ -404,17 +404,17 @@ GEM
rack
rack-test (0.6.3)
rack (>= 1.0)
- rails (5.0.0.1)
- actioncable (= 5.0.0.1)
- actionmailer (= 5.0.0.1)
- actionpack (= 5.0.0.1)
- actionview (= 5.0.0.1)
- activejob (= 5.0.0.1)
- activemodel (= 5.0.0.1)
- activerecord (= 5.0.0.1)
- activesupport (= 5.0.0.1)
+ rails (5.0.1)
+ actioncable (= 5.0.1)
+ actionmailer (= 5.0.1)
+ actionpack (= 5.0.1)
+ actionview (= 5.0.1)
+ activejob (= 5.0.1)
+ activemodel (= 5.0.1)
+ activerecord (= 5.0.1)
+ activesupport (= 5.0.1)
bundler (>= 1.3.0, < 2.0)
- railties (= 5.0.0.1)
+ railties (= 5.0.1)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.1)
actionpack (~> 5.x)
@@ -425,14 +425,14 @@ GEM
nokogiri (~> 1.6.0)
rails-html-sanitizer (1.0.3)
loofah (~> 2.0)
- railties (5.0.0.1)
- actionpack (= 5.0.0.1)
- activesupport (= 5.0.0.1)
+ railties (5.0.1)
+ actionpack (= 5.0.1)
+ activesupport (= 5.0.1)
method_source
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
raindrops (0.17.0)
- rake (11.2.2)
+ rake (12.0.0)
rb-fsevent (0.9.7)
rb-inotify (0.9.5)
ffi (>= 0.5.0)
@@ -516,7 +516,7 @@ GEM
spring-watcher-listen (2.0.0)
listen (>= 2.7, < 4.0)
spring (~> 1.2)
- sprockets (3.7.0)
+ sprockets (3.7.1)
concurrent-ruby (~> 1.0)
rack (> 1, < 3)
sprockets-rails (3.2.0)
@@ -533,7 +533,7 @@ GEM
therubyracer (0.12.2)
libv8 (~> 3.16.14.0)
ref
- thor (0.19.1)
+ thor (0.19.4)
thread_safe (0.3.5)
tilt (2.0.5)
tins (1.10.1)
@@ -665,7 +665,7 @@ DEPENDENCIES
pry-byebug
pry-rails
rack-livereload (~> 0.3.16)
- rails (~> 5.0.0.1)
+ rails (~> 5.0.1)
rails-controller-testing
rb-kqueue (>= 0.2)
rr
@@ -695,7 +695,7 @@ DEPENDENCIES
uglifier (~> 2.7.2)
unicorn (~> 5.1.0)
vcr
- web-console
+ web-console (>= 3.3.0)
webmock (~> 1.17.4)
weibo_2!
wunderground (~> 1.2.0)
From 070946e879872de609524b6434cc170207cada12 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Tue, 27 Dec 2016 23:35:29 +0100
Subject: [PATCH 23/23] Work around possible regression in rails
`pluck` used to always trigger a database query. With Rails 5.0.1 it uses data from
already loaded associations, this also includes new records for which we can not
generate agent_paths
---
app/views/layouts/application.html.erb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index 561f9a94..69f81378 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -52,7 +52,7 @@
window.agentPaths = {};
window.agentNames = [];
<% if current_user.present? -%>
- var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| m[a.first] = agent_path(a.last); m }) %>;
+ var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| next if a.last.nil?; m[a.first] = agent_path(a.last); m }) %>;
var myScenarios = <%= Utils.jsonify(current_user.scenarios.pluck(:name, :id).inject({}) {|m, s| m[s.first + " Scenario"] = scenario_path(s.last); m }) %>;
$.extend(window.agentPaths, myAgents);
$.extend(window.agentPaths, myScenarios);