From 7ac691652b90893bc84d89f31d996edc290d5259 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sat, 26 Nov 2016 13:26:09 +0900
Subject: [PATCH 01/23] Spec that force_encoding works with encoding
 declaration in RssAgent

---
 spec/models/agents/rss_agent_spec.rb | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index a70565f8..31c6737e 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -295,6 +295,13 @@ describe Agents::RssAgent do
         event = agent.events.first
         expect(event.payload['title']).to eq('Mëkanïk Zaïn')
       end
+
+      it "decodes the content properly with force_encoding specified" do
+        @valid_options['force_encoding'] = 'iso-8859-1'
+        agent.check
+        event = agent.events.first
+        expect(event.payload['title']).to eq('Mëkanïk Zaïn')
+      end
     end
   end
 

From f530305edc39b730b9a8a607bbc61589c60da4a9 Mon Sep 17 00:00:00 2001
From: bobbysteel <github@robk.com>
Date: Sat, 26 Nov 2016 20:13:00 +0000
Subject: [PATCH 02/23] Add class of service chooser for Google Flights Agent
 (#1778)

* Add class of service chooser

* Add cabin chooser test

* Fix preferredCabin

* Per @cantino feedback taking out check
---
 app/models/agents/google_flights_agent.rb | 6 ++++--
 spec/models/agents/google_flights_spec.rb | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/app/models/agents/google_flights_agent.rb b/app/models/agents/google_flights_agent.rb
index f7a82521..ccdc66b4 100644
--- a/app/models/agents/google_flights_agent.rb
+++ b/app/models/agents/google_flights_agent.rb
@@ -60,6 +60,7 @@ module Agents
         'seniorCount'=> 0,
         'return_date' => '2016-04-18',
         'roundtrip' => true,
+        'preferredCabin' => 'COACH',
         'solutions'=> 3
       }
     end
@@ -69,6 +70,7 @@ module Agents
     form_configurable :origin, type: :string
     form_configurable :destination, type: :string
     form_configurable :date, type: :string
+    form_configurable :preferredCabin, type: :array, values: %w(COACH PREMIUM_COACH BUSINESS FIRST)
     form_configurable :childCount
     form_configurable :infantInSeatCount
     form_configurable :infantInLapCount
@@ -101,9 +103,9 @@ module Agents
 
     def post_params
       if round_trip?
-        post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s } ], :solutions=> interpolated["solutions"]}}
+        post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[ {:origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }, {:origin=> interpolated["destination"].to_s , :destination=> interpolated["origin"].to_s , :date=> interpolated["return_date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s} ], :solutions=> interpolated["solutions"]}}
       else
-        post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s }], :solutions=> interpolated["solutions"]}}
+        post_params = {:request=>{:passengers=>{:kind=>"qpxexpress#passengerCounts", :adultCount=> interpolated["adultCount"], :childCount=> interpolated["childCount"], :infantInLapCount=>interpolated["infantInLapCount"], :infantInSeatCount=>interpolated['infantInSeatCount'], :seniorCount=>interpolated["seniorCount"]}, :slice=>[{:kind=>"qpxexpress#sliceInput", :origin=> interpolated["origin"].to_s , :destination=> interpolated["destination"].to_s , :date=> interpolated["date"].to_s , :preferredCabin=> interpolated["preferredCabin"].to_s }], :solutions=> interpolated["solutions"]}}
       end
     end
 
diff --git a/spec/models/agents/google_flights_spec.rb b/spec/models/agents/google_flights_spec.rb
index 28f5994d..a0b970b8 100644
--- a/spec/models/agents/google_flights_spec.rb
+++ b/spec/models/agents/google_flights_spec.rb
@@ -15,6 +15,7 @@ describe Agents::GoogleFlightsAgent do
       'origin' => 'BOS',
       'destination' => 'SFO',
       'date' => '2016-04-11',
+      'preferredCabin' => 'COACH',
       'childCount' => 0,
       'infantInSeatCount' => 0,
       'infantInLapCount'=> 0,

From 3d91469733bf2b0c0d00815a33faf0ac688e18ee Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 13:33:24 +0900
Subject: [PATCH 03/23] Make WebsiteAgent merge `template` with the results of
 `extract` (#1816)

A new extraction option `hidden` is added so that keys with it gets
excluded from the final payloads while they can be used in `template`.
---
 app/models/agents/website_agent.rb            | 82 +++++++++++------
 ...onvert_website_agent_template_for_merge.rb | 36 ++++++++
 ...t_website_agent_template_for_merge_spec.rb | 92 +++++++++++++++++++
 spec/models/agents/website_agent_spec.rb      | 23 ++++-
 4 files changed, 201 insertions(+), 32 deletions(-)
 create mode 100644 db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
 create mode 100644 spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb

diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index 437df864..f36d70e3 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -37,6 +37,8 @@ module Agents
 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor except when it has `repeat` set to true.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
 
+      For extractors with `hidden` set to true, they will be excluded from the payloads of events created by the Agent, but can be used and interpolated in the `template` option explained below.
+
       For extractors with `repeat` set to true, their first matches will be included in all extracts.  This is useful such as when you want to include the title of a page in all events created from the page.
 
       # Scraping HTML and XML
@@ -116,11 +118,9 @@ module Agents
 
       Set `http_success_codes` to an array of status codes (e.g., `[404, 422]`) to treat HTTP response codes beyond 200 as successes.
 
-      If a `template` option is given, it is used as a Liquid template for each event created by this Agent, instead of directly emitting the results of extraction as events.  In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section.  For example:
+      If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload.  In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section.  For example:
 
           "template": {
-            "url": "{{ url }}",
-            "title": "{{ title }}",
             "description": "{{ body_text }}",
             "last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
           }
@@ -159,7 +159,11 @@ module Agents
     end
 
     def event_keys
-      (options['template'].presence || options['extract']).try(:keys)
+      extract = options['extract'] or return nil
+
+      extract.each_with_object([]) { |(key, value), keys|
+        keys << key unless boolify(value['hidden'])
+      } | (options['template'].presence.try!(:keys) || [])
     end
 
     def working?
@@ -382,33 +386,19 @@ module Agents
             extract_xml(doc)
         end
 
-      num_tuples = output.each_value.inject(nil) { |num, value|
-        case size = value.size
-        when Float::INFINITY
-          num
-        when Integer
-          if num && num != size
-            raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
-          end
-          size
-        end
-      } or raise "At least one non-repeat key is required"
+      num_tuples = output.size or
+        raise "At least one non-repeat key is required"
 
       old_events = previous_payloads num_tuples
 
       template = options['template'].presence
 
-      num_tuples.times.zip(*output.values) do |index, *values|
-        extracted = output.each_key.lazy.zip(values).to_h
+      output.each do |extracted|
+        result = extracted.except(*output.hidden_keys)
 
-        result =
-          if template
-            interpolate_with(extracted) do
-              interpolate_options(template)
-            end
-          else
-            extracted
-          end
+        if template
+          result.update(interpolate_options(template, extracted))
+        end
 
         # url may be URI, string or nil
         if (payload_url = result['url'].presence) && (url = url.presence)
@@ -528,7 +518,7 @@ module Agents
     end
 
     def extract_each(&block)
-      interpolated['extract'].each_with_object({}) { |(name, extraction_details), output|
+      interpolated['extract'].each_with_object(Output.new) { |(name, extraction_details), output|
         if boolify(extraction_details['repeat'])
           values = Repeater.new { |repeater|
             block.call(extraction_details, repeater)
@@ -538,7 +528,13 @@ module Agents
           block.call(extraction_details, values)
         end
         log "Values extracted: #{values}"
-        output[name] = values
+        begin
+          output[name] = values
+        rescue UnevenSizeError
+          raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
+        else
+          output.hidden_keys << name if boolify(extraction_details['hidden'])
+        end
       }
     end
 
@@ -617,6 +613,38 @@ module Agents
       false
     end
 
+    class UnevenSizeError < ArgumentError
+    end
+
+    class Output
+      def initialize
+        @hash = {}
+        @size = nil
+        @hidden_keys = []
+      end
+
+      attr_reader :size
+      attr_reader :hidden_keys
+
+      def []=(key, value)
+        case size = value.size
+        when Integer
+          if @size && @size != size
+            raise UnevenSizeError, 'got an uneven size'
+          end
+          @size = size
+        end
+
+        @hash[key] = value
+      end
+
+      def each
+        @size.times.zip(*@hash.values) do |index, *values|
+          yield @hash.each_key.lazy.zip(values).to_h
+        end
+      end
+    end
+
     class Repeater < Enumerator
       # Repeater.new { |y|
       #   # ...
diff --git a/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
new file mode 100644
index 00000000..65b22f4d
--- /dev/null
+++ b/db/migrate/20161124061256_convert_website_agent_template_for_merge.rb
@@ -0,0 +1,36 @@
+class ConvertWebsiteAgentTemplateForMerge < ActiveRecord::Migration[5.0]
+  def up
+    Agents::WebsiteAgent.find_each do |agent|
+      extract = agent.options['extract'].presence
+      template = agent.options['template'].presence
+      next unless extract.is_a?(Hash) && template.is_a?(Hash)
+
+      (extract.keys - template.keys).each do |key|
+        extract[key]['hidden'] = true
+      end
+
+      template.delete_if { |key, value|
+        extract.key?(key) &&
+          value.match(/\A\{\{\s*#{Regexp.quote(key)}\s*\}\}\z/)
+      }
+
+      agent.save!(validate: false)
+    end
+  end
+
+  def down
+    Agents::WebsiteAgent.find_each do |agent|
+      extract = agent.options['extract'].presence
+      template = agent.options['template'].presence
+      next unless extract.is_a?(Hash) && template.is_a?(Hash)
+
+      (extract.keys - template.keys).each do |key|
+        unless extract[key].delete('hidden').in?([true, 'true'])
+          template[key] = "{{ #{key} }}"
+        end
+      end
+
+      agent.save!(validate: false)
+    end
+  end
+end
diff --git a/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb
new file mode 100644
index 00000000..af6c137f
--- /dev/null
+++ b/spec/migrations/20161124061256_convert_website_agent_template_for_merge_spec.rb
@@ -0,0 +1,92 @@
+load 'spec/rails_helper.rb'
+load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
+
+describe ConvertWebsiteAgentTemplateForMerge do
+  let :old_extract do
+    {
+      'url' => { 'css' => "#comic img", 'value' => "@src" },
+      'title' => { 'css' => "#comic img", 'value' => "@alt" },
+      'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
+    }
+  end
+
+  let :new_extract do
+    {
+      'url' => { 'css' => "#comic img", 'value' => "@src" },
+      'title' => { 'css' => "#comic img", 'value' => "@alt" },
+      'hovertext' => { 'css' => "#comic img", 'value' => "@title", 'hidden' => true }
+    }
+  end
+
+  let :reverted_extract do
+    old_extract
+  end
+
+  let :old_template do
+    {
+      'url' => '{{url}}',
+      'title' => '{{ title }}',
+      'description' => '{{ hovertext }}',
+      'comment' => '{{ comment }}'
+    }
+  end
+
+  let :new_template do
+    {
+      'description' => '{{ hovertext }}',
+      'comment' => '{{ comment }}'
+    }
+  end
+
+  let :reverted_template do
+    old_template.merge('url' => '{{ url }}')
+  end
+
+  let :valid_options do
+    {
+      'name' => "XKCD",
+      'expected_update_period_in_days' => "2",
+      'type' => "html",
+      'url' => "{{ url | default: 'http://xkcd.com/' }}",
+      'mode' => 'on_change',
+      'extract' => old_extract,
+      'template' => old_template
+    }
+  end
+
+  let :agent do
+    Agents::WebsiteAgent.create!(
+      user: users(:bob),
+      name: "xkcd",
+      options: valid_options,
+      keep_events_for: 2.days
+    )
+  end
+
+  describe 'up' do
+    it 'should update extract and template options for an existing WebsiteAgent' do
+      expect(agent.options).to include('extract' => old_extract,
+                                       'template' => old_template)
+      ConvertWebsiteAgentTemplateForMerge.new.up
+      agent.reload
+      expect(agent.options).to include('extract' => new_extract,
+                                       'template' => new_template)
+    end
+  end
+
+  describe 'down' do
+    let :valid_options do
+      super().merge('extract' => new_extract,
+                    'template' => new_template)
+    end
+
+    it 'should revert extract and template options for an updated WebsiteAgent' do
+      expect(agent.options).to include('extract' => new_extract,
+                                       'template' => new_template)
+      ConvertWebsiteAgentTemplateForMerge.new.down
+      agent.reload
+      expect(agent.options).to include('extract' => reverted_extract,
+                                       'template' => reverted_template)
+    end
+  end
+end
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 9f25752d..01d9f223 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -651,9 +651,22 @@ describe Agents::WebsiteAgent do
         @checker.options = @valid_options
         @checker.check
         event = Event.last
-        expect(event.payload['url']).to eq("http://imgs.xkcd.com/comics/evolving.png")
-        expect(event.payload['title']).to eq("Evolving")
-        expect(event.payload['hovertext']).to match(/^Biologists play reverse/)
+        expect(event.payload).to match(
+          'url' => 'http://imgs.xkcd.com/comics/evolving.png',
+          'title' => 'Evolving',
+          'hovertext' => /^Biologists play reverse/
+        )
+      end
+
+      it "should exclude hidden keys" do
+        @valid_options['extract']['hovertext']['hidden'] = true
+        @checker.options = @valid_options
+        @checker.check
+        event = Event.last
+        expect(event.payload).to match(
+          'url' => 'http://imgs.xkcd.com/comics/evolving.png',
+          'title' => 'Evolving'
+        )
       end
 
       it "should turn relative urls to absolute" do
@@ -749,9 +762,9 @@ describe Agents::WebsiteAgent do
         expect(event.payload['original_url']).to eq('http://xkcd.com/index')
       end
 
-      it "should be formatted by template after extraction" do
+      it "should format and merge values in template after extraction" do
+        @valid_options['extract']['hovertext']['hidden'] = true
         @valid_options['template'] = {
-          'url' => '{{url}}',
           'title' => '{{title | upcase}}',
           'summary' => '{{title}}: {{hovertext | truncate: 20}}',
         }

From 3a0c9e6274bf146af23aa14a32b8899b41910bc7 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 2 Nov 2016 21:03:31 +0900
Subject: [PATCH 04/23] Disable automatic URL normalization and absolutization
 on `url`

This was discussed in #1766.

For backward compatibility, existing WebsiteAgents with a key named
`url` will be given a `template` to resolve `url`.
---
 app/models/agents/website_agent.rb            | 50 ++++++++++++-------
 ...1124065838_add_templates_to_resolve_url.rb | 16 ++++++
 ...65838_add_templates_to_resolve_url_spec.rb | 39 +++++++++++++++
 spec/models/agents/website_agent_spec.rb      | 38 ++++++--------
 4 files changed, 104 insertions(+), 39 deletions(-)
 create mode 100644 db/migrate/20161124065838_add_templates_to_resolve_url.rb
 create mode 100644 spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb

diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index f36d70e3..1918186e 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -27,8 +27,6 @@ module Agents
       * Alternatively, set `data_from_event` to a Liquid template to use data directly without fetching any URL.  (For example, set it to `{{ html }}` to use HTML contained in the `html` key of the incoming Event.)
       * If you specify `merge` for the `mode` option, Huginn will retain the old payload and update it with new values.
 
-      If a created Event has a key named `url` containing a relative URL, it is automatically resolved using the request URL as base.
-
       # Supported Document Types
 
       The `type` value can be `xml`, `html`, `json`, or `text`.
@@ -121,6 +119,7 @@ module Agents
       If a `template` option is given, its value must be a hash, whose key-value pairs are interpolated after extraction for each iteration and merged with the payload.  In the template, keys of extracted data can be interpolated, and some additional variables are also available as explained in the next section.  For example:
 
           "template": {
+            "url": "{{ url | to_uri: _request_.url }}",
             "description": "{{ body_text }}",
             "last_modified": "{{ _response_.headers.Last-Modified | date: '%FT%T' }}"
           }
@@ -129,17 +128,17 @@ module Agents
 
       # Liquid Templating
 
-      In Liquid templating, the following variables are available except when invoked by `data_from_event`:
+      In Liquid templating, the following variables are available:
 
-      * `_url_`: The URL specified to fetch the content from.
+      * `_url_`: The URL specified to fetch the content from.  When parsing `data_from_event`, this is not set.
 
       * `_response_`: A response object with the following keys:
 
-          * `status`: HTTP status as integer. (Almost always 200)
+          * `status`: HTTP status as integer. (Almost always 200)  When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event.
 
-          * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.
+          * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.  When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
 
-          * `url`: The final URL of the fetched page, following redirects.  Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
+          * `url`: The final URL of the fetched page, following redirects.  When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event.  Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
 
       # Ordering Events
 
@@ -366,6 +365,8 @@ module Agents
     end
 
     def handle_data(body, url, existing_payload)
+      # Beware, url may be a URI object, string or nil
+
       doc = parse(body)
 
       if extract_full_json?
@@ -400,15 +401,6 @@ module Agents
           result.update(interpolate_options(template, extracted))
         end
 
-        # url may be URI, string or nil
-        if (payload_url = result['url'].presence) && (url = url.presence)
-          begin
-            result['url'] = (Utils.normalize_uri(url) + Utils.normalize_uri(payload_url)).to_s
-          rescue URI::Error
-            error "Cannot resolve url: <#{payload_url}> on <#{url}>"
-          end
-        end
-
         if store_payload!(old_events, result)
           log "Storing new parsed result for '#{name}': #{result.inspect}"
           create_event payload: existing_payload.merge(result)
@@ -450,7 +442,10 @@ module Agents
     end
 
     def handle_event_data(data, event, existing_payload)
-      handle_data(data, event.payload['url'], existing_payload)
+      interpolation_context.stack {
+        interpolation_context['_response_'] = ResponseFromEventDrop.new(event)
+        handle_data(data, event.payload['url'].presence, existing_payload)
+      }
     rescue => e
       error "Error when handling event data: #{e.message}\n#{e.backtrace.join("\n")}", inbound_event: event
     end
@@ -687,6 +682,27 @@ module Agents
       end
     end
 
+    class ResponseFromEventDrop < LiquidDroppable::Drop
+      def headers
+        case headers = @object.payload[:headers]
+        when Hash
+          HeaderDrop.new(Faraday::Utils::Headers.from(headers))
+        else
+          HeaderDrop.new({})
+        end
+      end
+
+      # Integer value of HTTP status
+      def status
+        @object.payload[:status]
+      end
+
+      # The URL
+      def url
+        @object.payload[:url]
+      end
+    end
+
     # Wraps Faraday::Utils::Headers
     class HeaderDrop < LiquidDroppable::Drop
       def before_method(name)
diff --git a/db/migrate/20161124065838_add_templates_to_resolve_url.rb b/db/migrate/20161124065838_add_templates_to_resolve_url.rb
new file mode 100644
index 00000000..a1e62522
--- /dev/null
+++ b/db/migrate/20161124065838_add_templates_to_resolve_url.rb
@@ -0,0 +1,16 @@
+class AddTemplatesToResolveUrl < ActiveRecord::Migration[5.0]
+  def up
+    Agents::WebsiteAgent.find_each do |agent|
+      if agent.event_keys.try!(:include?, 'url')
+        agent.options['template'] = (agent.options['template'] || {}).tap { |template|
+          template['url'] ||= '{{ url | to_uri: _response_.url }}'
+        }
+        agent.save!(validate: false)
+      end
+    end
+  end
+
+  def down
+    # No need to revert
+  end
+end
diff --git a/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb
new file mode 100644
index 00000000..6d0bd14b
--- /dev/null
+++ b/spec/migrations/20161124065838_add_templates_to_resolve_url_spec.rb
@@ -0,0 +1,39 @@
+load 'spec/rails_helper.rb'
+load File.join('db/migrate', File.basename(__FILE__, '_spec.rb') + '.rb')
+
+describe AddTemplatesToResolveUrl do
+  let :valid_options do
+    {
+      'name' => "XKCD",
+      'expected_update_period_in_days' => "2",
+      'type' => "html",
+      'url' => "http://xkcd.com",
+      'mode' => 'on_change',
+      'extract' => {
+        'url' => { 'css' => "#comic img", 'value' => "@src" },
+        'title' => { 'css' => "#comic img", 'value' => "@alt" },
+        'hovertext' => { 'css' => "#comic img", 'value' => "@title" }
+      }
+    }
+  end
+
+  let :agent do
+    Agents::WebsiteAgent.create!(
+      user: users(:bob),
+      name: "xkcd",
+      options: valid_options,
+      keep_events_for: 2.days
+    )
+  end
+
+  it 'should add a template for an existing WebsiteAgent with `url`' do
+    expect(agent.options).not_to include('template')
+    AddTemplatesToResolveUrl.new.up
+    agent.reload
+    expect(agent.options).to include(
+      'template' => {
+        'url' => '{{ url | to_uri: _response_.url }}'
+      }
+    )
+  end
+end
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 01d9f223..d943ca77 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -669,25 +669,6 @@ describe Agents::WebsiteAgent do
         )
       end
 
-      it "should turn relative urls to absolute" do
-        rel_site = {
-          'name' => "XKCD",
-          'expected_update_period_in_days' => "2",
-          'type' => "html",
-          'url' => "http://xkcd.com",
-          'mode' => "on_change",
-          'extract' => {
-            'url' => {'css' => "#topLeft a", 'value' => "@href"},
-          }
-        }
-        rel = Agents::WebsiteAgent.new(:name => "xkcd", :options => rel_site)
-        rel.user = users(:bob)
-        rel.save!
-        rel.check
-        event = Event.last
-        expect(event.payload['url']).to eq("http://xkcd.com/about")
-      end
-
       it "should return an integer value if XPath evaluates to one" do
         rel_site = {
           'name' => "XKCD",
@@ -1198,7 +1179,11 @@ fire: hot
               'some_object' => {
                 'some_data' => { hello: 'world', href: '/world' }.to_json
               },
-              url: 'http://example.com/'
+              url: 'http://example.com/',
+              'headers' => {
+                'Content-Type' => 'application/json'
+              },
+              'status' => 200
             }
             @event.save!
 
@@ -1208,6 +1193,12 @@ fire: hot
               'extract' => {
                 'value' => { 'path' => 'hello' },
                 'url' => { 'path' => 'href' },
+              },
+              'template' => {
+                'value' => '{{ value }}',
+                'url' => '{{ url | to_uri: _response_.url }}',
+                'type' => '{{ _response_.headers.content_type }}',
+                'status' => '{{ _response_.status }}'
               }
             )
           end
@@ -1216,7 +1207,7 @@ fire: hot
             expect {
               @checker.receive([@event])
             }.to change { Event.count }.by(1)
-            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world' })
+            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' })
           end
 
           it "should support merge mode" do
@@ -1225,7 +1216,7 @@ fire: hot
             expect {
               @checker.receive([@event])
             }.to change { Event.count }.by(1)
-            expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world'))
+            expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200'))
           end
 
           it "should output an error when nothing can be found at the path" do
@@ -1362,6 +1353,9 @@ fire: hot
         'mode' => 'all',
         'extract' => {
           'url' => { 'css' => "a", 'value' => "@href" },
+        },
+        'template' => {
+          'url' => '{{ url | to_uri }}',
         }
       }
       @checker = Agents::WebsiteAgent.new(:name => "ua", :options => @valid_options)

From a94cd7fd6d749cf37851f9db2eecc9bf99e00568 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sat, 12 Nov 2016 14:56:39 +0900
Subject: [PATCH 05/23] Allow an empty or null base URI

---
 app/concerns/liquid_interpolatable.rb       | 7 ++++---
 spec/concerns/liquid_interpolatable_spec.rb | 9 +++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/app/concerns/liquid_interpolatable.rb b/app/concerns/liquid_interpolatable.rb
index 83d93dec..4bc0d091 100644
--- a/app/concerns/liquid_interpolatable.rb
+++ b/app/concerns/liquid_interpolatable.rb
@@ -129,10 +129,11 @@ module LiquidInterpolatable
     # userinfo, host, port, registry, path, opaque, query, and
     # fragment.
     def to_uri(uri, base_uri = nil)
-      if base_uri
-        Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
-      else
+      case base_uri
+      when nil, ''
         Utils.normalize_uri(uri.to_s)
+      else
+        Utils.normalize_uri(base_uri) + Utils.normalize_uri(uri.to_s)
       end
     rescue URI::Error
       nil
diff --git a/spec/concerns/liquid_interpolatable_spec.rb b/spec/concerns/liquid_interpolatable_spec.rb
index 6c7c9a71..b58b9996 100644
--- a/spec/concerns/liquid_interpolatable_spec.rb
+++ b/spec/concerns/liquid_interpolatable_spec.rb
@@ -119,6 +119,15 @@ describe LiquidInterpolatable::Filters do
       @agent.interpolation_context['s'] = 'foo/index.html'
       expect(@agent.interpolated['foo']).to eq('/dir/foo/index.html')
     end
+
+    it 'should normalize a URI value if an empty base URI is given' do
+      @agent.options['foo'] = '{{ u | to_uri: b }}'
+      @agent.interpolation_context['u'] = "\u{3042}"
+      @agent.interpolation_context['b'] = ""
+      expect(@agent.interpolated['foo']).to eq('%E3%81%82')
+      @agent.interpolation_context['b'] = nil
+      expect(@agent.interpolated['foo']).to eq('%E3%81%82')
+    end
   end
 
   describe 'uri_expand' do

From 9074f3115ea572c2aa9be771997f48d4865daf22 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 23 Nov 2016 11:16:14 +0900
Subject: [PATCH 06/23] Make sure `status` is an integer when set

---
 app/models/agents/website_agent.rb       | 4 ++--
 spec/models/agents/website_agent_spec.rb | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index 1918186e..e8b97686 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -134,7 +134,7 @@ module Agents
 
       * `_response_`: A response object with the following keys:
 
-          * `status`: HTTP status as integer. (Almost always 200)  When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event.
+          * `status`: HTTP status as integer. (Almost always 200)  When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
 
           * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.  When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
 
@@ -694,7 +694,7 @@ module Agents
 
       # Integer value of HTTP status
       def status
-        @object.payload[:status]
+        Integer(@object.payload[:status]) rescue nil
       end
 
       # The URL
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index d943ca77..556f9c33 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -1198,7 +1198,7 @@ fire: hot
                 'value' => '{{ value }}',
                 'url' => '{{ url | to_uri: _response_.url }}',
                 'type' => '{{ _response_.headers.content_type }}',
-                'status' => '{{ _response_.status }}'
+                'status' => '{{ _response_.status | as_object }}'
               }
             )
           end
@@ -1207,7 +1207,7 @@ fire: hot
             expect {
               @checker.receive([@event])
             }.to change { Event.count }.by(1)
-            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200' })
+            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200 })
           end
 
           it "should support merge mode" do
@@ -1216,7 +1216,7 @@ fire: hot
             expect {
               @checker.receive([@event])
             }.to change { Event.count }.by(1)
-            expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => '200'))
+            expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
           end
 
           it "should output an error when nothing can be found at the path" do

From 5f5f3cd38f7b67bb3d33ac298cb3ebe4c17d33bf Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 23 Nov 2016 11:17:38 +0900
Subject: [PATCH 07/23] Do not err if `headers` is a valid headers hash

---
 app/models/agents/website_agent.rb       | 11 ++++-------
 spec/models/agents/website_agent_spec.rb | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index e8b97686..37fd780a 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -136,7 +136,7 @@ module Agents
 
           * `status`: HTTP status as integer. (Almost always 200)  When parsing `data_from_event`, this is set to the value of the `status` key in the incoming Event, if it is a number or a string convertible to an integer.
 
-          * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.  When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event.
+          * `headers`: Response headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insensitive to cases and -/_.  When parsing `data_from_event`, this is constructed from the value of the `headers` key in the incoming Event, if it is a hash.
 
           * `url`: The final URL of the fetched page, following redirects.  When parsing `data_from_event`, this is set to the value of the `url` key in the incoming Event.  Using this in the `template` option, you can resolve relative URLs extracted from a document like `{{ link | to_uri: _request_.url }}` and `{{ content | rebase_hrefs: _request_.url }}`.
 
@@ -684,12 +684,9 @@ module Agents
 
     class ResponseFromEventDrop < LiquidDroppable::Drop
       def headers
-        case headers = @object.payload[:headers]
-        when Hash
-          HeaderDrop.new(Faraday::Utils::Headers.from(headers))
-        else
-          HeaderDrop.new({})
-        end
+        headers = Faraday::Utils::Headers.from(@object.payload[:headers]) rescue {}
+
+        HeaderDrop.new(headers)
       end
 
       # Integer value of HTTP status
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 556f9c33..afceb250 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -1219,6 +1219,24 @@ fire: hot
             expect(@checker.events.last.payload).to eq(@event.payload.merge('value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/json', 'status' => 200))
           end
 
+          it "should convert headers and status in the event data properly" do
+            @event.payload[:status] = '201'
+            @event.payload[:headers] = [['Content-Type', 'application/rss+xml']]
+            expect {
+              @checker.receive([@event])
+            }.to change { Event.count }.by(1)
+            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => 'application/rss+xml', 'status' => 201 })
+          end
+
+          it "should ignore inconvertible headers and status in the event data" do
+            @event.payload[:status] = 'ok'
+            @event.payload[:headers] = ['Content-Type', 'Content-Length']
+            expect {
+              @checker.receive([@event])
+            }.to change { Event.count }.by(1)
+            expect(@checker.events.last.payload).to eq({ 'value' => 'world', 'url' => 'http://example.com/world', 'type' => '', 'status' => nil })
+          end
+
           it "should output an error when nothing can be found at the path" do
             @checker.options = @checker.options.merge(
               'data_from_event' => '{{ some_object.mistake }}'

From f4308afdf70ae9d98f22c9853553ed1bf75fd8c5 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 22:12:40 +0900
Subject: [PATCH 08/23] Use faraday_middleware 0.10.1, which was recently
 released

---
 Gemfile      |  2 +-
 Gemfile.lock | 12 +++---------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/Gemfile b/Gemfile
index 5a1005ac..c931b5db 100644
--- a/Gemfile
+++ b/Gemfile
@@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br
 gem 'devise','~> 4.2.0'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
-gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
+gem 'faraday_middleware', '>= 0.10.1'
 gem 'feedjira', '~> 2.0'
 gem 'font-awesome-sass', '~> 4.3.2'
 gem 'foreman', '~> 0.63.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index c18cf56a..99dc0984 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,14 +37,6 @@ GIT
       oauth2 (~> 1)
       rest-client (~> 1.8)
 
-GIT
-  remote: git://github.com/lostisland/faraday_middleware.git
-  revision: c5836ae55857272732b33eb0e0a98d60e995a376
-  branch: master
-  specs:
-    faraday_middleware (0.10.0)
-      faraday (>= 0.7.4, < 0.10)
-
 GIT
   remote: git://github.com/tumblr/tumblr_client.git
   revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
@@ -214,6 +206,8 @@ GEM
     extlib (0.9.16)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.1)
+      faraday (>= 0.7.4, < 1.0)
     feedjira (2.0.0)
       faraday (~> 0.9)
       faraday_middleware (~> 0.9)
@@ -618,7 +612,7 @@ DEPENDENCIES
   em-http-request (~> 1.1.2)
   evernote_oauth
   faraday (~> 0.9.0)
-  faraday_middleware!
+  faraday_middleware (>= 0.10.1)
   feedjira (~> 2.0)
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.3.2)

From 08b43cc8b9bacdb61ec22bdd9c8ffd985e80d2ca Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 22:14:02 +0900
Subject: [PATCH 09/23] Update the RUBY VERSION with 2.3.3p222

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 99dc0984..aef7d8bc 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -694,7 +694,7 @@ DEPENDENCIES
   xmpp4r (~> 0.5.6)
 
 RUBY VERSION
-   ruby 2.3.1p112
+   ruby 2.3.3p222
 
 BUNDLED WITH
    1.13.6

From 8e36fdd60c54f32f9b440cfdcf6616050e830d60 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Sun, 27 Nov 2016 23:22:42 +0900
Subject: [PATCH 10/23] Revert "Use faraday_middleware 0.10.1, which was
 recently released"

This reverts commit f4308afdf70ae9d98f22c9853553ed1bf75fd8c5.

I realized 0.10.1 did not include my fix for raw deflate.
---
 Gemfile      |  2 +-
 Gemfile.lock | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/Gemfile b/Gemfile
index c931b5db..5a1005ac 100644
--- a/Gemfile
+++ b/Gemfile
@@ -91,7 +91,7 @@ gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', br
 gem 'devise','~> 4.2.0'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
-gem 'faraday_middleware', '>= 0.10.1'
+gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
 gem 'feedjira', '~> 2.0'
 gem 'font-awesome-sass', '~> 4.3.2'
 gem 'foreman', '~> 0.63.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index aef7d8bc..90078534 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,6 +37,14 @@ GIT
       oauth2 (~> 1)
       rest-client (~> 1.8)
 
+GIT
+  remote: git://github.com/lostisland/faraday_middleware.git
+  revision: c5836ae55857272732b33eb0e0a98d60e995a376
+  branch: master
+  specs:
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
+
 GIT
   remote: git://github.com/tumblr/tumblr_client.git
   revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
@@ -206,8 +214,6 @@ GEM
     extlib (0.9.16)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.10.1)
-      faraday (>= 0.7.4, < 1.0)
     feedjira (2.0.0)
       faraday (~> 0.9)
       faraday_middleware (~> 0.9)
@@ -612,7 +618,7 @@ DEPENDENCIES
   em-http-request (~> 1.1.2)
   evernote_oauth
   faraday (~> 0.9.0)
-  faraday_middleware (>= 0.10.1)
+  faraday_middleware!
   feedjira (~> 2.0)
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.3.2)

From 8a14a57e00716c12d060a036240cbfda789ab010 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@users.noreply.github.com>
Date: Sun, 27 Nov 2016 15:30:50 -0500
Subject: [PATCH 11/23] Beeper.io is no more (#1808)

* Beeper.io is no more

* Avoid event propagation or scheduling of missing agents

* Update undefined_agents.html.erb with a link to the wiki
---
 app/models/agent.rb                           |   8 +-
 app/models/agents/beeper_agent.rb             | 129 ----------------
 .../application/undefined_agents.html.erb     |   8 +-
 spec/models/agent_spec.rb                     |  33 +++-
 spec/models/agents/beeper_agent_spec.rb       | 145 ------------------
 5 files changed, 41 insertions(+), 282 deletions(-)
 delete mode 100644 app/models/agents/beeper_agent.rb
 delete mode 100644 spec/models/agents/beeper_agent_spec.rb

diff --git a/app/models/agent.rb b/app/models/agent.rb
index b8d32ee9..66672d28 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -370,7 +370,7 @@ class Agent < ActiveRecord::Base
     def receive!(options={})
       Agent.transaction do
         scope = Agent.
-                select("agents.id AS receiver_agent_id, events.id AS event_id").
+                select("agents.id AS receiver_agent_id, sources.type AS source_agent_type, agents.type AS receiver_agent_type, events.id AS event_id").
                 joins("JOIN links ON (links.receiver_id = agents.id)").
                 joins("JOIN agents AS sources ON (links.source_id = sources.id)").
                 joins("JOIN events ON (events.agent_id = sources.id AND events.id > links.event_id_at_creation)").
@@ -379,10 +379,11 @@ class Agent < ActiveRecord::Base
           scope = scope.where("agents.id in (?)", options[:only_receivers])
         end
 
-        sql = scope.to_sql()
+        sql = scope.to_sql
 
         agents_to_events = {}
-        Agent.connection.select_rows(sql).each do |receiver_agent_id, event_id|
+        Agent.connection.select_rows(sql).each do |receiver_agent_id, source_agent_type, receiver_agent_type, event_id|
+          next unless const_defined?(source_agent_type) && const_defined?(receiver_agent_type)
           agents_to_events[receiver_agent_id.to_i] ||= []
           agents_to_events[receiver_agent_id.to_i] << event_id
         end
@@ -417,6 +418,7 @@ class Agent < ActiveRecord::Base
       return if schedule == 'never'
       types = where(:schedule => schedule).group(:type).pluck(:type)
       types.each do |type|
+        next unless const_defined?(type)
         type.constantize.bulk_check(schedule)
       end
     end
diff --git a/app/models/agents/beeper_agent.rb b/app/models/agents/beeper_agent.rb
deleted file mode 100644
index 4cd5a41e..00000000
--- a/app/models/agents/beeper_agent.rb
+++ /dev/null
@@ -1,129 +0,0 @@
-module Agents
-  class BeeperAgent < Agent
-    cannot_be_scheduled!
-    cannot_create_events!
-    no_bulk_receive!
-
-    description <<-MD
-      Beeper agent sends messages to Beeper app on your mobile device via Push notifications.
-
-      You need a Beeper Application ID (`app_id`), Beeper REST API Key (`api_key`) and Beeper Sender ID (`sender_id`) [https://beeper.io](https://beeper.io)
-
-      You have to provide phone number (`phone`) of the recipient which have a mobile device with Beeper installed, or a `group_id` – Beeper Group ID
-
-      Also you have to provide a message `type` which has to be `message`, `image`, `event`, `location` or `task`.
-
-      Depending on message type you have to provide additional fields:
-
-      ##### Message
-      * `text` – **required**
-
-      ##### Image
-      * `image` – **required** (Image URL or Base64-encoded image)
-      * `text` – optional
-
-      ##### Event
-      * `text` – **required**
-      * `start_time` – **required** (Corresponding to ISO 8601)
-      * `end_time` – optional (Corresponding to ISO 8601)
-
-      ##### Location
-      * `latitude` – **required**
-      * `longitude` – **required**
-      * `text` – optional
-
-      ##### Task
-      * `text` – **required**
-
-      You can see additional documentation at [Beeper website](https://beeper.io/docs)
-    MD
-
-    BASE_URL = 'https://api.beeper.io/api'
-
-    TYPE_ATTRIBUTES = {
-      'message'  => %w(text),
-      'image'    => %w(text image),
-      'event'    => %w(text start_time end_time),
-      'location' => %w(text latitude longitude),
-      'task'     => %w(text)
-    }
-
-    MESSAGE_TYPES = TYPE_ATTRIBUTES.keys
-
-    TYPE_REQUIRED_ATTRIBUTES = {
-      'message'  => %w(text),
-      'image'    => %w(image),
-      'event'    => %w(text start_time),
-      'location' => %w(latitude longitude),
-      'task'     => %w(text)
-    }
-
-    def default_options
-      {
-        'type'      => 'message',
-        'app_id'    => '',
-        'api_key'   => '',
-        'sender_id' => '',
-        'phone'     => '',
-        'text'      => '{{title}}'
-      }
-    end
-
-    def validate_options
-      %w(app_id api_key sender_id type).each do |attr|
-        errors.add(:base, "you need to specify a #{attr}") if options[attr].blank?
-      end
-
-      if options['type'].in?(MESSAGE_TYPES)
-        required_attributes = TYPE_REQUIRED_ATTRIBUTES[options['type']]
-        if required_attributes.any? { |attr| options[attr].blank? }
-          errors.add(:base, "you need to specify a #{required_attributes.join(', ')}")
-        end
-      else
-        errors.add(:base, 'you need to specify a valid message type')
-      end
-
-      unless options['group_id'].blank? ^ options['phone'].blank?
-        errors.add(:base, 'you need to specify a phone or group_id')
-      end
-    end
-
-    def working?
-      received_event_without_error? && !recent_error_logs?
-    end
-
-    def receive(incoming_events)
-      incoming_events.each do |event|
-        send_message(event)
-      end
-    end
-
-    def send_message(event)
-      mo = interpolated(event)
-      begin
-        response = HTTParty.post(endpoint_for(mo['type']), body: payload_for(mo), headers: headers)
-        error(response.body) if response.code != 201
-      rescue HTTParty::Error => e
-        error(e.message)
-      end
-    end
-
-    private
-
-    def headers
-      {
-        'X-Beeper-Application-Id' => options['app_id'],
-        'X-Beeper-REST-API-Key'   => options['api_key'],
-        'Content-Type' => 'application/json'
-      }
-    end
-
-    def payload_for(mo)
-      mo.slice(*TYPE_ATTRIBUTES[mo['type']], 'sender_id', 'phone', 'group_id').to_json
-    end
-
-    def endpoint_for(type)
-      "#{BASE_URL}/#{type}s.json"
-    end
-  end
-end
\ No newline at end of file
diff --git a/app/views/application/undefined_agents.html.erb b/app/views/application/undefined_agents.html.erb
index 3912af7d..229f3e19 100644
--- a/app/views/application/undefined_agents.html.erb
+++ b/app/views/application/undefined_agents.html.erb
@@ -19,20 +19,20 @@
         </ul>
         <br/>
         <p>
-          The issue most probably occurred because of one or more of the following reasons:
+          This issue probably occurred for one or more of the following reasons:
         </p>
         <ul>
+          <li>If the respective Agent is distributed as part of the Huginn application codebase, it may have been removed or moved to an Agent gem. Please see <a href="https://github.com/cantino/huginn/wiki/Dealing-with-Deleted-Agent-Types" target="_blank">this wiki page for more information</a>.</li>
           <li>If the respective Agent is distributed as a Ruby gem, it might have been removed from the <code>ADDITIONAL_GEMS</code> environment setting.</li>
-          <li>If the respective Agent is distributed as part of the Huginn application codebase, it might have been removed from that either on purpose (because the Agent has been deprecated or been moved to an Agent gem) or accidentally. Please check if the Agent(s) in question are available in your Huginn codebase under the path <code>app/models/agents/</code>.</li>
         </ul>
         <br/>
         <p>
-          You can fix the issue by adding the Agent(s) back to the application codebase by
+          You can fix this issue by:
         </p>
         <ul>
+          <li>deleting the respective Agent(s) from the database using the button below.</li>
           <li>adding the respective Agent(s) to the the <code>ADDITIONAL_GEMS</code> environment setting. Please see <a href="https://github.com/cantino/huginn_agent" target="_blank">https://github.com/cantino/huginn_agent</a> for documentation on how to properly set it.</li>
           <li>adding the respective Agent(s) code to the Huginn application codebase (in case it was deleted accidentally).</li>
-          <li>deleting the respective Agent(s) from the database using the button below.</li>
         </ul>
         <br/>
         <div class="btn-group">
diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb
index fa98b45b..7a67a587 100644
--- a/spec/models/agent_spec.rb
+++ b/spec/models/agent_spec.rb
@@ -74,6 +74,13 @@ describe Agent do
       Agent.run_schedule("midnight")
     end
 
+    it "ignores unknown types" do
+      Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
+      mock(Agents::WeatherAgent).bulk_check("midnight").once
+      mock(Agents::WebsiteAgent).bulk_check("midnight").once
+      Agent.run_schedule("midnight")
+    end
+
     it "only runs agents with the given schedule" do
       do_not_allow(Agents::WebsiteAgent).async_check
       Agent.run_schedule("blah")
@@ -283,13 +290,37 @@ describe Agent do
         Agent.receive!
       end
 
-      it "should not propogate to disabled Agents" do
+      it "should not propagate to disabled Agents" do
         Agent.async_check(agents(:bob_weather_agent).id)
         agents(:bob_rain_notifier_agent).update_attribute :disabled, true
         mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
         Agent.receive!
       end
 
+      it "should not propagate to Agents with unknown types" do
+        Agent.async_check(agents(:jane_weather_agent).id)
+        Agent.async_check(agents(:bob_weather_agent).id)
+
+        Agent.where(id: agents(:bob_rain_notifier_agent).id).update_all type: 'UnknownTypeAgent'
+
+        mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
+        mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
+
+        Agent.receive!
+      end
+
+      it "should not propagate from Agents with unknown types" do
+        Agent.async_check(agents(:jane_weather_agent).id)
+        Agent.async_check(agents(:bob_weather_agent).id)
+
+        Agent.where(id: agents(:bob_weather_agent).id).update_all type: 'UnknownTypeAgent'
+
+        mock(Agent).async_receive(agents(:bob_rain_notifier_agent).id, anything).times(0)
+        mock(Agent).async_receive(agents(:jane_rain_notifier_agent).id, anything).times(1)
+
+        Agent.receive!
+      end
+
       it "should log exceptions" do
         mock.any_instance_of(Agents::TriggerAgent).receive(anything).once {
           raise "foo"
diff --git a/spec/models/agents/beeper_agent_spec.rb b/spec/models/agents/beeper_agent_spec.rb
deleted file mode 100644
index e2c2aba5..00000000
--- a/spec/models/agents/beeper_agent_spec.rb
+++ /dev/null
@@ -1,145 +0,0 @@
-require 'rails_helper'
-
-
-describe Agents::BeeperAgent do
-  let(:base_params) {
-    {
-      'type'      => 'message',
-      'app_id'    => 'some-app-id',
-      'api_key'   => 'some-api-key',
-      'sender_id' => 'sender-id',
-      'phone'     => '+111111111111',
-      'text'      => 'Some text'
-    }
-  }
-
-  subject {
-    agent = described_class.new(name: 'beeper-agent', options: base_params)
-    agent.user = users(:jane)
-    agent.save! and return agent
-  }
-
-  context 'validation' do
-    it 'valid' do
-      expect(subject).to be_valid
-    end
-
-    [:type, :app_id, :api_key, :sender_id].each do |attr|
-      it "invalid without #{attr}" do
-        subject.options[attr] = nil
-        expect(subject).not_to be_valid
-      end
-    end
-
-    it 'invalid with group_id and phone' do
-      subject.options['group_id'] ='some-group-id'
-      expect(subject).not_to be_valid
-    end
-
-    context '#message' do
-      it 'requires text' do
-        subject.options[:text] = nil
-        expect(subject).not_to be_valid
-      end
-    end
-
-    context '#image' do
-      before(:each) do
-        subject.options[:type] = 'image'
-      end
-
-      it 'invalid without image' do
-        expect(subject).not_to be_valid
-      end
-
-      it 'valid with image' do
-        subject.options[:image] = 'some-url'
-        expect(subject).to be_valid
-      end
-    end
-
-    context '#event' do
-      before(:each) do
-        subject.options[:type] = 'event'
-      end
-
-      it 'invalid without start_time' do
-        expect(subject).not_to be_valid
-      end
-
-      it 'valid with start_time' do
-        subject.options[:start_time] = Time.now
-        expect(subject).to be_valid
-      end
-    end
-
-    context '#location' do
-      before(:each) do
-        subject.options[:type] = 'location'
-      end
-
-      it 'invalid without latitude and longitude' do
-        expect(subject).not_to be_valid
-      end
-
-      it 'valid with latitude and longitude' do
-        subject.options[:latitude] = 15.0
-        subject.options[:longitude] = 16.0
-        expect(subject).to be_valid
-      end
-    end
-
-    context '#task' do
-      before(:each) do
-        subject.options[:type] = 'task'
-      end
-
-      it 'valid with text' do
-        expect(subject).to be_valid
-      end
-    end
-  end
-
-  context 'payload_for' do
-    it 'removes unwanted attributes' do
-      result = subject.send(:payload_for, {'type' => 'message', 'text' => 'text',
-        'sender_id' => 'sender', 'phone' => '+1', 'random_attribute' => 'unwanted'})
-      expect(result).to eq('{"text":"text","sender_id":"sender","phone":"+1"}')
-    end
-  end
-
-  context 'headers' do
-    it 'sets X-Beeper-Application-Id header with app_id' do
-      expect(subject.send(:headers)['X-Beeper-Application-Id']).to eq(base_params['app_id'])
-    end
-
-    it 'sets X-Beeper-REST-API-Key header with api_key' do
-      expect(subject.send(:headers)['X-Beeper-REST-API-Key']).to eq(base_params['api_key'])
-    end
-
-    it 'sets Content-Type' do
-      expect(subject.send(:headers)['Content-Type']).to eq('application/json')
-    end
-  end
-
-  context 'endpoint_for' do
-    it 'returns valid URL for message' do
-      expect(subject.send(:endpoint_for, 'message')).to eq('https://api.beeper.io/api/messages.json')
-    end
-
-    it 'returns valid URL for image' do
-      expect(subject.send(:endpoint_for, 'image')).to eq('https://api.beeper.io/api/images.json')
-    end
-
-    it 'returns valid URL for event' do
-      expect(subject.send(:endpoint_for, 'event')).to eq('https://api.beeper.io/api/events.json')
-    end
-
-    it 'returns valid URL for location' do
-      expect(subject.send(:endpoint_for, 'location')).to eq('https://api.beeper.io/api/locations.json')
-    end
-    it 'returns valid URL for task' do
-      expect(subject.send(:endpoint_for, 'task')).to eq('https://api.beeper.io/api/tasks.json')
-    end
-  end
-end

From 3612ba7333ba4b07bba59acbac5181eae997a81e Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Mon, 7 Nov 2016 22:34:06 +0900
Subject: [PATCH 12/23] Add podcast tags to events emitted by RssAgent

The keys are only added when the feed is a podcast.
---
 Gemfile                              |   2 +-
 Gemfile.lock                         |  17 +--
 app/models/agents/rss_agent.rb       |  83 ++++++++++++++-
 lib/feedjira_extension.rb            |  17 +++
 spec/data_fixtures/podcast.rss       |  76 ++++++++++++++
 spec/models/agents/rss_agent_spec.rb | 152 +++++++++++++++++++++++++++
 6 files changed, 339 insertions(+), 8 deletions(-)
 create mode 100644 spec/data_fixtures/podcast.rss

diff --git a/Gemfile b/Gemfile
index 5a1005ac..ffddd76a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
 gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
-gem 'feedjira', '~> 2.0'
+gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1']
 gem 'font-awesome-sass', '~> 4.3.2'
 gem 'foreman', '~> 0.63.0'
 gem 'geokit', '~> 1.8.4'
diff --git a/Gemfile.lock b/Gemfile.lock
index 90078534..2b871768 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,6 +37,16 @@ GIT
       oauth2 (~> 1)
       rest-client (~> 1.8)
 
+GIT
+  remote: git://github.com/feedjira/feedjira.git
+  revision: 849b7809455c084b06f9c4c2d368a752677882b0
+  specs:
+    feedjira (2.0.0)
+      faraday (>= 0.9)
+      faraday_middleware (>= 0.9)
+      loofah (>= 2.0)
+      sax-machine (>= 1.0)
+
 GIT
   remote: git://github.com/lostisland/faraday_middleware.git
   revision: c5836ae55857272732b33eb0e0a98d60e995a376
@@ -214,11 +224,6 @@ GEM
     extlib (0.9.16)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    feedjira (2.0.0)
-      faraday (~> 0.9)
-      faraday_middleware (~> 0.9)
-      loofah (~> 2.0)
-      sax-machine (~> 1.0)
     ffi (1.9.10)
     font-awesome-sass (4.3.2.1)
       sass (~> 3.2)
@@ -619,7 +624,7 @@ DEPENDENCIES
   evernote_oauth
   faraday (~> 0.9.0)
   faraday_middleware!
-  feedjira (~> 2.0)
+  feedjira!
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.3.2)
   forecast_io (~> 2.0.0)
diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb
index 0ed34c42..0ab5719c 100644
--- a/app/models/agents/rss_agent.rb
+++ b/app/models/agents/rss_agent.rb
@@ -66,6 +66,22 @@ module Agents
               "copyright": "...",
               "icon": "http://example.com/icon.png",
               "authors": [ "..." ],
+
+              "itunes_block": "no",
+              "itunes_categories": [
+                "Technology", "Gadgets",
+                "TV & Film",
+                "Arts", "Food"
+              ],
+              "itunes_complete": "yes",
+              "itunes_explicit": "yes",
+              "itunes_image": "http://...",
+              "itunes_new_feed_url": "http://...",
+              "itunes_owners": [ "John Doe <john.doe@example.com>" ],
+              "itunes_subtitle": "...",
+              "itunes_summary": "...",
+              "language": "en-US",
+
               "date_published": "2014-09-11T01:30:00-07:00",
               "last_updated": "2014-09-11T01:30:00-07:00"
             },
@@ -84,6 +100,16 @@ module Agents
             "enclosure": {
               "url" => "http://example.com/file.mp3", "type" => "audio/mpeg", "length" => "123456789"
             },
+
+            "itunes_block": "no",
+            "itunes_closed_captioned": "yes",
+            "itunes_duration": "04:34",
+            "itunes_explicit": "yes",
+            "itunes_image": "http://...",
+            "itunes_order": "1",
+            "itunes_subtitle": "...",
+            "itunes_summary": "...",
+
             "date_published": "2014-09-11T01:30:00-0700",
             "last_updated": "2014-09-11T01:30:00-0700"
           }
@@ -91,7 +117,8 @@ module Agents
       Some notes:
 
       - The `feed` key is present only if `include_feed_info` is set to true.
-      - Each element in `authors` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
+      - The keys starting with `itunes_`, and `language` are only present when the feed is a podcast.  See [Podcasts Connect Help](https://help.apple.com/itc/podcasts_connect/#/itcb54353390) for details.
+      - Each element in `authors` and `itunes_owners` is a string normalized in the format "*name* <*email*> (*url*)", where each space-separated part is optional.
       - Timestamps are converted to the ISO 8601 format.
     MD
 
@@ -206,9 +233,40 @@ module Agents
         authors: feed.authors,
         date_published: feed.date_published,
         last_updated: feed.last_updated,
+        **itunes_feed_data(feed)
       }
     end
 
+    def itunes_feed_data(feed)
+      data = {}
+      case feed
+      when Feedjira::Parser::ITunesRSS
+        %i[
+          itunes_block
+          itunes_categories
+          itunes_complete
+          itunes_explicit
+          itunes_image
+          itunes_new_feed_url
+          itunes_owners
+          itunes_subtitle
+          itunes_summary
+          language
+        ].each { |attr|
+          if value = feed.try(attr).presence
+            data[attr] =
+              case attr
+              when :itunes_summary
+                clean_fragment(value)
+              else
+                value
+              end
+          end
+        }
+      end
+      data
+    end
+
     def entry_data(entry)
       {
         id: entry.id,
@@ -224,9 +282,32 @@ module Agents
         categories: Array(entry.try(:categories)),
         date_published: entry.date_published,
         last_updated: entry.last_updated,
+        **itunes_entry_data(entry)
       }
     end
 
+    def itunes_entry_data(entry)
+      data = {}
+      case entry
+      when Feedjira::Parser::ITunesRSSItem
+        %i[
+          itunes_block
+          itunes_closed_captioned
+          itunes_duration
+          itunes_explicit
+          itunes_image
+          itunes_order
+          itunes_subtitle
+          itunes_summary
+        ].each { |attr|
+          if value = entry.try(attr).presence
+            data[attr] = value
+          end
+        }
+      end
+      data
+    end
+
     def feed_to_events(feed)
       payload_base = {}
 
diff --git a/lib/feedjira_extension.rb b/lib/feedjira_extension.rb
index a9a9ac11..5eab2981 100644
--- a/lib/feedjira_extension.rb
+++ b/lib/feedjira_extension.rb
@@ -57,6 +57,13 @@ module FeedjiraExtension
     value :content
   end
 
+  class ITunesRssOwner < Author
+    include SAXMachine
+
+    element :'itunes:name', as: :name
+    element :'itunes:email', as: :email
+  end
+
   class Enclosure
     include SAXMachine
 
@@ -290,6 +297,16 @@ module FeedjiraExtension
           def copyright
             @copyright || super
           end
+
+          if /ITunes/ === name
+            sax_config.collection_elements['itunes:owner'].clear
+            elements :"itunes:owner", as: :_itunes_owners, class: ITunesRssOwner
+            private :_itunes_owners
+
+            def itunes_owners
+              _itunes_owners.reject(&:empty?)
+            end
+          end
         end
 
         sax_config.collection_elements.each_value do |collection_elements|
diff --git a/spec/data_fixtures/podcast.rss b/spec/data_fixtures/podcast.rss
new file mode 100644
index 00000000..bb64933e
--- /dev/null
+++ b/spec/data_fixtures/podcast.rss
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+  <channel>
+    <title>All About Everything</title>
+    <link>http://www.example.com/podcasts/everything/index.html</link>
+    <language>en-us</language>
+    <copyright>&#x2117; &amp; &#xA9; 2014 John Doe &amp; Family</copyright>
+    <itunes:subtitle>A show about everything</itunes:subtitle>
+    <itunes:author>John Doe</itunes:author>
+    <itunes:summary>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</itunes:summary>
+    <description>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store</description>
+    <itunes:owner>
+      <itunes:name>John Doe</itunes:name>
+      <itunes:email>john.doe@example.com</itunes:email>
+    </itunes:owner>
+    <itunes:complete>yes</itunes:complete>
+    <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg"/>
+    <itunes:category text="Technology">
+      <itunes:category text="Gadgets"/>
+    </itunes:category>
+    <itunes:category text="TV &amp; Film"/>
+    <itunes:category text="Arts">
+      <itunes:category text="Food"/>
+    </itunes:category>
+    <itunes:explicit>no</itunes:explicit>
+    <item>
+      <title>Shake Shake Shake Your Spices</title>
+      <itunes:author>John Doe</itunes:author>
+      <itunes:subtitle>A short primer on table spices</itunes:subtitle>
+      <itunes:summary><![CDATA[This week we talk about <a href="https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!]]></itunes:summary>
+      <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg"/>
+      <enclosure length="8727310" type="audio/x-m4a" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a"/>
+      <guid>http://example.com/podcasts/archive/aae20140615.m4a</guid>
+      <pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
+      <itunes:duration>07:04</itunes:duration>
+      <itunes:explicit>no</itunes:explicit>
+    </item>
+    <item>
+      <title>Socket Wrench Shootout</title>
+      <itunes:author>Jane Doe</itunes:author>
+      <itunes:subtitle>Comparing socket wrenches is fun!</itunes:subtitle>
+      <itunes:summary>This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here.</itunes:summary>
+      <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg"/>
+      <enclosure length="5650889" type="video/mp4" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4"/>
+      <guid>http://example.com/podcasts/archive/aae20140608.mp4</guid>
+      <pubDate>Wed, 09 Mar 2016 13:00:00 EST</pubDate>
+      <itunes:duration>04:34</itunes:duration>
+      <itunes:explicit>no</itunes:explicit>
+    </item>
+    <item>
+      <title>The Best Chili</title>
+      <itunes:author>Jane Doe</itunes:author>
+      <itunes:subtitle>Jane and Eric</itunes:subtitle>
+      <itunes:summary>This week we talk about the best Chili in the world. Which chili is better?</itunes:summary>
+      <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg"/>
+      <enclosure length="5650889" type="video/x-m4v" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v"/>
+      <guid>http://example.com/podcasts/archive/aae20140697.m4v</guid>
+      <pubDate>Thu, 10 Mar 2016 02:00:00 -0700</pubDate>
+      <itunes:duration>04:34</itunes:duration>
+      <itunes:explicit>no</itunes:explicit>
+      <itunes:isClosedCaptioned>Yes</itunes:isClosedCaptioned>
+    </item>
+    <item>
+      <title>Red,Whine, &amp; Blue</title>
+      <itunes:author>Various</itunes:author>
+      <itunes:subtitle>Red + Blue != Purple</itunes:subtitle>
+      <itunes:summary>This week we talk about surviving in a Red state if you are a Blue person. Or vice versa.</itunes:summary>
+      <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg"/>
+      <enclosure length="498537" type="audio/mpeg" url="http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3"/>
+      <guid>http://example.com/podcasts/archive/aae20140601.mp3</guid>
+      <pubDate>Fri, 11 Mar 2016 01:15:00 +3000</pubDate>
+      <itunes:duration>03:59</itunes:duration>
+      <itunes:explicit>no</itunes:explicit>
+    </item>
+  </channel>
+</rss>
diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index 31c6737e..ac651166 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -12,7 +12,11 @@ describe Agents::RssAgent do
     stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
     stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
     stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
+<<<<<<< HEAD
     stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
+=======
+    stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
+>>>>>>> Add podcast tags to events emitted by RssAgent
   end
 
   let(:agent) do
@@ -303,6 +307,154 @@ describe Agents::RssAgent do
         expect(event.payload['title']).to eq('Mëkanïk Zaïn')
       end
     end
+
+    context 'with podcast elements' do
+      before do
+        @valid_options['url'] = 'http://example.com/podcast.rss'
+        @valid_options['include_feed_info'] = true
+      end
+
+      let :feed_info do
+        {
+          "id" => nil,
+          "type" => "rss",
+          "url" => "http://www.example.com/podcasts/everything/index.html",
+          "links" => [ { "href" => "http://www.example.com/podcasts/everything/index.html" } ],
+          "title" => "All About Everything",
+          "description" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
+          "copyright" => "℗ & © 2014 John Doe & Family",
+          "generator" => nil,
+          "icon" => nil,
+          "authors" => [
+            "John Doe"
+          ],
+          "date_published" => nil,
+          "last_updated" => nil,
+          "itunes_categories" => [
+            "Technology", "Gadgets",
+            "TV & Film",
+            "Arts", "Food"
+          ],
+          "itunes_complete" => "yes",
+          "itunes_explicit" => "no",
+          "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything.jpg",
+          "itunes_owners" => ["John Doe <john.doe@example.com>"],
+          "itunes_subtitle" => "A show about everything",
+          "itunes_summary" => "All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our podcast in the Podcasts app or in the iTunes Store",
+          "language" => "en-us"
+        }
+      end
+
+      it "is parsed correctly" do
+        expect {
+          agent.check
+        }.to change { agent.events.count }.by(4)
+
+        expect(agent.events.map(&:payload)).to match([
+          {
+            "feed" => feed_info,
+            "id" => "http://example.com/podcasts/archive/aae20140601.mp3",
+            "url" => nil,
+            "urls" => [],
+            "links" => [],
+            "title" => "Red,Whine, & Blue",
+            "description" => nil,
+            "content" => nil,
+            "image" => nil,
+            "enclosure" => {
+              "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode4.mp3",
+              "type" => "audio/mpeg",
+              "length" => "498537"
+            },
+            "authors" => ["<Various>"],
+            "categories" => [],
+            "date_published" => "2016-03-11T01:15:00+00:00",
+            "last_updated" => "2016-03-11T01:15:00+00:00",
+            "itunes_duration" => "03:59",
+            "itunes_explicit" => "no",
+            "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode4.jpg",
+            "itunes_subtitle" => "Red + Blue != Purple",
+            "itunes_summary" => "This week we talk about surviving in a Red state if you are a Blue person. Or vice versa."
+          },
+          {
+            "feed" => feed_info,
+            "id" => "http://example.com/podcasts/archive/aae20140697.m4v",
+            "url" => nil,
+            "urls" => [],
+            "links" => [],
+            "title" => "The Best Chili",
+            "description" => nil,
+            "content" => nil,
+            "image" => nil,
+            "enclosure" => {
+              "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.m4v",
+              "type" => "video/x-m4v",
+              "length" => "5650889"
+            },
+            "authors" => ["Jane Doe"],
+            "categories" => [],
+            "date_published" => "2016-03-10T02:00:00-07:00",
+            "last_updated" => "2016-03-10T02:00:00-07:00",
+            "itunes_closed_captioned" => "Yes",
+            "itunes_duration" => "04:34",
+            "itunes_explicit" => "no",
+            "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode3.jpg",
+            "itunes_subtitle" => "Jane and Eric",
+            "itunes_summary" => "This week we talk about the best Chili in the world. Which chili is better?"
+          },
+          {
+            "feed" => feed_info,
+            "id" => "http://example.com/podcasts/archive/aae20140608.mp4",
+            "url" => nil,
+            "urls" => [],
+            "links" => [],
+            "title" => "Socket Wrench Shootout",
+            "description" => nil,
+            "content" => nil,
+            "image" => nil,
+            "enclosure" => {
+              "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp4",
+              "type" => "video/mp4",
+              "length" => "5650889"
+            },
+            "authors" => ["Jane Doe"],
+            "categories" => [],
+            "date_published" => "2016-03-09T13:00:00-05:00",
+            "last_updated" => "2016-03-09T13:00:00-05:00",
+            "itunes_duration" => "04:34",
+            "itunes_explicit" => "no",
+            "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode2.jpg",
+            "itunes_subtitle" => "Comparing socket wrenches is fun!",
+            "itunes_summary" => "This week we talk about metric vs. Old English socket wrenches. Which one is better? Do you really need both? Get all of your answers here."
+          },
+          {
+            "feed" => feed_info,
+            "id" => "http://example.com/podcasts/archive/aae20140615.m4a",
+            "url" => nil,
+            "urls" => [],
+            "links" => [],
+            "title" => "Shake Shake Shake Your Spices",
+            "description" => nil,
+            "content" => nil,
+            "image" => nil,
+            "enclosure" => {
+              "url" => "http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a",
+              "type" => "audio/x-m4a",
+              "length" => "8727310"
+            },
+            "authors" => ["John Doe"],
+            "categories" => [],
+            "date_published" => "2016-03-08T12:00:00+00:00",
+            "last_updated" => "2016-03-08T12:00:00+00:00",
+            "itunes_duration" => "07:04",
+            "itunes_explicit" => "no",
+            "itunes_image" => "http://example.com/podcasts/everything/AllAboutEverything/Episode1.jpg",
+            "itunes_subtitle" => "A short primer on table spices",
+            "itunes_summary" => "This week we talk about <a href=\"https://itunes/apple.com/us/book/antique-trader-salt-pepper/id429691295?mt=11\">salt and pepper shakers</a>, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!"
+          }
+        ])
+      end
+    end
   end
 
   describe 'logging errors with the feed url' do

From e26c07e75bd996ab27673aac67914c423204e7cf Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Wed, 30 Nov 2016 01:24:36 +0900
Subject: [PATCH 13/23] Fix a merge conflict

---
 spec/models/agents/rss_agent_spec.rb | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/spec/models/agents/rss_agent_spec.rb b/spec/models/agents/rss_agent_spec.rb
index ac651166..399cf819 100644
--- a/spec/models/agents/rss_agent_spec.rb
+++ b/spec/models/agents/rss_agent_spec.rb
@@ -12,11 +12,8 @@ describe Agents::RssAgent do
     stub_request(:any, /SlickdealsnetFP/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/slickdeals.atom")), :status => 200)
     stub_request(:any, /onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")), status: 200)
     stub_request(:any, /bad.onethingwell.org/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/onethingwell.rss")).gsub(/(?<=<link>)[^<]*/, ''), status: 200)
-<<<<<<< HEAD
     stub_request(:any, /iso-8859-1/).to_return(body: File.binread(Rails.root.join("spec/data_fixtures/iso-8859-1.rss")), headers: { 'Content-Type' => 'application/rss+xml; charset=ISO-8859-1' }, status: 200)
-=======
     stub_request(:any, /podcast/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/podcast.rss")), status: 200)
->>>>>>> Add podcast tags to events emitted by RssAgent
   end
 
   let(:agent) do

From 15347c8a3ce7a6e33db98835619b161172960cc7 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 13 Dec 2016 18:08:16 +0900
Subject: [PATCH 14/23] Feedjira 2.1.0 is out, yay!

---
 Gemfile      |  2 +-
 Gemfile.lock | 17 ++++++-----------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/Gemfile b/Gemfile
index ffddd76a..a83fc492 100644
--- a/Gemfile
+++ b/Gemfile
@@ -92,7 +92,7 @@ gem 'devise','~> 4.2.0'
 gem 'em-http-request', '~> 1.1.2'
 gem 'faraday', '~> 0.9.0'
 gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
-gem 'feedjira', github: 'feedjira/feedjira' # ['~> 2.0', '>= 2.0.1']
+gem 'feedjira', '~> 2.1'
 gem 'font-awesome-sass', '~> 4.3.2'
 gem 'foreman', '~> 0.63.0'
 gem 'geokit', '~> 1.8.4'
diff --git a/Gemfile.lock b/Gemfile.lock
index 2b871768..ba055445 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -37,16 +37,6 @@ GIT
       oauth2 (~> 1)
       rest-client (~> 1.8)
 
-GIT
-  remote: git://github.com/feedjira/feedjira.git
-  revision: 849b7809455c084b06f9c4c2d368a752677882b0
-  specs:
-    feedjira (2.0.0)
-      faraday (>= 0.9)
-      faraday_middleware (>= 0.9)
-      loofah (>= 2.0)
-      sax-machine (>= 1.0)
-
 GIT
   remote: git://github.com/lostisland/faraday_middleware.git
   revision: c5836ae55857272732b33eb0e0a98d60e995a376
@@ -224,6 +214,11 @@ GEM
     extlib (0.9.16)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
+    feedjira (2.1.0)
+      faraday (>= 0.9)
+      faraday_middleware (>= 0.9)
+      loofah (>= 2.0)
+      sax-machine (>= 1.0)
     ffi (1.9.10)
     font-awesome-sass (4.3.2.1)
       sass (~> 3.2)
@@ -624,7 +619,7 @@ DEPENDENCIES
   evernote_oauth
   faraday (~> 0.9.0)
   faraday_middleware!
-  feedjira!
+  feedjira (~> 2.1)
   ffi (>= 1.9.4)
   font-awesome-sass (~> 4.3.2)
   forecast_io (~> 2.0.0)

From fd18e1453883768e2bf848a055528fcf82740243 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 13 Dec 2016 19:31:32 +0900
Subject: [PATCH 15/23] Fast-forward faraday_middleware

The version constraint on faraday is relaxed while at it, although
some gems still have the strict constraint.
---
 Gemfile      | 2 +-
 Gemfile.lock | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Gemfile b/Gemfile
index a83fc492..f6fec106 100644
--- a/Gemfile
+++ b/Gemfile
@@ -90,7 +90,7 @@ gem 'delayed_job', '~> 4.1.0'
 gem 'delayed_job_active_record', github: 'dsander/delayed_job_active_record', branch: 'rails5'
 gem 'devise','~> 4.2.0'
 gem 'em-http-request', '~> 1.1.2'
-gem 'faraday', '~> 0.9.0'
+gem 'faraday', '~> 0.9'
 gem 'faraday_middleware', github: 'lostisland/faraday_middleware', branch: 'master'  # '>= 0.10.1'
 gem 'feedjira', '~> 2.1'
 gem 'font-awesome-sass', '~> 4.3.2'
diff --git a/Gemfile.lock b/Gemfile.lock
index ba055445..ee694741 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -39,11 +39,11 @@ GIT
 
 GIT
   remote: git://github.com/lostisland/faraday_middleware.git
-  revision: c5836ae55857272732b33eb0e0a98d60e995a376
+  revision: 59088da02940d0ee2010b2e3156343346767c31e
   branch: master
   specs:
     faraday_middleware (0.10.0)
-      faraday (>= 0.7.4, < 0.10)
+      faraday (>= 0.7.4, < 1.0)
 
 GIT
   remote: git://github.com/tumblr/tumblr_client.git
@@ -617,7 +617,7 @@ DEPENDENCIES
   dropbox-api
   em-http-request (~> 1.1.2)
   evernote_oauth
-  faraday (~> 0.9.0)
+  faraday (~> 0.9)
   faraday_middleware!
   feedjira (~> 2.1)
   ffi (>= 1.9.4)

From 3159b547c7854365455b194233baf562f9d2f8e2 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Mon, 19 Dec 2016 14:51:32 +0900
Subject: [PATCH 16/23] Use the builtin mysql on Travis CI

---
 .travis.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1d4abefa..eeb4f722 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ sudo: required
 language: ruby
 services:
   - docker
+  - mysql
   - postgresql
 env:
   global:
@@ -37,9 +38,6 @@ rvm:
 - 2.3.1
 cache: bundler
 bundler_args: --without development production
-before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get install -y mysql-server
 script:
   - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake db:create db:migrate; else true; fi
   - if [ -z "${DOCKER_IMAGE}" ]; then bundle exec rake $RSPEC_TASK; else ./build_docker_image.sh; fi

From 2a524abff57d3d17f8e69d31b66e7f3997560470 Mon Sep 17 00:00:00 2001
From: Akinori MUSHA <knu@idaemons.org>
Date: Tue, 20 Dec 2016 10:26:39 +0900
Subject: [PATCH 17/23] Fix the regex pattern to reduce backtracking

This should fix #1832.
---
 app/models/agents/rss_agent.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb
index 0ab5719c..8e4a3a83 100644
--- a/app/models/agents/rss_agent.rb
+++ b/app/models/agents/rss_agent.rb
@@ -206,7 +206,7 @@ module Agents
       else
         # Encoding is already known, so do not let the parser detect
         # it from the XML declaration in the content.
-        body.sub!(/(\A\u{FEFF}?\s*<\?xml(?:\s+\w+\s*=\s*(['"]).*?\2)*)\s+encoding\s*=\s*(['"]).*?\3/, '\\1')
+        body.sub!(/(?<noenc>\A\u{FEFF}?\s*<\?xml(?:\s+\w+(?<av>\s*=\s*(?:'[^']*'|"[^"]*")))*?)\s+encoding\g<av>/, '\\k<noenc>')
       end
       body
     end

From 4e2d1775a6f1fc9ae8f6c7a6fe212813b8d7b5df Mon Sep 17 00:00:00 2001
From: Irfan Charania <irfancharania@users.noreply.github.com>
Date: Tue, 20 Dec 2016 09:44:54 -0800
Subject: [PATCH 18/23] PhantomJs Cloud Agent (#1503)

* Initial draft of PhantomJsCloudAgent

Generates event with url for fetching html/plainText content

* Add options

* Pass in event instead of url
Fix hash syntax
Remove whitespace
Add mode merge

* Add some tests

* Style changes

- Add link to wiki entry for manually creating agent with full set of
options
---
 app/models/agents/phantom_js_cloud_agent.rb   | 162 ++++++++++++++++++
 .../agents/phantom_js_cloud_agent_spec.rb     | 117 +++++++++++++
 2 files changed, 279 insertions(+)
 create mode 100644 app/models/agents/phantom_js_cloud_agent.rb
 create mode 100644 spec/models/agents/phantom_js_cloud_agent_spec.rb

diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb
new file mode 100644
index 00000000..f0c9e605
--- /dev/null
+++ b/app/models/agents/phantom_js_cloud_agent.rb
@@ -0,0 +1,162 @@
+require 'json'
+require 'uri'
+
+module Agents
+  class PhantomJsCloudAgent < Agent
+    include ERB::Util
+    include FormConfigurable
+    include WebRequestConcern
+
+    can_dry_run!
+
+    default_schedule 'every_12h'
+
+    description <<-MD
+      [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages.
+
+      The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
+      This url can then be supplied to Website Agent to fetch and parse content.
+
+      [Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
+
+
+      Options:
+
+      * `Api key` - PhantomJs Cloud API Key credential stored in Huginn
+      * `Url` - The url to render
+      * `Mode` - Create a new `clean` event or `merge` old payload with new values (default: `clean`)
+      * `Render type` - Render as html or plain text without html tags (default: `html`)
+      * `Output as json` - Return the page conents and metadata as a JSON object (default: `false`)
+      * `Ignore images` - Skip loading of inlined images (default: `false`)
+      * `Url agent` - A custom User-Agent name (default: `#{default_user_agent}`)
+      * `Wait interval` - Milliseconds to delay rendering after the last resource is finished loading.
+      This is useful in case there are any AJAX requests or animations that need to finish up.
+      This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
+
+
+      As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
+
+    MD
+
+    event_description <<-MD
+      Events look like this:
+          {
+            "url": "..."
+          }
+    MD
+
+    def default_options
+      {
+        'mode' => 'clean',
+        'url' => 'http://xkcd.com',
+        'render_type' => 'html',
+        'output_as_json' => false,
+        'ignore_images' => false,
+        'user_agent' => self.class.default_user_agent,
+        'wait_interval' => '1000'
+      }
+    end
+
+    form_configurable :mode, type: :array, values: ['clean', 'merge']
+    form_configurable :api_key, roles: :completable
+    form_configurable :url
+    form_configurable :render_type, type: :array, values: ['html', 'plainText']
+    form_configurable :output_as_json, type: :boolean
+    form_configurable :ignore_images, type: :boolean
+    form_configurable :user_agent, type: :text
+    form_configurable :wait_interval
+
+    def mode
+      interpolated['mode'].presence || default_options['mode']
+    end
+
+    def render_type
+      interpolated['render_type'].presence || default_options['render_type']
+    end
+
+    def output_as_json
+      boolify(interpolated['output_as_json'].presence ||
+      default_options['output_as_json'])
+    end
+
+    def ignore_images
+      boolify(interpolated['ignore_images'].presence ||
+      default_options['ignore_images'])
+    end
+
+    def user_agent
+      interpolated['user_agent'].presence || self.class.default_user_agent
+    end
+
+    def wait_interval
+      interpolated['wait_interval'].presence || default_options['wait_interval']
+    end
+
+    def page_request_settings
+      prs = {}
+
+      prs[:ignoreImages] = ignore_images if ignore_images
+      prs[:userAgent] = user_agent if user_agent.present?
+
+      if wait_interval != default_options['wait_interval']
+        prs[:wait_interval] = wait_interval
+      end
+
+      prs
+    end
+
+    def build_phantom_url(interpolated)
+      api_key = interpolated[:api_key]
+      page_request_hash = {
+        url: interpolated[:url],
+        renderType: render_type
+      }
+
+      page_request_hash[:outputAsJson] = output_as_json if output_as_json
+
+      page_request_settings_hash = page_request_settings
+
+      if page_request_settings_hash.any?
+        page_request_hash[:requestSettings] = page_request_settings_hash
+      end
+
+      request = page_request_hash.to_json
+      log "Generated request: #{request}"
+
+      encoded = url_encode(request)
+      "https://phantomjscloud.com/api/browser/v2/#{api_key}/?request=#{encoded}"
+    end
+
+    def check
+      phantom_url = build_phantom_url(interpolated)
+
+      create_event payload: { 'url' => phantom_url }
+    end
+
+    def receive(incoming_events)
+      incoming_events.each do |event|
+        interpolate_with(event) do
+          existing_payload = interpolated['mode'].to_s == 'merge' ? event.payload : {}
+          phantom_url = build_phantom_url(interpolated)
+
+          result = { 'url' => phantom_url }
+          create_event payload: existing_payload.merge(result)
+        end
+      end
+    end
+
+    def complete_api_key
+      user.user_credentials.map { |c| { text: c.credential_name, id: "{% credential #{c.credential_name} %}" } }
+    end
+
+    def working?
+      !recent_error_logs? || received_event_without_error?
+    end
+
+    def validate_options
+      # Check for required fields
+      errors.add(:base, 'Url is required') unless options['url'].present?
+      errors.add(:base, 'API key (credential) is required') unless options['api_key'].present?
+    end
+  end
+end
diff --git a/spec/models/agents/phantom_js_cloud_agent_spec.rb b/spec/models/agents/phantom_js_cloud_agent_spec.rb
new file mode 100644
index 00000000..07a79c2b
--- /dev/null
+++ b/spec/models/agents/phantom_js_cloud_agent_spec.rb
@@ -0,0 +1,117 @@
+require 'rails_helper'
+
+describe Agents::PhantomJsCloudAgent do
+  before do
+
+    @valid_options = {
+        'name' => "XKCD",
+        'render_type' => "html",
+        'url' => "http://xkcd.com",
+        'mode' => 'clean',
+        'api_key' => '1234567890'
+      }
+
+    @checker = Agents::PhantomJsCloudAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days)
+    @checker.user = users(:jane)
+    @checker.save!
+  end
+
+  describe "validations" do
+    before do
+      expect(@checker).to be_valid
+    end
+
+    it "should validate the presence of url" do
+      @checker.options['url'] = "http://google.com"
+      expect(@checker).to be_valid
+
+      @checker.options['url'] = ""
+      expect(@checker).not_to be_valid
+
+      @checker.options['url'] = nil
+      expect(@checker).not_to be_valid
+    end
+
+  end
+
+  describe "emitting event" do
+    it "should emit url as event" do
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+    end
+
+    it "should set render type as plain text" do
+      @checker.options['render_type'] = 'plainText'
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22plainText%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+    end
+
+    it "should set output as json" do
+      @checker.options['output_as_json'] = true
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22outputAsJson%22%3Atrue%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+    end
+
+    it "should not set ignore images" do
+      @checker.options['ignore_images'] = false
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+    end
+
+    it "should set ignore images" do
+      @checker.options['ignore_images'] = true
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22ignoreImages%22%3Atrue%2C%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%7D%7D")
+    end
+
+    it "should set wait interval to zero" do
+      @checker.options['wait_interval'] = '0'
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Huginn%20-%20https%3A%2F%2Fgithub.com%2Fcantino%2Fhuginn%22%2C%22wait_interval%22%3A%220%22%7D%7D")
+    end
+
+    it "should set user agent to BlackBerry" do
+      @checker.options['user_agent'] = 'Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+'
+
+      expect {
+        @checker.check
+      }.to change { @checker.events.count }.by(1)
+
+      item,* = @checker.events.last(1)
+      expect(item.payload['url']).to eq("https://phantomjscloud.com/api/browser/v2/1234567890/?request=%7B%22url%22%3A%22http%3A%2F%2Fxkcd.com%22%2C%22renderType%22%3A%22html%22%2C%22requestSettings%22%3A%7B%22userAgent%22%3A%22Mozilla%2F5.0%20%28BlackBerry%3B%20U%3B%20BlackBerry%209900%3B%20en%29%20AppleWebKit%2F534.11%2B%20%28KHTML%2C%20like%20Gecko%29%20Version%2F7.1.0.346%20Mobile%20Safari%2F534.11%2B%22%7D%7D")
+    end
+
+
+
+  end
+
+end

From 65cea03062dfc4aa7d0ebdacf2edde5a78e75f82 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@users.noreply.github.com>
Date: Tue, 20 Dec 2016 13:32:48 -0500
Subject: [PATCH 19/23] Fix spec failures

---
 app/models/agents/phantom_js_cloud_agent.rb | 9 ++++-----
 spec/features/create_an_agent_spec.rb       | 9 ++++-----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/app/models/agents/phantom_js_cloud_agent.rb b/app/models/agents/phantom_js_cloud_agent.rb
index f0c9e605..727de10e 100644
--- a/app/models/agents/phantom_js_cloud_agent.rb
+++ b/app/models/agents/phantom_js_cloud_agent.rb
@@ -12,13 +12,14 @@ module Agents
     default_schedule 'every_12h'
 
     description <<-MD
-      [PhantomJs Cloud](https://phantomjscloud.com/) renders webpages in much the same way as a browser would, and allows the Website Agent to properly scrape dynamic content from javascript-heavy pages.
+      This Agent generates [PhantomJs Cloud](https://phantomjscloud.com/) URLs that can be used to render JavaScript-heavy webpages for content extraction.
 
-      The Phantom Js Cloud Agent is used to formulate a url in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
-      This url can then be supplied to Website Agent to fetch and parse content.
+      URLs generated by this Agent are formulated in accordance with the [PhantomJs Cloud API](https://phantomjscloud.com/docs/index.html).
+      The generated URLs can then be supplied to a Website Agent to fetch and parse the content.
 
       [Sign up](https://dashboard.phantomjscloud.com/dash.html#/signup) to get an api key, and add it in Huginn credentials.
 
+      Please see the [Huginn Wiki for more info](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud).
 
       Options:
 
@@ -33,9 +34,7 @@ module Agents
       This is useful in case there are any AJAX requests or animations that need to finish up.
       This can safely be set to 0 if you know there are no AJAX or animations you need to wait for (default: `1000`ms)
 
-
       As this agent only provides a limited subset of the most commonly used options, you can follow [this guide](https://github.com/cantino/huginn/wiki/Browser-Emulation-Using-PhantomJS-Cloud) to make full use of additional options PhantomJsCloud provides.
-
     MD
 
     event_description <<-MD
diff --git a/spec/features/create_an_agent_spec.rb b/spec/features/create_an_agent_spec.rb
index ac2ce295..ae725630 100644
--- a/spec/features/create_an_agent_spec.rb
+++ b/spec/features/create_an_agent_spec.rb
@@ -38,7 +38,7 @@ describe "Creating a new agent", js: true do
       visit new_agent_path
     end
     it "shows all options for agents that can be scheduled, create and receive events" do
-      select2("Website Agent", from: "Type")
+      select2("Website Agent scrapes", from: "Type")
       expect(page).not_to have_content('This type of Agent cannot create events.')
     end
 
@@ -49,9 +49,8 @@ describe "Creating a new agent", js: true do
   end
 
   it "allows to click on on the agent name in select2 tags" do
-    agent = agents(:bob_weather_agent)
     visit new_agent_path
-    select2("Website Agent", from: "Type")
+    select2("Website Agent scrapes", from: "Type")
     select2("SF Weather", from: 'Sources')
     click_on "SF Weather"
     expect(page).to have_content "Editing your WeatherAgent"
@@ -63,7 +62,7 @@ describe "Creating a new agent", js: true do
     end
 
     it "does not send previously configured sources when the current agent does not support them" do
-      select2("Website Agent", from: "Type")
+      select2("Website Agent scrapes", from: "Type")
       select2("SF Weather", from: 'Sources')
       select2("Webhook Agent", from: "Type")
       fill_in(:agent_name, with: "No sources")
@@ -85,7 +84,7 @@ describe "Creating a new agent", js: true do
     end
 
     it "does not send previously configured receivers when the current agent does not support them" do
-      select2("Website Agent", from: "Type")
+      select2("Website Agent scrapes", from: "Type")
       select2("ZKCD", from: 'Receivers')
       select2("Email Agent", from: "Type")
       fill_in(:agent_name, with: "No receivers")

From db4de696b4d71169ca9733ce0fd473a5cdc7c334 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Sun, 27 Nov 2016 12:30:45 +0100
Subject: [PATCH 20/23] Override `git_source` method in the Gemfile

When the bundler version is below 2 override `git_source` to ensure
`github` remotes always use HTTPS
---
 Gemfile | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Gemfile b/Gemfile
index f6fec106..95608b5b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -3,6 +3,12 @@ source 'https://rubygems.org'
 # Ruby 2.2.2 is the minimum requirement
 ruby ['2.2.2', RUBY_VERSION].max
 
+# Ensure github repositories are fetched using HTTPS
+git_source(:github) do |repo_name|
+  repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/")
+  "https://github.com/#{repo_name}.git"
+end if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('2')
+
 # Load vendored dotenv gem and .env file
 require File.join(File.dirname(__FILE__), 'lib/gemfile_helper.rb')
 GemfileHelper.load_dotenv do |dotenv_dir|
@@ -50,7 +56,7 @@ gem 'twitter-stream', github: 'cantino/twitter-stream', branch: 'huginn'
 gem 'omniauth-twitter', '~> 1.2.1'
 
 # Tumblr Agents
-gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master'  # '>= 0.8.5'
+gem 'tumblr_client', github: 'tumblr/tumblr_client', branch: 'master', ref: '0c59b04e49f2a8c89860613b18cf4e8f978d8dc7'  # '>= 0.8.5'
 gem 'omniauth-tumblr', '~> 1.2'
 
 # Dropbox Agents
@@ -103,7 +109,7 @@ gem 'jquery-rails', '~> 4.2.1'
 gem 'huginn_agent', '~> 0.4.0'
 gem 'json', '~> 1.8.1'
 gem 'jsonpathv2', '~> 0.0.8'
-gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable'
+gem 'kaminari', github: "amatsuda/kaminari", branch: '0-17-stable', ref: 'abbf93d557208ee1d0b612c612cd079f86ed54f4'
 gem 'kramdown', '~> 1.3.3'
 gem 'liquid', '~> 3.0.3'
 gem 'loofah', '~> 2.0'

From f813a73dd067ee33dedd9fe46ffc6569a960ae3b Mon Sep 17 00:00:00 2001
From: Alex Jordan <alex@strugee.net>
Date: Mon, 31 Oct 2016 18:39:34 -0700
Subject: [PATCH 21/23] Update Gemfile.lock with HTTPS URLs for git gems

---
 Gemfile.lock | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index ee694741..1542b084 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,7 @@
 GIT
-  remote: git://github.com/amatsuda/kaminari.git
+  remote: https://github.com/amatsuda/kaminari.git
   revision: abbf93d557208ee1d0b612c612cd079f86ed54f4
+  ref: abbf93d557208ee1d0b612c612cd079f86ed54f4
   branch: 0-17-stable
   specs:
     kaminari (0.17.0)
@@ -8,7 +9,7 @@ GIT
       activesupport (>= 3.0.0)
 
 GIT
-  remote: git://github.com/cantino/twitter-stream.git
+  remote: https://github.com/cantino/twitter-stream.git
   revision: f7e7edb0bae013bffabf3598e7147773d9fd370f
   branch: huginn
   specs:
@@ -18,7 +19,7 @@ GIT
       simple_oauth (~> 0.3.0)
 
 GIT
-  remote: git://github.com/dsander/delayed_job_active_record.git
+  remote: https://github.com/dsander/delayed_job_active_record.git
   revision: b314972ccc92e0e8b03b1589174d8fb9a82b3cd0
   branch: rails5
   specs:
@@ -27,7 +28,7 @@ GIT
       delayed_job (>= 3.0, < 5)
 
 GIT
-  remote: git://github.com/dsander/weibo_2.git
+  remote: https://github.com/dsander/weibo_2.git
   revision: e5b77f21a7e9a666b582c48e16b1e96fca198cf8
   branch: master
   specs:
@@ -38,7 +39,7 @@ GIT
       rest-client (~> 1.8)
 
 GIT
-  remote: git://github.com/lostisland/faraday_middleware.git
+  remote: https://github.com/lostisland/faraday_middleware.git
   revision: 59088da02940d0ee2010b2e3156343346767c31e
   branch: master
   specs:
@@ -46,8 +47,9 @@ GIT
       faraday (>= 0.7.4, < 1.0)
 
 GIT
-  remote: git://github.com/tumblr/tumblr_client.git
+  remote: https://github.com/tumblr/tumblr_client.git
   revision: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
+  ref: 0c59b04e49f2a8c89860613b18cf4e8f978d8dc7
   branch: master
   specs:
     tumblr_client (0.8.5)
@@ -703,4 +705,4 @@ RUBY VERSION
    ruby 2.3.3p222
 
 BUNDLED WITH
-   1.13.6
+   1.13.7

From 0285a1a9933db7b05ae7e0ead5c0e7660e72e731 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Tue, 27 Dec 2016 23:35:06 +0100
Subject: [PATCH 22/23] Update rails to 5.0.1

---
 Gemfile      |  4 +--
 Gemfile.lock | 80 ++++++++++++++++++++++++++--------------------------
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/Gemfile b/Gemfile
index 95608b5b..016c72dd 100644
--- a/Gemfile
+++ b/Gemfile
@@ -117,7 +117,7 @@ gem 'mini_magick'
 gem 'multi_xml'
 gem 'nokogiri'
 gem 'omniauth', '~> 1.3.1'
-gem 'rails', '~> 5.0.0.1'
+gem 'rails', '~> 5.0.1'
 gem 'rufus-scheduler', '~> 3.0.8', require: false
 gem 'sass-rails',   '~> 5.0.6'
 gem 'select2-rails', '~> 3.5.4'
@@ -134,7 +134,7 @@ group :development do
   gem 'guard-rspec', '~> 4.6.4'
   gem 'rack-livereload', '~> 0.3.16'
   gem 'letter_opener_web', '~> 1.3.0'
-  gem 'web-console'
+  gem 'web-console', '>= 3.3.0'
 
   gem 'capistrano', '~> 3.4.0'
   gem 'capistrano-rails', '~> 1.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1542b084..4bf8109d 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -71,45 +71,45 @@ GEM
   remote: https://rubygems.org/
   specs:
     ace-rails-ap (2.0.1)
-    actioncable (5.0.0.1)
-      actionpack (= 5.0.0.1)
+    actioncable (5.0.1)
+      actionpack (= 5.0.1)
       nio4r (~> 1.2)
       websocket-driver (~> 0.6.1)
-    actionmailer (5.0.0.1)
-      actionpack (= 5.0.0.1)
-      actionview (= 5.0.0.1)
-      activejob (= 5.0.0.1)
+    actionmailer (5.0.1)
+      actionpack (= 5.0.1)
+      actionview (= 5.0.1)
+      activejob (= 5.0.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.0.0.1)
-      actionview (= 5.0.0.1)
-      activesupport (= 5.0.0.1)
+    actionpack (5.0.1)
+      actionview (= 5.0.1)
+      activesupport (= 5.0.1)
       rack (~> 2.0)
       rack-test (~> 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.0.0.1)
-      activesupport (= 5.0.0.1)
+    actionview (5.0.1)
+      activesupport (= 5.0.1)
       builder (~> 3.1)
       erubis (~> 2.7.0)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    activejob (5.0.0.1)
-      activesupport (= 5.0.0.1)
+    activejob (5.0.1)
+      activesupport (= 5.0.1)
       globalid (>= 0.3.6)
-    activemodel (5.0.0.1)
-      activesupport (= 5.0.0.1)
-    activerecord (5.0.0.1)
-      activemodel (= 5.0.0.1)
-      activesupport (= 5.0.0.1)
+    activemodel (5.0.1)
+      activesupport (= 5.0.1)
+    activerecord (5.0.1)
+      activemodel (= 5.0.1)
+      activesupport (= 5.0.1)
       arel (~> 7.0)
-    activesupport (5.0.0.1)
+    activesupport (5.0.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     addressable (2.3.8)
-    arel (7.1.1)
+    arel (7.1.4)
     autoparse (0.3.3)
       addressable (>= 2.3.1)
       extlib (>= 0.9.15)
@@ -159,7 +159,7 @@ GEM
       execjs
     coffee-script-source (1.10.0)
     colorize (0.7.7)
-    concurrent-ruby (1.0.2)
+    concurrent-ruby (1.0.3)
     cookiejar (0.3.2)
     coveralls (0.7.12)
       multi_json (~> 1.10)
@@ -329,7 +329,7 @@ GEM
     mimemagic (0.3.1)
     mini_magick (4.2.3)
     mini_portile2 (2.1.0)
-    minitest (5.9.0)
+    minitest (5.10.1)
     mqtt (0.3.1)
     multi_json (1.12.1)
     multi_xml (0.5.5)
@@ -404,17 +404,17 @@ GEM
       rack
     rack-test (0.6.3)
       rack (>= 1.0)
-    rails (5.0.0.1)
-      actioncable (= 5.0.0.1)
-      actionmailer (= 5.0.0.1)
-      actionpack (= 5.0.0.1)
-      actionview (= 5.0.0.1)
-      activejob (= 5.0.0.1)
-      activemodel (= 5.0.0.1)
-      activerecord (= 5.0.0.1)
-      activesupport (= 5.0.0.1)
+    rails (5.0.1)
+      actioncable (= 5.0.1)
+      actionmailer (= 5.0.1)
+      actionpack (= 5.0.1)
+      actionview (= 5.0.1)
+      activejob (= 5.0.1)
+      activemodel (= 5.0.1)
+      activerecord (= 5.0.1)
+      activesupport (= 5.0.1)
       bundler (>= 1.3.0, < 2.0)
-      railties (= 5.0.0.1)
+      railties (= 5.0.1)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.1)
       actionpack (~> 5.x)
@@ -425,14 +425,14 @@ GEM
       nokogiri (~> 1.6.0)
     rails-html-sanitizer (1.0.3)
       loofah (~> 2.0)
-    railties (5.0.0.1)
-      actionpack (= 5.0.0.1)
-      activesupport (= 5.0.0.1)
+    railties (5.0.1)
+      actionpack (= 5.0.1)
+      activesupport (= 5.0.1)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
     raindrops (0.17.0)
-    rake (11.2.2)
+    rake (12.0.0)
     rb-fsevent (0.9.7)
     rb-inotify (0.9.5)
       ffi (>= 0.5.0)
@@ -516,7 +516,7 @@ GEM
     spring-watcher-listen (2.0.0)
       listen (>= 2.7, < 4.0)
       spring (~> 1.2)
-    sprockets (3.7.0)
+    sprockets (3.7.1)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.0)
@@ -533,7 +533,7 @@ GEM
     therubyracer (0.12.2)
       libv8 (~> 3.16.14.0)
       ref
-    thor (0.19.1)
+    thor (0.19.4)
     thread_safe (0.3.5)
     tilt (2.0.5)
     tins (1.10.1)
@@ -665,7 +665,7 @@ DEPENDENCIES
   pry-byebug
   pry-rails
   rack-livereload (~> 0.3.16)
-  rails (~> 5.0.0.1)
+  rails (~> 5.0.1)
   rails-controller-testing
   rb-kqueue (>= 0.2)
   rr
@@ -695,7 +695,7 @@ DEPENDENCIES
   uglifier (~> 2.7.2)
   unicorn (~> 5.1.0)
   vcr
-  web-console
+  web-console (>= 3.3.0)
   webmock (~> 1.17.4)
   weibo_2!
   wunderground (~> 1.2.0)

From 070946e879872de609524b6434cc170207cada12 Mon Sep 17 00:00:00 2001
From: Dominik Sander <git@dsander.de>
Date: Tue, 27 Dec 2016 23:35:29 +0100
Subject: [PATCH 23/23] Work around possible regression in rails

`pluck` used to always trigger a database query. With Rails 5.0.1 it uses data from
already loaded associations, this also includes new records for which we can not
generate agent_paths
---
 app/views/layouts/application.html.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index 561f9a94..69f81378 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -52,7 +52,7 @@
       window.agentPaths = {};
       window.agentNames = [];
       <% if current_user.present? -%>
-        var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| m[a.first] = agent_path(a.last); m }) %>;
+        var myAgents = <%= Utils.jsonify(current_user.agents.pluck(:name, :id).inject({}) {|m, a| next if a.last.nil?; m[a.first] = agent_path(a.last); m }) %>;
         var myScenarios = <%= Utils.jsonify(current_user.scenarios.pluck(:name, :id).inject({}) {|m, s| m[s.first + " Scenario"] = scenario_path(s.last); m }) %>;
         $.extend(window.agentPaths, myAgents);
         $.extend(window.agentPaths, myScenarios);