From 77882908c4b1b0fc0a3dca1773d8471f156864d4 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Sun, 22 Dec 2013 15:36:29 -0800
Subject: [PATCH 01/10] in order to avoid leaking memory, remove all
 symbolization

---
 app/concerns/email_concern.rb                 |  2 +-
 app/models/agent.rb                           |  6 +--
 app/models/agents/adioso_agent.rb             |  2 +-
 app/models/agents/human_task_agent.rb         | 28 +++++------
 app/models/agents/peak_detector_agent.rb      |  2 +-
 app/models/agents/twilio_agent.rb             |  6 +--
 app/models/agents/twitter_stream_agent.rb     | 10 ++--
 app/models/event.rb                           | 12 ++---
 .../recursively_symbolize_keys.rb             |  7 ---
 lib/serialize_and_normalize.rb                | 46 +++++++++++++++++++
 lib/serialize_and_symbolize.rb                | 43 -----------------
 lib/utils.rb                                  | 11 -----
 spec/controllers/agents_controller_spec.rb    |  2 +-
 .../user_location_updates_controller_spec.rb  |  6 +--
 spec/controllers/webhooks_controller_spec.rb  |  4 +-
 spec/models/agents/human_task_agent_spec.rb   | 38 +++++++--------
 spec/models/agents/sentiment_agent_spec.rb    |  2 +-
 .../agents/twitter_stream_agent_spec.rb       | 14 +++---
 18 files changed, 112 insertions(+), 129 deletions(-)
 delete mode 100644 config/initializers/recursively_symbolize_keys.rb
 create mode 100644 lib/serialize_and_normalize.rb
 delete mode 100644 lib/serialize_and_symbolize.rb

diff --git a/app/concerns/email_concern.rb b/app/concerns/email_concern.rb
index 9a311b80..455acdcb 100644
--- a/app/concerns/email_concern.rb
+++ b/app/concerns/email_concern.rb
@@ -1,7 +1,7 @@
 module EmailConcern
   extend ActiveSupport::Concern
 
-  MAIN_KEYS = %w[title message text main value].map(&:to_sym)
+  MAIN_KEYS = %w[title message text main value]
 
   included do
     self.validate :validate_email_options
diff --git a/app/models/agent.rb b/app/models/agent.rb
index 8ee4f9cf..ad4d04fb 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -1,14 +1,14 @@
-require 'serialize_and_symbolize'
+require 'serialize_and_normalize'
 require 'assignable_types'
 require 'markdown_class_attributes'
 require 'utils'
 
 class Agent < ActiveRecord::Base
-  include SerializeAndSymbolize
+  include SerializeAndNormalize
   include AssignableTypes
   include MarkdownClassAttributes
 
-  serialize_and_symbolize :options, :memory
+  serialize_and_normalize :options, :memory
   markdown_class_attributes :description, :event_description
 
   load_types_in "Agents"
diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb
index 2b08feec..3055d2f5 100644
--- a/app/models/agents/adioso_agent.rb
+++ b/app/models/agents/adioso_agent.rb
@@ -44,7 +44,7 @@ module Agents
     end
 
     def validate_options
-			unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field.to_sym].present? }
+			unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field].present? }
 				errors.add(:base, "All fields are required")
 			end
 		end
diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb
index 65a73c5f..116347e6 100644
--- a/app/models/agents/human_task_agent.rb
+++ b/app/models/agents/human_task_agent.rb
@@ -214,12 +214,12 @@ module Agents
           end
 
           event = create_event :payload => payload
-          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
+          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id])
 
           assignments.each(&:approve!)
           hit.dispose!
 
-          memory[:hits].delete(hit_id.to_sym)
+          memory[:hits].delete(hit_id)
         end
       end
     end
@@ -268,34 +268,34 @@ module Agents
         @questions.each.with_index do |question, index|
           Question do
             QuestionIdentifier do
-              text question[:key] || "question_#{index}"
+              text question['key'] || "question_#{index}"
             end
             DisplayName do
-              text question[:name] || "Question ##{index}"
+              text question['name'] || "Question ##{index}"
             end
             IsRequired do
-              text question[:required] || 'true'
+              text question['required'] || 'true'
             end
             QuestionContent do
               Text do
-                text question[:question]
+                text question['question']
               end
             end
             AnswerSpecification do
-              if question[:type] == "selection"
+              if question['type'] == "selection"
 
                 SelectionAnswer do
                   StyleSuggestion do
                     text 'radiobutton'
                   end
                   Selections do
-                    question[:selections].each do |selection|
+                    question['selections'].each do |selection|
                       Selection do
                         SelectionIdentifier do
-                          text selection[:key]
+                          text selection['key']
                         end
                         Text do
-                          text selection[:text]
+                          text selection['text']
                         end
                       end
                     end
@@ -305,18 +305,18 @@ module Agents
               else
 
                 FreeTextAnswer do
-                  if question[:min_length].present? || question[:max_length].present?
+                  if question['min_length'].present? || question['max_length'].present?
                     Constraints do
                       lengths = {}
-                      lengths[:minLength] = question[:min_length].to_s if question[:min_length].present?
-                      lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present?
+                      lengths['minLength'] = question['min_length'].to_s if question['min_length'].present?
+                      lengths['maxLength'] = question['max_length'].to_s if question['max_length'].present?
                       Length lengths
                     end
                   end
 
                   if question[:default].present?
                     DefaultText do
-                      text question[:default]
+                      text question['default']
                     end
                   end
                 end
diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb
index a7f3c4e3..7c7e7468 100644
--- a/app/models/agents/peak_detector_agent.rb
+++ b/app/models/agents/peak_detector_agent.rb
@@ -114,7 +114,7 @@ module Agents
     end
 
     def group_for(event)
-      ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group').to_sym
+      ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group')
     end
 
     def remember(group, event)
diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb
index f22ef31b..82076e89 100644
--- a/app/models/agents/twilio_agent.rb
+++ b/app/models/agents/twilio_agent.rb
@@ -79,9 +79,9 @@ module Agents
     end
 
     def receive_webhook(params)
-      if memory[:pending_calls].has_key? params[:secret].to_sym
-        response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret].to_sym], :voice => 'woman'}
-        memory[:pending_calls].delete params[:secret].to_sym
+      if memory[:pending_calls].has_key? params[:secret]
+        response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'}
+        memory[:pending_calls].delete params[:secret]
         [response.text, 200]
       end
     end
diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb
index d21bcf75..96afae25 100644
--- a/app/models/agents/twitter_stream_agent.rb
+++ b/app/models/agents/twitter_stream_agent.rb
@@ -85,12 +85,12 @@ module Agents
           # Avoid memory pollution by reloading the Agent.
           agent = Agent.find(id)
           agent.memory[:filter_counts] ||= {}
-          agent.memory[:filter_counts][filter.to_sym] ||= 0
-          agent.memory[:filter_counts][filter.to_sym] += 1
+          agent.memory[:filter_counts][filter] ||= 0
+          agent.memory[:filter_counts][filter] += 1
           remove_unused_keys!(agent, :filter_counts)
           agent.save!
         else
-          create_event :payload => status.merge(:filter => filter.to_s)
+          create_event :payload => status.merge(:filter => filter)
         end
       end
     end
@@ -98,7 +98,7 @@ module Agents
     def check
       if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0
         memory[:filter_counts].each do |filter, count|
-          create_event :payload => { :filter => filter.to_s, :count => count, :time => Time.now.to_i }
+          create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i }
         end
       end
       memory[:filter_counts] = {}
@@ -120,7 +120,7 @@ module Agents
 
     def remove_unused_keys!(agent, base)
       if agent.memory[base]
-        (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_sym : f.to_sym }).each do |removed_key|
+        (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
           agent.memory[base].delete(removed_key)
         end
       end
diff --git a/app/models/event.rb b/app/models/event.rb
index 4aaddf1c..f7c99256 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -1,23 +1,21 @@
+require 'serialize_and_normalize'
+
 class Event < ActiveRecord::Base
+  include SerializeAndNormalize
+
   attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at
 
   acts_as_mappable
 
-  serialize :payload
+  serialize_and_normalize :payload
 
   belongs_to :user
   belongs_to :agent, :counter_cache => true
 
-  before_save :symbolize_payload
-
   scope :recent, lambda { |timespan = 12.hours.ago|
     where("events.created_at > ?", timespan)
   }
 
-  def symbolize_payload
-    self.payload = payload.recursively_symbolize_keys if payload.is_a?(Hash)
-  end
-
   def reemit!
     agent.create_event :payload => payload, :lat => lat, :lng => lng
   end
diff --git a/config/initializers/recursively_symbolize_keys.rb b/config/initializers/recursively_symbolize_keys.rb
deleted file mode 100644
index 939e55a1..00000000
--- a/config/initializers/recursively_symbolize_keys.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-require 'utils'
-
-class Hash
-  def recursively_symbolize_keys
-    Utils.recursively_symbolize_keys self
-  end
-end
diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb
new file mode 100644
index 00000000..842c95dc
--- /dev/null
+++ b/lib/serialize_and_normalize.rb
@@ -0,0 +1,46 @@
+module SerializeAndNormalize
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def serialize_and_normalize(*column_names)
+      column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
+        setup_name = "setup_#{column_name}".to_sym
+        normalize_name = "normalize_#{column_name}".to_sym
+        validate_name = "validate_#{column_name}".to_sym
+
+        serialize column_name
+        after_initialize setup_name
+        before_validation normalize_name
+        before_save normalize_name
+        validate validate_name
+
+        class_eval <<-RUBY
+          def #{setup_name}
+            self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new
+          end
+
+          def #{validate_name}
+            # Implement me in your subclass.
+          end
+
+          def #{normalize_name}
+            self.#{column_name} = self[:#{column_name}]
+          end
+
+          def #{column_name}=(data)
+            data = (JSON.parse(data) rescue data) if data.is_a?(String)
+
+            case data
+              when ActiveSupport::HashWithIndifferentAccess
+                self[:#{column_name}] = data
+              when Hash
+                self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data)
+              else
+                self[:#{column_name}] = data
+            end
+          end
+        RUBY
+      end
+    end
+  end
+end
diff --git a/lib/serialize_and_symbolize.rb b/lib/serialize_and_symbolize.rb
deleted file mode 100644
index f051abb5..00000000
--- a/lib/serialize_and_symbolize.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-module SerializeAndSymbolize
-  extend ActiveSupport::Concern
-
-  module ClassMethods
-    def serialize_and_symbolize(*column_names)
-      column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
-        setup_name = "setup_#{column_name}".to_sym
-        symbolize_name = "symbolize_#{column_name}".to_sym
-        validate_name = "validate_#{column_name}".to_sym
-
-        serialize column_name
-        after_initialize setup_name
-        before_validation symbolize_name
-        before_save symbolize_name
-        validate validate_name
-
-        class_eval <<-RUBY
-          def #{setup_name}
-            self[:#{column_name}] ||= {}
-          end
-
-          def #{validate_name}
-            # Implement me in your subclass.
-          end
-
-          def #{symbolize_name}
-            self.#{column_name} = self[:#{column_name}]
-          end
-
-          def #{column_name}=(data)
-            if data.is_a?(String)
-              self[:#{column_name}] = JSON.parse(data).recursively_symbolize_keys rescue {}
-            elsif data.is_a?(Hash)
-              self[:#{column_name}] = data.recursively_symbolize_keys
-            else
-              self[:#{column_name}] = data
-            end
-          end
-        RUBY
-      end
-    end
-  end
-end
diff --git a/lib/utils.rb b/lib/utils.rb
index fae7577e..0d031ae2 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -21,17 +21,6 @@ module Utils
     end
   end
 
-  def self.recursively_symbolize_keys(object)
-    case object
-      when Hash
-        object.inject({}) {|memo, (k, v)| memo[String === k ? k.to_sym : k] = recursively_symbolize_keys(v); memo }
-      when Array
-        object.map { |item| recursively_symbolize_keys item }
-      else
-        object
-    end
-  end
-
   def self.interpolate_jsonpaths(value, data)
     value.gsub(/<[^>]+>/).each { |jsonpath|
       Utils.values_at(data, jsonpath[1..-2]).first.to_s
diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb
index e36efa6b..4b5c164a 100644
--- a/spec/controllers/agents_controller_spec.rb
+++ b/spec/controllers/agents_controller_spec.rb
@@ -23,7 +23,7 @@ describe AgentsController do
       sign_in users(:bob)
       post :handle_details_post, :id => agents(:bob_manual_event_agent).to_param, :payload => { :foo => "bar" }
       JSON.parse(response.body).should == { "success" => true }
-      agents(:bob_manual_event_agent).events.last.payload.should == { :foo => "bar" }
+      agents(:bob_manual_event_agent).events.last.payload.should == { 'foo' => "bar" }
     end
 
     it "can only be accessed by the Agent's owner" do
diff --git a/spec/controllers/user_location_updates_controller_spec.rb b/spec/controllers/user_location_updates_controller_spec.rb
index 79c211a4..08ca9192 100644
--- a/spec/controllers/user_location_updates_controller_spec.rb
+++ b/spec/controllers/user_location_updates_controller_spec.rb
@@ -8,7 +8,7 @@ describe UserLocationUpdatesController do
 
   it "should create events without requiring login" do
     post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else"
-    @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
+    @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
     @agent.events.last.lat.should == 45
     @agent.events.last.lng.should == 123
   end
@@ -18,7 +18,7 @@ describe UserLocationUpdatesController do
     @jane_agent.save!
 
     post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else"
-    @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
+    @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
     @jane_agent.events.should be_empty
   end
 
@@ -33,7 +33,7 @@ describe UserLocationUpdatesController do
 
     lambda {
       post :create, :user_id => users(:bob).to_param, :secret => "my_secret2", :longitude => 123, :latitude => 45, :something => "else"
-      @agent2.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" }
+      @agent2.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" }
     }.should_not change { @agent.events.count }
   end
 end
\ No newline at end of file
diff --git a/spec/controllers/webhooks_controller_spec.rb b/spec/controllers/webhooks_controller_spec.rb
index 8a42f8ec..754d8e7d 100644
--- a/spec/controllers/webhooks_controller_spec.rb
+++ b/spec/controllers/webhooks_controller_spec.rb
@@ -32,12 +32,12 @@ describe WebhooksController do
 
   it "should call receive_webhook" do
     post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5"
-    @agent.reload.memory[:webhook_values].should == { :key => "value", :another_key => "5" }
+    @agent.reload.memory[:webhook_values].should == { 'key' => "value", 'another_key' => "5" }
     response.body.should == "success"
     response.should be_success
 
     post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "not_my_secret", :no => "go"
-    @agent.reload.memory[:webhook_values].should_not == { :no => "go" }
+    @agent.reload.memory[:webhook_values].should_not == { 'no' => "go" }
     response.body.should == "failure"
     response.should be_missing
   end
diff --git a/spec/models/agents/human_task_agent_spec.rb b/spec/models/agents/human_task_agent_spec.rb
index 38cb6932..054ea4cf 100644
--- a/spec/models/agents/human_task_agent_spec.rb
+++ b/spec/models/agents/human_task_agent_spec.rb
@@ -288,7 +288,7 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
+      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
     end
 
     it "shouldn't do anything if an assignment is missing" do
@@ -306,7 +306,7 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
+      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
     end
 
     it "should create events when all assignments are ready" do
@@ -328,8 +328,8 @@ describe Agents::HumanTaskAgent do
       hit.should be_disposed
 
       @checker.events.last.payload[:answers].should == [
-        {:sentiment => "neutral", :feedback => ""},
-        {:sentiment => "happy", :feedback => "Take 2"}
+        {'sentiment' => "neutral", 'feedback' => ""},
+        {'sentiment' => "happy", 'feedback' => "Take 2"}
       ]
 
       @checker.memory[:hits].should == {}
@@ -338,7 +338,7 @@ describe Agents::HumanTaskAgent do
     describe "taking majority votes" do
       before do
         @checker.options[:take_majority] = "true"
-        @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
+        @checker.memory[:hits] = { "JH3132836336DHG" => @event.id }
         mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       end
 
@@ -372,14 +372,14 @@ describe Agents::HumanTaskAgent do
         assignments.all? {|a| a.approved == true }.should be_true
 
         @checker.events.last.payload[:answers].should == [
-          { :sentiment => "sad", :age_range => "<50" },
-          { :sentiment => "neutral", :age_range => ">50" },
-          { :sentiment => "happy", :age_range => ">50" },
-          { :sentiment => "happy", :age_range => ">50" }
+          { 'sentiment' => "sad", 'age_range' => "<50" },
+          { 'sentiment' => "neutral", 'age_range' => ">50" },
+          { 'sentiment' => "happy", 'age_range' => ">50" },
+          { 'sentiment' => "happy", 'age_range' => ">50" }
         ]
 
-        @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } }
-        @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" }
+        @checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
+        @checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" }
         @checker.events.last.payload.should_not have_key(:average_answer)
 
         @checker.memory[:hits].should == {}
@@ -421,16 +421,16 @@ describe Agents::HumanTaskAgent do
         assignments.all? {|a| a.approved == true }.should be_true
 
         @checker.events.last.payload[:answers].should == [
-          { :rating => "1" },
-          { :rating => "3" },
-          { :rating => "5.1" },
-          { :rating => "2" },
-          { :rating => "2" }
+          { 'rating' => "1" },
+          { 'rating' => "3" },
+          { 'rating' => "5.1" },
+          { 'rating' => "2" },
+          { 'rating' => "2" }
         ]
 
-        @checker.events.last.payload[:counts].should == { :rating => { :"1" => 1, :"2" => 2, :"3" => 1, :"4" => 0, :"5.1" => 1 } }
-        @checker.events.last.payload[:majority_answer].should == { :rating => "2" }
-        @checker.events.last.payload[:average_answer].should == { :rating => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
+        @checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
+        @checker.events.last.payload[:majority_answer].should == { 'rating' => "2" }
+        @checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
 
         @checker.memory[:hits].should == {}
       end
diff --git a/spec/models/agents/sentiment_agent_spec.rb b/spec/models/agents/sentiment_agent_spec.rb
index 142abad8..f4afbf52 100644
--- a/spec/models/agents/sentiment_agent_spec.rb
+++ b/spec/models/agents/sentiment_agent_spec.rb
@@ -53,7 +53,7 @@ describe Agents::SentimentAgent do
         it "checks if content key is working fine" do
             @checker.receive([@event])
             Event.last.payload[:content].should == "value1"
-            Event.last.payload[:original_event].should == {:message => "value1"}
+            Event.last.payload[:original_event].should == { 'message' => "value1" }
         end
         it "should handle multiple events" do
             event1 = Event.new
diff --git a/spec/models/agents/twitter_stream_agent_spec.rb b/spec/models/agents/twitter_stream_agent_spec.rb
index 353a0258..816decb9 100644
--- a/spec/models/agents/twitter_stream_agent_spec.rb
+++ b/spec/models/agents/twitter_stream_agent_spec.rb
@@ -53,7 +53,7 @@ describe Agents::TwitterStreamAgent do
         @agent.memory[:filter_counts] = {:keyword1 => 2, :keyword2 => 3, :keyword3 => 4}
         @agent.save!
         @agent.process_tweet('keyword1', {:text => "something", :user => {:name => "Mr. Someone"}})
-        @agent.reload.memory[:filter_counts].should == {:keyword1 => 3, :keyword2 => 3}
+        @agent.reload.memory[:filter_counts].should == { 'keyword1' => 3, 'keyword2' => 3 }
       end
     end
 
@@ -64,9 +64,9 @@ describe Agents::TwitterStreamAgent do
         }.should change { @agent.events.count }.by(1)
 
         @agent.events.last.payload.should == {
-          :filter => 'keyword1',
-          :text => "something",
-          :user => {:name => "Mr. Someone"}
+          'filter' => 'keyword1',
+          'text' => "something",
+          'user' => { 'name' => "Mr. Someone" }
         }
       end
 
@@ -79,9 +79,9 @@ describe Agents::TwitterStreamAgent do
         }.should change { @agent.events.count }.by(1)
 
         @agent.events.last.payload.should == {
-          :filter => 'keyword1-1',
-          :text => "something",
-          :user => {:name => "Mr. Someone"}
+          'filter' => 'keyword1-1',
+          'text' => "something",
+          'user' => { 'name' => "Mr. Someone" }
         }
       end
     end

From ec32d7f9794209802fe650f83ba4cf6426fe0279 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Mon, 23 Dec 2013 00:54:20 -0500
Subject: [PATCH 02/10] switch from yaml to json serialization

---
 ...1223032112_switch_to_json_serialization.rb | 41 +++++++++++++++++++
 lib/serialize_and_normalize.rb                |  2 +-
 2 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20131223032112_switch_to_json_serialization.rb

diff --git a/db/migrate/20131223032112_switch_to_json_serialization.rb b/db/migrate/20131223032112_switch_to_json_serialization.rb
new file mode 100644
index 00000000..c132d532
--- /dev/null
+++ b/db/migrate/20131223032112_switch_to_json_serialization.rb
@@ -0,0 +1,41 @@
+class SwitchToJsonSerialization < ActiveRecord::Migration
+  FIELDS = {
+    :agents => [:options, :memory],
+    :events => [:payload]
+  }
+
+  def up
+    puts "This migration will update Agent and Event storage from YAML to JSON.  It should work, but please make a backup"
+    puts "before proceeding."
+    print "Continue? (y/n) "
+    STDOUT.flush
+    exit unless STDIN.gets =~ /^y/i
+
+    translate YAML, JSON
+  end
+
+  def down
+    translate JSON, YAML
+  end
+
+  def translate(from, to)
+    FIELDS.each do |table, fields|
+      quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table)
+      fields = fields.map { |f| ActiveRecord::Base.connection.quote_column_name(f) }
+
+      rows = ActiveRecord::Base.connection.select_rows("SELECT id, #{fields.join(", ")} FROM #{quoted_table_name}")
+      rows.each do |row|
+        id, *field_data = row
+
+        yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) }
+
+        update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?"
+
+        sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id]
+
+        ActiveRecord::Base.connection.execute sanitized_update_sql
+      end
+    end
+
+  end
+end
diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb
index 842c95dc..857a777f 100644
--- a/lib/serialize_and_normalize.rb
+++ b/lib/serialize_and_normalize.rb
@@ -8,7 +8,7 @@ module SerializeAndNormalize
         normalize_name = "normalize_#{column_name}".to_sym
         validate_name = "validate_#{column_name}".to_sym
 
-        serialize column_name
+        serialize column_name, JSON
         after_initialize setup_name
         before_validation normalize_name
         before_save normalize_name

From a408ae48eae8807246658582bb57f51c5871af2e Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Tue, 24 Dec 2013 17:37:26 -0500
Subject: [PATCH 03/10] globally avoid using symbols since we're moving to json
 storage

---
 app/concerns/email_concern.rb                 |   4 +-
 app/concerns/twitter_concern.rb               |  16 +-
 app/concerns/weibo_concern.rb                 |  12 +-
 app/models/agent.rb                           |  20 +-
 app/models/agents/adioso_agent.rb             |  24 +--
 app/models/agents/digest_email_agent.rb       |  28 +--
 app/models/agents/email_agent.rb              |   8 +-
 app/models/agents/event_formatting_agent.rb   |  44 ++--
 app/models/agents/human_task_agent.rb         | 136 ++++++------
 app/models/agents/manual_event_agent.rb       |   4 +-
 app/models/agents/peak_detector_agent.rb      |  54 ++---
 app/models/agents/post_agent.rb               |   8 +-
 app/models/agents/sentiment_agent.rb          |  20 +-
 app/models/agents/translation_agent.rb        |  30 +--
 app/models/agents/trigger_agent.rb            |  48 ++---
 app/models/agents/twilio_agent.rb             |  50 ++---
 app/models/agents/twitter_publish_agent.rb    |  40 ++--
 app/models/agents/twitter_stream_agent.rb     |  46 ++---
 app/models/agents/twitter_user_agent.rb       |  24 +--
 app/models/agents/user_location_agent.rb      |   4 +-
 app/models/agents/weather_agent.rb            |  16 +-
 app/models/agents/website_agent.rb            |  78 +++----
 app/models/agents/weibo_publish_agent.rb      |  38 ++--
 app/models/agents/weibo_user_agent.rb         |  22 +-
 app/models/event.rb                           |  11 +-
 lib/json_with_indifferent_access.rb           |   9 +
 lib/serialize_and_normalize.rb                |  46 -----
 spec/controllers/agents_controller_spec.rb    |   2 +-
 spec/fixtures/agents.yml                      |  14 +-
 spec/fixtures/events.yml                      |   4 +-
 spec/models/agent_spec.rb                     |   4 +-
 spec/models/agents/digest_email_agent_spec.rb |  10 +-
 spec/models/agents/email_agent_spec.rb        |   8 +-
 spec/models/agents/human_task_agent_spec.rb   | 194 +++++++++---------
 .../models/agents/peak_detector_agent_spec.rb |  58 +++---
 spec/models/agents/trigger_agent_spec.rb      | 106 +++++-----
 36 files changed, 610 insertions(+), 630 deletions(-)
 create mode 100644 lib/json_with_indifferent_access.rb
 delete mode 100644 lib/serialize_and_normalize.rb

diff --git a/app/concerns/email_concern.rb b/app/concerns/email_concern.rb
index 455acdcb..01cca474 100644
--- a/app/concerns/email_concern.rb
+++ b/app/concerns/email_concern.rb
@@ -8,11 +8,11 @@ module EmailConcern
   end
 
   def validate_email_options
-    errors.add(:base, "subject and expected_receive_period_in_days are required") unless options[:subject].present? && options[:expected_receive_period_in_days].present?
+    errors.add(:base, "subject and expected_receive_period_in_days are required") unless options['subject'].present? && options['expected_receive_period_in_days'].present?
   end
 
   def working?
-    last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+    last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
   end
 
   def present(payload)
diff --git a/app/concerns/twitter_concern.rb b/app/concerns/twitter_concern.rb
index 0a596af3..aedc351a 100644
--- a/app/concerns/twitter_concern.rb
+++ b/app/concerns/twitter_concern.rb
@@ -7,20 +7,20 @@ module TwitterConcern
   end
 
   def validate_twitter_options
-    unless options[:consumer_key].present? &&
-      options[:consumer_secret].present? &&
-      options[:oauth_token].present? &&
-      options[:oauth_token_secret].present?
+    unless options['consumer_key'].present? &&
+      options['consumer_secret'].present? &&
+      options['oauth_token'].present? &&
+      options['oauth_token_secret'].present?
       errors.add(:base, "consumer_key, consumer_secret, oauth_token and oauth_token_secret are required to authenticate with the Twitter API")
     end
   end
 
   def configure_twitter
     Twitter.configure do |config|
-      config.consumer_key = options[:consumer_key]
-      config.consumer_secret = options[:consumer_secret]
-      config.oauth_token = options[:oauth_token] || options[:access_key]
-      config.oauth_token_secret = options[:oauth_token_secret] || options[:access_secret]
+      config.consumer_key = options['consumer_key']
+      config.consumer_secret = options['consumer_secret']
+      config.oauth_token = options['oauth_token'] || options['access_key']
+      config.oauth_token_secret = options['oauth_token_secret'] || options['access_secret']
     end
   end
 
diff --git a/app/concerns/weibo_concern.rb b/app/concerns/weibo_concern.rb
index 029071e7..eea56237 100644
--- a/app/concerns/weibo_concern.rb
+++ b/app/concerns/weibo_concern.rb
@@ -6,19 +6,19 @@ module WeiboConcern
   end
 
   def validate_weibo_options
-    unless options[:app_key].present? &&
-        options[:app_secret].present? &&
-        options[:access_token].present?
+    unless options['app_key'].present? &&
+        options['app_secret'].present? &&
+        options['access_token'].present?
         errors.add(:base, "app_key, app_secret and access_token are required")
     end
   end
 
   def weibo_client
     unless @weibo_client
-      WeiboOAuth2::Config.api_key = options[:app_key] # WEIBO_APP_KEY
-      WeiboOAuth2::Config.api_secret = options[:app_secret] # WEIBO_APP_SECRET
+      WeiboOAuth2::Config.api_key = options['app_key'] # WEIBO_APP_KEY
+      WeiboOAuth2::Config.api_secret = options['app_secret'] # WEIBO_APP_SECRET
       @weibo_client = WeiboOAuth2::Client.new
-      @weibo_client.get_token_from_hash :access_token => options[:access_token]
+      @weibo_client.get_token_from_hash :access_token => options['access_token']
     end
     @weibo_client
   end
diff --git a/app/models/agent.rb b/app/models/agent.rb
index ad4d04fb..9d6a9f34 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -1,14 +1,12 @@
-require 'serialize_and_normalize'
+require 'json_with_indifferent_access'
 require 'assignable_types'
 require 'markdown_class_attributes'
 require 'utils'
 
 class Agent < ActiveRecord::Base
-  include SerializeAndNormalize
   include AssignableTypes
   include MarkdownClassAttributes
 
-  serialize_and_normalize :options, :memory
   markdown_class_attributes :description, :event_description
 
   load_types_in "Agents"
@@ -18,9 +16,21 @@ class Agent < ActiveRecord::Base
 
   attr_accessible :options, :memory, :name, :type, :schedule, :source_ids
 
+  serialize :options, JSONWithIndifferentAccess
+  serialize :memory, JSONWithIndifferentAccess
+
+  def options=(o)
+    self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
+  def memory=(o)
+    self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
   validates_presence_of :name, :user
   validate :sources_are_owned
   validate :validate_schedule
+  validate :validate_options
 
   after_initialize :set_default_schedule
   before_validation :set_default_schedule
@@ -74,6 +84,10 @@ class Agent < ActiveRecord::Base
     raise "Implement me in your subclass"
   end
 
+  def validate_options
+    # Implement me in your subclass to test for valid options.
+  end
+
   def event_created_within(days)
     event = most_recent_event
     event && event.created_at > days.to_i.days.ago && event.payload.present? && event
diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb
index 3055d2f5..662bb5e4 100644
--- a/app/models/agents/adioso_agent.rb
+++ b/app/models/agents/adioso_agent.rb
@@ -29,18 +29,18 @@ module Agents
 
     def default_options
       {
-        :start_date => Date.today.httpdate[0..15],
-        :end_date   => Date.today.plus_with_duration(100).httpdate[0..15],
-        :from       => "New York",
-        :to         => "Chicago",
-        :username   => "xx",
-        :password   => "xx",
-				:expected_update_period_in_days => "1"
+        'start_date' => Date.today.httpdate[0..15],
+        'end_date'   => Date.today.plus_with_duration(100).httpdate[0..15],
+        'from'       => "New York",
+        'to'         => "Chicago",
+        'username'   => "xx",
+        'password'   => "xx",
+				'expected_update_period_in_days' => "1"
       }
     end
 
     def working?
-      event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
+      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def validate_options
@@ -54,9 +54,9 @@ module Agents
     end
 
     def check
-      auth_options = {:basic_auth => {:username =>options[:username], :password=>options[:password]}}
-      parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options[:from])}+to+#{URI.encode(options[:to])}", auth_options
-      fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options[:end_date])}\\3#{date_to_unix_epoch(options[:start_date])}"
+      auth_options = {:basic_auth => {:username =>options[:username], :password=>options['password']}}
+      parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options['from'])}+to+#{URI.encode(options['to'])}", auth_options
+      fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options['end_date'])}\\3#{date_to_unix_epoch(options['start_date'])}"
       fare = HTTParty.get fare_request, auth_options
 
 			if fare["warnings"]
@@ -64,7 +64,7 @@ module Agents
 			else
 				event = fare["results"].min {|a,b| a["cost"] <=> b["cost"]}
 				event["date"]  = Time.at(event["date"]).to_date.httpdate[0..15]
-				event["route"] = "#{options[:from]} to #{options[:to]}" 
+				event["route"] = "#{options['from']} to #{options['to']}"
 				create_event :payload => event
 			end
     end
diff --git a/app/models/agents/digest_email_agent.rb b/app/models/agents/digest_email_agent.rb
index 009564df..62204d68 100644
--- a/app/models/agents/digest_email_agent.rb
+++ b/app/models/agents/digest_email_agent.rb
@@ -9,7 +9,7 @@ module Agents
     description <<-MD
       The DigestEmailAgent collects any Events sent to it and sends them all via email when run.
       The email will be sent to your account's address and will have a `subject` and an optional `headline` before
-      listing the Events.  If the Events' payloads contain a `:message`, that will be highlighted, otherwise everything in
+      listing the Events.  If the Events' payloads contain a `message`, that will be highlighted, otherwise everything in
       their payloads will be shown.
 
       Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
@@ -17,29 +17,29 @@ module Agents
 
     def default_options
       {
-          :subject => "You have some notifications!",
-          :headline => "Your notifications:",
-          :expected_receive_period_in_days => "2"
+          'subject' => "You have some notifications!",
+          'headline' => "Your notifications:",
+          'expected_receive_period_in_days' => "2"
       }
     end
 
     def receive(incoming_events)
       incoming_events.each do |event|
-        self.memory[:queue] ||= []
-        self.memory[:queue] << event.payload
-        self.memory[:events] ||= []
-        self.memory[:events] << event.id
+        self.memory['queue'] ||= []
+        self.memory['queue'] << event.payload
+        self.memory['events'] ||= []
+        self.memory['events'] << event.id
       end
     end
 
     def check
-      if self.memory[:queue] && self.memory[:queue].length > 0
-        ids = self.memory[:events].join(",")
-        groups = self.memory[:queue].map { |payload| present(payload) }
+      if self.memory['queue'] && self.memory['queue'].length > 0
+        ids = self.memory['events'].join(",")
+        groups = self.memory['queue'].map { |payload| present(payload) }
         log "Sending digest mail to #{user.email} with events [#{ids}]"
-        SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => groups)
-        self.memory[:queue] = []
-        self.memory[:events] = []
+        SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => groups)
+        self.memory['queue'] = []
+        self.memory['events'] = []
       end
     end
   end
diff --git a/app/models/agents/email_agent.rb b/app/models/agents/email_agent.rb
index e1c79dce..64f81164 100644
--- a/app/models/agents/email_agent.rb
+++ b/app/models/agents/email_agent.rb
@@ -16,16 +16,16 @@ module Agents
 
     def default_options
       {
-          :subject => "You have a notification!",
-          :headline => "Your notification:",
-          :expected_receive_period_in_days => "2"
+          'subject' => "You have a notification!",
+          'headline' => "Your notification:",
+          'expected_receive_period_in_days' => "2"
       }
     end
 
     def receive(incoming_events)
       incoming_events.each do |event|
         log "Sending digest mail to #{user.email} with event #{event.id}"
-        SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => [present(event.payload)])
+        SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => [present(event.payload)])
       end
     end
   end
diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb
index cc27c5da..947c00bb 100644
--- a/app/models/agents/event_formatting_agent.rb
+++ b/app/models/agents/event_formatting_agent.rb
@@ -8,20 +8,20 @@ module Agents
       For example, here is a possible Event:
 
           {
-            :high => {
-              :celsius => "18",
-              :fahreinheit => "64"
+            "high": {
+              "celsius": "18",
+              "fahreinheit": "64"
             },
-            :conditions => "Rain showers",
-            :data   => "This is some data"
+            "conditions": "Rain showers",
+            "data": "This is some data"
           }
 
       You may want to send this event to another Agent, for example a Twilio Agent, which expects a `message` key.
       You can use an Event Formatting Agent's `instructions` setting to do this in the following way:
 
-          instructions: {
-            message: "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.",
-            subject: "$.data"
+          "instructions": {
+            "message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.",
+            "subject": "$.data"
           }
 
       JSONPaths must be between < and > . Make sure that you don't use these symbols anywhere else.
@@ -29,8 +29,8 @@ module Agents
       Events generated by this possible Event Formatting Agent will look like:
 
           {
-            :message => "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.",
-            :subject => "This is some data"
+            "message": "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.",
+            "subject": "This is some data"
           }
 
       If you want to retain original contents of events and only add new keys, then set `mode` to `merge`, otherwise set it to `clean`.
@@ -40,25 +40,25 @@ module Agents
       To CGI escape output (for example when creating a link), prefix with `escape`, like so:
 
           {
-            :message => "A peak was on Twitter in <$.group_by>.  Search: https://twitter.com/search?q=<escape $.group_by>"
+            "message": "A peak was on Twitter in <$.group_by>.  Search: https://twitter.com/search?q=<escape $.group_by>"
           }
     MD
 
     event_description "User defined"
 
     def validate_options
-      errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options[:instructions].present? and options[:mode].present? and options[:skip_agent].present? and options[:skip_created_at].present?
+      errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options['instructions'].present? and options['mode'].present? and options['skip_agent'].present? and options['skip_created_at'].present?
     end
 
     def default_options
       {
-        :instructions => {
-          :message =>  "You received a text <$.text> from <$.fields.from>",
-          :some_other_field => "Looks like the weather is going to be <$.fields.weather>"
+        'instructions' => {
+          'message' =>  "You received a text <$.text> from <$.fields.from>",
+          'some_other_field' => "Looks like the weather is going to be <$.fields.weather>"
         },
-        :mode => "clean",
-        :skip_agent => "false",
-        :skip_created_at => "false"
+        'mode' => "clean",
+        'skip_agent' => "false",
+        'skip_created_at' => "false"
       }
     end
 
@@ -68,10 +68,10 @@ module Agents
 
     def receive(incoming_events)
       incoming_events.each do |event|
-        formatted_event = options[:mode].to_s == "merge" ? event.payload : {}
-        options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
-        formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true"
-        formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true"
+        formatted_event = options['mode'].to_s == "merge" ? event.payload : {}
+        options['instructions'].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
+        formatted_event['agent'] = Agent.find(event.agent_id).type.slice!(8..-1) unless options['skip_agent'].to_s == "true"
+        formatted_event['created_at'] = event.created_at unless options['skip_created_at'].to_s == "true"
         create_event :payload => formatted_event
       end
     end
diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb
index 116347e6..0d27327b 100644
--- a/app/models/agents/human_task_agent.rb
+++ b/app/models/agents/human_task_agent.rb
@@ -74,69 +74,69 @@ module Agents
     MD
 
     def validate_options
-      options[:hit] ||= {}
-      options[:hit][:questions] ||= []
+      options['hit'] ||= {}
+      options['hit']['questions'] ||= []
 
-      errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on])
-      errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options[:hit][:assignments].present? && options[:hit][:assignments].to_i > 0
-      errors.add(:base, "'hit.title' must be provided") unless options[:hit][:title].present?
-      errors.add(:base, "'hit.description' must be provided") unless options[:hit][:description].present?
-      errors.add(:base, "'hit.questions' must be provided") unless options[:hit][:questions].present? && options[:hit][:questions].length > 0
+      errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options['trigger_on'])
+      errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options['hit']['assignments'].present? && options['hit']['assignments'].to_i > 0
+      errors.add(:base, "'hit.title' must be provided") unless options['hit']['title'].present?
+      errors.add(:base, "'hit.description' must be provided") unless options['hit']['description'].present?
+      errors.add(:base, "'hit.questions' must be provided") unless options['hit']['questions'].present? && options['hit']['questions'].length > 0
 
-      if options[:trigger_on] == "event"
-        errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present?
-      elsif options[:trigger_on] == "schedule"
-        errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0
+      if options['trigger_on'] == "event"
+        errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options['expected_receive_period_in_days'].present?
+      elsif options['trigger_on'] == "schedule"
+        errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options['submission_period'].present? && options['submission_period'].to_i > 0
       end
 
-      if options[:hit][:questions].any? { |question| [:key, :name, :required, :type, :question].any? {|k| !question[k].present? } }
+      if options['hit']['questions'].any? { |question| %w[key name required type question].any? {|k| !question[k].present? } }
         errors.add(:base, "all questions must set 'key', 'name', 'required', 'type', and 'question'")
       end
 
-      if options[:hit][:questions].any? { |question| question[:type] == "selection" && (!question[:selections].present? || question[:selections].length == 0 || !question[:selections].all? {|s| s[:key].present? } || !question[:selections].all? { |s| s[:text].present? })}
+      if options['hit']['questions'].any? { |question| question['type'] == "selection" && (!question['selections'].present? || question['selections'].length == 0 || !question['selections'].all? {|s| s['key'].present? } || !question['selections'].all? { |s| s['text'].present? })}
         errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
       end
 
-      if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" }
+      if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" }
         errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
       end
     end
 
     def default_options
       {
-        :expected_receive_period_in_days => 2,
-        :trigger_on => "event",
-        :hit =>
+        'expected_receive_period_in_days' => 2,
+        'trigger_on' => "event",
+        'hit' =>
           {
-            :assignments => 1,
-            :title => "Sentiment evaluation",
-            :description => "Please rate the sentiment of this message: '<$.message>'",
-            :reward => 0.05,
-            :lifetime_in_seconds => 24 * 60 * 60,
-            :questions =>
+            'assignments' => 1,
+            'title' => "Sentiment evaluation",
+            'description' => "Please rate the sentiment of this message: '<$.message>'",
+            'reward' => 0.05,
+            'lifetime_in_seconds' => 24 * 60 * 60,
+            'questions' =>
               [
                 {
-                  :type => "selection",
-                  :key => "sentiment",
-                  :name => "Sentiment",
-                  :required => "true",
-                  :question => "Please select the best sentiment value:",
-                  :selections =>
+                  'type' => "selection",
+                  'key' => "sentiment",
+                  'name' => "Sentiment",
+                  'required' => "true",
+                  'question' => "Please select the best sentiment value:",
+                  'selections' =>
                     [
-                      { :key => "happy", :text => "Happy" },
-                      { :key => "sad", :text => "Sad" },
-                      { :key => "neutral", :text => "Neutral" }
+                      { 'key' => "happy", 'text' => "Happy" },
+                      { 'key' => "sad", 'text' => "Sad" },
+                      { 'key' => "neutral", 'text' => "Neutral" }
                     ]
                 },
                 {
-                  :type => "free_text",
-                  :key => "feedback",
-                  :name => "Have any feedback for us?",
-                  :required => "false",
-                  :question => "Feedback",
-                  :default => "Type here...",
-                  :min_length => "2",
-                  :max_length => "2000"
+                  'type' => "free_text",
+                  'key' => "feedback",
+                  'name' => "Have any feedback for us?",
+                  'required' => "false",
+                  'question' => "Feedback",
+                  'default' => "Type here...",
+                  'min_length' => "2",
+                  'max_length' => "2000"
                 }
               ]
           }
@@ -144,20 +144,20 @@ module Agents
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def check
       review_hits
 
-      if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60
-        memory[:last_schedule] = Time.now.to_i
+      if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60
+        memory['last_schedule'] = Time.now.to_i
         create_hit
       end
     end
 
     def receive(incoming_events)
-      if options[:trigger_on] == "event"
+      if options['trigger_on'] == "event"
         incoming_events.each do |event|
           create_hit event
         end
@@ -168,7 +168,7 @@ module Agents
 
     def review_hits
       reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
-      my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s)
+      my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys
       if reviewable_hit_ids.length > 0
         log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]"
       end
@@ -178,26 +178,26 @@ module Agents
 
         log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
         if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
-          payload = { :answers => assignments.map(&:answers) }
+          payload = { 'answers' => assignments.map(&:answers) }
 
-          if options[:take_majority] == "true"
+          if options['take_majority'] == "true"
             counts = {}
-            options[:hit][:questions].each do |question|
-              question_counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
+            options['hit']['questions'].each do |question|
+              question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
               assignments.each do |assignment|
                 answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
-                answer = answers[question[:key]]
+                answer = answers[question['key']]
                 question_counts[answer] += 1
               end
-              counts[question[:key]] = question_counts
+              counts[question['key']] = question_counts
             end
-            payload[:counts] = counts
+            payload['counts'] = counts
 
             majority_answer = counts.inject({}) do |memo, (key, question_counts)|
               memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
               memo
             end
-            payload[:majority_answer] = majority_answer
+            payload['majority_answer'] = majority_answer
 
             if all_questions_are_numeric?
               average_answer = counts.inject({}) do |memo, (key, question_counts)|
@@ -209,44 +209,44 @@ module Agents
                 memo[key] = sum / divisor.to_f
                 memo
               end
-              payload[:average_answer] = average_answer
+              payload['average_answer'] = average_answer
             end
           end
 
           event = create_event :payload => payload
-          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id])
+          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id])
 
           assignments.each(&:approve!)
           hit.dispose!
 
-          memory[:hits].delete(hit_id)
+          memory['hits'].delete(hit_id)
         end
       end
     end
 
     def all_questions_are_numeric?
-      options[:hit][:questions].all? do |question|
-        question[:selections].all? do |selection|
-          selection[:key] == selection[:key].to_f.to_s || selection[:key] == selection[:key].to_i.to_s
+      options['hit']['questions'].all? do |question|
+        question['selections'].all? do |selection|
+          selection['key'] == selection['key'].to_f.to_s || selection['key'] == selection['key'].to_i.to_s
         end
       end
     end
 
     def create_hit(event = nil)
       payload = event ? event.payload : {}
-      title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip
-      description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip
-      questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload)
+      title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip
+      description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip
+      questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload)
       hit = RTurk::Hit.create(:title => title) do |hit|
-        hit.max_assignments = (options[:hit][:assignments] || 1).to_i
+        hit.max_assignments = (options['hit']['assignments'] || 1).to_i
         hit.description = description
-        hit.lifetime = (options[:hit][:lifetime_in_seconds] || 24 * 60 * 60).to_i
+        hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i
         hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
-        hit.reward = (options[:hit][:reward] || 0.05).to_f
+        hit.reward = (options['hit']['reward'] || 0.05).to_f
         #hit.qualifications.add :approval_rate, { :gt => 80 }
       end
-      memory[:hits] ||= {}
-      memory[:hits][hit.id] = event && event.id
+      memory['hits'] ||= {}
+      memory['hits'][hit.id] = event && event.id
       log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
     end
 
@@ -314,7 +314,7 @@ module Agents
                     end
                   end
 
-                  if question[:default].present?
+                  if question['default'].present?
                     DefaultText do
                       text question['default']
                     end
diff --git a/app/models/agents/manual_event_agent.rb b/app/models/agents/manual_event_agent.rb
index 7b9f6f14..7c045a97 100644
--- a/app/models/agents/manual_event_agent.rb
+++ b/app/models/agents/manual_event_agent.rb
@@ -14,8 +14,8 @@ module Agents
     end
 
     def handle_details_post(params)
-      if params[:payload]
-        create_event(:payload => params[:payload])
+      if params['payload']
+        create_event(:payload => params['payload'])
         { :success => true }
       else
         { :success => false, :error => "You must provide a JSON payload" }
diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb
index 7c7e7468..bf277c2c 100644
--- a/app/models/agents/peak_detector_agent.rb
+++ b/app/models/agents/peak_detector_agent.rb
@@ -28,22 +28,22 @@ module Agents
     MD
 
     def validate_options
-      unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:value_path].present?
+      unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['value_path'].present?
         errors.add(:base, "expected_receive_period_in_days, value_path, and message are required")
       end
     end
 
     def default_options
       {
-          :expected_receive_period_in_days => "2",
-          :group_by_path => "filter",
-          :value_path => "count",
-          :message => "A peak was found"
+        'expected_receive_period_in_days' => "2",
+        'group_by_path' => "filter",
+        'value_path' => "count",
+        'message' => "A peak was found"
       }
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def receive(incoming_events)
@@ -57,25 +57,25 @@ module Agents
     private
 
     def check_for_peak(group, event)
-      memory[:peaks] ||= {}
-      memory[:peaks][group] ||= []
+      memory['peaks'] ||= {}
+      memory['peaks'][group] ||= []
 
-      if memory[:data][group].length > 4 && (memory[:peaks][group].empty? || memory[:peaks][group].last < event.created_at.to_i - peak_spacing)
+      if memory['data'][group].length > 4 && (memory['peaks'][group].empty? || memory['peaks'][group].last < event.created_at.to_i - peak_spacing)
         average_value, standard_deviation = stats_for(group, :skip_last => 1)
-        newest_value, newest_time = memory[:data][group][-1].map(&:to_f)
+        newest_value, newest_time = memory['data'][group][-1].map(&:to_f)
 
         #p [newest_value, average_value, average_value + std_multiple * standard_deviation, standard_deviation]
 
         if newest_value > average_value + std_multiple * standard_deviation
-          memory[:peaks][group] << newest_time
-          memory[:peaks][group].reject! { |p| p <= newest_time - window_duration }
-          create_event :payload => {:message => options[:message], :peak => newest_value, :peak_time => newest_time, :grouped_by => group.to_s}
+          memory['peaks'][group] << newest_time
+          memory['peaks'][group].reject! { |p| p <= newest_time - window_duration }
+          create_event :payload => { 'message' => options['message'], 'peak' => newest_value, 'peak_time' => newest_time, 'grouped_by' => group.to_s }
         end
       end
     end
 
     def stats_for(group, options = {})
-      data = memory[:data][group].map { |d| d.first.to_f }
+      data = memory['data'][group].map { |d| d.first.to_f }
       data = data[0...(data.length - (options[:skip_last] || 0))]
       length = data.length.to_f
       mean = 0
@@ -94,39 +94,39 @@ module Agents
     end
 
     def window_duration
-      if options[:window_duration].present? # The older option
-        options[:window_duration].to_i
+      if options['window_duration'].present? # The older option
+        options['window_duration'].to_i
       else
-        (options[:window_duration_in_days] || 14).to_f.days
+        (options['window_duration_in_days'] || 14).to_f.days
       end
     end
 
     def std_multiple
-      (options[:std_multiple] || 3).to_f
+      (options['std_multiple'] || 3).to_f
     end
 
     def peak_spacing
-      if options[:peak_spacing].present? # The older option
-        options[:peak_spacing].to_i
+      if options['peak_spacing'].present? # The older option
+        options['peak_spacing'].to_i
       else
-        (options[:min_peak_spacing_in_days] || 2).to_f.days
+        (options['min_peak_spacing_in_days'] || 2).to_f.days
       end
     end
 
     def group_for(event)
-      ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group')
+      ((options['group_by_path'].present? && Utils.value_at(event.payload, options['group_by_path'])) || 'no_group')
     end
 
     def remember(group, event)
-      memory[:data] ||= {}
-      memory[:data][group] ||= []
-      memory[:data][group] << [Utils.value_at(event.payload, options[:value_path]), event.created_at.to_i]
+      memory['data'] ||= {}
+      memory['data'][group] ||= []
+      memory['data'][group] << [ Utils.value_at(event.payload, options['value_path']), event.created_at.to_i ]
       cleanup group
     end
 
     def cleanup(group)
-      newest_time = memory[:data][group].last.last
-      memory[:data][group].reject! { |value, time| time <= newest_time - window_duration }
+      newest_time = memory['data'][group].last.last
+      memory['data'][group].reject! { |value, time| time <= newest_time - window_duration }
     end
   end
 end
\ No newline at end of file
diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb
index d3582e25..3d5ae824 100644
--- a/app/models/agents/post_agent.rb
+++ b/app/models/agents/post_agent.rb
@@ -11,17 +11,17 @@ module Agents
 
     def default_options
       {
-        :post_url => "http://www.example.com",
-        :expected_receive_period_in_days => 1
+        'post_url' => "http://www.example.com",
+        'expected_receive_period_in_days' => 1
       }
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def validate_options
-      unless options[:post_url].present? && options[:expected_receive_period_in_days].present?
+      unless options['post_url'].present? && options['expected_receive_period_in_days'].present?
         errors.add(:base, "post_url and expected_receive_period_in_days are required fields")
       end
     end
diff --git a/app/models/agents/sentiment_agent.rb b/app/models/agents/sentiment_agent.rb
index c6a917b0..c0f6c5be 100644
--- a/app/models/agents/sentiment_agent.rb
+++ b/app/models/agents/sentiment_agent.rb
@@ -28,31 +28,31 @@ module Agents
 
     def default_options
       {
-        :content => "$.message.text[*]",
-        :expected_receive_period_in_days => 1
+        'content' => "$.message.text[*]",
+        'expected_receive_period_in_days' => 1
       }
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def receive(incoming_events)
       anew = self.class.sentiment_hash
       incoming_events.each do |event|
-        Utils.values_at(event.payload, options[:content]).each do |content|
+        Utils.values_at(event.payload, options['content']).each do |content|
           sent_values = sentiment_values anew, content
-          create_event :payload => { :content => content,
-                                     :valence => sent_values[0],
-                                     :arousal => sent_values[1],
-                                     :dominance => sent_values[2],
-                                     :original_event => event.payload }
+          create_event :payload => { 'content' => content,
+                                     'valence' => sent_values[0],
+                                     'arousal' => sent_values[1],
+                                     'dominance' => sent_values[2],
+                                     'original_event' => event.payload }
         end
       end
     end
 
     def validate_options
-      errors.add(:base, "content and expected_receive_period_in_days must be present") unless options[:content].present? && options[:expected_receive_period_in_days].present?
+      errors.add(:base, "content and expected_receive_period_in_days must be present") unless options['content'].present? && options['expected_receive_period_in_days'].present?
     end
 
     def self.sentiment_hash
diff --git a/app/models/agents/translation_agent.rb b/app/models/agents/translation_agent.rb
index 2fc073f3..7bbdb05d 100644
--- a/app/models/agents/translation_agent.rb
+++ b/app/models/agents/translation_agent.rb
@@ -17,26 +17,26 @@ module Agents
 
     def default_options
       {
-        :client_id => "xxxxxx",
-        :client_secret => "xxxxxx",
-        :to => "fi",
-        :expected_receive_period_in_days => 1,
-        :content => {
-          :text => "$.message.text",
-          :content => "$.xyz"
+        'client_id' => "xxxxxx",
+        'client_secret' => "xxxxxx",
+        'to' => "fi",
+        'expected_receive_period_in_days' => 1,
+        'content' => {
+          'text' => "$.message.text",
+          'content' => "$.xyz"
         }
       }
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def translate(text, to, access_token)
       translate_uri = URI 'http://api.microsofttranslator.com/v2/Ajax.svc/Translate'
       params = {
-          :text => text,
-          :to => to
+        'text' => text,
+        'to' => to
       }
       translate_uri.query = URI.encode_www_form params
       request = Net::HTTP::Get.new translate_uri.request_uri
@@ -47,7 +47,7 @@ module Agents
     end
 
     def validate_options
-      unless options[:client_id].present? && options[:client_secret].present? && options[:to].present? && options[:content].present? && options[:expected_receive_period_in_days].present?
+      unless options['client_id'].present? && options['client_secret'].present? && options['to'].present? && options['content'].present? && options['expected_receive_period_in_days'].present?
         errors.add :base, "client_id,client_secret,to,expected_receive_period_in_days and content are all required"
       end
     end
@@ -60,16 +60,16 @@ module Agents
 
     def receive(incoming_events)
       auth_uri = URI "https://datamarket.accesscontrol.windows.net/v2/OAuth2-13"
-      response = postform auth_uri, :client_id => options[:client_id],
-                                    :client_secret => options[:client_secret],
+      response = postform auth_uri, :client_id => options['client_id'],
+                                    :client_secret => options['client_secret'],
                                     :scope => "http://api.microsofttranslator.com",
                                     :grant_type => "client_credentials"
       access_token = JSON.parse(response.body)["access_token"]
       incoming_events.each do |event|
         translated_event = {}
-        options[:content].each_pair do |key, value|
+        options['content'].each_pair do |key, value|
           to_be_translated = Utils.values_at event.payload, value
-          translated_event[key] = translate to_be_translated.first, options[:to], access_token
+          translated_event[key] = translate to_be_translated.first, options['to'], access_token
         end
         create_event :payload => translated_event
       end
diff --git a/app/models/agents/trigger_agent.rb b/app/models/agents/trigger_agent.rb
index f8190c33..b32fff6e 100644
--- a/app/models/agents/trigger_agent.rb
+++ b/app/models/agents/trigger_agent.rb
@@ -23,57 +23,57 @@ module Agents
     MD
 
     def validate_options
-      unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:rules].present? &&
-          options[:rules].all? { |rule| rule[:type].present? && VALID_COMPARISON_TYPES.include?(rule[:type]) && rule[:value].present? && rule[:path].present? }
+      unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['rules'].present? &&
+          options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? }
         errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required")
       end
     end
 
     def default_options
       {
-          :expected_receive_period_in_days => "2",
-          :rules => [{
-                         :type => "regex",
-                         :value => "foo\\d+bar",
-                         :path => "topkey.subkey.subkey.goal",
-                     }],
-          :message => "Looks like your pattern matched in '<value>'!"
+        'expected_receive_period_in_days' => "2",
+        'rules' => [{
+                      'type' => "regex",
+                      'value' => "foo\\d+bar",
+                      'path' => "topkey.subkey.subkey.goal",
+                    }],
+        'message' => "Looks like your pattern matched in '<value>'!"
       }
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def receive(incoming_events)
       incoming_events.each do |event|
-        match = options[:rules].all? do |rule|
-          value_at_path = Utils.value_at(event[:payload], rule[:path])
-          case rule[:type]
+        match = options['rules'].all? do |rule|
+          value_at_path = Utils.value_at(event['payload'], rule['path'])
+          case rule['type']
             when "regex"
-              value_at_path.to_s =~ Regexp.new(rule[:value], Regexp::IGNORECASE)
+              value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
             when "!regex"
-              value_at_path.to_s !~ Regexp.new(rule[:value], Regexp::IGNORECASE)
+              value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
             when "field>value"
-              value_at_path.to_f > rule[:value].to_f
+              value_at_path.to_f > rule['value'].to_f
             when "field>=value"
-              value_at_path.to_f >= rule[:value].to_f
+              value_at_path.to_f >= rule['value'].to_f
             when "field<value"
-              value_at_path.to_f < rule[:value].to_f
+              value_at_path.to_f < rule['value'].to_f
             when "field<=value"
-              value_at_path.to_f <= rule[:value].to_f
+              value_at_path.to_f <= rule['value'].to_f
             when "field==value"
-              value_at_path.to_s == rule[:value].to_s
+              value_at_path.to_s == rule['value'].to_s
             when "field!=value"
-              value_at_path.to_s != rule[:value].to_s
+              value_at_path.to_s != rule['value'].to_s
             else
-              raise "Invalid :type of #{rule[:type]} in TriggerAgent##{id}"
+              raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
           end
         end
 
         if match
-          create_event :payload => { :message => make_message(event[:payload]) } # Maybe this should include the
-                                                                                 # original event as well?
+          create_event :payload => { 'message' => make_message(event[:payload]) } # Maybe this should include the
+                                                                                  # original event as well?
         end
       end
     end
diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb
index 82076e89..9cec3268 100644
--- a/app/models/agents/twilio_agent.rb
+++ b/app/models/agents/twilio_agent.rb
@@ -9,7 +9,7 @@ module Agents
     description <<-MD
       The TwilioAgent receives and collects events and sends them via text message or gives you a call when scheduled.
 
-      It is assumed that events have a `:message`, `:text`, or `:sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
+      It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
 
       Set `receiver_cell` to the number to receive text messages/call and `sender_cell` to the number sending them.
 
@@ -22,35 +22,35 @@ module Agents
 
     def default_options
       {
-        :account_sid => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
-        :auth_token => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
-        :sender_cell => 'xxxxxxxxxx',
-        :receiver_cell => 'xxxxxxxxxx',
-        :server_url    => 'http://somename.com:3000',
-        :receive_text  => 'true',
-        :receive_call  => 'false',
-        :expected_receive_period_in_days => '1'
+        'account_sid' => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
+        'auth_token' => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
+        'sender_cell' => 'xxxxxxxxxx',
+        'receiver_cell' => 'xxxxxxxxxx',
+        'server_url'    => 'http://somename.com:3000',
+        'receive_text'  => 'true',
+        'receive_call'  => 'false',
+        'expected_receive_period_in_days' => '1'
       }
     end
 
     def validate_options
-      unless options[:account_sid].present? && options[:auth_token].present? && options[:sender_cell].present? && options[:receiver_cell].present? && options[:expected_receive_period_in_days].present? && options[:receive_call].present? && options[:receive_text].present?
+      unless options['account_sid'].present? && options['auth_token'].present? && options['sender_cell'].present? && options['receiver_cell'].present? && options['expected_receive_period_in_days'].present? && options['receive_call'].present? && options['receive_text'].present?
         errors.add(:base, 'account_sid, auth_token, sender_cell, receiver_cell, receive_text, receive_call and expected_receive_period_in_days are all required')
       end
     end
 
     def receive(incoming_events)
-      @client = Twilio::REST::Client.new options[:account_sid], options[:auth_token]
-      memory[:pending_calls] ||= {}
+      @client = Twilio::REST::Client.new options['account_sid'], options['auth_token']
+      memory['pending_calls'] ||= {}
       incoming_events.each do |event|
-        message = (event.payload[:message] || event.payload[:text] || event.payload[:sms]).to_s
+        message = (event.payload['message'] || event.payload['text'] || event.payload['sms']).to_s
         if message != ""
-          if options[:receive_call].to_s == 'true'
+          if options['receive_call'].to_s == 'true'
             secret = SecureRandom.hex 3
-            memory[:pending_calls][secret] = message
+            memory['pending_calls'][secret] = message
             make_call secret
           end
-          if options[:receive_text].to_s == 'true'
+          if options['receive_text'].to_s == 'true'
             message = message.slice 0..160
             send_message message
           end
@@ -59,19 +59,19 @@ module Agents
     end
 
     def working?
-      last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
+      last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
     end
 
     def send_message(message)
-      @client.account.sms.messages.create :from => options[:sender_cell],
-                                          :to => options[:receiver_cell],
+      @client.account.sms.messages.create :from => options['sender_cell'],
+                                          :to => options['receiver_cell'],
                                           :body => message
     end
 
     def make_call(secret)
-      @client.account.calls.create :from => options[:sender_cell],
-                                   :to => options[:receiver_cell],
-                                   :url => post_url(options[:server_url],secret)
+      @client.account.calls.create :from => options['sender_cell'],
+                                   :to => options['receiver_cell'],
+                                   :url => post_url(options['server_url'],secret)
     end
 
     def post_url(server_url,secret)
@@ -79,9 +79,9 @@ module Agents
     end
 
     def receive_webhook(params)
-      if memory[:pending_calls].has_key? params[:secret]
-        response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'}
-        memory[:pending_calls].delete params[:secret]
+      if memory['pending_calls'].has_key? params['secret']
+        response = Twilio::TwiML::Response.new {|r| r.Say memory['pending_calls'][params['secret']], :voice => 'woman'}
+        memory['pending_calls'].delete params['secret']
         [response.text, 200]
       end
     end
diff --git a/app/models/agents/twitter_publish_agent.rb b/app/models/agents/twitter_publish_agent.rb
index 823b504e..d03d7984 100644
--- a/app/models/agents/twitter_publish_agent.rb
+++ b/app/models/agents/twitter_publish_agent.rb
@@ -19,25 +19,25 @@ module Agents
     MD
 
     def validate_options
-      unless options[:username].present? &&
-        options[:expected_update_period_in_days].present?
+      unless options['username'].present? &&
+        options['expected_update_period_in_days'].present?
         errors.add(:base, "username and expected_update_period_in_days are required")
       end      
     end
 
     def working?
-      (event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs?
+      (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
     end
 
     def default_options
       {
-          :username => "",
-          :expected_update_period_in_days => "10",
-          :consumer_key => "---",
-          :consumer_secret => "---",
-          :oauth_token => "---",
-          :oauth_token_secret => "---",
-          :message_path => "text"
+        'username' => "",
+        'expected_update_period_in_days' => "10",
+        'consumer_key' => "---",
+        'consumer_secret' => "---",
+        'oauth_token' => "---",
+        'oauth_token_secret' => "---",
+        'message_path' => "text"
       }
     end
 
@@ -47,22 +47,22 @@ module Agents
         incoming_events = incoming_events.first(20)
       end
       incoming_events.each do |event|
-        tweet_text = Utils.value_at(event.payload, options[:message_path])
+        tweet_text = Utils.value_at(event.payload, options['message_path'])
         begin
           publish_tweet tweet_text
           create_event :payload => {
-            :success => true,
-            :published_tweet => tweet_text,
-            :agent_id => event.agent_id,
-            :event_id => event.id
+            'success' => true,
+            'published_tweet' => tweet_text,
+            'agent_id' => event.agent_id,
+            'event_id' => event.id
           }
         rescue Twitter::Error => e
           create_event :payload => {
-            :success => false,
-            :error => e.message,
-            :failed_tweet => tweet_text,
-            :agent_id => event.agent_id,
-            :event_id => event.id
+            'success' => false,
+            'error' => e.message,
+            'failed_tweet' => tweet_text,
+            'agent_id' => event.agent_id,
+            'event_id' => event.id
           }
         end
       end
diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb
index 96afae25..f369b891 100644
--- a/app/models/agents/twitter_stream_agent.rb
+++ b/app/models/agents/twitter_stream_agent.rb
@@ -54,26 +54,26 @@ module Agents
     default_schedule "11pm"
 
     def validate_options
-      unless options[:filters].present? &&
-             options[:expected_update_period_in_days].present? &&
-             options[:generate].present?
+      unless options['filters'].present? &&
+             options['expected_update_period_in_days'].present? &&
+             options['generate'].present?
         errors.add(:base, "expected_update_period_in_days, generate, and filters are required fields")
       end
     end
 
     def working?
-      event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
+      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
       {
-          :consumer_key => "---",
-          :consumer_secret => "---",
-          :oauth_token => "---",
-          :oauth_token_secret => "---",
-          :filters => %w[keyword1 keyword2],
-          :expected_update_period_in_days => "2",
-          :generate => "events"
+          'consumer_key' => "---",
+          'consumer_secret' => "---",
+          'oauth_token' => "---",
+          'oauth_token_secret' => "---",
+          'filters' => %w[keyword1 keyword2],
+          'expected_update_period_in_days' => "2",
+          'generate' => "events"
       }
     end
 
@@ -81,33 +81,33 @@ module Agents
       filter = lookup_filter(filter)
 
       if filter
-        if options[:generate] == "counts"
+        if options['generate'] == "counts"
           # Avoid memory pollution by reloading the Agent.
           agent = Agent.find(id)
-          agent.memory[:filter_counts] ||= {}
-          agent.memory[:filter_counts][filter] ||= 0
-          agent.memory[:filter_counts][filter] += 1
-          remove_unused_keys!(agent, :filter_counts)
+          agent.memory['filter_counts'] ||= {}
+          agent.memory['filter_counts'][filter] ||= 0
+          agent.memory['filter_counts'][filter] += 1
+          remove_unused_keys!(agent, 'filter_counts')
           agent.save!
         else
-          create_event :payload => status.merge(:filter => filter)
+          create_event :payload => status.merge('filter' => filter)
         end
       end
     end
 
     def check
-      if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0
-        memory[:filter_counts].each do |filter, count|
-          create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i }
+      if options['generate'] == "counts" && memory['filter_counts'] && memory['filter_counts'].length > 0
+        memory['filter_counts'].each do |filter, count|
+          create_event :payload => { 'filter' => filter, 'count' => count, 'time' => Time.now.to_i }
         end
       end
-      memory[:filter_counts] = {}
+      memory['filter_counts'] = {}
     end
 
     protected
 
     def lookup_filter(filter)
-      options[:filters].each do |known_filter|
+      options['filters'].each do |known_filter|
         if known_filter == filter
           return filter
         elsif known_filter.is_a?(Array)
@@ -120,7 +120,7 @@ module Agents
 
     def remove_unused_keys!(agent, base)
       if agent.memory[base]
-        (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
+        (agent.memory[base].keys - agent.options['filters'].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
           agent.memory[base].delete(removed_key)
         end
       end
diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb
index 5f3cc627..52d10a7b 100644
--- a/app/models/agents/twitter_user_agent.rb
+++ b/app/models/agents/twitter_user_agent.rb
@@ -41,36 +41,36 @@ module Agents
     default_schedule "every_1h"
 
     def validate_options
-      unless options[:username].present? &&
-        options[:expected_update_period_in_days].present?
+      unless options['username'].present? &&
+        options['expected_update_period_in_days'].present?
         errors.add(:base, "username and expected_update_period_in_days are required")
       end      
     end
 
     def working?
-      event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
+      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
       {
-          :username => "tectonic",
-          :expected_update_period_in_days => "2",
-          :consumer_key => "---",
-          :consumer_secret => "---",
-          :oauth_token => "---",
-          :oauth_token_secret => "---"
+          'username' => "tectonic",
+          'expected_update_period_in_days' => "2",
+          'consumer_key' => "---",
+          'consumer_secret' => "---",
+          'oauth_token' => "---",
+          'oauth_token_secret' => "---"
       }
     end
 
     def check
-      since_id = memory[:since_id] || nil
+      since_id = memory['since_id'] || nil
       opts = {:count => 200, :include_rts => true, :exclude_replies => false, :include_entities => true, :contributor_details => true}
       opts.merge! :since_id => since_id unless since_id.nil?
 
-      tweets = Twitter.user_timeline(options[:username], opts)
+      tweets = Twitter.user_timeline(options['username'], opts)
 
       tweets.each do |tweet|
-        memory[:since_id] = tweet.id if !memory[:since_id] || (tweet.id > memory[:since_id])
+        memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id'])
 
         create_event :payload => tweet.attrs
       end
diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb
index f1a6a601..2330c307 100644
--- a/app/models/agents/user_location_agent.rb
+++ b/app/models/agents/user_location_agent.rb
@@ -34,11 +34,11 @@ module Agents
     end
 
     def default_options
-      { :secret => SecureRandom.hex(7) }
+      { 'secret' => SecureRandom.hex(7) }
     end
 
     def validate_options
-      errors.add(:base, "secret is required and must be longer than 4 characters") unless options[:secret].present? && options[:secret].length > 4
+      errors.add(:base, "secret is required and must be longer than 4 characters") unless options['secret'].present? && options['secret'].length > 4
     end
   end
 end
\ No newline at end of file
diff --git a/app/models/agents/weather_agent.rb b/app/models/agents/weather_agent.rb
index 34058e04..5a94128b 100644
--- a/app/models/agents/weather_agent.rb
+++ b/app/models/agents/weather_agent.rb
@@ -45,30 +45,30 @@ module Agents
     end
 
     def wunderground
-      Wunderground.new(options[:api_key]) if key_setup?
+      Wunderground.new(options['api_key']) if key_setup?
     end
 
     def key_setup?
-      options[:api_key] && options[:api_key] != "your-key"
+      options['api_key'] && options['api_key'] != "your-key"
     end
 
     def default_options
       {
-        :api_key => "your-key",
-        :location => "94103"
+        'api_key' => "your-key",
+        'location' => "94103"
       }
     end
 
     def validate_options
-      errors.add(:base, "location is required") unless options[:location].present? || options[:zipcode].present?
-      errors.add(:base, "api_key is required") unless options[:api_key].present?
+      errors.add(:base, "location is required") unless options['location'].present? || options['zipcode'].present?
+      errors.add(:base, "api_key is required") unless options['api_key'].present?
     end
 
     def check
       if key_setup?
-        wunderground.forecast_for(options[:location] || options[:zipcode])["forecast"]["simpleforecast"]["forecastday"].each do |day|
+        wunderground.forecast_for(options['location'] || options['zipcode'])["forecast"]["simpleforecast"]["forecastday"].each do |day|
           if is_tomorrow?(day)
-            create_event :payload => day.merge(:location => options[:location] || options[:zipcode])
+            create_event :payload => day.merge('location' => options['location'] || options['zipcode'])
           end
         end
       end
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index 096baa40..c0cef555 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -15,19 +15,19 @@ module Agents
 
       To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
 
-      When parsing HTML or XML, these sub-hashes specify how to extract with a `:css` CSS selector and either `:text => true` or `attr` pointing to an attribute name to grab.  An example:
+      When parsing HTML or XML, these sub-hashes specify how to extract with a `css` CSS selector and either `'text': true` or `attr` pointing to an attribute name to grab.  An example:
 
-          :extract => {
-            :url => { :css => "#comic img", :attr => "src" },
-            :title => { :css => "#comic img", :attr => "title" },
-            :body_text => { :css => "div.main", :text => true }
+          'extract': {
+            'url': { 'css': "#comic img", 'attr': "src" },
+            'title': { 'css': "#comic img", 'attr': "title" },
+            'body_text': { 'css': "div.main", 'text': true }
           }
 
       When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about.  For example:
 
-          :extract => {
-            :title => { :path => "results.data[*].title" },
-            :description => { :path => "results.data[*].description" }
+          'extract': {
+            'title': { 'path': "results.data[*].title" },
+            'description': { 'path': "results.data[*].description" }
           }
 
       Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor.  E.g., if you're extracting rows, all extractors must match all rows.  For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
@@ -36,7 +36,7 @@ module Agents
     MD
 
     event_description do
-      "Events will have the fields you specified.  Your options look like:\n\n    #{Utils.pretty_print options[:extract]}"
+      "Events will have the fields you specified.  Your options look like:\n\n    #{Utils.pretty_print options['extract']}"
     end
 
     default_schedule "every_12h"
@@ -44,33 +44,33 @@ module Agents
     UNIQUENESS_LOOK_BACK = 30
 
     def working?
-      event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
+      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
       {
-          :expected_update_period_in_days => "2",
-          :url => "http://xkcd.com",
-          :type => "html",
-          :mode => :on_change,
-          :extract => {
-              :url => {:css => "#comic img", :attr => "src"},
-              :title => {:css => "#comic img", :attr => "title"}
+          'expected_update_period_in_days' => "2",
+          'url' => "http://xkcd.com",
+          'type' => "html",
+          'mode' => :on_change,
+          'extract' => {
+            'url' => {'css' => "#comic img", 'attr' => "src"},
+            'title' => {'css' => "#comic img", 'attr' => "title"}
           }
       }
     end
 
     def validate_options
-      errors.add(:base, "url and expected_update_period_in_days are required") unless options[:expected_update_period_in_days].present? && options[:url].present?
-      if !options[:extract].present? && extraction_type != "json"
+      errors.add(:base, "url and expected_update_period_in_days are required") unless options['expected_update_period_in_days'].present? && options['url'].present?
+      if !options['extract'].present? && extraction_type != "json"
         errors.add(:base, "extract is required for all types except json")
       end
     end
 
     def check
       hydra = Typhoeus::Hydra.new
-      log "Fetching #{options[:url]}"
-      request = Typhoeus::Request.new(options[:url], :followlocation => true)
+      log "Fetching #{options['url']}"
+      request = Typhoeus::Request.new(options['url'], :followlocation => true)
       request.on_failure do |response|
         error "Failed: #{response.inspect}"
       end
@@ -85,37 +85,37 @@ module Agents
           end
         else
           output = {}
-          options[:extract].each do |name, extraction_details|
+          options['extract'].each do |name, extraction_details|
             result = if extraction_type == "json"
-                       output[name] = Utils.values_at(doc, extraction_details[:path])
+                       output[name] = Utils.values_at(doc, extraction_details['path'])
                      else
-                       output[name] = doc.css(extraction_details[:css]).map { |node|
-                         if extraction_details[:attr]
-                           node.attr(extraction_details[:attr])
-                         elsif extraction_details[:text]
+                       output[name] = doc.css(extraction_details['css']).map { |node|
+                         if extraction_details['attr']
+                           node.attr(extraction_details['attr'])
+                         elsif extraction_details['text']
                            node.text()
                          else
-                           error ":attr or :text is required on HTML or XML extraction patterns"
+                           error "'attr' or 'text' is required on HTML or XML extraction patterns"
                            return
                          end
                        }
                      end
-            log "Extracting #{extraction_type} at #{extraction_details[:path] || extraction_details[:css]}: #{result}"
+            log "Extracting #{extraction_type} at #{extraction_details['path'] || extraction_details['css']}: #{result}"
           end
 
-          num_unique_lengths = options[:extract].keys.map { |name| output[name].length }.uniq
+          num_unique_lengths = options['extract'].keys.map { |name| output[name].length }.uniq
 
           if num_unique_lengths.length != 1
-            error "Got an uneven number of matches for #{options[:name]}: #{options[:extract].inspect}"
+            error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}"
             return
           end
       
           num_unique_lengths.first.times do |index|
             result = {}
-            options[:extract].keys.each do |name|
+            options['extract'].keys.each do |name|
               result[name] = output[name][index]
               if name.to_s == 'url'
-                result[name] = URI.join(options[:url], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
+                result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
               end
             end
 
@@ -133,22 +133,22 @@ module Agents
     private
 
     def store_payload? result
-      !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json))
+      !options['mode'] || options['mode'].to_s == "all" || (options['mode'].to_s == "on_change" && !previous_payloads.include?(result.to_json))
     end
 
     def previous_payloads
-      events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change"
+      events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options['mode'].to_s == "on_change"
     end
 
     def extract_full_json?
-      (!options[:extract].present? && extraction_type == "json")
+      (!options['extract'].present? && extraction_type == "json")
     end
 
     def extraction_type
-      (options[:type] || begin
-        if options[:url] =~ /\.(rss|xml)$/i
+      (options['type'] || begin
+        if options['url'] =~ /\.(rss|xml)$/i
           "xml"
-        elsif options[:url] =~ /\.json$/i
+        elsif options['url'] =~ /\.json$/i
           "json"
         else
           "html"
diff --git a/app/models/agents/weibo_publish_agent.rb b/app/models/agents/weibo_publish_agent.rb
index bc0fc07e..4930c93b 100644
--- a/app/models/agents/weibo_publish_agent.rb
+++ b/app/models/agents/weibo_publish_agent.rb
@@ -20,24 +20,24 @@ module Agents
     MD
 
     def validate_options
-      unless options[:uid].present? &&
-        options[:expected_update_period_in_days].present?
+      unless options['uid'].present? &&
+        options['expected_update_period_in_days'].present?
         errors.add(:base, "expected_update_period_in_days and uid are required")
       end
     end
 
     def working?
-      (event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs?
+      (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
     end
 
     def default_options
       {
-          :uid => "",
-          :access_token => "---",
-          :app_key => "---",
-          :app_secret => "---",
-          :expected_update_period_in_days => "10",
-          :message_path => "text"
+        'uid' => "",
+        'access_token' => "---",
+        'app_key' => "---",
+        'app_secret' => "---",
+        'expected_update_period_in_days' => "10",
+        'message_path' => "text"
       }
     end
 
@@ -47,25 +47,25 @@ module Agents
         incoming_events = incoming_events.first(20)
       end
       incoming_events.each do |event|
-        tweet_text = Utils.value_at(event.payload, options[:message_path])
+        tweet_text = Utils.value_at(event.payload, options['message_path'])
         if event.agent.type == "Agents::TwitterUserAgent"
           tweet_text = unwrap_tco_urls(tweet_text, event.payload)
         end
         begin
           publish_tweet tweet_text
           create_event :payload => {
-            :success => true,
-            :published_tweet => tweet_text,
-            :agent_id => event.agent_id,
-            :event_id => event.id
+            'success' => true,
+            'published_tweet' => tweet_text,
+            'agent_id' => event.agent_id,
+            'event_id' => event.id
           }
         rescue OAuth2::Error => e
           create_event :payload => {
-            :success => false,
-            :error => e.message,
-            :failed_tweet => tweet_text,
-            :agent_id => event.agent_id,
-            :event_id => event.id
+            'success' => false,
+            'error' => e.message,
+            'failed_tweet' => tweet_text,
+            'agent_id' => event.agent_id,
+            'event_id' => event.id
           }
         end
       end
diff --git a/app/models/agents/weibo_user_agent.rb b/app/models/agents/weibo_user_agent.rb
index 27c94b29..d00120dd 100644
--- a/app/models/agents/weibo_user_agent.rb
+++ b/app/models/agents/weibo_user_agent.rb
@@ -70,29 +70,29 @@ module Agents
     default_schedule "every_1h"
 
     def validate_options
-      unless options[:uid].present? &&
-        options[:expected_update_period_in_days].present?
+      unless options['uid'].present? &&
+        options['expected_update_period_in_days'].present?
         errors.add(:base, "expected_update_period_in_days and uid are required")
       end
     end
 
     def working?
-      event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
+      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
       {
-          :uid => "",
-          :access_token => "---",
-          :app_key => "---",
-          :app_secret => "---",
-          :expected_update_period_in_days => "2"
+        'uid' => "",
+        'access_token' => "---",
+        'app_key' => "---",
+        'app_secret' => "---",
+        'expected_update_period_in_days' => "2"
       }
     end
 
     def check
-      since_id = memory[:since_id] || nil
-      opts = {:uid => options[:uid].to_i}
+      since_id = memory['since_id'] || nil
+      opts = {:uid => options['uid'].to_i}
       opts.merge! :since_id => since_id unless since_id.nil?
 
       # http://open.weibo.com/wiki/2/statuses/user_timeline/en
@@ -101,7 +101,7 @@ module Agents
 
 
         resp[:statuses].each do |status|
-          memory[:since_id] = status.id if !memory[:since_id] || (status.id > memory[:since_id])
+          memory['since_id'] = status.id if !memory['since_id'] || (status.id > memory['since_id'])
 
           create_event :payload => status.as_json
         end
diff --git a/app/models/event.rb b/app/models/event.rb
index f7c99256..883b8a26 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -1,13 +1,16 @@
-require 'serialize_and_normalize'
+require 'json_with_indifferent_access'
 
 class Event < ActiveRecord::Base
-  include SerializeAndNormalize
-
   attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at
 
   acts_as_mappable
 
-  serialize_and_normalize :payload
+  serialize :payload, JSONWithIndifferentAccess
+
+  def payload=(o)
+    self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
 
   belongs_to :user
   belongs_to :agent, :counter_cache => true
diff --git a/lib/json_with_indifferent_access.rb b/lib/json_with_indifferent_access.rb
new file mode 100644
index 00000000..ee02daf2
--- /dev/null
+++ b/lib/json_with_indifferent_access.rb
@@ -0,0 +1,9 @@
+class JSONWithIndifferentAccess
+  def self.load(json)
+    ActiveSupport::HashWithIndifferentAccess.new(JSON.load(json || '{}'))
+  end
+
+  def self.dump(hash)
+    JSON.dump(hash)
+  end
+end
\ No newline at end of file
diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb
deleted file mode 100644
index 857a777f..00000000
--- a/lib/serialize_and_normalize.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-module SerializeAndNormalize
-  extend ActiveSupport::Concern
-
-  module ClassMethods
-    def serialize_and_normalize(*column_names)
-      column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
-        setup_name = "setup_#{column_name}".to_sym
-        normalize_name = "normalize_#{column_name}".to_sym
-        validate_name = "validate_#{column_name}".to_sym
-
-        serialize column_name, JSON
-        after_initialize setup_name
-        before_validation normalize_name
-        before_save normalize_name
-        validate validate_name
-
-        class_eval <<-RUBY
-          def #{setup_name}
-            self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new
-          end
-
-          def #{validate_name}
-            # Implement me in your subclass.
-          end
-
-          def #{normalize_name}
-            self.#{column_name} = self[:#{column_name}]
-          end
-
-          def #{column_name}=(data)
-            data = (JSON.parse(data) rescue data) if data.is_a?(String)
-
-            case data
-              when ActiveSupport::HashWithIndifferentAccess
-                self[:#{column_name}] = data
-              when Hash
-                self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data)
-              else
-                self[:#{column_name}] = data
-            end
-          end
-        RUBY
-      end
-    end
-  end
-end
diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb
index 4b5c164a..5e073583 100644
--- a/spec/controllers/agents_controller_spec.rb
+++ b/spec/controllers/agents_controller_spec.rb
@@ -5,7 +5,7 @@ describe AgentsController do
     {
         :type => "Agents::WebsiteAgent",
         :name => "Something",
-        :options => agents(:bob_website_agent).options.to_json,
+        :options => agents(:bob_website_agent).options,
         :source_ids => [agents(:bob_weather_agent).id, ""]
     }.merge(options)
   end
diff --git a/spec/fixtures/agents.yml b/spec/fixtures/agents.yml
index df47d77f..0066ec73 100644
--- a/spec/fixtures/agents.yml
+++ b/spec/fixtures/agents.yml
@@ -11,7 +11,7 @@ jane_website_agent:
                      :title => {:css => "item title", :text => true},
                      :url => {:css => "item link", :text => true}
                  }
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 bob_website_agent:
   type: Agents::WebsiteAgent
@@ -26,7 +26,7 @@ bob_website_agent:
                    :url => {:css => "#comic img", :attr => "src"},
                    :title => {:css => "#comic img", :attr => "title"}
                  }
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 bob_weather_agent:
   type: Agents::WeatherAgent
@@ -38,7 +38,7 @@ bob_weather_agent:
                  :lat => 37.779329,
                  :lng => -122.41915,
                  :api_key => 'test'
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 jane_weather_agent:
   type: Agents::WeatherAgent
@@ -50,7 +50,7 @@ jane_weather_agent:
                  :lat => 37.779329,
                  :lng => -122.41915,
                  :api_key => 'test'
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 jane_rain_notifier_agent:
   type: Agents::TriggerAgent
@@ -64,7 +64,7 @@ jane_rain_notifier_agent:
                    :path => "conditions"
                  }],
                  :message => "Just so you know, it looks like '<conditions>' tomorrow in <location>"
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 bob_rain_notifier_agent:
   type: Agents::TriggerAgent
@@ -78,7 +78,7 @@ bob_rain_notifier_agent:
                    :path => "conditions"
                   }],
                  :message => "Just so you know, it looks like '<conditions>' tomorrow in <location>"
-               }.to_yaml.inspect %>
+               }.to_json.inspect %>
 
 bob_twitter_user_agent:
   type: Agents::TwitterUserAgent
@@ -91,7 +91,7 @@ bob_twitter_user_agent:
       :consumer_secret => "---",
       :oauth_token => "---",
       :oauth_token_secret => "---"
-    }.to_yaml.inspect %>
+    }.to_json.inspect %>
 
 bob_manual_event_agent:
   type: Agents::ManualEventAgent
diff --git a/spec/fixtures/events.yml b/spec/fixtures/events.yml
index 51adfde7..4ed2e86d 100644
--- a/spec/fixtures/events.yml
+++ b/spec/fixtures/events.yml
@@ -1,9 +1,9 @@
 bob_website_agent_event:
   user: bob
   agent: bob_website_agent
-  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %>
+  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>
 
 jane_website_agent_event:
   user: jane
   agent: jane_website_agent
-  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %>
+  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>
diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb
index 34be7129..6e9e9efc 100644
--- a/spec/models/agent_spec.rb
+++ b/spec/models/agent_spec.rb
@@ -261,9 +261,9 @@ describe Agent do
       it "symbolizes memory before validating" do
         agent = Agents::SomethingSource.new(:name => "something")
         agent.user = users(:bob)
-        agent.memory["bad"] = :hello
+        agent.memory["bad"] = 2
         agent.save
-        agent.memory[:bad].should == :hello
+        agent.memory[:bad].should == 2
       end
 
       it "should not allow agents owned by other people" do
diff --git a/spec/models/agents/digest_email_agent_spec.rb b/spec/models/agents/digest_email_agent_spec.rb
index 86ddf9d2..68e4b9b3 100644
--- a/spec/models/agents/digest_email_agent_spec.rb
+++ b/spec/models/agents/digest_email_agent_spec.rb
@@ -19,16 +19,16 @@ describe Agents::DigestEmailAgent do
     it "queues any payloads it receives" do
       event1 = Event.new
       event1.agent = agents(:bob_rain_notifier_agent)
-      event1.payload = "Something you should know about"
+      event1.payload = { :data => "Something you should know about" }
       event1.save!
 
       event2 = Event.new
       event2.agent = agents(:bob_weather_agent)
-      event2.payload = "Something else you should know about"
+      event2.payload = { :data => "Something else you should know about" }
       event2.save!
 
       Agents::DigestEmailAgent.async_receive(@checker.id, [event1.id, event2.id])
-      @checker.reload.memory[:queue].should == ["Something you should know about", "Something else you should know about"]
+      @checker.reload.memory[:queue].should == [{ 'data' => "Something you should know about" }, { 'data' => "Something else you should know about" }]
     end
   end
 
@@ -37,7 +37,7 @@ describe Agents::DigestEmailAgent do
       Agents::DigestEmailAgent.async_check(@checker.id)
       ActionMailer::Base.deliveries.should == []
 
-      @checker.memory[:queue] = ["Something you should know about",
+      @checker.memory[:queue] = [{ :data => "Something you should know about" },
                                  { :title => "Foo", :url => "http://google.com", :bar => 2 },
                                  { "message" => "hi", :woah => "there" },
                                  { "test" => 2 }]
@@ -47,7 +47,7 @@ describe Agents::DigestEmailAgent do
       Agents::DigestEmailAgent.async_check(@checker.id)
       ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"]
       ActionMailer::Base.deliveries.last.subject.should == "something interesting"
-      get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something you should know about\n\nFoo\n  bar: 2\n  url: http://google.com\n\nhi\n  woah: there\n\nEvent\n  test: 2"
+      get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n  data: Something you should know about\n\nFoo\n  bar: 2\n  url: http://google.com\n\nhi\n  woah: there\n\nEvent\n  test: 2"
       @checker.reload.memory[:queue].should be_empty
     end
 
diff --git a/spec/models/agents/email_agent_spec.rb b/spec/models/agents/email_agent_spec.rb
index 2a7c3d40..fcc3429a 100644
--- a/spec/models/agents/email_agent_spec.rb
+++ b/spec/models/agents/email_agent_spec.rb
@@ -21,12 +21,12 @@ describe Agents::EmailAgent do
 
       event1 = Event.new
       event1.agent = agents(:bob_rain_notifier_agent)
-      event1.payload = "Something you should know about"
+      event1.payload = { :data => "Something you should know about" }
       event1.save!
 
       event2 = Event.new
       event2.agent = agents(:bob_weather_agent)
-      event2.payload = "Something else you should know about"
+      event2.payload = { :data => "Something else you should know about" }
       event2.save!
 
       Agents::EmailAgent.async_receive(@checker.id, [event1.id])
@@ -35,8 +35,8 @@ describe Agents::EmailAgent do
       ActionMailer::Base.deliveries.count.should == 2
       ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"]
       ActionMailer::Base.deliveries.last.subject.should == "something interesting"
-      get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something else you should know about"
-      get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Something you should know about"
+      get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n  data: Something else you should know about"
+      get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Event\n  data: Something you should know about"
     end
 
     it "can receive complex events and send them on" do
diff --git a/spec/models/agents/human_task_agent_spec.rb b/spec/models/agents/human_task_agent_spec.rb
index 054ea4cf..89639316 100644
--- a/spec/models/agents/human_task_agent_spec.rb
+++ b/spec/models/agents/human_task_agent_spec.rb
@@ -9,8 +9,8 @@ describe Agents::HumanTaskAgent do
 
     @event = Event.new
     @event.agent = agents(:bob_rain_notifier_agent)
-    @event.payload = { :foo => { "bar" => { :baz => "a2b" } },
-                       :name => "Joe" }
+    @event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" } },
+                       'name' => "Joe" }
     @event.id = 345
 
     @checker.should be_valid
@@ -18,110 +18,110 @@ describe Agents::HumanTaskAgent do
 
   describe "validations" do
     it "validates that trigger_on is 'schedule' or 'event'" do
-      @checker.options[:trigger_on] = "foo"
+      @checker.options['trigger_on'] = "foo"
       @checker.should_not be_valid
     end
 
     it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
-      @checker.options[:trigger_on] = "event"
-      @checker.options[:expected_receive_period_in_days] = nil
+      @checker.options['trigger_on'] = "event"
+      @checker.options['expected_receive_period_in_days'] = nil
       @checker.should_not be_valid
-      @checker.options[:expected_receive_period_in_days] = 2
+      @checker.options['expected_receive_period_in_days'] = 2
       @checker.should be_valid
     end
 
     it "requires a positive submission_period when trigger_on is set to 'schedule'" do
-      @checker.options[:trigger_on] = "schedule"
-      @checker.options[:submission_period] = nil
+      @checker.options['trigger_on'] = "schedule"
+      @checker.options['submission_period'] = nil
       @checker.should_not be_valid
-      @checker.options[:submission_period] = 2
+      @checker.options['submission_period'] = 2
       @checker.should be_valid
     end
 
     it "requires a hit.title" do
-      @checker.options[:hit][:title] = ""
+      @checker.options['hit']['title'] = ""
       @checker.should_not be_valid
     end
 
     it "requires a hit.description" do
-      @checker.options[:hit][:description] = ""
+      @checker.options['hit']['description'] = ""
       @checker.should_not be_valid
     end
 
     it "requires hit.assignments" do
-      @checker.options[:hit][:assignments] = ""
+      @checker.options['hit']['assignments'] = ""
       @checker.should_not be_valid
-      @checker.options[:hit][:assignments] = 0
+      @checker.options['hit']['assignments'] = 0
       @checker.should_not be_valid
-      @checker.options[:hit][:assignments] = "moose"
+      @checker.options['hit']['assignments'] = "moose"
       @checker.should_not be_valid
-      @checker.options[:hit][:assignments] = "2"
+      @checker.options['hit']['assignments'] = "2"
       @checker.should be_valid
     end
 
     it "requires hit.questions" do
-      old_questions = @checker.options[:hit][:questions]
-      @checker.options[:hit][:questions] = nil
+      old_questions = @checker.options['hit']['questions']
+      @checker.options['hit']['questions'] = nil
       @checker.should_not be_valid
-      @checker.options[:hit][:questions] = []
+      @checker.options['hit']['questions'] = []
       @checker.should_not be_valid
-      @checker.options[:hit][:questions] = [old_questions[0]]
+      @checker.options['hit']['questions'] = [old_questions[0]]
       @checker.should be_valid
     end
 
     it "requires that all questions have key, name, required, type, and question" do
-      old_questions = @checker.options[:hit][:questions]
-      @checker.options[:hit][:questions].first[:key] = ""
+      old_questions = @checker.options['hit']['questions']
+      @checker.options['hit']['questions'].first['key'] = ""
       @checker.should_not be_valid
 
-      @checker.options[:hit][:questions] = old_questions
-      @checker.options[:hit][:questions].first[:name] = ""
+      @checker.options['hit']['questions'] = old_questions
+      @checker.options['hit']['questions'].first['name'] = ""
       @checker.should_not be_valid
 
-      @checker.options[:hit][:questions] = old_questions
-      @checker.options[:hit][:questions].first[:required] = nil
+      @checker.options['hit']['questions'] = old_questions
+      @checker.options['hit']['questions'].first['required'] = nil
       @checker.should_not be_valid
 
-      @checker.options[:hit][:questions] = old_questions
-      @checker.options[:hit][:questions].first[:type] = ""
+      @checker.options['hit']['questions'] = old_questions
+      @checker.options['hit']['questions'].first['type'] = ""
       @checker.should_not be_valid
 
-      @checker.options[:hit][:questions] = old_questions
-      @checker.options[:hit][:questions].first[:question] = ""
+      @checker.options['hit']['questions'] = old_questions
+      @checker.options['hit']['questions'].first['question'] = ""
       @checker.should_not be_valid
     end
 
     it "requires that all questions of type 'selection' have a selections array with keys and text" do
-      @checker.options[:hit][:questions][0][:selections] = []
+      @checker.options['hit']['questions'][0]['selections'] = []
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{}]
+      @checker.options['hit']['questions'][0]['selections'] = [{}]
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }]
+      @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "" }]
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }]
+      @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "hi" }]
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }]
+      @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "" }]
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }]
+      @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }]
       @checker.should be_valid
-      @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}]
+      @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }, {}]
       @checker.should_not be_valid
     end
 
     it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
-      @checker.options[:take_majority] = "true"
+      @checker.options['take_majority'] = "true"
       @checker.should_not be_valid
-      @checker.options[:hit][:questions][1][:type] = "selection"
-      @checker.options[:hit][:questions][1][:selections] = @checker.options[:hit][:questions][0][:selections]
+      @checker.options['hit']['questions'][1]['type'] = "selection"
+      @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
       @checker.should be_valid
     end
   end
 
   describe "when 'trigger_on' is set to 'schedule'" do
     before do
-      @checker.options[:trigger_on] = "schedule"
-      @checker.options[:submission_period] = "2"
-      @checker.options.delete(:expected_receive_period_in_days)
+      @checker.options['trigger_on'] = "schedule"
+      @checker.options['submission_period'] = "2"
+      @checker.options.delete('expected_receive_period_in_days')
     end
 
     it "should check for reviewable HITs frequently" do
@@ -151,7 +151,7 @@ describe Agents::HumanTaskAgent do
 
   describe "when 'trigger_on' is set to 'event'" do
     it "should not create HITs during check but should check for reviewable HITs" do
-      @checker.options[:submission_period] = "2"
+      @checker.options['submission_period'] = "2"
       now = Time.now
       stub(Time).now { now }
       mock(@checker).review_hits.times(3)
@@ -171,9 +171,9 @@ describe Agents::HumanTaskAgent do
 
   describe "creating hits" do
     it "can create HITs based on events, interpolating their values" do
-      @checker.options[:hit][:title] = "Hi <.name>"
-      @checker.options[:hit][:description] = "Make something for <.name>"
-      @checker.options[:hit][:questions][0][:name] = "<.name> Question 1"
+      @checker.options['hit']['title'] = "Hi <.name>"
+      @checker.options['hit']['description'] = "Make something for <.name>"
+      @checker.options['hit']['questions'][0]['name'] = "<.name> Question 1"
 
       question_form = nil
       hitInterface = OpenStruct.new
@@ -183,8 +183,8 @@ describe Agents::HumanTaskAgent do
 
       @checker.send :create_hit, @event
 
-      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
-      hitInterface.reward.should == @checker.options[:hit][:reward]
+      hitInterface.max_assignments.should == @checker.options['hit']['assignments']
+      hitInterface.reward.should == @checker.options['hit']['reward']
       hitInterface.description.should == "Make something for Joe"
 
       xml = question_form.to_xml
@@ -192,18 +192,18 @@ describe Agents::HumanTaskAgent do
       xml.should include("<Text>Make something for Joe</Text>")
       xml.should include("<DisplayName>Joe Question 1</DisplayName>")
 
-      @checker.memory[:hits][123].should == @event.id
+      @checker.memory['hits'][123].should == @event.id
     end
 
     it "works without an event too" do
-      @checker.options[:hit][:title] = "Hi <.name>"
+      @checker.options['hit']['title'] = "Hi <.name>"
       hitInterface = OpenStruct.new
       hitInterface.id = 123
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
       mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
       @checker.send :create_hit
-      hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
-      hitInterface.reward.should == @checker.options[:hit][:reward]
+      hitInterface.max_assignments.should == @checker.options['hit']['assignments']
+      hitInterface.reward.should == @checker.options['hit']['reward']
     end
   end
 
@@ -253,14 +253,14 @@ describe Agents::HumanTaskAgent do
     it "should work on multiple HITs" do
       event2 = Event.new
       event2.agent = agents(:bob_rain_notifier_agent)
-      event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } },
-                          :name2 => "Joe2" }
+      event2.payload = { 'foo2' => { "bar2" => { 'baz2' => "a2b2" } },
+                         'name2' => "Joe2" }
       event2.id = 3452
 
       # It knows about two HITs from two different events.
-      @checker.memory[:hits] = {}
-      @checker.memory[:hits][:"JH3132836336DHG"] = @event.id
-      @checker.memory[:hits][:"JH39AA63836DHG"] = event2.id
+      @checker.memory['hits'] = {}
+      @checker.memory['hits']["JH3132836336DHG"] = @event.id
+      @checker.memory['hits']["JH39AA63836DHG"] = event2.id
 
       hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
@@ -273,7 +273,7 @@ describe Agents::HumanTaskAgent do
     end
 
     it "shouldn't do anything if an assignment isn't ready" do
-      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Accepted", :answers => {}),
@@ -288,11 +288,11 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
     end
 
     it "shouldn't do anything if an assignment is missing" do
-      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
@@ -306,11 +306,11 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
     end
 
     it "should create events when all assignments are ready" do
-      @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
@@ -327,32 +327,32 @@ describe Agents::HumanTaskAgent do
       assignments.all? {|a| a.approved == true }.should be_true
       hit.should be_disposed
 
-      @checker.events.last.payload[:answers].should == [
+      @checker.events.last.payload['answers'].should == [
         {'sentiment' => "neutral", 'feedback' => ""},
         {'sentiment' => "happy", 'feedback' => "Take 2"}
       ]
 
-      @checker.memory[:hits].should == {}
+      @checker.memory['hits'].should == {}
     end
 
     describe "taking majority votes" do
       before do
-        @checker.options[:take_majority] = "true"
-        @checker.memory[:hits] = { "JH3132836336DHG" => @event.id }
+        @checker.options['take_majority'] = "true"
+        @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
         mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       end
 
       it "should take the majority votes of all questions" do
-        @checker.options[:hit][:questions][1] = {
-          :type => "selection",
-          :key => "age_range",
-          :name => "Age Range",
-          :required => "true",
-          :question => "Please select your age range:",
-          :selections =>
+        @checker.options['hit']['questions'][1] = {
+          'type' => "selection",
+          'key' => "age_range",
+          'name' => "Age Range",
+          'required' => "true",
+          'question' => "Please select your age range:",
+          'selections' =>
             [
-              { :key => "<50", :text => "50 years old or younger" },
-              { :key => ">50", :text => "Over 50 years old" }
+              { 'key' => "<50", 'text' => "50 years old or younger" },
+              { 'key' => ">50", 'text' => "Over 50 years old" }
             ]
         }
 
@@ -371,35 +371,35 @@ describe Agents::HumanTaskAgent do
 
         assignments.all? {|a| a.approved == true }.should be_true
 
-        @checker.events.last.payload[:answers].should == [
+        @checker.events.last.payload['answers'].should == [
           { 'sentiment' => "sad", 'age_range' => "<50" },
           { 'sentiment' => "neutral", 'age_range' => ">50" },
           { 'sentiment' => "happy", 'age_range' => ">50" },
           { 'sentiment' => "happy", 'age_range' => ">50" }
         ]
 
-        @checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
-        @checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" }
-        @checker.events.last.payload.should_not have_key(:average_answer)
+        @checker.events.last.payload['counts'].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
+        @checker.events.last.payload['majority_answer'].should == { 'sentiment' => "happy", 'age_range' => ">50" }
+        @checker.events.last.payload.should_not have_key('average_answer')
 
-        @checker.memory[:hits].should == {}
+        @checker.memory['hits'].should == {}
       end
 
       it "should also provide an average answer when all questions are numeric" do
-        @checker.options[:hit][:questions] = [
+        @checker.options['hit']['questions'] = [
           {
-            :type => "selection",
-            :key => "rating",
-            :name => "Rating",
-            :required => "true",
-            :question => "Please select a rating:",
-            :selections =>
+            'type' => "selection",
+            'key' => "rating",
+            'name' => "Rating",
+            'required' => "true",
+            'question' => "Please select a rating:",
+            'selections' =>
               [
-                { :key => "1", :text => "One" },
-                { :key => "2", :text => "Two" },
-                { :key => "3", :text => "Three" },
-                { :key => "4", :text => "Four" },
-                { :key => "5.1", :text => "Five Point One" }
+                { 'key' => "1", 'text' => "One" },
+                { 'key' => "2", 'text' => "Two" },
+                { 'key' => "3", 'text' => "Three" },
+                { 'key' => "4", 'text' => "Four" },
+                { 'key' => "5.1", 'text' => "Five Point One" }
               ]
           }
         ]
@@ -420,7 +420,7 @@ describe Agents::HumanTaskAgent do
 
         assignments.all? {|a| a.approved == true }.should be_true
 
-        @checker.events.last.payload[:answers].should == [
+        @checker.events.last.payload['answers'].should == [
           { 'rating' => "1" },
           { 'rating' => "3" },
           { 'rating' => "5.1" },
@@ -428,11 +428,11 @@ describe Agents::HumanTaskAgent do
           { 'rating' => "2" }
         ]
 
-        @checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
-        @checker.events.last.payload[:majority_answer].should == { 'rating' => "2" }
-        @checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
+        @checker.events.last.payload['counts'].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
+        @checker.events.last.payload['majority_answer'].should == { 'rating' => "2" }
+        @checker.events.last.payload['average_answer'].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
 
-        @checker.memory[:hits].should == {}
+        @checker.memory['hits'].should == {}
       end
     end
   end
diff --git a/spec/models/agents/peak_detector_agent_spec.rb b/spec/models/agents/peak_detector_agent_spec.rb
index 53663e27..b55e2b77 100644
--- a/spec/models/agents/peak_detector_agent_spec.rb
+++ b/spec/models/agents/peak_detector_agent_spec.rb
@@ -3,12 +3,12 @@ require 'spec_helper'
 describe Agents::PeakDetectorAgent do
   before do
     @valid_params = {
-        :name => "my peak detector agent",
-        :options => {
-            :expected_receive_period_in_days => "2",
-            :group_by_path => "filter",
-            :value_path => "count",
-            :message => "A peak was found"
+        'name' => "my peak detector agent",
+        'options' => {
+          'expected_receive_period_in_days' => "2",
+          'group_by_path' => "filter",
+          'value_path' => "count",
+          'message' => "A peak was found"
         }
     }
 
@@ -19,54 +19,54 @@ describe Agents::PeakDetectorAgent do
 
   describe "#receive" do
     it "tracks and groups by the group_by_path" do
-      events = build_events(:keys => [:count, :filter],
+      events = build_events(:keys => ['count', 'filter'],
                             :values => [[1, "something"], [2, "something"], [3, "else"]])
       @agent.receive events
-      @agent.memory[:data][:something].map(&:first).should == [1, 2]
-      @agent.memory[:data][:something].last.last.should be_within(10).of((100 - 1).hours.ago.to_i)
-      @agent.memory[:data][:else].first.first.should == 3
-      @agent.memory[:data][:else].first.last.should be_within(10).of((100 - 2).hours.ago.to_i)
+      @agent.memory['data']['something'].map(&:first).should == [1, 2]
+      @agent.memory['data']['something'].last.last.should be_within(10).of((100 - 1).hours.ago.to_i)
+      @agent.memory['data']['else'].first.first.should == 3
+      @agent.memory['data']['else'].first.last.should be_within(10).of((100 - 2).hours.ago.to_i)
     end
 
     it "works without a group_by_path as well" do
-      @agent.options[:group_by_path] = ""
-      events = build_events(:keys => [:count], :values => [[1], [2]])
+      @agent.options['group_by_path'] = ""
+      events = build_events(:keys => ['count'], :values => [[1], [2]])
       @agent.receive events
-      @agent.memory[:data][:no_group].map(&:first).should == [1, 2]
+      @agent.memory['data']['no_group'].map(&:first).should == [1, 2]
     end
 
     it "keeps a rolling window of data" do
-      @agent.options[:window_duration_in_days] = 5/24.0
-      @agent.receive build_events(:keys => [:count],
+      @agent.options['window_duration_in_days'] = 5/24.0
+      @agent.receive build_events(:keys => ['count'],
                                   :values => [1, 2, 3, 4, 5, 6, 7, 8].map {|i| [i]},
-                                  :pattern => { :filter => "something" })
-      @agent.memory[:data][:something].map(&:first).should == [4, 5, 6, 7, 8]
+                                  :pattern => { 'filter' => "something" })
+      @agent.memory['data']['something'].map(&:first).should == [4, 5, 6, 7, 8]
     end
 
     it "finds peaks" do
-      build_events(:keys => [:count],
+      build_events(:keys => ['count'],
                    :values => [5, 6,
                                4, 5,
                                4, 5,
                                15, 11, # peak
                                8, 50, # ignored because it's too close to the first peak
                                4, 5].map {|i| [i]},
-                   :pattern => { :filter => "something" }).each.with_index do |event, index|
+                   :pattern => { 'filter' => "something" }).each.with_index do |event, index|
         lambda {
           @agent.receive([event])
         }.should change { @agent.events.count }.by( index == 6 ? 1 : 0 )
       end
 
-      @agent.events.last.payload[:peak].should == 15.0
-      @agent.memory[:peaks][:something].length.should == 1
+      @agent.events.last.payload['peak'].should == 15.0
+      @agent.memory['peaks']['something'].length.should == 1
     end
 
     it "keeps a rolling window of peaks" do
-      @agent.options[:min_peak_spacing_in_days] = 1/24.0
-      @agent.receive build_events(:keys => [:count],
+      @agent.options['min_peak_spacing_in_days'] = 1/24.0
+      @agent.receive build_events(:keys => ['count'],
                                   :values => [1, 1, 1, 1, 1, 1, 10, 1, 1, 1, 1, 1, 1, 1, 10, 1].map {|i| [i]},
-                                  :pattern => { :filter => "something" })
-      @agent.memory[:peaks][:something].length.should == 2
+                                  :pattern => { 'filter' => "something" })
+      @agent.memory['peaks']['something'].length.should == 2
     end
   end
 
@@ -76,17 +76,17 @@ describe Agents::PeakDetectorAgent do
     end
 
     it "should validate presence of message" do
-      @agent.options[:message] = nil
+      @agent.options['message'] = nil
       @agent.should_not be_valid
     end
 
     it "should validate presence of expected_receive_period_in_days" do
-      @agent.options[:expected_receive_period_in_days] = ""
+      @agent.options['expected_receive_period_in_days'] = ""
       @agent.should_not be_valid
     end
 
     it "should validate presence of value_path" do
-      @agent.options[:value_path] = ""
+      @agent.options['value_path'] = ""
       @agent.should_not be_valid
     end
   end
diff --git a/spec/models/agents/trigger_agent_spec.rb b/spec/models/agents/trigger_agent_spec.rb
index f525110a..22b3b6ec 100644
--- a/spec/models/agents/trigger_agent_spec.rb
+++ b/spec/models/agents/trigger_agent_spec.rb
@@ -3,16 +3,16 @@ require 'spec_helper'
 describe Agents::TriggerAgent do
   before do
     @valid_params = {
-        :name => "my trigger agent",
-        :options => {
-            :expected_receive_period_in_days => 2,
-            :rules => [{
-                           :type => "regex",
-                           'value' => "a\\db",
-                           :path => "foo.bar.baz",
-                       }],
-            :message => "I saw '<foo.bar.baz>' from <name>"
-        }
+      'name' => "my trigger agent",
+      'options' => {
+        'expected_receive_period_in_days' => 2,
+        'rules' => [{
+                      'type' => "regex",
+                      'value' => "a\\db",
+                      'path' => "foo.bar.baz",
+                    }],
+        'message' => "I saw '<foo.bar.baz>' from <name>"
+      }
     }
 
     @checker = Agents::TriggerAgent.new(@valid_params)
@@ -21,8 +21,8 @@ describe Agents::TriggerAgent do
 
     @event = Event.new
     @event.agent = agents(:bob_rain_notifier_agent)
-    @event.payload = { :foo => { "bar" => { :baz => "a2b" }},
-                       :name => "Joe" }
+    @event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" }},
+                       'name' => "Joe" }
   end
 
   describe "validation" do
@@ -31,22 +31,22 @@ describe Agents::TriggerAgent do
     end
 
     it "should validate presence of options" do
-      @checker.options[:message] = nil
+      @checker.options['message'] = nil
       @checker.should_not be_valid
     end
 
     it "should validate the three fields in each rule" do
-      @checker.options[:rules] << { :path => "foo", :type => "fake", :value => "6" }
+      @checker.options['rules'] << { 'path' => "foo", 'type' => "fake", 'value' => "6" }
       @checker.should_not be_valid
-      @checker.options[:rules].last[:type] = "field>=value"
+      @checker.options['rules'].last['type'] = "field>=value"
       @checker.should be_valid
-      @checker.options[:rules].last.delete(:value)
+      @checker.options['rules'].last.delete('value')
       @checker.should_not be_valid
     end
   end
 
   describe "#working?" do
-    it "checks to see if the Agent has received any events in the last :expected_receive_period_in_days days" do
+    it "checks to see if the Agent has received any events in the last 'expected_receive_period_in_days' days" do
       @event.save!
 
       @checker.should_not be_working # no events have ever been received
@@ -60,30 +60,30 @@ describe Agents::TriggerAgent do
 
   describe "#receive" do
     it "handles regex" do
-      @event.payload[:foo]["bar"][:baz] = "a222b"
+      @event.payload['foo']['bar']['baz'] = "a222b"
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
 
-      @event.payload[:foo]["bar"][:baz] = "a2b"
+      @event.payload['foo']['bar']['baz'] = "a2b"
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
     end
 
     it "handles negated regex" do
-      @event.payload[:foo]["bar"][:baz] = "a2b"
-      @checker.options[:rules][0] = {
-                                      :type => "!regex",
-                                      :value => "a\\db",
-                                      :path => "foo.bar.baz",
-                                    }
+      @event.payload['foo']['bar']['baz'] = "a2b"
+      @checker.options['rules'][0] = {
+        'type' => "!regex",
+        'value' => "a\\db",
+        'path' => "foo.bar.baz",
+      }
 
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
 
-      @event.payload[:foo]["bar"][:baz] = "a22b"
+      @event.payload['foo']['bar']['baz'] = "a22b"
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
@@ -91,49 +91,49 @@ describe Agents::TriggerAgent do
 
     it "puts can extract values into the message based on paths" do
       @checker.receive([@event])
-      Event.last.payload[:message].should == "I saw 'a2b' from Joe"
+      Event.last.payload['message'].should == "I saw 'a2b' from Joe"
     end
 
     it "handles numerical comparisons" do
-      @event.payload[:foo]["bar"][:baz] = "5"
-      @checker.options[:rules].first[:value] = 6
-      @checker.options[:rules].first[:type] = "field<value"
+      @event.payload['foo']['bar']['baz'] = "5"
+      @checker.options['rules'].first['value'] = 6
+      @checker.options['rules'].first['type'] = "field<value"
 
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
 
-      @checker.options[:rules].first[:value] = 3
+      @checker.options['rules'].first['value'] = 3
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
     end
 
     it "handles exact comparisons" do
-      @event.payload[:foo]["bar"][:baz] = "hello world"
-      @checker.options[:rules].first[:type] = "field==value"
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field==value"
 
-      @checker.options[:rules].first[:value] = "hello there"
+      @checker.options['rules'].first['value'] = "hello there"
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
 
-      @checker.options[:rules].first[:value] = "hello world"
+      @checker.options['rules'].first['value'] = "hello world"
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
     end
 
     it "handles negated comparisons" do
-      @event.payload[:foo]["bar"][:baz] = "hello world"
-      @checker.options[:rules].first[:type] = "field!=value"
-      @checker.options[:rules].first[:value] = "hello world"
+      @event.payload['foo']['bar']['baz'] = "hello world"
+      @checker.options['rules'].first['type'] = "field!=value"
+      @checker.options['rules'].first['value'] = "hello world"
 
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
 
-      @checker.options[:rules].first[:value] = "hello there"
+      @checker.options['rules'].first['value'] = "hello there"
 
       lambda {
         @checker.receive([@event])
@@ -141,20 +141,20 @@ describe Agents::TriggerAgent do
     end
 
     it "does fine without dots in the path" do
-      @event.payload = { :hello => "world" }
-      @checker.options[:rules].first[:type] = "field==value"
-      @checker.options[:rules].first[:path] = "hello"
-      @checker.options[:rules].first[:value] = "world"
+      @event.payload = { 'hello' => "world" }
+      @checker.options['rules'].first['type'] = "field==value"
+      @checker.options['rules'].first['path'] = "hello"
+      @checker.options['rules'].first['value'] = "world"
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
 
-      @checker.options[:rules].first[:path] = "foo"
+      @checker.options['rules'].first['path'] = "foo"
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
 
-      @checker.options[:rules].first[:value] = "hi"
+      @checker.options['rules'].first['value'] = "hi"
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }
@@ -163,11 +163,11 @@ describe Agents::TriggerAgent do
     it "handles multiple events" do
       event2 = Event.new
       event2.agent = agents(:bob_weather_agent)
-      event2.payload = { :foo => { "bar" => { :baz => "a2b" }}}
+      event2.payload = { 'foo' => { 'bar' => { 'baz' => "a2b" }}}
 
       event3 = Event.new
       event3.agent = agents(:bob_weather_agent)
-      event3.payload = { :foo => { "bar" => { :baz => "a222b" }}}
+      event3.payload = { 'foo' => { 'bar' => { 'baz' => "a222b" }}}
 
       lambda {
         @checker.receive([@event, event2, event3])
@@ -175,19 +175,19 @@ describe Agents::TriggerAgent do
     end
 
     it "handles ANDing rules together" do
-      @checker.options[:rules] << {
-          :type => "field>=value",
-          :value => "4",
-          :path => "foo.bing"
+      @checker.options['rules'] << {
+        'type' => "field>=value",
+        'value' => "4",
+        'path' => "foo.bing"
       }
 
-      @event.payload[:foo]["bing"] = "5"
+      @event.payload['foo']["bing"] = "5"
 
       lambda {
         @checker.receive([@event])
       }.should change { Event.count }.by(1)
 
-      @checker.options[:rules].last[:value] = 6
+      @checker.options['rules'].last['value'] = 6
       lambda {
         @checker.receive([@event])
       }.should_not change { Event.count }

From ba0967e44934a7ef52952eef23424780130fe02d Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Wed, 25 Dec 2013 00:08:39 -0500
Subject: [PATCH 04/10] puts

---
 app/models/agents/peak_detector_agent.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb
index bf277c2c..d8e058db 100644
--- a/app/models/agents/peak_detector_agent.rb
+++ b/app/models/agents/peak_detector_agent.rb
@@ -64,8 +64,6 @@ module Agents
         average_value, standard_deviation = stats_for(group, :skip_last => 1)
         newest_value, newest_time = memory['data'][group][-1].map(&:to_f)
 
-        #p [newest_value, average_value, average_value + std_multiple * standard_deviation, standard_deviation]
-
         if newest_value > average_value + std_multiple * standard_deviation
           memory['peaks'][group] << newest_time
           memory['peaks'][group].reject! { |p| p <= newest_time - window_duration }

From 21c0e5911dac0e05eb18877cf532234097274972 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Thu, 26 Dec 2013 18:36:09 -0500
Subject: [PATCH 05/10] transform and clean utf-8

---
 ...1223032112_switch_to_json_serialization.rb | 40 +++++++++++++++----
 1 file changed, 33 insertions(+), 7 deletions(-)

diff --git a/db/migrate/20131223032112_switch_to_json_serialization.rb b/db/migrate/20131223032112_switch_to_json_serialization.rb
index c132d532..13c46ac6 100644
--- a/db/migrate/20131223032112_switch_to_json_serialization.rb
+++ b/db/migrate/20131223032112_switch_to_json_serialization.rb
@@ -5,17 +5,41 @@ class SwitchToJsonSerialization < ActiveRecord::Migration
   }
 
   def up
-    puts "This migration will update Agent and Event storage from YAML to JSON.  It should work, but please make a backup"
-    puts "before proceeding."
-    print "Continue? (y/n) "
-    STDOUT.flush
-    exit unless STDIN.gets =~ /^y/i
+    if data_exists?
+      puts "This migration will update tables to use UTF-8 encoding and will update Agent and Event storage from YAML to JSON."
+      puts "It should work, but please make a backup before proceeding!"
+      print "Continue? (y/n) "
+      STDOUT.flush
+      exit unless STDIN.gets =~ /^y/i
 
-    translate YAML, JSON
+      set_to_utf8
+      translate YAML, JSON
+    end
   end
 
   def down
-    translate JSON, YAML
+    if data_exists?
+      translate JSON, YAML
+    end
+  end
+
+  def set_to_utf8
+    if mysql?
+      %w[agent_logs agents delayed_jobs events links taggings tags users].each do |table_name|
+        quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table_name)
+        execute "ALTER TABLE #{quoted_table_name} CONVERT TO CHARACTER SET utf8"
+      end
+    end
+  end
+
+  def mysql?
+    ActiveRecord::Base.connection.adapter_name =~ /mysql/i
+  end
+
+  def data_exists?
+    events = ActiveRecord::Base.connection.select_rows("SELECT count(*) FROM #{ActiveRecord::Base.connection.quote_table_name("events")}").first.first
+    agents = ActiveRecord::Base.connection.select_rows("SELECT count(*) FROM #{ActiveRecord::Base.connection.quote_table_name("agents")}").first.first
+    agents + events > 0
   end
 
   def translate(from, to)
@@ -29,6 +53,8 @@ class SwitchToJsonSerialization < ActiveRecord::Migration
 
         yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) }
 
+        yaml_fields.map! {|f| f.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '??') }
+
         update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?"
 
         sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id]

From bca562513b9aa5533193c5910f8f71fd71bbfc9b Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Thu, 26 Dec 2013 18:49:15 -0500
Subject: [PATCH 06/10] Only load the fields we need for the autocomplete.

---
 app/views/layouts/application.html.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index fcfc0d56..2ab0e63e 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -36,7 +36,7 @@
     <script>
       var agentPaths = {};
       <% if current_user -%>
-        var myAgents = <%= Utils.jsonify(current_user.agents.inject({}) {|m, a| m[a.name] = agent_path(a) unless a.new_record?; m }) %>;
+        var myAgents = <%= Utils.jsonify(current_user.agents.select([:name, :id, :schedule]).inject({}) {|m, a| m[a.name] = agent_path(a); m }) %>;
         $.extend(agentPaths, myAgents);
       <% end -%>
       agentPaths["All Agents Index"] = <%= Utils.jsonify agents_path %>;

From 00b7423dd7ee76729cac876c30af30dfac6484ef Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Thu, 26 Dec 2013 20:15:29 -0500
Subject: [PATCH 07/10] add cached columns for event creation and last errors,
 reducing the number of SQL queries

---
 .../javascripts/application.js.coffee.erb     |  1 +
 .../stylesheets/application.css.scss.erb      |  4 +++
 app/controllers/events_controller.rb          |  2 +-
 app/controllers/logs_controller.rb            |  2 +-
 app/models/agent.rb                           | 33 +++++++++----------
 app/models/agent_log.rb                       |  3 ++
 app/models/agents/adioso_agent.rb             |  2 +-
 app/models/agents/twitter_publish_agent.rb    |  2 +-
 app/models/agents/twitter_stream_agent.rb     |  2 +-
 app/models/agents/twitter_user_agent.rb       |  2 +-
 app/models/agents/user_location_agent.rb      |  2 +-
 app/models/agents/weather_agent.rb            |  2 +-
 app/models/agents/website_agent.rb            |  2 +-
 app/models/agents/weibo_publish_agent.rb      |  2 +-
 app/models/agents/weibo_user_agent.rb         |  2 +-
 app/models/event.rb                           | 11 +++----
 app/views/agents/show.html.erb                |  2 +-
 ...0131227000021_add_cached_dates_to_agent.rb | 14 ++++++++
 spec/controllers/logs_controller_spec.rb      |  4 ++-
 spec/models/agent_log_spec.rb                 |  8 +++++
 spec/models/agent_spec.rb                     | 26 +++++++++++++++
 spec/models/agents/website_agent_spec.rb      | 14 +++++---
 22 files changed, 101 insertions(+), 41 deletions(-)
 create mode 100644 db/migrate/20131227000021_add_cached_dates_to_agent.rb

diff --git a/app/assets/javascripts/application.js.coffee.erb b/app/assets/javascripts/application.js.coffee.erb
index 5cb719a2..55afb965 100644
--- a/app/assets/javascripts/application.js.coffee.erb
+++ b/app/assets/javascripts/application.js.coffee.erb
@@ -102,6 +102,7 @@ $(document).ready ->
       $("#logs .refresh, #logs .clear").hide()
       $.post "/agents/#{agentId}/logs/clear", { "_method": "DELETE" }, (html) =>
         $("#logs .logs").html html
+        $("#show-tabs li a.recent-errors").removeClass 'recent-errors'
         $("#logs .spinner").stop(true, true).fadeOut ->
           $("#logs .refresh, #logs .clear").show()
 
diff --git a/app/assets/stylesheets/application.css.scss.erb b/app/assets/stylesheets/application.css.scss.erb
index c72938ca..588debf3 100644
--- a/app/assets/stylesheets/application.css.scss.erb
+++ b/app/assets/stylesheets/application.css.scss.erb
@@ -122,3 +122,7 @@ span.not-applicable:after {
     margin: 0 10px;
   }
 }
+
+#show-tabs li a.recent-errors {
+  font-weight: bold;
+}
diff --git a/app/controllers/events_controller.rb b/app/controllers/events_controller.rb
index e1860a26..20635378 100644
--- a/app/controllers/events_controller.rb
+++ b/app/controllers/events_controller.rb
@@ -6,7 +6,7 @@ class EventsController < ApplicationController
       @agent = current_user.agents.find(params[:agent])
       @events = @agent.events.page(params[:page])
     else
-      @events = current_user.events.page(params[:page])
+      @events = current_user.events.preload(:agent).page(params[:page])
     end
 
     respond_to do |format|
diff --git a/app/controllers/logs_controller.rb b/app/controllers/logs_controller.rb
index c3fd4dfc..45da4177 100644
--- a/app/controllers/logs_controller.rb
+++ b/app/controllers/logs_controller.rb
@@ -7,7 +7,7 @@ class LogsController < ApplicationController
   end
 
   def clear
-    @agent.logs.delete_all
+    @agent.delete_logs!
     index
   end
 
diff --git a/app/models/agent.rb b/app/models/agent.rb
index 9d6a9f34..e472debc 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -19,14 +19,6 @@ class Agent < ActiveRecord::Base
   serialize :options, JSONWithIndifferentAccess
   serialize :memory, JSONWithIndifferentAccess
 
-  def options=(o)
-    self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
-  def memory=(o)
-    self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
   validates_presence_of :name, :user
   validate :sources_are_owned
   validate :validate_schedule
@@ -42,7 +34,6 @@ class Agent < ActiveRecord::Base
   has_many :events, :dependent => :delete_all, :inverse_of => :agent, :order => "events.id desc"
   has_one  :most_recent_event, :inverse_of => :agent, :class_name => "Event", :order => "events.id desc"
   has_many :logs, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog", :order => "agent_logs.id desc"
-  has_one  :most_recent_log, :inverse_of => :agent, :class_name => "AgentLog", :order => "agent_logs.id desc"
   has_many :received_events, :through => :sources, :class_name => "Event", :source => :events, :order => "events.id desc"
   has_many :links_as_source, :dependent => :delete_all, :foreign_key => "source_id", :class_name => "Link", :inverse_of => :source
   has_many :links_as_receiver, :dependent => :delete_all, :foreign_key => "receiver_id", :class_name => "Link", :inverse_of => :receiver
@@ -88,13 +79,20 @@ class Agent < ActiveRecord::Base
     # Implement me in your subclass to test for valid options.
   end
 
-  def event_created_within(days)
-    event = most_recent_event
-    event && event.created_at > days.to_i.days.ago && event.payload.present? && event
+  def options=(o)
+    self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
+  def memory=(o)
+    self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
+  def event_created_within?(days)
+    last_event_at && last_event_at > days.to_i.days.ago
   end
 
   def recent_error_logs?
-    most_recent_log.try(:level) == 4
+    last_event_at && last_error_log_at && last_error_log_at > (last_event_at - 2.minutes)
   end
 
   def sources_are_owned
@@ -134,10 +132,6 @@ class Agent < ActiveRecord::Base
     self.schedule = nil if cannot_be_scheduled?
   end
 
-  def last_event_at
-    @memoized_last_event_at ||= most_recent_event.try(:created_at)
-  end
-
   def default_schedule
     self.class.default_schedule
   end
@@ -172,6 +166,11 @@ class Agent < ActiveRecord::Base
     end
   end
 
+  def delete_logs!
+    logs.delete_all
+    update_column :last_error_log_at, nil
+  end
+
   def log(message, options = {})
     puts "Agent##{id}: #{message}" unless Rails.env.test?
     AgentLog.log_for_agent(self, message, options)
diff --git a/app/models/agent_log.rb b/app/models/agent_log.rb
index 7d10f6fd..ce9c000c 100644
--- a/app/models/agent_log.rb
+++ b/app/models/agent_log.rb
@@ -14,6 +14,9 @@ class AgentLog < ActiveRecord::Base
       oldest_id_to_keep = agent.logs.limit(1).offset(log_length - 1).pluck("agent_logs.id")
       agent.logs.where("agent_logs.id < ?", oldest_id_to_keep).delete_all
     end
+
+    agent.update_column :last_error_log_at, Time.now if log.level >= 4
+
     log
   end
 
diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb
index 662bb5e4..7cb64f1f 100644
--- a/app/models/agents/adioso_agent.rb
+++ b/app/models/agents/adioso_agent.rb
@@ -40,7 +40,7 @@ module Agents
     end
 
     def working?
-      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def validate_options
diff --git a/app/models/agents/twitter_publish_agent.rb b/app/models/agents/twitter_publish_agent.rb
index d03d7984..0576c469 100644
--- a/app/models/agents/twitter_publish_agent.rb
+++ b/app/models/agents/twitter_publish_agent.rb
@@ -26,7 +26,7 @@ module Agents
     end
 
     def working?
-      (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && most_recent_event.payload['success'] == true && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb
index f369b891..eb0bbd26 100644
--- a/app/models/agents/twitter_stream_agent.rb
+++ b/app/models/agents/twitter_stream_agent.rb
@@ -62,7 +62,7 @@ module Agents
     end
 
     def working?
-      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb
index 52d10a7b..0d7f57d2 100644
--- a/app/models/agents/twitter_user_agent.rb
+++ b/app/models/agents/twitter_user_agent.rb
@@ -48,7 +48,7 @@ module Agents
     end
 
     def working?
-      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb
index 2330c307..e1b53ffd 100644
--- a/app/models/agents/user_location_agent.rb
+++ b/app/models/agents/user_location_agent.rb
@@ -30,7 +30,7 @@ module Agents
     MD
 
     def working?
-      event_created_within(2) && !recent_error_logs?
+      event_created_within?(2) && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/weather_agent.rb b/app/models/agents/weather_agent.rb
index 5a94128b..44db74f0 100644
--- a/app/models/agents/weather_agent.rb
+++ b/app/models/agents/weather_agent.rb
@@ -41,7 +41,7 @@ module Agents
     default_schedule "8pm"
 
     def working?
-      event_created_within(2) && !recent_error_logs?
+      event_created_within?(2) && !recent_error_logs?
     end
 
     def wunderground
diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb
index c0cef555..d171e85b 100644
--- a/app/models/agents/website_agent.rb
+++ b/app/models/agents/website_agent.rb
@@ -44,7 +44,7 @@ module Agents
     UNIQUENESS_LOOK_BACK = 30
 
     def working?
-      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/weibo_publish_agent.rb b/app/models/agents/weibo_publish_agent.rb
index 4930c93b..d95f97b9 100644
--- a/app/models/agents/weibo_publish_agent.rb
+++ b/app/models/agents/weibo_publish_agent.rb
@@ -27,7 +27,7 @@ module Agents
     end
 
     def working?
-      (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && most_recent_event.payload['success'] == true && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/agents/weibo_user_agent.rb b/app/models/agents/weibo_user_agent.rb
index d00120dd..6c4de7ff 100644
--- a/app/models/agents/weibo_user_agent.rb
+++ b/app/models/agents/weibo_user_agent.rb
@@ -77,7 +77,7 @@ module Agents
     end
 
     def working?
-      event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
+      event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
     end
 
     def default_options
diff --git a/app/models/event.rb b/app/models/event.rb
index 883b8a26..adb77577 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -7,18 +7,17 @@ class Event < ActiveRecord::Base
 
   serialize :payload, JSONWithIndifferentAccess
 
-  def payload=(o)
-    self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
-
   belongs_to :user
-  belongs_to :agent, :counter_cache => true
+  belongs_to :agent, :counter_cache => true, :touch => :last_event_at
 
   scope :recent, lambda { |timespan = 12.hours.ago|
     where("events.created_at > ?", timespan)
   }
 
+  def payload=(o)
+    self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o)
+  end
+
   def reemit!
     agent.create_event :payload => payload, :lat => lat, :lng => lng
   end
diff --git a/app/views/agents/show.html.erb b/app/views/agents/show.html.erb
index a07d03a4..6f1b6359 100644
--- a/app/views/agents/show.html.erb
+++ b/app/views/agents/show.html.erb
@@ -11,7 +11,7 @@
             <li class='disabled'><a><i class='icon-picture'></i> Summary</a></li>
             <li class='active'><a href="#details" data-toggle="tab"><i class='icon-indent-left'></i> Details</a></li>
           <% end %>
-          <li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>"><i class='icon-list-alt'></i> Logs</a></li>
+          <li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>" class='<%= @agent.recent_error_logs? ? 'recent-errors' : '' %>'><i class='icon-list-alt'></i> Logs</a></li>
 
           <% if @agent.can_create_events? && @agent.events.count > 0 %>
             <li><%= link_to '<i class="icon-random"></i> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
diff --git a/db/migrate/20131227000021_add_cached_dates_to_agent.rb b/db/migrate/20131227000021_add_cached_dates_to_agent.rb
new file mode 100644
index 00000000..6c2b6f62
--- /dev/null
+++ b/db/migrate/20131227000021_add_cached_dates_to_agent.rb
@@ -0,0 +1,14 @@
+class AddCachedDatesToAgent < ActiveRecord::Migration
+  def up
+    add_column :agents, :last_event_at, :datetime
+    execute "UPDATE agents SET last_event_at = (SELECT created_at FROM events WHERE events.agent_id = agents.id ORDER BY id DESC LIMIT 1)"
+
+    add_column :agents, :last_error_log_at, :datetime
+    execute "UPDATE agents SET last_error_log_at = (SELECT created_at FROM agent_logs WHERE agent_logs.agent_id = agents.id AND agent_logs.level >= 4 ORDER BY id DESC LIMIT 1)"
+  end
+
+  def down
+    remove_column :agents, :last_event_at
+    remove_column :agents, :last_error_log_at
+  end
+end
diff --git a/spec/controllers/logs_controller_spec.rb b/spec/controllers/logs_controller_spec.rb
index 849ce628..117bd312 100644
--- a/spec/controllers/logs_controller_spec.rb
+++ b/spec/controllers/logs_controller_spec.rb
@@ -19,12 +19,14 @@ describe LogsController do
 
   describe "DELETE clear" do
     it "deletes all logs for a specific Agent" do
+      agents(:bob_weather_agent).last_error_log_at = 2.hours.ago
       sign_in users(:bob)
       lambda {
         delete :clear, :agent_id => agents(:bob_weather_agent).id
       }.should change { AgentLog.count }.by(-1 * agents(:bob_weather_agent).logs.count)
       assigns(:logs).length.should == 0
-      agents(:bob_weather_agent).logs.count.should == 0
+      agents(:bob_weather_agent).reload.logs.count.should == 0
+      agents(:bob_weather_agent).last_error_log_at.should be_nil
     end
 
     it "only deletes logs for an Agent owned by the current user" do
diff --git a/spec/models/agent_log_spec.rb b/spec/models/agent_log_spec.rb
index d6b6057f..939d580c 100644
--- a/spec/models/agent_log_spec.rb
+++ b/spec/models/agent_log_spec.rb
@@ -67,6 +67,14 @@ describe AgentLog do
       agents(:jane_website_agent).logs.order("agent_logs.id desc").first.message.should == "message 6"
       agents(:jane_website_agent).logs.order("agent_logs.id desc").last.message.should == "message 3"
     end
+
+    it "updates Agents' last_error_log_at when an error is logged" do
+      AgentLog.log_for_agent(agents(:jane_website_agent), "some message", :level => 3, :outbound_event => events(:jane_website_agent_event))
+      agents(:jane_website_agent).reload.last_error_log_at.should be_nil
+
+      AgentLog.log_for_agent(agents(:jane_website_agent), "some message", :level => 4, :outbound_event => events(:jane_website_agent_event))
+      agents(:jane_website_agent).reload.last_error_log_at.to_i.should be_within(2).of(Time.now.to_i)
+    end
   end
 
   describe "#log_length" do
diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb
index 6e9e9efc..05d71e98 100644
--- a/spec/models/agent_spec.rb
+++ b/spec/models/agent_spec.rb
@@ -279,6 +279,32 @@ describe Agent do
     end
   end
 
+  describe "recent_error_logs?" do
+    it "returns true if last_error_log_at is near last_event_at" do
+      agent = Agent.new
+
+      agent.last_error_log_at = 10.minutes.ago
+      agent.last_event_at = 10.minutes.ago
+      agent.recent_error_logs?.should be_true
+
+      agent.last_error_log_at = 11.minutes.ago
+      agent.last_event_at = 10.minutes.ago
+      agent.recent_error_logs?.should be_true
+
+      agent.last_error_log_at = 5.minutes.ago
+      agent.last_event_at = 10.minutes.ago
+      agent.recent_error_logs?.should be_true
+
+      agent.last_error_log_at = 15.minutes.ago
+      agent.last_event_at = 10.minutes.ago
+      agent.recent_error_logs?.should be_false
+
+      agent.last_error_log_at = 2.days.ago
+      agent.last_event_at = 10.minutes.ago
+      agent.recent_error_logs?.should be_false
+    end
+  end
+
   describe "scopes" do
     describe "of_type" do
       it "should accept classes" do
diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb
index 3c29beb2..42fd26bf 100644
--- a/spec/models/agents/website_agent_spec.rb
+++ b/spec/models/agents/website_agent_spec.rb
@@ -44,18 +44,22 @@ describe Agents::WebsiteAgent do
 
   describe '#working?' do
     it 'checks if events have been received within the expected receive period' do
+      stubbed_time = Time.now
+      stub(Time).now { stubbed_time }
+
       @checker.should_not be_working # No events created
       @checker.check
       @checker.reload.should be_working # Just created events
 
       @checker.error "oh no!"
-      @checker.reload.should_not be_working # The most recent log is an error
+      @checker.reload.should_not be_working # There is a recent error
 
-      @checker.log "ok now"
-      @checker.reload.should be_working # The most recent log is no longer an error
+      stubbed_time = 20.minutes.from_now
+      @checker.events.delete_all
+      @checker.check
+      @checker.reload.should be_working # There is a newer event now
 
-      two_days_from_now = 2.days.from_now
-      stub(Time).now { two_days_from_now }
+      stubbed_time = 2.days.from_now
       @checker.reload.should_not be_working # Two days have passed without a new event having been created
     end
   end

From 6a276e0d76b1fadd9a08d53a8c9374b3a4ca3542 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Sun, 29 Dec 2013 22:28:10 -0800
Subject: [PATCH 08/10] add new JSONSerializedField Concern and better specs
 around handling of json string assignment to memory and options

---
 app/models/agent.rb                 | 14 +++-------
 app/models/event.rb                 | 10 +++----
 lib/json_serialized_field.rb        | 42 +++++++++++++++++++++++++++++
 lib/json_with_indifferent_access.rb |  2 +-
 spec/models/agent_spec.rb           | 42 +++++++++++++++++++++++++++++
 5 files changed, 92 insertions(+), 18 deletions(-)
 create mode 100644 lib/json_serialized_field.rb

diff --git a/app/models/agent.rb b/app/models/agent.rb
index e472debc..216d0fca 100644
--- a/app/models/agent.rb
+++ b/app/models/agent.rb
@@ -1,4 +1,4 @@
-require 'json_with_indifferent_access'
+require 'json_serialized_field'
 require 'assignable_types'
 require 'markdown_class_attributes'
 require 'utils'
@@ -6,6 +6,7 @@ require 'utils'
 class Agent < ActiveRecord::Base
   include AssignableTypes
   include MarkdownClassAttributes
+  include JSONSerializedField
 
   markdown_class_attributes :description, :event_description
 
@@ -16,8 +17,7 @@ class Agent < ActiveRecord::Base
 
   attr_accessible :options, :memory, :name, :type, :schedule, :source_ids
 
-  serialize :options, JSONWithIndifferentAccess
-  serialize :memory, JSONWithIndifferentAccess
+  json_serialize :options, :memory
 
   validates_presence_of :name, :user
   validate :sources_are_owned
@@ -79,14 +79,6 @@ class Agent < ActiveRecord::Base
     # Implement me in your subclass to test for valid options.
   end
 
-  def options=(o)
-    self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
-  def memory=(o)
-    self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
   def event_created_within?(days)
     last_event_at && last_event_at > days.to_i.days.ago
   end
diff --git a/app/models/event.rb b/app/models/event.rb
index adb77577..f9a7baad 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -1,11 +1,13 @@
-require 'json_with_indifferent_access'
+require 'json_serialized_field'
 
 class Event < ActiveRecord::Base
+  include JSONSerializedField
+
   attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at
 
   acts_as_mappable
 
-  serialize :payload, JSONWithIndifferentAccess
+  json_serialize :payload
 
   belongs_to :user
   belongs_to :agent, :counter_cache => true, :touch => :last_event_at
@@ -14,10 +16,6 @@ class Event < ActiveRecord::Base
     where("events.created_at > ?", timespan)
   }
 
-  def payload=(o)
-    self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o)
-  end
-
   def reemit!
     agent.create_event :payload => payload, :lat => lat, :lng => lng
   end
diff --git a/lib/json_serialized_field.rb b/lib/json_serialized_field.rb
new file mode 100644
index 00000000..d9dcfc3a
--- /dev/null
+++ b/lib/json_serialized_field.rb
@@ -0,0 +1,42 @@
+require 'json_with_indifferent_access'
+
+module JSONSerializedField
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def json_serialize(*fields)
+      fields.each do |field|
+        class_eval <<-CODE
+          serialize :#{field}, JSONWithIndifferentAccess
+
+          validate :#{field}_has_no_errors
+
+          def #{field}=(input)
+            @#{field}_assignment_error = false
+            case input
+              when String
+                if input.strip.length == 0
+                  self[:#{field}] = ActiveSupport::HashWithIndifferentAccess.new
+                else
+                  json = JSON.parse(input) rescue nil
+                  if json
+                    self[:#{field}] = ActiveSupport::HashWithIndifferentAccess.new(json)
+                  else
+                    @#{field}_assignment_error = "was assigned invalid JSON"
+                  end
+                end
+              when Hash
+                self[:#{field}] = ActiveSupport::HashWithIndifferentAccess.new(input)
+              else
+                @#{field}_assignment_error = "cannot be set to an instance of \#{input.class}"
+            end
+          end
+
+          def #{field}_has_no_errors
+            errors.add(:#{field}, @#{field}_assignment_error) if @#{field}_assignment_error
+          end
+        CODE
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/json_with_indifferent_access.rb b/lib/json_with_indifferent_access.rb
index ee02daf2..67387282 100644
--- a/lib/json_with_indifferent_access.rb
+++ b/lib/json_with_indifferent_access.rb
@@ -1,6 +1,6 @@
 class JSONWithIndifferentAccess
   def self.load(json)
-    ActiveSupport::HashWithIndifferentAccess.new(JSON.load(json || '{}'))
+    ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(json || '{}'))
   end
 
   def self.dump(hash)
diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb
index 05d71e98..fe0cd36b 100644
--- a/spec/models/agent_spec.rb
+++ b/spec/models/agent_spec.rb
@@ -266,6 +266,48 @@ describe Agent do
         agent.memory[:bad].should == 2
       end
 
+      it "should work when assigned a hash or JSON string" do
+        agent = Agents::SomethingSource.new(:name => "something")
+        agent.memory = {}
+        agent.memory.should == {}
+        agent.memory["foo"].should be_nil
+
+        agent.memory = ""
+        agent.memory["foo"].should be_nil
+        agent.memory.should == {}
+
+        agent.memory = '{"hi": "there"}'
+        agent.memory.should == { "hi" => "there" }
+
+        agent.memory = '{invalid}'
+        agent.memory.should == { "hi" => "there" }
+        agent.should have(1).errors_on(:memory)
+
+        agent.memory = "{}"
+        agent.memory["foo"].should be_nil
+        agent.memory.should == {}
+        agent.should have(0).errors_on(:memory)
+
+        agent.options = "{}"
+        agent.options["foo"].should be_nil
+        agent.options.should == {}
+        agent.should have(0).errors_on(:options)
+
+        agent.options = '{"hi": 2}'
+        agent.options["hi"].should == 2
+        agent.should have(0).errors_on(:options)
+
+        agent.options = '{"hi": wut}'
+        agent.options["hi"].should == 2
+        agent.should have(1).errors_on(:options)
+        agent.errors_on(:options).should include("was assigned invalid JSON")
+
+        agent.options = 5
+        agent.options["hi"].should == 2
+        agent.should have(1).errors_on(:options)
+        agent.errors_on(:options).should include("cannot be set to an instance of Fixnum")
+      end
+
       it "should not allow agents owned by other people" do
         agent = Agents::SomethingSource.new(:name => "something")
         agent.user = users(:bob)

From de83040dee356c6743b400c17bc9aca84270e531 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Wed, 1 Jan 2014 13:16:38 -0800
Subject: [PATCH 09/10] fix migration

---
 db/migrate/20131223032112_switch_to_json_serialization.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db/migrate/20131223032112_switch_to_json_serialization.rb b/db/migrate/20131223032112_switch_to_json_serialization.rb
index 13c46ac6..87675258 100644
--- a/db/migrate/20131223032112_switch_to_json_serialization.rb
+++ b/db/migrate/20131223032112_switch_to_json_serialization.rb
@@ -25,7 +25,7 @@ class SwitchToJsonSerialization < ActiveRecord::Migration
 
   def set_to_utf8
     if mysql?
-      %w[agent_logs agents delayed_jobs events links taggings tags users].each do |table_name|
+      %w[agent_logs agents delayed_jobs events links users].each do |table_name|
         quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table_name)
         execute "ALTER TABLE #{quoted_table_name} CONVERT TO CHARACTER SET utf8"
       end

From 48a024845a7207032431321356921d5e36d40b30 Mon Sep 17 00:00:00 2001
From: Andrew Cantino <cantino@gmail.com>
Date: Wed, 1 Jan 2014 13:39:08 -0800
Subject: [PATCH 10/10] update VERSION, README, and CHANGES

---
 CHANGES.md | 3 ++-
 README.md  | 6 ++++--
 VERSION    | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 71b0c61b..6c63da61 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,8 +1,9 @@
 # Changes
 
+* 0.3 (Jan 1, 2014)    - Remove symbolization of memory, options, and payloads; convert memory, options, and payloads to JSON from YAML.  Migration will perform conversion and adjust tables to be UTF-8.  Recommend making a DB backup before migrating.
 * 0.2 (Nov 6, 2013)    - PeakDetectorAgent now uses `window_duration_in_days` and `min_peak_spacing_in_days`.  Additionally, peaks trigger when the time series rises over the standard deviation multiple, not after it starts to fall.
 * June 29, 2013        - Removed rails\_admin because it was causing deployment issues. Better to have people install their favorite admin tool if they want one.
 * June, 2013           - A number of new agents have been contributed, including interfaces to Weibo, Twitter, and Twilio, as well as Agents for translation, sentiment analysis, and for posting and receiving webhooks.
-* March 24, 2013 (0.1) - Refactored loading of Agents for `check` and `receive` to use ids instead of full objects.  This should fix the too-large delayed_job issues.  Added `system_timer` and `fastercsv` to the Gemfile for the Ruby 1.8 platform.
+* March 24, 2013 (0.1) - Refactored loading of Agents for `check` and `receive` to use ids instead of full objects.  This should fix the too-large delayed\_job issues.  Added `system_timer` and `fastercsv` to the Gemfile for the Ruby 1.8 platform.
 * March 18, 2013       - Added Wiki page about the [Agent API](https://github.com/cantino/huginn/wiki/Creating-a-new-agent).
 * March 17, 2013       - Switched to JSONPath for defining paths through JSON structures.  The WebsiteAgent can now scrape and parse JSON.
diff --git a/README.md b/README.md
index ea10af61..6c9fbc25 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,10 @@ Huginn is a system for building agents that perform automated tasks for you onli
 * Track the weather and get an email when it's going to rain (or snow) tomorrow
 * Follow your project names on Twitter and get updates when people mention them
 * Scrape websites and receive emails when they change
+* Compose digest emails about things you care about to be sent at specific times of the day
+* Track counts of high frequency events and SMS on changes, such as the term "san francisco emergency"
 * Track your location over time
+* Create Amazon Mechanical Turk tasks as the input, or output, of events
 
 Follow [@tectonic](https://twitter.com/tectonic) for updates as Huginn evolves, and join us in \#huginn on Freenode IRC to discuss the project.
 
@@ -84,7 +87,7 @@ In order to use the WeatherAgent you need an [API key with Wunderground](http://
 
 You can use [Post Location](https://github.com/cantino/post_location) on your iPhone to post your location to an instance of the UserLocationAgent.  Make a new one to see instructions.
 
-#### Enable DelayedJobWeb for handy delayed_job monitoring and control
+#### Enable DelayedJobWeb for handy delayed\_job monitoring and control
 
 * Edit `config.ru`, uncomment the DelayedJobWeb section, and change the DelayedJobWeb username and password.
 * Uncomment `match "/delayed_job" => DelayedJobWeb, :anchor => false` in `config/routes.rb`.
@@ -110,6 +113,5 @@ Please fork, add specs, and send pull requests!
 
 [![Build Status](https://travis-ci.org/cantino/huginn.png)](https://travis-ci.org/cantino/huginn) [![Code Climate](https://codeclimate.com/github/cantino/huginn.png)](https://codeclimate.com/github/cantino/huginn)
 
-
 [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/cantino/huginn/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
 
diff --git a/VERSION b/VERSION
index 3b04cfb6..be586341 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.2
+0.3