From 77882908c4b1b0fc0a3dca1773d8471f156864d4 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 22 Dec 2013 15:36:29 -0800 Subject: [PATCH 01/10] in order to avoid leaking memory, remove all symbolization --- app/concerns/email_concern.rb | 2 +- app/models/agent.rb | 6 +-- app/models/agents/adioso_agent.rb | 2 +- app/models/agents/human_task_agent.rb | 28 +++++------ app/models/agents/peak_detector_agent.rb | 2 +- app/models/agents/twilio_agent.rb | 6 +-- app/models/agents/twitter_stream_agent.rb | 10 ++-- app/models/event.rb | 12 ++--- .../recursively_symbolize_keys.rb | 7 --- lib/serialize_and_normalize.rb | 46 +++++++++++++++++++ lib/serialize_and_symbolize.rb | 43 ----------------- lib/utils.rb | 11 ----- spec/controllers/agents_controller_spec.rb | 2 +- .../user_location_updates_controller_spec.rb | 6 +-- spec/controllers/webhooks_controller_spec.rb | 4 +- spec/models/agents/human_task_agent_spec.rb | 38 +++++++-------- spec/models/agents/sentiment_agent_spec.rb | 2 +- .../agents/twitter_stream_agent_spec.rb | 14 +++--- 18 files changed, 112 insertions(+), 129 deletions(-) delete mode 100644 config/initializers/recursively_symbolize_keys.rb create mode 100644 lib/serialize_and_normalize.rb delete mode 100644 lib/serialize_and_symbolize.rb diff --git a/app/concerns/email_concern.rb b/app/concerns/email_concern.rb index 9a311b80..455acdcb 100644 --- a/app/concerns/email_concern.rb +++ b/app/concerns/email_concern.rb @@ -1,7 +1,7 @@ module EmailConcern extend ActiveSupport::Concern - MAIN_KEYS = %w[title message text main value].map(&:to_sym) + MAIN_KEYS = %w[title message text main value] included do self.validate :validate_email_options diff --git a/app/models/agent.rb b/app/models/agent.rb index 8ee4f9cf..ad4d04fb 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -1,14 +1,14 @@ -require 'serialize_and_symbolize' +require 'serialize_and_normalize' require 'assignable_types' require 'markdown_class_attributes' require 'utils' class Agent < ActiveRecord::Base - include SerializeAndSymbolize + include SerializeAndNormalize include AssignableTypes include MarkdownClassAttributes - serialize_and_symbolize :options, :memory + serialize_and_normalize :options, :memory markdown_class_attributes :description, :event_description load_types_in "Agents" diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb index 2b08feec..3055d2f5 100644 --- a/app/models/agents/adioso_agent.rb +++ b/app/models/agents/adioso_agent.rb @@ -44,7 +44,7 @@ module Agents end def validate_options - unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field.to_sym].present? } + unless %w[start_date end_date from to username password expected_update_period_in_days].all? { |field| options[field].present? } errors.add(:base, "All fields are required") end end diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb index 65a73c5f..116347e6 100644 --- a/app/models/agents/human_task_agent.rb +++ b/app/models/agents/human_task_agent.rb @@ -214,12 +214,12 @@ module Agents end event = create_event :payload => payload - log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym]) + log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id]) assignments.each(&:approve!) hit.dispose! - memory[:hits].delete(hit_id.to_sym) + memory[:hits].delete(hit_id) end end end @@ -268,34 +268,34 @@ module Agents @questions.each.with_index do |question, index| Question do QuestionIdentifier do - text question[:key] || "question_#{index}" + text question['key'] || "question_#{index}" end DisplayName do - text question[:name] || "Question ##{index}" + text question['name'] || "Question ##{index}" end IsRequired do - text question[:required] || 'true' + text question['required'] || 'true' end QuestionContent do Text do - text question[:question] + text question['question'] end end AnswerSpecification do - if question[:type] == "selection" + if question['type'] == "selection" SelectionAnswer do StyleSuggestion do text 'radiobutton' end Selections do - question[:selections].each do |selection| + question['selections'].each do |selection| Selection do SelectionIdentifier do - text selection[:key] + text selection['key'] end Text do - text selection[:text] + text selection['text'] end end end @@ -305,18 +305,18 @@ module Agents else FreeTextAnswer do - if question[:min_length].present? || question[:max_length].present? + if question['min_length'].present? || question['max_length'].present? Constraints do lengths = {} - lengths[:minLength] = question[:min_length].to_s if question[:min_length].present? - lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present? + lengths['minLength'] = question['min_length'].to_s if question['min_length'].present? + lengths['maxLength'] = question['max_length'].to_s if question['max_length'].present? Length lengths end end if question[:default].present? DefaultText do - text question[:default] + text question['default'] end end end diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb index a7f3c4e3..7c7e7468 100644 --- a/app/models/agents/peak_detector_agent.rb +++ b/app/models/agents/peak_detector_agent.rb @@ -114,7 +114,7 @@ module Agents end def group_for(event) - ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group').to_sym + ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group') end def remember(group, event) diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb index f22ef31b..82076e89 100644 --- a/app/models/agents/twilio_agent.rb +++ b/app/models/agents/twilio_agent.rb @@ -79,9 +79,9 @@ module Agents end def receive_webhook(params) - if memory[:pending_calls].has_key? params[:secret].to_sym - response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret].to_sym], :voice => 'woman'} - memory[:pending_calls].delete params[:secret].to_sym + if memory[:pending_calls].has_key? params[:secret] + response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'} + memory[:pending_calls].delete params[:secret] [response.text, 200] end end diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb index d21bcf75..96afae25 100644 --- a/app/models/agents/twitter_stream_agent.rb +++ b/app/models/agents/twitter_stream_agent.rb @@ -85,12 +85,12 @@ module Agents # Avoid memory pollution by reloading the Agent. agent = Agent.find(id) agent.memory[:filter_counts] ||= {} - agent.memory[:filter_counts][filter.to_sym] ||= 0 - agent.memory[:filter_counts][filter.to_sym] += 1 + agent.memory[:filter_counts][filter] ||= 0 + agent.memory[:filter_counts][filter] += 1 remove_unused_keys!(agent, :filter_counts) agent.save! else - create_event :payload => status.merge(:filter => filter.to_s) + create_event :payload => status.merge(:filter => filter) end end end @@ -98,7 +98,7 @@ module Agents def check if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0 memory[:filter_counts].each do |filter, count| - create_event :payload => { :filter => filter.to_s, :count => count, :time => Time.now.to_i } + create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i } end end memory[:filter_counts] = {} @@ -120,7 +120,7 @@ module Agents def remove_unused_keys!(agent, base) if agent.memory[base] - (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_sym : f.to_sym }).each do |removed_key| + (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key| agent.memory[base].delete(removed_key) end end diff --git a/app/models/event.rb b/app/models/event.rb index 4aaddf1c..f7c99256 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,23 +1,21 @@ +require 'serialize_and_normalize' + class Event < ActiveRecord::Base + include SerializeAndNormalize + attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at acts_as_mappable - serialize :payload + serialize_and_normalize :payload belongs_to :user belongs_to :agent, :counter_cache => true - before_save :symbolize_payload - scope :recent, lambda { |timespan = 12.hours.ago| where("events.created_at > ?", timespan) } - def symbolize_payload - self.payload = payload.recursively_symbolize_keys if payload.is_a?(Hash) - end - def reemit! agent.create_event :payload => payload, :lat => lat, :lng => lng end diff --git a/config/initializers/recursively_symbolize_keys.rb b/config/initializers/recursively_symbolize_keys.rb deleted file mode 100644 index 939e55a1..00000000 --- a/config/initializers/recursively_symbolize_keys.rb +++ /dev/null @@ -1,7 +0,0 @@ -require 'utils' - -class Hash - def recursively_symbolize_keys - Utils.recursively_symbolize_keys self - end -end diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb new file mode 100644 index 00000000..842c95dc --- /dev/null +++ b/lib/serialize_and_normalize.rb @@ -0,0 +1,46 @@ +module SerializeAndNormalize + extend ActiveSupport::Concern + + module ClassMethods + def serialize_and_normalize(*column_names) + column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name| + setup_name = "setup_#{column_name}".to_sym + normalize_name = "normalize_#{column_name}".to_sym + validate_name = "validate_#{column_name}".to_sym + + serialize column_name + after_initialize setup_name + before_validation normalize_name + before_save normalize_name + validate validate_name + + class_eval <<-RUBY + def #{setup_name} + self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new + end + + def #{validate_name} + # Implement me in your subclass. + end + + def #{normalize_name} + self.#{column_name} = self[:#{column_name}] + end + + def #{column_name}=(data) + data = (JSON.parse(data) rescue data) if data.is_a?(String) + + case data + when ActiveSupport::HashWithIndifferentAccess + self[:#{column_name}] = data + when Hash + self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data) + else + self[:#{column_name}] = data + end + end + RUBY + end + end + end +end diff --git a/lib/serialize_and_symbolize.rb b/lib/serialize_and_symbolize.rb deleted file mode 100644 index f051abb5..00000000 --- a/lib/serialize_and_symbolize.rb +++ /dev/null @@ -1,43 +0,0 @@ -module SerializeAndSymbolize - extend ActiveSupport::Concern - - module ClassMethods - def serialize_and_symbolize(*column_names) - column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name| - setup_name = "setup_#{column_name}".to_sym - symbolize_name = "symbolize_#{column_name}".to_sym - validate_name = "validate_#{column_name}".to_sym - - serialize column_name - after_initialize setup_name - before_validation symbolize_name - before_save symbolize_name - validate validate_name - - class_eval <<-RUBY - def #{setup_name} - self[:#{column_name}] ||= {} - end - - def #{validate_name} - # Implement me in your subclass. - end - - def #{symbolize_name} - self.#{column_name} = self[:#{column_name}] - end - - def #{column_name}=(data) - if data.is_a?(String) - self[:#{column_name}] = JSON.parse(data).recursively_symbolize_keys rescue {} - elsif data.is_a?(Hash) - self[:#{column_name}] = data.recursively_symbolize_keys - else - self[:#{column_name}] = data - end - end - RUBY - end - end - end -end diff --git a/lib/utils.rb b/lib/utils.rb index fae7577e..0d031ae2 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -21,17 +21,6 @@ module Utils end end - def self.recursively_symbolize_keys(object) - case object - when Hash - object.inject({}) {|memo, (k, v)| memo[String === k ? k.to_sym : k] = recursively_symbolize_keys(v); memo } - when Array - object.map { |item| recursively_symbolize_keys item } - else - object - end - end - def self.interpolate_jsonpaths(value, data) value.gsub(/<[^>]+>/).each { |jsonpath| Utils.values_at(data, jsonpath[1..-2]).first.to_s diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb index e36efa6b..4b5c164a 100644 --- a/spec/controllers/agents_controller_spec.rb +++ b/spec/controllers/agents_controller_spec.rb @@ -23,7 +23,7 @@ describe AgentsController do sign_in users(:bob) post :handle_details_post, :id => agents(:bob_manual_event_agent).to_param, :payload => { :foo => "bar" } JSON.parse(response.body).should == { "success" => true } - agents(:bob_manual_event_agent).events.last.payload.should == { :foo => "bar" } + agents(:bob_manual_event_agent).events.last.payload.should == { 'foo' => "bar" } end it "can only be accessed by the Agent's owner" do diff --git a/spec/controllers/user_location_updates_controller_spec.rb b/spec/controllers/user_location_updates_controller_spec.rb index 79c211a4..08ca9192 100644 --- a/spec/controllers/user_location_updates_controller_spec.rb +++ b/spec/controllers/user_location_updates_controller_spec.rb @@ -8,7 +8,7 @@ describe UserLocationUpdatesController do it "should create events without requiring login" do post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else" - @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" } + @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" } @agent.events.last.lat.should == 45 @agent.events.last.lng.should == 123 end @@ -18,7 +18,7 @@ describe UserLocationUpdatesController do @jane_agent.save! post :create, :user_id => users(:bob).to_param, :secret => "my_secret", :longitude => 123, :latitude => 45, :something => "else" - @agent.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" } + @agent.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" } @jane_agent.events.should be_empty end @@ -33,7 +33,7 @@ describe UserLocationUpdatesController do lambda { post :create, :user_id => users(:bob).to_param, :secret => "my_secret2", :longitude => 123, :latitude => 45, :something => "else" - @agent2.events.last.payload.should == { :longitude => "123", :latitude => "45", :something => "else" } + @agent2.events.last.payload.should == { 'longitude' => "123", 'latitude' => "45", 'something' => "else" } }.should_not change { @agent.events.count } end end \ No newline at end of file diff --git a/spec/controllers/webhooks_controller_spec.rb b/spec/controllers/webhooks_controller_spec.rb index 8a42f8ec..754d8e7d 100644 --- a/spec/controllers/webhooks_controller_spec.rb +++ b/spec/controllers/webhooks_controller_spec.rb @@ -32,12 +32,12 @@ describe WebhooksController do it "should call receive_webhook" do post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "my_secret", :key => "value", :another_key => "5" - @agent.reload.memory[:webhook_values].should == { :key => "value", :another_key => "5" } + @agent.reload.memory[:webhook_values].should == { 'key' => "value", 'another_key' => "5" } response.body.should == "success" response.should be_success post :create, :user_id => users(:bob).to_param, :agent_id => @agent.id, :secret => "not_my_secret", :no => "go" - @agent.reload.memory[:webhook_values].should_not == { :no => "go" } + @agent.reload.memory[:webhook_values].should_not == { 'no' => "go" } response.body.should == "failure" response.should be_missing end diff --git a/spec/models/agents/human_task_agent_spec.rb b/spec/models/agents/human_task_agent_spec.rb index 38cb6932..054ea4cf 100644 --- a/spec/models/agents/human_task_agent_spec.rb +++ b/spec/models/agents/human_task_agent_spec.rb @@ -288,7 +288,7 @@ describe Agents::HumanTaskAgent do @checker.send :review_hits assignments.all? {|a| a.approved == true }.should be_false - @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id } + @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id } end it "shouldn't do anything if an assignment is missing" do @@ -306,7 +306,7 @@ describe Agents::HumanTaskAgent do @checker.send :review_hits assignments.all? {|a| a.approved == true }.should be_false - @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id } + @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id } end it "should create events when all assignments are ready" do @@ -328,8 +328,8 @@ describe Agents::HumanTaskAgent do hit.should be_disposed @checker.events.last.payload[:answers].should == [ - {:sentiment => "neutral", :feedback => ""}, - {:sentiment => "happy", :feedback => "Take 2"} + {'sentiment' => "neutral", 'feedback' => ""}, + {'sentiment' => "happy", 'feedback' => "Take 2"} ] @checker.memory[:hits].should == {} @@ -338,7 +338,7 @@ describe Agents::HumanTaskAgent do describe "taking majority votes" do before do @checker.options[:take_majority] = "true" - @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + @checker.memory[:hits] = { "JH3132836336DHG" => @event.id } mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } end @@ -372,14 +372,14 @@ describe Agents::HumanTaskAgent do assignments.all? {|a| a.approved == true }.should be_true @checker.events.last.payload[:answers].should == [ - { :sentiment => "sad", :age_range => "<50" }, - { :sentiment => "neutral", :age_range => ">50" }, - { :sentiment => "happy", :age_range => ">50" }, - { :sentiment => "happy", :age_range => ">50" } + { 'sentiment' => "sad", 'age_range' => "<50" }, + { 'sentiment' => "neutral", 'age_range' => ">50" }, + { 'sentiment' => "happy", 'age_range' => ">50" }, + { 'sentiment' => "happy", 'age_range' => ">50" } ] - @checker.events.last.payload[:counts].should == { :sentiment => { :happy => 2, :sad => 1, :neutral => 1 }, :age_range => { :">50" => 3, :"<50" => 1 } } - @checker.events.last.payload[:majority_answer].should == { :sentiment => "happy", :age_range => ">50" } + @checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } } + @checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" } @checker.events.last.payload.should_not have_key(:average_answer) @checker.memory[:hits].should == {} @@ -421,16 +421,16 @@ describe Agents::HumanTaskAgent do assignments.all? {|a| a.approved == true }.should be_true @checker.events.last.payload[:answers].should == [ - { :rating => "1" }, - { :rating => "3" }, - { :rating => "5.1" }, - { :rating => "2" }, - { :rating => "2" } + { 'rating' => "1" }, + { 'rating' => "3" }, + { 'rating' => "5.1" }, + { 'rating' => "2" }, + { 'rating' => "2" } ] - @checker.events.last.payload[:counts].should == { :rating => { :"1" => 1, :"2" => 2, :"3" => 1, :"4" => 0, :"5.1" => 1 } } - @checker.events.last.payload[:majority_answer].should == { :rating => "2" } - @checker.events.last.payload[:average_answer].should == { :rating => (1 + 2 + 2 + 3 + 5.1) / 5.0 } + @checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } } + @checker.events.last.payload[:majority_answer].should == { 'rating' => "2" } + @checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 } @checker.memory[:hits].should == {} end diff --git a/spec/models/agents/sentiment_agent_spec.rb b/spec/models/agents/sentiment_agent_spec.rb index 142abad8..f4afbf52 100644 --- a/spec/models/agents/sentiment_agent_spec.rb +++ b/spec/models/agents/sentiment_agent_spec.rb @@ -53,7 +53,7 @@ describe Agents::SentimentAgent do it "checks if content key is working fine" do @checker.receive([@event]) Event.last.payload[:content].should == "value1" - Event.last.payload[:original_event].should == {:message => "value1"} + Event.last.payload[:original_event].should == { 'message' => "value1" } end it "should handle multiple events" do event1 = Event.new diff --git a/spec/models/agents/twitter_stream_agent_spec.rb b/spec/models/agents/twitter_stream_agent_spec.rb index 353a0258..816decb9 100644 --- a/spec/models/agents/twitter_stream_agent_spec.rb +++ b/spec/models/agents/twitter_stream_agent_spec.rb @@ -53,7 +53,7 @@ describe Agents::TwitterStreamAgent do @agent.memory[:filter_counts] = {:keyword1 => 2, :keyword2 => 3, :keyword3 => 4} @agent.save! @agent.process_tweet('keyword1', {:text => "something", :user => {:name => "Mr. Someone"}}) - @agent.reload.memory[:filter_counts].should == {:keyword1 => 3, :keyword2 => 3} + @agent.reload.memory[:filter_counts].should == { 'keyword1' => 3, 'keyword2' => 3 } end end @@ -64,9 +64,9 @@ describe Agents::TwitterStreamAgent do }.should change { @agent.events.count }.by(1) @agent.events.last.payload.should == { - :filter => 'keyword1', - :text => "something", - :user => {:name => "Mr. Someone"} + 'filter' => 'keyword1', + 'text' => "something", + 'user' => { 'name' => "Mr. Someone" } } end @@ -79,9 +79,9 @@ describe Agents::TwitterStreamAgent do }.should change { @agent.events.count }.by(1) @agent.events.last.payload.should == { - :filter => 'keyword1-1', - :text => "something", - :user => {:name => "Mr. Someone"} + 'filter' => 'keyword1-1', + 'text' => "something", + 'user' => { 'name' => "Mr. Someone" } } end end From ec32d7f9794209802fe650f83ba4cf6426fe0279 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 23 Dec 2013 00:54:20 -0500 Subject: [PATCH 02/10] switch from yaml to json serialization --- ...1223032112_switch_to_json_serialization.rb | 41 +++++++++++++++++++ lib/serialize_and_normalize.rb | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20131223032112_switch_to_json_serialization.rb diff --git a/db/migrate/20131223032112_switch_to_json_serialization.rb b/db/migrate/20131223032112_switch_to_json_serialization.rb new file mode 100644 index 00000000..c132d532 --- /dev/null +++ b/db/migrate/20131223032112_switch_to_json_serialization.rb @@ -0,0 +1,41 @@ +class SwitchToJsonSerialization < ActiveRecord::Migration + FIELDS = { + :agents => [:options, :memory], + :events => [:payload] + } + + def up + puts "This migration will update Agent and Event storage from YAML to JSON. It should work, but please make a backup" + puts "before proceeding." + print "Continue? (y/n) " + STDOUT.flush + exit unless STDIN.gets =~ /^y/i + + translate YAML, JSON + end + + def down + translate JSON, YAML + end + + def translate(from, to) + FIELDS.each do |table, fields| + quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table) + fields = fields.map { |f| ActiveRecord::Base.connection.quote_column_name(f) } + + rows = ActiveRecord::Base.connection.select_rows("SELECT id, #{fields.join(", ")} FROM #{quoted_table_name}") + rows.each do |row| + id, *field_data = row + + yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) } + + update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?" + + sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id] + + ActiveRecord::Base.connection.execute sanitized_update_sql + end + end + + end +end diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb index 842c95dc..857a777f 100644 --- a/lib/serialize_and_normalize.rb +++ b/lib/serialize_and_normalize.rb @@ -8,7 +8,7 @@ module SerializeAndNormalize normalize_name = "normalize_#{column_name}".to_sym validate_name = "validate_#{column_name}".to_sym - serialize column_name + serialize column_name, JSON after_initialize setup_name before_validation normalize_name before_save normalize_name From a408ae48eae8807246658582bb57f51c5871af2e Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Tue, 24 Dec 2013 17:37:26 -0500 Subject: [PATCH 03/10] globally avoid using symbols since we're moving to json storage --- app/concerns/email_concern.rb | 4 +- app/concerns/twitter_concern.rb | 16 +- app/concerns/weibo_concern.rb | 12 +- app/models/agent.rb | 20 +- app/models/agents/adioso_agent.rb | 24 +-- app/models/agents/digest_email_agent.rb | 28 +-- app/models/agents/email_agent.rb | 8 +- app/models/agents/event_formatting_agent.rb | 44 ++-- app/models/agents/human_task_agent.rb | 136 ++++++------ app/models/agents/manual_event_agent.rb | 4 +- app/models/agents/peak_detector_agent.rb | 54 ++--- app/models/agents/post_agent.rb | 8 +- app/models/agents/sentiment_agent.rb | 20 +- app/models/agents/translation_agent.rb | 30 +-- app/models/agents/trigger_agent.rb | 48 ++--- app/models/agents/twilio_agent.rb | 50 ++--- app/models/agents/twitter_publish_agent.rb | 40 ++-- app/models/agents/twitter_stream_agent.rb | 46 ++--- app/models/agents/twitter_user_agent.rb | 24 +-- app/models/agents/user_location_agent.rb | 4 +- app/models/agents/weather_agent.rb | 16 +- app/models/agents/website_agent.rb | 78 +++---- app/models/agents/weibo_publish_agent.rb | 38 ++-- app/models/agents/weibo_user_agent.rb | 22 +- app/models/event.rb | 11 +- lib/json_with_indifferent_access.rb | 9 + lib/serialize_and_normalize.rb | 46 ----- spec/controllers/agents_controller_spec.rb | 2 +- spec/fixtures/agents.yml | 14 +- spec/fixtures/events.yml | 4 +- spec/models/agent_spec.rb | 4 +- spec/models/agents/digest_email_agent_spec.rb | 10 +- spec/models/agents/email_agent_spec.rb | 8 +- spec/models/agents/human_task_agent_spec.rb | 194 +++++++++--------- .../models/agents/peak_detector_agent_spec.rb | 58 +++--- spec/models/agents/trigger_agent_spec.rb | 106 +++++----- 36 files changed, 610 insertions(+), 630 deletions(-) create mode 100644 lib/json_with_indifferent_access.rb delete mode 100644 lib/serialize_and_normalize.rb diff --git a/app/concerns/email_concern.rb b/app/concerns/email_concern.rb index 455acdcb..01cca474 100644 --- a/app/concerns/email_concern.rb +++ b/app/concerns/email_concern.rb @@ -8,11 +8,11 @@ module EmailConcern end def validate_email_options - errors.add(:base, "subject and expected_receive_period_in_days are required") unless options[:subject].present? && options[:expected_receive_period_in_days].present? + errors.add(:base, "subject and expected_receive_period_in_days are required") unless options['subject'].present? && options['expected_receive_period_in_days'].present? end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def present(payload) diff --git a/app/concerns/twitter_concern.rb b/app/concerns/twitter_concern.rb index 0a596af3..aedc351a 100644 --- a/app/concerns/twitter_concern.rb +++ b/app/concerns/twitter_concern.rb @@ -7,20 +7,20 @@ module TwitterConcern end def validate_twitter_options - unless options[:consumer_key].present? && - options[:consumer_secret].present? && - options[:oauth_token].present? && - options[:oauth_token_secret].present? + unless options['consumer_key'].present? && + options['consumer_secret'].present? && + options['oauth_token'].present? && + options['oauth_token_secret'].present? errors.add(:base, "consumer_key, consumer_secret, oauth_token and oauth_token_secret are required to authenticate with the Twitter API") end end def configure_twitter Twitter.configure do |config| - config.consumer_key = options[:consumer_key] - config.consumer_secret = options[:consumer_secret] - config.oauth_token = options[:oauth_token] || options[:access_key] - config.oauth_token_secret = options[:oauth_token_secret] || options[:access_secret] + config.consumer_key = options['consumer_key'] + config.consumer_secret = options['consumer_secret'] + config.oauth_token = options['oauth_token'] || options['access_key'] + config.oauth_token_secret = options['oauth_token_secret'] || options['access_secret'] end end diff --git a/app/concerns/weibo_concern.rb b/app/concerns/weibo_concern.rb index 029071e7..eea56237 100644 --- a/app/concerns/weibo_concern.rb +++ b/app/concerns/weibo_concern.rb @@ -6,19 +6,19 @@ module WeiboConcern end def validate_weibo_options - unless options[:app_key].present? && - options[:app_secret].present? && - options[:access_token].present? + unless options['app_key'].present? && + options['app_secret'].present? && + options['access_token'].present? errors.add(:base, "app_key, app_secret and access_token are required") end end def weibo_client unless @weibo_client - WeiboOAuth2::Config.api_key = options[:app_key] # WEIBO_APP_KEY - WeiboOAuth2::Config.api_secret = options[:app_secret] # WEIBO_APP_SECRET + WeiboOAuth2::Config.api_key = options['app_key'] # WEIBO_APP_KEY + WeiboOAuth2::Config.api_secret = options['app_secret'] # WEIBO_APP_SECRET @weibo_client = WeiboOAuth2::Client.new - @weibo_client.get_token_from_hash :access_token => options[:access_token] + @weibo_client.get_token_from_hash :access_token => options['access_token'] end @weibo_client end diff --git a/app/models/agent.rb b/app/models/agent.rb index ad4d04fb..9d6a9f34 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -1,14 +1,12 @@ -require 'serialize_and_normalize' +require 'json_with_indifferent_access' require 'assignable_types' require 'markdown_class_attributes' require 'utils' class Agent < ActiveRecord::Base - include SerializeAndNormalize include AssignableTypes include MarkdownClassAttributes - serialize_and_normalize :options, :memory markdown_class_attributes :description, :event_description load_types_in "Agents" @@ -18,9 +16,21 @@ class Agent < ActiveRecord::Base attr_accessible :options, :memory, :name, :type, :schedule, :source_ids + serialize :options, JSONWithIndifferentAccess + serialize :memory, JSONWithIndifferentAccess + + def options=(o) + self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o) + end + + def memory=(o) + self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o) + end + validates_presence_of :name, :user validate :sources_are_owned validate :validate_schedule + validate :validate_options after_initialize :set_default_schedule before_validation :set_default_schedule @@ -74,6 +84,10 @@ class Agent < ActiveRecord::Base raise "Implement me in your subclass" end + def validate_options + # Implement me in your subclass to test for valid options. + end + def event_created_within(days) event = most_recent_event event && event.created_at > days.to_i.days.ago && event.payload.present? && event diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb index 3055d2f5..662bb5e4 100644 --- a/app/models/agents/adioso_agent.rb +++ b/app/models/agents/adioso_agent.rb @@ -29,18 +29,18 @@ module Agents def default_options { - :start_date => Date.today.httpdate[0..15], - :end_date => Date.today.plus_with_duration(100).httpdate[0..15], - :from => "New York", - :to => "Chicago", - :username => "xx", - :password => "xx", - :expected_update_period_in_days => "1" + 'start_date' => Date.today.httpdate[0..15], + 'end_date' => Date.today.plus_with_duration(100).httpdate[0..15], + 'from' => "New York", + 'to' => "Chicago", + 'username' => "xx", + 'password' => "xx", + 'expected_update_period_in_days' => "1" } end def working? - event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs? + event_created_within(options['expected_update_period_in_days']) && !recent_error_logs? end def validate_options @@ -54,9 +54,9 @@ module Agents end def check - auth_options = {:basic_auth => {:username =>options[:username], :password=>options[:password]}} - parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options[:from])}+to+#{URI.encode(options[:to])}", auth_options - fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options[:end_date])}\\3#{date_to_unix_epoch(options[:start_date])}" + auth_options = {:basic_auth => {:username =>options[:username], :password=>options['password']}} + parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options['from'])}+to+#{URI.encode(options['to'])}", auth_options + fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options['end_date'])}\\3#{date_to_unix_epoch(options['start_date'])}" fare = HTTParty.get fare_request, auth_options if fare["warnings"] @@ -64,7 +64,7 @@ module Agents else event = fare["results"].min {|a,b| a["cost"] <=> b["cost"]} event["date"] = Time.at(event["date"]).to_date.httpdate[0..15] - event["route"] = "#{options[:from]} to #{options[:to]}" + event["route"] = "#{options['from']} to #{options['to']}" create_event :payload => event end end diff --git a/app/models/agents/digest_email_agent.rb b/app/models/agents/digest_email_agent.rb index 009564df..62204d68 100644 --- a/app/models/agents/digest_email_agent.rb +++ b/app/models/agents/digest_email_agent.rb @@ -9,7 +9,7 @@ module Agents description <<-MD The DigestEmailAgent collects any Events sent to it and sends them all via email when run. The email will be sent to your account's address and will have a `subject` and an optional `headline` before - listing the Events. If the Events' payloads contain a `:message`, that will be highlighted, otherwise everything in + listing the Events. If the Events' payloads contain a `message`, that will be highlighted, otherwise everything in their payloads will be shown. Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent. @@ -17,29 +17,29 @@ module Agents def default_options { - :subject => "You have some notifications!", - :headline => "Your notifications:", - :expected_receive_period_in_days => "2" + 'subject' => "You have some notifications!", + 'headline' => "Your notifications:", + 'expected_receive_period_in_days' => "2" } end def receive(incoming_events) incoming_events.each do |event| - self.memory[:queue] ||= [] - self.memory[:queue] << event.payload - self.memory[:events] ||= [] - self.memory[:events] << event.id + self.memory['queue'] ||= [] + self.memory['queue'] << event.payload + self.memory['events'] ||= [] + self.memory['events'] << event.id end end def check - if self.memory[:queue] && self.memory[:queue].length > 0 - ids = self.memory[:events].join(",") - groups = self.memory[:queue].map { |payload| present(payload) } + if self.memory['queue'] && self.memory['queue'].length > 0 + ids = self.memory['events'].join(",") + groups = self.memory['queue'].map { |payload| present(payload) } log "Sending digest mail to #{user.email} with events [#{ids}]" - SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => groups) - self.memory[:queue] = [] - self.memory[:events] = [] + SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => groups) + self.memory['queue'] = [] + self.memory['events'] = [] end end end diff --git a/app/models/agents/email_agent.rb b/app/models/agents/email_agent.rb index e1c79dce..64f81164 100644 --- a/app/models/agents/email_agent.rb +++ b/app/models/agents/email_agent.rb @@ -16,16 +16,16 @@ module Agents def default_options { - :subject => "You have a notification!", - :headline => "Your notification:", - :expected_receive_period_in_days => "2" + 'subject' => "You have a notification!", + 'headline' => "Your notification:", + 'expected_receive_period_in_days' => "2" } end def receive(incoming_events) incoming_events.each do |event| log "Sending digest mail to #{user.email} with event #{event.id}" - SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => [present(event.payload)]) + SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => [present(event.payload)]) end end end diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb index cc27c5da..947c00bb 100644 --- a/app/models/agents/event_formatting_agent.rb +++ b/app/models/agents/event_formatting_agent.rb @@ -8,20 +8,20 @@ module Agents For example, here is a possible Event: { - :high => { - :celsius => "18", - :fahreinheit => "64" + "high": { + "celsius": "18", + "fahreinheit": "64" }, - :conditions => "Rain showers", - :data => "This is some data" + "conditions": "Rain showers", + "data": "This is some data" } You may want to send this event to another Agent, for example a Twilio Agent, which expects a `message` key. You can use an Event Formatting Agent's `instructions` setting to do this in the following way: - instructions: { - message: "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.", - subject: "$.data" + "instructions": { + "message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.", + "subject": "$.data" } JSONPaths must be between < and > . Make sure that you don't use these symbols anywhere else. @@ -29,8 +29,8 @@ module Agents Events generated by this possible Event Formatting Agent will look like: { - :message => "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.", - :subject => "This is some data" + "message": "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.", + "subject": "This is some data" } If you want to retain original contents of events and only add new keys, then set `mode` to `merge`, otherwise set it to `clean`. @@ -40,25 +40,25 @@ module Agents To CGI escape output (for example when creating a link), prefix with `escape`, like so: { - :message => "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=" + "message": "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=" } MD event_description "User defined" def validate_options - errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options[:instructions].present? and options[:mode].present? and options[:skip_agent].present? and options[:skip_created_at].present? + errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options['instructions'].present? and options['mode'].present? and options['skip_agent'].present? and options['skip_created_at'].present? end def default_options { - :instructions => { - :message => "You received a text <$.text> from <$.fields.from>", - :some_other_field => "Looks like the weather is going to be <$.fields.weather>" + 'instructions' => { + 'message' => "You received a text <$.text> from <$.fields.from>", + 'some_other_field' => "Looks like the weather is going to be <$.fields.weather>" }, - :mode => "clean", - :skip_agent => "false", - :skip_created_at => "false" + 'mode' => "clean", + 'skip_agent' => "false", + 'skip_created_at' => "false" } end @@ -68,10 +68,10 @@ module Agents def receive(incoming_events) incoming_events.each do |event| - formatted_event = options[:mode].to_s == "merge" ? event.payload : {} - options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) } - formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true" - formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true" + formatted_event = options['mode'].to_s == "merge" ? event.payload : {} + options['instructions'].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) } + formatted_event['agent'] = Agent.find(event.agent_id).type.slice!(8..-1) unless options['skip_agent'].to_s == "true" + formatted_event['created_at'] = event.created_at unless options['skip_created_at'].to_s == "true" create_event :payload => formatted_event end end diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb index 116347e6..0d27327b 100644 --- a/app/models/agents/human_task_agent.rb +++ b/app/models/agents/human_task_agent.rb @@ -74,69 +74,69 @@ module Agents MD def validate_options - options[:hit] ||= {} - options[:hit][:questions] ||= [] + options['hit'] ||= {} + options['hit']['questions'] ||= [] - errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on]) - errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options[:hit][:assignments].present? && options[:hit][:assignments].to_i > 0 - errors.add(:base, "'hit.title' must be provided") unless options[:hit][:title].present? - errors.add(:base, "'hit.description' must be provided") unless options[:hit][:description].present? - errors.add(:base, "'hit.questions' must be provided") unless options[:hit][:questions].present? && options[:hit][:questions].length > 0 + errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options['trigger_on']) + errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options['hit']['assignments'].present? && options['hit']['assignments'].to_i > 0 + errors.add(:base, "'hit.title' must be provided") unless options['hit']['title'].present? + errors.add(:base, "'hit.description' must be provided") unless options['hit']['description'].present? + errors.add(:base, "'hit.questions' must be provided") unless options['hit']['questions'].present? && options['hit']['questions'].length > 0 - if options[:trigger_on] == "event" - errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present? - elsif options[:trigger_on] == "schedule" - errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0 + if options['trigger_on'] == "event" + errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options['expected_receive_period_in_days'].present? + elsif options['trigger_on'] == "schedule" + errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options['submission_period'].present? && options['submission_period'].to_i > 0 end - if options[:hit][:questions].any? { |question| [:key, :name, :required, :type, :question].any? {|k| !question[k].present? } } + if options['hit']['questions'].any? { |question| %w[key name required type question].any? {|k| !question[k].present? } } errors.add(:base, "all questions must set 'key', 'name', 'required', 'type', and 'question'") end - if options[:hit][:questions].any? { |question| question[:type] == "selection" && (!question[:selections].present? || question[:selections].length == 0 || !question[:selections].all? {|s| s[:key].present? } || !question[:selections].all? { |s| s[:text].present? })} + if options['hit']['questions'].any? { |question| question['type'] == "selection" && (!question['selections'].present? || question['selections'].length == 0 || !question['selections'].all? {|s| s['key'].present? } || !question['selections'].all? { |s| s['text'].present? })} errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'") end - if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" } + if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" } errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option") end end def default_options { - :expected_receive_period_in_days => 2, - :trigger_on => "event", - :hit => + 'expected_receive_period_in_days' => 2, + 'trigger_on' => "event", + 'hit' => { - :assignments => 1, - :title => "Sentiment evaluation", - :description => "Please rate the sentiment of this message: '<$.message>'", - :reward => 0.05, - :lifetime_in_seconds => 24 * 60 * 60, - :questions => + 'assignments' => 1, + 'title' => "Sentiment evaluation", + 'description' => "Please rate the sentiment of this message: '<$.message>'", + 'reward' => 0.05, + 'lifetime_in_seconds' => 24 * 60 * 60, + 'questions' => [ { - :type => "selection", - :key => "sentiment", - :name => "Sentiment", - :required => "true", - :question => "Please select the best sentiment value:", - :selections => + 'type' => "selection", + 'key' => "sentiment", + 'name' => "Sentiment", + 'required' => "true", + 'question' => "Please select the best sentiment value:", + 'selections' => [ - { :key => "happy", :text => "Happy" }, - { :key => "sad", :text => "Sad" }, - { :key => "neutral", :text => "Neutral" } + { 'key' => "happy", 'text' => "Happy" }, + { 'key' => "sad", 'text' => "Sad" }, + { 'key' => "neutral", 'text' => "Neutral" } ] }, { - :type => "free_text", - :key => "feedback", - :name => "Have any feedback for us?", - :required => "false", - :question => "Feedback", - :default => "Type here...", - :min_length => "2", - :max_length => "2000" + 'type' => "free_text", + 'key' => "feedback", + 'name' => "Have any feedback for us?", + 'required' => "false", + 'question' => "Feedback", + 'default' => "Type here...", + 'min_length' => "2", + 'max_length' => "2000" } ] } @@ -144,20 +144,20 @@ module Agents end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def check review_hits - if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60 - memory[:last_schedule] = Time.now.to_i + if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60 + memory['last_schedule'] = Time.now.to_i create_hit end end def receive(incoming_events) - if options[:trigger_on] == "event" + if options['trigger_on'] == "event" incoming_events.each do |event| create_hit event end @@ -168,7 +168,7 @@ module Agents def review_hits reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids - my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s) + my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys if reviewable_hit_ids.length > 0 log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]" end @@ -178,26 +178,26 @@ module Agents log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}" if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" } - payload = { :answers => assignments.map(&:answers) } + payload = { 'answers' => assignments.map(&:answers) } - if options[:take_majority] == "true" + if options['take_majority'] == "true" counts = {} - options[:hit][:questions].each do |question| - question_counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo } + options['hit']['questions'].each do |question| + question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo } assignments.each do |assignment| answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers) - answer = answers[question[:key]] + answer = answers[question['key']] question_counts[answer] += 1 end - counts[question[:key]] = question_counts + counts[question['key']] = question_counts end - payload[:counts] = counts + payload['counts'] = counts majority_answer = counts.inject({}) do |memo, (key, question_counts)| memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first memo end - payload[:majority_answer] = majority_answer + payload['majority_answer'] = majority_answer if all_questions_are_numeric? average_answer = counts.inject({}) do |memo, (key, question_counts)| @@ -209,44 +209,44 @@ module Agents memo[key] = sum / divisor.to_f memo end - payload[:average_answer] = average_answer + payload['average_answer'] = average_answer end end event = create_event :payload => payload - log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id]) + log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id]) assignments.each(&:approve!) hit.dispose! - memory[:hits].delete(hit_id) + memory['hits'].delete(hit_id) end end end def all_questions_are_numeric? - options[:hit][:questions].all? do |question| - question[:selections].all? do |selection| - selection[:key] == selection[:key].to_f.to_s || selection[:key] == selection[:key].to_i.to_s + options['hit']['questions'].all? do |question| + question['selections'].all? do |selection| + selection['key'] == selection['key'].to_f.to_s || selection['key'] == selection['key'].to_i.to_s end end end def create_hit(event = nil) payload = event ? event.payload : {} - title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip - description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip - questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload) + title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip + description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip + questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload) hit = RTurk::Hit.create(:title => title) do |hit| - hit.max_assignments = (options[:hit][:assignments] || 1).to_i + hit.max_assignments = (options['hit']['assignments'] || 1).to_i hit.description = description - hit.lifetime = (options[:hit][:lifetime_in_seconds] || 24 * 60 * 60).to_i + hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions) - hit.reward = (options[:hit][:reward] || 0.05).to_f + hit.reward = (options['hit']['reward'] || 0.05).to_f #hit.qualifications.add :approval_rate, { :gt => 80 } end - memory[:hits] ||= {} - memory[:hits][hit.id] = event && event.id + memory['hits'] ||= {} + memory['hits'][hit.id] = event && event.id log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event end @@ -314,7 +314,7 @@ module Agents end end - if question[:default].present? + if question['default'].present? DefaultText do text question['default'] end diff --git a/app/models/agents/manual_event_agent.rb b/app/models/agents/manual_event_agent.rb index 7b9f6f14..7c045a97 100644 --- a/app/models/agents/manual_event_agent.rb +++ b/app/models/agents/manual_event_agent.rb @@ -14,8 +14,8 @@ module Agents end def handle_details_post(params) - if params[:payload] - create_event(:payload => params[:payload]) + if params['payload'] + create_event(:payload => params['payload']) { :success => true } else { :success => false, :error => "You must provide a JSON payload" } diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb index 7c7e7468..bf277c2c 100644 --- a/app/models/agents/peak_detector_agent.rb +++ b/app/models/agents/peak_detector_agent.rb @@ -28,22 +28,22 @@ module Agents MD def validate_options - unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:value_path].present? + unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['value_path'].present? errors.add(:base, "expected_receive_period_in_days, value_path, and message are required") end end def default_options { - :expected_receive_period_in_days => "2", - :group_by_path => "filter", - :value_path => "count", - :message => "A peak was found" + 'expected_receive_period_in_days' => "2", + 'group_by_path' => "filter", + 'value_path' => "count", + 'message' => "A peak was found" } end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def receive(incoming_events) @@ -57,25 +57,25 @@ module Agents private def check_for_peak(group, event) - memory[:peaks] ||= {} - memory[:peaks][group] ||= [] + memory['peaks'] ||= {} + memory['peaks'][group] ||= [] - if memory[:data][group].length > 4 && (memory[:peaks][group].empty? || memory[:peaks][group].last < event.created_at.to_i - peak_spacing) + if memory['data'][group].length > 4 && (memory['peaks'][group].empty? || memory['peaks'][group].last < event.created_at.to_i - peak_spacing) average_value, standard_deviation = stats_for(group, :skip_last => 1) - newest_value, newest_time = memory[:data][group][-1].map(&:to_f) + newest_value, newest_time = memory['data'][group][-1].map(&:to_f) #p [newest_value, average_value, average_value + std_multiple * standard_deviation, standard_deviation] if newest_value > average_value + std_multiple * standard_deviation - memory[:peaks][group] << newest_time - memory[:peaks][group].reject! { |p| p <= newest_time - window_duration } - create_event :payload => {:message => options[:message], :peak => newest_value, :peak_time => newest_time, :grouped_by => group.to_s} + memory['peaks'][group] << newest_time + memory['peaks'][group].reject! { |p| p <= newest_time - window_duration } + create_event :payload => { 'message' => options['message'], 'peak' => newest_value, 'peak_time' => newest_time, 'grouped_by' => group.to_s } end end end def stats_for(group, options = {}) - data = memory[:data][group].map { |d| d.first.to_f } + data = memory['data'][group].map { |d| d.first.to_f } data = data[0...(data.length - (options[:skip_last] || 0))] length = data.length.to_f mean = 0 @@ -94,39 +94,39 @@ module Agents end def window_duration - if options[:window_duration].present? # The older option - options[:window_duration].to_i + if options['window_duration'].present? # The older option + options['window_duration'].to_i else - (options[:window_duration_in_days] || 14).to_f.days + (options['window_duration_in_days'] || 14).to_f.days end end def std_multiple - (options[:std_multiple] || 3).to_f + (options['std_multiple'] || 3).to_f end def peak_spacing - if options[:peak_spacing].present? # The older option - options[:peak_spacing].to_i + if options['peak_spacing'].present? # The older option + options['peak_spacing'].to_i else - (options[:min_peak_spacing_in_days] || 2).to_f.days + (options['min_peak_spacing_in_days'] || 2).to_f.days end end def group_for(event) - ((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group') + ((options['group_by_path'].present? && Utils.value_at(event.payload, options['group_by_path'])) || 'no_group') end def remember(group, event) - memory[:data] ||= {} - memory[:data][group] ||= [] - memory[:data][group] << [Utils.value_at(event.payload, options[:value_path]), event.created_at.to_i] + memory['data'] ||= {} + memory['data'][group] ||= [] + memory['data'][group] << [ Utils.value_at(event.payload, options['value_path']), event.created_at.to_i ] cleanup group end def cleanup(group) - newest_time = memory[:data][group].last.last - memory[:data][group].reject! { |value, time| time <= newest_time - window_duration } + newest_time = memory['data'][group].last.last + memory['data'][group].reject! { |value, time| time <= newest_time - window_duration } end end end \ No newline at end of file diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index d3582e25..3d5ae824 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -11,17 +11,17 @@ module Agents def default_options { - :post_url => "http://www.example.com", - :expected_receive_period_in_days => 1 + 'post_url' => "http://www.example.com", + 'expected_receive_period_in_days' => 1 } end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def validate_options - unless options[:post_url].present? && options[:expected_receive_period_in_days].present? + unless options['post_url'].present? && options['expected_receive_period_in_days'].present? errors.add(:base, "post_url and expected_receive_period_in_days are required fields") end end diff --git a/app/models/agents/sentiment_agent.rb b/app/models/agents/sentiment_agent.rb index c6a917b0..c0f6c5be 100644 --- a/app/models/agents/sentiment_agent.rb +++ b/app/models/agents/sentiment_agent.rb @@ -28,31 +28,31 @@ module Agents def default_options { - :content => "$.message.text[*]", - :expected_receive_period_in_days => 1 + 'content' => "$.message.text[*]", + 'expected_receive_period_in_days' => 1 } end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def receive(incoming_events) anew = self.class.sentiment_hash incoming_events.each do |event| - Utils.values_at(event.payload, options[:content]).each do |content| + Utils.values_at(event.payload, options['content']).each do |content| sent_values = sentiment_values anew, content - create_event :payload => { :content => content, - :valence => sent_values[0], - :arousal => sent_values[1], - :dominance => sent_values[2], - :original_event => event.payload } + create_event :payload => { 'content' => content, + 'valence' => sent_values[0], + 'arousal' => sent_values[1], + 'dominance' => sent_values[2], + 'original_event' => event.payload } end end end def validate_options - errors.add(:base, "content and expected_receive_period_in_days must be present") unless options[:content].present? && options[:expected_receive_period_in_days].present? + errors.add(:base, "content and expected_receive_period_in_days must be present") unless options['content'].present? && options['expected_receive_period_in_days'].present? end def self.sentiment_hash diff --git a/app/models/agents/translation_agent.rb b/app/models/agents/translation_agent.rb index 2fc073f3..7bbdb05d 100644 --- a/app/models/agents/translation_agent.rb +++ b/app/models/agents/translation_agent.rb @@ -17,26 +17,26 @@ module Agents def default_options { - :client_id => "xxxxxx", - :client_secret => "xxxxxx", - :to => "fi", - :expected_receive_period_in_days => 1, - :content => { - :text => "$.message.text", - :content => "$.xyz" + 'client_id' => "xxxxxx", + 'client_secret' => "xxxxxx", + 'to' => "fi", + 'expected_receive_period_in_days' => 1, + 'content' => { + 'text' => "$.message.text", + 'content' => "$.xyz" } } end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def translate(text, to, access_token) translate_uri = URI 'http://api.microsofttranslator.com/v2/Ajax.svc/Translate' params = { - :text => text, - :to => to + 'text' => text, + 'to' => to } translate_uri.query = URI.encode_www_form params request = Net::HTTP::Get.new translate_uri.request_uri @@ -47,7 +47,7 @@ module Agents end def validate_options - unless options[:client_id].present? && options[:client_secret].present? && options[:to].present? && options[:content].present? && options[:expected_receive_period_in_days].present? + unless options['client_id'].present? && options['client_secret'].present? && options['to'].present? && options['content'].present? && options['expected_receive_period_in_days'].present? errors.add :base, "client_id,client_secret,to,expected_receive_period_in_days and content are all required" end end @@ -60,16 +60,16 @@ module Agents def receive(incoming_events) auth_uri = URI "https://datamarket.accesscontrol.windows.net/v2/OAuth2-13" - response = postform auth_uri, :client_id => options[:client_id], - :client_secret => options[:client_secret], + response = postform auth_uri, :client_id => options['client_id'], + :client_secret => options['client_secret'], :scope => "http://api.microsofttranslator.com", :grant_type => "client_credentials" access_token = JSON.parse(response.body)["access_token"] incoming_events.each do |event| translated_event = {} - options[:content].each_pair do |key, value| + options['content'].each_pair do |key, value| to_be_translated = Utils.values_at event.payload, value - translated_event[key] = translate to_be_translated.first, options[:to], access_token + translated_event[key] = translate to_be_translated.first, options['to'], access_token end create_event :payload => translated_event end diff --git a/app/models/agents/trigger_agent.rb b/app/models/agents/trigger_agent.rb index f8190c33..b32fff6e 100644 --- a/app/models/agents/trigger_agent.rb +++ b/app/models/agents/trigger_agent.rb @@ -23,57 +23,57 @@ module Agents MD def validate_options - unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:rules].present? && - options[:rules].all? { |rule| rule[:type].present? && VALID_COMPARISON_TYPES.include?(rule[:type]) && rule[:value].present? && rule[:path].present? } + unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['rules'].present? && + options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? } errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required") end end def default_options { - :expected_receive_period_in_days => "2", - :rules => [{ - :type => "regex", - :value => "foo\\d+bar", - :path => "topkey.subkey.subkey.goal", - }], - :message => "Looks like your pattern matched in ''!" + 'expected_receive_period_in_days' => "2", + 'rules' => [{ + 'type' => "regex", + 'value' => "foo\\d+bar", + 'path' => "topkey.subkey.subkey.goal", + }], + 'message' => "Looks like your pattern matched in ''!" } end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def receive(incoming_events) incoming_events.each do |event| - match = options[:rules].all? do |rule| - value_at_path = Utils.value_at(event[:payload], rule[:path]) - case rule[:type] + match = options['rules'].all? do |rule| + value_at_path = Utils.value_at(event['payload'], rule['path']) + case rule['type'] when "regex" - value_at_path.to_s =~ Regexp.new(rule[:value], Regexp::IGNORECASE) + value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE) when "!regex" - value_at_path.to_s !~ Regexp.new(rule[:value], Regexp::IGNORECASE) + value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE) when "field>value" - value_at_path.to_f > rule[:value].to_f + value_at_path.to_f > rule['value'].to_f when "field>=value" - value_at_path.to_f >= rule[:value].to_f + value_at_path.to_f >= rule['value'].to_f when "field { :message => make_message(event[:payload]) } # Maybe this should include the - # original event as well? + create_event :payload => { 'message' => make_message(event[:payload]) } # Maybe this should include the + # original event as well? end end end diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb index 82076e89..9cec3268 100644 --- a/app/models/agents/twilio_agent.rb +++ b/app/models/agents/twilio_agent.rb @@ -9,7 +9,7 @@ module Agents description <<-MD The TwilioAgent receives and collects events and sends them via text message or gives you a call when scheduled. - It is assumed that events have a `:message`, `:text`, or `:sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys. + It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys. Set `receiver_cell` to the number to receive text messages/call and `sender_cell` to the number sending them. @@ -22,35 +22,35 @@ module Agents def default_options { - :account_sid => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', - :auth_token => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', - :sender_cell => 'xxxxxxxxxx', - :receiver_cell => 'xxxxxxxxxx', - :server_url => 'http://somename.com:3000', - :receive_text => 'true', - :receive_call => 'false', - :expected_receive_period_in_days => '1' + 'account_sid' => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + 'auth_token' => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + 'sender_cell' => 'xxxxxxxxxx', + 'receiver_cell' => 'xxxxxxxxxx', + 'server_url' => 'http://somename.com:3000', + 'receive_text' => 'true', + 'receive_call' => 'false', + 'expected_receive_period_in_days' => '1' } end def validate_options - unless options[:account_sid].present? && options[:auth_token].present? && options[:sender_cell].present? && options[:receiver_cell].present? && options[:expected_receive_period_in_days].present? && options[:receive_call].present? && options[:receive_text].present? + unless options['account_sid'].present? && options['auth_token'].present? && options['sender_cell'].present? && options['receiver_cell'].present? && options['expected_receive_period_in_days'].present? && options['receive_call'].present? && options['receive_text'].present? errors.add(:base, 'account_sid, auth_token, sender_cell, receiver_cell, receive_text, receive_call and expected_receive_period_in_days are all required') end end def receive(incoming_events) - @client = Twilio::REST::Client.new options[:account_sid], options[:auth_token] - memory[:pending_calls] ||= {} + @client = Twilio::REST::Client.new options['account_sid'], options['auth_token'] + memory['pending_calls'] ||= {} incoming_events.each do |event| - message = (event.payload[:message] || event.payload[:text] || event.payload[:sms]).to_s + message = (event.payload['message'] || event.payload['text'] || event.payload['sms']).to_s if message != "" - if options[:receive_call].to_s == 'true' + if options['receive_call'].to_s == 'true' secret = SecureRandom.hex 3 - memory[:pending_calls][secret] = message + memory['pending_calls'][secret] = message make_call secret end - if options[:receive_text].to_s == 'true' + if options['receive_text'].to_s == 'true' message = message.slice 0..160 send_message message end @@ -59,19 +59,19 @@ module Agents end def working? - last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs? end def send_message(message) - @client.account.sms.messages.create :from => options[:sender_cell], - :to => options[:receiver_cell], + @client.account.sms.messages.create :from => options['sender_cell'], + :to => options['receiver_cell'], :body => message end def make_call(secret) - @client.account.calls.create :from => options[:sender_cell], - :to => options[:receiver_cell], - :url => post_url(options[:server_url],secret) + @client.account.calls.create :from => options['sender_cell'], + :to => options['receiver_cell'], + :url => post_url(options['server_url'],secret) end def post_url(server_url,secret) @@ -79,9 +79,9 @@ module Agents end def receive_webhook(params) - if memory[:pending_calls].has_key? params[:secret] - response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'} - memory[:pending_calls].delete params[:secret] + if memory['pending_calls'].has_key? params['secret'] + response = Twilio::TwiML::Response.new {|r| r.Say memory['pending_calls'][params['secret']], :voice => 'woman'} + memory['pending_calls'].delete params['secret'] [response.text, 200] end end diff --git a/app/models/agents/twitter_publish_agent.rb b/app/models/agents/twitter_publish_agent.rb index 823b504e..d03d7984 100644 --- a/app/models/agents/twitter_publish_agent.rb +++ b/app/models/agents/twitter_publish_agent.rb @@ -19,25 +19,25 @@ module Agents MD def validate_options - unless options[:username].present? && - options[:expected_update_period_in_days].present? + unless options['username'].present? && + options['expected_update_period_in_days'].present? errors.add(:base, "username and expected_update_period_in_days are required") end end def working? - (event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs? + (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs? end def default_options { - :username => "", - :expected_update_period_in_days => "10", - :consumer_key => "---", - :consumer_secret => "---", - :oauth_token => "---", - :oauth_token_secret => "---", - :message_path => "text" + 'username' => "", + 'expected_update_period_in_days' => "10", + 'consumer_key' => "---", + 'consumer_secret' => "---", + 'oauth_token' => "---", + 'oauth_token_secret' => "---", + 'message_path' => "text" } end @@ -47,22 +47,22 @@ module Agents incoming_events = incoming_events.first(20) end incoming_events.each do |event| - tweet_text = Utils.value_at(event.payload, options[:message_path]) + tweet_text = Utils.value_at(event.payload, options['message_path']) begin publish_tweet tweet_text create_event :payload => { - :success => true, - :published_tweet => tweet_text, - :agent_id => event.agent_id, - :event_id => event.id + 'success' => true, + 'published_tweet' => tweet_text, + 'agent_id' => event.agent_id, + 'event_id' => event.id } rescue Twitter::Error => e create_event :payload => { - :success => false, - :error => e.message, - :failed_tweet => tweet_text, - :agent_id => event.agent_id, - :event_id => event.id + 'success' => false, + 'error' => e.message, + 'failed_tweet' => tweet_text, + 'agent_id' => event.agent_id, + 'event_id' => event.id } end end diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb index 96afae25..f369b891 100644 --- a/app/models/agents/twitter_stream_agent.rb +++ b/app/models/agents/twitter_stream_agent.rb @@ -54,26 +54,26 @@ module Agents default_schedule "11pm" def validate_options - unless options[:filters].present? && - options[:expected_update_period_in_days].present? && - options[:generate].present? + unless options['filters'].present? && + options['expected_update_period_in_days'].present? && + options['generate'].present? errors.add(:base, "expected_update_period_in_days, generate, and filters are required fields") end end def working? - event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs? + event_created_within(options['expected_update_period_in_days']) && !recent_error_logs? end def default_options { - :consumer_key => "---", - :consumer_secret => "---", - :oauth_token => "---", - :oauth_token_secret => "---", - :filters => %w[keyword1 keyword2], - :expected_update_period_in_days => "2", - :generate => "events" + 'consumer_key' => "---", + 'consumer_secret' => "---", + 'oauth_token' => "---", + 'oauth_token_secret' => "---", + 'filters' => %w[keyword1 keyword2], + 'expected_update_period_in_days' => "2", + 'generate' => "events" } end @@ -81,33 +81,33 @@ module Agents filter = lookup_filter(filter) if filter - if options[:generate] == "counts" + if options['generate'] == "counts" # Avoid memory pollution by reloading the Agent. agent = Agent.find(id) - agent.memory[:filter_counts] ||= {} - agent.memory[:filter_counts][filter] ||= 0 - agent.memory[:filter_counts][filter] += 1 - remove_unused_keys!(agent, :filter_counts) + agent.memory['filter_counts'] ||= {} + agent.memory['filter_counts'][filter] ||= 0 + agent.memory['filter_counts'][filter] += 1 + remove_unused_keys!(agent, 'filter_counts') agent.save! else - create_event :payload => status.merge(:filter => filter) + create_event :payload => status.merge('filter' => filter) end end end def check - if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0 - memory[:filter_counts].each do |filter, count| - create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i } + if options['generate'] == "counts" && memory['filter_counts'] && memory['filter_counts'].length > 0 + memory['filter_counts'].each do |filter, count| + create_event :payload => { 'filter' => filter, 'count' => count, 'time' => Time.now.to_i } end end - memory[:filter_counts] = {} + memory['filter_counts'] = {} end protected def lookup_filter(filter) - options[:filters].each do |known_filter| + options['filters'].each do |known_filter| if known_filter == filter return filter elsif known_filter.is_a?(Array) @@ -120,7 +120,7 @@ module Agents def remove_unused_keys!(agent, base) if agent.memory[base] - (agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key| + (agent.memory[base].keys - agent.options['filters'].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key| agent.memory[base].delete(removed_key) end end diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 5f3cc627..52d10a7b 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -41,36 +41,36 @@ module Agents default_schedule "every_1h" def validate_options - unless options[:username].present? && - options[:expected_update_period_in_days].present? + unless options['username'].present? && + options['expected_update_period_in_days'].present? errors.add(:base, "username and expected_update_period_in_days are required") end end def working? - event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs? + event_created_within(options['expected_update_period_in_days']) && !recent_error_logs? end def default_options { - :username => "tectonic", - :expected_update_period_in_days => "2", - :consumer_key => "---", - :consumer_secret => "---", - :oauth_token => "---", - :oauth_token_secret => "---" + 'username' => "tectonic", + 'expected_update_period_in_days' => "2", + 'consumer_key' => "---", + 'consumer_secret' => "---", + 'oauth_token' => "---", + 'oauth_token_secret' => "---" } end def check - since_id = memory[:since_id] || nil + since_id = memory['since_id'] || nil opts = {:count => 200, :include_rts => true, :exclude_replies => false, :include_entities => true, :contributor_details => true} opts.merge! :since_id => since_id unless since_id.nil? - tweets = Twitter.user_timeline(options[:username], opts) + tweets = Twitter.user_timeline(options['username'], opts) tweets.each do |tweet| - memory[:since_id] = tweet.id if !memory[:since_id] || (tweet.id > memory[:since_id]) + memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id']) create_event :payload => tweet.attrs end diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb index f1a6a601..2330c307 100644 --- a/app/models/agents/user_location_agent.rb +++ b/app/models/agents/user_location_agent.rb @@ -34,11 +34,11 @@ module Agents end def default_options - { :secret => SecureRandom.hex(7) } + { 'secret' => SecureRandom.hex(7) } end def validate_options - errors.add(:base, "secret is required and must be longer than 4 characters") unless options[:secret].present? && options[:secret].length > 4 + errors.add(:base, "secret is required and must be longer than 4 characters") unless options['secret'].present? && options['secret'].length > 4 end end end \ No newline at end of file diff --git a/app/models/agents/weather_agent.rb b/app/models/agents/weather_agent.rb index 34058e04..5a94128b 100644 --- a/app/models/agents/weather_agent.rb +++ b/app/models/agents/weather_agent.rb @@ -45,30 +45,30 @@ module Agents end def wunderground - Wunderground.new(options[:api_key]) if key_setup? + Wunderground.new(options['api_key']) if key_setup? end def key_setup? - options[:api_key] && options[:api_key] != "your-key" + options['api_key'] && options['api_key'] != "your-key" end def default_options { - :api_key => "your-key", - :location => "94103" + 'api_key' => "your-key", + 'location' => "94103" } end def validate_options - errors.add(:base, "location is required") unless options[:location].present? || options[:zipcode].present? - errors.add(:base, "api_key is required") unless options[:api_key].present? + errors.add(:base, "location is required") unless options['location'].present? || options['zipcode'].present? + errors.add(:base, "api_key is required") unless options['api_key'].present? end def check if key_setup? - wunderground.forecast_for(options[:location] || options[:zipcode])["forecast"]["simpleforecast"]["forecastday"].each do |day| + wunderground.forecast_for(options['location'] || options['zipcode'])["forecast"]["simpleforecast"]["forecastday"].each do |day| if is_tomorrow?(day) - create_event :payload => day.merge(:location => options[:location] || options[:zipcode]) + create_event :payload => day.merge('location' => options['location'] || options['zipcode']) end end end diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 096baa40..c0cef555 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -15,19 +15,19 @@ module Agents To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes. - When parsing HTML or XML, these sub-hashes specify how to extract with a `:css` CSS selector and either `:text => true` or `attr` pointing to an attribute name to grab. An example: + When parsing HTML or XML, these sub-hashes specify how to extract with a `css` CSS selector and either `'text': true` or `attr` pointing to an attribute name to grab. An example: - :extract => { - :url => { :css => "#comic img", :attr => "src" }, - :title => { :css => "#comic img", :attr => "title" }, - :body_text => { :css => "div.main", :text => true } + 'extract': { + 'url': { 'css': "#comic img", 'attr': "src" }, + 'title': { 'css': "#comic img", 'attr': "title" }, + 'body_text': { 'css': "div.main", 'text': true } } When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about. For example: - :extract => { - :title => { :path => "results.data[*].title" }, - :description => { :path => "results.data[*].description" } + 'extract': { + 'title': { 'path': "results.data[*].title" }, + 'description': { 'path': "results.data[*].description" } } Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful. @@ -36,7 +36,7 @@ module Agents MD event_description do - "Events will have the fields you specified. Your options look like:\n\n #{Utils.pretty_print options[:extract]}" + "Events will have the fields you specified. Your options look like:\n\n #{Utils.pretty_print options['extract']}" end default_schedule "every_12h" @@ -44,33 +44,33 @@ module Agents UNIQUENESS_LOOK_BACK = 30 def working? - event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs? + event_created_within(options['expected_update_period_in_days']) && !recent_error_logs? end def default_options { - :expected_update_period_in_days => "2", - :url => "http://xkcd.com", - :type => "html", - :mode => :on_change, - :extract => { - :url => {:css => "#comic img", :attr => "src"}, - :title => {:css => "#comic img", :attr => "title"} + 'expected_update_period_in_days' => "2", + 'url' => "http://xkcd.com", + 'type' => "html", + 'mode' => :on_change, + 'extract' => { + 'url' => {'css' => "#comic img", 'attr' => "src"}, + 'title' => {'css' => "#comic img", 'attr' => "title"} } } end def validate_options - errors.add(:base, "url and expected_update_period_in_days are required") unless options[:expected_update_period_in_days].present? && options[:url].present? - if !options[:extract].present? && extraction_type != "json" + errors.add(:base, "url and expected_update_period_in_days are required") unless options['expected_update_period_in_days'].present? && options['url'].present? + if !options['extract'].present? && extraction_type != "json" errors.add(:base, "extract is required for all types except json") end end def check hydra = Typhoeus::Hydra.new - log "Fetching #{options[:url]}" - request = Typhoeus::Request.new(options[:url], :followlocation => true) + log "Fetching #{options['url']}" + request = Typhoeus::Request.new(options['url'], :followlocation => true) request.on_failure do |response| error "Failed: #{response.inspect}" end @@ -85,37 +85,37 @@ module Agents end else output = {} - options[:extract].each do |name, extraction_details| + options['extract'].each do |name, extraction_details| result = if extraction_type == "json" - output[name] = Utils.values_at(doc, extraction_details[:path]) + output[name] = Utils.values_at(doc, extraction_details['path']) else - output[name] = doc.css(extraction_details[:css]).map { |node| - if extraction_details[:attr] - node.attr(extraction_details[:attr]) - elsif extraction_details[:text] + output[name] = doc.css(extraction_details['css']).map { |node| + if extraction_details['attr'] + node.attr(extraction_details['attr']) + elsif extraction_details['text'] node.text() else - error ":attr or :text is required on HTML or XML extraction patterns" + error "'attr' or 'text' is required on HTML or XML extraction patterns" return end } end - log "Extracting #{extraction_type} at #{extraction_details[:path] || extraction_details[:css]}: #{result}" + log "Extracting #{extraction_type} at #{extraction_details['path'] || extraction_details['css']}: #{result}" end - num_unique_lengths = options[:extract].keys.map { |name| output[name].length }.uniq + num_unique_lengths = options['extract'].keys.map { |name| output[name].length }.uniq if num_unique_lengths.length != 1 - error "Got an uneven number of matches for #{options[:name]}: #{options[:extract].inspect}" + error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}" return end num_unique_lengths.first.times do |index| result = {} - options[:extract].keys.each do |name| + options['extract'].keys.each do |name| result[name] = output[name][index] if name.to_s == 'url' - result[name] = URI.join(options[:url], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil? + result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil? end end @@ -133,22 +133,22 @@ module Agents private def store_payload? result - !options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json)) + !options['mode'] || options['mode'].to_s == "all" || (options['mode'].to_s == "on_change" && !previous_payloads.include?(result.to_json)) end def previous_payloads - events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change" + events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options['mode'].to_s == "on_change" end def extract_full_json? - (!options[:extract].present? && extraction_type == "json") + (!options['extract'].present? && extraction_type == "json") end def extraction_type - (options[:type] || begin - if options[:url] =~ /\.(rss|xml)$/i + (options['type'] || begin + if options['url'] =~ /\.(rss|xml)$/i "xml" - elsif options[:url] =~ /\.json$/i + elsif options['url'] =~ /\.json$/i "json" else "html" diff --git a/app/models/agents/weibo_publish_agent.rb b/app/models/agents/weibo_publish_agent.rb index bc0fc07e..4930c93b 100644 --- a/app/models/agents/weibo_publish_agent.rb +++ b/app/models/agents/weibo_publish_agent.rb @@ -20,24 +20,24 @@ module Agents MD def validate_options - unless options[:uid].present? && - options[:expected_update_period_in_days].present? + unless options['uid'].present? && + options['expected_update_period_in_days'].present? errors.add(:base, "expected_update_period_in_days and uid are required") end end def working? - (event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs? + (event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs? end def default_options { - :uid => "", - :access_token => "---", - :app_key => "---", - :app_secret => "---", - :expected_update_period_in_days => "10", - :message_path => "text" + 'uid' => "", + 'access_token' => "---", + 'app_key' => "---", + 'app_secret' => "---", + 'expected_update_period_in_days' => "10", + 'message_path' => "text" } end @@ -47,25 +47,25 @@ module Agents incoming_events = incoming_events.first(20) end incoming_events.each do |event| - tweet_text = Utils.value_at(event.payload, options[:message_path]) + tweet_text = Utils.value_at(event.payload, options['message_path']) if event.agent.type == "Agents::TwitterUserAgent" tweet_text = unwrap_tco_urls(tweet_text, event.payload) end begin publish_tweet tweet_text create_event :payload => { - :success => true, - :published_tweet => tweet_text, - :agent_id => event.agent_id, - :event_id => event.id + 'success' => true, + 'published_tweet' => tweet_text, + 'agent_id' => event.agent_id, + 'event_id' => event.id } rescue OAuth2::Error => e create_event :payload => { - :success => false, - :error => e.message, - :failed_tweet => tweet_text, - :agent_id => event.agent_id, - :event_id => event.id + 'success' => false, + 'error' => e.message, + 'failed_tweet' => tweet_text, + 'agent_id' => event.agent_id, + 'event_id' => event.id } end end diff --git a/app/models/agents/weibo_user_agent.rb b/app/models/agents/weibo_user_agent.rb index 27c94b29..d00120dd 100644 --- a/app/models/agents/weibo_user_agent.rb +++ b/app/models/agents/weibo_user_agent.rb @@ -70,29 +70,29 @@ module Agents default_schedule "every_1h" def validate_options - unless options[:uid].present? && - options[:expected_update_period_in_days].present? + unless options['uid'].present? && + options['expected_update_period_in_days'].present? errors.add(:base, "expected_update_period_in_days and uid are required") end end def working? - event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs? + event_created_within(options['expected_update_period_in_days']) && !recent_error_logs? end def default_options { - :uid => "", - :access_token => "---", - :app_key => "---", - :app_secret => "---", - :expected_update_period_in_days => "2" + 'uid' => "", + 'access_token' => "---", + 'app_key' => "---", + 'app_secret' => "---", + 'expected_update_period_in_days' => "2" } end def check - since_id = memory[:since_id] || nil - opts = {:uid => options[:uid].to_i} + since_id = memory['since_id'] || nil + opts = {:uid => options['uid'].to_i} opts.merge! :since_id => since_id unless since_id.nil? # http://open.weibo.com/wiki/2/statuses/user_timeline/en @@ -101,7 +101,7 @@ module Agents resp[:statuses].each do |status| - memory[:since_id] = status.id if !memory[:since_id] || (status.id > memory[:since_id]) + memory['since_id'] = status.id if !memory['since_id'] || (status.id > memory['since_id']) create_event :payload => status.as_json end diff --git a/app/models/event.rb b/app/models/event.rb index f7c99256..883b8a26 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -1,13 +1,16 @@ -require 'serialize_and_normalize' +require 'json_with_indifferent_access' class Event < ActiveRecord::Base - include SerializeAndNormalize - attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at acts_as_mappable - serialize_and_normalize :payload + serialize :payload, JSONWithIndifferentAccess + + def payload=(o) + self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o) + end + belongs_to :user belongs_to :agent, :counter_cache => true diff --git a/lib/json_with_indifferent_access.rb b/lib/json_with_indifferent_access.rb new file mode 100644 index 00000000..ee02daf2 --- /dev/null +++ b/lib/json_with_indifferent_access.rb @@ -0,0 +1,9 @@ +class JSONWithIndifferentAccess + def self.load(json) + ActiveSupport::HashWithIndifferentAccess.new(JSON.load(json || '{}')) + end + + def self.dump(hash) + JSON.dump(hash) + end +end \ No newline at end of file diff --git a/lib/serialize_and_normalize.rb b/lib/serialize_and_normalize.rb deleted file mode 100644 index 857a777f..00000000 --- a/lib/serialize_and_normalize.rb +++ /dev/null @@ -1,46 +0,0 @@ -module SerializeAndNormalize - extend ActiveSupport::Concern - - module ClassMethods - def serialize_and_normalize(*column_names) - column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name| - setup_name = "setup_#{column_name}".to_sym - normalize_name = "normalize_#{column_name}".to_sym - validate_name = "validate_#{column_name}".to_sym - - serialize column_name, JSON - after_initialize setup_name - before_validation normalize_name - before_save normalize_name - validate validate_name - - class_eval <<-RUBY - def #{setup_name} - self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new - end - - def #{validate_name} - # Implement me in your subclass. - end - - def #{normalize_name} - self.#{column_name} = self[:#{column_name}] - end - - def #{column_name}=(data) - data = (JSON.parse(data) rescue data) if data.is_a?(String) - - case data - when ActiveSupport::HashWithIndifferentAccess - self[:#{column_name}] = data - when Hash - self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data) - else - self[:#{column_name}] = data - end - end - RUBY - end - end - end -end diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb index 4b5c164a..5e073583 100644 --- a/spec/controllers/agents_controller_spec.rb +++ b/spec/controllers/agents_controller_spec.rb @@ -5,7 +5,7 @@ describe AgentsController do { :type => "Agents::WebsiteAgent", :name => "Something", - :options => agents(:bob_website_agent).options.to_json, + :options => agents(:bob_website_agent).options, :source_ids => [agents(:bob_weather_agent).id, ""] }.merge(options) end diff --git a/spec/fixtures/agents.yml b/spec/fixtures/agents.yml index df47d77f..0066ec73 100644 --- a/spec/fixtures/agents.yml +++ b/spec/fixtures/agents.yml @@ -11,7 +11,7 @@ jane_website_agent: :title => {:css => "item title", :text => true}, :url => {:css => "item link", :text => true} } - }.to_yaml.inspect %> + }.to_json.inspect %> bob_website_agent: type: Agents::WebsiteAgent @@ -26,7 +26,7 @@ bob_website_agent: :url => {:css => "#comic img", :attr => "src"}, :title => {:css => "#comic img", :attr => "title"} } - }.to_yaml.inspect %> + }.to_json.inspect %> bob_weather_agent: type: Agents::WeatherAgent @@ -38,7 +38,7 @@ bob_weather_agent: :lat => 37.779329, :lng => -122.41915, :api_key => 'test' - }.to_yaml.inspect %> + }.to_json.inspect %> jane_weather_agent: type: Agents::WeatherAgent @@ -50,7 +50,7 @@ jane_weather_agent: :lat => 37.779329, :lng => -122.41915, :api_key => 'test' - }.to_yaml.inspect %> + }.to_json.inspect %> jane_rain_notifier_agent: type: Agents::TriggerAgent @@ -64,7 +64,7 @@ jane_rain_notifier_agent: :path => "conditions" }], :message => "Just so you know, it looks like '' tomorrow in " - }.to_yaml.inspect %> + }.to_json.inspect %> bob_rain_notifier_agent: type: Agents::TriggerAgent @@ -78,7 +78,7 @@ bob_rain_notifier_agent: :path => "conditions" }], :message => "Just so you know, it looks like '' tomorrow in " - }.to_yaml.inspect %> + }.to_json.inspect %> bob_twitter_user_agent: type: Agents::TwitterUserAgent @@ -91,7 +91,7 @@ bob_twitter_user_agent: :consumer_secret => "---", :oauth_token => "---", :oauth_token_secret => "---" - }.to_yaml.inspect %> + }.to_json.inspect %> bob_manual_event_agent: type: Agents::ManualEventAgent diff --git a/spec/fixtures/events.yml b/spec/fixtures/events.yml index 51adfde7..4ed2e86d 100644 --- a/spec/fixtures/events.yml +++ b/spec/fixtures/events.yml @@ -1,9 +1,9 @@ bob_website_agent_event: user: bob agent: bob_website_agent - payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %> + payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %> jane_website_agent_event: user: jane agent: jane_website_agent - payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %> + payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %> diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb index 34be7129..6e9e9efc 100644 --- a/spec/models/agent_spec.rb +++ b/spec/models/agent_spec.rb @@ -261,9 +261,9 @@ describe Agent do it "symbolizes memory before validating" do agent = Agents::SomethingSource.new(:name => "something") agent.user = users(:bob) - agent.memory["bad"] = :hello + agent.memory["bad"] = 2 agent.save - agent.memory[:bad].should == :hello + agent.memory[:bad].should == 2 end it "should not allow agents owned by other people" do diff --git a/spec/models/agents/digest_email_agent_spec.rb b/spec/models/agents/digest_email_agent_spec.rb index 86ddf9d2..68e4b9b3 100644 --- a/spec/models/agents/digest_email_agent_spec.rb +++ b/spec/models/agents/digest_email_agent_spec.rb @@ -19,16 +19,16 @@ describe Agents::DigestEmailAgent do it "queues any payloads it receives" do event1 = Event.new event1.agent = agents(:bob_rain_notifier_agent) - event1.payload = "Something you should know about" + event1.payload = { :data => "Something you should know about" } event1.save! event2 = Event.new event2.agent = agents(:bob_weather_agent) - event2.payload = "Something else you should know about" + event2.payload = { :data => "Something else you should know about" } event2.save! Agents::DigestEmailAgent.async_receive(@checker.id, [event1.id, event2.id]) - @checker.reload.memory[:queue].should == ["Something you should know about", "Something else you should know about"] + @checker.reload.memory[:queue].should == [{ 'data' => "Something you should know about" }, { 'data' => "Something else you should know about" }] end end @@ -37,7 +37,7 @@ describe Agents::DigestEmailAgent do Agents::DigestEmailAgent.async_check(@checker.id) ActionMailer::Base.deliveries.should == [] - @checker.memory[:queue] = ["Something you should know about", + @checker.memory[:queue] = [{ :data => "Something you should know about" }, { :title => "Foo", :url => "http://google.com", :bar => 2 }, { "message" => "hi", :woah => "there" }, { "test" => 2 }] @@ -47,7 +47,7 @@ describe Agents::DigestEmailAgent do Agents::DigestEmailAgent.async_check(@checker.id) ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"] ActionMailer::Base.deliveries.last.subject.should == "something interesting" - get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something you should know about\n\nFoo\n bar: 2\n url: http://google.com\n\nhi\n woah: there\n\nEvent\n test: 2" + get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n data: Something you should know about\n\nFoo\n bar: 2\n url: http://google.com\n\nhi\n woah: there\n\nEvent\n test: 2" @checker.reload.memory[:queue].should be_empty end diff --git a/spec/models/agents/email_agent_spec.rb b/spec/models/agents/email_agent_spec.rb index 2a7c3d40..fcc3429a 100644 --- a/spec/models/agents/email_agent_spec.rb +++ b/spec/models/agents/email_agent_spec.rb @@ -21,12 +21,12 @@ describe Agents::EmailAgent do event1 = Event.new event1.agent = agents(:bob_rain_notifier_agent) - event1.payload = "Something you should know about" + event1.payload = { :data => "Something you should know about" } event1.save! event2 = Event.new event2.agent = agents(:bob_weather_agent) - event2.payload = "Something else you should know about" + event2.payload = { :data => "Something else you should know about" } event2.save! Agents::EmailAgent.async_receive(@checker.id, [event1.id]) @@ -35,8 +35,8 @@ describe Agents::EmailAgent do ActionMailer::Base.deliveries.count.should == 2 ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"] ActionMailer::Base.deliveries.last.subject.should == "something interesting" - get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something else you should know about" - get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Something you should know about" + get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n data: Something else you should know about" + get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Event\n data: Something you should know about" end it "can receive complex events and send them on" do diff --git a/spec/models/agents/human_task_agent_spec.rb b/spec/models/agents/human_task_agent_spec.rb index 054ea4cf..89639316 100644 --- a/spec/models/agents/human_task_agent_spec.rb +++ b/spec/models/agents/human_task_agent_spec.rb @@ -9,8 +9,8 @@ describe Agents::HumanTaskAgent do @event = Event.new @event.agent = agents(:bob_rain_notifier_agent) - @event.payload = { :foo => { "bar" => { :baz => "a2b" } }, - :name => "Joe" } + @event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" } }, + 'name' => "Joe" } @event.id = 345 @checker.should be_valid @@ -18,110 +18,110 @@ describe Agents::HumanTaskAgent do describe "validations" do it "validates that trigger_on is 'schedule' or 'event'" do - @checker.options[:trigger_on] = "foo" + @checker.options['trigger_on'] = "foo" @checker.should_not be_valid end it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do - @checker.options[:trigger_on] = "event" - @checker.options[:expected_receive_period_in_days] = nil + @checker.options['trigger_on'] = "event" + @checker.options['expected_receive_period_in_days'] = nil @checker.should_not be_valid - @checker.options[:expected_receive_period_in_days] = 2 + @checker.options['expected_receive_period_in_days'] = 2 @checker.should be_valid end it "requires a positive submission_period when trigger_on is set to 'schedule'" do - @checker.options[:trigger_on] = "schedule" - @checker.options[:submission_period] = nil + @checker.options['trigger_on'] = "schedule" + @checker.options['submission_period'] = nil @checker.should_not be_valid - @checker.options[:submission_period] = 2 + @checker.options['submission_period'] = 2 @checker.should be_valid end it "requires a hit.title" do - @checker.options[:hit][:title] = "" + @checker.options['hit']['title'] = "" @checker.should_not be_valid end it "requires a hit.description" do - @checker.options[:hit][:description] = "" + @checker.options['hit']['description'] = "" @checker.should_not be_valid end it "requires hit.assignments" do - @checker.options[:hit][:assignments] = "" + @checker.options['hit']['assignments'] = "" @checker.should_not be_valid - @checker.options[:hit][:assignments] = 0 + @checker.options['hit']['assignments'] = 0 @checker.should_not be_valid - @checker.options[:hit][:assignments] = "moose" + @checker.options['hit']['assignments'] = "moose" @checker.should_not be_valid - @checker.options[:hit][:assignments] = "2" + @checker.options['hit']['assignments'] = "2" @checker.should be_valid end it "requires hit.questions" do - old_questions = @checker.options[:hit][:questions] - @checker.options[:hit][:questions] = nil + old_questions = @checker.options['hit']['questions'] + @checker.options['hit']['questions'] = nil @checker.should_not be_valid - @checker.options[:hit][:questions] = [] + @checker.options['hit']['questions'] = [] @checker.should_not be_valid - @checker.options[:hit][:questions] = [old_questions[0]] + @checker.options['hit']['questions'] = [old_questions[0]] @checker.should be_valid end it "requires that all questions have key, name, required, type, and question" do - old_questions = @checker.options[:hit][:questions] - @checker.options[:hit][:questions].first[:key] = "" + old_questions = @checker.options['hit']['questions'] + @checker.options['hit']['questions'].first['key'] = "" @checker.should_not be_valid - @checker.options[:hit][:questions] = old_questions - @checker.options[:hit][:questions].first[:name] = "" + @checker.options['hit']['questions'] = old_questions + @checker.options['hit']['questions'].first['name'] = "" @checker.should_not be_valid - @checker.options[:hit][:questions] = old_questions - @checker.options[:hit][:questions].first[:required] = nil + @checker.options['hit']['questions'] = old_questions + @checker.options['hit']['questions'].first['required'] = nil @checker.should_not be_valid - @checker.options[:hit][:questions] = old_questions - @checker.options[:hit][:questions].first[:type] = "" + @checker.options['hit']['questions'] = old_questions + @checker.options['hit']['questions'].first['type'] = "" @checker.should_not be_valid - @checker.options[:hit][:questions] = old_questions - @checker.options[:hit][:questions].first[:question] = "" + @checker.options['hit']['questions'] = old_questions + @checker.options['hit']['questions'].first['question'] = "" @checker.should_not be_valid end it "requires that all questions of type 'selection' have a selections array with keys and text" do - @checker.options[:hit][:questions][0][:selections] = [] + @checker.options['hit']['questions'][0]['selections'] = [] @checker.should_not be_valid - @checker.options[:hit][:questions][0][:selections] = [{}] + @checker.options['hit']['questions'][0]['selections'] = [{}] @checker.should_not be_valid - @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }] + @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "" }] @checker.should_not be_valid - @checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }] + @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "hi" }] @checker.should_not be_valid - @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }] + @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "" }] @checker.should_not be_valid - @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }] + @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }] @checker.should be_valid - @checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}] + @checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }, {}] @checker.should_not be_valid end it "requires that all questions be of type 'selection' when `take_majority` is `true`" do - @checker.options[:take_majority] = "true" + @checker.options['take_majority'] = "true" @checker.should_not be_valid - @checker.options[:hit][:questions][1][:type] = "selection" - @checker.options[:hit][:questions][1][:selections] = @checker.options[:hit][:questions][0][:selections] + @checker.options['hit']['questions'][1]['type'] = "selection" + @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections'] @checker.should be_valid end end describe "when 'trigger_on' is set to 'schedule'" do before do - @checker.options[:trigger_on] = "schedule" - @checker.options[:submission_period] = "2" - @checker.options.delete(:expected_receive_period_in_days) + @checker.options['trigger_on'] = "schedule" + @checker.options['submission_period'] = "2" + @checker.options.delete('expected_receive_period_in_days') end it "should check for reviewable HITs frequently" do @@ -151,7 +151,7 @@ describe Agents::HumanTaskAgent do describe "when 'trigger_on' is set to 'event'" do it "should not create HITs during check but should check for reviewable HITs" do - @checker.options[:submission_period] = "2" + @checker.options['submission_period'] = "2" now = Time.now stub(Time).now { now } mock(@checker).review_hits.times(3) @@ -171,9 +171,9 @@ describe Agents::HumanTaskAgent do describe "creating hits" do it "can create HITs based on events, interpolating their values" do - @checker.options[:hit][:title] = "Hi <.name>" - @checker.options[:hit][:description] = "Make something for <.name>" - @checker.options[:hit][:questions][0][:name] = "<.name> Question 1" + @checker.options['hit']['title'] = "Hi <.name>" + @checker.options['hit']['description'] = "Make something for <.name>" + @checker.options['hit']['questions'][0]['name'] = "<.name> Question 1" question_form = nil hitInterface = OpenStruct.new @@ -183,8 +183,8 @@ describe Agents::HumanTaskAgent do @checker.send :create_hit, @event - hitInterface.max_assignments.should == @checker.options[:hit][:assignments] - hitInterface.reward.should == @checker.options[:hit][:reward] + hitInterface.max_assignments.should == @checker.options['hit']['assignments'] + hitInterface.reward.should == @checker.options['hit']['reward'] hitInterface.description.should == "Make something for Joe" xml = question_form.to_xml @@ -192,18 +192,18 @@ describe Agents::HumanTaskAgent do xml.should include("Make something for Joe") xml.should include("Joe Question 1") - @checker.memory[:hits][123].should == @event.id + @checker.memory['hits'][123].should == @event.id end it "works without an event too" do - @checker.options[:hit][:title] = "Hi <.name>" + @checker.options['hit']['title'] = "Hi <.name>" hitInterface = OpenStruct.new hitInterface.id = 123 mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface } @checker.send :create_hit - hitInterface.max_assignments.should == @checker.options[:hit][:assignments] - hitInterface.reward.should == @checker.options[:hit][:reward] + hitInterface.max_assignments.should == @checker.options['hit']['assignments'] + hitInterface.reward.should == @checker.options['hit']['reward'] end end @@ -253,14 +253,14 @@ describe Agents::HumanTaskAgent do it "should work on multiple HITs" do event2 = Event.new event2.agent = agents(:bob_rain_notifier_agent) - event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } }, - :name2 => "Joe2" } + event2.payload = { 'foo2' => { "bar2" => { 'baz2' => "a2b2" } }, + 'name2' => "Joe2" } event2.id = 3452 # It knows about two HITs from two different events. - @checker.memory[:hits] = {} - @checker.memory[:hits][:"JH3132836336DHG"] = @event.id - @checker.memory[:hits][:"JH39AA63836DHG"] = event2.id + @checker.memory['hits'] = {} + @checker.memory['hits']["JH3132836336DHG"] = @event.id + @checker.memory['hits']["JH39AA63836DHG"] = event2.id hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs. @@ -273,7 +273,7 @@ describe Agents::HumanTaskAgent do end it "shouldn't do anything if an assignment isn't ready" do - @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + @checker.memory['hits'] = { "JH3132836336DHG" => @event.id } mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } assignments = [ FakeAssignment.new(:status => "Accepted", :answers => {}), @@ -288,11 +288,11 @@ describe Agents::HumanTaskAgent do @checker.send :review_hits assignments.all? {|a| a.approved == true }.should be_false - @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id } + @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id } end it "shouldn't do anything if an assignment is missing" do - @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + @checker.memory['hits'] = { "JH3132836336DHG" => @event.id } mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } assignments = [ FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"}) @@ -306,11 +306,11 @@ describe Agents::HumanTaskAgent do @checker.send :review_hits assignments.all? {|a| a.approved == true }.should be_false - @checker.memory[:hits].should == { "JH3132836336DHG" => @event.id } + @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id } end it "should create events when all assignments are ready" do - @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + @checker.memory['hits'] = { "JH3132836336DHG" => @event.id } mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } assignments = [ FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}), @@ -327,32 +327,32 @@ describe Agents::HumanTaskAgent do assignments.all? {|a| a.approved == true }.should be_true hit.should be_disposed - @checker.events.last.payload[:answers].should == [ + @checker.events.last.payload['answers'].should == [ {'sentiment' => "neutral", 'feedback' => ""}, {'sentiment' => "happy", 'feedback' => "Take 2"} ] - @checker.memory[:hits].should == {} + @checker.memory['hits'].should == {} end describe "taking majority votes" do before do - @checker.options[:take_majority] = "true" - @checker.memory[:hits] = { "JH3132836336DHG" => @event.id } + @checker.options['take_majority'] = "true" + @checker.memory['hits'] = { "JH3132836336DHG" => @event.id } mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } end it "should take the majority votes of all questions" do - @checker.options[:hit][:questions][1] = { - :type => "selection", - :key => "age_range", - :name => "Age Range", - :required => "true", - :question => "Please select your age range:", - :selections => + @checker.options['hit']['questions'][1] = { + 'type' => "selection", + 'key' => "age_range", + 'name' => "Age Range", + 'required' => "true", + 'question' => "Please select your age range:", + 'selections' => [ - { :key => "<50", :text => "50 years old or younger" }, - { :key => ">50", :text => "Over 50 years old" } + { 'key' => "<50", 'text' => "50 years old or younger" }, + { 'key' => ">50", 'text' => "Over 50 years old" } ] } @@ -371,35 +371,35 @@ describe Agents::HumanTaskAgent do assignments.all? {|a| a.approved == true }.should be_true - @checker.events.last.payload[:answers].should == [ + @checker.events.last.payload['answers'].should == [ { 'sentiment' => "sad", 'age_range' => "<50" }, { 'sentiment' => "neutral", 'age_range' => ">50" }, { 'sentiment' => "happy", 'age_range' => ">50" }, { 'sentiment' => "happy", 'age_range' => ">50" } ] - @checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } } - @checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" } - @checker.events.last.payload.should_not have_key(:average_answer) + @checker.events.last.payload['counts'].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } } + @checker.events.last.payload['majority_answer'].should == { 'sentiment' => "happy", 'age_range' => ">50" } + @checker.events.last.payload.should_not have_key('average_answer') - @checker.memory[:hits].should == {} + @checker.memory['hits'].should == {} end it "should also provide an average answer when all questions are numeric" do - @checker.options[:hit][:questions] = [ + @checker.options['hit']['questions'] = [ { - :type => "selection", - :key => "rating", - :name => "Rating", - :required => "true", - :question => "Please select a rating:", - :selections => + 'type' => "selection", + 'key' => "rating", + 'name' => "Rating", + 'required' => "true", + 'question' => "Please select a rating:", + 'selections' => [ - { :key => "1", :text => "One" }, - { :key => "2", :text => "Two" }, - { :key => "3", :text => "Three" }, - { :key => "4", :text => "Four" }, - { :key => "5.1", :text => "Five Point One" } + { 'key' => "1", 'text' => "One" }, + { 'key' => "2", 'text' => "Two" }, + { 'key' => "3", 'text' => "Three" }, + { 'key' => "4", 'text' => "Four" }, + { 'key' => "5.1", 'text' => "Five Point One" } ] } ] @@ -420,7 +420,7 @@ describe Agents::HumanTaskAgent do assignments.all? {|a| a.approved == true }.should be_true - @checker.events.last.payload[:answers].should == [ + @checker.events.last.payload['answers'].should == [ { 'rating' => "1" }, { 'rating' => "3" }, { 'rating' => "5.1" }, @@ -428,11 +428,11 @@ describe Agents::HumanTaskAgent do { 'rating' => "2" } ] - @checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } } - @checker.events.last.payload[:majority_answer].should == { 'rating' => "2" } - @checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 } + @checker.events.last.payload['counts'].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } } + @checker.events.last.payload['majority_answer'].should == { 'rating' => "2" } + @checker.events.last.payload['average_answer'].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 } - @checker.memory[:hits].should == {} + @checker.memory['hits'].should == {} end end end diff --git a/spec/models/agents/peak_detector_agent_spec.rb b/spec/models/agents/peak_detector_agent_spec.rb index 53663e27..b55e2b77 100644 --- a/spec/models/agents/peak_detector_agent_spec.rb +++ b/spec/models/agents/peak_detector_agent_spec.rb @@ -3,12 +3,12 @@ require 'spec_helper' describe Agents::PeakDetectorAgent do before do @valid_params = { - :name => "my peak detector agent", - :options => { - :expected_receive_period_in_days => "2", - :group_by_path => "filter", - :value_path => "count", - :message => "A peak was found" + 'name' => "my peak detector agent", + 'options' => { + 'expected_receive_period_in_days' => "2", + 'group_by_path' => "filter", + 'value_path' => "count", + 'message' => "A peak was found" } } @@ -19,54 +19,54 @@ describe Agents::PeakDetectorAgent do describe "#receive" do it "tracks and groups by the group_by_path" do - events = build_events(:keys => [:count, :filter], + events = build_events(:keys => ['count', 'filter'], :values => [[1, "something"], [2, "something"], [3, "else"]]) @agent.receive events - @agent.memory[:data][:something].map(&:first).should == [1, 2] - @agent.memory[:data][:something].last.last.should be_within(10).of((100 - 1).hours.ago.to_i) - @agent.memory[:data][:else].first.first.should == 3 - @agent.memory[:data][:else].first.last.should be_within(10).of((100 - 2).hours.ago.to_i) + @agent.memory['data']['something'].map(&:first).should == [1, 2] + @agent.memory['data']['something'].last.last.should be_within(10).of((100 - 1).hours.ago.to_i) + @agent.memory['data']['else'].first.first.should == 3 + @agent.memory['data']['else'].first.last.should be_within(10).of((100 - 2).hours.ago.to_i) end it "works without a group_by_path as well" do - @agent.options[:group_by_path] = "" - events = build_events(:keys => [:count], :values => [[1], [2]]) + @agent.options['group_by_path'] = "" + events = build_events(:keys => ['count'], :values => [[1], [2]]) @agent.receive events - @agent.memory[:data][:no_group].map(&:first).should == [1, 2] + @agent.memory['data']['no_group'].map(&:first).should == [1, 2] end it "keeps a rolling window of data" do - @agent.options[:window_duration_in_days] = 5/24.0 - @agent.receive build_events(:keys => [:count], + @agent.options['window_duration_in_days'] = 5/24.0 + @agent.receive build_events(:keys => ['count'], :values => [1, 2, 3, 4, 5, 6, 7, 8].map {|i| [i]}, - :pattern => { :filter => "something" }) - @agent.memory[:data][:something].map(&:first).should == [4, 5, 6, 7, 8] + :pattern => { 'filter' => "something" }) + @agent.memory['data']['something'].map(&:first).should == [4, 5, 6, 7, 8] end it "finds peaks" do - build_events(:keys => [:count], + build_events(:keys => ['count'], :values => [5, 6, 4, 5, 4, 5, 15, 11, # peak 8, 50, # ignored because it's too close to the first peak 4, 5].map {|i| [i]}, - :pattern => { :filter => "something" }).each.with_index do |event, index| + :pattern => { 'filter' => "something" }).each.with_index do |event, index| lambda { @agent.receive([event]) }.should change { @agent.events.count }.by( index == 6 ? 1 : 0 ) end - @agent.events.last.payload[:peak].should == 15.0 - @agent.memory[:peaks][:something].length.should == 1 + @agent.events.last.payload['peak'].should == 15.0 + @agent.memory['peaks']['something'].length.should == 1 end it "keeps a rolling window of peaks" do - @agent.options[:min_peak_spacing_in_days] = 1/24.0 - @agent.receive build_events(:keys => [:count], + @agent.options['min_peak_spacing_in_days'] = 1/24.0 + @agent.receive build_events(:keys => ['count'], :values => [1, 1, 1, 1, 1, 1, 10, 1, 1, 1, 1, 1, 1, 1, 10, 1].map {|i| [i]}, - :pattern => { :filter => "something" }) - @agent.memory[:peaks][:something].length.should == 2 + :pattern => { 'filter' => "something" }) + @agent.memory['peaks']['something'].length.should == 2 end end @@ -76,17 +76,17 @@ describe Agents::PeakDetectorAgent do end it "should validate presence of message" do - @agent.options[:message] = nil + @agent.options['message'] = nil @agent.should_not be_valid end it "should validate presence of expected_receive_period_in_days" do - @agent.options[:expected_receive_period_in_days] = "" + @agent.options['expected_receive_period_in_days'] = "" @agent.should_not be_valid end it "should validate presence of value_path" do - @agent.options[:value_path] = "" + @agent.options['value_path'] = "" @agent.should_not be_valid end end diff --git a/spec/models/agents/trigger_agent_spec.rb b/spec/models/agents/trigger_agent_spec.rb index f525110a..22b3b6ec 100644 --- a/spec/models/agents/trigger_agent_spec.rb +++ b/spec/models/agents/trigger_agent_spec.rb @@ -3,16 +3,16 @@ require 'spec_helper' describe Agents::TriggerAgent do before do @valid_params = { - :name => "my trigger agent", - :options => { - :expected_receive_period_in_days => 2, - :rules => [{ - :type => "regex", - 'value' => "a\\db", - :path => "foo.bar.baz", - }], - :message => "I saw '' from " - } + 'name' => "my trigger agent", + 'options' => { + 'expected_receive_period_in_days' => 2, + 'rules' => [{ + 'type' => "regex", + 'value' => "a\\db", + 'path' => "foo.bar.baz", + }], + 'message' => "I saw '' from " + } } @checker = Agents::TriggerAgent.new(@valid_params) @@ -21,8 +21,8 @@ describe Agents::TriggerAgent do @event = Event.new @event.agent = agents(:bob_rain_notifier_agent) - @event.payload = { :foo => { "bar" => { :baz => "a2b" }}, - :name => "Joe" } + @event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" }}, + 'name' => "Joe" } end describe "validation" do @@ -31,22 +31,22 @@ describe Agents::TriggerAgent do end it "should validate presence of options" do - @checker.options[:message] = nil + @checker.options['message'] = nil @checker.should_not be_valid end it "should validate the three fields in each rule" do - @checker.options[:rules] << { :path => "foo", :type => "fake", :value => "6" } + @checker.options['rules'] << { 'path' => "foo", 'type' => "fake", 'value' => "6" } @checker.should_not be_valid - @checker.options[:rules].last[:type] = "field>=value" + @checker.options['rules'].last['type'] = "field>=value" @checker.should be_valid - @checker.options[:rules].last.delete(:value) + @checker.options['rules'].last.delete('value') @checker.should_not be_valid end end describe "#working?" do - it "checks to see if the Agent has received any events in the last :expected_receive_period_in_days days" do + it "checks to see if the Agent has received any events in the last 'expected_receive_period_in_days' days" do @event.save! @checker.should_not be_working # no events have ever been received @@ -60,30 +60,30 @@ describe Agents::TriggerAgent do describe "#receive" do it "handles regex" do - @event.payload[:foo]["bar"][:baz] = "a222b" + @event.payload['foo']['bar']['baz'] = "a222b" lambda { @checker.receive([@event]) }.should_not change { Event.count } - @event.payload[:foo]["bar"][:baz] = "a2b" + @event.payload['foo']['bar']['baz'] = "a2b" lambda { @checker.receive([@event]) }.should change { Event.count }.by(1) end it "handles negated regex" do - @event.payload[:foo]["bar"][:baz] = "a2b" - @checker.options[:rules][0] = { - :type => "!regex", - :value => "a\\db", - :path => "foo.bar.baz", - } + @event.payload['foo']['bar']['baz'] = "a2b" + @checker.options['rules'][0] = { + 'type' => "!regex", + 'value' => "a\\db", + 'path' => "foo.bar.baz", + } lambda { @checker.receive([@event]) }.should_not change { Event.count } - @event.payload[:foo]["bar"][:baz] = "a22b" + @event.payload['foo']['bar']['baz'] = "a22b" lambda { @checker.receive([@event]) }.should change { Event.count }.by(1) @@ -91,49 +91,49 @@ describe Agents::TriggerAgent do it "puts can extract values into the message based on paths" do @checker.receive([@event]) - Event.last.payload[:message].should == "I saw 'a2b' from Joe" + Event.last.payload['message'].should == "I saw 'a2b' from Joe" end it "handles numerical comparisons" do - @event.payload[:foo]["bar"][:baz] = "5" - @checker.options[:rules].first[:value] = 6 - @checker.options[:rules].first[:type] = "field "world" } - @checker.options[:rules].first[:type] = "field==value" - @checker.options[:rules].first[:path] = "hello" - @checker.options[:rules].first[:value] = "world" + @event.payload = { 'hello' => "world" } + @checker.options['rules'].first['type'] = "field==value" + @checker.options['rules'].first['path'] = "hello" + @checker.options['rules'].first['value'] = "world" lambda { @checker.receive([@event]) }.should change { Event.count }.by(1) - @checker.options[:rules].first[:path] = "foo" + @checker.options['rules'].first['path'] = "foo" lambda { @checker.receive([@event]) }.should_not change { Event.count } - @checker.options[:rules].first[:value] = "hi" + @checker.options['rules'].first['value'] = "hi" lambda { @checker.receive([@event]) }.should_not change { Event.count } @@ -163,11 +163,11 @@ describe Agents::TriggerAgent do it "handles multiple events" do event2 = Event.new event2.agent = agents(:bob_weather_agent) - event2.payload = { :foo => { "bar" => { :baz => "a2b" }}} + event2.payload = { 'foo' => { 'bar' => { 'baz' => "a2b" }}} event3 = Event.new event3.agent = agents(:bob_weather_agent) - event3.payload = { :foo => { "bar" => { :baz => "a222b" }}} + event3.payload = { 'foo' => { 'bar' => { 'baz' => "a222b" }}} lambda { @checker.receive([@event, event2, event3]) @@ -175,19 +175,19 @@ describe Agents::TriggerAgent do end it "handles ANDing rules together" do - @checker.options[:rules] << { - :type => "field>=value", - :value => "4", - :path => "foo.bing" + @checker.options['rules'] << { + 'type' => "field>=value", + 'value' => "4", + 'path' => "foo.bing" } - @event.payload[:foo]["bing"] = "5" + @event.payload['foo']["bing"] = "5" lambda { @checker.receive([@event]) }.should change { Event.count }.by(1) - @checker.options[:rules].last[:value] = 6 + @checker.options['rules'].last['value'] = 6 lambda { @checker.receive([@event]) }.should_not change { Event.count } From ba0967e44934a7ef52952eef23424780130fe02d Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Wed, 25 Dec 2013 00:08:39 -0500 Subject: [PATCH 04/10] puts --- app/models/agents/peak_detector_agent.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb index bf277c2c..d8e058db 100644 --- a/app/models/agents/peak_detector_agent.rb +++ b/app/models/agents/peak_detector_agent.rb @@ -64,8 +64,6 @@ module Agents average_value, standard_deviation = stats_for(group, :skip_last => 1) newest_value, newest_time = memory['data'][group][-1].map(&:to_f) - #p [newest_value, average_value, average_value + std_multiple * standard_deviation, standard_deviation] - if newest_value > average_value + std_multiple * standard_deviation memory['peaks'][group] << newest_time memory['peaks'][group].reject! { |p| p <= newest_time - window_duration } From 21c0e5911dac0e05eb18877cf532234097274972 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 26 Dec 2013 18:36:09 -0500 Subject: [PATCH 05/10] transform and clean utf-8 --- ...1223032112_switch_to_json_serialization.rb | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/db/migrate/20131223032112_switch_to_json_serialization.rb b/db/migrate/20131223032112_switch_to_json_serialization.rb index c132d532..13c46ac6 100644 --- a/db/migrate/20131223032112_switch_to_json_serialization.rb +++ b/db/migrate/20131223032112_switch_to_json_serialization.rb @@ -5,17 +5,41 @@ class SwitchToJsonSerialization < ActiveRecord::Migration } def up - puts "This migration will update Agent and Event storage from YAML to JSON. It should work, but please make a backup" - puts "before proceeding." - print "Continue? (y/n) " - STDOUT.flush - exit unless STDIN.gets =~ /^y/i + if data_exists? + puts "This migration will update tables to use UTF-8 encoding and will update Agent and Event storage from YAML to JSON." + puts "It should work, but please make a backup before proceeding!" + print "Continue? (y/n) " + STDOUT.flush + exit unless STDIN.gets =~ /^y/i - translate YAML, JSON + set_to_utf8 + translate YAML, JSON + end end def down - translate JSON, YAML + if data_exists? + translate JSON, YAML + end + end + + def set_to_utf8 + if mysql? + %w[agent_logs agents delayed_jobs events links taggings tags users].each do |table_name| + quoted_table_name = ActiveRecord::Base.connection.quote_table_name(table_name) + execute "ALTER TABLE #{quoted_table_name} CONVERT TO CHARACTER SET utf8" + end + end + end + + def mysql? + ActiveRecord::Base.connection.adapter_name =~ /mysql/i + end + + def data_exists? + events = ActiveRecord::Base.connection.select_rows("SELECT count(*) FROM #{ActiveRecord::Base.connection.quote_table_name("events")}").first.first + agents = ActiveRecord::Base.connection.select_rows("SELECT count(*) FROM #{ActiveRecord::Base.connection.quote_table_name("agents")}").first.first + agents + events > 0 end def translate(from, to) @@ -29,6 +53,8 @@ class SwitchToJsonSerialization < ActiveRecord::Migration yaml_fields = field_data.map { |f| from.load(f) }.map { |f| to.dump(f) } + yaml_fields.map! {|f| f.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '??') } + update_sql = "UPDATE #{quoted_table_name} SET #{fields.map {|f| "#{f}=?"}.join(", ")} WHERE id = ?" sanitized_update_sql = ActiveRecord::Base.send :sanitize_sql_array, [update_sql, *yaml_fields, id] From bca562513b9aa5533193c5910f8f71fd71bbfc9b Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 26 Dec 2013 18:49:15 -0500 Subject: [PATCH 06/10] Only load the fields we need for the autocomplete. --- app/views/layouts/application.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index fcfc0d56..2ab0e63e 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -36,7 +36,7 @@