globally avoid using symbols since we're moving to json storage

This commit is contained in:
Andrew Cantino 2013-12-24 17:37:26 -05:00
parent ec32d7f979
commit a408ae48ea
36 changed files with 610 additions and 630 deletions

View file

@ -8,11 +8,11 @@ module EmailConcern
end
def validate_email_options
errors.add(:base, "subject and expected_receive_period_in_days are required") unless options[:subject].present? && options[:expected_receive_period_in_days].present?
errors.add(:base, "subject and expected_receive_period_in_days are required") unless options['subject'].present? && options['expected_receive_period_in_days'].present?
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def present(payload)

View file

@ -7,20 +7,20 @@ module TwitterConcern
end
def validate_twitter_options
unless options[:consumer_key].present? &&
options[:consumer_secret].present? &&
options[:oauth_token].present? &&
options[:oauth_token_secret].present?
unless options['consumer_key'].present? &&
options['consumer_secret'].present? &&
options['oauth_token'].present? &&
options['oauth_token_secret'].present?
errors.add(:base, "consumer_key, consumer_secret, oauth_token and oauth_token_secret are required to authenticate with the Twitter API")
end
end
def configure_twitter
Twitter.configure do |config|
config.consumer_key = options[:consumer_key]
config.consumer_secret = options[:consumer_secret]
config.oauth_token = options[:oauth_token] || options[:access_key]
config.oauth_token_secret = options[:oauth_token_secret] || options[:access_secret]
config.consumer_key = options['consumer_key']
config.consumer_secret = options['consumer_secret']
config.oauth_token = options['oauth_token'] || options['access_key']
config.oauth_token_secret = options['oauth_token_secret'] || options['access_secret']
end
end

View file

@ -6,19 +6,19 @@ module WeiboConcern
end
def validate_weibo_options
unless options[:app_key].present? &&
options[:app_secret].present? &&
options[:access_token].present?
unless options['app_key'].present? &&
options['app_secret'].present? &&
options['access_token'].present?
errors.add(:base, "app_key, app_secret and access_token are required")
end
end
def weibo_client
unless @weibo_client
WeiboOAuth2::Config.api_key = options[:app_key] # WEIBO_APP_KEY
WeiboOAuth2::Config.api_secret = options[:app_secret] # WEIBO_APP_SECRET
WeiboOAuth2::Config.api_key = options['app_key'] # WEIBO_APP_KEY
WeiboOAuth2::Config.api_secret = options['app_secret'] # WEIBO_APP_SECRET
@weibo_client = WeiboOAuth2::Client.new
@weibo_client.get_token_from_hash :access_token => options[:access_token]
@weibo_client.get_token_from_hash :access_token => options['access_token']
end
@weibo_client
end

View file

@ -1,14 +1,12 @@
require 'serialize_and_normalize'
require 'json_with_indifferent_access'
require 'assignable_types'
require 'markdown_class_attributes'
require 'utils'
class Agent < ActiveRecord::Base
include SerializeAndNormalize
include AssignableTypes
include MarkdownClassAttributes
serialize_and_normalize :options, :memory
markdown_class_attributes :description, :event_description
load_types_in "Agents"
@ -18,9 +16,21 @@ class Agent < ActiveRecord::Base
attr_accessible :options, :memory, :name, :type, :schedule, :source_ids
serialize :options, JSONWithIndifferentAccess
serialize :memory, JSONWithIndifferentAccess
def options=(o)
self[:options] = ActiveSupport::HashWithIndifferentAccess.new(o)
end
def memory=(o)
self[:memory] = ActiveSupport::HashWithIndifferentAccess.new(o)
end
validates_presence_of :name, :user
validate :sources_are_owned
validate :validate_schedule
validate :validate_options
after_initialize :set_default_schedule
before_validation :set_default_schedule
@ -74,6 +84,10 @@ class Agent < ActiveRecord::Base
raise "Implement me in your subclass"
end
def validate_options
# Implement me in your subclass to test for valid options.
end
def event_created_within(days)
event = most_recent_event
event && event.created_at > days.to_i.days.ago && event.payload.present? && event

View file

@ -29,18 +29,18 @@ module Agents
def default_options
{
:start_date => Date.today.httpdate[0..15],
:end_date => Date.today.plus_with_duration(100).httpdate[0..15],
:from => "New York",
:to => "Chicago",
:username => "xx",
:password => "xx",
:expected_update_period_in_days => "1"
'start_date' => Date.today.httpdate[0..15],
'end_date' => Date.today.plus_with_duration(100).httpdate[0..15],
'from' => "New York",
'to' => "Chicago",
'username' => "xx",
'password' => "xx",
'expected_update_period_in_days' => "1"
}
end
def working?
event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
end
def validate_options
@ -54,9 +54,9 @@ module Agents
end
def check
auth_options = {:basic_auth => {:username =>options[:username], :password=>options[:password]}}
parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options[:from])}+to+#{URI.encode(options[:to])}", auth_options
fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options[:end_date])}\\3#{date_to_unix_epoch(options[:start_date])}"
auth_options = {:basic_auth => {:username =>options[:username], :password=>options['password']}}
parse_response = HTTParty.get "http://api.adioso.com/v2/search/parse?q=#{URI.encode(options['from'])}+to+#{URI.encode(options['to'])}", auth_options
fare_request = parse_response["search_url"].gsub /(end=)(\d*)([^\d]*)(\d*)/, "\\1#{date_to_unix_epoch(options['end_date'])}\\3#{date_to_unix_epoch(options['start_date'])}"
fare = HTTParty.get fare_request, auth_options
if fare["warnings"]
@ -64,7 +64,7 @@ module Agents
else
event = fare["results"].min {|a,b| a["cost"] <=> b["cost"]}
event["date"] = Time.at(event["date"]).to_date.httpdate[0..15]
event["route"] = "#{options[:from]} to #{options[:to]}"
event["route"] = "#{options['from']} to #{options['to']}"
create_event :payload => event
end
end

View file

@ -9,7 +9,7 @@ module Agents
description <<-MD
The DigestEmailAgent collects any Events sent to it and sends them all via email when run.
The email will be sent to your account's address and will have a `subject` and an optional `headline` before
listing the Events. If the Events' payloads contain a `:message`, that will be highlighted, otherwise everything in
listing the Events. If the Events' payloads contain a `message`, that will be highlighted, otherwise everything in
their payloads will be shown.
Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent.
@ -17,29 +17,29 @@ module Agents
def default_options
{
:subject => "You have some notifications!",
:headline => "Your notifications:",
:expected_receive_period_in_days => "2"
'subject' => "You have some notifications!",
'headline' => "Your notifications:",
'expected_receive_period_in_days' => "2"
}
end
def receive(incoming_events)
incoming_events.each do |event|
self.memory[:queue] ||= []
self.memory[:queue] << event.payload
self.memory[:events] ||= []
self.memory[:events] << event.id
self.memory['queue'] ||= []
self.memory['queue'] << event.payload
self.memory['events'] ||= []
self.memory['events'] << event.id
end
end
def check
if self.memory[:queue] && self.memory[:queue].length > 0
ids = self.memory[:events].join(",")
groups = self.memory[:queue].map { |payload| present(payload) }
if self.memory['queue'] && self.memory['queue'].length > 0
ids = self.memory['events'].join(",")
groups = self.memory['queue'].map { |payload| present(payload) }
log "Sending digest mail to #{user.email} with events [#{ids}]"
SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => groups)
self.memory[:queue] = []
self.memory[:events] = []
SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => groups)
self.memory['queue'] = []
self.memory['events'] = []
end
end
end

View file

@ -16,16 +16,16 @@ module Agents
def default_options
{
:subject => "You have a notification!",
:headline => "Your notification:",
:expected_receive_period_in_days => "2"
'subject' => "You have a notification!",
'headline' => "Your notification:",
'expected_receive_period_in_days' => "2"
}
end
def receive(incoming_events)
incoming_events.each do |event|
log "Sending digest mail to #{user.email} with event #{event.id}"
SystemMailer.delay.send_message(:to => user.email, :subject => options[:subject], :headline => options[:headline], :groups => [present(event.payload)])
SystemMailer.delay.send_message(:to => user.email, :subject => options['subject'], :headline => options['headline'], :groups => [present(event.payload)])
end
end
end

View file

@ -8,20 +8,20 @@ module Agents
For example, here is a possible Event:
{
:high => {
:celsius => "18",
:fahreinheit => "64"
"high": {
"celsius": "18",
"fahreinheit": "64"
},
:conditions => "Rain showers",
:data => "This is some data"
"conditions": "Rain showers",
"data": "This is some data"
}
You may want to send this event to another Agent, for example a Twilio Agent, which expects a `message` key.
You can use an Event Formatting Agent's `instructions` setting to do this in the following way:
instructions: {
message: "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.",
subject: "$.data"
"instructions": {
"message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.",
"subject": "$.data"
}
JSONPaths must be between < and > . Make sure that you don't use these symbols anywhere else.
@ -29,8 +29,8 @@ module Agents
Events generated by this possible Event Formatting Agent will look like:
{
:message => "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.",
:subject => "This is some data"
"message": "Today's conditions look like Rain showers with a high temperature of 18 degrees Celsius.",
"subject": "This is some data"
}
If you want to retain original contents of events and only add new keys, then set `mode` to `merge`, otherwise set it to `clean`.
@ -40,25 +40,25 @@ module Agents
To CGI escape output (for example when creating a link), prefix with `escape`, like so:
{
:message => "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=<escape $.group_by>"
"message": "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=<escape $.group_by>"
}
MD
event_description "User defined"
def validate_options
errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options[:instructions].present? and options[:mode].present? and options[:skip_agent].present? and options[:skip_created_at].present?
errors.add(:base, "instructions, mode, skip_agent, and skip_created_at all need to be present.") unless options['instructions'].present? and options['mode'].present? and options['skip_agent'].present? and options['skip_created_at'].present?
end
def default_options
{
:instructions => {
:message => "You received a text <$.text> from <$.fields.from>",
:some_other_field => "Looks like the weather is going to be <$.fields.weather>"
'instructions' => {
'message' => "You received a text <$.text> from <$.fields.from>",
'some_other_field' => "Looks like the weather is going to be <$.fields.weather>"
},
:mode => "clean",
:skip_agent => "false",
:skip_created_at => "false"
'mode' => "clean",
'skip_agent' => "false",
'skip_created_at' => "false"
}
end
@ -68,10 +68,10 @@ module Agents
def receive(incoming_events)
incoming_events.each do |event|
formatted_event = options[:mode].to_s == "merge" ? event.payload : {}
options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true"
formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true"
formatted_event = options['mode'].to_s == "merge" ? event.payload : {}
options['instructions'].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
formatted_event['agent'] = Agent.find(event.agent_id).type.slice!(8..-1) unless options['skip_agent'].to_s == "true"
formatted_event['created_at'] = event.created_at unless options['skip_created_at'].to_s == "true"
create_event :payload => formatted_event
end
end

View file

@ -74,69 +74,69 @@ module Agents
MD
def validate_options
options[:hit] ||= {}
options[:hit][:questions] ||= []
options['hit'] ||= {}
options['hit']['questions'] ||= []
errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on])
errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options[:hit][:assignments].present? && options[:hit][:assignments].to_i > 0
errors.add(:base, "'hit.title' must be provided") unless options[:hit][:title].present?
errors.add(:base, "'hit.description' must be provided") unless options[:hit][:description].present?
errors.add(:base, "'hit.questions' must be provided") unless options[:hit][:questions].present? && options[:hit][:questions].length > 0
errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options['trigger_on'])
errors.add(:base, "'hit.assignments' should specify the number of HIT assignments to create") unless options['hit']['assignments'].present? && options['hit']['assignments'].to_i > 0
errors.add(:base, "'hit.title' must be provided") unless options['hit']['title'].present?
errors.add(:base, "'hit.description' must be provided") unless options['hit']['description'].present?
errors.add(:base, "'hit.questions' must be provided") unless options['hit']['questions'].present? && options['hit']['questions'].length > 0
if options[:trigger_on] == "event"
errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present?
elsif options[:trigger_on] == "schedule"
errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0
if options['trigger_on'] == "event"
errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options['expected_receive_period_in_days'].present?
elsif options['trigger_on'] == "schedule"
errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options['submission_period'].present? && options['submission_period'].to_i > 0
end
if options[:hit][:questions].any? { |question| [:key, :name, :required, :type, :question].any? {|k| !question[k].present? } }
if options['hit']['questions'].any? { |question| %w[key name required type question].any? {|k| !question[k].present? } }
errors.add(:base, "all questions must set 'key', 'name', 'required', 'type', and 'question'")
end
if options[:hit][:questions].any? { |question| question[:type] == "selection" && (!question[:selections].present? || question[:selections].length == 0 || !question[:selections].all? {|s| s[:key].present? } || !question[:selections].all? { |s| s[:text].present? })}
if options['hit']['questions'].any? { |question| question['type'] == "selection" && (!question['selections'].present? || question['selections'].length == 0 || !question['selections'].all? {|s| s['key'].present? } || !question['selections'].all? { |s| s['text'].present? })}
errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
end
if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" }
if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" }
errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
end
end
def default_options
{
:expected_receive_period_in_days => 2,
:trigger_on => "event",
:hit =>
'expected_receive_period_in_days' => 2,
'trigger_on' => "event",
'hit' =>
{
:assignments => 1,
:title => "Sentiment evaluation",
:description => "Please rate the sentiment of this message: '<$.message>'",
:reward => 0.05,
:lifetime_in_seconds => 24 * 60 * 60,
:questions =>
'assignments' => 1,
'title' => "Sentiment evaluation",
'description' => "Please rate the sentiment of this message: '<$.message>'",
'reward' => 0.05,
'lifetime_in_seconds' => 24 * 60 * 60,
'questions' =>
[
{
:type => "selection",
:key => "sentiment",
:name => "Sentiment",
:required => "true",
:question => "Please select the best sentiment value:",
:selections =>
'type' => "selection",
'key' => "sentiment",
'name' => "Sentiment",
'required' => "true",
'question' => "Please select the best sentiment value:",
'selections' =>
[
{ :key => "happy", :text => "Happy" },
{ :key => "sad", :text => "Sad" },
{ :key => "neutral", :text => "Neutral" }
{ 'key' => "happy", 'text' => "Happy" },
{ 'key' => "sad", 'text' => "Sad" },
{ 'key' => "neutral", 'text' => "Neutral" }
]
},
{
:type => "free_text",
:key => "feedback",
:name => "Have any feedback for us?",
:required => "false",
:question => "Feedback",
:default => "Type here...",
:min_length => "2",
:max_length => "2000"
'type' => "free_text",
'key' => "feedback",
'name' => "Have any feedback for us?",
'required' => "false",
'question' => "Feedback",
'default' => "Type here...",
'min_length' => "2",
'max_length' => "2000"
}
]
}
@ -144,20 +144,20 @@ module Agents
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def check
review_hits
if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60
memory[:last_schedule] = Time.now.to_i
if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60
memory['last_schedule'] = Time.now.to_i
create_hit
end
end
def receive(incoming_events)
if options[:trigger_on] == "event"
if options['trigger_on'] == "event"
incoming_events.each do |event|
create_hit event
end
@ -168,7 +168,7 @@ module Agents
def review_hits
reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s)
my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys
if reviewable_hit_ids.length > 0
log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]"
end
@ -178,26 +178,26 @@ module Agents
log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
payload = { :answers => assignments.map(&:answers) }
payload = { 'answers' => assignments.map(&:answers) }
if options[:take_majority] == "true"
if options['take_majority'] == "true"
counts = {}
options[:hit][:questions].each do |question|
question_counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
options['hit']['questions'].each do |question|
question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
assignments.each do |assignment|
answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
answer = answers[question[:key]]
answer = answers[question['key']]
question_counts[answer] += 1
end
counts[question[:key]] = question_counts
counts[question['key']] = question_counts
end
payload[:counts] = counts
payload['counts'] = counts
majority_answer = counts.inject({}) do |memo, (key, question_counts)|
memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
memo
end
payload[:majority_answer] = majority_answer
payload['majority_answer'] = majority_answer
if all_questions_are_numeric?
average_answer = counts.inject({}) do |memo, (key, question_counts)|
@ -209,44 +209,44 @@ module Agents
memo[key] = sum / divisor.to_f
memo
end
payload[:average_answer] = average_answer
payload['average_answer'] = average_answer
end
end
event = create_event :payload => payload
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id])
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id])
assignments.each(&:approve!)
hit.dispose!
memory[:hits].delete(hit_id)
memory['hits'].delete(hit_id)
end
end
end
def all_questions_are_numeric?
options[:hit][:questions].all? do |question|
question[:selections].all? do |selection|
selection[:key] == selection[:key].to_f.to_s || selection[:key] == selection[:key].to_i.to_s
options['hit']['questions'].all? do |question|
question['selections'].all? do |selection|
selection['key'] == selection['key'].to_f.to_s || selection['key'] == selection['key'].to_i.to_s
end
end
end
def create_hit(event = nil)
payload = event ? event.payload : {}
title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip
description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip
questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload)
title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip
description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip
questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload)
hit = RTurk::Hit.create(:title => title) do |hit|
hit.max_assignments = (options[:hit][:assignments] || 1).to_i
hit.max_assignments = (options['hit']['assignments'] || 1).to_i
hit.description = description
hit.lifetime = (options[:hit][:lifetime_in_seconds] || 24 * 60 * 60).to_i
hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i
hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
hit.reward = (options[:hit][:reward] || 0.05).to_f
hit.reward = (options['hit']['reward'] || 0.05).to_f
#hit.qualifications.add :approval_rate, { :gt => 80 }
end
memory[:hits] ||= {}
memory[:hits][hit.id] = event && event.id
memory['hits'] ||= {}
memory['hits'][hit.id] = event && event.id
log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
end
@ -314,7 +314,7 @@ module Agents
end
end
if question[:default].present?
if question['default'].present?
DefaultText do
text question['default']
end

View file

@ -14,8 +14,8 @@ module Agents
end
def handle_details_post(params)
if params[:payload]
create_event(:payload => params[:payload])
if params['payload']
create_event(:payload => params['payload'])
{ :success => true }
else
{ :success => false, :error => "You must provide a JSON payload" }

View file

@ -28,22 +28,22 @@ module Agents
MD
def validate_options
unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:value_path].present?
unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['value_path'].present?
errors.add(:base, "expected_receive_period_in_days, value_path, and message are required")
end
end
def default_options
{
:expected_receive_period_in_days => "2",
:group_by_path => "filter",
:value_path => "count",
:message => "A peak was found"
'expected_receive_period_in_days' => "2",
'group_by_path' => "filter",
'value_path' => "count",
'message' => "A peak was found"
}
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def receive(incoming_events)
@ -57,25 +57,25 @@ module Agents
private
def check_for_peak(group, event)
memory[:peaks] ||= {}
memory[:peaks][group] ||= []
memory['peaks'] ||= {}
memory['peaks'][group] ||= []
if memory[:data][group].length > 4 && (memory[:peaks][group].empty? || memory[:peaks][group].last < event.created_at.to_i - peak_spacing)
if memory['data'][group].length > 4 && (memory['peaks'][group].empty? || memory['peaks'][group].last < event.created_at.to_i - peak_spacing)
average_value, standard_deviation = stats_for(group, :skip_last => 1)
newest_value, newest_time = memory[:data][group][-1].map(&:to_f)
newest_value, newest_time = memory['data'][group][-1].map(&:to_f)
#p [newest_value, average_value, average_value + std_multiple * standard_deviation, standard_deviation]
if newest_value > average_value + std_multiple * standard_deviation
memory[:peaks][group] << newest_time
memory[:peaks][group].reject! { |p| p <= newest_time - window_duration }
create_event :payload => {:message => options[:message], :peak => newest_value, :peak_time => newest_time, :grouped_by => group.to_s}
memory['peaks'][group] << newest_time
memory['peaks'][group].reject! { |p| p <= newest_time - window_duration }
create_event :payload => { 'message' => options['message'], 'peak' => newest_value, 'peak_time' => newest_time, 'grouped_by' => group.to_s }
end
end
end
def stats_for(group, options = {})
data = memory[:data][group].map { |d| d.first.to_f }
data = memory['data'][group].map { |d| d.first.to_f }
data = data[0...(data.length - (options[:skip_last] || 0))]
length = data.length.to_f
mean = 0
@ -94,39 +94,39 @@ module Agents
end
def window_duration
if options[:window_duration].present? # The older option
options[:window_duration].to_i
if options['window_duration'].present? # The older option
options['window_duration'].to_i
else
(options[:window_duration_in_days] || 14).to_f.days
(options['window_duration_in_days'] || 14).to_f.days
end
end
def std_multiple
(options[:std_multiple] || 3).to_f
(options['std_multiple'] || 3).to_f
end
def peak_spacing
if options[:peak_spacing].present? # The older option
options[:peak_spacing].to_i
if options['peak_spacing'].present? # The older option
options['peak_spacing'].to_i
else
(options[:min_peak_spacing_in_days] || 2).to_f.days
(options['min_peak_spacing_in_days'] || 2).to_f.days
end
end
def group_for(event)
((options[:group_by_path].present? && Utils.value_at(event.payload, options[:group_by_path])) || 'no_group')
((options['group_by_path'].present? && Utils.value_at(event.payload, options['group_by_path'])) || 'no_group')
end
def remember(group, event)
memory[:data] ||= {}
memory[:data][group] ||= []
memory[:data][group] << [Utils.value_at(event.payload, options[:value_path]), event.created_at.to_i]
memory['data'] ||= {}
memory['data'][group] ||= []
memory['data'][group] << [ Utils.value_at(event.payload, options['value_path']), event.created_at.to_i ]
cleanup group
end
def cleanup(group)
newest_time = memory[:data][group].last.last
memory[:data][group].reject! { |value, time| time <= newest_time - window_duration }
newest_time = memory['data'][group].last.last
memory['data'][group].reject! { |value, time| time <= newest_time - window_duration }
end
end
end

View file

@ -11,17 +11,17 @@ module Agents
def default_options
{
:post_url => "http://www.example.com",
:expected_receive_period_in_days => 1
'post_url' => "http://www.example.com",
'expected_receive_period_in_days' => 1
}
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def validate_options
unless options[:post_url].present? && options[:expected_receive_period_in_days].present?
unless options['post_url'].present? && options['expected_receive_period_in_days'].present?
errors.add(:base, "post_url and expected_receive_period_in_days are required fields")
end
end

View file

@ -28,31 +28,31 @@ module Agents
def default_options
{
:content => "$.message.text[*]",
:expected_receive_period_in_days => 1
'content' => "$.message.text[*]",
'expected_receive_period_in_days' => 1
}
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def receive(incoming_events)
anew = self.class.sentiment_hash
incoming_events.each do |event|
Utils.values_at(event.payload, options[:content]).each do |content|
Utils.values_at(event.payload, options['content']).each do |content|
sent_values = sentiment_values anew, content
create_event :payload => { :content => content,
:valence => sent_values[0],
:arousal => sent_values[1],
:dominance => sent_values[2],
:original_event => event.payload }
create_event :payload => { 'content' => content,
'valence' => sent_values[0],
'arousal' => sent_values[1],
'dominance' => sent_values[2],
'original_event' => event.payload }
end
end
end
def validate_options
errors.add(:base, "content and expected_receive_period_in_days must be present") unless options[:content].present? && options[:expected_receive_period_in_days].present?
errors.add(:base, "content and expected_receive_period_in_days must be present") unless options['content'].present? && options['expected_receive_period_in_days'].present?
end
def self.sentiment_hash

View file

@ -17,26 +17,26 @@ module Agents
def default_options
{
:client_id => "xxxxxx",
:client_secret => "xxxxxx",
:to => "fi",
:expected_receive_period_in_days => 1,
:content => {
:text => "$.message.text",
:content => "$.xyz"
'client_id' => "xxxxxx",
'client_secret' => "xxxxxx",
'to' => "fi",
'expected_receive_period_in_days' => 1,
'content' => {
'text' => "$.message.text",
'content' => "$.xyz"
}
}
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def translate(text, to, access_token)
translate_uri = URI 'http://api.microsofttranslator.com/v2/Ajax.svc/Translate'
params = {
:text => text,
:to => to
'text' => text,
'to' => to
}
translate_uri.query = URI.encode_www_form params
request = Net::HTTP::Get.new translate_uri.request_uri
@ -47,7 +47,7 @@ module Agents
end
def validate_options
unless options[:client_id].present? && options[:client_secret].present? && options[:to].present? && options[:content].present? && options[:expected_receive_period_in_days].present?
unless options['client_id'].present? && options['client_secret'].present? && options['to'].present? && options['content'].present? && options['expected_receive_period_in_days'].present?
errors.add :base, "client_id,client_secret,to,expected_receive_period_in_days and content are all required"
end
end
@ -60,16 +60,16 @@ module Agents
def receive(incoming_events)
auth_uri = URI "https://datamarket.accesscontrol.windows.net/v2/OAuth2-13"
response = postform auth_uri, :client_id => options[:client_id],
:client_secret => options[:client_secret],
response = postform auth_uri, :client_id => options['client_id'],
:client_secret => options['client_secret'],
:scope => "http://api.microsofttranslator.com",
:grant_type => "client_credentials"
access_token = JSON.parse(response.body)["access_token"]
incoming_events.each do |event|
translated_event = {}
options[:content].each_pair do |key, value|
options['content'].each_pair do |key, value|
to_be_translated = Utils.values_at event.payload, value
translated_event[key] = translate to_be_translated.first, options[:to], access_token
translated_event[key] = translate to_be_translated.first, options['to'], access_token
end
create_event :payload => translated_event
end

View file

@ -23,57 +23,57 @@ module Agents
MD
def validate_options
unless options[:expected_receive_period_in_days].present? && options[:message].present? && options[:rules].present? &&
options[:rules].all? { |rule| rule[:type].present? && VALID_COMPARISON_TYPES.include?(rule[:type]) && rule[:value].present? && rule[:path].present? }
unless options['expected_receive_period_in_days'].present? && options['message'].present? && options['rules'].present? &&
options['rules'].all? { |rule| rule['type'].present? && VALID_COMPARISON_TYPES.include?(rule['type']) && rule['value'].present? && rule['path'].present? }
errors.add(:base, "expected_receive_period_in_days, message, and rules, with a type, value, and path for every rule, are required")
end
end
def default_options
{
:expected_receive_period_in_days => "2",
:rules => [{
:type => "regex",
:value => "foo\\d+bar",
:path => "topkey.subkey.subkey.goal",
}],
:message => "Looks like your pattern matched in '<value>'!"
'expected_receive_period_in_days' => "2",
'rules' => [{
'type' => "regex",
'value' => "foo\\d+bar",
'path' => "topkey.subkey.subkey.goal",
}],
'message' => "Looks like your pattern matched in '<value>'!"
}
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def receive(incoming_events)
incoming_events.each do |event|
match = options[:rules].all? do |rule|
value_at_path = Utils.value_at(event[:payload], rule[:path])
case rule[:type]
match = options['rules'].all? do |rule|
value_at_path = Utils.value_at(event['payload'], rule['path'])
case rule['type']
when "regex"
value_at_path.to_s =~ Regexp.new(rule[:value], Regexp::IGNORECASE)
value_at_path.to_s =~ Regexp.new(rule['value'], Regexp::IGNORECASE)
when "!regex"
value_at_path.to_s !~ Regexp.new(rule[:value], Regexp::IGNORECASE)
value_at_path.to_s !~ Regexp.new(rule['value'], Regexp::IGNORECASE)
when "field>value"
value_at_path.to_f > rule[:value].to_f
value_at_path.to_f > rule['value'].to_f
when "field>=value"
value_at_path.to_f >= rule[:value].to_f
value_at_path.to_f >= rule['value'].to_f
when "field<value"
value_at_path.to_f < rule[:value].to_f
value_at_path.to_f < rule['value'].to_f
when "field<=value"
value_at_path.to_f <= rule[:value].to_f
value_at_path.to_f <= rule['value'].to_f
when "field==value"
value_at_path.to_s == rule[:value].to_s
value_at_path.to_s == rule['value'].to_s
when "field!=value"
value_at_path.to_s != rule[:value].to_s
value_at_path.to_s != rule['value'].to_s
else
raise "Invalid :type of #{rule[:type]} in TriggerAgent##{id}"
raise "Invalid type of #{rule['type']} in TriggerAgent##{id}"
end
end
if match
create_event :payload => { :message => make_message(event[:payload]) } # Maybe this should include the
# original event as well?
create_event :payload => { 'message' => make_message(event[:payload]) } # Maybe this should include the
# original event as well?
end
end
end

View file

@ -9,7 +9,7 @@ module Agents
description <<-MD
The TwilioAgent receives and collects events and sends them via text message or gives you a call when scheduled.
It is assumed that events have a `:message`, `:text`, or `:sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use Event Formatting Agent if your event does not provide these keys.
Set `receiver_cell` to the number to receive text messages/call and `sender_cell` to the number sending them.
@ -22,35 +22,35 @@ module Agents
def default_options
{
:account_sid => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
:auth_token => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
:sender_cell => 'xxxxxxxxxx',
:receiver_cell => 'xxxxxxxxxx',
:server_url => 'http://somename.com:3000',
:receive_text => 'true',
:receive_call => 'false',
:expected_receive_period_in_days => '1'
'account_sid' => 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
'auth_token' => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
'sender_cell' => 'xxxxxxxxxx',
'receiver_cell' => 'xxxxxxxxxx',
'server_url' => 'http://somename.com:3000',
'receive_text' => 'true',
'receive_call' => 'false',
'expected_receive_period_in_days' => '1'
}
end
def validate_options
unless options[:account_sid].present? && options[:auth_token].present? && options[:sender_cell].present? && options[:receiver_cell].present? && options[:expected_receive_period_in_days].present? && options[:receive_call].present? && options[:receive_text].present?
unless options['account_sid'].present? && options['auth_token'].present? && options['sender_cell'].present? && options['receiver_cell'].present? && options['expected_receive_period_in_days'].present? && options['receive_call'].present? && options['receive_text'].present?
errors.add(:base, 'account_sid, auth_token, sender_cell, receiver_cell, receive_text, receive_call and expected_receive_period_in_days are all required')
end
end
def receive(incoming_events)
@client = Twilio::REST::Client.new options[:account_sid], options[:auth_token]
memory[:pending_calls] ||= {}
@client = Twilio::REST::Client.new options['account_sid'], options['auth_token']
memory['pending_calls'] ||= {}
incoming_events.each do |event|
message = (event.payload[:message] || event.payload[:text] || event.payload[:sms]).to_s
message = (event.payload['message'] || event.payload['text'] || event.payload['sms']).to_s
if message != ""
if options[:receive_call].to_s == 'true'
if options['receive_call'].to_s == 'true'
secret = SecureRandom.hex 3
memory[:pending_calls][secret] = message
memory['pending_calls'][secret] = message
make_call secret
end
if options[:receive_text].to_s == 'true'
if options['receive_text'].to_s == 'true'
message = message.slice 0..160
send_message message
end
@ -59,19 +59,19 @@ module Agents
end
def working?
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago && !recent_error_logs?
end
def send_message(message)
@client.account.sms.messages.create :from => options[:sender_cell],
:to => options[:receiver_cell],
@client.account.sms.messages.create :from => options['sender_cell'],
:to => options['receiver_cell'],
:body => message
end
def make_call(secret)
@client.account.calls.create :from => options[:sender_cell],
:to => options[:receiver_cell],
:url => post_url(options[:server_url],secret)
@client.account.calls.create :from => options['sender_cell'],
:to => options['receiver_cell'],
:url => post_url(options['server_url'],secret)
end
def post_url(server_url,secret)
@ -79,9 +79,9 @@ module Agents
end
def receive_webhook(params)
if memory[:pending_calls].has_key? params[:secret]
response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret]], :voice => 'woman'}
memory[:pending_calls].delete params[:secret]
if memory['pending_calls'].has_key? params['secret']
response = Twilio::TwiML::Response.new {|r| r.Say memory['pending_calls'][params['secret']], :voice => 'woman'}
memory['pending_calls'].delete params['secret']
[response.text, 200]
end
end

View file

@ -19,25 +19,25 @@ module Agents
MD
def validate_options
unless options[:username].present? &&
options[:expected_update_period_in_days].present?
unless options['username'].present? &&
options['expected_update_period_in_days'].present?
errors.add(:base, "username and expected_update_period_in_days are required")
end
end
def working?
(event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs?
(event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
end
def default_options
{
:username => "",
:expected_update_period_in_days => "10",
:consumer_key => "---",
:consumer_secret => "---",
:oauth_token => "---",
:oauth_token_secret => "---",
:message_path => "text"
'username' => "",
'expected_update_period_in_days' => "10",
'consumer_key' => "---",
'consumer_secret' => "---",
'oauth_token' => "---",
'oauth_token_secret' => "---",
'message_path' => "text"
}
end
@ -47,22 +47,22 @@ module Agents
incoming_events = incoming_events.first(20)
end
incoming_events.each do |event|
tweet_text = Utils.value_at(event.payload, options[:message_path])
tweet_text = Utils.value_at(event.payload, options['message_path'])
begin
publish_tweet tweet_text
create_event :payload => {
:success => true,
:published_tweet => tweet_text,
:agent_id => event.agent_id,
:event_id => event.id
'success' => true,
'published_tweet' => tweet_text,
'agent_id' => event.agent_id,
'event_id' => event.id
}
rescue Twitter::Error => e
create_event :payload => {
:success => false,
:error => e.message,
:failed_tweet => tweet_text,
:agent_id => event.agent_id,
:event_id => event.id
'success' => false,
'error' => e.message,
'failed_tweet' => tweet_text,
'agent_id' => event.agent_id,
'event_id' => event.id
}
end
end

View file

@ -54,26 +54,26 @@ module Agents
default_schedule "11pm"
def validate_options
unless options[:filters].present? &&
options[:expected_update_period_in_days].present? &&
options[:generate].present?
unless options['filters'].present? &&
options['expected_update_period_in_days'].present? &&
options['generate'].present?
errors.add(:base, "expected_update_period_in_days, generate, and filters are required fields")
end
end
def working?
event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
end
def default_options
{
:consumer_key => "---",
:consumer_secret => "---",
:oauth_token => "---",
:oauth_token_secret => "---",
:filters => %w[keyword1 keyword2],
:expected_update_period_in_days => "2",
:generate => "events"
'consumer_key' => "---",
'consumer_secret' => "---",
'oauth_token' => "---",
'oauth_token_secret' => "---",
'filters' => %w[keyword1 keyword2],
'expected_update_period_in_days' => "2",
'generate' => "events"
}
end
@ -81,33 +81,33 @@ module Agents
filter = lookup_filter(filter)
if filter
if options[:generate] == "counts"
if options['generate'] == "counts"
# Avoid memory pollution by reloading the Agent.
agent = Agent.find(id)
agent.memory[:filter_counts] ||= {}
agent.memory[:filter_counts][filter] ||= 0
agent.memory[:filter_counts][filter] += 1
remove_unused_keys!(agent, :filter_counts)
agent.memory['filter_counts'] ||= {}
agent.memory['filter_counts'][filter] ||= 0
agent.memory['filter_counts'][filter] += 1
remove_unused_keys!(agent, 'filter_counts')
agent.save!
else
create_event :payload => status.merge(:filter => filter)
create_event :payload => status.merge('filter' => filter)
end
end
end
def check
if options[:generate] == "counts" && memory[:filter_counts] && memory[:filter_counts].length > 0
memory[:filter_counts].each do |filter, count|
create_event :payload => { :filter => filter, :count => count, :time => Time.now.to_i }
if options['generate'] == "counts" && memory['filter_counts'] && memory['filter_counts'].length > 0
memory['filter_counts'].each do |filter, count|
create_event :payload => { 'filter' => filter, 'count' => count, 'time' => Time.now.to_i }
end
end
memory[:filter_counts] = {}
memory['filter_counts'] = {}
end
protected
def lookup_filter(filter)
options[:filters].each do |known_filter|
options['filters'].each do |known_filter|
if known_filter == filter
return filter
elsif known_filter.is_a?(Array)
@ -120,7 +120,7 @@ module Agents
def remove_unused_keys!(agent, base)
if agent.memory[base]
(agent.memory[base].keys - agent.options[:filters].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
(agent.memory[base].keys - agent.options['filters'].map {|f| f.is_a?(Array) ? f.first.to_s : f.to_s }).each do |removed_key|
agent.memory[base].delete(removed_key)
end
end

View file

@ -41,36 +41,36 @@ module Agents
default_schedule "every_1h"
def validate_options
unless options[:username].present? &&
options[:expected_update_period_in_days].present?
unless options['username'].present? &&
options['expected_update_period_in_days'].present?
errors.add(:base, "username and expected_update_period_in_days are required")
end
end
def working?
event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
end
def default_options
{
:username => "tectonic",
:expected_update_period_in_days => "2",
:consumer_key => "---",
:consumer_secret => "---",
:oauth_token => "---",
:oauth_token_secret => "---"
'username' => "tectonic",
'expected_update_period_in_days' => "2",
'consumer_key' => "---",
'consumer_secret' => "---",
'oauth_token' => "---",
'oauth_token_secret' => "---"
}
end
def check
since_id = memory[:since_id] || nil
since_id = memory['since_id'] || nil
opts = {:count => 200, :include_rts => true, :exclude_replies => false, :include_entities => true, :contributor_details => true}
opts.merge! :since_id => since_id unless since_id.nil?
tweets = Twitter.user_timeline(options[:username], opts)
tweets = Twitter.user_timeline(options['username'], opts)
tweets.each do |tweet|
memory[:since_id] = tweet.id if !memory[:since_id] || (tweet.id > memory[:since_id])
memory['since_id'] = tweet.id if !memory['since_id'] || (tweet.id > memory['since_id'])
create_event :payload => tweet.attrs
end

View file

@ -34,11 +34,11 @@ module Agents
end
def default_options
{ :secret => SecureRandom.hex(7) }
{ 'secret' => SecureRandom.hex(7) }
end
def validate_options
errors.add(:base, "secret is required and must be longer than 4 characters") unless options[:secret].present? && options[:secret].length > 4
errors.add(:base, "secret is required and must be longer than 4 characters") unless options['secret'].present? && options['secret'].length > 4
end
end
end

View file

@ -45,30 +45,30 @@ module Agents
end
def wunderground
Wunderground.new(options[:api_key]) if key_setup?
Wunderground.new(options['api_key']) if key_setup?
end
def key_setup?
options[:api_key] && options[:api_key] != "your-key"
options['api_key'] && options['api_key'] != "your-key"
end
def default_options
{
:api_key => "your-key",
:location => "94103"
'api_key' => "your-key",
'location' => "94103"
}
end
def validate_options
errors.add(:base, "location is required") unless options[:location].present? || options[:zipcode].present?
errors.add(:base, "api_key is required") unless options[:api_key].present?
errors.add(:base, "location is required") unless options['location'].present? || options['zipcode'].present?
errors.add(:base, "api_key is required") unless options['api_key'].present?
end
def check
if key_setup?
wunderground.forecast_for(options[:location] || options[:zipcode])["forecast"]["simpleforecast"]["forecastday"].each do |day|
wunderground.forecast_for(options['location'] || options['zipcode'])["forecast"]["simpleforecast"]["forecastday"].each do |day|
if is_tomorrow?(day)
create_event :payload => day.merge(:location => options[:location] || options[:zipcode])
create_event :payload => day.merge('location' => options['location'] || options['zipcode'])
end
end
end

View file

@ -15,19 +15,19 @@ module Agents
To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
When parsing HTML or XML, these sub-hashes specify how to extract with a `:css` CSS selector and either `:text => true` or `attr` pointing to an attribute name to grab. An example:
When parsing HTML or XML, these sub-hashes specify how to extract with a `css` CSS selector and either `'text': true` or `attr` pointing to an attribute name to grab. An example:
:extract => {
:url => { :css => "#comic img", :attr => "src" },
:title => { :css => "#comic img", :attr => "title" },
:body_text => { :css => "div.main", :text => true }
'extract': {
'url': { 'css': "#comic img", 'attr': "src" },
'title': { 'css': "#comic img", 'attr': "title" },
'body_text': { 'css': "div.main", 'text': true }
}
When parsing JSON, these sub-hashes specify [JSONPaths](http://goessner.net/articles/JsonPath/) to the values that you care about. For example:
:extract => {
:title => { :path => "results.data[*].title" },
:description => { :path => "results.data[*].description" }
'extract': {
'title': { 'path': "results.data[*].title" },
'description': { 'path': "results.data[*].description" }
}
Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
@ -36,7 +36,7 @@ module Agents
MD
event_description do
"Events will have the fields you specified. Your options look like:\n\n #{Utils.pretty_print options[:extract]}"
"Events will have the fields you specified. Your options look like:\n\n #{Utils.pretty_print options['extract']}"
end
default_schedule "every_12h"
@ -44,33 +44,33 @@ module Agents
UNIQUENESS_LOOK_BACK = 30
def working?
event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
end
def default_options
{
:expected_update_period_in_days => "2",
:url => "http://xkcd.com",
:type => "html",
:mode => :on_change,
:extract => {
:url => {:css => "#comic img", :attr => "src"},
:title => {:css => "#comic img", :attr => "title"}
'expected_update_period_in_days' => "2",
'url' => "http://xkcd.com",
'type' => "html",
'mode' => :on_change,
'extract' => {
'url' => {'css' => "#comic img", 'attr' => "src"},
'title' => {'css' => "#comic img", 'attr' => "title"}
}
}
end
def validate_options
errors.add(:base, "url and expected_update_period_in_days are required") unless options[:expected_update_period_in_days].present? && options[:url].present?
if !options[:extract].present? && extraction_type != "json"
errors.add(:base, "url and expected_update_period_in_days are required") unless options['expected_update_period_in_days'].present? && options['url'].present?
if !options['extract'].present? && extraction_type != "json"
errors.add(:base, "extract is required for all types except json")
end
end
def check
hydra = Typhoeus::Hydra.new
log "Fetching #{options[:url]}"
request = Typhoeus::Request.new(options[:url], :followlocation => true)
log "Fetching #{options['url']}"
request = Typhoeus::Request.new(options['url'], :followlocation => true)
request.on_failure do |response|
error "Failed: #{response.inspect}"
end
@ -85,37 +85,37 @@ module Agents
end
else
output = {}
options[:extract].each do |name, extraction_details|
options['extract'].each do |name, extraction_details|
result = if extraction_type == "json"
output[name] = Utils.values_at(doc, extraction_details[:path])
output[name] = Utils.values_at(doc, extraction_details['path'])
else
output[name] = doc.css(extraction_details[:css]).map { |node|
if extraction_details[:attr]
node.attr(extraction_details[:attr])
elsif extraction_details[:text]
output[name] = doc.css(extraction_details['css']).map { |node|
if extraction_details['attr']
node.attr(extraction_details['attr'])
elsif extraction_details['text']
node.text()
else
error ":attr or :text is required on HTML or XML extraction patterns"
error "'attr' or 'text' is required on HTML or XML extraction patterns"
return
end
}
end
log "Extracting #{extraction_type} at #{extraction_details[:path] || extraction_details[:css]}: #{result}"
log "Extracting #{extraction_type} at #{extraction_details['path'] || extraction_details['css']}: #{result}"
end
num_unique_lengths = options[:extract].keys.map { |name| output[name].length }.uniq
num_unique_lengths = options['extract'].keys.map { |name| output[name].length }.uniq
if num_unique_lengths.length != 1
error "Got an uneven number of matches for #{options[:name]}: #{options[:extract].inspect}"
error "Got an uneven number of matches for #{options['name']}: #{options['extract'].inspect}"
return
end
num_unique_lengths.first.times do |index|
result = {}
options[:extract].keys.each do |name|
options['extract'].keys.each do |name|
result[name] = output[name][index]
if name.to_s == 'url'
result[name] = URI.join(options[:url], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
result[name] = URI.join(options['url'], result[name]).to_s if (result[name] =~ URI::DEFAULT_PARSER.regexp[:ABS_URI]).nil?
end
end
@ -133,22 +133,22 @@ module Agents
private
def store_payload? result
!options[:mode] || options[:mode].to_s == "all" || (options[:mode].to_s == "on_change" && !previous_payloads.include?(result.to_json))
!options['mode'] || options['mode'].to_s == "all" || (options['mode'].to_s == "on_change" && !previous_payloads.include?(result.to_json))
end
def previous_payloads
events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options[:mode].to_s == "on_change"
events.order("id desc").limit(UNIQUENESS_LOOK_BACK).pluck(:payload).map(&:to_json) if options['mode'].to_s == "on_change"
end
def extract_full_json?
(!options[:extract].present? && extraction_type == "json")
(!options['extract'].present? && extraction_type == "json")
end
def extraction_type
(options[:type] || begin
if options[:url] =~ /\.(rss|xml)$/i
(options['type'] || begin
if options['url'] =~ /\.(rss|xml)$/i
"xml"
elsif options[:url] =~ /\.json$/i
elsif options['url'] =~ /\.json$/i
"json"
else
"html"

View file

@ -20,24 +20,24 @@ module Agents
MD
def validate_options
unless options[:uid].present? &&
options[:expected_update_period_in_days].present?
unless options['uid'].present? &&
options['expected_update_period_in_days'].present?
errors.add(:base, "expected_update_period_in_days and uid are required")
end
end
def working?
(event = event_created_within(options[:expected_update_period_in_days])) && event.payload[:success] == true && !recent_error_logs?
(event = event_created_within(options['expected_update_period_in_days'])) && event.payload['success'] == true && !recent_error_logs?
end
def default_options
{
:uid => "",
:access_token => "---",
:app_key => "---",
:app_secret => "---",
:expected_update_period_in_days => "10",
:message_path => "text"
'uid' => "",
'access_token' => "---",
'app_key' => "---",
'app_secret' => "---",
'expected_update_period_in_days' => "10",
'message_path' => "text"
}
end
@ -47,25 +47,25 @@ module Agents
incoming_events = incoming_events.first(20)
end
incoming_events.each do |event|
tweet_text = Utils.value_at(event.payload, options[:message_path])
tweet_text = Utils.value_at(event.payload, options['message_path'])
if event.agent.type == "Agents::TwitterUserAgent"
tweet_text = unwrap_tco_urls(tweet_text, event.payload)
end
begin
publish_tweet tweet_text
create_event :payload => {
:success => true,
:published_tweet => tweet_text,
:agent_id => event.agent_id,
:event_id => event.id
'success' => true,
'published_tweet' => tweet_text,
'agent_id' => event.agent_id,
'event_id' => event.id
}
rescue OAuth2::Error => e
create_event :payload => {
:success => false,
:error => e.message,
:failed_tweet => tweet_text,
:agent_id => event.agent_id,
:event_id => event.id
'success' => false,
'error' => e.message,
'failed_tweet' => tweet_text,
'agent_id' => event.agent_id,
'event_id' => event.id
}
end
end

View file

@ -70,29 +70,29 @@ module Agents
default_schedule "every_1h"
def validate_options
unless options[:uid].present? &&
options[:expected_update_period_in_days].present?
unless options['uid'].present? &&
options['expected_update_period_in_days'].present?
errors.add(:base, "expected_update_period_in_days and uid are required")
end
end
def working?
event_created_within(options[:expected_update_period_in_days]) && !recent_error_logs?
event_created_within(options['expected_update_period_in_days']) && !recent_error_logs?
end
def default_options
{
:uid => "",
:access_token => "---",
:app_key => "---",
:app_secret => "---",
:expected_update_period_in_days => "2"
'uid' => "",
'access_token' => "---",
'app_key' => "---",
'app_secret' => "---",
'expected_update_period_in_days' => "2"
}
end
def check
since_id = memory[:since_id] || nil
opts = {:uid => options[:uid].to_i}
since_id = memory['since_id'] || nil
opts = {:uid => options['uid'].to_i}
opts.merge! :since_id => since_id unless since_id.nil?
# http://open.weibo.com/wiki/2/statuses/user_timeline/en
@ -101,7 +101,7 @@ module Agents
resp[:statuses].each do |status|
memory[:since_id] = status.id if !memory[:since_id] || (status.id > memory[:since_id])
memory['since_id'] = status.id if !memory['since_id'] || (status.id > memory['since_id'])
create_event :payload => status.as_json
end

View file

@ -1,13 +1,16 @@
require 'serialize_and_normalize'
require 'json_with_indifferent_access'
class Event < ActiveRecord::Base
include SerializeAndNormalize
attr_accessible :lat, :lng, :payload, :user_id, :user, :expires_at
acts_as_mappable
serialize_and_normalize :payload
serialize :payload, JSONWithIndifferentAccess
def payload=(o)
self[:payload] = ActiveSupport::HashWithIndifferentAccess.new(o)
end
belongs_to :user
belongs_to :agent, :counter_cache => true

View file

@ -0,0 +1,9 @@
class JSONWithIndifferentAccess
def self.load(json)
ActiveSupport::HashWithIndifferentAccess.new(JSON.load(json || '{}'))
end
def self.dump(hash)
JSON.dump(hash)
end
end

View file

@ -1,46 +0,0 @@
module SerializeAndNormalize
extend ActiveSupport::Concern
module ClassMethods
def serialize_and_normalize(*column_names)
column_names.flatten.uniq.compact.map(&:to_sym).each do |column_name|
setup_name = "setup_#{column_name}".to_sym
normalize_name = "normalize_#{column_name}".to_sym
validate_name = "validate_#{column_name}".to_sym
serialize column_name, JSON
after_initialize setup_name
before_validation normalize_name
before_save normalize_name
validate validate_name
class_eval <<-RUBY
def #{setup_name}
self[:#{column_name}] ||= ActiveSupport::HashWithIndifferentAccess.new
end
def #{validate_name}
# Implement me in your subclass.
end
def #{normalize_name}
self.#{column_name} = self[:#{column_name}]
end
def #{column_name}=(data)
data = (JSON.parse(data) rescue data) if data.is_a?(String)
case data
when ActiveSupport::HashWithIndifferentAccess
self[:#{column_name}] = data
when Hash
self[:#{column_name}] = ActiveSupport::HashWithIndifferentAccess.new(data)
else
self[:#{column_name}] = data
end
end
RUBY
end
end
end
end

View file

@ -5,7 +5,7 @@ describe AgentsController do
{
:type => "Agents::WebsiteAgent",
:name => "Something",
:options => agents(:bob_website_agent).options.to_json,
:options => agents(:bob_website_agent).options,
:source_ids => [agents(:bob_weather_agent).id, ""]
}.merge(options)
end

View file

@ -11,7 +11,7 @@ jane_website_agent:
:title => {:css => "item title", :text => true},
:url => {:css => "item link", :text => true}
}
}.to_yaml.inspect %>
}.to_json.inspect %>
bob_website_agent:
type: Agents::WebsiteAgent
@ -26,7 +26,7 @@ bob_website_agent:
:url => {:css => "#comic img", :attr => "src"},
:title => {:css => "#comic img", :attr => "title"}
}
}.to_yaml.inspect %>
}.to_json.inspect %>
bob_weather_agent:
type: Agents::WeatherAgent
@ -38,7 +38,7 @@ bob_weather_agent:
:lat => 37.779329,
:lng => -122.41915,
:api_key => 'test'
}.to_yaml.inspect %>
}.to_json.inspect %>
jane_weather_agent:
type: Agents::WeatherAgent
@ -50,7 +50,7 @@ jane_weather_agent:
:lat => 37.779329,
:lng => -122.41915,
:api_key => 'test'
}.to_yaml.inspect %>
}.to_json.inspect %>
jane_rain_notifier_agent:
type: Agents::TriggerAgent
@ -64,7 +64,7 @@ jane_rain_notifier_agent:
:path => "conditions"
}],
:message => "Just so you know, it looks like '<conditions>' tomorrow in <location>"
}.to_yaml.inspect %>
}.to_json.inspect %>
bob_rain_notifier_agent:
type: Agents::TriggerAgent
@ -78,7 +78,7 @@ bob_rain_notifier_agent:
:path => "conditions"
}],
:message => "Just so you know, it looks like '<conditions>' tomorrow in <location>"
}.to_yaml.inspect %>
}.to_json.inspect %>
bob_twitter_user_agent:
type: Agents::TwitterUserAgent
@ -91,7 +91,7 @@ bob_twitter_user_agent:
:consumer_secret => "---",
:oauth_token => "---",
:oauth_token_secret => "---"
}.to_yaml.inspect %>
}.to_json.inspect %>
bob_manual_event_agent:
type: Agents::ManualEventAgent

View file

@ -1,9 +1,9 @@
bob_website_agent_event:
user: bob
agent: bob_website_agent
payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %>
payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>
jane_website_agent_event:
user: jane
agent: jane_website_agent
payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_yaml.inspect %>
payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>

View file

@ -261,9 +261,9 @@ describe Agent do
it "symbolizes memory before validating" do
agent = Agents::SomethingSource.new(:name => "something")
agent.user = users(:bob)
agent.memory["bad"] = :hello
agent.memory["bad"] = 2
agent.save
agent.memory[:bad].should == :hello
agent.memory[:bad].should == 2
end
it "should not allow agents owned by other people" do

View file

@ -19,16 +19,16 @@ describe Agents::DigestEmailAgent do
it "queues any payloads it receives" do
event1 = Event.new
event1.agent = agents(:bob_rain_notifier_agent)
event1.payload = "Something you should know about"
event1.payload = { :data => "Something you should know about" }
event1.save!
event2 = Event.new
event2.agent = agents(:bob_weather_agent)
event2.payload = "Something else you should know about"
event2.payload = { :data => "Something else you should know about" }
event2.save!
Agents::DigestEmailAgent.async_receive(@checker.id, [event1.id, event2.id])
@checker.reload.memory[:queue].should == ["Something you should know about", "Something else you should know about"]
@checker.reload.memory[:queue].should == [{ 'data' => "Something you should know about" }, { 'data' => "Something else you should know about" }]
end
end
@ -37,7 +37,7 @@ describe Agents::DigestEmailAgent do
Agents::DigestEmailAgent.async_check(@checker.id)
ActionMailer::Base.deliveries.should == []
@checker.memory[:queue] = ["Something you should know about",
@checker.memory[:queue] = [{ :data => "Something you should know about" },
{ :title => "Foo", :url => "http://google.com", :bar => 2 },
{ "message" => "hi", :woah => "there" },
{ "test" => 2 }]
@ -47,7 +47,7 @@ describe Agents::DigestEmailAgent do
Agents::DigestEmailAgent.async_check(@checker.id)
ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"]
ActionMailer::Base.deliveries.last.subject.should == "something interesting"
get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something you should know about\n\nFoo\n bar: 2\n url: http://google.com\n\nhi\n woah: there\n\nEvent\n test: 2"
get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n data: Something you should know about\n\nFoo\n bar: 2\n url: http://google.com\n\nhi\n woah: there\n\nEvent\n test: 2"
@checker.reload.memory[:queue].should be_empty
end

View file

@ -21,12 +21,12 @@ describe Agents::EmailAgent do
event1 = Event.new
event1.agent = agents(:bob_rain_notifier_agent)
event1.payload = "Something you should know about"
event1.payload = { :data => "Something you should know about" }
event1.save!
event2 = Event.new
event2.agent = agents(:bob_weather_agent)
event2.payload = "Something else you should know about"
event2.payload = { :data => "Something else you should know about" }
event2.save!
Agents::EmailAgent.async_receive(@checker.id, [event1.id])
@ -35,8 +35,8 @@ describe Agents::EmailAgent do
ActionMailer::Base.deliveries.count.should == 2
ActionMailer::Base.deliveries.last.to.should == ["bob@example.com"]
ActionMailer::Base.deliveries.last.subject.should == "something interesting"
get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Something else you should know about"
get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Something you should know about"
get_message_part(ActionMailer::Base.deliveries.last, /plain/).strip.should == "Event\n data: Something else you should know about"
get_message_part(ActionMailer::Base.deliveries.first, /plain/).strip.should == "Event\n data: Something you should know about"
end
it "can receive complex events and send them on" do

View file

@ -9,8 +9,8 @@ describe Agents::HumanTaskAgent do
@event = Event.new
@event.agent = agents(:bob_rain_notifier_agent)
@event.payload = { :foo => { "bar" => { :baz => "a2b" } },
:name => "Joe" }
@event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" } },
'name' => "Joe" }
@event.id = 345
@checker.should be_valid
@ -18,110 +18,110 @@ describe Agents::HumanTaskAgent do
describe "validations" do
it "validates that trigger_on is 'schedule' or 'event'" do
@checker.options[:trigger_on] = "foo"
@checker.options['trigger_on'] = "foo"
@checker.should_not be_valid
end
it "requires expected_receive_period_in_days when trigger_on is set to 'event'" do
@checker.options[:trigger_on] = "event"
@checker.options[:expected_receive_period_in_days] = nil
@checker.options['trigger_on'] = "event"
@checker.options['expected_receive_period_in_days'] = nil
@checker.should_not be_valid
@checker.options[:expected_receive_period_in_days] = 2
@checker.options['expected_receive_period_in_days'] = 2
@checker.should be_valid
end
it "requires a positive submission_period when trigger_on is set to 'schedule'" do
@checker.options[:trigger_on] = "schedule"
@checker.options[:submission_period] = nil
@checker.options['trigger_on'] = "schedule"
@checker.options['submission_period'] = nil
@checker.should_not be_valid
@checker.options[:submission_period] = 2
@checker.options['submission_period'] = 2
@checker.should be_valid
end
it "requires a hit.title" do
@checker.options[:hit][:title] = ""
@checker.options['hit']['title'] = ""
@checker.should_not be_valid
end
it "requires a hit.description" do
@checker.options[:hit][:description] = ""
@checker.options['hit']['description'] = ""
@checker.should_not be_valid
end
it "requires hit.assignments" do
@checker.options[:hit][:assignments] = ""
@checker.options['hit']['assignments'] = ""
@checker.should_not be_valid
@checker.options[:hit][:assignments] = 0
@checker.options['hit']['assignments'] = 0
@checker.should_not be_valid
@checker.options[:hit][:assignments] = "moose"
@checker.options['hit']['assignments'] = "moose"
@checker.should_not be_valid
@checker.options[:hit][:assignments] = "2"
@checker.options['hit']['assignments'] = "2"
@checker.should be_valid
end
it "requires hit.questions" do
old_questions = @checker.options[:hit][:questions]
@checker.options[:hit][:questions] = nil
old_questions = @checker.options['hit']['questions']
@checker.options['hit']['questions'] = nil
@checker.should_not be_valid
@checker.options[:hit][:questions] = []
@checker.options['hit']['questions'] = []
@checker.should_not be_valid
@checker.options[:hit][:questions] = [old_questions[0]]
@checker.options['hit']['questions'] = [old_questions[0]]
@checker.should be_valid
end
it "requires that all questions have key, name, required, type, and question" do
old_questions = @checker.options[:hit][:questions]
@checker.options[:hit][:questions].first[:key] = ""
old_questions = @checker.options['hit']['questions']
@checker.options['hit']['questions'].first['key'] = ""
@checker.should_not be_valid
@checker.options[:hit][:questions] = old_questions
@checker.options[:hit][:questions].first[:name] = ""
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['name'] = ""
@checker.should_not be_valid
@checker.options[:hit][:questions] = old_questions
@checker.options[:hit][:questions].first[:required] = nil
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['required'] = nil
@checker.should_not be_valid
@checker.options[:hit][:questions] = old_questions
@checker.options[:hit][:questions].first[:type] = ""
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['type'] = ""
@checker.should_not be_valid
@checker.options[:hit][:questions] = old_questions
@checker.options[:hit][:questions].first[:question] = ""
@checker.options['hit']['questions'] = old_questions
@checker.options['hit']['questions'].first['question'] = ""
@checker.should_not be_valid
end
it "requires that all questions of type 'selection' have a selections array with keys and text" do
@checker.options[:hit][:questions][0][:selections] = []
@checker.options['hit']['questions'][0]['selections'] = []
@checker.should_not be_valid
@checker.options[:hit][:questions][0][:selections] = [{}]
@checker.options['hit']['questions'][0]['selections'] = [{}]
@checker.should_not be_valid
@checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "" }]
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "" }]
@checker.should_not be_valid
@checker.options[:hit][:questions][0][:selections] = [{ :key => "", :text => "hi" }]
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "", 'text' => "hi" }]
@checker.should_not be_valid
@checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "" }]
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "" }]
@checker.should_not be_valid
@checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }]
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }]
@checker.should be_valid
@checker.options[:hit][:questions][0][:selections] = [{ :key => "hi", :text => "hi" }, {}]
@checker.options['hit']['questions'][0]['selections'] = [{ 'key' => "hi", 'text' => "hi" }, {}]
@checker.should_not be_valid
end
it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
@checker.options[:take_majority] = "true"
@checker.options['take_majority'] = "true"
@checker.should_not be_valid
@checker.options[:hit][:questions][1][:type] = "selection"
@checker.options[:hit][:questions][1][:selections] = @checker.options[:hit][:questions][0][:selections]
@checker.options['hit']['questions'][1]['type'] = "selection"
@checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
@checker.should be_valid
end
end
describe "when 'trigger_on' is set to 'schedule'" do
before do
@checker.options[:trigger_on] = "schedule"
@checker.options[:submission_period] = "2"
@checker.options.delete(:expected_receive_period_in_days)
@checker.options['trigger_on'] = "schedule"
@checker.options['submission_period'] = "2"
@checker.options.delete('expected_receive_period_in_days')
end
it "should check for reviewable HITs frequently" do
@ -151,7 +151,7 @@ describe Agents::HumanTaskAgent do
describe "when 'trigger_on' is set to 'event'" do
it "should not create HITs during check but should check for reviewable HITs" do
@checker.options[:submission_period] = "2"
@checker.options['submission_period'] = "2"
now = Time.now
stub(Time).now { now }
mock(@checker).review_hits.times(3)
@ -171,9 +171,9 @@ describe Agents::HumanTaskAgent do
describe "creating hits" do
it "can create HITs based on events, interpolating their values" do
@checker.options[:hit][:title] = "Hi <.name>"
@checker.options[:hit][:description] = "Make something for <.name>"
@checker.options[:hit][:questions][0][:name] = "<.name> Question 1"
@checker.options['hit']['title'] = "Hi <.name>"
@checker.options['hit']['description'] = "Make something for <.name>"
@checker.options['hit']['questions'][0]['name'] = "<.name> Question 1"
question_form = nil
hitInterface = OpenStruct.new
@ -183,8 +183,8 @@ describe Agents::HumanTaskAgent do
@checker.send :create_hit, @event
hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
hitInterface.reward.should == @checker.options[:hit][:reward]
hitInterface.max_assignments.should == @checker.options['hit']['assignments']
hitInterface.reward.should == @checker.options['hit']['reward']
hitInterface.description.should == "Make something for Joe"
xml = question_form.to_xml
@ -192,18 +192,18 @@ describe Agents::HumanTaskAgent do
xml.should include("<Text>Make something for Joe</Text>")
xml.should include("<DisplayName>Joe Question 1</DisplayName>")
@checker.memory[:hits][123].should == @event.id
@checker.memory['hits'][123].should == @event.id
end
it "works without an event too" do
@checker.options[:hit][:title] = "Hi <.name>"
@checker.options['hit']['title'] = "Hi <.name>"
hitInterface = OpenStruct.new
hitInterface.id = 123
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
@checker.send :create_hit
hitInterface.max_assignments.should == @checker.options[:hit][:assignments]
hitInterface.reward.should == @checker.options[:hit][:reward]
hitInterface.max_assignments.should == @checker.options['hit']['assignments']
hitInterface.reward.should == @checker.options['hit']['reward']
end
end
@ -253,14 +253,14 @@ describe Agents::HumanTaskAgent do
it "should work on multiple HITs" do
event2 = Event.new
event2.agent = agents(:bob_rain_notifier_agent)
event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } },
:name2 => "Joe2" }
event2.payload = { 'foo2' => { "bar2" => { 'baz2' => "a2b2" } },
'name2' => "Joe2" }
event2.id = 3452
# It knows about two HITs from two different events.
@checker.memory[:hits] = {}
@checker.memory[:hits][:"JH3132836336DHG"] = @event.id
@checker.memory[:hits][:"JH39AA63836DHG"] = event2.id
@checker.memory['hits'] = {}
@checker.memory['hits']["JH3132836336DHG"] = @event.id
@checker.memory['hits']["JH39AA63836DHG"] = event2.id
hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
@ -273,7 +273,7 @@ describe Agents::HumanTaskAgent do
end
it "shouldn't do anything if an assignment isn't ready" do
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
assignments = [
FakeAssignment.new(:status => "Accepted", :answers => {}),
@ -288,11 +288,11 @@ describe Agents::HumanTaskAgent do
@checker.send :review_hits
assignments.all? {|a| a.approved == true }.should be_false
@checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
@checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
end
it "shouldn't do anything if an assignment is missing" do
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
assignments = [
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
@ -306,11 +306,11 @@ describe Agents::HumanTaskAgent do
@checker.send :review_hits
assignments.all? {|a| a.approved == true }.should be_false
@checker.memory[:hits].should == { "JH3132836336DHG" => @event.id }
@checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
end
it "should create events when all assignments are ready" do
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
assignments = [
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
@ -327,32 +327,32 @@ describe Agents::HumanTaskAgent do
assignments.all? {|a| a.approved == true }.should be_true
hit.should be_disposed
@checker.events.last.payload[:answers].should == [
@checker.events.last.payload['answers'].should == [
{'sentiment' => "neutral", 'feedback' => ""},
{'sentiment' => "happy", 'feedback' => "Take 2"}
]
@checker.memory[:hits].should == {}
@checker.memory['hits'].should == {}
end
describe "taking majority votes" do
before do
@checker.options[:take_majority] = "true"
@checker.memory[:hits] = { "JH3132836336DHG" => @event.id }
@checker.options['take_majority'] = "true"
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
end
it "should take the majority votes of all questions" do
@checker.options[:hit][:questions][1] = {
:type => "selection",
:key => "age_range",
:name => "Age Range",
:required => "true",
:question => "Please select your age range:",
:selections =>
@checker.options['hit']['questions'][1] = {
'type' => "selection",
'key' => "age_range",
'name' => "Age Range",
'required' => "true",
'question' => "Please select your age range:",
'selections' =>
[
{ :key => "<50", :text => "50 years old or younger" },
{ :key => ">50", :text => "Over 50 years old" }
{ 'key' => "<50", 'text' => "50 years old or younger" },
{ 'key' => ">50", 'text' => "Over 50 years old" }
]
}
@ -371,35 +371,35 @@ describe Agents::HumanTaskAgent do
assignments.all? {|a| a.approved == true }.should be_true
@checker.events.last.payload[:answers].should == [
@checker.events.last.payload['answers'].should == [
{ 'sentiment' => "sad", 'age_range' => "<50" },
{ 'sentiment' => "neutral", 'age_range' => ">50" },
{ 'sentiment' => "happy", 'age_range' => ">50" },
{ 'sentiment' => "happy", 'age_range' => ">50" }
]
@checker.events.last.payload[:counts].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
@checker.events.last.payload[:majority_answer].should == { 'sentiment' => "happy", 'age_range' => ">50" }
@checker.events.last.payload.should_not have_key(:average_answer)
@checker.events.last.payload['counts'].should == { 'sentiment' => { 'happy' => 2, 'sad' => 1, 'neutral' => 1 }, 'age_range' => { ">50" => 3, "<50" => 1 } }
@checker.events.last.payload['majority_answer'].should == { 'sentiment' => "happy", 'age_range' => ">50" }
@checker.events.last.payload.should_not have_key('average_answer')
@checker.memory[:hits].should == {}
@checker.memory['hits'].should == {}
end
it "should also provide an average answer when all questions are numeric" do
@checker.options[:hit][:questions] = [
@checker.options['hit']['questions'] = [
{
:type => "selection",
:key => "rating",
:name => "Rating",
:required => "true",
:question => "Please select a rating:",
:selections =>
'type' => "selection",
'key' => "rating",
'name' => "Rating",
'required' => "true",
'question' => "Please select a rating:",
'selections' =>
[
{ :key => "1", :text => "One" },
{ :key => "2", :text => "Two" },
{ :key => "3", :text => "Three" },
{ :key => "4", :text => "Four" },
{ :key => "5.1", :text => "Five Point One" }
{ 'key' => "1", 'text' => "One" },
{ 'key' => "2", 'text' => "Two" },
{ 'key' => "3", 'text' => "Three" },
{ 'key' => "4", 'text' => "Four" },
{ 'key' => "5.1", 'text' => "Five Point One" }
]
}
]
@ -420,7 +420,7 @@ describe Agents::HumanTaskAgent do
assignments.all? {|a| a.approved == true }.should be_true
@checker.events.last.payload[:answers].should == [
@checker.events.last.payload['answers'].should == [
{ 'rating' => "1" },
{ 'rating' => "3" },
{ 'rating' => "5.1" },
@ -428,11 +428,11 @@ describe Agents::HumanTaskAgent do
{ 'rating' => "2" }
]
@checker.events.last.payload[:counts].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
@checker.events.last.payload[:majority_answer].should == { 'rating' => "2" }
@checker.events.last.payload[:average_answer].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
@checker.events.last.payload['counts'].should == { 'rating' => { "1" => 1, "2" => 2, "3" => 1, "4" => 0, "5.1" => 1 } }
@checker.events.last.payload['majority_answer'].should == { 'rating' => "2" }
@checker.events.last.payload['average_answer'].should == { 'rating' => (1 + 2 + 2 + 3 + 5.1) / 5.0 }
@checker.memory[:hits].should == {}
@checker.memory['hits'].should == {}
end
end
end

View file

@ -3,12 +3,12 @@ require 'spec_helper'
describe Agents::PeakDetectorAgent do
before do
@valid_params = {
:name => "my peak detector agent",
:options => {
:expected_receive_period_in_days => "2",
:group_by_path => "filter",
:value_path => "count",
:message => "A peak was found"
'name' => "my peak detector agent",
'options' => {
'expected_receive_period_in_days' => "2",
'group_by_path' => "filter",
'value_path' => "count",
'message' => "A peak was found"
}
}
@ -19,54 +19,54 @@ describe Agents::PeakDetectorAgent do
describe "#receive" do
it "tracks and groups by the group_by_path" do
events = build_events(:keys => [:count, :filter],
events = build_events(:keys => ['count', 'filter'],
:values => [[1, "something"], [2, "something"], [3, "else"]])
@agent.receive events
@agent.memory[:data][:something].map(&:first).should == [1, 2]
@agent.memory[:data][:something].last.last.should be_within(10).of((100 - 1).hours.ago.to_i)
@agent.memory[:data][:else].first.first.should == 3
@agent.memory[:data][:else].first.last.should be_within(10).of((100 - 2).hours.ago.to_i)
@agent.memory['data']['something'].map(&:first).should == [1, 2]
@agent.memory['data']['something'].last.last.should be_within(10).of((100 - 1).hours.ago.to_i)
@agent.memory['data']['else'].first.first.should == 3
@agent.memory['data']['else'].first.last.should be_within(10).of((100 - 2).hours.ago.to_i)
end
it "works without a group_by_path as well" do
@agent.options[:group_by_path] = ""
events = build_events(:keys => [:count], :values => [[1], [2]])
@agent.options['group_by_path'] = ""
events = build_events(:keys => ['count'], :values => [[1], [2]])
@agent.receive events
@agent.memory[:data][:no_group].map(&:first).should == [1, 2]
@agent.memory['data']['no_group'].map(&:first).should == [1, 2]
end
it "keeps a rolling window of data" do
@agent.options[:window_duration_in_days] = 5/24.0
@agent.receive build_events(:keys => [:count],
@agent.options['window_duration_in_days'] = 5/24.0
@agent.receive build_events(:keys => ['count'],
:values => [1, 2, 3, 4, 5, 6, 7, 8].map {|i| [i]},
:pattern => { :filter => "something" })
@agent.memory[:data][:something].map(&:first).should == [4, 5, 6, 7, 8]
:pattern => { 'filter' => "something" })
@agent.memory['data']['something'].map(&:first).should == [4, 5, 6, 7, 8]
end
it "finds peaks" do
build_events(:keys => [:count],
build_events(:keys => ['count'],
:values => [5, 6,
4, 5,
4, 5,
15, 11, # peak
8, 50, # ignored because it's too close to the first peak
4, 5].map {|i| [i]},
:pattern => { :filter => "something" }).each.with_index do |event, index|
:pattern => { 'filter' => "something" }).each.with_index do |event, index|
lambda {
@agent.receive([event])
}.should change { @agent.events.count }.by( index == 6 ? 1 : 0 )
end
@agent.events.last.payload[:peak].should == 15.0
@agent.memory[:peaks][:something].length.should == 1
@agent.events.last.payload['peak'].should == 15.0
@agent.memory['peaks']['something'].length.should == 1
end
it "keeps a rolling window of peaks" do
@agent.options[:min_peak_spacing_in_days] = 1/24.0
@agent.receive build_events(:keys => [:count],
@agent.options['min_peak_spacing_in_days'] = 1/24.0
@agent.receive build_events(:keys => ['count'],
:values => [1, 1, 1, 1, 1, 1, 10, 1, 1, 1, 1, 1, 1, 1, 10, 1].map {|i| [i]},
:pattern => { :filter => "something" })
@agent.memory[:peaks][:something].length.should == 2
:pattern => { 'filter' => "something" })
@agent.memory['peaks']['something'].length.should == 2
end
end
@ -76,17 +76,17 @@ describe Agents::PeakDetectorAgent do
end
it "should validate presence of message" do
@agent.options[:message] = nil
@agent.options['message'] = nil
@agent.should_not be_valid
end
it "should validate presence of expected_receive_period_in_days" do
@agent.options[:expected_receive_period_in_days] = ""
@agent.options['expected_receive_period_in_days'] = ""
@agent.should_not be_valid
end
it "should validate presence of value_path" do
@agent.options[:value_path] = ""
@agent.options['value_path'] = ""
@agent.should_not be_valid
end
end

View file

@ -3,16 +3,16 @@ require 'spec_helper'
describe Agents::TriggerAgent do
before do
@valid_params = {
:name => "my trigger agent",
:options => {
:expected_receive_period_in_days => 2,
:rules => [{
:type => "regex",
'value' => "a\\db",
:path => "foo.bar.baz",
}],
:message => "I saw '<foo.bar.baz>' from <name>"
}
'name' => "my trigger agent",
'options' => {
'expected_receive_period_in_days' => 2,
'rules' => [{
'type' => "regex",
'value' => "a\\db",
'path' => "foo.bar.baz",
}],
'message' => "I saw '<foo.bar.baz>' from <name>"
}
}
@checker = Agents::TriggerAgent.new(@valid_params)
@ -21,8 +21,8 @@ describe Agents::TriggerAgent do
@event = Event.new
@event.agent = agents(:bob_rain_notifier_agent)
@event.payload = { :foo => { "bar" => { :baz => "a2b" }},
:name => "Joe" }
@event.payload = { 'foo' => { "bar" => { 'baz' => "a2b" }},
'name' => "Joe" }
end
describe "validation" do
@ -31,22 +31,22 @@ describe Agents::TriggerAgent do
end
it "should validate presence of options" do
@checker.options[:message] = nil
@checker.options['message'] = nil
@checker.should_not be_valid
end
it "should validate the three fields in each rule" do
@checker.options[:rules] << { :path => "foo", :type => "fake", :value => "6" }
@checker.options['rules'] << { 'path' => "foo", 'type' => "fake", 'value' => "6" }
@checker.should_not be_valid
@checker.options[:rules].last[:type] = "field>=value"
@checker.options['rules'].last['type'] = "field>=value"
@checker.should be_valid
@checker.options[:rules].last.delete(:value)
@checker.options['rules'].last.delete('value')
@checker.should_not be_valid
end
end
describe "#working?" do
it "checks to see if the Agent has received any events in the last :expected_receive_period_in_days days" do
it "checks to see if the Agent has received any events in the last 'expected_receive_period_in_days' days" do
@event.save!
@checker.should_not be_working # no events have ever been received
@ -60,30 +60,30 @@ describe Agents::TriggerAgent do
describe "#receive" do
it "handles regex" do
@event.payload[:foo]["bar"][:baz] = "a222b"
@event.payload['foo']['bar']['baz'] = "a222b"
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@event.payload[:foo]["bar"][:baz] = "a2b"
@event.payload['foo']['bar']['baz'] = "a2b"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "handles negated regex" do
@event.payload[:foo]["bar"][:baz] = "a2b"
@checker.options[:rules][0] = {
:type => "!regex",
:value => "a\\db",
:path => "foo.bar.baz",
}
@event.payload['foo']['bar']['baz'] = "a2b"
@checker.options['rules'][0] = {
'type' => "!regex",
'value' => "a\\db",
'path' => "foo.bar.baz",
}
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@event.payload[:foo]["bar"][:baz] = "a22b"
@event.payload['foo']['bar']['baz'] = "a22b"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@ -91,49 +91,49 @@ describe Agents::TriggerAgent do
it "puts can extract values into the message based on paths" do
@checker.receive([@event])
Event.last.payload[:message].should == "I saw 'a2b' from Joe"
Event.last.payload['message'].should == "I saw 'a2b' from Joe"
end
it "handles numerical comparisons" do
@event.payload[:foo]["bar"][:baz] = "5"
@checker.options[:rules].first[:value] = 6
@checker.options[:rules].first[:type] = "field<value"
@event.payload['foo']['bar']['baz'] = "5"
@checker.options['rules'].first['value'] = 6
@checker.options['rules'].first['type'] = "field<value"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@checker.options[:rules].first[:value] = 3
@checker.options['rules'].first['value'] = 3
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
end
it "handles exact comparisons" do
@event.payload[:foo]["bar"][:baz] = "hello world"
@checker.options[:rules].first[:type] = "field==value"
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field==value"
@checker.options[:rules].first[:value] = "hello there"
@checker.options['rules'].first['value'] = "hello there"
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@checker.options[:rules].first[:value] = "hello world"
@checker.options['rules'].first['value'] = "hello world"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "handles negated comparisons" do
@event.payload[:foo]["bar"][:baz] = "hello world"
@checker.options[:rules].first[:type] = "field!=value"
@checker.options[:rules].first[:value] = "hello world"
@event.payload['foo']['bar']['baz'] = "hello world"
@checker.options['rules'].first['type'] = "field!=value"
@checker.options['rules'].first['value'] = "hello world"
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@checker.options[:rules].first[:value] = "hello there"
@checker.options['rules'].first['value'] = "hello there"
lambda {
@checker.receive([@event])
@ -141,20 +141,20 @@ describe Agents::TriggerAgent do
end
it "does fine without dots in the path" do
@event.payload = { :hello => "world" }
@checker.options[:rules].first[:type] = "field==value"
@checker.options[:rules].first[:path] = "hello"
@checker.options[:rules].first[:value] = "world"
@event.payload = { 'hello' => "world" }
@checker.options['rules'].first['type'] = "field==value"
@checker.options['rules'].first['path'] = "hello"
@checker.options['rules'].first['value'] = "world"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@checker.options[:rules].first[:path] = "foo"
@checker.options['rules'].first['path'] = "foo"
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@checker.options[:rules].first[:value] = "hi"
@checker.options['rules'].first['value'] = "hi"
lambda {
@checker.receive([@event])
}.should_not change { Event.count }
@ -163,11 +163,11 @@ describe Agents::TriggerAgent do
it "handles multiple events" do
event2 = Event.new
event2.agent = agents(:bob_weather_agent)
event2.payload = { :foo => { "bar" => { :baz => "a2b" }}}
event2.payload = { 'foo' => { 'bar' => { 'baz' => "a2b" }}}
event3 = Event.new
event3.agent = agents(:bob_weather_agent)
event3.payload = { :foo => { "bar" => { :baz => "a222b" }}}
event3.payload = { 'foo' => { 'bar' => { 'baz' => "a222b" }}}
lambda {
@checker.receive([@event, event2, event3])
@ -175,19 +175,19 @@ describe Agents::TriggerAgent do
end
it "handles ANDing rules together" do
@checker.options[:rules] << {
:type => "field>=value",
:value => "4",
:path => "foo.bing"
@checker.options['rules'] << {
'type' => "field>=value",
'value' => "4",
'path' => "foo.bing"
}
@event.payload[:foo]["bing"] = "5"
@event.payload['foo']["bing"] = "5"
lambda {
@checker.receive([@event])
}.should change { Event.count }.by(1)
@checker.options[:rules].last[:value] = 6
@checker.options['rules'].last['value'] = 6
lambda {
@checker.receive([@event])
}.should_not change { Event.count }