mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Merge branch 'master' into do_not_symbolize
Conflicts: app/models/agents/human_task_agent.rb spec/models/agents/human_task_agent_spec.rb
This commit is contained in:
commit
73e8406d4f
6 changed files with 494 additions and 66 deletions
|
@ -63,9 +63,9 @@ GEM
|
|||
railties (>= 3.2.6, < 5)
|
||||
warden (~> 1.2.3)
|
||||
diff-lcs (1.2.4)
|
||||
dotenv (0.8.0)
|
||||
dotenv-rails (0.8.0)
|
||||
dotenv (= 0.8.0)
|
||||
dotenv (0.9.0)
|
||||
dotenv-rails (0.9.0)
|
||||
dotenv (= 0.9.0)
|
||||
em-http-request (1.0.3)
|
||||
addressable (>= 2.2.3)
|
||||
cookiejar
|
||||
|
|
|
@ -9,9 +9,13 @@ module Agents
|
|||
|
||||
HITs can be created in response to events, or on a schedule. Set `trigger_on` to either `schedule` or `event`.
|
||||
|
||||
# Schedule
|
||||
|
||||
The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one. To configure how often a new HIT
|
||||
should be submitted when in `schedule` mode, set `submission_period` to a number of hours.
|
||||
|
||||
# Example
|
||||
|
||||
If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
|
||||
For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
|
||||
|
||||
|
@ -58,8 +62,52 @@ module Agents
|
|||
which contain `key` and `text`. For _free\\_text_, the special configuration options are all optional, and are
|
||||
`default`, `min_length`, and `max_length`.
|
||||
|
||||
If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
|
||||
automatically select the majority vote for each question across all `assignments`. If all selections are numeric, an `average_answer` will also be generated.
|
||||
# Combining answers
|
||||
|
||||
There are a couple of ways to combine HITs that have multiple `assignments`, all of which involve setting `combination_mode` at the top level.
|
||||
|
||||
## Taking the majority
|
||||
|
||||
Option 1: if all of your `questions` are of `type` _selection_, you can set `combination_mode` to `take_majority`.
|
||||
This will cause the Agent to automatically select the majority vote for each question across all `assignments` and return it as `majority_answer`.
|
||||
If all selections are numeric, an `average_answer` will also be generated.
|
||||
|
||||
Option 2: you can have the Agent ask additional human workers to rank the `assignments` and return the most highly ranked answer.
|
||||
To do this, set `combination_mode` to `poll` and provide a `poll_options` object. Here is an example:
|
||||
|
||||
{
|
||||
"trigger_on": "schedule",
|
||||
"submission_period": 12,
|
||||
"combination_mode": "poll",
|
||||
"poll_options": {
|
||||
"title": "Take a poll about some jokes",
|
||||
"instructions": "Please rank these jokes from most funny (5) to least funny (1)",
|
||||
"assignments": 3,
|
||||
"row_template": "<$.joke>"
|
||||
},
|
||||
"hit": {
|
||||
"assignments": 5,
|
||||
"title": "Tell a joke",
|
||||
"description": "Please tell me a joke",
|
||||
"reward": 0.05,
|
||||
"lifetime_in_seconds": "3600",
|
||||
"questions": [
|
||||
{
|
||||
"type": "free_text",
|
||||
"key": "joke",
|
||||
"name": "Your joke",
|
||||
"required": "true",
|
||||
"question": "Joke",
|
||||
"min_length": "2",
|
||||
"max_length": "2000"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Resulting events will have the original `answers`, as well as the `poll` results, and a field called `best_answer` that contains the best answer as determined by the poll.
|
||||
|
||||
# Other settings
|
||||
|
||||
`lifetime_in_seconds` is the number of seconds a HIT is left on Amazon before it's automatically closed. The default is 1 day.
|
||||
|
||||
|
@ -70,6 +118,12 @@ module Agents
|
|||
Events look like:
|
||||
|
||||
{
|
||||
"answers": [
|
||||
{
|
||||
"feedback": "Hello!",
|
||||
"sentiment": "happy"
|
||||
}
|
||||
]
|
||||
}
|
||||
MD
|
||||
|
||||
|
@ -97,9 +151,13 @@ module Agents
|
|||
errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
|
||||
end
|
||||
|
||||
if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" }
|
||||
if take_majority? && options['hit']['questions'].any? { |question| question['type'] != "selection" }
|
||||
errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
|
||||
end
|
||||
|
||||
if create_poll?
|
||||
errors.add(:base, "poll_options is required when combination_mode is set to 'poll' and must have the keys 'title', 'instructions', 'row_template', and 'assignments'") unless options['poll_options'].is_a?(Hash) && options['poll_options']['title'].present? && options['poll_options']['instructions'].present? && options['poll_options']['row_template'].present? && options['poll_options']['assignments'].to_i > 0
|
||||
end
|
||||
end
|
||||
|
||||
def default_options
|
||||
|
@ -152,70 +210,152 @@ module Agents
|
|||
|
||||
if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60
|
||||
memory['last_schedule'] = Time.now.to_i
|
||||
create_hit
|
||||
create_basic_hit
|
||||
end
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
if options['trigger_on'] == "event"
|
||||
incoming_events.each do |event|
|
||||
create_hit event
|
||||
create_basic_hit event
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def take_majority?
|
||||
options['combination_mode'] == "take_majority" || options['take_majority'] == "true"
|
||||
end
|
||||
|
||||
def create_poll?
|
||||
options['combination_mode'] == "poll"
|
||||
end
|
||||
|
||||
def event_for_hit(hit_id)
|
||||
if memory['hits'][hit_id].is_a?(Hash)
|
||||
Event.find_by_id(memory['hits'][hit_id]['event_id'])
|
||||
else
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def hit_type(hit_id)
|
||||
if memory['hits'][hit_id].is_a?(Hash) && memory['hits'][hit_id]['type']
|
||||
memory['hits'][hit_id]['type']
|
||||
else
|
||||
'user'
|
||||
end
|
||||
end
|
||||
|
||||
def review_hits
|
||||
reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
|
||||
my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys
|
||||
if reviewable_hit_ids.length > 0
|
||||
log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]"
|
||||
end
|
||||
|
||||
my_reviewed_hit_ids.each do |hit_id|
|
||||
hit = RTurk::Hit.new(hit_id)
|
||||
assignments = hit.assignments
|
||||
|
||||
log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
|
||||
if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
|
||||
payload = { 'answers' => assignments.map(&:answers) }
|
||||
inbound_event = event_for_hit(hit_id)
|
||||
|
||||
if options['take_majority'] == "true"
|
||||
counts = {}
|
||||
options['hit']['questions'].each do |question|
|
||||
question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
|
||||
assignments.each do |assignment|
|
||||
answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
|
||||
answer = answers[question['key']]
|
||||
question_counts[answer] += 1
|
||||
if hit_type(hit_id) == 'poll'
|
||||
# handle completed polls
|
||||
|
||||
log "Handling a poll: #{hit_id}"
|
||||
|
||||
scores = {}
|
||||
assignments.each do |assignment|
|
||||
assignment.answers.each do |index, rating|
|
||||
scores[index] ||= 0
|
||||
scores[index] += rating.to_i
|
||||
end
|
||||
counts[question['key']] = question_counts
|
||||
end
|
||||
payload['counts'] = counts
|
||||
|
||||
majority_answer = counts.inject({}) do |memo, (key, question_counts)|
|
||||
memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
|
||||
memo
|
||||
end
|
||||
payload['majority_answer'] = majority_answer
|
||||
top_answer = scores.to_a.sort {|b, a| a.last <=> b.last }.first.first
|
||||
|
||||
if all_questions_are_numeric?
|
||||
average_answer = counts.inject({}) do |memo, (key, question_counts)|
|
||||
sum = divisor = 0
|
||||
question_counts.to_a.each do |num, count|
|
||||
sum += num.to_s.to_f * count
|
||||
divisor += count
|
||||
payload = {
|
||||
'answers' => memory['hits'][hit_id]['answers'],
|
||||
'poll' => assignments.map(&:answers),
|
||||
'best_answer' => memory['hits'][hit_id]['answers'][top_answer.to_i - 1]
|
||||
}
|
||||
|
||||
event = create_event :payload => payload
|
||||
log "Event emitted with answer(s) for poll", :outbound_event => event, :inbound_event => inbound_event
|
||||
else
|
||||
# handle normal completed HITs
|
||||
payload = { 'answers' => assignments.map(&:answers) }
|
||||
|
||||
if take_majority?
|
||||
counts = {}
|
||||
options['hit']['questions'].each do |question|
|
||||
question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
|
||||
assignments.each do |assignment|
|
||||
answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
|
||||
answer = answers[question['key']]
|
||||
question_counts[answer] += 1
|
||||
end
|
||||
memo[key] = sum / divisor.to_f
|
||||
counts[question['key']] = question_counts
|
||||
end
|
||||
payload['counts'] = counts
|
||||
|
||||
majority_answer = counts.inject({}) do |memo, (key, question_counts)|
|
||||
memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
|
||||
memo
|
||||
end
|
||||
payload['average_answer'] = average_answer
|
||||
payload['majority_answer'] = majority_answer
|
||||
|
||||
if all_questions_are_numeric?
|
||||
average_answer = counts.inject({}) do |memo, (key, question_counts)|
|
||||
sum = divisor = 0
|
||||
question_counts.to_a.each do |num, count|
|
||||
sum += num.to_s.to_f * count
|
||||
divisor += count
|
||||
end
|
||||
memo[key] = sum / divisor.to_f
|
||||
memo
|
||||
end
|
||||
payload['average_answer'] = average_answer
|
||||
end
|
||||
end
|
||||
|
||||
if create_poll?
|
||||
questions = []
|
||||
selections = 5.times.map { |i| { 'key' => i+1, 'text' => i+1 } }.reverse
|
||||
assignments.length.times do |index|
|
||||
questions << {
|
||||
'type' => "selection",
|
||||
'name' => "Item #{index + 1}",
|
||||
'key' => index,
|
||||
'required' => "true",
|
||||
'question' => Utils.interpolate_jsonpaths(options['poll_options']['row_template'], assignments[index].answers),
|
||||
'selections' => selections
|
||||
}
|
||||
end
|
||||
|
||||
poll_hit = create_hit 'title' => options['poll_options']['title'],
|
||||
'description' => options['poll_options']['instructions'],
|
||||
'questions' => questions,
|
||||
'assignments' => options['poll_options']['assignments'],
|
||||
'lifetime_in_seconds' => options['poll_options']['lifetime_in_seconds'],
|
||||
'reward' => options['poll_options']['reward'],
|
||||
'payload' => inbound_event && inbound_event.payload,
|
||||
'metadata' => { 'type' => 'poll',
|
||||
'original_hit' => hit_id,
|
||||
'answers' => assignments.map(&:answers),
|
||||
'event_id' => inbound_event && inbound_event.id }
|
||||
|
||||
log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}. Original HIT: #{hit_id}", :inbound_event => inbound_event
|
||||
else
|
||||
event = create_event :payload => payload
|
||||
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event
|
||||
end
|
||||
end
|
||||
|
||||
event = create_event :payload => payload
|
||||
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id])
|
||||
|
||||
assignments.each(&:approve!)
|
||||
hit.dispose!
|
||||
|
||||
|
@ -232,22 +372,35 @@ module Agents
|
|||
end
|
||||
end
|
||||
|
||||
def create_hit(event = nil)
|
||||
payload = event ? event.payload : {}
|
||||
title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip
|
||||
description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip
|
||||
questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload)
|
||||
def create_basic_hit(event = nil)
|
||||
hit = create_hit 'title' => options['hit']['title'],
|
||||
'description' => options['hit']['description'],
|
||||
'questions' => options['hit']['questions'],
|
||||
'assignments' => options['hit']['assignments'],
|
||||
'lifetime_in_seconds' => options['hit']['lifetime_in_seconds'],
|
||||
'reward' => options['hit']['reward'],
|
||||
'payload' => event && event.payload,
|
||||
'metadata' => { 'event_id' => event && event.id }
|
||||
|
||||
log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
|
||||
end
|
||||
|
||||
def create_hit(opts = {})
|
||||
payload = opts['payload'] || {}
|
||||
title = Utils.interpolate_jsonpaths(opts['title'], payload).strip
|
||||
description = Utils.interpolate_jsonpaths(opts['description'], payload).strip
|
||||
questions = Utils.recursively_interpolate_jsonpaths(opts['questions'], payload)
|
||||
hit = RTurk::Hit.create(:title => title) do |hit|
|
||||
hit.max_assignments = (options['hit']['assignments'] || 1).to_i
|
||||
hit.max_assignments = (opts['assignments'] || 1).to_i
|
||||
hit.description = description
|
||||
hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i
|
||||
hit.lifetime = (opts['lifetime_in_seconds'] || 24 * 60 * 60).to_i
|
||||
hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
|
||||
hit.reward = (options['hit']['reward'] || 0.05).to_f
|
||||
hit.reward = (opts['reward'] || 0.05).to_f
|
||||
#hit.qualifications.add :approval_rate, { :gt => 80 }
|
||||
end
|
||||
memory['hits'] ||= {}
|
||||
memory['hits'][hit.id] = event && event.id
|
||||
log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
|
||||
memory['hits'][hit.id] = opts['metadata'] || {}
|
||||
hit
|
||||
end
|
||||
|
||||
# RTurk Question Form
|
||||
|
@ -328,4 +481,4 @@ module Agents
|
|||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
62
app/models/agents/webhook_agent.rb
Normal file
62
app/models/agents/webhook_agent.rb
Normal file
|
@ -0,0 +1,62 @@
|
|||
module Agents
|
||||
class WebhookAgent < Agent
|
||||
cannot_be_scheduled!
|
||||
|
||||
description do
|
||||
<<-MD
|
||||
Use this Agent to create events by receiving webhooks from any source.
|
||||
|
||||
In order to create events with this agent, make a POST request to:
|
||||
```
|
||||
https://#{ENV['DOMAIN']}/users/#{user.id}/webhooks/#{id || '<id>'}/:secret
|
||||
``` where `:secret` is specified in your options.
|
||||
|
||||
The
|
||||
|
||||
Options:
|
||||
|
||||
* `secret` - A token that the host will provide for authentication.
|
||||
* `expected_receive_period_in_days` - How often you expect to receive
|
||||
events this way. Used to determine if the agent is working.
|
||||
* `payload_path` - JSONPath of the attribute in the POST body to be
|
||||
used as the Event payload.
|
||||
MD
|
||||
end
|
||||
|
||||
event_description do
|
||||
<<-MD
|
||||
The event payload is base on the value of the `payload_path` option,
|
||||
which is set to `#{options['payload_path']}`.
|
||||
MD
|
||||
end
|
||||
|
||||
def default_options
|
||||
{ "secret" => "supersecretstring",
|
||||
"expected_receive_period_in_days" => 1,
|
||||
"payload_path" => "payload"}
|
||||
end
|
||||
|
||||
def receive_webhook(params)
|
||||
secret = params.delete('secret')
|
||||
return ["Not Authorized", 401] unless secret == options['secret']
|
||||
|
||||
create_event(:payload => payload_for(params))
|
||||
|
||||
['Event Created', 201]
|
||||
end
|
||||
|
||||
def working?
|
||||
event_created_within(options['expected_receive_period_in_days']) && !recent_error_logs?
|
||||
end
|
||||
|
||||
def validate_options
|
||||
unless options['secret'].present?
|
||||
errors.add(:base, "Must specify a secret for 'Authenticating' requests")
|
||||
end
|
||||
end
|
||||
|
||||
def payload_for(params)
|
||||
Utils.value_at(params, options['payload_path']) || {}
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,5 +1,6 @@
|
|||
require 'yaml'
|
||||
require 'pathname'
|
||||
require 'dotenv'
|
||||
|
||||
# Edited by Andrew Cantino. Based on: https://gist.github.com/339471
|
||||
|
||||
|
@ -99,6 +100,28 @@ namespace :sync do
|
|||
return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
|
||||
end
|
||||
|
||||
# Used by remote_database_config to parse the remote .env file. Depends on the dotenv-rails gem.
|
||||
class RemoteEnvLoader < Dotenv::Environment
|
||||
def initialize(data)
|
||||
@data = data
|
||||
load
|
||||
end
|
||||
|
||||
def with_loaded_env
|
||||
begin
|
||||
saved_env = ENV.to_hash.dup
|
||||
ENV.update(self)
|
||||
yield
|
||||
ensure
|
||||
ENV.replace(saved_env)
|
||||
end
|
||||
end
|
||||
|
||||
def read
|
||||
@data.split("\n")
|
||||
end
|
||||
end
|
||||
|
||||
#
|
||||
# Reads the database credentials from the remote config/database.yml file
|
||||
# +db+ the name of the environment to get the credentials for
|
||||
|
@ -106,7 +129,13 @@ namespace :sync do
|
|||
#
|
||||
def remote_database_config(db)
|
||||
remote_config = capture("cat #{current_path}/config/database.yml")
|
||||
database = YAML::load(remote_config)
|
||||
remote_env = capture("cat #{current_path}/.env")
|
||||
|
||||
database = nil
|
||||
RemoteEnvLoader.new(remote_env).with_loaded_env do
|
||||
database = YAML::load(ERB.new(remote_config).result)
|
||||
end
|
||||
|
||||
return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
|
||||
end
|
||||
|
||||
|
|
|
@ -108,7 +108,43 @@ describe Agents::HumanTaskAgent do
|
|||
@checker.should_not be_valid
|
||||
end
|
||||
|
||||
it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
|
||||
it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
|
||||
@checker.options['combination_mode'] = "poll"
|
||||
@checker.should_not be_valid
|
||||
@checker.options['poll_options'] = {}
|
||||
@checker.should_not be_valid
|
||||
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
|
||||
'instructions' => "Rank these by how funny they are",
|
||||
'assignments' => 3,
|
||||
'row_template' => "<$.joke>" }
|
||||
@checker.should be_valid
|
||||
@checker.options['poll_options'] = { 'instructions' => "Rank these by how funny they are",
|
||||
'assignments' => 3,
|
||||
'row_template' => "<$.joke>" }
|
||||
@checker.should_not be_valid
|
||||
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
|
||||
'assignments' => 3,
|
||||
'row_template' => "<$.joke>" }
|
||||
@checker.should_not be_valid
|
||||
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
|
||||
'instructions' => "Rank these by how funny they are",
|
||||
'row_template' => "<$.joke>" }
|
||||
@checker.should_not be_valid
|
||||
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
|
||||
'instructions' => "Rank these by how funny they are",
|
||||
'assignments' => 3}
|
||||
@checker.should_not be_valid
|
||||
end
|
||||
|
||||
it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
|
||||
@checker.options['combination_mode'] = "take_majority"
|
||||
@checker.should_not be_valid
|
||||
@checker.options['hit']['questions'][1]['type'] = "selection"
|
||||
@checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
|
||||
@checker.should be_valid
|
||||
end
|
||||
|
||||
it "accepts 'take_majority': 'true' for legacy support" do
|
||||
@checker.options['take_majority'] = "true"
|
||||
@checker.should_not be_valid
|
||||
@checker.options['hit']['questions'][1]['type'] = "selection"
|
||||
|
@ -126,7 +162,7 @@ describe Agents::HumanTaskAgent do
|
|||
|
||||
it "should check for reviewable HITs frequently" do
|
||||
mock(@checker).review_hits.twice
|
||||
mock(@checker).create_hit.once
|
||||
mock(@checker).create_basic_hit.once
|
||||
@checker.check
|
||||
@checker.check
|
||||
end
|
||||
|
@ -135,7 +171,7 @@ describe Agents::HumanTaskAgent do
|
|||
now = Time.now
|
||||
stub(Time).now { now }
|
||||
mock(@checker).review_hits.times(3)
|
||||
mock(@checker).create_hit.twice
|
||||
mock(@checker).create_basic_hit.twice
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
|
@ -144,7 +180,7 @@ describe Agents::HumanTaskAgent do
|
|||
end
|
||||
|
||||
it "should ignore events" do
|
||||
mock(@checker).create_hit(anything).times(0)
|
||||
mock(@checker).create_basic_hit(anything).times(0)
|
||||
@checker.receive([events(:bob_website_agent_event)])
|
||||
end
|
||||
end
|
||||
|
@ -155,7 +191,7 @@ describe Agents::HumanTaskAgent do
|
|||
now = Time.now
|
||||
stub(Time).now { now }
|
||||
mock(@checker).review_hits.times(3)
|
||||
mock(@checker).create_hit.times(0)
|
||||
mock(@checker).create_basic_hit.times(0)
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
|
@ -164,7 +200,7 @@ describe Agents::HumanTaskAgent do
|
|||
end
|
||||
|
||||
it "should create HITs based on events" do
|
||||
mock(@checker).create_hit(events(:bob_website_agent_event)).times(1)
|
||||
mock(@checker).create_basic_hit(events(:bob_website_agent_event)).times(1)
|
||||
@checker.receive([events(:bob_website_agent_event)])
|
||||
end
|
||||
end
|
||||
|
@ -181,7 +217,7 @@ describe Agents::HumanTaskAgent do
|
|||
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
|
||||
mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
|
||||
|
||||
@checker.send :create_hit, @event
|
||||
@checker.send :create_basic_hit, @event
|
||||
|
||||
hitInterface.max_assignments.should == @checker.options['hit']['assignments']
|
||||
hitInterface.reward.should == @checker.options['hit']['reward']
|
||||
|
@ -192,7 +228,7 @@ describe Agents::HumanTaskAgent do
|
|||
xml.should include("<Text>Make something for Joe</Text>")
|
||||
xml.should include("<DisplayName>Joe Question 1</DisplayName>")
|
||||
|
||||
@checker.memory['hits'][123].should == @event.id
|
||||
@checker.memory['hits'][123]['event_id'].should == @event.id
|
||||
end
|
||||
|
||||
it "works without an event too" do
|
||||
|
@ -201,7 +237,7 @@ describe Agents::HumanTaskAgent do
|
|||
hitInterface.id = 123
|
||||
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
|
||||
mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
|
||||
@checker.send :create_hit
|
||||
@checker.send :create_basic_hit
|
||||
hitInterface.max_assignments.should == @checker.options['hit']['assignments']
|
||||
hitInterface.reward.should == @checker.options['hit']['reward']
|
||||
end
|
||||
|
@ -259,8 +295,8 @@ describe Agents::HumanTaskAgent do
|
|||
|
||||
# It knows about two HITs from two different events.
|
||||
@checker.memory['hits'] = {}
|
||||
@checker.memory['hits']["JH3132836336DHG"] = @event.id
|
||||
@checker.memory['hits']["JH39AA63836DHG"] = event2.id
|
||||
@checker.memory['hits']["JH3132836336DHG"] = { 'event_id' => @event.id }
|
||||
@checker.memory['hits']["JH39AA63836DHG"] = { 'event_id' => event2.id }
|
||||
|
||||
hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
|
||||
|
@ -273,7 +309,7 @@ describe Agents::HumanTaskAgent do
|
|||
end
|
||||
|
||||
it "shouldn't do anything if an assignment isn't ready" do
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Accepted", :answers => {}),
|
||||
|
@ -288,11 +324,11 @@ describe Agents::HumanTaskAgent do
|
|||
@checker.send :review_hits
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_false
|
||||
@checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
|
||||
@checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
end
|
||||
|
||||
it "shouldn't do anything if an assignment is missing" do
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
|
||||
|
@ -306,11 +342,11 @@ describe Agents::HumanTaskAgent do
|
|||
@checker.send :review_hits
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_false
|
||||
@checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
|
||||
@checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
end
|
||||
|
||||
it "should create events when all assignments are ready" do
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
|
||||
|
@ -337,8 +373,8 @@ describe Agents::HumanTaskAgent do
|
|||
|
||||
describe "taking majority votes" do
|
||||
before do
|
||||
@checker.options['take_majority'] = "true"
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
|
||||
@checker.options['combination_mode'] = "take_majority"
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
end
|
||||
|
||||
|
@ -386,6 +422,10 @@ describe Agents::HumanTaskAgent do
|
|||
end
|
||||
|
||||
it "should also provide an average answer when all questions are numeric" do
|
||||
# it should accept 'take_majority': 'true' as well for legacy support. Demonstrating that here.
|
||||
@checker.options.delete :combination_mode
|
||||
@checker.options['take_majority'] = "true"
|
||||
|
||||
@checker.options['hit']['questions'] = [
|
||||
{
|
||||
'type' => "selection",
|
||||
|
@ -435,5 +475,118 @@ describe Agents::HumanTaskAgent do
|
|||
@checker.memory['hits'].should == {}
|
||||
end
|
||||
end
|
||||
|
||||
describe "creating and reviewing polls" do
|
||||
before do
|
||||
@checker.options['combination_mode'] = "poll"
|
||||
@checker.options['poll_options'] = {
|
||||
'title' => "Hi!",
|
||||
'instructions' => "hello!",
|
||||
'assignments' => 2,
|
||||
'row_template' => "This is <.sentiment>"
|
||||
}
|
||||
@event.save!
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
end
|
||||
|
||||
it "creates a poll using the row_template, message, and correct number of assignments" do
|
||||
@checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
|
||||
|
||||
# Mock out the HIT's submitted assignments.
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad", "feedback"=>"This is my feedback 1"}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>"This is my feedback 2"}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"This is my feedback 3"}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"This is my feedback 4"})
|
||||
]
|
||||
hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
|
||||
mock(RTurk::Hit).new("JH3132836336DHG") { hit }
|
||||
|
||||
@checker.memory['hits']["JH3132836336DHG"].should be_present
|
||||
|
||||
# Setup mocks for HIT creation
|
||||
|
||||
question_form = nil
|
||||
hitInterface = OpenStruct.new
|
||||
hitInterface.id = "JH39AA63836DH12345"
|
||||
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
|
||||
mock(RTurk::Hit).create(:title => "Hi!").yields(hitInterface) { hitInterface }
|
||||
|
||||
# And finally, the test.
|
||||
|
||||
lambda {
|
||||
@checker.send :review_hits
|
||||
}.should change { Event.count }.by(0) # it does not emit an event until all poll results are in
|
||||
|
||||
# it approves the existing assignments
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_true
|
||||
hit.should be_disposed
|
||||
|
||||
# it creates a new HIT for the poll
|
||||
|
||||
hitInterface.max_assignments.should == @checker.options['poll_options']['assignments']
|
||||
hitInterface.description.should == @checker.options['poll_options']['instructions']
|
||||
|
||||
xml = question_form.to_xml
|
||||
xml.should include("<Text>This is happy</Text>")
|
||||
xml.should include("<Text>This is neutral</Text>")
|
||||
xml.should include("<Text>This is sad</Text>")
|
||||
|
||||
@checker.save
|
||||
@checker.reload
|
||||
@checker.memory['hits']["JH3132836336DHG"].should_not be_present
|
||||
@checker.memory['hits']["JH39AA63836DH12345"].should be_present
|
||||
@checker.memory['hits']["JH39AA63836DH12345"]['event_id'].should == @event.id
|
||||
@checker.memory['hits']["JH39AA63836DH12345"]['type'].should == "poll"
|
||||
@checker.memory['hits']["JH39AA63836DH12345"]['original_hit'].should == "JH3132836336DHG"
|
||||
@checker.memory['hits']["JH39AA63836DH12345"]['answers'].length.should == 4
|
||||
end
|
||||
|
||||
it "emits an event when all poll results are in, containing the data from the best answer, plus all others" do
|
||||
original_answers = [
|
||||
{ 'sentiment' => "sad", 'feedback' => "This is my feedback 1"},
|
||||
{ 'sentiment' => "neutral", 'feedback' => "This is my feedback 2"},
|
||||
{ 'sentiment' => "happy", 'feedback' => "This is my feedback 3"},
|
||||
{ 'sentiment' => "happy", 'feedback' => "This is my feedback 4"}
|
||||
]
|
||||
|
||||
@checker.memory['hits'] = {
|
||||
'JH39AA63836DH12345' => {
|
||||
'type' => 'poll',
|
||||
'original_hit' => "JH3132836336DHG",
|
||||
'answers' => original_answers,
|
||||
'event_id' => 345
|
||||
}
|
||||
}
|
||||
|
||||
# Mock out the HIT's submitted assignments.
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"1" => "2", "2" => "5", "3" => "3", "4" => "2"}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"1" => "3", "2" => "4", "3" => "1", "4" => "4"})
|
||||
]
|
||||
hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
|
||||
mock(RTurk::Hit).new("JH39AA63836DH12345") { hit }
|
||||
|
||||
@checker.memory['hits']["JH39AA63836DH12345"].should be_present
|
||||
|
||||
lambda {
|
||||
@checker.send :review_hits
|
||||
}.should change { Event.count }.by(1)
|
||||
|
||||
# It emits an event
|
||||
|
||||
@checker.events.last.payload['answers'].should == original_answers
|
||||
@checker.events.last.payload['poll'].should == [{"1" => "2", "2" => "5", "3" => "3", "4" => "2"}, {"1" => "3", "2" => "4", "3" => "1", "4" => "4"}]
|
||||
@checker.events.last.payload['best_answer'].should == {'sentiment' => "neutral", 'feedback' => "This is my feedback 2"}
|
||||
|
||||
# it approves the existing assignments
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_true
|
||||
hit.should be_disposed
|
||||
|
||||
@checker.memory['hits'].should be_empty
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
31
spec/models/agents/webhook_agent_spec.rb
Normal file
31
spec/models/agents/webhook_agent_spec.rb
Normal file
|
@ -0,0 +1,31 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Agents::WebhookAgent do
|
||||
let(:agent) do
|
||||
_agent = Agents::WebhookAgent.new(:name => 'webhook',
|
||||
:options => { 'secret' => 'foobar', 'payload_path' => 'payload' })
|
||||
_agent.user = users(:bob)
|
||||
_agent.save!
|
||||
_agent
|
||||
end
|
||||
let(:payload) { {'some' => 'info'} }
|
||||
|
||||
describe 'receive_webhook' do
|
||||
it 'should create event if secret matches' do
|
||||
out = nil
|
||||
lambda {
|
||||
out = agent.receive_webhook('secret' => 'foobar', 'payload' => payload)
|
||||
}.should change { Event.count }.by(1)
|
||||
out.should eq(['Event Created', 201])
|
||||
Event.last.payload.should eq(payload)
|
||||
end
|
||||
|
||||
it 'should not create event if secrets dont match' do
|
||||
out = nil
|
||||
lambda {
|
||||
out = agent.receive_webhook('secret' => 'bazbat', 'payload' => payload)
|
||||
}.should change { Event.count }.by(0)
|
||||
out.should eq(['Not Authorized', 401])
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Add table
Reference in a new issue