mirror of
https://github.com/Fishwaldo/huginn.git
synced 2025-03-15 19:31:26 +00:00
Basic HuamnTaskAgent with specs
This commit is contained in:
parent
690eb0f979
commit
0120ccb9c1
11 changed files with 661 additions and 63 deletions
30
.env.example
30
.env.example
|
@ -7,7 +7,10 @@ APP_SECRET_TOKEN=REPLACE_ME_NOW!
|
|||
# for development, but it needs to be changed when you deploy to a production environment.
|
||||
DOMAIN=localhost:3000
|
||||
|
||||
# Database Setup
|
||||
############################
|
||||
# Database Setup #
|
||||
############################
|
||||
|
||||
DATABASE_ADAPTER=mysql2
|
||||
DATABASE_ENCODING=utf8
|
||||
DATABASE_RECONNECT=true
|
||||
|
@ -24,6 +27,10 @@ DATABASE_PASSWORD=""
|
|||
# Configure Rails environment. This should only be needed in production and may cause errors in development.
|
||||
# RAILS_ENV=production
|
||||
|
||||
#############################
|
||||
# Email Configuration #
|
||||
#############################
|
||||
|
||||
# Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com
|
||||
# as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.
|
||||
SMTP_DOMAIN=your-domain-here.com
|
||||
|
@ -37,9 +44,28 @@ SMTP_ENABLE_STARTTLS_AUTO=true
|
|||
# The address from which system emails will appear to be sent.
|
||||
EMAIL_FROM_ADDRESS=from_address@gmail.com
|
||||
|
||||
############################
|
||||
# Allowing Signups #
|
||||
############################
|
||||
|
||||
# This invitation code will be required for users to signup with your Huginn installation.
|
||||
# You can see its use in user.rb.
|
||||
INVITATION_CODE=try-huginn
|
||||
|
||||
###########################
|
||||
# Agent Logging #
|
||||
###########################
|
||||
|
||||
# Number of lines of log messages to keep per Agent
|
||||
AGENT_LOG_LENGTH=100
|
||||
AGENT_LOG_LENGTH=200
|
||||
|
||||
#############################
|
||||
# AWS and Mechanical Turk #
|
||||
#############################
|
||||
|
||||
# AWS Credentials for MTurk
|
||||
AWS_ACCESS_KEY_ID="your aws access key id"
|
||||
AWS_ACCESS_KEY="your aws access key"
|
||||
|
||||
# Set AWS_SANDBOX to true if you're developing Huginn code.
|
||||
AWS_SANDBOX=false
|
||||
|
|
1
Gemfile
1
Gemfile
|
@ -32,6 +32,7 @@ gem 'kramdown'
|
|||
gem "typhoeus"
|
||||
gem 'nokogiri'
|
||||
gem 'wunderground'
|
||||
gem 'rturk'
|
||||
|
||||
gem "twitter"
|
||||
gem 'twitter-stream', '>=0.1.16'
|
||||
|
|
11
Gemfile.lock
11
Gemfile.lock
|
@ -74,6 +74,8 @@ GEM
|
|||
http_parser.rb (>= 0.5.3)
|
||||
em-socksify (0.3.0)
|
||||
eventmachine (>= 1.0.0.beta.4)
|
||||
erector (0.9.0)
|
||||
treetop (>= 1.2.3)
|
||||
erubis (2.7.0)
|
||||
ethon (0.5.12)
|
||||
ffi (>= 1.3.0)
|
||||
|
@ -118,7 +120,7 @@ GEM
|
|||
mime-types (~> 1.16)
|
||||
treetop (~> 1.4.8)
|
||||
method_source (0.8.1)
|
||||
mime-types (1.23)
|
||||
mime-types (1.24)
|
||||
mini_portile (0.5.1)
|
||||
multi_json (1.7.9)
|
||||
multi_xml (0.5.5)
|
||||
|
@ -182,6 +184,10 @@ GEM
|
|||
rspec-core (~> 2.14.0)
|
||||
rspec-expectations (~> 2.14.0)
|
||||
rspec-mocks (~> 2.14.0)
|
||||
rturk (2.11.0)
|
||||
erector
|
||||
nokogiri
|
||||
rest-client
|
||||
rufus-scheduler (2.0.22)
|
||||
tzinfo (>= 0.3.23)
|
||||
safe_yaml (0.9.5)
|
||||
|
@ -205,7 +211,7 @@ GEM
|
|||
system_timer (1.2.4)
|
||||
thor (0.18.1)
|
||||
tilt (1.4.1)
|
||||
treetop (1.4.14)
|
||||
treetop (1.4.15)
|
||||
polyglot
|
||||
polyglot (>= 0.3.1)
|
||||
twilio-ruby (3.10.0)
|
||||
|
@ -269,6 +275,7 @@ DEPENDENCIES
|
|||
rr
|
||||
rspec
|
||||
rspec-rails
|
||||
rturk
|
||||
rufus-scheduler
|
||||
sass-rails (~> 3.2.3)
|
||||
select2-rails
|
||||
|
|
|
@ -8,7 +8,7 @@ $ ->
|
|||
|
||||
if json.pending? && json.pending > 0
|
||||
tooltipOptions = {
|
||||
title: "#{json.pending} pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
|
||||
title: "#{json.pending} jobs pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures"
|
||||
delay: 0
|
||||
placement: "bottom"
|
||||
trigger: "hover"
|
||||
|
|
|
@ -147,6 +147,7 @@ class Agent < ActiveRecord::Base
|
|||
end
|
||||
|
||||
def log(message, options = {})
|
||||
puts "Agent##{id}: #{message}" unless Rails.env.test?
|
||||
AgentLog.log_for_agent(self, message, options)
|
||||
end
|
||||
|
||||
|
|
|
@ -66,16 +66,10 @@ module Agents
|
|||
!recent_error_logs?
|
||||
end
|
||||
|
||||
def value_constructor(value, payload)
|
||||
value.gsub(/<[^>]+>/).each { |jsonpath|
|
||||
Utils.values_at(payload, jsonpath[1..-2]).first.to_s
|
||||
}
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
incoming_events.each do |event|
|
||||
formatted_event = options[:mode].to_s == "merge" ? event.payload : {}
|
||||
options[:instructions].each_pair {|key, value| formatted_event[key] = value_constructor value, event.payload }
|
||||
options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) }
|
||||
formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true"
|
||||
formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true"
|
||||
create_event :payload => formatted_event
|
||||
|
|
285
app/models/agents/human_task_agent.rb
Normal file
285
app/models/agents/human_task_agent.rb
Normal file
|
@ -0,0 +1,285 @@
|
|||
require 'rturk'
|
||||
|
||||
module Agents
|
||||
class HumanTaskAgent < Agent
|
||||
default_schedule "every_10m"
|
||||
|
||||
description <<-MD
|
||||
You can use a HumanTaskAgent to create Human Intelligence Tasks (HITs) on Mechanical Turk.
|
||||
|
||||
HITs can be created in response to events, or on a schedule. Set `trigger_on` to either `schedule` or `event`.
|
||||
|
||||
The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one. To configure how often a new HIT
|
||||
should be submitted when in `schedule` mode, set `submission_period` to a number of hours.
|
||||
|
||||
If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
|
||||
For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
|
||||
|
||||
{
|
||||
"expected_receive_period_in_days": 2,
|
||||
"trigger_on": "event",
|
||||
"hit": {
|
||||
"max_assignments": 1,
|
||||
"title": "Sentiment evaluation",
|
||||
"description": "Please rate the sentiment of this message: '<$.message>'",
|
||||
"reward": 0.05,
|
||||
"questions": [
|
||||
{
|
||||
"type": "selection",
|
||||
"key": "sentiment",
|
||||
"name": "Sentiment",
|
||||
"required": "true",
|
||||
"question": "Please select the best sentiment value:",
|
||||
"selections": [
|
||||
{ "key": "happy", "text": "Happy" },
|
||||
{ "key": "sad", "text": "Sad" },
|
||||
{ "key": "neutral", "text": "Neutral" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "free_text",
|
||||
"key": "feedback",
|
||||
"name": "Have any feedback for us?",
|
||||
"required": "false",
|
||||
"question": "Feedback",
|
||||
"default": "Type here...",
|
||||
"min_length": "2",
|
||||
"max_length": "2000"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
As you can see, you configure the created HIT with the `hit` option. Required fields are `title`, which is the
|
||||
title of the created HIT, `description`, which is the description of the HIT, and `questions` which is an array of
|
||||
questions. Questions can be of `type` _selection_ or _free\\_text_. Both types require the `key`, `name`, `required`,
|
||||
`type`, and `question` configuration options. Additionally, _selection_ requires a `selections` array of options, each of
|
||||
which contain `key` and `text`. For _free\\_text_, the special configuration options are all optional, and are
|
||||
`default`, `min_length`, and `max_length`.
|
||||
|
||||
If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
|
||||
automatically select the majority vote for each question across all `max_assignments`.
|
||||
|
||||
As with most Agents, `expected_receive_period_in_days` is required if `trigger_on` is set to `event`.
|
||||
MD
|
||||
|
||||
event_description <<-MD
|
||||
Events look like:
|
||||
|
||||
{
|
||||
}
|
||||
MD
|
||||
|
||||
def validate_options
|
||||
errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on])
|
||||
|
||||
if options[:trigger_on] == "event"
|
||||
errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present?
|
||||
elsif options[:trigger_on] == "schedule"
|
||||
errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0
|
||||
end
|
||||
|
||||
if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" }
|
||||
errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
|
||||
end
|
||||
end
|
||||
|
||||
def default_options
|
||||
{
|
||||
:expected_receive_period_in_days => 2,
|
||||
:trigger_on => "event",
|
||||
:hit =>
|
||||
{
|
||||
:max_assignments => 1,
|
||||
:title => "Sentiment evaluation",
|
||||
:description => "Please rate the sentiment of this message: '<$.message>'",
|
||||
:reward => 0.05,
|
||||
:questions =>
|
||||
[
|
||||
{
|
||||
:type => "selection",
|
||||
:key => "sentiment",
|
||||
:name => "Sentiment",
|
||||
:required => "true",
|
||||
:question => "Please select the best sentiment value:",
|
||||
:selections =>
|
||||
[
|
||||
{ :key => "happy", :text => "Happy" },
|
||||
{ :key => "sad", :text => "Sad" },
|
||||
{ :key => "neutral", :text => "Neutral" }
|
||||
]
|
||||
},
|
||||
{
|
||||
:type => "free_text",
|
||||
:key => "feedback",
|
||||
:name => "Have any feedback for us?",
|
||||
:required => "false",
|
||||
:question => "Feedback",
|
||||
:default => "Type here...",
|
||||
:min_length => "2",
|
||||
:max_length => "2000"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
def working?
|
||||
last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs?
|
||||
end
|
||||
|
||||
def check
|
||||
setup!
|
||||
review_hits
|
||||
|
||||
if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60
|
||||
memory[:last_schedule] = Time.now.to_i
|
||||
create_hit
|
||||
end
|
||||
end
|
||||
|
||||
def receive(incoming_events)
|
||||
if options[:trigger_on] == "event"
|
||||
setup!
|
||||
|
||||
incoming_events.each do |event|
|
||||
create_hit event
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# To be moved either into an initilizer or a per-agent setting.
|
||||
def setup!
|
||||
RTurk::logger.level = Logger::DEBUG
|
||||
RTurk.setup(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_ACCESS_KEY'], :sandbox => ENV['AWS_SANDBOX'] == "true") unless Rails.env.test?
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def review_hits
|
||||
reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
|
||||
my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s)
|
||||
log "MTurk reports the following HITs [#{reviewable_hit_ids.to_sentence}], of which I own [#{my_reviewed_hit_ids.to_sentence}]"
|
||||
my_reviewed_hit_ids.each do |hit_id|
|
||||
hit = RTurk::Hit.new(hit_id)
|
||||
assignments = hit.assignments
|
||||
|
||||
log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
|
||||
if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
|
||||
if options[:take_majority] == "true"
|
||||
options[:hit][:questions].each do |question|
|
||||
counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo }
|
||||
assignments.each do |assignment|
|
||||
answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
|
||||
answer = answers[question[:key]]
|
||||
counts[answer] += 1
|
||||
end
|
||||
end
|
||||
else
|
||||
event = create_event :payload => { :answers => assignments.map(&:answers) }
|
||||
log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym])
|
||||
end
|
||||
|
||||
assignments.each(&:approve!)
|
||||
|
||||
memory[:hits].delete(hit_id.to_sym)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def create_hit(event = nil)
|
||||
payload = event ? event.payload : {}
|
||||
title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip
|
||||
description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip
|
||||
questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload)
|
||||
hit = RTurk::Hit.create(:title => title) do |hit|
|
||||
hit.max_assignments = (options[:hit][:max_assignments] || 1).to_i
|
||||
hit.description = description
|
||||
hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
|
||||
hit.reward = (options[:hit][:reward] || 0.05).to_f
|
||||
#hit.qualifications.add :approval_rate, { :gt => 80 }
|
||||
end
|
||||
memory[:hits] ||= {}
|
||||
memory[:hits][hit.id] = event && event.id
|
||||
log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
|
||||
end
|
||||
|
||||
# RTurk Question Form
|
||||
|
||||
class AgentQuestionForm < RTurk::QuestionForm
|
||||
needs :title, :description, :questions
|
||||
|
||||
def question_form_content
|
||||
Overview do
|
||||
Title do
|
||||
text @title
|
||||
end
|
||||
Text do
|
||||
text @description
|
||||
end
|
||||
end
|
||||
|
||||
@questions.each.with_index do |question, index|
|
||||
Question do
|
||||
QuestionIdentifier do
|
||||
text question[:key] || "question_#{index}"
|
||||
end
|
||||
DisplayName do
|
||||
text question[:name] || "Question ##{index}"
|
||||
end
|
||||
IsRequired do
|
||||
text question[:required] || 'true'
|
||||
end
|
||||
QuestionContent do
|
||||
Text do
|
||||
text question[:question]
|
||||
end
|
||||
end
|
||||
AnswerSpecification do
|
||||
if question[:type] == "selection"
|
||||
|
||||
SelectionAnswer do
|
||||
StyleSuggestion do
|
||||
text 'radiobutton'
|
||||
end
|
||||
Selections do
|
||||
question[:selections].each do |selection|
|
||||
Selection do
|
||||
SelectionIdentifier do
|
||||
text selection[:key]
|
||||
end
|
||||
Text do
|
||||
text selection[:text]
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
else
|
||||
|
||||
FreeTextAnswer do
|
||||
if question[:min_length].present? || question[:max_length].present?
|
||||
Constraints do
|
||||
lengths = {}
|
||||
lengths[:minLength] = question[:min_length].to_s if question[:min_length].present?
|
||||
lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present?
|
||||
Length lengths
|
||||
end
|
||||
end
|
||||
|
||||
if question[:default].present?
|
||||
DefaultText do
|
||||
text question[:default]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
19
lib/utils.rb
19
lib/utils.rb
|
@ -32,6 +32,25 @@ module Utils
|
|||
end
|
||||
end
|
||||
|
||||
def self.interpolate_jsonpaths(value, data)
|
||||
value.gsub(/<[^>]+>/).each { |jsonpath|
|
||||
Utils.values_at(data, jsonpath[1..-2]).first.to_s
|
||||
}
|
||||
end
|
||||
|
||||
def self.recursively_interpolate_jsonpaths(struct, data)
|
||||
case struct
|
||||
when Hash
|
||||
struct.inject({}) {|memo, (key, value)| memo[key] = recursively_interpolate_jsonpaths(value, data); memo }
|
||||
when Array
|
||||
struct.map {|elem| recursively_interpolate_jsonpaths(elem, data) }
|
||||
when String
|
||||
interpolate_jsonpaths(struct, data)
|
||||
else
|
||||
struct
|
||||
end
|
||||
end
|
||||
|
||||
def self.value_at(data, path)
|
||||
values_at(data, path).first
|
||||
end
|
||||
|
|
|
@ -27,6 +27,36 @@ describe Utils do
|
|||
end
|
||||
end
|
||||
|
||||
describe "#interpolate_jsonpaths" do
|
||||
it "interpolates jsonpath expressions between matching <>'s" do
|
||||
Utils.interpolate_jsonpaths("hello <$.there.world> this <escape works>", { :there => { :world => "WORLD" }, :works => "should work" }).should == "hello WORLD this should+work"
|
||||
end
|
||||
end
|
||||
|
||||
describe "#recursively_interpolate_jsonpaths" do
|
||||
it "interpolates all string values in a structure" do
|
||||
struct = {
|
||||
:int => 5,
|
||||
:string => "this <escape $.works>",
|
||||
:array => ["<works>", "now", "<$.there.world>"],
|
||||
:deep => {
|
||||
:string => "hello <there.world>",
|
||||
:hello => :world
|
||||
}
|
||||
}
|
||||
data = { :there => { :world => "WORLD" }, :works => "should work" }
|
||||
Utils.recursively_interpolate_jsonpaths(struct, data).should == {
|
||||
:int => 5,
|
||||
:string => "this should+work",
|
||||
:array => ["should work", "now", "WORLD"],
|
||||
:deep => {
|
||||
:string => "hello WORLD",
|
||||
:hello => :world
|
||||
}
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
describe "#value_at" do
|
||||
it "returns the value at a JSON path" do
|
||||
Utils.value_at({ :foo => { :bar => :baz }}.to_json, "foo.bar").should == "baz"
|
||||
|
|
235
spec/models/agents/human_task_agent_spec.rb
Normal file
235
spec/models/agents/human_task_agent_spec.rb
Normal file
|
@ -0,0 +1,235 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Agents::HumanTaskAgent do
|
||||
before do
|
||||
@checker = Agents::HumanTaskAgent.new(:name => "my human task agent")
|
||||
@checker.options = @checker.default_options
|
||||
@checker.user = users(:bob)
|
||||
@checker.save!
|
||||
|
||||
@event = Event.new
|
||||
@event.agent = agents(:bob_rain_notifier_agent)
|
||||
@event.payload = { :foo => { "bar" => { :baz => "a2b" } },
|
||||
:name => "Joe" }
|
||||
@event.id = 345
|
||||
end
|
||||
|
||||
describe "when 'trigger_on' is set to 'schedule'" do
|
||||
before do
|
||||
@checker.options[:trigger_on] = "schedule"
|
||||
@checker.options[:submission_period] = "2"
|
||||
@checker.options.delete(:expected_receive_period_in_days)
|
||||
end
|
||||
|
||||
it "should check for reviewable HITs frequently" do
|
||||
mock(@checker).review_hits.twice
|
||||
mock(@checker).create_hit.once
|
||||
@checker.check
|
||||
@checker.check
|
||||
end
|
||||
|
||||
it "should create HITs every 'submission_period' hours" do
|
||||
now = Time.now
|
||||
stub(Time).now { now }
|
||||
mock(@checker).review_hits.times(3)
|
||||
mock(@checker).create_hit.twice
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
end
|
||||
|
||||
it "should ignore events" do
|
||||
mock(@checker).create_hit(anything).times(0)
|
||||
@checker.receive([events(:bob_website_agent_event)])
|
||||
end
|
||||
end
|
||||
|
||||
describe "when 'trigger_on' is set to 'event'" do
|
||||
it "should not create HITs during check but should check for reviewable HITs" do
|
||||
@checker.options[:submission_period] = "2"
|
||||
now = Time.now
|
||||
stub(Time).now { now }
|
||||
mock(@checker).review_hits.times(3)
|
||||
mock(@checker).create_hit.times(0)
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
now += 1 * 60 * 60
|
||||
@checker.check
|
||||
end
|
||||
|
||||
it "should create HITs based on events" do
|
||||
mock(@checker).create_hit(events(:bob_website_agent_event)).times(1)
|
||||
@checker.receive([events(:bob_website_agent_event)])
|
||||
end
|
||||
end
|
||||
|
||||
describe "creating hits" do
|
||||
it "can create HITs based on events, interpolating their values" do
|
||||
@checker.options[:hit][:title] = "Hi <.name>"
|
||||
@checker.options[:hit][:description] = "Make something for <.name>"
|
||||
@checker.options[:hit][:questions][0][:name] = "<.name> Question 1"
|
||||
|
||||
question_form = nil
|
||||
hitInterface = OpenStruct.new
|
||||
hitInterface.id = 123
|
||||
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
|
||||
mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
|
||||
|
||||
@checker.send :create_hit, @event
|
||||
|
||||
hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
|
||||
hitInterface.reward.should == @checker.options[:hit][:reward]
|
||||
hitInterface.description.should == "Make something for Joe"
|
||||
|
||||
xml = question_form.to_xml
|
||||
xml.should include("<Title>Hi Joe</Title>")
|
||||
xml.should include("<Text>Make something for Joe</Text>")
|
||||
xml.should include("<DisplayName>Joe Question 1</DisplayName>")
|
||||
|
||||
@checker.memory[:hits][123].should == @event.id
|
||||
end
|
||||
|
||||
it "works without an event too" do
|
||||
@checker.options[:hit][:title] = "Hi <.name>"
|
||||
hitInterface = OpenStruct.new
|
||||
hitInterface.id = 123
|
||||
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
|
||||
mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
|
||||
@checker.send :create_hit
|
||||
hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments]
|
||||
hitInterface.reward.should == @checker.options[:hit][:reward]
|
||||
end
|
||||
end
|
||||
|
||||
describe "reviewing HITs" do
|
||||
class FakeHit
|
||||
def initialize(options = {})
|
||||
@options = options
|
||||
end
|
||||
|
||||
def assignments
|
||||
@options[:assignments] || []
|
||||
end
|
||||
|
||||
def max_assignments
|
||||
@options[:max_assignments] || 1
|
||||
end
|
||||
end
|
||||
|
||||
class FakeAssignment
|
||||
attr_accessor :approved
|
||||
|
||||
def initialize(options = {})
|
||||
@options = options
|
||||
end
|
||||
|
||||
def answers
|
||||
@options[:answers] || {}
|
||||
end
|
||||
|
||||
def status
|
||||
@options[:status] || ""
|
||||
end
|
||||
|
||||
def approve!
|
||||
@approved = true
|
||||
end
|
||||
end
|
||||
|
||||
it "should work on multiple HITs" do
|
||||
event2 = Event.new
|
||||
event2.agent = agents(:bob_rain_notifier_agent)
|
||||
event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } },
|
||||
:name2 => "Joe2" }
|
||||
event2.id = 3452
|
||||
|
||||
# It knows about two HITs from two different events.
|
||||
@checker.memory[:hits] = {}
|
||||
@checker.memory[:hits][:"JH3132836336DHG"] = @event.id
|
||||
@checker.memory[:hits][:"JH39AA63836DHG"] = event2.id
|
||||
|
||||
hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
|
||||
|
||||
# It looksup the two HITs that it owns. Neither are ready yet.
|
||||
mock(RTurk::Hit).new("JH3132836336DHG") { FakeHit.new }
|
||||
mock(RTurk::Hit).new("JH39AA63836DHG") { FakeHit.new }
|
||||
|
||||
@checker.send :review_hits
|
||||
end
|
||||
|
||||
it "shouldn't do anything if an assignment isn't ready" do
|
||||
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Accepted", :answers => {}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
|
||||
]
|
||||
hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
|
||||
mock(RTurk::Hit).new("JH3132836336DHG") { hit }
|
||||
|
||||
# One of the assignments isn't set to "Submitted", so this should get skipped for now.
|
||||
mock.any_instance_of(FakeAssignment).answers.times(0)
|
||||
|
||||
@checker.send :review_hits
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_false
|
||||
@checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
|
||||
end
|
||||
|
||||
it "shouldn't do anything if an assignment is missing" do
|
||||
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
|
||||
]
|
||||
hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
|
||||
mock(RTurk::Hit).new("JH3132836336DHG") { hit }
|
||||
|
||||
# One of the assignments hasn't shown up yet, so this should get skipped for now.
|
||||
mock.any_instance_of(FakeAssignment).answers.times(0)
|
||||
|
||||
@checker.send :review_hits
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_false
|
||||
@checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id }
|
||||
end
|
||||
|
||||
it "should create events when all assignments are ready" do
|
||||
@checker.memory[:hits] = { :"JH3132836336DHG" => @event.id }
|
||||
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
|
||||
assignments = [
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
|
||||
FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
|
||||
]
|
||||
hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
|
||||
mock(RTurk::Hit).new("JH3132836336DHG") { hit }
|
||||
|
||||
lambda {
|
||||
@checker.send :review_hits
|
||||
}.should change { Event.count }.by(1)
|
||||
|
||||
assignments.all? {|a| a.approved == true }.should be_true
|
||||
|
||||
@checker.events.last.payload[:answers].should == [
|
||||
{:sentiment => "neutral", :feedback => ""},
|
||||
{:sentiment => "happy", :feedback => "Take 2"}
|
||||
]
|
||||
|
||||
@checker.memory[:hits].should == {}
|
||||
end
|
||||
|
||||
describe "taking majority votes" do
|
||||
it "should only be valid when all questions are of type 'selection'" do
|
||||
|
||||
end
|
||||
|
||||
it "should take the majority votes of all questions" do
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,14 +1,14 @@
|
|||
require 'spec_helper'
|
||||
|
||||
describe Agents::PostAgent do
|
||||
before do
|
||||
@valid_params = {
|
||||
:name => "somename",
|
||||
:options => {
|
||||
:post_url => "http://www.example.com",
|
||||
:expected_receive_period_in_days => 1
|
||||
}
|
||||
}
|
||||
before do
|
||||
@valid_params = {
|
||||
:name => "somename",
|
||||
:options => {
|
||||
:post_url => "http://www.example.com",
|
||||
:expected_receive_period_in_days => 1
|
||||
}
|
||||
}
|
||||
|
||||
@checker = Agents::PostAgent.new(@valid_params)
|
||||
@checker.user = users(:jane)
|
||||
|
@ -17,55 +17,55 @@ describe Agents::PostAgent do
|
|||
@event = Event.new
|
||||
@event.agent = agents(:jane_weather_agent)
|
||||
@event.payload = {
|
||||
:somekey => "somevalue",
|
||||
:someotherkey => {
|
||||
:somekey => "value"
|
||||
}
|
||||
:somekey => "somevalue",
|
||||
:someotherkey => {
|
||||
:somekey => "value"
|
||||
}
|
||||
}
|
||||
|
||||
@sent_messages = []
|
||||
stub.any_instance_of(Agents::PostAgent).post_event { |uri,event| @sent_messages << event}
|
||||
stub.any_instance_of(Agents::PostAgent).post_event { |uri, event| @sent_messages << event }
|
||||
end
|
||||
|
||||
describe "#receive" do
|
||||
it "checks if it can handle multiple events" do
|
||||
event1 = Event.new
|
||||
event1.agent = agents(:bob_weather_agent)
|
||||
event1.payload = {
|
||||
:xyz => "value1",
|
||||
:message => "value2"
|
||||
}
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event, event1])
|
||||
}.should change { @sent_messages.length }.by(2)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#working?" do
|
||||
it "checks if events have been received within expected receive period" do
|
||||
@checker.should_not be_working
|
||||
Agents::PostAgent.async_receive @checker.id, [@event.id]
|
||||
@checker.reload.should be_working
|
||||
two_days_from_now = 2.days.from_now
|
||||
stub(Time).now { two_days_from_now }
|
||||
@checker.reload.should_not be_working
|
||||
end
|
||||
end
|
||||
|
||||
describe "validation" do
|
||||
before do
|
||||
@checker.should be_valid
|
||||
end
|
||||
|
||||
describe "#receive" do
|
||||
it "checks if it can handle multiple events" do
|
||||
event1 = Event.new
|
||||
event1.agent = agents(:bob_weather_agent)
|
||||
event1.payload = {
|
||||
:xyz => "value1",
|
||||
:message => "value2"
|
||||
}
|
||||
|
||||
lambda {
|
||||
@checker.receive([@event,event1])
|
||||
}.should change { @sent_messages.length }.by(2)
|
||||
end
|
||||
it "should validate presence of post_url" do
|
||||
@checker.options[:post_url] = ""
|
||||
@checker.should_not be_valid
|
||||
end
|
||||
|
||||
describe "#working?" do
|
||||
it "checks if events have been received within expected receive period" do
|
||||
@checker.should_not be_working
|
||||
Agents::PostAgent.async_receive @checker.id, [@event.id]
|
||||
@checker.reload.should be_working
|
||||
two_days_from_now = 2.days.from_now
|
||||
stub(Time).now { two_days_from_now }
|
||||
@checker.reload.should_not be_working
|
||||
end
|
||||
end
|
||||
|
||||
describe "validation" do
|
||||
before do
|
||||
@checker.should be_valid
|
||||
end
|
||||
|
||||
it "should validate presence of post_url" do
|
||||
@checker.options[:post_url] = ""
|
||||
@checker.should_not be_valid
|
||||
end
|
||||
|
||||
it "should validate presence of expected_receive_period_in_days" do
|
||||
@checker.options[:expected_receive_period_in_days] = ""
|
||||
@checker.should_not be_valid
|
||||
end
|
||||
it "should validate presence of expected_receive_period_in_days" do
|
||||
@checker.options[:expected_receive_period_in_days] = ""
|
||||
@checker.should_not be_valid
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Add table
Reference in a new issue