From 0120ccb9c174dc3093f3c0f1c6bfaadd071eb3c8 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Wed, 28 Aug 2013 18:02:47 -0600 Subject: [PATCH] Basic HuamnTaskAgent with specs --- .env.example | 30 +- Gemfile | 1 + Gemfile.lock | 11 +- .../javascripts/worker-checker.js.coffee | 2 +- app/models/agent.rb | 1 + app/models/agents/event_formatting_agent.rb | 8 +- app/models/agents/human_task_agent.rb | 285 ++++++++++++++++++ lib/utils.rb | 19 ++ spec/lib/utils_spec.rb | 30 ++ spec/models/agents/human_task_agent_spec.rb | 235 +++++++++++++++ spec/models/agents/post_agent_spec.rb | 102 +++---- 11 files changed, 661 insertions(+), 63 deletions(-) create mode 100644 app/models/agents/human_task_agent.rb create mode 100644 spec/models/agents/human_task_agent_spec.rb diff --git a/.env.example b/.env.example index cdd35f34..51eb1727 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,10 @@ APP_SECRET_TOKEN=REPLACE_ME_NOW! # for development, but it needs to be changed when you deploy to a production environment. DOMAIN=localhost:3000 -# Database Setup +############################ +# Database Setup # +############################ + DATABASE_ADAPTER=mysql2 DATABASE_ENCODING=utf8 DATABASE_RECONNECT=true @@ -24,6 +27,10 @@ DATABASE_PASSWORD="" # Configure Rails environment. This should only be needed in production and may cause errors in development. # RAILS_ENV=production +############################# +# Email Configuration # +############################# + # Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. SMTP_DOMAIN=your-domain-here.com @@ -37,9 +44,28 @@ SMTP_ENABLE_STARTTLS_AUTO=true # The address from which system emails will appear to be sent. EMAIL_FROM_ADDRESS=from_address@gmail.com +############################ +# Allowing Signups # +############################ + # This invitation code will be required for users to signup with your Huginn installation. # You can see its use in user.rb. INVITATION_CODE=try-huginn +########################### +# Agent Logging # +########################### + # Number of lines of log messages to keep per Agent -AGENT_LOG_LENGTH=100 +AGENT_LOG_LENGTH=200 + +############################# +# AWS and Mechanical Turk # +############################# + +# AWS Credentials for MTurk +AWS_ACCESS_KEY_ID="your aws access key id" +AWS_ACCESS_KEY="your aws access key" + +# Set AWS_SANDBOX to true if you're developing Huginn code. +AWS_SANDBOX=false diff --git a/Gemfile b/Gemfile index a24ed760..4d6856cd 100644 --- a/Gemfile +++ b/Gemfile @@ -32,6 +32,7 @@ gem 'kramdown' gem "typhoeus" gem 'nokogiri' gem 'wunderground' +gem 'rturk' gem "twitter" gem 'twitter-stream', '>=0.1.16' diff --git a/Gemfile.lock b/Gemfile.lock index 0b834b24..c44bea33 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -74,6 +74,8 @@ GEM http_parser.rb (>= 0.5.3) em-socksify (0.3.0) eventmachine (>= 1.0.0.beta.4) + erector (0.9.0) + treetop (>= 1.2.3) erubis (2.7.0) ethon (0.5.12) ffi (>= 1.3.0) @@ -118,7 +120,7 @@ GEM mime-types (~> 1.16) treetop (~> 1.4.8) method_source (0.8.1) - mime-types (1.23) + mime-types (1.24) mini_portile (0.5.1) multi_json (1.7.9) multi_xml (0.5.5) @@ -182,6 +184,10 @@ GEM rspec-core (~> 2.14.0) rspec-expectations (~> 2.14.0) rspec-mocks (~> 2.14.0) + rturk (2.11.0) + erector + nokogiri + rest-client rufus-scheduler (2.0.22) tzinfo (>= 0.3.23) safe_yaml (0.9.5) @@ -205,7 +211,7 @@ GEM system_timer (1.2.4) thor (0.18.1) tilt (1.4.1) - treetop (1.4.14) + treetop (1.4.15) polyglot polyglot (>= 0.3.1) twilio-ruby (3.10.0) @@ -269,6 +275,7 @@ DEPENDENCIES rr rspec rspec-rails + rturk rufus-scheduler sass-rails (~> 3.2.3) select2-rails diff --git a/app/assets/javascripts/worker-checker.js.coffee b/app/assets/javascripts/worker-checker.js.coffee index 5869d26e..7e80cd88 100644 --- a/app/assets/javascripts/worker-checker.js.coffee +++ b/app/assets/javascripts/worker-checker.js.coffee @@ -8,7 +8,7 @@ $ -> if json.pending? && json.pending > 0 tooltipOptions = { - title: "#{json.pending} pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures" + title: "#{json.pending} jobs pending, #{json.awaiting_retry} awaiting retry, and #{json.recent_failures} recent failures" delay: 0 placement: "bottom" trigger: "hover" diff --git a/app/models/agent.rb b/app/models/agent.rb index b4fcb83e..7ea745b5 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -147,6 +147,7 @@ class Agent < ActiveRecord::Base end def log(message, options = {}) + puts "Agent##{id}: #{message}" unless Rails.env.test? AgentLog.log_for_agent(self, message, options) end diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb index f794bc66..cc27c5da 100644 --- a/app/models/agents/event_formatting_agent.rb +++ b/app/models/agents/event_formatting_agent.rb @@ -66,16 +66,10 @@ module Agents !recent_error_logs? end - def value_constructor(value, payload) - value.gsub(/<[^>]+>/).each { |jsonpath| - Utils.values_at(payload, jsonpath[1..-2]).first.to_s - } - end - def receive(incoming_events) incoming_events.each do |event| formatted_event = options[:mode].to_s == "merge" ? event.payload : {} - options[:instructions].each_pair {|key, value| formatted_event[key] = value_constructor value, event.payload } + options[:instructions].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, event.payload) } formatted_event[:agent] = Agent.find(event.agent_id).type.slice!(8..-1) unless options[:skip_agent].to_s == "true" formatted_event[:created_at] = event.created_at unless options[:skip_created_at].to_s == "true" create_event :payload => formatted_event diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb new file mode 100644 index 00000000..2b20706b --- /dev/null +++ b/app/models/agents/human_task_agent.rb @@ -0,0 +1,285 @@ +require 'rturk' + +module Agents + class HumanTaskAgent < Agent + default_schedule "every_10m" + + description <<-MD + You can use a HumanTaskAgent to create Human Intelligence Tasks (HITs) on Mechanical Turk. + + HITs can be created in response to events, or on a schedule. Set `trigger_on` to either `schedule` or `event`. + + The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one. To configure how often a new HIT + should be submitted when in `schedule` mode, set `submission_period` to a number of hours. + + If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters. + For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this: + + { + "expected_receive_period_in_days": 2, + "trigger_on": "event", + "hit": { + "max_assignments": 1, + "title": "Sentiment evaluation", + "description": "Please rate the sentiment of this message: '<$.message>'", + "reward": 0.05, + "questions": [ + { + "type": "selection", + "key": "sentiment", + "name": "Sentiment", + "required": "true", + "question": "Please select the best sentiment value:", + "selections": [ + { "key": "happy", "text": "Happy" }, + { "key": "sad", "text": "Sad" }, + { "key": "neutral", "text": "Neutral" } + ] + }, + { + "type": "free_text", + "key": "feedback", + "name": "Have any feedback for us?", + "required": "false", + "question": "Feedback", + "default": "Type here...", + "min_length": "2", + "max_length": "2000" + } + ] + } + } + + As you can see, you configure the created HIT with the `hit` option. Required fields are `title`, which is the + title of the created HIT, `description`, which is the description of the HIT, and `questions` which is an array of + questions. Questions can be of `type` _selection_ or _free\\_text_. Both types require the `key`, `name`, `required`, + `type`, and `question` configuration options. Additionally, _selection_ requires a `selections` array of options, each of + which contain `key` and `text`. For _free\\_text_, the special configuration options are all optional, and are + `default`, `min_length`, and `max_length`. + + If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to + automatically select the majority vote for each question across all `max_assignments`. + + As with most Agents, `expected_receive_period_in_days` is required if `trigger_on` is set to `event`. + MD + + event_description <<-MD + Events look like: + + { + } + MD + + def validate_options + errors.add(:base, "'trigger_on' must be one of 'schedule' or 'event'") unless %w[schedule event].include?(options[:trigger_on]) + + if options[:trigger_on] == "event" + errors.add(:base, "'expected_receive_period_in_days' is required when 'trigger_on' is set to 'event'") unless options[:expected_receive_period_in_days].present? + elsif options[:trigger_on] == "schedule" + errors.add(:base, "'submission_period' must be set to a positive number of hours when 'trigger_on' is set to 'schedule'") unless options[:submission_period].present? && options[:submission_period].to_i > 0 + end + + if options[:take_majority] == "true" && options[:hit][:questions].any? { |question| question[:type] != "selection" } + errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option") + end + end + + def default_options + { + :expected_receive_period_in_days => 2, + :trigger_on => "event", + :hit => + { + :max_assignments => 1, + :title => "Sentiment evaluation", + :description => "Please rate the sentiment of this message: '<$.message>'", + :reward => 0.05, + :questions => + [ + { + :type => "selection", + :key => "sentiment", + :name => "Sentiment", + :required => "true", + :question => "Please select the best sentiment value:", + :selections => + [ + { :key => "happy", :text => "Happy" }, + { :key => "sad", :text => "Sad" }, + { :key => "neutral", :text => "Neutral" } + ] + }, + { + :type => "free_text", + :key => "feedback", + :name => "Have any feedback for us?", + :required => "false", + :question => "Feedback", + :default => "Type here...", + :min_length => "2", + :max_length => "2000" + } + ] + } + } + end + + def working? + last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago && !recent_error_logs? + end + + def check + setup! + review_hits + + if options[:trigger_on] == "schedule" && (memory[:last_schedule] || 0) <= Time.now.to_i - options[:submission_period].to_i * 60 * 60 + memory[:last_schedule] = Time.now.to_i + create_hit + end + end + + def receive(incoming_events) + if options[:trigger_on] == "event" + setup! + + incoming_events.each do |event| + create_hit event + end + end + end + + # To be moved either into an initilizer or a per-agent setting. + def setup! + RTurk::logger.level = Logger::DEBUG + RTurk.setup(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_ACCESS_KEY'], :sandbox => ENV['AWS_SANDBOX'] == "true") unless Rails.env.test? + end + + protected + + def review_hits + reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids + my_reviewed_hit_ids = reviewable_hit_ids & (memory[:hits] || {}).keys.map(&:to_s) + log "MTurk reports the following HITs [#{reviewable_hit_ids.to_sentence}], of which I own [#{my_reviewed_hit_ids.to_sentence}]" + my_reviewed_hit_ids.each do |hit_id| + hit = RTurk::Hit.new(hit_id) + assignments = hit.assignments + + log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}" + if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" } + if options[:take_majority] == "true" + options[:hit][:questions].each do |question| + counts = question[:selections].inject({}) { |memo, selection| memo[selection[:key]] = 0; memo } + assignments.each do |assignment| + answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers) + answer = answers[question[:key]] + counts[answer] += 1 + end + end + else + event = create_event :payload => { :answers => assignments.map(&:answers) } + log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory[:hits][hit_id.to_sym]) + end + + assignments.each(&:approve!) + + memory[:hits].delete(hit_id.to_sym) + end + end + end + + def create_hit(event = nil) + payload = event ? event.payload : {} + title = Utils.interpolate_jsonpaths(options[:hit][:title], payload).strip + description = Utils.interpolate_jsonpaths(options[:hit][:description], payload).strip + questions = Utils.recursively_interpolate_jsonpaths(options[:hit][:questions], payload) + hit = RTurk::Hit.create(:title => title) do |hit| + hit.max_assignments = (options[:hit][:max_assignments] || 1).to_i + hit.description = description + hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions) + hit.reward = (options[:hit][:reward] || 0.05).to_f + #hit.qualifications.add :approval_rate, { :gt => 80 } + end + memory[:hits] ||= {} + memory[:hits][hit.id] = event && event.id + log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event + end + + # RTurk Question Form + + class AgentQuestionForm < RTurk::QuestionForm + needs :title, :description, :questions + + def question_form_content + Overview do + Title do + text @title + end + Text do + text @description + end + end + + @questions.each.with_index do |question, index| + Question do + QuestionIdentifier do + text question[:key] || "question_#{index}" + end + DisplayName do + text question[:name] || "Question ##{index}" + end + IsRequired do + text question[:required] || 'true' + end + QuestionContent do + Text do + text question[:question] + end + end + AnswerSpecification do + if question[:type] == "selection" + + SelectionAnswer do + StyleSuggestion do + text 'radiobutton' + end + Selections do + question[:selections].each do |selection| + Selection do + SelectionIdentifier do + text selection[:key] + end + Text do + text selection[:text] + end + end + end + end + end + + else + + FreeTextAnswer do + if question[:min_length].present? || question[:max_length].present? + Constraints do + lengths = {} + lengths[:minLength] = question[:min_length].to_s if question[:min_length].present? + lengths[:maxLength] = question[:max_length].to_s if question[:max_length].present? + Length lengths + end + end + + if question[:default].present? + DefaultText do + text question[:default] + end + end + end + + end + end + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/utils.rb b/lib/utils.rb index 327165ea..99322346 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -32,6 +32,25 @@ module Utils end end + def self.interpolate_jsonpaths(value, data) + value.gsub(/<[^>]+>/).each { |jsonpath| + Utils.values_at(data, jsonpath[1..-2]).first.to_s + } + end + + def self.recursively_interpolate_jsonpaths(struct, data) + case struct + when Hash + struct.inject({}) {|memo, (key, value)| memo[key] = recursively_interpolate_jsonpaths(value, data); memo } + when Array + struct.map {|elem| recursively_interpolate_jsonpaths(elem, data) } + when String + interpolate_jsonpaths(struct, data) + else + struct + end + end + def self.value_at(data, path) values_at(data, path).first end diff --git a/spec/lib/utils_spec.rb b/spec/lib/utils_spec.rb index 57de0def..377dbc54 100644 --- a/spec/lib/utils_spec.rb +++ b/spec/lib/utils_spec.rb @@ -27,6 +27,36 @@ describe Utils do end end + describe "#interpolate_jsonpaths" do + it "interpolates jsonpath expressions between matching <>'s" do + Utils.interpolate_jsonpaths("hello <$.there.world> this ", { :there => { :world => "WORLD" }, :works => "should work" }).should == "hello WORLD this should+work" + end + end + + describe "#recursively_interpolate_jsonpaths" do + it "interpolates all string values in a structure" do + struct = { + :int => 5, + :string => "this ", + :array => ["", "now", "<$.there.world>"], + :deep => { + :string => "hello ", + :hello => :world + } + } + data = { :there => { :world => "WORLD" }, :works => "should work" } + Utils.recursively_interpolate_jsonpaths(struct, data).should == { + :int => 5, + :string => "this should+work", + :array => ["should work", "now", "WORLD"], + :deep => { + :string => "hello WORLD", + :hello => :world + } + } + end + end + describe "#value_at" do it "returns the value at a JSON path" do Utils.value_at({ :foo => { :bar => :baz }}.to_json, "foo.bar").should == "baz" diff --git a/spec/models/agents/human_task_agent_spec.rb b/spec/models/agents/human_task_agent_spec.rb new file mode 100644 index 00000000..e624538f --- /dev/null +++ b/spec/models/agents/human_task_agent_spec.rb @@ -0,0 +1,235 @@ +require 'spec_helper' + +describe Agents::HumanTaskAgent do + before do + @checker = Agents::HumanTaskAgent.new(:name => "my human task agent") + @checker.options = @checker.default_options + @checker.user = users(:bob) + @checker.save! + + @event = Event.new + @event.agent = agents(:bob_rain_notifier_agent) + @event.payload = { :foo => { "bar" => { :baz => "a2b" } }, + :name => "Joe" } + @event.id = 345 + end + + describe "when 'trigger_on' is set to 'schedule'" do + before do + @checker.options[:trigger_on] = "schedule" + @checker.options[:submission_period] = "2" + @checker.options.delete(:expected_receive_period_in_days) + end + + it "should check for reviewable HITs frequently" do + mock(@checker).review_hits.twice + mock(@checker).create_hit.once + @checker.check + @checker.check + end + + it "should create HITs every 'submission_period' hours" do + now = Time.now + stub(Time).now { now } + mock(@checker).review_hits.times(3) + mock(@checker).create_hit.twice + @checker.check + now += 1 * 60 * 60 + @checker.check + now += 1 * 60 * 60 + @checker.check + end + + it "should ignore events" do + mock(@checker).create_hit(anything).times(0) + @checker.receive([events(:bob_website_agent_event)]) + end + end + + describe "when 'trigger_on' is set to 'event'" do + it "should not create HITs during check but should check for reviewable HITs" do + @checker.options[:submission_period] = "2" + now = Time.now + stub(Time).now { now } + mock(@checker).review_hits.times(3) + mock(@checker).create_hit.times(0) + @checker.check + now += 1 * 60 * 60 + @checker.check + now += 1 * 60 * 60 + @checker.check + end + + it "should create HITs based on events" do + mock(@checker).create_hit(events(:bob_website_agent_event)).times(1) + @checker.receive([events(:bob_website_agent_event)]) + end + end + + describe "creating hits" do + it "can create HITs based on events, interpolating their values" do + @checker.options[:hit][:title] = "Hi <.name>" + @checker.options[:hit][:description] = "Make something for <.name>" + @checker.options[:hit][:questions][0][:name] = "<.name> Question 1" + + question_form = nil + hitInterface = OpenStruct.new + hitInterface.id = 123 + mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance } + mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface } + + @checker.send :create_hit, @event + + hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments] + hitInterface.reward.should == @checker.options[:hit][:reward] + hitInterface.description.should == "Make something for Joe" + + xml = question_form.to_xml + xml.should include("Hi Joe") + xml.should include("Make something for Joe") + xml.should include("Joe Question 1") + + @checker.memory[:hits][123].should == @event.id + end + + it "works without an event too" do + @checker.options[:hit][:title] = "Hi <.name>" + hitInterface = OpenStruct.new + hitInterface.id = 123 + mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) + mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface } + @checker.send :create_hit + hitInterface.max_assignments.should == @checker.options[:hit][:max_assignments] + hitInterface.reward.should == @checker.options[:hit][:reward] + end + end + + describe "reviewing HITs" do + class FakeHit + def initialize(options = {}) + @options = options + end + + def assignments + @options[:assignments] || [] + end + + def max_assignments + @options[:max_assignments] || 1 + end + end + + class FakeAssignment + attr_accessor :approved + + def initialize(options = {}) + @options = options + end + + def answers + @options[:answers] || {} + end + + def status + @options[:status] || "" + end + + def approve! + @approved = true + end + end + + it "should work on multiple HITs" do + event2 = Event.new + event2.agent = agents(:bob_rain_notifier_agent) + event2.payload = { :foo2 => { "bar2" => { :baz2 => "a2b2" } }, + :name2 => "Joe2" } + event2.id = 3452 + + # It knows about two HITs from two different events. + @checker.memory[:hits] = {} + @checker.memory[:hits][:"JH3132836336DHG"] = @event.id + @checker.memory[:hits][:"JH39AA63836DHG"] = event2.id + + hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] + mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs. + + # It looksup the two HITs that it owns. Neither are ready yet. + mock(RTurk::Hit).new("JH3132836336DHG") { FakeHit.new } + mock(RTurk::Hit).new("JH39AA63836DHG") { FakeHit.new } + + @checker.send :review_hits + end + + it "shouldn't do anything if an assignment isn't ready" do + @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } + assignments = [ + FakeAssignment.new(:status => "Accepted", :answers => {}), + FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"}) + ] + hit = FakeHit.new(:max_assignments => 2, :assignments => assignments) + mock(RTurk::Hit).new("JH3132836336DHG") { hit } + + # One of the assignments isn't set to "Submitted", so this should get skipped for now. + mock.any_instance_of(FakeAssignment).answers.times(0) + + @checker.send :review_hits + + assignments.all? {|a| a.approved == true }.should be_false + @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id } + end + + it "shouldn't do anything if an assignment is missing" do + @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } + assignments = [ + FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"}) + ] + hit = FakeHit.new(:max_assignments => 2, :assignments => assignments) + mock(RTurk::Hit).new("JH3132836336DHG") { hit } + + # One of the assignments hasn't shown up yet, so this should get skipped for now. + mock.any_instance_of(FakeAssignment).answers.times(0) + + @checker.send :review_hits + + assignments.all? {|a| a.approved == true }.should be_false + @checker.memory[:hits].should == { :"JH3132836336DHG" => @event.id } + end + + it "should create events when all assignments are ready" do + @checker.memory[:hits] = { :"JH3132836336DHG" => @event.id } + mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } } + assignments = [ + FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}), + FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"}) + ] + hit = FakeHit.new(:max_assignments => 2, :assignments => assignments) + mock(RTurk::Hit).new("JH3132836336DHG") { hit } + + lambda { + @checker.send :review_hits + }.should change { Event.count }.by(1) + + assignments.all? {|a| a.approved == true }.should be_true + + @checker.events.last.payload[:answers].should == [ + {:sentiment => "neutral", :feedback => ""}, + {:sentiment => "happy", :feedback => "Take 2"} + ] + + @checker.memory[:hits].should == {} + end + + describe "taking majority votes" do + it "should only be valid when all questions are of type 'selection'" do + + end + + it "should take the majority votes of all questions" do + + end + end + end +end \ No newline at end of file diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb index 5a662930..04cc6309 100644 --- a/spec/models/agents/post_agent_spec.rb +++ b/spec/models/agents/post_agent_spec.rb @@ -1,14 +1,14 @@ require 'spec_helper' describe Agents::PostAgent do - before do - @valid_params = { - :name => "somename", - :options => { - :post_url => "http://www.example.com", - :expected_receive_period_in_days => 1 - } - } + before do + @valid_params = { + :name => "somename", + :options => { + :post_url => "http://www.example.com", + :expected_receive_period_in_days => 1 + } + } @checker = Agents::PostAgent.new(@valid_params) @checker.user = users(:jane) @@ -17,55 +17,55 @@ describe Agents::PostAgent do @event = Event.new @event.agent = agents(:jane_weather_agent) @event.payload = { - :somekey => "somevalue", - :someotherkey => { - :somekey => "value" - } + :somekey => "somevalue", + :someotherkey => { + :somekey => "value" + } } @sent_messages = [] - stub.any_instance_of(Agents::PostAgent).post_event { |uri,event| @sent_messages << event} + stub.any_instance_of(Agents::PostAgent).post_event { |uri, event| @sent_messages << event } + end + + describe "#receive" do + it "checks if it can handle multiple events" do + event1 = Event.new + event1.agent = agents(:bob_weather_agent) + event1.payload = { + :xyz => "value1", + :message => "value2" + } + + lambda { + @checker.receive([@event, event1]) + }.should change { @sent_messages.length }.by(2) + end + end + + describe "#working?" do + it "checks if events have been received within expected receive period" do + @checker.should_not be_working + Agents::PostAgent.async_receive @checker.id, [@event.id] + @checker.reload.should be_working + two_days_from_now = 2.days.from_now + stub(Time).now { two_days_from_now } + @checker.reload.should_not be_working + end + end + + describe "validation" do + before do + @checker.should be_valid end - describe "#receive" do - it "checks if it can handle multiple events" do - event1 = Event.new - event1.agent = agents(:bob_weather_agent) - event1.payload = { - :xyz => "value1", - :message => "value2" - } - - lambda { - @checker.receive([@event,event1]) - }.should change { @sent_messages.length }.by(2) - end + it "should validate presence of post_url" do + @checker.options[:post_url] = "" + @checker.should_not be_valid end - describe "#working?" do - it "checks if events have been received within expected receive period" do - @checker.should_not be_working - Agents::PostAgent.async_receive @checker.id, [@event.id] - @checker.reload.should be_working - two_days_from_now = 2.days.from_now - stub(Time).now { two_days_from_now } - @checker.reload.should_not be_working - end - end - - describe "validation" do - before do - @checker.should be_valid - end - - it "should validate presence of post_url" do - @checker.options[:post_url] = "" - @checker.should_not be_valid - end - - it "should validate presence of expected_receive_period_in_days" do - @checker.options[:expected_receive_period_in_days] = "" - @checker.should_not be_valid - end + it "should validate presence of expected_receive_period_in_days" do + @checker.options[:expected_receive_period_in_days] = "" + @checker.should_not be_valid end + end end \ No newline at end of file