Merge branch 'master' into do_not_symbolize

Conflicts:
	app/models/agents/human_task_agent.rb
	spec/models/agents/human_task_agent_spec.rb

Gemfile.lock

@@ -63,9 +63,9 @@ GEM
       railties (>= 3.2.6, < 5)
       warden (~> 1.2.3)
     diff-lcs (1.2.4)
-    dotenv (0.8.0)
-    dotenv-rails (0.8.0)
-      dotenv (= 0.8.0)
+    dotenv (0.9.0)
+    dotenv-rails (0.9.0)
+      dotenv (= 0.9.0)
     em-http-request (1.0.3)
       addressable (>= 2.2.3)
       cookiejar

app/models/agents/human_task_agent.rb

@@ -9,9 +9,13 @@ module Agents
 
       HITs can be created in response to events, or on a schedule.  Set `trigger_on` to either `schedule` or `event`.
 
+      # Schedule
+
       The schedule of this Agent is how often it should check for completed HITs, __NOT__ how often to submit one.  To configure how often a new HIT
       should be submitted when in `schedule` mode, set `submission_period` to a number of hours.
 
+      # Example
+
       If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
       For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
 
@@ -58,8 +62,52 @@ module Agents
       which contain `key` and `text`.  For _free\\_text_, the special configuration options are all optional, and are
       `default`, `min_length`, and `max_length`.
 
-      If all of the `questions` are of `type` _selection_, you can set `take_majority` to _true_ at the top level to
-      automatically select the majority vote for each question across all `assignments`.  If all selections are numeric, an `average_answer` will also be generated.
+      # Combining answers
+
+      There are a couple of ways to combine HITs that have multiple `assignments`, all of which involve setting `combination_mode` at the top level.
+
+      ## Taking the majority
+
+      Option 1: if all of your `questions` are of `type` _selection_, you can set `combination_mode` to `take_majority`.
+      This will cause the Agent to automatically select the majority vote for each question across all `assignments` and return it as `majority_answer`.
+      If all selections are numeric, an `average_answer` will also be generated.
+
+      Option 2: you can have the Agent ask additional human workers to rank the `assignments` and return the most highly ranked answer.
+      To do this, set `combination_mode` to `poll` and provide a `poll_options` object.  Here is an example:
+
+          {
+            "trigger_on": "schedule",
+            "submission_period": 12,
+            "combination_mode": "poll",
+            "poll_options": {
+              "title": "Take a poll about some jokes",
+              "instructions": "Please rank these jokes from most funny (5) to least funny (1)",
+              "assignments": 3,
+              "row_template": "<$.joke>"
+            },
+            "hit": {
+              "assignments": 5,
+              "title": "Tell a joke",
+              "description": "Please tell me a joke",
+              "reward": 0.05,
+              "lifetime_in_seconds": "3600",
+              "questions": [
+                {
+                  "type": "free_text",
+                  "key": "joke",
+                  "name": "Your joke",
+                  "required": "true",
+                  "question": "Joke",
+                  "min_length": "2",
+                  "max_length": "2000"
+                }
+              ]
+            }
+          }
+
+      Resulting events will have the original `answers`, as well as the `poll` results, and a field called `best_answer` that contains the best answer as determined by the poll.
+
+      # Other settings
 
       `lifetime_in_seconds` is the number of seconds a HIT is left on Amazon before it's automatically closed.  The default is 1 day.
 
@@ -70,6 +118,12 @@ module Agents
       Events look like:
 
           {
+            "answers": [
+              {
+                "feedback": "Hello!",
+                "sentiment": "happy"
+              }
+            ]
           }
     MD
 
@@ -97,9 +151,13 @@ module Agents
         errors.add(:base, "all questions of type 'selection' must have a selections array with selections that set 'key' and 'name'")
       end
 
-      if options['take_majority'] == "true" && options['hit']['questions'].any? { |question| question['type'] != "selection" }
+      if take_majority? && options['hit']['questions'].any? { |question| question['type'] != "selection" }
         errors.add(:base, "all questions must be of type 'selection' to use the 'take_majority' option")
       end
+
+      if create_poll?
+        errors.add(:base, "poll_options is required when combination_mode is set to 'poll' and must have the keys 'title', 'instructions', 'row_template', and 'assignments'") unless options['poll_options'].is_a?(Hash) && options['poll_options']['title'].present? && options['poll_options']['instructions'].present? && options['poll_options']['row_template'].present? && options['poll_options']['assignments'].to_i > 0
+      end
     end
 
     def default_options
@@ -152,70 +210,152 @@ module Agents
 
       if options['trigger_on'] == "schedule" && (memory['last_schedule'] || 0) <= Time.now.to_i - options['submission_period'].to_i * 60 * 60
         memory['last_schedule'] = Time.now.to_i
-        create_hit
+        create_basic_hit
       end
     end
 
     def receive(incoming_events)
       if options['trigger_on'] == "event"
         incoming_events.each do |event|
-          create_hit event
+          create_basic_hit event
         end
       end
     end
 
     protected
 
+    def take_majority?
+      options['combination_mode'] == "take_majority" || options['take_majority'] == "true"
+    end
+
+    def create_poll?
+      options['combination_mode'] == "poll"
+    end
+
+    def event_for_hit(hit_id)
+      if memory['hits'][hit_id].is_a?(Hash)
+        Event.find_by_id(memory['hits'][hit_id]['event_id'])
+      else
+        nil
+      end
+    end
+
+    def hit_type(hit_id)
+      if memory['hits'][hit_id].is_a?(Hash) && memory['hits'][hit_id]['type']
+        memory['hits'][hit_id]['type']
+      else
+        'user'
+      end
+    end
+
     def review_hits
       reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids
       my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys
       if reviewable_hit_ids.length > 0
         log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]"
       end
+
       my_reviewed_hit_ids.each do |hit_id|
         hit = RTurk::Hit.new(hit_id)
         assignments = hit.assignments
 
         log "Looking at HIT #{hit_id}.  I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}"
         if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" }
-          payload = { 'answers' => assignments.map(&:answers) }
+          inbound_event = event_for_hit(hit_id)
 
-          if options['take_majority'] == "true"
-            counts = {}
-            options['hit']['questions'].each do |question|
-              question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
-              assignments.each do |assignment|
-                answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
-                answer = answers[question['key']]
-                question_counts[answer] += 1
+          if hit_type(hit_id) == 'poll'
+            # handle completed polls
+
+            log "Handling a poll: #{hit_id}"
+
+            scores = {}
+            assignments.each do |assignment|
+              assignment.answers.each do |index, rating|
+                scores[index] ||= 0
+                scores[index] += rating.to_i
               end
-              counts[question['key']] = question_counts
             end
-            payload['counts'] = counts
 
-            majority_answer = counts.inject({}) do |memo, (key, question_counts)|
-              memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
-              memo
-            end
-            payload['majority_answer'] = majority_answer
+            top_answer = scores.to_a.sort {|b, a| a.last <=> b.last }.first.first
 
-            if all_questions_are_numeric?
-              average_answer = counts.inject({}) do |memo, (key, question_counts)|
-                sum = divisor = 0
-                question_counts.to_a.each do |num, count|
-                  sum += num.to_s.to_f * count
-                  divisor += count
+            payload = {
+              'answers' => memory['hits'][hit_id]['answers'],
+              'poll' => assignments.map(&:answers),
+              'best_answer' => memory['hits'][hit_id]['answers'][top_answer.to_i - 1]
+            }
+
+            event = create_event :payload => payload
+            log "Event emitted with answer(s) for poll", :outbound_event => event, :inbound_event => inbound_event
+          else
+            # handle normal completed HITs
+            payload = { 'answers' => assignments.map(&:answers) }
+
+            if take_majority?
+              counts = {}
+              options['hit']['questions'].each do |question|
+                question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo }
+                assignments.each do |assignment|
+                  answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers)
+                  answer = answers[question['key']]
+                  question_counts[answer] += 1
                 end
-                memo[key] = sum / divisor.to_f
+                counts[question['key']] = question_counts
+              end
+              payload['counts'] = counts
+
+              majority_answer = counts.inject({}) do |memo, (key, question_counts)|
+                memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first
                 memo
               end
-              payload['average_answer'] = average_answer
+              payload['majority_answer'] = majority_answer
+
+              if all_questions_are_numeric?
+                average_answer = counts.inject({}) do |memo, (key, question_counts)|
+                  sum = divisor = 0
+                  question_counts.to_a.each do |num, count|
+                    sum += num.to_s.to_f * count
+                    divisor += count
+                  end
+                  memo[key] = sum / divisor.to_f
+                  memo
+                end
+                payload['average_answer'] = average_answer
+              end
+            end
+
+            if create_poll?
+              questions = []
+              selections = 5.times.map { |i| { 'key' => i+1, 'text' => i+1 } }.reverse
+              assignments.length.times do |index|
+                questions << {
+                  'type' => "selection",
+                  'name' => "Item #{index + 1}",
+                  'key' => index,
+                  'required' => "true",
+                  'question' => Utils.interpolate_jsonpaths(options['poll_options']['row_template'], assignments[index].answers),
+                  'selections' => selections
+                }
+              end
+
+              poll_hit = create_hit 'title' => options['poll_options']['title'],
+                                    'description' => options['poll_options']['instructions'],
+                                    'questions' => questions,
+                                    'assignments' => options['poll_options']['assignments'],
+                                    'lifetime_in_seconds' => options['poll_options']['lifetime_in_seconds'],
+                                    'reward' => options['poll_options']['reward'],
+                                    'payload' => inbound_event && inbound_event.payload,
+                                    'metadata' => { 'type' => 'poll',
+                                                    'original_hit' => hit_id,
+                                                    'answers' => assignments.map(&:answers),
+                                                    'event_id' => inbound_event && inbound_event.id }
+
+              log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}.  Original HIT: #{hit_id}", :inbound_event => inbound_event
+            else
+              event = create_event :payload => payload
+              log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event
             end
           end
 
-          event = create_event :payload => payload
-          log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => Event.find_by_id(memory['hits'][hit_id])
-
           assignments.each(&:approve!)
           hit.dispose!
 
@@ -232,22 +372,35 @@ module Agents
       end
     end
 
-    def create_hit(event = nil)
-      payload = event ? event.payload : {}
-      title = Utils.interpolate_jsonpaths(options['hit']['title'], payload).strip
-      description = Utils.interpolate_jsonpaths(options['hit']['description'], payload).strip
-      questions = Utils.recursively_interpolate_jsonpaths(options['hit']['questions'], payload)
+    def create_basic_hit(event = nil)
+      hit = create_hit 'title' => options['hit']['title'],
+                       'description' => options['hit']['description'],
+                       'questions' => options['hit']['questions'],
+                       'assignments' => options['hit']['assignments'],
+                       'lifetime_in_seconds' => options['hit']['lifetime_in_seconds'],
+                       'reward' => options['hit']['reward'],
+                       'payload' => event && event.payload,
+                       'metadata' => { 'event_id' => event && event.id }
+
+      log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
+    end
+
+    def create_hit(opts = {})
+      payload = opts['payload'] || {}
+      title = Utils.interpolate_jsonpaths(opts['title'], payload).strip
+      description = Utils.interpolate_jsonpaths(opts['description'], payload).strip
+      questions = Utils.recursively_interpolate_jsonpaths(opts['questions'], payload)
       hit = RTurk::Hit.create(:title => title) do |hit|
-        hit.max_assignments = (options['hit']['assignments'] || 1).to_i
+        hit.max_assignments = (opts['assignments'] || 1).to_i
         hit.description = description
-        hit.lifetime = (options['hit']['lifetime_in_seconds'] || 24 * 60 * 60).to_i
+        hit.lifetime = (opts['lifetime_in_seconds'] || 24 * 60 * 60).to_i
         hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions)
-        hit.reward = (options['hit']['reward'] || 0.05).to_f
+        hit.reward = (opts['reward'] || 0.05).to_f
         #hit.qualifications.add :approval_rate, { :gt => 80 }
       end
       memory['hits'] ||= {}
-      memory['hits'][hit.id] = event && event.id
-      log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event
+      memory['hits'][hit.id] = opts['metadata'] || {}
+      hit
     end
 
     # RTurk Question Form
@@ -328,4 +481,4 @@ module Agents
       end
     end
   end
-end
+end

app/models/agents/webhook_agent.rb

@@ -0,0 +1,62 @@
+module Agents
+  class WebhookAgent < Agent
+    cannot_be_scheduled!
+
+    description  do
+        <<-MD
+        Use this Agent to create events by receiving webhooks from any source.
+
+        In order to create events with this agent, make a POST request to:
+        ```
+           https://#{ENV['DOMAIN']}/users/#{user.id}/webhooks/#{id || '<id>'}/:secret
+        ``` where `:secret` is specified in your options.
+
+        The
+
+        Options:
+
+          * `secret` - A token that the host will provide for authentication.
+          * `expected_receive_period_in_days` - How often you expect to receive
+            events this way. Used to determine if the agent is working.
+          * `payload_path` - JSONPath of the attribute in the POST body to be
+            used as the Event payload.
+      MD
+    end
+
+    event_description do
+      <<-MD
+        The event payload is base on the value of the `payload_path` option,
+        which is set to `#{options['payload_path']}`.
+      MD
+    end
+
+    def default_options
+      { "secret" => "supersecretstring",
+        "expected_receive_period_in_days" => 1,
+        "payload_path" => "payload"}
+    end
+
+    def receive_webhook(params)
+      secret = params.delete('secret')
+      return ["Not Authorized", 401] unless secret == options['secret']
+
+      create_event(:payload => payload_for(params))
+
+      ['Event Created', 201]
+    end
+
+    def working?
+      event_created_within(options['expected_receive_period_in_days']) && !recent_error_logs?
+    end
+
+    def validate_options
+      unless options['secret'].present?
+        errors.add(:base, "Must specify a secret for 'Authenticating' requests")
+      end
+    end
+
+    def payload_for(params)
+      Utils.value_at(params, options['payload_path']) || {}
+    end
+  end
+end

lib/capistrano/sync.rb

@@ -1,5 +1,6 @@
 require 'yaml'
 require 'pathname'
+require 'dotenv'
 
 # Edited by Andrew Cantino.  Based on: https://gist.github.com/339471
 
@@ -99,6 +100,28 @@ namespace :sync do
     return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
   end
 
+  # Used by remote_database_config to parse the remote .env file.  Depends on the dotenv-rails gem.
+  class RemoteEnvLoader < Dotenv::Environment
+    def initialize(data)
+      @data = data
+      load
+    end
+
+    def with_loaded_env
+      begin
+        saved_env = ENV.to_hash.dup
+        ENV.update(self)
+        yield
+      ensure
+        ENV.replace(saved_env)
+      end
+    end
+
+    def read
+      @data.split("\n")
+    end
+  end
+
   #
   # Reads the database credentials from the remote config/database.yml file
   # +db+ the name of the environment to get the credentials for
@@ -106,7 +129,13 @@ namespace :sync do
   #
   def remote_database_config(db)
     remote_config = capture("cat #{current_path}/config/database.yml")
-    database = YAML::load(remote_config)
+    remote_env = capture("cat #{current_path}/.env")
+
+    database = nil
+    RemoteEnvLoader.new(remote_env).with_loaded_env do
+      database = YAML::load(ERB.new(remote_config).result)
+    end
+
     return database["#{db}"]['username'], database["#{db}"]['password'], database["#{db}"]['database'], database["#{db}"]['host']
   end
 

spec/models/agents/human_task_agent_spec.rb

@@ -108,7 +108,43 @@ describe Agents::HumanTaskAgent do
       @checker.should_not be_valid
     end
 
-    it "requires that all questions be of type 'selection' when `take_majority` is `true`" do
+    it "requires that 'poll_options' be present and populated when 'combination_mode' is set to 'poll'" do
+      @checker.options['combination_mode'] = "poll"
+      @checker.should_not be_valid
+      @checker.options['poll_options'] = {}
+      @checker.should_not be_valid
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
+                                           'instructions' => "Rank these by how funny they are",
+                                           'assignments' => 3,
+                                           'row_template' => "<$.joke>" }
+      @checker.should be_valid
+      @checker.options['poll_options'] = { 'instructions' => "Rank these by how funny they are",
+                                           'assignments' => 3,
+                                           'row_template' => "<$.joke>" }
+      @checker.should_not be_valid
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
+                                           'assignments' => 3,
+                                           'row_template' => "<$.joke>" }
+      @checker.should_not be_valid
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
+                                           'instructions' => "Rank these by how funny they are",
+                                           'row_template' => "<$.joke>" }
+      @checker.should_not be_valid
+      @checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
+                                           'instructions' => "Rank these by how funny they are",
+                                           'assignments' => 3}
+      @checker.should_not be_valid
+    end
+
+    it "requires that all questions be of type 'selection' when 'combination_mode' is 'take_majority'" do
+      @checker.options['combination_mode'] = "take_majority"
+      @checker.should_not be_valid
+      @checker.options['hit']['questions'][1]['type'] = "selection"
+      @checker.options['hit']['questions'][1]['selections'] = @checker.options['hit']['questions'][0]['selections']
+      @checker.should be_valid
+    end
+
+    it "accepts 'take_majority': 'true' for legacy support" do
       @checker.options['take_majority'] = "true"
       @checker.should_not be_valid
       @checker.options['hit']['questions'][1]['type'] = "selection"
@@ -126,7 +162,7 @@ describe Agents::HumanTaskAgent do
 
     it "should check for reviewable HITs frequently" do
       mock(@checker).review_hits.twice
-      mock(@checker).create_hit.once
+      mock(@checker).create_basic_hit.once
       @checker.check
       @checker.check
     end
@@ -135,7 +171,7 @@ describe Agents::HumanTaskAgent do
       now = Time.now
       stub(Time).now { now }
       mock(@checker).review_hits.times(3)
-      mock(@checker).create_hit.twice
+      mock(@checker).create_basic_hit.twice
       @checker.check
       now += 1 * 60 * 60
       @checker.check
@@ -144,7 +180,7 @@ describe Agents::HumanTaskAgent do
     end
 
     it "should ignore events" do
-      mock(@checker).create_hit(anything).times(0)
+      mock(@checker).create_basic_hit(anything).times(0)
       @checker.receive([events(:bob_website_agent_event)])
     end
   end
@@ -155,7 +191,7 @@ describe Agents::HumanTaskAgent do
       now = Time.now
       stub(Time).now { now }
       mock(@checker).review_hits.times(3)
-      mock(@checker).create_hit.times(0)
+      mock(@checker).create_basic_hit.times(0)
       @checker.check
       now += 1 * 60 * 60
       @checker.check
@@ -164,7 +200,7 @@ describe Agents::HumanTaskAgent do
     end
 
     it "should create HITs based on events" do
-      mock(@checker).create_hit(events(:bob_website_agent_event)).times(1)
+      mock(@checker).create_basic_hit(events(:bob_website_agent_event)).times(1)
       @checker.receive([events(:bob_website_agent_event)])
     end
   end
@@ -181,7 +217,7 @@ describe Agents::HumanTaskAgent do
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
       mock(RTurk::Hit).create(:title => "Hi Joe").yields(hitInterface) { hitInterface }
 
-      @checker.send :create_hit, @event
+      @checker.send :create_basic_hit, @event
 
       hitInterface.max_assignments.should == @checker.options['hit']['assignments']
       hitInterface.reward.should == @checker.options['hit']['reward']
@@ -192,7 +228,7 @@ describe Agents::HumanTaskAgent do
       xml.should include("<Text>Make something for Joe</Text>")
       xml.should include("<DisplayName>Joe Question 1</DisplayName>")
 
-      @checker.memory['hits'][123].should == @event.id
+      @checker.memory['hits'][123]['event_id'].should == @event.id
     end
 
     it "works without an event too" do
@@ -201,7 +237,7 @@ describe Agents::HumanTaskAgent do
       hitInterface.id = 123
       mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
       mock(RTurk::Hit).create(:title => "Hi").yields(hitInterface) { hitInterface }
-      @checker.send :create_hit
+      @checker.send :create_basic_hit
       hitInterface.max_assignments.should == @checker.options['hit']['assignments']
       hitInterface.reward.should == @checker.options['hit']['reward']
     end
@@ -259,8 +295,8 @@ describe Agents::HumanTaskAgent do
 
       # It knows about two HITs from two different events.
       @checker.memory['hits'] = {}
-      @checker.memory['hits']["JH3132836336DHG"] = @event.id
-      @checker.memory['hits']["JH39AA63836DHG"] = event2.id
+      @checker.memory['hits']["JH3132836336DHG"] = { 'event_id' => @event.id }
+      @checker.memory['hits']["JH39AA63836DHG"] = { 'event_id' => event2.id }
 
       hit_ids = %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345]
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { hit_ids } } # It sees 3 HITs.
@@ -273,7 +309,7 @@ describe Agents::HumanTaskAgent do
     end
 
     it "shouldn't do anything if an assignment isn't ready" do
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Accepted", :answers => {}),
@@ -288,11 +324,11 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
     end
 
     it "shouldn't do anything if an assignment is missing" do
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy", "feedback"=>"Take 2"})
@@ -306,11 +342,11 @@ describe Agents::HumanTaskAgent do
       @checker.send :review_hits
 
       assignments.all? {|a| a.approved == true }.should be_false
-      @checker.memory['hits'].should == { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'].should == { "JH3132836336DHG" => { 'event_id' => @event.id } }
     end
 
     it "should create events when all assignments are ready" do
-      @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
+      @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
       mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       assignments = [
         FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>""}),
@@ -337,8 +373,8 @@ describe Agents::HumanTaskAgent do
 
     describe "taking majority votes" do
       before do
-        @checker.options['take_majority'] = "true"
-        @checker.memory['hits'] = { "JH3132836336DHG" => @event.id }
+        @checker.options['combination_mode'] = "take_majority"
+        @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
         mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
       end
 
@@ -386,6 +422,10 @@ describe Agents::HumanTaskAgent do
       end
 
       it "should also provide an average answer when all questions are numeric" do
+        # it should accept 'take_majority': 'true' as well for legacy support.  Demonstrating that here.
+        @checker.options.delete :combination_mode
+        @checker.options['take_majority'] = "true"
+
         @checker.options['hit']['questions'] = [
           {
             'type' => "selection",
@@ -435,5 +475,118 @@ describe Agents::HumanTaskAgent do
         @checker.memory['hits'].should == {}
       end
     end
+
+    describe "creating and reviewing polls" do
+      before do
+        @checker.options['combination_mode'] = "poll"
+        @checker.options['poll_options'] = {
+          'title' => "Hi!",
+          'instructions' => "hello!",
+          'assignments' => 2,
+          'row_template' => "This is <.sentiment>"
+        }
+        @event.save!
+        mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }
+      end
+
+      it "creates a poll using the row_template, message, and correct number of assignments" do
+        @checker.memory['hits'] = { "JH3132836336DHG" => { 'event_id' => @event.id } }
+
+        # Mock out the HIT's submitted assignments.
+        assignments = [
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"sad",     "feedback"=>"This is my feedback 1"}),
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"neutral", "feedback"=>"This is my feedback 2"}),
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 3"}),
+          FakeAssignment.new(:status => "Submitted", :answers => {"sentiment"=>"happy",   "feedback"=>"This is my feedback 4"})
+        ]
+        hit = FakeHit.new(:max_assignments => 4, :assignments => assignments)
+        mock(RTurk::Hit).new("JH3132836336DHG") { hit }
+
+        @checker.memory['hits']["JH3132836336DHG"].should be_present
+
+        # Setup mocks for HIT creation
+
+        question_form = nil
+        hitInterface = OpenStruct.new
+        hitInterface.id = "JH39AA63836DH12345"
+        mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm) { |agent_question_form_instance| question_form = agent_question_form_instance }
+        mock(RTurk::Hit).create(:title => "Hi!").yields(hitInterface) { hitInterface }
+
+        # And finally, the test.
+
+        lambda {
+          @checker.send :review_hits
+        }.should change { Event.count }.by(0) # it does not emit an event until all poll results are in
+
+        # it approves the existing assignments
+
+        assignments.all? {|a| a.approved == true }.should be_true
+        hit.should be_disposed
+
+        # it creates a new HIT for the poll
+
+        hitInterface.max_assignments.should == @checker.options['poll_options']['assignments']
+        hitInterface.description.should == @checker.options['poll_options']['instructions']
+
+        xml = question_form.to_xml
+        xml.should include("<Text>This is happy</Text>")
+        xml.should include("<Text>This is neutral</Text>")
+        xml.should include("<Text>This is sad</Text>")
+
+        @checker.save
+        @checker.reload
+        @checker.memory['hits']["JH3132836336DHG"].should_not be_present
+        @checker.memory['hits']["JH39AA63836DH12345"].should be_present
+        @checker.memory['hits']["JH39AA63836DH12345"]['event_id'].should == @event.id
+        @checker.memory['hits']["JH39AA63836DH12345"]['type'].should == "poll"
+        @checker.memory['hits']["JH39AA63836DH12345"]['original_hit'].should == "JH3132836336DHG"
+        @checker.memory['hits']["JH39AA63836DH12345"]['answers'].length.should == 4
+      end
+
+      it "emits an event when all poll results are in, containing the data from the best answer, plus all others" do
+        original_answers = [
+          { 'sentiment' => "sad",     'feedback' => "This is my feedback 1"},
+          { 'sentiment' => "neutral", 'feedback' => "This is my feedback 2"},
+          { 'sentiment' => "happy",   'feedback' => "This is my feedback 3"},
+          { 'sentiment' => "happy",   'feedback' => "This is my feedback 4"}
+        ]
+
+        @checker.memory['hits'] = {
+          'JH39AA63836DH12345' => {
+            'type' => 'poll',
+            'original_hit' => "JH3132836336DHG",
+            'answers' => original_answers,
+            'event_id' => 345
+          }
+        }
+
+        # Mock out the HIT's submitted assignments.
+        assignments = [
+          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "2", "2" => "5", "3" => "3", "4" => "2"}),
+          FakeAssignment.new(:status => "Submitted", :answers => {"1" => "3", "2" => "4", "3" => "1", "4" => "4"})
+        ]
+        hit = FakeHit.new(:max_assignments => 2, :assignments => assignments)
+        mock(RTurk::Hit).new("JH39AA63836DH12345") { hit }
+
+        @checker.memory['hits']["JH39AA63836DH12345"].should be_present
+
+        lambda {
+          @checker.send :review_hits
+        }.should change { Event.count }.by(1)
+
+        # It emits an event
+
+        @checker.events.last.payload['answers'].should == original_answers
+        @checker.events.last.payload['poll'].should == [{"1" => "2", "2" => "5", "3" => "3", "4" => "2"}, {"1" => "3", "2" => "4", "3" => "1", "4" => "4"}]
+        @checker.events.last.payload['best_answer'].should == {'sentiment' => "neutral", 'feedback' => "This is my feedback 2"}
+
+        # it approves the existing assignments
+
+        assignments.all? {|a| a.approved == true }.should be_true
+        hit.should be_disposed
+
+        @checker.memory['hits'].should be_empty
+      end
+    end
   end
-end
+end

spec/models/agents/webhook_agent_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Agents::WebhookAgent do
+  let(:agent) do
+    _agent = Agents::WebhookAgent.new(:name => 'webhook',
+                                      :options => { 'secret' => 'foobar', 'payload_path' => 'payload' })
+    _agent.user = users(:bob)
+    _agent.save!
+    _agent
+  end
+  let(:payload) { {'some' => 'info'} }
+
+  describe 'receive_webhook' do
+    it 'should create event if secret matches' do
+      out = nil
+      lambda {
+        out = agent.receive_webhook('secret' => 'foobar', 'payload' => payload)
+      }.should change { Event.count }.by(1)
+      out.should eq(['Event Created', 201])
+      Event.last.payload.should eq(payload)
+    end
+
+    it 'should not create event if secrets dont match' do
+      out = nil
+      lambda {
+        out = agent.receive_webhook('secret' => 'bazbat', 'payload' => payload)
+      }.should change { Event.count }.by(0)
+      out.should eq(['Not Authorized', 401])
+    end
+  end
+end