From eb52eeeeea2aa0b74835ca91d0b66c00c7b132ed Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 21 Sep 2014 17:34:37 -0700 Subject: [PATCH 1/2] experimental branch to support using fewer gems for saving RAM --- Gemfile | 146 +++--- Gemfile.lock | 5 +- .../stylesheets/application.css.scss.erb | 2 +- app/concerns/twitter_concern.rb | 2 +- app/concerns/weibo_concern.rb | 6 +- app/helpers/application_helper.rb | 2 + app/helpers/dot_helper.rb | 10 +- app/helpers/service_helper.rb | 5 - app/models/agent.rb | 21 +- app/models/agents/ftpsite_agent.rb | 18 +- .../agents/google_calendar_publish_agent.rb | 3 + app/models/agents/growl_agent.rb | 9 +- app/models/agents/hipchat_agent.rb | 8 +- app/models/agents/human_task_agent.rb | 468 +++++++++--------- app/models/agents/jabber_agent.rb | 3 + app/models/agents/mqtt_agent.rb | 4 +- app/models/agents/slack_agent.rb | 8 +- app/models/agents/twilio_agent.rb | 21 +- app/models/agents/weather_agent.rb | 3 + app/models/agents/weibo_publish_agent.rb | 5 +- app/models/agents/weibo_user_agent.rb | 2 +- app/views/agents/_table.html.erb | 12 +- app/views/services/index.html.erb | 6 +- config/initializers/aws.rb | 2 +- config/initializers/omniauth.rb | 26 +- lib/huginn_scheduler.rb | 2 +- 26 files changed, 425 insertions(+), 374 deletions(-) delete mode 100644 app/helpers/service_helper.rb diff --git a/Gemfile b/Gemfile index ec59f960..c802f7f9 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,24 @@ source 'https://rubygems.org' +# Optional libraries. To conserve RAM, comment out any that you don't need, +# then run `bundle` and commit the updated Gemfile and Gemfile.lock. +gem 'twilio-ruby', '~> 3.11.5' # TwilioAgent +gem 'ruby-growl', '~> 4.1.0' # GrowlAgent +gem 'net-ftp-list', '~> 3.2.8' # FtpsiteAgent +gem 'wunderground', '~> 1.2.0' # WeatherAgent +gem 'forecast_io', '~> 2.0.0' # WeatherAgent +gem 'rturk', '~> 2.12.1' # HumanTaskAgent +gem 'weibo_2', '~> 0.1.4' # Weibo Agents +gem 'hipchat', '~> 1.2.0' # HipchatAgent +gem 'xmpp4r', '~> 0.5.6' # JabberAgent +gem "google-api-client" # GoogleCalendarPublishAgent +gem 'mqtt' # MQTTAgent +gem 'slack-notifier', '~> 0.5.0' # SlackAgent + +# Optional Services. +gem 'omniauth-37signals' # BasecampAgent +# gem 'omniauth-github' + # Bundler <1.5 does not recognize :x64_mingw as a valid platform name. # Unfortunately, it can't self-update because it errors when encountering :x64_mingw. unless Gem::Version.new(Bundler::VERSION) >= Gem::Version.new('1.5.0') @@ -7,109 +26,67 @@ unless Gem::Version.new(Bundler::VERSION) >= Gem::Version.new('1.5.0') exit 1 end -gem 'bundler', '>= 1.5.0' +gem 'protected_attributes', '~>1.0.8' # This must be loaded before some other gems, like delayed_job. -gem 'protected_attributes', '~>1.0.8' - -gem 'rails' , '4.1.5' - -case RUBY_PLATFORM -when /freebsd|netbsd|openbsd/ - # ffi (required by typhoeus via ethon) merged fixes for bugs fatal - # on these platforms after 1.9.3; no following release as yet. - gem 'ffi', github: 'ffi/ffi', branch: 'master' - - # tzinfo 1.2.0 has added support for reading zoneinfo on these - # platforms. - gem 'tzinfo', '>= 1.2.0' -when /solaris/ - # ditto - gem 'tzinfo', '>= 1.2.0' -end - -# Windows does not have zoneinfo files, so bundle the tzinfo-data gem. -gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw] - -gem 'mysql2', '~> 0.3.16' -gem 'devise', '~> 3.2.4' -gem 'kaminari', '~> 0.16.1' +gem 'ace-rails-ap', '~> 2.0.1' gem 'bootstrap-kaminari-views', '~> 0.0.3' -gem 'rufus-scheduler', '~> 3.0.8', require: false -gem 'json', '~> 1.8.1' -gem 'jsonpath', '~> 0.5.6' -gem 'twilio-ruby', '~> 3.11.5' -gem 'ruby-growl', '~> 4.1.0' -gem 'liquid', '~> 2.6.1' - +gem 'bundler', '>= 1.5.0' +gem 'cantino-twitter-stream', github: 'cantino/twitter-stream', branch: 'master' +gem 'coffee-rails', '~> 4.0.0' +gem 'daemons', '~> 1.1.9' gem 'delayed_job', '~> 4.0.0' gem 'delayed_job_active_record', '~> 4.0.0' -gem 'daemons', '~> 1.1.9' - +gem 'devise', '~> 3.2.4' +gem 'em-http-request', '~> 1.1.2' +gem 'faraday', '~> 0.9.0' +gem 'faraday_middleware' +gem 'feed-normalizer' gem 'foreman', '~> 0.63.0' - -gem 'sass-rails', '~> 4.0.0' -gem 'coffee-rails', '~> 4.0.0' -gem 'uglifier', '>= 1.3.0' -gem 'select2-rails', '~> 3.5.4' -gem 'jquery-rails', '~> 3.1.0' -gem 'ace-rails-ap', '~> 2.0.1' -gem 'spectrum-rails' - - # geokit-rails doesn't work with geokit 1.8.X but it specifies ~> 1.5 # in its own Gemfile. gem 'geokit', '~> 1.8.4' gem 'geokit-rails', '~> 2.0.1' - +gem 'jquery-rails', '~> 3.1.0' +gem 'json', '~> 1.8.1' +gem 'jsonpath', '~> 0.5.6' +gem 'kaminari', '~> 0.16.1' gem 'kramdown', '~> 1.3.3' -gem 'faraday', '~> 0.9.0' -gem 'faraday_middleware' -gem 'typhoeus', '~> 0.6.3' +gem 'liquid', '~> 2.6.1' +gem 'mysql2', '~> 0.3.16' +gem 'multi_xml' gem 'nokogiri', '~> 1.6.1' -gem 'net-ftp-list', '~> 3.2.8' - -gem 'wunderground', '~> 1.2.0' -gem 'forecast_io', '~> 2.0.0' -gem 'rturk', '~> 2.12.1' - -gem "google-api-client" - -gem 'twitter', '~> 5.8.0' -gem 'cantino-twitter-stream', github: 'cantino/twitter-stream', branch: 'master' -gem 'em-http-request', '~> 1.1.2' -gem 'weibo_2', '~> 0.1.4' -gem 'hipchat', '~> 1.2.0' -gem 'xmpp4r', '~> 0.5.6' -gem 'feed-normalizer' -gem 'slack-notifier', '~> 0.5.0' -gem 'therubyracer', '~> 0.12.1' -gem 'mqtt' - gem 'omniauth' gem 'omniauth-twitter' -gem 'omniauth-37signals' -gem 'omniauth-github' +gem 'rails' , '4.1.5' +gem 'rufus-scheduler', '~> 3.0.8', require: false +gem 'sass-rails', '~> 4.0.0' +gem 'select2-rails', '~> 3.5.4' +gem 'spectrum-rails' +gem 'therubyracer', '~> 0.12.1' +gem 'twitter', '~> 5.8.0' +gem 'typhoeus', '~> 0.6.3' +gem 'uglifier', '>= 1.3.0' group :development do - gem 'binding_of_caller' gem 'better_errors', '~> 1.1' + gem 'binding_of_caller' gem 'quiet_assets' end group :development, :test do - gem 'vcr' + gem 'coveralls', require: false + gem 'delorean' gem 'dotenv-rails' gem 'pry' - gem 'rspec-rails', '~> 2.99' + gem 'rr' gem 'rspec', '~> 2.99' gem 'rspec-collection_matchers' + gem 'rspec-rails', '~> 2.99' gem 'shoulda-matchers' - gem 'rr' - gem 'delorean' - gem 'webmock', '~> 1.17.4', require: false - gem 'coveralls', require: false gem 'spring' gem 'spring-commands-rspec' + gem 'vcr' + gem 'webmock', '~> 1.17.4', require: false end group :production do @@ -117,6 +94,23 @@ group :production do gem 'rack' end +case RUBY_PLATFORM + when /freebsd|netbsd|openbsd/ + # ffi (required by typhoeus via ethon) merged fixes for bugs fatal + # on these platforms after 1.9.3; no following release as yet. + gem 'ffi', github: 'ffi/ffi', branch: 'master' + + # tzinfo 1.2.0 has added support for reading zoneinfo on these + # platforms. + gem 'tzinfo', '>= 1.2.0' + when /solaris/ + # ditto + gem 'tzinfo', '>= 1.2.0' +end + +# Windows does not have zoneinfo files, so bundle the tzinfo-data gem. +gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw] + # This hack needs some explanation. When on Heroku, use the pg, unicorn, and rails12factor gems. # When not on Heroku, we still want our Gemfile.lock to include these gems, so we scope them to # an unsupported platform. diff --git a/Gemfile.lock b/Gemfile.lock index 42b1ee29..8264c44e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -204,9 +204,6 @@ GEM omniauth-37signals (1.0.5) omniauth (~> 1.0) omniauth-oauth2 (~> 1.0) - omniauth-github (1.1.2) - omniauth (~> 1.0) - omniauth-oauth2 (~> 1.1) omniauth-oauth (1.0.1) oauth omniauth (~> 1.0) @@ -423,12 +420,12 @@ DEPENDENCIES kramdown (~> 1.3.3) liquid (~> 2.6.1) mqtt + multi_xml mysql2 (~> 0.3.16) net-ftp-list (~> 3.2.8) nokogiri (~> 1.6.1) omniauth omniauth-37signals - omniauth-github omniauth-twitter pg protected_attributes (~> 1.0.8) diff --git a/app/assets/stylesheets/application.css.scss.erb b/app/assets/stylesheets/application.css.scss.erb index b4b35df1..d960778b 100644 --- a/app/assets/stylesheets/application.css.scss.erb +++ b/app/assets/stylesheets/application.css.scss.erb @@ -170,7 +170,7 @@ span.not-applicable:after { // Disabled -.agent-disabled { +.agent-unavailable { opacity: 0.5; } diff --git a/app/concerns/twitter_concern.rb b/app/concerns/twitter_concern.rb index 1f3359d2..bd432e63 100644 --- a/app/concerns/twitter_concern.rb +++ b/app/concerns/twitter_concern.rb @@ -5,7 +5,7 @@ module TwitterConcern include Oauthable validate :validate_twitter_options - valid_oauth_providers :twitter + valid_oauth_providers 'twitter' end def validate_twitter_options diff --git a/app/concerns/weibo_concern.rb b/app/concerns/weibo_concern.rb index eea56237..228a565b 100644 --- a/app/concerns/weibo_concern.rb +++ b/app/concerns/weibo_concern.rb @@ -2,6 +2,8 @@ module WeiboConcern extend ActiveSupport::Concern included do + gem_dependency_check { defined?(WeiboOAuth2) } + self.validate :validate_weibo_options end @@ -22,8 +24,4 @@ module WeiboConcern end @weibo_client end - - module ClassMethods - - end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 2e2074c0..94df27a9 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -32,6 +32,8 @@ module ApplicationHelper def working(agent) if agent.disabled? link_to 'Disabled', agent_path(agent), class: 'label label-warning' + elsif agent.dependencies_missing? + content_tag :span, 'Missing Gems', class: 'label label-danger' elsif agent.working? content_tag :span, 'Yes', class: 'label label-success' else diff --git a/app/helpers/dot_helper.rb b/app/helpers/dot_helper.rb index 126798a8..e7ea0a50 100644 --- a/app/helpers/dot_helper.rb +++ b/app/helpers/dot_helper.rb @@ -137,9 +137,9 @@ module DotHelper label: agent_label[agent], tooltip: (agent.short_type.titleize if rich), URL: (agent_url[agent] if rich), - style: ('rounded,dashed' if agent.disabled?), - color: (@disabled if agent.disabled?), - fontcolor: (@disabled if agent.disabled?)) + style: ('rounded,dashed' if agent.unavailable?), + color: (@disabled if agent.unavailable?), + fontcolor: (@disabled if agent.unavailable?)) end def agent_edge(agent, receiver) @@ -148,7 +148,7 @@ module DotHelper style: ('dashed' unless receiver.propagate_immediately?), label: (" #{agent.control_action}s " if agent.can_control_other_agents?), arrowhead: ('empty' if agent.can_control_other_agents?), - color: (@disabled if agent.disabled? || receiver.disabled?)) + color: (@disabled if agent.unavailable? || receiver.unavailable?)) end block('digraph', 'Agent Event Flow') { @@ -218,7 +218,7 @@ module DotHelper # a dummy label only to obtain the background color label['class'] = [ 'label', - if agent.disabled? + if agent.unavailable? 'label-warning' elsif agent.working? 'label-success' diff --git a/app/helpers/service_helper.rb b/app/helpers/service_helper.rb deleted file mode 100644 index b5e690e7..00000000 --- a/app/helpers/service_helper.rb +++ /dev/null @@ -1,5 +0,0 @@ -module ServiceHelper - def has_oauth_configuration_for(provider) - ENV["#{provider.upcase}_OAUTH_KEY"].present? && ENV["#{provider.upcase}_OAUTH_SECRET"].present? - end -end \ No newline at end of file diff --git a/app/models/agent.rb b/app/models/agent.rb index 0f24e8fb..e33c0d81 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -150,6 +150,14 @@ class Agent < ActiveRecord::Base end end + def unavailable? + disabled? || dependencies_missing? + end + + def dependencies_missing? + self.class.dependencies_missing? + end + def default_schedule self.class.default_schedule end @@ -317,6 +325,15 @@ class Agent < ActiveRecord::Base include? AgentControllerConcern end + def gem_dependency_check + @gem_dependencies_checked = true + @gem_dependencies_met = yield + end + + def dependencies_missing? + @gem_dependencies_checked && !@gem_dependencies_met + end + # Find all Agents that have received Events since the last execution of this method. Update those Agents with # their new `last_checked_event_id` and queue each of the Agents to be called with #receive using `async_receive`. # This is called by bin/schedule.rb periodically. @@ -362,7 +379,7 @@ class Agent < ActiveRecord::Base def async_receive(agent_id, event_ids) agent = Agent.find(agent_id) begin - return if agent.disabled? + return if agent.unavailable? agent.receive(Event.where(:id => event_ids)) agent.last_receive_at = Time.now agent.save! @@ -400,7 +417,7 @@ class Agent < ActiveRecord::Base def async_check(agent_id) agent = Agent.find(agent_id) begin - return if agent.disabled? + return if agent.unavailable? agent.check agent.last_check_at = Time.now agent.save! diff --git a/app/models/agents/ftpsite_agent.rb b/app/models/agents/ftpsite_agent.rb index 8af1262d..0dceb3c5 100644 --- a/app/models/agents/ftpsite_agent.rb +++ b/app/models/agents/ftpsite_agent.rb @@ -1,15 +1,15 @@ -require 'net/ftp' -require 'net/ftp/list' require 'uri' require 'time' module Agents class FtpsiteAgent < Agent cannot_receive_events! - default_schedule "every_12h" + gem_dependency_check { defined?(Net::FTP) && defined?(Net::FTP::List) } + description <<-MD + #{'## Include `net-ftp-list` in your Gemfile to use this Agent!' if dependencies_missing?} The FtpsiteAgent checks a FTP site and creates Events based on newly uploaded files in a directory. Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names. @@ -35,12 +35,12 @@ module Agents def default_options { - 'expected_update_period_in_days' => "1", - 'url' => "ftp://example.org/pub/releases/", - 'patterns' => [ - 'foo-*.tar.gz', - ], - 'after' => Time.now.iso8601, + 'expected_update_period_in_days' => "1", + 'url' => "ftp://example.org/pub/releases/", + 'patterns' => [ + 'foo-*.tar.gz', + ], + 'after' => Time.now.iso8601, } end diff --git a/app/models/agents/google_calendar_publish_agent.rb b/app/models/agents/google_calendar_publish_agent.rb index 9d0c38db..633d18ef 100644 --- a/app/models/agents/google_calendar_publish_agent.rb +++ b/app/models/agents/google_calendar_publish_agent.rb @@ -4,7 +4,10 @@ module Agents class GoogleCalendarPublishAgent < Agent cannot_be_scheduled! + gem_dependency_check { defined?(GoogleCalendar) } + description <<-MD + #{'## Include `google-api-client` in your Gemfile to use this Agent!' if dependencies_missing?} The GoogleCalendarPublishAgent creates events on your google calendar. This agent relies on service accounts, rather than oauth. diff --git a/app/models/agents/growl_agent.rb b/app/models/agents/growl_agent.rb index 35f5e33f..4fb7931a 100644 --- a/app/models/agents/growl_agent.rb +++ b/app/models/agents/growl_agent.rb @@ -1,5 +1,3 @@ -require 'ruby-growl' - module Agents class GrowlAgent < Agent attr_reader :growler @@ -7,7 +5,10 @@ module Agents cannot_be_scheduled! cannot_create_events! + gem_dependency_check { defined?(Growl) } + description <<-MD + #{'## Include `ruby-growl` in your Gemfile to use this Agent!' if dependencies_missing?} The GrowlAgent sends any events it receives to a Growl GNTP server immediately. It is assumed that events have a `message` or `text` key, which will hold the body of the growl notification, and a `subject` key, which will have the headline of the Growl notification. You can use Event Formatting Agent if your event does not provide these keys. @@ -34,13 +35,13 @@ module Agents errors.add(:base, "growl_server and expected_receive_period_in_days are required fields") end end - + def register_growl @growler = Growl.new interpolated['growl_server'], interpolated['growl_app_name'], "GNTP" @growler.password = interpolated['growl_password'] @growler.add_notification interpolated['growl_notification_name'] end - + def notify_growl(subject, message) @growler.notify(interpolated['growl_notification_name'], subject, message) end diff --git a/app/models/agents/hipchat_agent.rb b/app/models/agents/hipchat_agent.rb index 68168929..cd8bcaac 100644 --- a/app/models/agents/hipchat_agent.rb +++ b/app/models/agents/hipchat_agent.rb @@ -3,7 +3,10 @@ module Agents cannot_be_scheduled! cannot_create_events! + gem_dependency_check { defined?(HipChat) } + description <<-MD + #{'## Include `hipchat` in your Gemfile to use this Agent!' if dependencies_missing?} The HipchatAgent sends messages to a Hipchat Room To authenticate you need to set the `auth_token`, you can get one at your Hipchat Group Admin page which you can find here: @@ -40,11 +43,14 @@ module Agents end def receive(incoming_events) - client = HipChat::Client.new(interpolated[:auth_token] || credential('hipchat_auth_token')) incoming_events.each do |event| mo = interpolated(event) client[mo[:room_name]].send(mo[:username][0..14], mo[:message], :notify => boolify(mo[:notify]), :color => mo[:color]) end end + + def client + @client ||= HipChat::Client.new(interpolated[:auth_token] || credential('hipchat_auth_token')) + end end end diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb index 0b4290ee..2a130b4d 100644 --- a/app/models/agents/human_task_agent.rb +++ b/app/models/agents/human_task_agent.rb @@ -1,10 +1,11 @@ -require 'rturk' - module Agents class HumanTaskAgent < Agent default_schedule "every_10m" + gem_dependency_check { defined?(RTurk) } + description <<-MD + #{'## Include `rturk` in your Gemfile to use this Agent!' if dependencies_missing?} You can use a HumanTaskAgent to create Human Intelligence Tasks (HITs) on Mechanical Turk. HITs can be created in response to events, or on a schedule. Set `trigger_on` to either `schedule` or `event`. @@ -226,266 +227,269 @@ module Agents protected - def take_majority? - interpolated['combination_mode'] == "take_majority" || interpolated['take_majority'] == "true" - end + if defined?(RTurk) - def create_poll? - interpolated['combination_mode'] == "poll" - end - - def event_for_hit(hit_id) - if memory['hits'][hit_id].is_a?(Hash) - Event.find_by_id(memory['hits'][hit_id]['event_id']) - else - nil - end - end - - def hit_type(hit_id) - if memory['hits'][hit_id].is_a?(Hash) && memory['hits'][hit_id]['type'] - memory['hits'][hit_id]['type'] - else - 'user' - end - end - - def review_hits - reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids - my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys - if reviewable_hit_ids.length > 0 - log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]" + def take_majority? + interpolated['combination_mode'] == "take_majority" || interpolated['take_majority'] == "true" end - my_reviewed_hit_ids.each do |hit_id| - hit = RTurk::Hit.new(hit_id) - assignments = hit.assignments + def create_poll? + interpolated['combination_mode'] == "poll" + end - log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}" - if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" } - inbound_event = event_for_hit(hit_id) + def event_for_hit(hit_id) + if memory['hits'][hit_id].is_a?(Hash) + Event.find_by_id(memory['hits'][hit_id]['event_id']) + else + nil + end + end - if hit_type(hit_id) == 'poll' - # handle completed polls + def hit_type(hit_id) + if memory['hits'][hit_id].is_a?(Hash) && memory['hits'][hit_id]['type'] + memory['hits'][hit_id]['type'] + else + 'user' + end + end - log "Handling a poll: #{hit_id}" + def review_hits + reviewable_hit_ids = RTurk::GetReviewableHITs.create.hit_ids + my_reviewed_hit_ids = reviewable_hit_ids & (memory['hits'] || {}).keys + if reviewable_hit_ids.length > 0 + log "MTurk reports #{reviewable_hit_ids.length} HITs, of which I own [#{my_reviewed_hit_ids.to_sentence}]" + end - scores = {} - assignments.each do |assignment| - assignment.answers.each do |index, rating| - scores[index] ||= 0 - scores[index] += rating.to_i - end - end + my_reviewed_hit_ids.each do |hit_id| + hit = RTurk::Hit.new(hit_id) + assignments = hit.assignments - top_answer = scores.to_a.sort {|b, a| a.last <=> b.last }.first.first + log "Looking at HIT #{hit_id}. I found #{assignments.length} assignments#{" with the statuses: #{assignments.map(&:status).to_sentence}" if assignments.length > 0}" + if assignments.length == hit.max_assignments && assignments.all? { |assignment| assignment.status == "Submitted" } + inbound_event = event_for_hit(hit_id) - payload = { - 'answers' => memory['hits'][hit_id]['answers'], - 'poll' => assignments.map(&:answers), - 'best_answer' => memory['hits'][hit_id]['answers'][top_answer.to_i - 1] - } + if hit_type(hit_id) == 'poll' + # handle completed polls - event = create_event :payload => payload - log "Event emitted with answer(s) for poll", :outbound_event => event, :inbound_event => inbound_event - else - # handle normal completed HITs - payload = { 'answers' => assignments.map(&:answers) } + log "Handling a poll: #{hit_id}" - if take_majority? - counts = {} - options['hit']['questions'].each do |question| - question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo } - assignments.each do |assignment| - answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers) - answer = answers[question['key']] - question_counts[answer] += 1 + scores = {} + assignments.each do |assignment| + assignment.answers.each do |index, rating| + scores[index] ||= 0 + scores[index] += rating.to_i end - counts[question['key']] = question_counts end - payload['counts'] = counts - majority_answer = counts.inject({}) do |memo, (key, question_counts)| - memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first - memo - end - payload['majority_answer'] = majority_answer + top_answer = scores.to_a.sort {|b, a| a.last <=> b.last }.first.first - if all_questions_are_numeric? - average_answer = counts.inject({}) do |memo, (key, question_counts)| - sum = divisor = 0 - question_counts.to_a.each do |num, count| - sum += num.to_s.to_f * count - divisor += count + payload = { + 'answers' => memory['hits'][hit_id]['answers'], + 'poll' => assignments.map(&:answers), + 'best_answer' => memory['hits'][hit_id]['answers'][top_answer.to_i - 1] + } + + event = create_event :payload => payload + log "Event emitted with answer(s) for poll", :outbound_event => event, :inbound_event => inbound_event + else + # handle normal completed HITs + payload = { 'answers' => assignments.map(&:answers) } + + if take_majority? + counts = {} + options['hit']['questions'].each do |question| + question_counts = question['selections'].inject({}) { |memo, selection| memo[selection['key']] = 0; memo } + assignments.each do |assignment| + answers = ActiveSupport::HashWithIndifferentAccess.new(assignment.answers) + answer = answers[question['key']] + question_counts[answer] += 1 end - memo[key] = sum / divisor.to_f + counts[question['key']] = question_counts + end + payload['counts'] = counts + + majority_answer = counts.inject({}) do |memo, (key, question_counts)| + memo[key] = question_counts.to_a.sort {|a, b| a.last <=> b.last }.last.first memo end - payload['average_answer'] = average_answer - end - end + payload['majority_answer'] = majority_answer - if create_poll? - questions = [] - selections = 5.times.map { |i| { 'key' => i+1, 'text' => i+1 } }.reverse - assignments.length.times do |index| - questions << { - 'type' => "selection", - 'name' => "Item #{index + 1}", - 'key' => index, - 'required' => "true", - 'question' => interpolate_string(options['poll_options']['row_template'], assignments[index].answers), - 'selections' => selections - } - end - - poll_hit = create_hit 'title' => options['poll_options']['title'], - 'description' => options['poll_options']['instructions'], - 'questions' => questions, - 'assignments' => options['poll_options']['assignments'], - 'lifetime_in_seconds' => options['poll_options']['lifetime_in_seconds'], - 'reward' => options['poll_options']['reward'], - 'payload' => inbound_event && inbound_event.payload, - 'metadata' => { 'type' => 'poll', - 'original_hit' => hit_id, - 'answers' => assignments.map(&:answers), - 'event_id' => inbound_event && inbound_event.id } - - log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}. Original HIT: #{hit_id}", :inbound_event => inbound_event - else - if options[:separate_answers] - payload['answers'].each.with_index do |answer, index| - sub_payload = payload.dup - sub_payload.delete('answers') - sub_payload['answer'] = answer - event = create_event :payload => sub_payload - log "Event emitted with answer ##{index}", :outbound_event => event, :inbound_event => inbound_event - end - else - event = create_event :payload => payload - log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event - end - end - end - - assignments.each(&:approve!) - hit.dispose! - - memory['hits'].delete(hit_id) - end - end - end - - def all_questions_are_numeric? - interpolated['hit']['questions'].all? do |question| - question['selections'].all? do |selection| - selection['key'] == selection['key'].to_f.to_s || selection['key'] == selection['key'].to_i.to_s - end - end - end - - def create_basic_hit(event = nil) - hit = create_hit 'title' => options['hit']['title'], - 'description' => options['hit']['description'], - 'questions' => options['hit']['questions'], - 'assignments' => options['hit']['assignments'], - 'lifetime_in_seconds' => options['hit']['lifetime_in_seconds'], - 'reward' => options['hit']['reward'], - 'payload' => event && event.payload, - 'metadata' => { 'event_id' => event && event.id } - - log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event - end - - def create_hit(opts = {}) - payload = opts['payload'] || {} - title = interpolate_string(opts['title'], payload).strip - description = interpolate_string(opts['description'], payload).strip - questions = interpolate_options(opts['questions'], payload) - hit = RTurk::Hit.create(:title => title) do |hit| - hit.max_assignments = (opts['assignments'] || 1).to_i - hit.description = description - hit.lifetime = (opts['lifetime_in_seconds'] || 24 * 60 * 60).to_i - hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions) - hit.reward = (opts['reward'] || 0.05).to_f - #hit.qualifications.add :approval_rate, { :gt => 80 } - end - memory['hits'] ||= {} - memory['hits'][hit.id] = opts['metadata'] || {} - hit - end - - # RTurk Question Form - - class AgentQuestionForm < RTurk::QuestionForm - needs :title, :description, :questions - - def question_form_content - Overview do - Title do - text @title - end - Text do - text @description - end - end - - @questions.each.with_index do |question, index| - Question do - QuestionIdentifier do - text question['key'] || "question_#{index}" - end - DisplayName do - text question['name'] || "Question ##{index}" - end - IsRequired do - text question['required'] || 'true' - end - QuestionContent do - Text do - text question['question'] - end - end - AnswerSpecification do - if question['type'] == "selection" - - SelectionAnswer do - StyleSuggestion do - text 'radiobutton' + if all_questions_are_numeric? + average_answer = counts.inject({}) do |memo, (key, question_counts)| + sum = divisor = 0 + question_counts.to_a.each do |num, count| + sum += num.to_s.to_f * count + divisor += count + end + memo[key] = sum / divisor.to_f + memo end - Selections do - question['selections'].each do |selection| - Selection do - SelectionIdentifier do - text selection['key'] - end - Text do - text selection['text'] + payload['average_answer'] = average_answer + end + end + + if create_poll? + questions = [] + selections = 5.times.map { |i| { 'key' => i+1, 'text' => i+1 } }.reverse + assignments.length.times do |index| + questions << { + 'type' => "selection", + 'name' => "Item #{index + 1}", + 'key' => index, + 'required' => "true", + 'question' => interpolate_string(options['poll_options']['row_template'], assignments[index].answers), + 'selections' => selections + } + end + + poll_hit = create_hit 'title' => options['poll_options']['title'], + 'description' => options['poll_options']['instructions'], + 'questions' => questions, + 'assignments' => options['poll_options']['assignments'], + 'lifetime_in_seconds' => options['poll_options']['lifetime_in_seconds'], + 'reward' => options['poll_options']['reward'], + 'payload' => inbound_event && inbound_event.payload, + 'metadata' => { 'type' => 'poll', + 'original_hit' => hit_id, + 'answers' => assignments.map(&:answers), + 'event_id' => inbound_event && inbound_event.id } + + log "Poll HIT created with ID #{poll_hit.id} and URL #{poll_hit.url}. Original HIT: #{hit_id}", :inbound_event => inbound_event + else + if options[:separate_answers] + payload['answers'].each.with_index do |answer, index| + sub_payload = payload.dup + sub_payload.delete('answers') + sub_payload['answer'] = answer + event = create_event :payload => sub_payload + log "Event emitted with answer ##{index}", :outbound_event => event, :inbound_event => inbound_event + end + else + event = create_event :payload => payload + log "Event emitted with answer(s)", :outbound_event => event, :inbound_event => inbound_event + end + end + end + + assignments.each(&:approve!) + hit.dispose! + + memory['hits'].delete(hit_id) + end + end + end + + def all_questions_are_numeric? + interpolated['hit']['questions'].all? do |question| + question['selections'].all? do |selection| + selection['key'] == selection['key'].to_f.to_s || selection['key'] == selection['key'].to_i.to_s + end + end + end + + def create_basic_hit(event = nil) + hit = create_hit 'title' => options['hit']['title'], + 'description' => options['hit']['description'], + 'questions' => options['hit']['questions'], + 'assignments' => options['hit']['assignments'], + 'lifetime_in_seconds' => options['hit']['lifetime_in_seconds'], + 'reward' => options['hit']['reward'], + 'payload' => event && event.payload, + 'metadata' => { 'event_id' => event && event.id } + + log "HIT created with ID #{hit.id} and URL #{hit.url}", :inbound_event => event + end + + def create_hit(opts = {}) + payload = opts['payload'] || {} + title = interpolate_string(opts['title'], payload).strip + description = interpolate_string(opts['description'], payload).strip + questions = interpolate_options(opts['questions'], payload) + hit = RTurk::Hit.create(:title => title) do |hit| + hit.max_assignments = (opts['assignments'] || 1).to_i + hit.description = description + hit.lifetime = (opts['lifetime_in_seconds'] || 24 * 60 * 60).to_i + hit.question_form AgentQuestionForm.new(:title => title, :description => description, :questions => questions) + hit.reward = (opts['reward'] || 0.05).to_f + #hit.qualifications.add :approval_rate, { :gt => 80 } + end + memory['hits'] ||= {} + memory['hits'][hit.id] = opts['metadata'] || {} + hit + end + + # RTurk Question Form + + class AgentQuestionForm < RTurk::QuestionForm + needs :title, :description, :questions + + def question_form_content + Overview do + Title do + text @title + end + Text do + text @description + end + end + + @questions.each.with_index do |question, index| + Question do + QuestionIdentifier do + text question['key'] || "question_#{index}" + end + DisplayName do + text question['name'] || "Question ##{index}" + end + IsRequired do + text question['required'] || 'true' + end + QuestionContent do + Text do + text question['question'] + end + end + AnswerSpecification do + if question['type'] == "selection" + + SelectionAnswer do + StyleSuggestion do + text 'radiobutton' + end + Selections do + question['selections'].each do |selection| + Selection do + SelectionIdentifier do + text selection['key'] + end + Text do + text selection['text'] + end end end end end - end - else + else - FreeTextAnswer do - if question['min_length'].present? || question['max_length'].present? - Constraints do - lengths = {} - lengths['minLength'] = question['min_length'].to_s if question['min_length'].present? - lengths['maxLength'] = question['max_length'].to_s if question['max_length'].present? - Length lengths + FreeTextAnswer do + if question['min_length'].present? || question['max_length'].present? + Constraints do + lengths = {} + lengths['minLength'] = question['min_length'].to_s if question['min_length'].present? + lengths['maxLength'] = question['max_length'].to_s if question['max_length'].present? + Length lengths + end + end + + if question['default'].present? + DefaultText do + text question['default'] + end end end - if question['default'].present? - DefaultText do - text question['default'] - end - end end - end end end diff --git a/app/models/agents/jabber_agent.rb b/app/models/agents/jabber_agent.rb index 6fd553c9..c8f82952 100644 --- a/app/models/agents/jabber_agent.rb +++ b/app/models/agents/jabber_agent.rb @@ -3,7 +3,10 @@ module Agents cannot_be_scheduled! cannot_create_events! + gem_dependency_check { defined?(Jabber) } + description <<-MD + #{'## Include `xmpp4r` in your Gemfile to use this Agent!' if dependencies_missing?} The JabberAgent will send any events it receives to your Jabber/XMPP IM account. Specify the `jabber_server` and `jabber_port` for your Jabber server. diff --git a/app/models/agents/mqtt_agent.rb b/app/models/agents/mqtt_agent.rb index 35c4005e..bfb28bf6 100644 --- a/app/models/agents/mqtt_agent.rb +++ b/app/models/agents/mqtt_agent.rb @@ -1,10 +1,12 @@ # encoding: utf-8 -require "mqtt" require "json" module Agents class MqttAgent < Agent + gem_dependency_check { defined?(MQTT) } + description <<-MD + #{'## Include `mqtt` in your Gemfile to use this Agent!' if dependencies_missing?} The MQTT agent allows both publication and subscription to an MQTT topic. MQTT is a generic transport protocol for machine to machine communication. diff --git a/app/models/agents/slack_agent.rb b/app/models/agents/slack_agent.rb index 10c0c013..c7ebf220 100644 --- a/app/models/agents/slack_agent.rb +++ b/app/models/agents/slack_agent.rb @@ -1,11 +1,15 @@ module Agents class SlackAgent < Agent + DEFAULT_WEBHOOK = 'incoming-webhook' + DEFAULT_USERNAME = 'Huginn' + cannot_be_scheduled! cannot_create_events! - DEFAULT_WEBHOOK = 'incoming-webhook' - DEFAULT_USERNAME = 'Huginn' + gem_dependency_check { defined?(Slack) } + description <<-MD + #{'## Include `slack-notifier` in your Gemfile to use this Agent!' if dependencies_missing?} The SlackAgent lets you receive events and send notifications to [slack](https://slack.com/). To get started, you will first need to setup an incoming webhook. diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb index 1641097a..91d1430a 100644 --- a/app/models/agents/twilio_agent.rb +++ b/app/models/agents/twilio_agent.rb @@ -1,4 +1,3 @@ -require 'twilio-ruby' require 'securerandom' module Agents @@ -6,7 +5,10 @@ module Agents cannot_be_scheduled! cannot_create_events! + gem_dependency_check { defined?(Twilio) } + description <<-MD + #{'## Include `twilio-ruby` in your Gemfile to use this Agent!' if dependencies_missing?} The TwilioAgent receives and collects events and sends them via text message (up to 160 characters) or gives you a call when scheduled. It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use the EventFormattingAgent if your event does not provide these keys. @@ -39,7 +41,6 @@ module Agents end def receive(incoming_events) - @client = Twilio::REST::Client.new interpolated['account_sid'], interpolated['auth_token'] memory['pending_calls'] ||= {} incoming_events.each do |event| message = (event.payload['message'].presence || event.payload['text'].presence || event.payload['sms'].presence).to_s @@ -63,15 +64,15 @@ module Agents end def send_message(message) - @client.account.sms.messages.create :from => interpolated['sender_cell'], - :to => interpolated['receiver_cell'], - :body => message + client.account.sms.messages.create :from => interpolated['sender_cell'], + :to => interpolated['receiver_cell'], + :body => message end def make_call(secret) - @client.account.calls.create :from => interpolated['sender_cell'], - :to => interpolated['receiver_cell'], - :url => post_url(interpolated['server_url'], secret) + client.account.calls.create :from => interpolated['sender_cell'], + :to => interpolated['receiver_cell'], + :url => post_url(interpolated['server_url'], secret) end def post_url(server_url, secret) @@ -85,5 +86,9 @@ module Agents [response.text, 200] end end + + def client + @client ||= Twilio::REST::Client.new interpolated['account_sid'], interpolated['auth_token'] + end end end diff --git a/app/models/agents/weather_agent.rb b/app/models/agents/weather_agent.rb index 58d1c381..2ee8457a 100644 --- a/app/models/agents/weather_agent.rb +++ b/app/models/agents/weather_agent.rb @@ -5,7 +5,10 @@ module Agents class WeatherAgent < Agent cannot_receive_events! + gem_dependency_check { defined?(Wunderground) && defined?(ForecastIO) } + description <<-MD + #{'## Include `forecast_io` and `wunderground` in your Gemfile to use this Agent!' if dependencies_missing?} The WeatherAgent creates an event for the day's weather at a given `location`. You also must select `which_day` you would like to get the weather for where the number 0 is for today and 1 is for tomorrow and so on. Weather is only returned for 1 week at a time. diff --git a/app/models/agents/weibo_publish_agent.rb b/app/models/agents/weibo_publish_agent.rb index 591a1a97..0dcae12c 100644 --- a/app/models/agents/weibo_publish_agent.rb +++ b/app/models/agents/weibo_publish_agent.rb @@ -1,5 +1,4 @@ # encoding: utf-8 -require "weibo_2" module Agents class WeiboPublishAgent < Agent @@ -8,6 +7,7 @@ module Agents cannot_be_scheduled! description <<-MD + #{'## Include `weibo_2` in your Gemfile to use this Agent!' if dependencies_missing?} The WeiboPublishAgent publishes tweets from the events it receives. You must first set up a Weibo app and generate an `acess_token` for the user to send statuses as. @@ -79,8 +79,7 @@ module Agents tweet_json[:entities][:urls].each do |url| text.gsub! url[:url], url[:expanded_url] end - return text + text end - end end diff --git a/app/models/agents/weibo_user_agent.rb b/app/models/agents/weibo_user_agent.rb index fc2219c7..6c8373a8 100644 --- a/app/models/agents/weibo_user_agent.rb +++ b/app/models/agents/weibo_user_agent.rb @@ -1,5 +1,4 @@ # encoding: utf-8 -require "weibo_2" module Agents class WeiboUserAgent < Agent @@ -8,6 +7,7 @@ module Agents cannot_receive_events! description <<-MD + #{'## Include `weibo_2` in your Gemfile to use this Agent!' if dependencies_missing?} The WeiboUserAgent follows the timeline of a specified Weibo user. It uses this endpoint: http://open.weibo.com/wiki/2/statuses/user_timeline/en You must first set up a Weibo app and generate an `acess_token` to authenticate with. Provide that, along with the `app_key` and `app_secret` for your Weibo app in the options. diff --git a/app/views/agents/_table.html.erb b/app/views/agents/_table.html.erb index 0be163c2..244e1a1c 100644 --- a/app/views/agents/_table.html.erb +++ b/app/views/agents/_table.html.erb @@ -13,7 +13,7 @@ <% @agents.each do |agent| %> - + <%= link_to agent.name, agent_path(agent) %>
<%= agent.short_type.titleize %> @@ -23,35 +23,35 @@ <% end %> - + <% if agent.can_be_scheduled? %> <%= agent_schedule(agent, ',
') %> <% else %> <% end %> - + <% if agent.can_be_scheduled? %> <%= agent.last_check_at ? time_ago_in_words(agent.last_check_at) + " ago" : "never" %> <% else %> <% end %> - + <% if agent.can_create_events? %> <%= agent.last_event_at ? time_ago_in_words(agent.last_event_at) + " ago" : "never" %> <% else %> <% end %> - + <% if agent.can_receive_events? %> <%= agent.last_receive_at ? time_ago_in_words(agent.last_receive_at) + " ago" : "never" %> <% else %> <% end %> - + <% if agent.can_create_events? %> <%= link_to(agent.events_count || 0, agent_events_path(agent)) %> <% else %> diff --git a/app/views/services/index.html.erb b/app/views/services/index.html.erb index 83a8cc41..910410db 100644 --- a/app/views/services/index.html.erb +++ b/app/views/services/index.html.erb @@ -11,13 +11,13 @@ <%= link_to 'wiki', 'https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications', target: :_blank %> for guidance.

- <% if has_oauth_configuration_for('twitter') %> + <% if has_oauth_configuration_for?('twitter') %>

<%= link_to "Authenticate with Twitter", "/auth/twitter" %>

<% end %> - <% if has_oauth_configuration_for('thirty_seven_signals') %> + <% if has_oauth_configuration_for?('thirty_seven_signals') %>

<%= link_to "Authenticate with 37Signals (Basecamp)", "/auth/37signals" %>

<% end -%> - <% if has_oauth_configuration_for('github') %> + <% if has_oauth_configuration_for?('github') %>

<%= link_to "Authenticate with Github", "/auth/github" %>

<% end -%>
diff --git a/config/initializers/aws.rb b/config/initializers/aws.rb index 64b70866..e28c4059 100644 --- a/config/initializers/aws.rb +++ b/config/initializers/aws.rb @@ -1,4 +1,4 @@ -unless Rails.env.test? +if defined?(RTurk) && !Rails.env.test? RTurk::logger.level = Logger::DEBUG RTurk.setup(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_ACCESS_KEY'], :sandbox => ENV['AWS_SANDBOX'] == "true") end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index 2143a089..32ceca30 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -1,5 +1,23 @@ -Rails.application.config.middleware.use OmniAuth::Builder do - provider :twitter, ENV['TWITTER_OAUTH_KEY'], ENV['TWITTER_OAUTH_SECRET'], authorize_params: {force_login: 'true', use_authorize: 'true'} - provider '37signals', ENV['THIRTY_SEVEN_SIGNALS_OAUTH_KEY'], ENV['THIRTY_SEVEN_SIGNALS_OAUTH_SECRET'] - provider :github, ENV['GITHUB_OAUTH_KEY'], ENV['GITHUB_OAUTH_SECRET'] +LOADED_OMNIAUTH_STRATEGIES = { + 'twitter' => defined?(OmniAuth::Strategies::Twitter), + '37signals' => defined?(OmniAuth::Strategies::ThirtySevenSignals), + 'github' => defined?(OmniAuth::Strategies::GitHub) +} + +def has_oauth_configuration_for?(provider) + LOADED_OMNIAUTH_STRATEGIES[provider.to_s] && ENV["#{provider.upcase}_OAUTH_KEY"].present? && ENV["#{provider.upcase}_OAUTH_SECRET"].present? +end + +Rails.application.config.middleware.use OmniAuth::Builder do + if has_oauth_configuration_for?('twitter') + provider 'twitter', ENV['TWITTER_OAUTH_KEY'], ENV['TWITTER_OAUTH_SECRET'], authorize_params: {force_login: 'true', use_authorize: 'true'} + end + + if has_oauth_configuration_for?('37signals') + provider '37signals', ENV['THIRTY_SEVEN_SIGNALS_OAUTH_KEY'], ENV['THIRTY_SEVEN_SIGNALS_OAUTH_SECRET'] + end + + if has_oauth_configuration_for?('github') + provider 'github', ENV['GITHUB_OAUTH_KEY'], ENV['GITHUB_OAUTH_SECRET'] + end end diff --git a/lib/huginn_scheduler.rb b/lib/huginn_scheduler.rb index 4de210a9..9b4bfb20 100644 --- a/lib/huginn_scheduler.rb +++ b/lib/huginn_scheduler.rb @@ -40,7 +40,7 @@ class Rufus::Scheduler def schedule_scheduler_agent(agent) job = scheduler_agent_job(agent) - if agent.disabled? + if agent.unavailable? if job puts "Unscheduling SchedulerAgent##{agent.id} (disabled)" job.unschedule From 025a41904a121041a3a3241179b79f8c8cfe8c45 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 22 Sep 2014 13:32:45 -0700 Subject: [PATCH 2/2] add httpparty to gemfile --- Gemfile | 1 + Gemfile.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/Gemfile b/Gemfile index c802f7f9..68ca8c1c 100644 --- a/Gemfile +++ b/Gemfile @@ -46,6 +46,7 @@ gem 'foreman', '~> 0.63.0' # in its own Gemfile. gem 'geokit', '~> 1.8.4' gem 'geokit-rails', '~> 2.0.1' +gem 'httparty', '~> 0.13' gem 'jquery-rails', '~> 3.1.0' gem 'json', '~> 1.8.1' gem 'jsonpath', '~> 0.5.6' diff --git a/Gemfile.lock b/Gemfile.lock index 8264c44e..b73378f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -413,6 +413,7 @@ DEPENDENCIES geokit-rails (~> 2.0.1) google-api-client hipchat (~> 1.2.0) + httparty (~> 0.13) jquery-rails (~> 3.1.0) json (~> 1.8.1) jsonpath (~> 0.5.6)