GitHub-Mirror/huginn
2
0
Fork 0
mirror of https://github.com/Fishwaldo/huginn.git synced 2025-03-15 19:31:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into set_charset_for_mysql

Browse source 
Conflicts:
	db/schema.rb
This commit is contained in:
Akinori MUSHA 2014-08-27 13:06:21 +09:00
commit 857b8ea969
63 changed files with 788 additions and 276 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.env.example
View file

@ -70,9 +70,11 @@ EMAIL_FROM_ADDRESS=from_address@gmail.com
# Number of lines of log messages to keep per Agent
AGENT_LOG_LENGTH=200
#############################
# OAuth Configuration #
#############################
########################################################################################################
# OAuth Configuration #
# More information at the wiki: https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications #
########################################################################################################
TWITTER_OAUTH_KEY=
TWITTER_OAUTH_SECRET=

4
.travis.yml
View file

@ -2,13 +2,13 @@ language: ruby
cache: bundler
bundler_args: --without development production
env:
- APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d TWITTER_OAUTH_KEY=twitteroauthkey TWITTER_OAUTH_SECRET=twitteroauthsecret
- APP_SECRET_TOKEN=b2724973fd81c2f4ac0f92ac48eb3f0152c4a11824c122bcf783419a4c51d8b9bba81c8ba6a66c7de599677c7f486242cf819775c433908e77c739c5c8ae118d
rvm:
- 2.0.0
- 2.1.1
- 1.9.3
before_install:
- travis_retry gem install bundler
- travis_retry gem install bundler
before_script:
- mysql -e 'create database huginn_test;'
- bundle exec rake db:migrate db:test:prepare

3
Gemfile
View file

@ -56,6 +56,8 @@ gem 'uglifier', '>= 1.3.0'
gem 'select2-rails', '~> 3.5.4'
gem 'jquery-rails', '~> 3.1.0'
gem 'ace-rails-ap', '~> 2.0.1'
gem 'spectrum-rails'
# geokit-rails doesn't work with geokit 1.8.X but it specifies ~> 1.5
# in its own Gemfile.
@ -128,4 +130,3 @@ else
gem 'unicorn', platform: :ruby_18
gem 'rails_12factor', platform: :ruby_18
end

3
Gemfile.lock
View file

@ -315,6 +315,8 @@ GEM
simplecov-html (0.8.0)
slack-notifier (0.5.0)
slop (3.6.0)
spectrum-rails (1.3.4)
railties (>= 3.1)
sprockets (2.11.0)
hike (~> 1.2)
multi_json (~> 1.0)
@ -444,6 +446,7 @@ DEPENDENCIES
select2-rails (~> 3.5.4)
shoulda-matchers
slack-notifier (~> 0.5.0)
spectrum-rails
therubyracer (~> 0.12.1)
twilio-ruby (~> 3.11.5)
twitter (~> 5.8.0)

7
app/assets/javascripts/application.js.coffee.erb
View file

@ -6,6 +6,7 @@
#= require json2
#= require jquery.json-editor
#= require latlon_and_geo
#= require spectrum
#= require ./worker-checker
#= require_self
@ -60,6 +61,10 @@ showEventDescriptions = ->
$(".event-descriptions").html("").hide()
$(document).ready ->
$('.navbar .dropdown.dropdown-hover').hover \
-> $(this).addClass('open'),
-> $(this).removeClass('open')
# JSON Editor
window.jsonEditor = setupJsonEditor()[0]
@ -164,7 +169,7 @@ $(document).ready ->
$(".description").html(json.description_html) if json.description_html?
$('.oauthable-form').html($(json.form).find('.oauthable-form').html()) if json.form?
$('.oauthable-form').html(json.form) if json.form?
if $("#agent_options").hasClass("showing-default") || $("#agent_options").val().match(/\A\s*(\{\s*\}|)\s*\Z/g)
window.jsonEditor.json = json.options

15
app/assets/stylesheets/application.css.scss.erb
View file

@ -12,6 +12,7 @@
*= require select2-bootstrap
*= require jquery.json-editor
*= require rickshaw
*= require spectrum
*= require_tree .
*= require_self
*/
@ -186,3 +187,17 @@ h2 .scenario, a span.label.scenario {
.color-success {
color: #5cb85c;
}
.form-group {
.sp-replacer {
@extend .form-control;
}
.sp-preview {
width: 100%;
}
.sp-dd {
display: none;
}
}

43
app/concerns/liquid_droppable.rb
View file

@ -1,9 +1,50 @@
module LiquidDroppable
extend ActiveSupport::Concern
# In subclasses of this base class, "locals" take precedence over
# methods.
class Drop < Liquid::Drop
def initialize(object)
class << self
def inherited(subclass)
class << subclass
attr_reader :drop_methods
# Make all public methods private so that #before_method
# catches everything.
def drop_methods!
return if @drop_methods
@drop_methods = Set.new
(public_instance_methods - Drop.public_instance_methods).each { |name|
@drop_methods << name.to_s
private name
}
end
end
end
end
def initialize(object, locals = nil)
self.class.drop_methods!
@object = object
@locals = locals || {}
end
def before_method(name)
if @locals.include?(name)
@locals[name]
elsif self.class.drop_methods.include?(name)
__send__(name)
end
end
def each
return to_enum(__method__) unless block_given?
self.class.drop_methods.each { |name|
yield [name, __send__(name)]
}
end
end

6
app/concerns/oauthable.rb
View file

@ -11,11 +11,11 @@ module Oauthable
true
end
def valid_services(current_user)
def valid_services_for(user)
if valid_oauth_providers == :all
current_user.available_services
user.available_services
else
current_user.available_services.where(provider: valid_oauth_providers)
user.available_services.where(provider: valid_oauth_providers)
end
end

4
app/concerns/twitter_concern.rb
View file

@ -25,11 +25,11 @@ module TwitterConcern
end
def twitter_oauth_token
self.service.token
service.token
end
def twitter_oauth_token_secret
self.service.secret
service.secret
end
def twitter

3
app/concerns/web_request_concern.rb
View file

@ -1,3 +1,6 @@
require 'faraday'
require 'faraday_middleware'
module WebRequestConcern
extend ActiveSupport::Concern

15
app/controllers/agents_controller.rb
View file

@ -31,14 +31,15 @@ class AgentsController < ApplicationController
end
def type_details
agent = Agent.build_for_type(params[:type], current_user, {})
@agent = Agent.build_for_type(params[:type], current_user, {})
render :json => {
:can_be_scheduled => agent.can_be_scheduled?,
:default_schedule => agent.default_schedule,
:can_receive_events => agent.can_receive_events?,
:can_create_events => agent.can_create_events?,
:options => agent.default_options,
:description_html => agent.html_description
:can_be_scheduled => @agent.can_be_scheduled?,
:default_schedule => @agent.default_schedule,
:can_receive_events => @agent.can_receive_events?,
:can_create_events => @agent.can_create_events?,
:options => @agent.default_options,
:description_html => @agent.html_description,
:form => render_to_string(partial: 'oauth_dropdown')
}
end

23
app/controllers/application_controller.rb
View file

@ -13,4 +13,27 @@ class ApplicationController < ActionController::Base
devise_parameter_sanitizer.for(:sign_in) { |u| u.permit(:login, :username, :email, :password, :remember_me) }
devise_parameter_sanitizer.for(:account_update) { |u| u.permit(:username, :email, :password, :password_confirmation, :current_password) }
end
def upgrade_warning
return unless current_user
twitter_oauth_check
basecamp_auth_check
end
private
def twitter_oauth_check
if ENV['TWITTER_OAUTH_KEY'].blank? || ENV['TWITTER_OAUTH_SECRET'].blank?
if @twitter_agent = current_user.agents.where("type like 'Agents::Twitter%'").first
@twitter_oauth_key = @twitter_agent.options['consumer_key'].presence || @twitter_agent.credential('twitter_consumer_key')
@twitter_oauth_secret = @twitter_agent.options['consumer_secret'].presence || @twitter_agent.credential('twitter_consumer_secret')
end
end
end
def basecamp_auth_check
if ENV['THIRTY_SEVEN_SIGNALS_OAUTH_KEY'].blank? || ENV['THIRTY_SEVEN_SIGNALS_OAUTH_SECRET'].blank?
@basecamp_agent = current_user.agents.where(type: 'Agents::BasecampAgent').first
end
end
end

2
app/controllers/home_controller.rb
View file

@ -1,6 +1,8 @@
class HomeController < ApplicationController
skip_before_filter :authenticate_user!
before_filter :upgrade_warning, only: :index
def index
end

2
app/controllers/scenarios_controller.rb
View file

@ -45,6 +45,8 @@ class ScenariosController < ApplicationController
@exporter = AgentsExporter.new(:name => @scenario.name,
:description => @scenario.description,
:guid => @scenario.guid,
:tag_fg_color => @scenario.tag_fg_color,
:tag_bg_color => @scenario.tag_bg_color,
:source_url => @scenario.public? && export_scenario_url(@scenario),
:agents => @scenario.agents)
response.headers['Content-Disposition'] = 'attachment; filename="' + @exporter.filename + '"'

1
app/controllers/services_controller.rb
View file

@ -1,4 +1,5 @@
class ServicesController < ApplicationController
before_filter :upgrade_warning, only: :index
def index
@services = current_user.services.page(params[:page])

4
app/helpers/agent_helper.rb
View file

@ -8,11 +8,11 @@ module AgentHelper
def scenario_links(agent)
agent.scenarios.map { |scenario|
link_to(scenario.name, scenario, class: "label label-info")
link_to(scenario.name, scenario, class: "label", style: style_colors(scenario))
}.join(" ").html_safe
end
def agent_show_class(agent)
agent.short_type.underscore.dasherize
end
end
end

7
app/helpers/markdown_helper.rb Normal file
View file

@ -0,0 +1,7 @@
module MarkdownHelper
def markdown(text)
Kramdown::Document.new(text, :auto_ids => false).to_html.html_safe
end
end

23
app/helpers/scenario_helper.rb Normal file
View file

@ -0,0 +1,23 @@
module ScenarioHelper
def style_colors(scenario)
colors = {
color: scenario.tag_fg_color || default_scenario_fg_color,
background_color: scenario.tag_bg_color || default_scenario_bg_color
}.map { |key, value| "#{key.to_s.dasherize}:#{value}" }.join(';')
end
def scenario_label(scenario, text = nil)
text ||= scenario.name
content_tag :span, text, class: 'label scenario', style: style_colors(scenario)
end
def default_scenario_bg_color
'#5BC0DE'
end
def default_scenario_fg_color
'#FFFFFF'
end
end

5
app/helpers/service_helper.rb Normal file
View file

@ -0,0 +1,5 @@
module ServiceHelper
def has_oauth_configuration_for(provider)
ENV["#{provider.upcase}_OAUTH_KEY"].present? && ENV["#{provider.upcase}_OAUTH_SECRET"].present?
end
end

16
app/models/agent.rb
View file

@ -44,7 +44,7 @@ class Agent < ActiveRecord::Base
after_save :possibly_update_event_expirations
belongs_to :user, :inverse_of => :agents
belongs_to :service
belongs_to :service, :inverse_of => :agents
has_many :events, -> { order("events.id desc") }, :dependent => :delete_all, :inverse_of => :agent
has_one :most_recent_event, :inverse_of => :agent, :class_name => "Event", :order => "events.id desc"
has_many :logs, -> { order("agent_logs.id desc") }, :dependent => :delete_all, :inverse_of => :agent, :class_name => "AgentLog"
@ -392,7 +392,7 @@ class AgentDrop
@object.short_type
end
METHODS = [
[
:name,
:type,
:options,
@ -403,19 +403,9 @@ class AgentDrop
:disabled,
:keep_events_for,
:propagate_immediately,
]
METHODS.each { |attr|
].each { |attr|
define_method(attr) {
@object.__send__(attr)
} unless method_defined?(attr)
}
def each(&block)
return to_enum(__method__) unless block
METHODS.each { |attr|
yield [attr, __sent__(attr)]
}
end
end

53
app/models/agents/basecamp_agent.rb
View file

@ -21,25 +21,25 @@ module Agents
event_description <<-MD
Events are the raw JSON provided by the Basecamp API. Should look something like:
{
"creator": {
"fullsize_avatar_url": "https://dge9rmgqjs8m1.cloudfront.net/global/dfsdfsdfdsf/original.gif?r=3",
"avatar_url": "http://dge9rmgqjs8m1.cloudfront.net/global/dfsdfsdfdsf/avatar.gif?r=3",
"name": "Dominik Sander",
"id": 123456
},
"attachments": [],
"raw_excerpt": "test test",
"excerpt": "test test",
"id": 6454342343,
"created_at": "2014-04-17T10:25:31.000+02:00",
"updated_at": "2014-04-17T10:25:31.000+02:00",
"summary": "commented on whaat",
"action": "commented on",
"target": "whaat",
"url": "https://basecamp.com/12456/api/v1/projects/76454545-explore-basecamp/messages/76454545-whaat.json",
"html_url": "https://basecamp.com/12456/projects/76454545-explore-basecamp/messages/76454545-whaat#comment_76454545"
}
{
"creator": {
"fullsize_avatar_url": "https://dge9rmgqjs8m1.cloudfront.net/global/dfsdfsdfdsf/original.gif?r=3",
"avatar_url": "http://dge9rmgqjs8m1.cloudfront.net/global/dfsdfsdfdsf/avatar.gif?r=3",
"name": "Dominik Sander",
"id": 123456
},
"attachments": [],
"raw_excerpt": "test test",
"excerpt": "test test",
"id": 6454342343,
"created_at": "2014-04-17T10:25:31.000+02:00",
"updated_at": "2014-04-17T10:25:31.000+02:00",
"summary": "commented on whaat",
"action": "commented on",
"target": "whaat",
"url": "https://basecamp.com/12456/api/v1/projects/76454545-explore-basecamp/messages/76454545-whaat.json",
"html_url": "https://basecamp.com/12456/projects/76454545-explore-basecamp/messages/76454545-whaat#comment_76454545"
}
MD
default_schedule "every_10m"
@ -59,28 +59,29 @@ module Agents
end
def check
self.service.prepare_request
service.prepare_request
reponse = HTTParty.get request_url, request_options.merge(query_parameters)
memory[:last_run] = Time.now.utc.iso8601
if last_check_at != nil
JSON.parse(reponse.body).each do |event|
events = JSON.parse(reponse.body)
if !memory[:last_event].nil?
events.each do |event|
create_event :payload => event
end
end
memory[:last_event] = events.first['created_at'] if events.length > 0
save!
end
private
def request_url
"https://basecamp.com/#{URI.encode(self.service.options[:user_id].to_s)}/api/v1/projects/#{URI.encode(interpolated[:project_id].to_s)}/events.json"
"https://basecamp.com/#{URI.encode(service.options[:user_id].to_s)}/api/v1/projects/#{URI.encode(interpolated[:project_id].to_s)}/events.json"
end
def request_options
{:headers => {"User-Agent" => "Huginn (https://github.com/cantino/huginn)", "Authorization" => "Bearer \"#{self.service.token}\""}}
{:headers => {"User-Agent" => "Huginn (https://github.com/cantino/huginn)", "Authorization" => "Bearer \"#{service.token}\""}}
end
def query_parameters
memory[:last_run].present? ? { :query => {:since => memory[:last_run]} } : {}
memory[:last_event].present? ? { :query => {:since => memory[:last_event]} } : {}
end
end
end

18
app/models/agents/event_formatting_agent.rb
View file

@ -51,7 +51,7 @@ module Agents
{
"path": "{{date.pretty}}",
"regexp": "\\A(?<time>\\d\\d:\\d\\d [AP]M [A-Z]+)",
"to": "pretty_date",
"to": "pretty_date"
}
]
}
@ -61,7 +61,7 @@ module Agents
"pretty_date": {
"time": "10:00 PM EST",
"0": "10:00 PM EST on January 11, 2013"
"1": "10:00 PM EST",
"1": "10:00 PM EST"
}
So you can use it in `instructions` like this:
@ -80,7 +80,19 @@ module Agents
}
MD
event_description "User defined"
event_description do
"Events will have the following fields%s:\n\n %s" % [
case options['mode'].to_s
when 'merged'
', merged with the original contents'
when /\{/
', conditionally merged with the original contents'
end,
Utils.pretty_print(Hash[options['instructions'].keys.map { |key|
[key, "..."]
}])
]
end
after_save :clear_matchers

2
app/models/agents/google_calendar_publish_agent.rb
View file

@ -62,7 +62,7 @@ module Agents
....
},
'agent_id' => 1234,
'event_id' => 3432,
'event_id' => 3432
}
MD

230
app/models/agents/website_agent.rb
View file

@ -1,6 +1,4 @@
require 'nokogiri'
require 'faraday'
require 'faraday_middleware'
require 'date'
module Agents
@ -19,7 +17,7 @@ module Agents
`url` can be a single url, or an array of urls (for example, for multiple pages with the exact same structure but different content to scrape)
The `type` value can be `xml`, `html`, or `json`.
The `type` value can be `xml`, `html`, `json`, or `text`.
To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes.
@ -40,6 +38,28 @@ module Agents
"description": { "path": "results.data[*].description" }
}
When parsing text, each sub-hash should contain a `regexp` and `index`. Output text is matched against the regular expression repeatedly from the beginning through to the end, collecting a captured group specified by `index` in each match. Each index should be either an integer or a string name which corresponds to `(?<_name_>...)`. For example, to parse lines of `_word_: _definition_`, the following should work:
"extract": {
"word": { "regexp": "^(.+?): (.+)$", index: 1 },
"definition": { "regexp": "^(.+?): (.+)$", index: 2 }
}
Or if you prefer names to numbers for index:
"extract": {
"word": { "regexp": "^(?<word>.+?): (?<definition>.+)$", index: 'word' },
"definition": { "regexp": "^(?<word>.+?): (?<definition>.+)$", index: 'definition' }
}
To extract the whole content as one event:
"extract": {
"content": { "regexp": "\A(?m:.)*\z", index: 0 }
}
Beware that `.` does not match the newline character (LF) unless the `m` flag is in effect, and `^`/`$` basically match every line beginning/end. See [this document](http://ruby-doc.org/core-#{RUBY_VERSION}/doc/regexp_rdoc.html) to learn the regular expression variant used in this service.
Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful.
Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`.
@ -58,7 +78,11 @@ module Agents
MD
event_description do
"Events will have the fields you specified. Your options look like:\n\n #{Utils.pretty_print interpolated['extract']}"
"Events will have the following fields:\n\n %s" % [
Utils.pretty_print(Hash[options['extract'].keys.map { |key|
[key, "..."]
}])
]
end
def working?
@ -137,77 +161,60 @@ module Agents
log "Storing new result for '#{name}': #{doc.inspect}"
create_event :payload => doc
end
else
output = {}
interpolated['extract'].each do |name, extraction_details|
if extraction_type == "json"
result = Utils.values_at(doc, extraction_details['path'])
log "Extracting #{extraction_type} at #{extraction_details['path']}: #{result}"
else
case
when css = extraction_details['css']
nodes = doc.css(css)
when xpath = extraction_details['xpath']
doc.remove_namespaces! # ignore xmlns, useful when parsing atom feeds
nodes = doc.xpath(xpath)
else
error '"css" or "xpath" is required for HTML or XML extraction'
return
end
case nodes
when Nokogiri::XML::NodeSet
result = nodes.map { |node|
case value = node.xpath(extraction_details['value'])
when Float
# Node#xpath() returns any numeric value as float;
# convert it to integer as appropriate.
value = value.to_i if value.to_i == value
end
value.to_s
}
else
error "The result of HTML/XML extraction was not a NodeSet"
return
end
log "Extracting #{extraction_type} at #{xpath || css}: #{result}"
end
output[name] = result
next
end
output =
case extraction_type
when 'json'
extract_json(doc)
when 'text'
extract_text(doc)
else
extract_xml(doc)
end
num_unique_lengths = interpolated['extract'].keys.map { |name| output[name].length }.uniq
num_unique_lengths = interpolated['extract'].keys.map { |name| output[name].length }.uniq
if num_unique_lengths.length != 1
error "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
return
if num_unique_lengths.length != 1
raise "Got an uneven number of matches for #{interpolated['name']}: #{interpolated['extract'].inspect}"
end
old_events = previous_payloads num_unique_lengths.first
num_unique_lengths.first.times do |index|
result = {}
interpolated['extract'].keys.each do |name|
result[name] = output[name][index]
if name.to_s == 'url'
result[name] = (response.env[:url] + result[name]).to_s
end
end
old_events = previous_payloads num_unique_lengths.first
num_unique_lengths.first.times do |index|
result = {}
interpolated['extract'].keys.each do |name|
result[name] = output[name][index]
if name.to_s == 'url'
result[name] = (response.env[:url] + result[name]).to_s
end
end
if store_payload!(old_events, result)
log "Storing new parsed result for '#{name}': #{result.inspect}"
create_event :payload => result
end
if store_payload!(old_events, result)
log "Storing new parsed result for '#{name}': #{result.inspect}"
create_event :payload => result
end
end
else
error "Failed: #{response.inspect}"
raise "Failed: #{response.inspect}"
end
end
rescue => e
error e.message
end
def receive(incoming_events)
incoming_events.each do |event|
Thread.current[:current_event] = event
url_to_scrape = event.payload['url']
check_url(url_to_scrape) if url_to_scrape =~ /^https?:\/\//i
end
ensure
Thread.current[:current_event] = nil
end
def interpolated(event = Thread.current[:current_event])
super
end
private
@ -216,22 +223,22 @@ module Agents
# If mode is set to 'on_change', this method may return false and update an existing
# event to expire further in the future.
def store_payload!(old_events, result)
if !interpolated['mode'].present?
return true
elsif interpolated['mode'].to_s == "all"
return true
elsif interpolated['mode'].to_s == "on_change"
case interpolated['mode'].presence
when 'on_change'
result_json = result.to_json
old_events.each do |old_event|
if old_event.payload.to_json == result_json
old_event.expires_at = new_event_expiration_date
old_event.save!
return false
end
end
end
return true
true
when 'all', ''
true
else
raise "Illegal options[mode]: #{interpolated['mode']}"
end
raise "Illegal options[mode]: " + interpolated['mode'].to_s
end
def previous_payloads(num_events)
@ -244,7 +251,7 @@ module Agents
look_back = UNIQUENESS_LOOK_BACK
end
end
events.order("id desc").limit(look_back) if interpolated['mode'].present? && interpolated['mode'].to_s == "on_change"
events.order("id desc").limit(look_back) if interpolated['mode'] == "on_change"
end
def extract_full_json?
@ -253,35 +260,94 @@ module Agents
def extraction_type
(interpolated['type'] || begin
if interpolated['url'] =~ /\.(rss|xml)$/i
case interpolated['url']
when /\.(rss|xml)$/i
"xml"
elsif interpolated['url'] =~ /\.json$/i
when /\.json$/i
"json"
when /\.(txt|text)$/i
"text"
else
"html"
end
end).to_s
end
def extract_each(doc, &block)
interpolated['extract'].each_with_object({}) { |(name, extraction_details), output|
output[name] = block.call(extraction_details)
}
end
def extract_json(doc)
extract_each(doc) { |extraction_details|
result = Utils.values_at(doc, extraction_details['path'])
log "Extracting #{extraction_type} at #{extraction_details['path']}: #{result}"
result
}
end
def extract_text(doc)
extract_each(doc) { |extraction_details|
regexp = Regexp.new(extraction_details['regexp'])
result = []
doc.scan(regexp) {
result << Regexp.last_match[extraction_details['index']]
}
log "Extracting #{extraction_type} at #{regexp}: #{result}"
result
}
end
def extract_xml(doc)
extract_each(doc) { |extraction_details|
case
when css = extraction_details['css']
nodes = doc.css(css)
when xpath = extraction_details['xpath']
doc.remove_namespaces! # ignore xmlns, useful when parsing atom feeds
nodes = doc.xpath(xpath)
else
raise '"css" or "xpath" is required for HTML or XML extraction'
end
case nodes
when Nokogiri::XML::NodeSet
result = nodes.map { |node|
case value = node.xpath(extraction_details['value'])
when Float
# Node#xpath() returns any numeric value as float;
# convert it to integer as appropriate.
value = value.to_i if value.to_i == value
end
value.to_s
}
else
raise "The result of HTML/XML extraction was not a NodeSet"
end
log "Extracting #{extraction_type} at #{xpath || css}: #{result}"
result
}
end
def parse(data)
case extraction_type
when "xml"
Nokogiri::XML(data)
when "json"
JSON.parse(data)
when "html"
Nokogiri::HTML(data)
else
raise "Unknown extraction type #{extraction_type}"
when "xml"
Nokogiri::XML(data)
when "json"
JSON.parse(data)
when "html"
Nokogiri::HTML(data)
when "text"
data
else
raise "Unknown extraction type #{extraction_type}"
end
end
def is_positive_integer?(value)
begin
Integer(value) >= 0
rescue
false
end
Integer(value) >= 0
rescue
false
end
end
end

29
app/models/event.rb
View file

@ -44,26 +44,21 @@ class Event < ActiveRecord::Base
end
class EventDrop
def initialize(event, payload = event.payload)
super(event)
@payload = payload
end
def before_method(key)
if @payload.key?(key)
@payload[key]
else
case key
when 'agent'
@object.agent
when 'created_at'
@object.created_at
end
end
def initialize(object, locals = nil)
locals ||= object.payload
super
end
def each(&block)
return to_enum(__method__) unless block
@payload.each(&block)
@locals.each(&block)
end
def agent
@object.agent
end
def created_at
@object.created_at
end
end

7
app/models/scenario.rb
View file

@ -1,7 +1,7 @@
class Scenario < ActiveRecord::Base
include HasGuid
attr_accessible :name, :agent_ids, :description, :public, :source_url
attr_accessible :name, :agent_ids, :description, :public, :source_url, :tag_fg_color, :tag_bg_color
belongs_to :user, :counter_cache => :scenario_count, :inverse_of => :scenarios
has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :scenario
@ -9,6 +9,11 @@ class Scenario < ActiveRecord::Base
validates_presence_of :name, :user
validates_format_of :tag_fg_color, :tag_bg_color,
# Regex adapted from: http://stackoverflow.com/a/1636354/3130625
:with => /\A#(?:[0-9a-fA-F]{3}){1,2}\z/, :allow_nil => true,
:message => "must be a valid hex color."
validate :agents_are_owned
protected

6
app/models/scenario_import.rb
View file

@ -60,10 +60,14 @@ class ScenarioImport
description = parsed_data['description']
name = parsed_data['name']
links = parsed_data['links']
tag_fg_color = parsed_data['tag_fg_color']
tag_bg_color = parsed_data['tag_bg_color']
source_url = parsed_data['source_url'].presence || nil
@scenario = user.scenarios.where(:guid => guid).first_or_initialize
@scenario.update_attributes!(:name => name, :description => description,
:source_url => source_url, :public => false)
:source_url => source_url, :public => false,
:tag_fg_color => tag_fg_color,
:tag_bg_color => tag_bg_color)
unless options[:skip_agents]
created_agents = agent_diffs.map do |agent_diff|

87
app/models/service.rb
View file

@ -1,17 +1,22 @@
class Service < ActiveRecord::Base
attr_accessible :provider, :name, :token, :secret, :refresh_token, :expires_at, :global, :options
PROVIDER_TO_ENV_MAP = {'37signals' => 'THIRTY_SEVEN_SIGNALS'}
attr_accessible :provider, :name, :token, :secret, :refresh_token, :expires_at, :global, :options, :uid
serialize :options, Hash
belongs_to :user
has_many :agents
belongs_to :user, :inverse_of => :services
has_many :agents, :inverse_of => :service
validates_presence_of :user_id, :provider, :name, :token
before_destroy :disable_agents
def disable_agents
self.agents.each do |agent|
scope :available_to_user, lambda { |user| where("services.user_id = ? or services.global = true", user.id) }
scope :by_name, lambda { |dir = 'desc'| order("services.name #{dir}") }
def disable_agents(conditions = {})
agents.where.not(conditions[:where_not] || {}).each do |agent|
agent.service_id = nil
agent.disabled = true
agent.save!(validate: false)
@ -19,52 +24,66 @@ class Service < ActiveRecord::Base
end
def toggle_availability!
disable_agents(where_not: {user_id: self.user_id}) if global
self.global = !self.global
self.save!
end
def prepare_request
if self.expires_at && Time.now > self.expires_at
self.refresh_token!
if expires_at && Time.now > expires_at
refresh_token!
end
end
def refresh_token!
response = HTTParty.post(endpoint, query: {
type: 'refresh',
client_id: ENV["#{self.provider.upcase}_OAUTH_KEY"],
client_secret: ENV["#{self.provider.upcase}_OAUTH_SECRET"],
refresh_token: self.refresh_token
client_id: oauth_key,
client_secret: oauth_secret,
refresh_token: refresh_token
})
data = JSON.parse(response.body)
self.update(expires_at: Time.now + data['expires_in'], token: data['access_token'], refresh_token: data['refresh_token'].presence || self.refresh_token)
update(expires_at: Time.now + data['expires_in'], token: data['access_token'], refresh_token: data['refresh_token'].presence || refresh_token)
end
def self.initialize_or_update_via_omniauth(omniauth)
case omniauth['provider']
when 'twitter'
find_or_initialize_by(provider: omniauth['provider'], name: omniauth['info']['nickname']).tap do |service|
service.assign_attributes(token: omniauth['credentials']['token'], secret: omniauth['credentials']['secret'])
end
when 'github'
find_or_initialize_by(provider: omniauth['provider'], name: omniauth['info']['nickname']).tap do |service|
service.assign_attributes(token: omniauth['credentials']['token'])
end
when '37signals'
find_or_initialize_by(provider: omniauth['provider'], name: omniauth['info']['name']).tap do |service|
service.assign_attributes(token: omniauth['credentials']['token'],
refresh_token: omniauth['credentials']['refresh_token'],
expires_at: Time.at(omniauth['credentials']['expires_at']),
options: {user_id: omniauth['extra']['accounts'][0]['id']})
end
else
false
end
end
private
def endpoint
client_options = "OmniAuth::Strategies::#{OmniAuth::Utils.camelize(self.provider)}".constantize.default_options['client_options']
URI.join(client_options['site'], client_options['token_url'])
end
def provider_to_env
PROVIDER_TO_ENV_MAP[provider].presence || provider.upcase
end
def oauth_key
ENV["#{provider_to_env}_OAUTH_KEY"]
end
def oauth_secret
ENV["#{provider_to_env}_OAUTH_SECRET"]
end
def self.provider_specific_options(omniauth)
case omniauth['provider']
when 'twitter', 'github'
{ name: omniauth['info']['nickname'] }
when '37signals'
{ user_id: omniauth['extra']['accounts'][0]['id'], name: omniauth['info']['name'] }
else
{ name: omniauth['info']['nickname'] }
end
end
def self.initialize_or_update_via_omniauth(omniauth)
options = provider_specific_options(omniauth)
find_or_initialize_by(provider: omniauth['provider'], uid: omniauth['uid'].to_s).tap do |service|
service.assign_attributes token: omniauth['credentials']['token'],
secret: omniauth['credentials']['secret'],
name: options[:name],
refresh_token: omniauth['credentials']['refresh_token'],
expires_at: omniauth['credentials']['expires_at'] && Time.at(omniauth['credentials']['expires_at']),
options: options
end
end
end

4
app/models/user.rb
View file

@ -27,10 +27,10 @@ class User < ActiveRecord::Base
has_many :agents, -> { order("agents.created_at desc") }, :dependent => :destroy, :inverse_of => :user
has_many :logs, :through => :agents, :class_name => "AgentLog"
has_many :scenarios, :inverse_of => :user, :dependent => :destroy
has_many :services, -> { order("services.name")}, :dependent => :destroy
has_many :services, -> { by_name('asc') }, :dependent => :destroy
def available_services
Service.where("user_id = ? or global = true", self.id).order("services.name desc")
Service.available_to_user(self).by_name
end
# Allow users to login via either email or username.

2
app/views/agents/_action_menu.html.erb
View file

@ -32,7 +32,7 @@
<% agent.scenarios.each do |scenario| %>
<li>
<%= link_to "<span class='color-warning glyphicon glyphicon-remove-circle'></span> Remove from <span class='scenario label label-info'>#{h scenario.name}</span>".html_safe, leave_scenario_agent_path(agent, :scenario_id => scenario.to_param, :return => returnTo), method: :put, :tabindex => "-1" %>
<%= link_to "<span class='color-warning glyphicon glyphicon-remove-circle'></span> Remove from #{scenario_label(scenario)}".html_safe, leave_scenario_agent_path(agent, :scenario_id => scenario.to_param, :return => returnTo), method: :put, :tabindex => "-1" %>
</li>
<% end %>
<% end %>

7
app/views/agents/_form.html.erb
View file

@ -31,12 +31,7 @@
</div>
<div class='oauthable-form'>
<% if @agent.try(:oauthable?) %>
<div class="form-group type-select">
<%= f.label :service %>
<%= f.select :service_id, options_for_select(@agent.valid_services(current_user).collect { |s| ["(#{s.provider}) #{s.name}", s.id]}, @agent.service_id),{}, class: 'form-control' %>
</div>
<% end %>
<%= render partial: 'oauth_dropdown' %>
</div>
<div class="form-group">

6
app/views/agents/_oauth_dropdown.html.erb Normal file
View file

@ -0,0 +1,6 @@
<% if @agent.try(:oauthable?) %>
<div class="form-group type-select">
<%= label_tag :service %>
<%= select_tag 'agent[service_id]', options_for_select(@agent.valid_services_for(current_user).collect { |s| ["(#{s.provider}) #{s.name}", s.id]}, @agent.service_id), class: 'form-control' %>
</div>
<% end %>

8
app/views/agents/show.html.erb
View file

@ -110,8 +110,8 @@
<% if @agent.can_receive_events? %>
<p>
<b>Event sources:</b>
<% if @agent.sources.length %>
<%= @agent.sources.map { |source_agent| link_to(source_agent.name, agent_path(source_agent)) }.to_sentence.html_safe %>
<% if (agents = @agent.sources).length > 0 %>
<%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %>
<% else %>
None
<% end %>
@ -126,8 +126,8 @@
<% if @agent.can_create_events? %>
<p>
<b>Event receivers:</b>
<% if @agent.receivers.length %>
<%= @agent.receivers.map { |receiver_agent| link_to(receiver_agent.name, agent_path(receiver_agent)) }.to_sentence.html_safe %>
<% if (agents = @agent.receivers).length > 0 %>
<%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %>
<% else %>
None
<% end %>

26
app/views/application/_upgrade_warning.html.erb Normal file
View file

@ -0,0 +1,26 @@
<% if @twitter_agent || @basecamp_agent %>
<div class="alert alert-danger" role="alert">
<p>
<b>Warning!</b> You need to update your Huginn configuration, so your agents continue to work with the new OAuth services.
</p>
<br/>
<% if @twitter_agent %>
<p>
To complete the migration of your <b>Twitter</b> agents you need to update your .env file and add the following two lines:
<pre>
TWITTER_OAUTH_KEY=<%= @twitter_oauth_key %>
TWITTER_OAUTH_SECRET=<%= @twitter_oauth_secret %>
</pre>
To authenticate new accounts with your twitter OAuth application you need to log in the to <a href="https://apps.twitter.com/" target="_blank">twitter application management page</a> and set the callback URL of your application to "http<%= ENV['FORCE_SSL'] == 'true' ? 's' : '' %>://<%= ENV['DOMAIN'] %>/auth/twitter/callback".
</p>
<% end %>
<% if @basecamp_agent %>
<p>
Your <b>Basecamp</b> agents could not be migrated automatically. You need to manually register an application with 37signals and authenticate Huginn to use it.<br/>
Have a look at the <%= link_to 'Wiki', 'https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications', target: '_blank' %> if you need help.
</p>
<% end %>
</div>
<% end -%>

5
app/views/layouts/application.html.erb
View file

@ -24,7 +24,10 @@
<%= render 'layouts/messages' %>
</div>
</div>
<% if user_signed_in? %>
<%= render "upgrade_warning" %>
<% end %>
<%= yield %>
</div>

10
app/views/scenario_imports/_step_two.html.erb
View file

@ -13,9 +13,8 @@
<div class="alert alert-warning">
<span class='glyphicon glyphicon-warning-sign'></span>
This Scenario already exists in your system. The import will update your existing
<span class='label label-info scenario'><%= @scenario_import.existing_scenario.name %></span> Scenario's title
and
description. Below you can customize how the individual agents get updated.
<%= scenario_label(@scenario_import.existing_scenario) %> Scenario's title,
description and tag colors. Below you can customize how the individual agents get updated.
</div>
<% end %>
@ -30,7 +29,7 @@
</div>
<% if @scenario_import.parsed_data["description"].present? %>
<blockquote><%= @scenario_import.parsed_data["description"] %></blockquote>
<blockquote><%= markdown(@scenario_import.parsed_data["description"]) %></blockquote>
<% end %>
</div>
@ -120,12 +119,13 @@
</div>
<% end %>
</div>
<% if agent_diff.requires_service? %>
<div class='row'>
<div class='col-md-4'>
<div class="form-group type-select">
<%= label_tag "scenario_import[merges][#{index}][service_id]", 'Service' %>
<%= select_tag "scenario_import[merges][#{index}][service_id]", options_for_select(agent_diff.agent_instance.valid_services(current_user).collect { |s| ["(#{s.provider}) #{s.name}", s.id]}, agent_diff.service_id.try(:current)), class: 'form-control' %>
<%= select_tag "scenario_import[merges][#{index}][service_id]", options_for_select(agent_diff.agent_instance.valid_services_for(current_user).collect { |s| ["(#{s.provider}) #{s.name}", s.id]}, agent_diff.agent.try(:service_id)), class: 'form-control' %>
</div>
</div>
</div>

14
app/views/scenarios/_form.html.erb
View file

@ -15,6 +15,18 @@
<%= f.text_field :name, :class => 'form-control', :placeholder => "Name your Scenario" %>
</div>
</div>
<div class="col-md-2">
<div class="form-group">
<%= f.label :tag_bg_color, "Tag Background Color" %>
<%= f.color_field :tag_bg_color, :class => 'form-control', :value => @scenario.tag_bg_color || default_scenario_bg_color %>
</div>
</div>
<div class="col-md-2">
<div class="form-group">
<%= f.label :tag_fg_color, "Tag Foreground Color" %>
<%= f.color_field :tag_fg_color, :class => 'form-control', :value => @scenario.tag_fg_color || default_scenario_fg_color %>
</div>
</div>
</div>
<div class="row">
@ -54,4 +66,4 @@
</div>
</div>
</div>
<% end %>
<% end %>

3
app/views/scenarios/index.html.erb
View file

@ -21,6 +21,7 @@
<% @scenarios.each do |scenario| %>
<tr>
<td>
<%= scenario_label(scenario, content_tag(:i, '', class: 'glyphicon glyphicon-font')) %>
<%= link_to(scenario.name, scenario) %>
</td>
<td><%= link_to pluralize(scenario.agents.count, "agent"), scenario %></td>
@ -47,4 +48,4 @@
</div>
</div>
</div>
</div>
</div>

4
app/views/scenarios/share.html.erb
View file

@ -2,7 +2,7 @@
<div class='row'>
<div class='col-md-12'>
<div class="page-header">
<h2>Share <span class='label label-info scenario'><%= @scenario.name %></span> with the world</h2>
<h2>Share <%= scenario_label(@scenario) %> with the world</h2>
</div>
<p>
@ -30,4 +30,4 @@
</div>
</div>
</div>
</div>
</div>

5
app/views/scenarios/show.html.erb
View file

@ -2,11 +2,12 @@
<div class='row'>
<div class='col-md-12'>
<div class="page-header">
<h2><span class='label label-info scenario'><%= @scenario.name %></span> <%= "Public" if @scenario.public? %> Scenario</h2>
<h2><%= scenario_label(@scenario) %> <%= "Public" if @scenario.public? %> Scenario</h2>
</div>
<% if @scenario.description.present? %>
<blockquote><%= @scenario.description %></blockquote>
<blockquote><%= markdown(@scenario.description) %></blockquote>
<% end %>
<%= render 'agents/table', :returnTo => scenario_path(@scenario) %>

18
app/views/services/index.html.erb
View file

@ -7,13 +7,19 @@
</h2>
</div>
<p>
Before you can authenticate with a service, you need to set it up. Have a look at the
Before you can authenticate with a service, you need to set it up. Have a look at the Huginn
<%= link_to 'wiki', 'https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications', target: :_blank %>
for guidance.
</p>
<p><%= link_to "Authenticate with Twitter", "/auth/twitter" %></p>
<p><%= link_to "Authenticate with 37Signals (Basecamp)", "/auth/37signals" %></p>
<p><%= link_to "Authenticate with Github", "/auth/github" %></p>
<% if has_oauth_configuration_for('twitter') %>
<p><%= link_to "Authenticate with Twitter", "/auth/twitter" %></p>
<% end %>
<% if has_oauth_configuration_for('thirty_seven_signals') %>
<p><%= link_to "Authenticate with 37Signals (Basecamp)", "/auth/37signals" %></p>
<% end -%>
<% if has_oauth_configuration_for('github') %>
<p><%= link_to "Authenticate with Github", "/auth/github" %></p>
<% end -%>
<hr>
<div class='table-responsive'>
@ -33,9 +39,9 @@
<td>
<div class="btn-group btn-group-xs">
<% if service.global %>
<%= link_to 'Make private', toggle_availability_service_path(service), method: :post, data: { confirm: 'Are you sure you want to remove the access to this service for every user?'}, class: "btn btn-default" %>
<%= link_to 'Make private', toggle_availability_service_path(service), method: :post, data: { confirm: 'Are you sure you want to remove access to your data on this service for other users?'}, class: "btn btn-default" %>
<% else %>
<%= link_to 'Make global', toggle_availability_service_path(service), method: :post, data: { confirm: 'Are you sure you want to grant every user access to this service?'}, class: "btn btn-default" %>
<%= link_to 'Make global', toggle_availability_service_path(service), method: :post, data: { confirm: 'Are you sure you want to grant every user on this system access to your data on this service?'}, class: "btn btn-default" %>
<% end %>
<%= link_to 'Delete', service_path(service), method: :delete, data: { confirm: 'Are you sure?' }, class: "btn btn-default btn-danger" %>
</div>

6
app/views/system_mailer/send_message.html.erb
View file

@ -5,14 +5,14 @@
</head>
<body>
<% if @headline %>
<h1><%= @headline %></h1>
<h1><%= sanitize @headline %></h1>
<% end %>
<% @groups.each do |group| %>
<div style='margin-bottom: 10px;'>
<div><%= group[:title] %></div>
<div><%= sanitize group[:title] %></div>
<% group[:entries].each do |entry| %>
<div style='margin-left: 10px;'>
<%= entry %>
<%= sanitize entry %>
</div>
<% end %>
</div>

8
db/migrate/20140525150140_migrate_agents_to_service_authentication.rb
View file

@ -26,6 +26,7 @@ class MigrateAgentsToServiceAuthentication < ActiveRecord::Migration
agent.service_id = service.id
agent.save!(validate: false)
end
migrated = false
if agents.length > 0
puts <<-EOF.strip_heredoc
@ -34,18 +35,23 @@ class MigrateAgentsToServiceAuthentication < ActiveRecord::Migration
TWITTER_OAUTH_KEY=#{twitter_consumer_key(agents.first)}
TWITTER_OAUTH_SECRET=#{twitter_consumer_secret(agents.first)}
To authenticate new accounts with your twitter OAuth application you need to log in the to twitter application management page (https://apps.twitter.com/)
and set the callback URL of your application to "http#{ENV['FORCE_SSL'] == 'true' ? 's' : ''}://#{ENV['DOMAIN']}/auth/twitter/callback"
EOF
migrated = true
end
if Agent.where(type: ['Agents::BasecampAgent']).count > 0
puts <<-EOF.strip_heredoc
Your Basecamp agents can not be migrated automatically. You need to manually register an application with 37signals and authenticate huginn to use it.
Your Basecamp agents can not be migrated automatically. You need to manually register an application with 37signals and authenticate Huginn to use it.
Have a look at the wiki (https://github.com/cantino/huginn/wiki/Configuring-OAuth-applications) if you need help.
EOF
migrated = true
end
sleep 20 if migrated
end
def down

5
db/migrate/20140809211540_remove_service_index_on_user_id.rb Normal file
View file

@ -0,0 +1,5 @@
class RemoveServiceIndexOnUserId < ActiveRecord::Migration
def change
remove_index :services, :user_id
end
end

7
db/migrate/20140811200922_add_uid_column_to_services.rb Normal file
View file

@ -0,0 +1,7 @@
class AddUidColumnToServices < ActiveRecord::Migration
def change
add_column :services, :uid, :string
add_index :services, :uid
add_index :services, :provider
end
end

6
db/migrate/20140820003139_add_tag_color_to_scenarios.rb Normal file
View file

@ -0,0 +1,6 @@
class AddTagColorToScenarios < ActiveRecord::Migration
def change
add_column :scenarios, :tag_bg_color, :string
add_column :scenarios, :tag_fg_color, :string
end
end

32
db/schema.rb
View file

@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20140813110107) do
ActiveRecord::Schema.define(version: 20140820003139) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -22,8 +22,8 @@ ActiveRecord::Schema.define(version: 20140813110107) do
t.integer "level", default: 3, null: false
t.integer "inbound_event_id"
t.integer "outbound_event_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at"
t.datetime "updated_at"
end
create_table "agents", force: true do |t|
@ -45,8 +45,8 @@ ActiveRecord::Schema.define(version: 20140813110107) do
t.datetime "last_error_log_at"
t.boolean "propagate_immediately", default: false, null: false
t.boolean "disabled", default: false, null: false
t.integer "service_id"
t.string "guid", null: false, charset: "ascii", collation: "ascii_bin"
t.integer "service_id"
end
add_index "agents", ["guid"], name: "index_agents_on_guid", using: :btree
@ -64,8 +64,8 @@ ActiveRecord::Schema.define(version: 20140813110107) do
t.datetime "failed_at"
t.string "locked_by"
t.string "queue"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at"
t.datetime "updated_at"
end
add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree
@ -88,8 +88,8 @@ ActiveRecord::Schema.define(version: 20140813110107) do
create_table "links", force: true do |t|
t.integer "source_id"
t.integer "receiver_id"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.datetime "created_at"
t.datetime "updated_at"
t.integer "event_id_at_creation", default: 0, null: false
end
@ -115,15 +115,17 @@ ActiveRecord::Schema.define(version: 20140813110107) do
t.boolean "public", default: false, null: false
t.string "guid", null: false, charset: "ascii", collation: "ascii_bin"
t.string "source_url"
t.string "tag_bg_color"
t.string "tag_fg_color"
end
add_index "scenarios", ["user_id", "guid"], name: "index_scenarios_on_user_id_and_guid", unique: true, using: :btree
create_table "services", force: true do |t|
t.integer "user_id"
t.string "provider"
t.string "name"
t.text "token"
t.integer "user_id", null: false
t.string "provider", null: false
t.string "name", null: false
t.text "token", null: false
t.text "secret"
t.text "refresh_token"
t.datetime "expires_at"
@ -131,10 +133,12 @@ ActiveRecord::Schema.define(version: 20140813110107) do
t.text "options"
t.datetime "created_at"
t.datetime "updated_at"
t.string "uid"
end
add_index "services", ["user_id", "global"], name: "index_accounts_on_user_id_and_global", using: :btree
add_index "services", ["user_id"], name: "index_accounts_on_user_id", using: :btree
add_index "services", ["provider"], name: "index_services_on_provider", using: :btree
add_index "services", ["uid"], name: "index_services_on_uid", using: :btree
add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree
create_table "user_credentials", force: true do |t|
t.integer "user_id", null: false

4
lib/agents_exporter.rb
View file

@ -16,6 +16,8 @@ class AgentsExporter
:description => options[:description].presence || 'No description provided',
:source_url => options[:source_url],
:guid => options[:guid],
:tag_fg_color => options[:tag_fg_color],
:tag_bg_color => options[:tag_bg_color],
:exported_at => Time.now.utc.iso8601,
:agents => agents.map { |agent| agent_as_json(agent) },
:links => links
@ -51,4 +53,4 @@ class AgentsExporter
options[:propagate_immediately] = agent.propagate_immediately if agent.can_receive_events?
end
end
end
end

15
spec/concerns/liquid_interpolatable_spec.rb Normal file
View file

@ -0,0 +1,15 @@
require 'spec_helper'
describe LiquidInterpolatable::Filters do
before do
@filter = Class.new do
include LiquidInterpolatable::Filters
end.new
end
describe 'uri_escape' do
it 'should escape a string for use in URI' do
@filter.uri_escape('abc:/?=').should == 'abc%3A%2F%3F%3D'
end
end
end

2
spec/controllers/scenarios_controller_spec.rb
View file

@ -50,6 +50,8 @@ describe ScenariosController do
assigns(:exporter).options[:description].should == scenarios(:bob_weather).description
assigns(:exporter).options[:agents].should == scenarios(:bob_weather).agents
assigns(:exporter).options[:guid].should == scenarios(:bob_weather).guid
assigns(:exporter).options[:tag_fg_color].should == scenarios(:bob_weather).tag_fg_color
assigns(:exporter).options[:tag_bg_color].should == scenarios(:bob_weather).tag_bg_color
assigns(:exporter).options[:source_url].should be_falsey
response.headers['Content-Disposition'].should == 'attachment; filename="bob-s-weather-alert-scenario.json"'
response.headers['Content-Type'].should == 'application/json; charset=utf-8'

9
spec/controllers/services_controller_spec.rb
View file

@ -10,7 +10,7 @@ describe ServicesController do
describe "GET index" do
it "only returns sevices of the current user" do
get :index
assigns(:services).all? {|i| i.user.should == users(:bob) }.should be_true
assigns(:services).all? {|i| i.user.should == users(:bob) }.should == true
end
end
@ -41,17 +41,18 @@ describe ServicesController do
end
describe "accepting a callback url" do
it "should update the users credentials" do
it "should update the user's credentials" do
expect {
get :callback, provider: 'twitter'
}.to change { users(:bob).services.count }.by(1)
end
it "should not work with an unknown provider" do
it "should work with an unknown provider (for now)" do
request.env["omniauth.auth"]['provider'] = 'unknown'
expect {
get :callback, provider: 'unknown'
}.to change { users(:bob).services.count }.by(0)
}.to change { users(:bob).services.count }.by(1)
users(:bob).services.first.provider.should == 'unknown'
end
end
end

6
spec/fixtures/agents.yml vendored
View file

@ -115,3 +115,9 @@ bob_basecamp_agent:
user: bob
service: generic
guid: <%= SecureRandom.hex %>
jane_basecamp_agent:
type: Agents::BasecampAgent
user: jane
service: generic
guid: <%= SecureRandom.hex %>

14
spec/helpers/markdown_helper_spec.rb Normal file
View file

@ -0,0 +1,14 @@
require 'spec_helper'
describe MarkdownHelper do
describe '#markdown' do
it 'renders HTML from a markdown text' do
markdown('# Header').should =~ /<h1>Header<\/h1>/
markdown('## Header 2').should =~ /<h2>Header 2<\/h2>/
end
end
end

30
spec/helpers/scenario_helper_spec.rb Normal file
View file

@ -0,0 +1,30 @@
require 'spec_helper'
describe ScenarioHelper do
let(:scenario) { users(:bob).scenarios.build(name: 'Scene', tag_fg_color: '#AAAAAA', tag_bg_color: '#000000') }
describe '#style_colors' do
it 'returns a css style-formated version of the scenario foreground and background colors' do
style_colors(scenario).should == "color:#AAAAAA;background-color:#000000"
end
it 'defauls foreground and background colors' do
scenario.tag_fg_color = nil
scenario.tag_bg_color = nil
style_colors(scenario).should == "color:#FFFFFF;background-color:#5BC0DE"
end
end
describe '#scenario_label' do
it 'creates a scenario label with the scenario name' do
scenario_label(scenario).should ==
'<span class="label scenario" style="color:#AAAAAA;background-color:#000000">Scene</span>'
end
it 'creates a scenario label with the given text' do
scenario_label(scenario, 'Other').should ==
'<span class="label scenario" style="color:#AAAAAA;background-color:#000000">Other</span>'
end
end
end

10
spec/lib/agents_exporter_spec.rb
View file

@ -7,9 +7,13 @@ describe AgentsExporter do
let(:name) { "My set of Agents" }
let(:description) { "These Agents work together nicely!" }
let(:guid) { "some-guid" }
let(:tag_fg_color) { "#ffffff" }
let(:tag_bg_color) { "#000000" }
let(:source_url) { "http://yourhuginn.com/scenarios/2/export.json" }
let(:agent_list) { [agents(:jane_weather_agent), agents(:jane_rain_notifier_agent)] }
let(:exporter) { AgentsExporter.new(:agents => agent_list, :name => name, :description => description, :source_url => source_url, :guid => guid) }
let(:exporter) { AgentsExporter.new(
:agents => agent_list, :name => name, :description => description, :source_url => source_url,
:guid => guid, :tag_fg_color => tag_fg_color, :tag_bg_color => tag_bg_color) }
it "outputs a structure containing name, description, the date, all agents & their links" do
data = exporter.as_json
@ -17,6 +21,8 @@ describe AgentsExporter do
data[:description].should == description
data[:source_url].should == source_url
data[:guid].should == guid
data[:tag_fg_color].should == tag_fg_color
data[:tag_bg_color].should == tag_bg_color
Time.parse(data[:exported_at]).should be_within(2).of(Time.now.utc)
data[:links].should == [{ :source => 0, :receiver => 1 }]
data[:agents].should == agent_list.map { |agent| exporter.agent_as_json(agent) }
@ -58,4 +64,4 @@ describe AgentsExporter do
AgentsExporter.new(:name => ",,").filename.should == "exported-agents.json"
end
end
end
end

9
spec/models/agents/basecamp_agent_spec.rb
View file

@ -6,7 +6,7 @@ describe Agents::BasecampAgent do
before(:each) do
stub_request(:get, /json$/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/basecamp.json")), :status => 200, :headers => {"Content-Type" => "text/json"})
stub_request(:get, /Z$/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/basecamp.json")), :status => 200, :headers => {"Content-Type" => "text/json"})
stub_request(:get, /02:00$/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/basecamp.json")), :status => 200, :headers => {"Content-Type" => "text/json"})
@valid_params = { :project_id => 6789 }
@checker = Agents::BasecampAgent.new(:name => "somename", :options => @valid_params)
@ -43,7 +43,7 @@ describe Agents::BasecampAgent do
it "should provide the since attribute after the first run" do
time = (Time.now-1.minute).iso8601
@checker.memory[:last_run] = time
@checker.memory[:last_event] = time
@checker.save
@checker.reload.send(:query_parameters).should == {:query => {:since => time}}
end
@ -51,9 +51,10 @@ describe Agents::BasecampAgent do
describe "#check" do
it "should not emit events on its first run" do
expect { @checker.check }.to change { Event.count }.by(0)
expect(@checker.memory[:last_event]).to eq '2014-04-17T10:25:31.000+02:00'
end
it "should check that initial run creates an event" do
@checker.last_check_at = Time.now - 1.minute
@checker.memory[:last_event] = '2014-04-17T10:25:31.000+02:00'
expect { @checker.check }.to change { Event.count }.by(1)
end
end
@ -61,7 +62,7 @@ describe Agents::BasecampAgent do
describe "#working?" do
it "it is working when at least one event was emited" do
@checker.should_not be_working
@checker.last_check_at = Time.now - 1.minute
@checker.memory[:last_event] = '2014-04-17T10:25:31.000+02:00'
@checker.check
@checker.reload.should be_working
end

70
spec/models/agents/website_agent_spec.rb
View file

@ -398,19 +398,87 @@ describe Agents::WebsiteAgent do
event.payload['response']['title'].should == "hello!"
end
end
describe "text parsing" do
before do
stub_request(:any, /text-site/).to_return(body: <<-EOF, status: 200)
water: wet
fire: hot
EOF
site = {
'name' => 'Some Text Response',
'expected_update_period_in_days' => '2',
'type' => 'text',
'url' => 'http://text-site.com',
'mode' => 'on_change',
'extract' => {
'word' => { 'regexp' => '^(.+?): (.+)$', index: 1 },
'property' => { 'regexp' => '^(.+?): (.+)$', index: 2 },
}
}
@checker = Agents::WebsiteAgent.new(name: 'Text Site', options: site)
@checker.user = users(:bob)
@checker.save!
end
it "works with regexp" do
@checker.options = @checker.options.merge('extract' => {
'word' => { 'regexp' => '^(?<word>.+?): (?<property>.+)$', index: 'word' },
'property' => { 'regexp' => '^(?<word>.+?): (?<property>.+)$', index: 'property' },
})
lambda {
@checker.check
}.should change { Event.count }.by(2)
event1, event2 = Event.last(2)
event1.payload['word'].should == 'water'
event1.payload['property'].should == 'wet'
event2.payload['word'].should == 'fire'
event2.payload['property'].should == 'hot'
end
it "works with regexp with named capture" do
lambda {
@checker.check
}.should change { Event.count }.by(2)
event1, event2 = Event.last(2)
event1.payload['word'].should == 'water'
event1.payload['property'].should == 'wet'
event2.payload['word'].should == 'fire'
event2.payload['property'].should == 'hot'
end
end
end
describe "#receive" do
it "should scrape from the url element in incoming event payload" do
before do
@event = Event.new
@event.agent = agents(:bob_rain_notifier_agent)
@event.payload = { 'url' => "http://xkcd.com" }
end
it "should scrape from the url element in incoming event payload" do
lambda {
@checker.options = @valid_options
@checker.receive([@event])
}.should change { Event.count }.by(1)
end
it "should interpolate values from incoming event payload" do
@event.payload['title'] = 'XKCD'
lambda {
@valid_options['extract']['site_title'] = {
'css' => "#comic img", 'value' => "'{{title}}'"
}
@checker.options = @valid_options
@checker.receive([@event])
}.should change { Event.count }.by(1)
Event.last.payload['site_title'].should == 'XKCD'
end
end
end

6
spec/models/concerns/oauthable.rb
View file

@ -16,14 +16,14 @@ shared_examples_for Oauthable do
@agent.oauthable?.should == true
end
describe "valid_services" do
describe "valid_services_for" do
it "should return all available services without specifying valid_oauth_providers" do
@agent = Agents::OauthableTestAgent.new
@agent.valid_services(users(:bob)).collect(&:id).sort.should == [services(:generic), services(:global)].collect(&:id).sort
@agent.valid_services_for(users(:bob)).collect(&:id).sort.should == [services(:generic), services(:global)].collect(&:id).sort
end
it "should filter the services based on the agent defaults" do
@agent.valid_services(users(:bob)).to_a.should == Service.where(provider: @agent.valid_oauth_providers)
@agent.valid_services_for(users(:bob)).to_a.should == Service.where(provider: @agent.valid_oauth_providers)
end
end
end

9
spec/models/event_spec.rb
View file

@ -102,6 +102,15 @@ describe EventDrop do
interpolate(t, @event).should eq('some title: http://some.site.example.org/')
end
it 'should use created_at from the payload if it exists' do
created_at = @event.created_at - 86400
# Avoid timezone issue by using %s
@event.payload['created_at'] = created_at.strftime("%s")
@event.save!
t = '{{created_at | date:"%s" }}'
interpolate(t, @event).should eq(created_at.strftime("%s"))
end
it 'should be iteratable' do
# to_liquid returns self
t = "{% for pair in to_liquid %}{{pair | join:':' }}\n{% endfor %}"

14
spec/models/scenario_import_spec.rb
View file

@ -3,6 +3,8 @@ require 'spec_helper'
describe ScenarioImport do
let(:user) { users(:bob) }
let(:guid) { "somescenarioguid" }
let(:tag_fg_color) { "#ffffff" }
let(:tag_bg_color) { "#000000" }
let(:description) { "This is a cool Huginn Scenario that does something useful!" }
let(:name) { "A useful Scenario" }
let(:source_url) { "http://example.com/scenarios/2/export.json" }
@ -58,10 +60,12 @@ describe ScenarioImport do
}
end
let(:valid_parsed_data) do
{
{
:name => name,
:description => description,
:guid => guid,
:tag_fg_color => tag_fg_color,
:tag_bg_color => tag_bg_color,
:source_url => source_url,
:exported_at => 2.days.ago.utc.iso8601,
:agents => [
@ -154,7 +158,7 @@ describe ScenarioImport do
end
end
end
describe "#dangerous?" do
it "returns false on most Agents" do
ScenarioImport.new(:data => valid_data).should_not be_dangerous
@ -183,6 +187,8 @@ describe ScenarioImport do
scenario_import.scenario.name.should == name
scenario_import.scenario.description.should == description
scenario_import.scenario.guid.should == guid
scenario_import.scenario.tag_fg_color.should == tag_fg_color
scenario_import.scenario.tag_bg_color.should == tag_bg_color
scenario_import.scenario.source_url.should == source_url
scenario_import.scenario.public.should be_falsey
end
@ -281,6 +287,8 @@ describe ScenarioImport do
existing_scenario.reload
existing_scenario.guid.should == guid
existing_scenario.tag_fg_color.should == tag_fg_color
existing_scenario.tag_bg_color.should == tag_bg_color
existing_scenario.description.should == description
existing_scenario.name.should == name
existing_scenario.source_url.should == source_url
@ -463,4 +471,4 @@ describe ScenarioImport do
end
end
end
end
end

24
spec/models/scenario_spec.rb
View file

@ -20,6 +20,30 @@ describe Scenario do
new_instance.should_not be_valid
end
it "validates tag_fg_color is hex color" do
new_instance.tag_fg_color = '#N07H3X'
new_instance.should_not be_valid
new_instance.tag_fg_color = '#BADA55'
new_instance.should be_valid
end
it "allows nil tag_fg_color" do
new_instance.tag_fg_color = nil
new_instance.should be_valid
end
it "validates tag_bg_color is hex color" do
new_instance.tag_bg_color = '#N07H3X'
new_instance.should_not be_valid
new_instance.tag_bg_color = '#BADA55'
new_instance.should be_valid
end
it "allows nil tag_bg_color" do
new_instance.tag_bg_color = nil
new_instance.should be_valid
end
it "only allows Agents owned by user" do
new_instance.agent_ids = [agents(:bob_website_agent).id]
new_instance.should be_valid

38
spec/models/service_spec.rb
View file

@ -5,13 +5,32 @@ describe Service do
@user = users(:bob)
end
it "should toggle the global flag" do
@service = services(:generic)
@service.global.should == false
@service.toggle_availability!
@service.global.should == true
@service.toggle_availability!
@service.global.should == false
describe "#toggle_availability!" do
it "should toggle the global flag" do
@service = services(:generic)
@service.global.should == false
@service.toggle_availability!
@service.global.should == true
@service.toggle_availability!
@service.global.should == false
end
it "disconnects agents and disables them if the previously global service is made private again", focus: true do
agent = agents(:bob_basecamp_agent)
jane_agent = agents(:jane_basecamp_agent)
service = agent.service
service.toggle_availability!
service.agents.length.should == 2
service.toggle_availability!
jane_agent.reload
jane_agent.service_id.should be_nil
jane_agent.disabled.should be true
service.reload
service.agents.length.should == 1
end
end
it "disables all agents before beeing destroyed" do
@ -20,7 +39,7 @@ describe Service do
service.destroy
agent.reload
agent.service_id.should be_nil
agent.disabled.should be_true
agent.disabled.should be true
end
describe "preparing for a request" do
@ -74,6 +93,7 @@ describe Service do
}.to change { @user.services.count }.by(1)
service = @user.services.first
service.name.should == 'johnqpublic'
service.uid.should == '123456'
service.provider.should == 'twitter'
service.token.should == 'a1b2c3d4...'
service.secret.should == 'abcdef1234'
@ -88,6 +108,7 @@ describe Service do
service.provider.should == '37signals'
service.name.should == 'Dominik Sander'
service.token.should == 'abcde'
service.uid.should == '12345'
service.refresh_token.should == 'fghrefresh'
service.options[:user_id].should == 12345
service.expires_at = Time.at(1401554352)
@ -101,6 +122,7 @@ describe Service do
service = @user.services.first
service.provider.should == 'github'
service.name.should == 'dsander'
service.uid.should == '12345'
service.token.should == 'agithubtoken'
end
end

2
spec/spec_helper.rb
View file

@ -10,7 +10,7 @@ end
# Required ENV variables that are normally set in .env are setup here for the test environment.
require 'dotenv'
Dotenv.load File.join(File.dirname(__FILE__), "env.test")
Dotenv.overload File.join(File.dirname(__FILE__), "env.test")
require File.expand_path("../../config/environment", __FILE__)
require 'rspec/rails'