Merge branch 'master' into threaded-background-workers

This commit is contained in:
Dominik Sander 2014-06-01 12:12:42 +02:00
commit a505a1c211
48 changed files with 1641 additions and 262 deletions

View file

@ -31,6 +31,7 @@ gem 'json', '~> 1.8.1'
gem 'jsonpath', '~> 0.5.3'
gem 'twilio-ruby', '~> 3.11.5'
gem 'ruby-growl', '~> 4.1.0'
gem 'liquid', '~> 2.6.1'
gem 'delayed_job', '~> 4.0.0'
gem 'delayed_job_active_record', '~> 4.0.0'

View file

@ -148,6 +148,7 @@ GEM
activesupport (>= 3.0.0)
kramdown (1.3.3)
libv8 (3.16.14.3)
liquid (2.6.1)
macaddr (1.7.1)
systemu (~> 2.6.2)
mail (2.5.4)
@ -338,6 +339,7 @@ DEPENDENCIES
jsonpath (~> 0.5.3)
kaminari (~> 0.15.1)
kramdown (~> 1.3.3)
liquid (~> 2.6.1)
mysql2 (~> 0.3.15)
nokogiri (~> 1.6.1)
protected_attributes (~> 1.0.7)

View file

@ -127,7 +127,7 @@ $(document).ready ->
if tab = window.location.href.match(/tab=(\w+)\b/i)?[1]
if tab in ["details", "logs"]
$(".agent-show .nav-tabs li a[href='##{tab}']").click()
$(".agent-show .nav-pills li a[href='##{tab}']").click()
# Editing Agents
$("#agent_source_ids").on "change", showEventDescriptions

View file

@ -136,10 +136,8 @@ span.not-applicable:after {
// Disabled
tr.agent-disabled {
td {
opacity: 0.5;
}
.agent-disabled {
opacity: 0.5;
}
// Fix JSON Editor
@ -147,3 +145,16 @@ tr.agent-disabled {
.json-editor blockquote {
font-size: 14px;
}
// Bootstrappy colour styles
.color-danger {
color: #d9534f;
}
.color-warning {
color: #f0ad4e;
}
.color-success {
color: #5cb85c;
}

View file

@ -1,39 +0,0 @@
module JsonPathOptionsOverwritable
extend ActiveSupport::Concern
# Using this concern allows providing optional `<attribute>_path` options hash
# attributes which will then (if not blank) be interpolated using the provided JSONPath.
#
# Example options Hash:
# {
# name: 'Huginn',
# name_path: '$.name',
# title: 'Hello from Huginn'
# title_path: ''
# }
# Example event payload:
# {
# name: 'dynamic huginn'
# }
# calling agent.merge_json_path_options(event) returns the following hash:
# {
# name: 'dynamic huginn'
# title: 'Hello from Huginn'
# }
private
def merge_json_path_options(event)
options.select { |k, v| options_with_path.include? k}.tap do |merged_options|
options_with_path.each do |a|
merged_options[a] = select_option(event, a)
end
end
end
def select_option(event, a)
if options[a.to_s + '_path'].present?
Utils.value_at(event.payload, options[a.to_s + '_path'])
else
options[a]
end
end
end

View file

@ -0,0 +1,49 @@
module LiquidInterpolatable
extend ActiveSupport::Concern
def interpolate_options(options, payload)
case options.class.to_s
when 'String'
interpolate_string(options, payload)
when 'ActiveSupport::HashWithIndifferentAccess', 'Hash'
duped_options = options.dup
duped_options.each do |key, value|
duped_options[key] = interpolate_options(value, payload)
end
when 'Array'
options.collect do |value|
interpolate_options(value, payload)
end
end
end
def interpolate_string(string, payload)
Liquid::Template.parse(string).render!(payload, registers: {agent: self})
end
require 'uri'
# Percent encoding for URI conforming to RFC 3986.
# Ref: http://tools.ietf.org/html/rfc3986#page-12
module Filters
def uri_escape(string)
CGI::escape string
end
end
Liquid::Template.register_filter(LiquidInterpolatable::Filters)
module Tags
class Credential < Liquid::Tag
def initialize(tag_name, name, tokens)
super
@credential_name = name.strip
end
def render(context)
credential = context.registers[:agent].credential(@credential_name)
raise "No user credential named '#{@credential_name}' defined" if credential.nil?
credential
end
end
end
Liquid::Template.register_tag('credential', LiquidInterpolatable::Tags::Credential)
end

View file

@ -7,7 +7,10 @@ class ApplicationController < ActionController::Base
helper :all
protected
def configure_permitted_parameters
devise_parameter_sanitizer.for(:sign_up) << [:username, :email, :invitation_code]
devise_parameter_sanitizer.for(:sign_up) { |u| u.permit(:username, :email, :password, :password_confirmation, :remember_me, :invitation_code) }
devise_parameter_sanitizer.for(:sign_in) { |u| u.permit(:login, :username, :email, :password, :remember_me) }
devise_parameter_sanitizer.for(:account_update) { |u| u.permit(:username, :email, :password, :password_confirmation, :current_password) }
end
end

View file

@ -9,11 +9,11 @@ module ApplicationHelper
def working(agent)
if agent.disabled?
'<span class="label label-warning">Disabled</span>'.html_safe
link_to 'Disabled', agent_path(agent), :class => 'label label-warning'
elsif agent.working?
'<span class="label label-success">Yes</span>'.html_safe
else
link_to '<span class="label btn-danger">No</span>'.html_safe, agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details'))
link_to 'No', agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details')), :class => 'label label-danger'
end
end
end

View file

@ -121,10 +121,6 @@ class Agent < ActiveRecord::Base
end
end
def make_message(payload, message = options[:message])
message.gsub(/<([^>]+)>/) { Utils.value_at(payload, $1) || "??" }
end
def trigger_web_request(params, method, format)
if respond_to?(:receive_webhook)
Rails.logger.warn "DEPRECATED: The .receive_webhook method is deprecated, please switch your Agent to use .receive_web_request."

View file

@ -1,5 +1,7 @@
module Agents
class DataOutputAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
description do
@ -19,7 +21,7 @@ module Agents
* `secrets` - An array of tokens that the requestor must provide for light-weight authentication.
* `expected_receive_period_in_days` - How often you expect data to be received by this Agent from other Agents.
* `template` - A JSON object representing a mapping between item output keys and incoming event JSONPath values. JSONPath values must start with `$`, or can be interpolated between `<` and `>` characters. The `item` key will be repeated for every Event.
* `template` - A JSON object representing a mapping between item output keys and incoming event values. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the values. The `item` key will be repeated for every Event.
MD
end
@ -31,9 +33,9 @@ module Agents
"title" => "XKCD comics as a feed",
"description" => "This is a feed of recent XKCD comics, generated by Huginn",
"item" => {
"title" => "$.title",
"description" => "Secret hovertext: <$.hovertext>",
"link" => "$.url",
"title" => "{{title}}",
"description" => "Secret hovertext: {{hovertext}}",
"link" => "{{url}}",
}
}
}
@ -82,7 +84,7 @@ module Agents
def receive_web_request(params, method, format)
if options['secrets'].include?(params['secret'])
items = received_events.order('id desc').limit(events_to_show).map do |event|
interpolated = Utils.recursively_interpolate_jsonpaths(options['template']['item'], event.payload, :leading_dollarsign_is_jsonpath => true)
interpolated = interpolate_options(options['template']['item'], event.payload)
interpolated['guid'] = event.id
interpolated['pubDate'] = event.created_at.rfc2822.to_s
interpolated

View file

@ -1,5 +1,6 @@
module Agents
class EventFormattingAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
description <<-MD
@ -24,11 +25,11 @@ module Agents
You can use an Event Formatting Agent's `instructions` setting to do this in the following way:
"instructions": {
"message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.",
"subject": "$.data"
"message": "Today's conditions look like {{conditions}} with a high temperature of {{high.celsius}} degrees Celsius.",
"subject": "{{data}}"
}
JSONPaths must be between < and > . Make sure that you don't use these symbols anywhere else.
Have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to learn more about liquid templating.
Events generated by this possible Event Formatting Agent will look like:
@ -42,7 +43,7 @@ module Agents
{
"matchers": [
{
"path": "$.date.pretty",
"path": "{{date.pretty}}",
"regexp": "\\A(?<time>\\d\\d:\\d\\d [AP]M [A-Z]+)",
"to": "pretty_date",
}
@ -60,18 +61,18 @@ module Agents
So you can use it in `instructions` like this:
"instructions": {
"message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius according to the forecast at <$.pretty_date.time>.",
"subject": "$.data"
"message": "Today's conditions look like <$.conditions> with a high temperature of {{high.celsius}} degrees Celsius according to the forecast at {{pretty_date.time}}.",
"subject": "{{data}}"
}
If you want to retain original contents of events and only add new keys, then set `mode` to `merge`, otherwise set it to `clean`.
By default, the output event will have `agent` and `created_at` fields added as well, reflecting the original Agent type and Event creation time. You can skip these outputs by setting `skip_agent` and `skip_created_at` to `true`.
To CGI escape output (for example when creating a link), prefix with `escape`, like so:
To CGI escape output (for example when creating a link), use the Liquid `uri_escape` filter, like so:
{
"message": "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=<escape $.group_by>"
"message": "A peak was on Twitter in {{group_by}}. Search: https://twitter.com/search?q={{group_by | uri_escape}}"
}
MD
@ -88,8 +89,8 @@ module Agents
def default_options
{
'instructions' => {
'message' => "You received a text <$.text> from <$.fields.from>",
'some_other_field' => "Looks like the weather is going to be <$.fields.weather>"
'message' => "You received a text {{text}} from {{fields.from}}",
'some_other_field' => "Looks like the weather is going to be {{fields.weather}}"
},
'matchers' => [],
'mode' => "clean",
@ -106,7 +107,7 @@ module Agents
incoming_events.each do |event|
formatted_event = options['mode'].to_s == "merge" ? event.payload.dup : {}
payload = perform_matching(event.payload)
options['instructions'].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, payload) }
formatted_event.merge! interpolate_options(options['instructions'], payload)
formatted_event['agent'] = Agent.find(event.agent_id).type.slice!(8..-1) unless options['skip_agent'].to_s == "true"
formatted_event['created_at'] = event.created_at unless options['skip_created_at'].to_s == "true"
create_event :payload => formatted_event
@ -161,7 +162,7 @@ module Agents
re = Regexp.new(regexp)
proc { |hash|
mhash = {}
value = Utils.value_at(hash, path)
value = interpolate_string(path, hash)
if value.is_a?(String) && (m = re.match(value))
m.to_a.each_with_index { |s, i|
mhash[i.to_s] = s

View file

@ -1,6 +1,6 @@
module Agents
class HipchatAgent < Agent
include JsonPathOptionsOverwritable
include LiquidInterpolatable
cannot_be_scheduled!
cannot_create_events!
@ -18,22 +18,17 @@ module Agents
If you want your message to notify the room members change `notify` to "true".
Modify the background color of your message via the `color` attribute (one of "yellow", "red", "green", "purple", "gray", or "random")
If you want to specify either of those attributes per event, you can provide a [JSONPath](http://goessner.net/articles/JsonPath/) for each of them (except the `auth_token`).
Have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to learn more about liquid templating.
MD
def default_options
{
'auth_token' => '',
'room_name' => '',
'room_name_path' => '',
'username' => "Huginn",
'username_path' => '',
'message' => "Hello from Huginn!",
'message_path' => '',
'notify' => false,
'notify_path' => '',
'color' => 'yellow',
'color_path' => '',
}
end
@ -49,14 +44,9 @@ module Agents
def receive(incoming_events)
client = HipChat::Client.new(options[:auth_token])
incoming_events.each do |event|
mo = merge_json_path_options event
mo = interpolate_options options, event.payload
client[mo[:room_name]].send(mo[:username], mo[:message], :notify => mo[:notify].to_s == 'true' ? 1 : 0, :color => mo[:color])
end
end
private
def options_with_path
[:room_name, :username, :message, :notify, :color]
end
end
end

View file

@ -2,6 +2,8 @@ require 'rturk'
module Agents
class HumanTaskAgent < Agent
include LiquidInterpolatable
default_schedule "every_10m"
description <<-MD
@ -16,7 +18,7 @@ module Agents
# Example
If created with an event, all HIT fields can contain interpolated values via [JSONPaths](http://goessner.net/articles/JsonPath/) placed between < and > characters.
If created with an event, all HIT fields can contain interpolated values via [liquid templating](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid).
For example, if the incoming event was a Twitter event, you could make a HITT to rate its sentiment like this:
{
@ -25,7 +27,7 @@ module Agents
"hit": {
"assignments": 1,
"title": "Sentiment evaluation",
"description": "Please rate the sentiment of this message: '<$.message>'",
"description": "Please rate the sentiment of this message: '{{message}}'",
"reward": 0.05,
"lifetime_in_seconds": "3600",
"questions": [
@ -83,7 +85,7 @@ module Agents
"title": "Take a poll about some jokes",
"instructions": "Please rank these jokes from most funny (5) to least funny (1)",
"assignments": 3,
"row_template": "<$.joke>"
"row_template": "{{joke}}"
},
"hit": {
"assignments": 5,
@ -168,7 +170,7 @@ module Agents
{
'assignments' => 1,
'title' => "Sentiment evaluation",
'description' => "Please rate the sentiment of this message: '<$.message>'",
'description' => "Please rate the sentiment of this message: '{{message}}'",
'reward' => 0.05,
'lifetime_in_seconds' => 24 * 60 * 60,
'questions' =>
@ -332,7 +334,7 @@ module Agents
'name' => "Item #{index + 1}",
'key' => index,
'required' => "true",
'question' => Utils.interpolate_jsonpaths(options['poll_options']['row_template'], assignments[index].answers),
'question' => interpolate_string(options['poll_options']['row_template'], assignments[index].answers),
'selections' => selections
}
end
@ -387,9 +389,9 @@ module Agents
def create_hit(opts = {})
payload = opts['payload'] || {}
title = Utils.interpolate_jsonpaths(opts['title'], payload).strip
description = Utils.interpolate_jsonpaths(opts['description'], payload).strip
questions = Utils.recursively_interpolate_jsonpaths(opts['questions'], payload)
title = interpolate_string(opts['title'], payload).strip
description = interpolate_string(opts['description'], payload).strip
questions = interpolate_options(opts['questions'], payload)
hit = RTurk::Hit.create(:title => title) do |hit|
hit.max_assignments = (opts['assignments'] || 1).to_i
hit.description = description

View file

@ -0,0 +1,453 @@
require 'delegate'
require 'net/imap'
require 'mail'
module Agents
class ImapFolderAgent < Agent
cannot_receive_events!
default_schedule "every_30m"
description <<-MD
The ImapFolderAgent checks an IMAP server in specified folders
and creates Events based on new unread mails.
Specify an IMAP server to connect with `host`, and set `ssl` to
true if the server supports IMAP over SSL. Specify `port` if
you need to connect to a port other than standard (143 or 993
depending on the `ssl` value).
Specify login credentials in `username` and `password`.
List the names of folders to check in `folders`.
To narrow mails by conditions, build a `conditions` hash with
the following keys:
- "subject"
- "body"
Specify a regular expression to match against the decoded
subject/body of each mail.
Use the `(?i)` directive for case-insensitive search. For
example, a pattern `(?i)alert` will match "alert", "Alert"
or "ALERT". You can also make only a part of a pattern to
work case-insensitively: `Re: (?i:alert)` will match either
"Re: Alert" or "Re: alert", but not "RE: alert".
When a mail has multiple non-attachment text parts, they are
prioritized according to the `mime_types` option (which see
below) and the first part that matches a "body" pattern, if
specified, will be chosen as the "body" value in a created
event.
Named captues will appear in the "matches" hash in a created
event.
- "from", "to", "cc"
Specify a shell glob pattern string that is matched against
mail addresses extracted from the corresponding header
values of each mail.
Patterns match addresses in case insensitive manner.
Multiple pattern strings can be specified in an array, in
which case a mail is selected if any of the patterns
matches. (i.e. patterns are OR'd)
- "mime_types"
Specify an array of MIME types to tell which non-attachment
part of a mail among its text/* parts should be used as mail
body. The default value is `['text/plain', 'text/enriched',
'text/html']`.
- "has_attachment"
Setting this to true or false means only mails that does or does
not have an attachment are selected.
If this key is unspecified or set to null, it is ignored.
Set `mark_as_read` to true to mark found mails as read.
Each agent instance memorizes a list of unread mails that are
found in the last run, so even if you change a set of conditions
so that it matches mails that are missed previously, they will
not show up as new events. Also, in order to avoid duplicated
notification it keeps a list of Message-Id's of 100 most recent
mails, so if multiple mails of the same Message-Id are found,
you will only see one event out of them.
MD
event_description <<-MD
Events look like this:
{
"folder": "INBOX",
"subject": "...",
"from": "Nanashi <nanashi.gombeh@example.jp>",
"to": ["Jane <jane.doe@example.com>"],
"cc": [],
"date": "2014-05-10T03:47:20+0900",
"mime_type": "text/plain",
"body": "Hello,\n\n...",
"matches": {
}
}
MD
IDCACHE_SIZE = 100
FNM_FLAGS = [:FNM_CASEFOLD, :FNM_EXTGLOB].inject(0) { |flags, sym|
if File.const_defined?(sym)
flags | File.const_get(sym)
else
flags
end
}
def working?
event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
end
def default_options
{
'expected_update_period_in_days' => "1",
'host' => 'imap.gmail.com',
'ssl' => true,
'username' => 'your.account',
'password' => 'your.password',
'folders' => %w[INBOX],
'conditions' => {}
}
end
def validate_options
%w[host username password].each { |key|
String === options[key] or
errors.add(:base, '%s is required and must be a string' % key)
}
if options['port'].present?
errors.add(:base, "port must be a positive integer") unless is_positive_integer?(options['port'])
end
%w[ssl mark_as_read].each { |key|
if options[key].present?
case options[key]
when true, false
else
errors.add(:base, '%s must be a boolean value' % key)
end
end
}
case mime_types = options['mime_types']
when nil
when Array
mime_types.all? { |mime_type|
String === mime_type && mime_type.start_with?('text/')
} or errors.add(:base, 'mime_types may only contain strings that match "text/*".')
if mime_types.empty?
errors.add(:base, 'mime_types should not be empty')
end
else
errors.add(:base, 'mime_types must be an array')
end
case folders = options['folders']
when nil
when Array
folders.all? { |folder|
String === folder
} or errors.add(:base, 'folders may only contain strings')
if folders.empty?
errors.add(:base, 'folders should not be empty')
end
else
errors.add(:base, 'folders must be an array')
end
case conditions = options['conditions']
when nil
when Hash
conditions.each { |key, value|
value.present? or next
case key
when 'subject', 'body'
case value
when String
begin
Regexp.new(value)
rescue
errors.add(:base, 'conditions.%s contains an invalid regexp' % key)
end
else
errors.add(:base, 'conditions.%s contains a non-string object' % key)
end
when 'from', 'to', 'cc'
Array(value).each { |pattern|
case pattern
when String
begin
glob_match?(pattern, '')
rescue
errors.add(:base, 'conditions.%s contains an invalid glob pattern' % key)
end
else
errors.add(:base, 'conditions.%s contains a non-string object' % key)
end
}
when 'has_attachment'
case value
when true, false
else
errors.add(:base, 'conditions.%s must be a boolean value or null' % key)
end
end
}
else
errors.add(:base, 'conditions must be a hash')
end
if options['expected_update_period_in_days'].present?
errors.add(:base, "Invalid expected_update_period_in_days format") unless is_positive_integer?(options['expected_update_period_in_days'])
end
end
def check
# 'seen' keeps a hash of { uidvalidity => uids, ... } which
# lists unread mails in watched folders.
seen = memory['seen'] || {}
new_seen = Hash.new { |hash, key|
hash[key] = []
}
# 'notified' keeps an array of message-ids of {IDCACHE_SIZE}
# most recent notified mails.
notified = memory['notified'] || []
each_unread_mail { |mail|
new_seen[mail.uidvalidity] << mail.uid
next if (uids = seen[mail.uidvalidity]) && uids.include?(mail.uid)
body_parts = mail.body_parts(mime_types)
matched_part = nil
matches = {}
options['conditions'].all? { |key, value|
case key
when 'subject'
value.present? or next true
re = Regexp.new(value)
if m = re.match(mail.subject)
m.names.each { |name|
matches[name] = m[name]
}
true
else
false
end
when 'body'
value.present? or next true
re = Regexp.new(value)
matched_part = body_parts.find { |part|
if m = re.match(part.decoded)
m.names.each { |name|
matches[name] = m[name]
}
true
else
false
end
}
when 'from', 'to', 'cc'
value.present? or next true
mail.header[key].addresses.any? { |address|
Array(value).any? { |pattern|
glob_match?(pattern, address)
}
}
when 'has_attachment'
value == mail.has_attachment?
else
log 'Unknown condition key ignored: %s' % key
true
end
} or next
unless notified.include?(mail.message_id)
matched_part ||= body_parts.first
if matched_part
mime_type = matched_part.mime_type
body = matched_part.decoded
else
mime_type = 'text/plain'
body = ''
end
create_event :payload => {
'folder' => mail.folder,
'subject' => mail.subject,
'from' => mail.from_addrs.first,
'to' => mail.to_addrs,
'cc' => mail.cc_addrs,
'date' => (mail.date.iso8601 rescue nil),
'mime_type' => mime_type,
'body' => body,
'matches' => matches,
'has_attachment' => mail.has_attachment?,
}
notified << mail.message_id if mail.message_id
end
if options['mark_as_read']
log 'Marking as read'
mail.mark_as_read
end
}
notified.slice!(0...-IDCACHE_SIZE) if notified.size > IDCACHE_SIZE
memory['seen'] = new_seen
memory['notified'] = notified
save!
end
def each_unread_mail
host, port, ssl, username = options.values_at(:host, :port, :ssl, :username)
log "Connecting to #{host}#{':%d' % port if port}#{' via SSL' if ssl}"
Client.open(host, Integer(port), ssl) { |imap|
log "Logging in as #{username}"
imap.login(username, options[:password])
options['folders'].each { |folder|
log "Selecting the folder: %s" % folder
imap.select(folder)
unseen = imap.search('UNSEEN')
if unseen.empty?
log "No unread mails"
next
end
imap.fetch_mails(unseen).each { |mail|
yield mail
}
}
}
ensure
log 'Connection closed'
end
def mime_types
options['mime_types'] || %w[text/plain text/enriched text/html]
end
private
def is_positive_integer?(value)
Integer(value) >= 0
rescue
false
end
def glob_match?(pattern, value)
File.fnmatch?(pattern, value, FNM_FLAGS)
end
class Client < ::Net::IMAP
class << self
def open(host, port, ssl)
imap = new(host, port, ssl)
yield imap
ensure
imap.disconnect unless imap.nil?
end
end
def select(folder)
ret = super(@folder = folder)
@uidvalidity = responses['UIDVALIDITY'].last
ret
end
def fetch_mails(set)
fetch(set, %w[UID RFC822.HEADER]).map { |data|
Message.new(self, data, folder: @folder, uidvalidity: @uidvalidity)
}
end
end
class Message < SimpleDelegator
DEFAULT_BODY_MIME_TYPES = %w[text/plain text/enriched text/html]
attr_reader :uid, :folder, :uidvalidity
def initialize(client, fetch_data, props = {})
@client = client
props.each { |key, value|
instance_variable_set(:"@#{key}", value)
}
attr = fetch_data.attr
@uid = attr['UID']
super(Mail.read_from_string(attr['RFC822.HEADER']))
end
def has_attachment?
@has_attachment ||=
begin
data = @client.uid_fetch(@uid, 'BODYSTRUCTURE').first
struct_has_attachment?(data.attr['BODYSTRUCTURE'])
end
end
def fetch
@parsed ||=
begin
data = @client.uid_fetch(@uid, 'BODY.PEEK[]').first
Mail.read_from_string(data.attr['BODY[]'])
end
end
def body_parts(mime_types = DEFAULT_BODY_MIME_TYPES)
mail = fetch
if mail.multipart?
mail.body.set_sort_order(mime_types)
mail.body.sort_parts!
mail.all_parts
else
[mail]
end.reject { |part|
part.multipart? || part.attachment? || !part.text? ||
!mime_types.include?(part.mime_type)
}
end
def mark_as_read
@client.uid_store(@uid, '+FLAGS', [:Seen])
end
private
def struct_has_attachment?(struct)
struct.multipart? && (
struct.subtype == 'MIXED' ||
struct.parts.any? { |part|
struct_has_attachment?(part)
}
)
end
end
end
end

View file

@ -1,5 +1,7 @@
module Agents
class JabberAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
cannot_create_events!
@ -10,7 +12,9 @@ module Agents
The `message` is sent from `jabber_sender` to `jaber_receiver`. This message
can contain any keys found in the source's payload, escaped using double curly braces.
ex: `"News Story: <$.title>: <$.url>"`
ex: `"News Story: {{title}}: {{url}}"`
Have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to learn more about liquid templating.
MD
def default_options
@ -20,7 +24,7 @@ module Agents
'jabber_sender' => 'huginn@localhost',
'jabber_receiver' => 'muninn@localhost',
'jabber_password' => '',
'message' => 'It will be <$.temp> out tomorrow',
'message' => 'It will be {{temp}} out tomorrow',
'expected_receive_period_in_days' => "2"
}
end
@ -58,7 +62,7 @@ module Agents
end
def body(event)
Utils.interpolate_jsonpaths(options['message'], event.payload)
interpolate_string(options['message'], event.payload)
end
end
end

View file

@ -0,0 +1,160 @@
#!/usr/bin/env ruby
require 'cgi'
require 'httparty'
require 'date'
module Agents
class JiraAgent < Agent
cannot_receive_events!
description <<-MD
The Jira Agent subscribes to Jira issue updates.
`jira_url` specifies the full URL of the jira installation, including https://
`jql` is an optional Jira Query Language-based filter to limit the flow of events. See [JQL Docs](https://confluence.atlassian.com/display/JIRA/Advanced+Searching) for details.
`username` and `password` are optional, and may need to be specified if your Jira instance is read-protected
`timeout` is an optional parameter that specifies how long the request processing may take in minutes.
The agent does periodic queries and emits the events containing the updated issues in JSON format.
NOTE: upon the first execution, the agent will fetch everything available by the JQL query. So if it's not desirable, limit the `jql` query by date.
MD
event_description <<-MD
Events are the raw JSON generated by Jira REST API
{
"expand": "editmeta,renderedFields,transitions,changelog,operations",
"id": "80127",
"self": "https://jira.atlassian.com/rest/api/2/issue/80127",
"key": "BAM-3512",
"fields": {
...
}
}
MD
default_schedule "every_10m"
MAX_EMPTY_REQUESTS = 10
def default_options
{
'username' => '',
'password' => '',
'jira_url' => 'https://jira.atlassian.com',
'jql' => '',
'expected_update_period_in_days' => '7',
'timeout' => '1'
}
end
def validate_options
errors.add(:base, "you need to specify password if user name is set") if options['username'].present? and not options['password'].present?
errors.add(:base, "you need to specify your jira URL") unless options['jira_url'].present?
errors.add(:base, "you need to specify the expected update period") unless options['expected_update_period_in_days'].present?
errors.add(:base, "you need to specify request timeout") unless options['timeout'].present?
end
def working?
event_created_within?(options['expected_update_period_in_days']) && !recent_error_logs?
end
def check
last_run = nil
current_run = Time.now.utc.iso8601
last_run = Time.parse(memory[:last_run]) if memory[:last_run]
issues = get_issues(last_run)
issues.each do |issue|
updated = Time.parse(issue['fields']['updated'])
# this check is more precise than in get_issues()
# see get_issues() for explanation
if not last_run or updated > last_run
create_event :payload => issue
end
end
memory[:last_run] = current_run
end
private
def request_url(jql, start_at)
"#{options[:jira_url]}/rest/api/2/search?jql=#{CGI::escape(jql)}&fields=*all&startAt=#{start_at}"
end
def request_options
ropts = {:headers => {"User-Agent" => "Huginn (https://github.com/cantino/huginn)"}}
if !options[:username].empty?
ropts = ropts.merge({:basic_auth => {:username =>options[:username], :password=>options[:password]}})
end
ropts
end
def get(url, options)
response = HTTParty.get(url, options)
if response.code == 400
raise RuntimeError.new("Jira error: #{response['errorMessages']}")
elsif response.code == 403
raise RuntimeError.new("Authentication failed: Forbidden (403)")
elsif response.code != 200
raise RuntimeError.new("Request failed: #{response}")
end
response
end
def get_issues(since)
startAt = 0
issues = []
# JQL doesn't have an ability to specify timezones
# Because of this we have to fetch issues 24 h
# earlier and filter out unnecessary ones at a later
# stage. Fortunately, the 'updated' field has GMT
# offset
since -= 24*60*60 if since
jql = ""
if !options[:jql].empty? && since
jql = "(#{options[:jql]}) and updated >= '#{since.strftime('%Y-%m-%d %H:%M')}'"
else
jql = options[:jql] if !options[:jql].empty?
jql = "updated >= '#{since.strftime('%Y-%m-%d %H:%M')}'" if since
end
start_time = Time.now
request_limit = 0
loop do
response = get(request_url(jql, startAt), request_options)
if response['issues'].length == 0
request_limit+=1
end
if request_limit > MAX_EMPTY_REQUESTS
raise RuntimeError.new("There is no progress while fetching issues")
end
if Time.now > start_time + options['timeout'].to_i * 60
raise RuntimeError.new("Timeout exceeded while fetching issues")
end
issues += response['issues']
startAt += response['issues'].length
break if startAt >= response['total']
end
issues
end
end
end

View file

@ -2,10 +2,12 @@ require 'pp'
module Agents
class PeakDetectorAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
description <<-MD
Use a PeakDetectorAgent to watch for peaks in an event stream. When a peak is detected, the resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
Use a PeakDetectorAgent to watch for peaks in an event stream. When a peak is detected, the resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: {{foo.bar}}`, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details.
The `value_path` value is a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value of interest. `group_by_path` is a hash path that will be used to group values, if present.
@ -67,7 +69,7 @@ module Agents
if newest_value > average_value + std_multiple * standard_deviation
memory['peaks'][group] << newest_time
memory['peaks'][group].reject! { |p| p <= newest_time - window_duration }
create_event :payload => { 'message' => options['message'], 'peak' => newest_value, 'peak_time' => newest_time, 'grouped_by' => group.to_s }
create_event :payload => { 'message' => interpolate_string(options['message'], event.payload), 'peak' => newest_value, 'peak_time' => newest_time, 'grouped_by' => group.to_s }
end
end
end

View file

@ -1,6 +1,6 @@
module Agents
class PushbulletAgent < Agent
include JsonPathOptionsOverwritable
include LiquidInterpolatable
cannot_be_scheduled!
cannot_create_events!
@ -20,7 +20,7 @@ module Agents
You can provide a `title` and a `body`.
If you want to specify `title` or `body` per event, you can provide a [JSONPath](http://goessner.net/articles/JsonPath/) for each of them.
In every value of the options hash you can use the liquid templating, learn more about it at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid).
MD
def default_options
@ -28,9 +28,7 @@ module Agents
'api_key' => '',
'device_id' => '',
'title' => "Hello from Huginn!",
'title_path' => '',
'body' => '',
'body_path' => '',
'body' => '{{body}}',
}
end
@ -52,16 +50,11 @@ module Agents
private
def query_options(event)
mo = merge_json_path_options event
basic_options.deep_merge(:body => {:title => mo[:title], :body => mo[:body]})
end
def basic_options
{:basic_auth => {:username =>options[:api_key], :password=>''}, :body => {:device_iden => options[:device_id], :type => 'note'}}
end
def options_with_path
[:title, :body]
mo = interpolate_options options, event.payload
{
:basic_auth => {:username =>mo[:api_key], :password=>''},
:body => {:device_iden => mo[:device_id], :title => mo[:title], :body => mo[:body], :type => 'note'}
}
end
end
end

View file

@ -1,5 +1,6 @@
module Agents
class TranslationAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
@ -8,7 +9,7 @@ module Agents
Services are provided using Microsoft Translator. You can [sign up](https://datamarket.azure.com/dataset/bing/microsofttranslator) and [register your application](https://datamarket.azure.com/developer/applications/register) to get `client_id` and `client_secret` which are required to use this agent.
`to` must be filled with a [translator language code](http://msdn.microsoft.com/en-us/library/hh456380.aspx).
Specify what you would like to translate in `content` field, by specifying key and JSONPath of content to be translated.
Specify what you would like to translate in `content` field, you can use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) specify which part of the payload you want to translate.
`expected_receive_period_in_days` is the maximum number of days you would allow to pass between events.
MD
@ -22,8 +23,8 @@ module Agents
'to' => "fi",
'expected_receive_period_in_days' => 1,
'content' => {
'text' => "$.message.text",
'content' => "$.xyz"
'text' => "{{message.text}}",
'content' => "{{xyz}}"
}
}
end
@ -68,8 +69,8 @@ module Agents
incoming_events.each do |event|
translated_event = {}
options['content'].each_pair do |key, value|
to_be_translated = Utils.values_at event.payload, value
translated_event[key] = translate to_be_translated.first, options['to'], access_token
to_be_translated = interpolate_string(value, event.payload)
translated_event[key] = translate(to_be_translated.first, options['to'], access_token)
end
create_event :payload => translated_event
end

View file

@ -1,5 +1,7 @@
module Agents
class TriggerAgent < Agent
include LiquidInterpolatable
cannot_be_scheduled!
VALID_COMPARISON_TYPES = %w[regex !regex field<value field<=value field==value field!=value field>=value field>value]
@ -13,7 +15,7 @@ module Agents
The `value` can be a single value or an array of values. In the case of an array, if one or more values match then the rule matches.
All rules must match for the Agent to match. The resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: <foo.bar>`
All rules must match for the Agent to match. The resulting Event will have a payload message of `message`. You can use liquid templating in the `message, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details.
Set `keep_event` to `true` if you'd like to re-emit the incoming event, optionally merged with 'message' when provided.
@ -46,7 +48,7 @@ module Agents
'value' => "foo\\d+bar",
'path' => "topkey.subkey.subkey.goal",
}],
'message' => "Looks like your pattern matched in '<value>'!"
'message' => "Looks like your pattern matched in '{{value}}'!"
}
end
@ -88,9 +90,9 @@ module Agents
if match
if keep_event?
payload = event.payload.dup
payload['message'] = make_message(event[:payload]) if options['message'].present?
payload['message'] = interpolate_string(options['message'], event.payload) if options['message'].present?
else
payload = { 'message' => make_message(event[:payload]) }
payload = { 'message' => interpolate_string(options['message'], event.payload) }
end
create_event :payload => payload

View file

@ -3,6 +3,7 @@ require "twitter"
module Agents
class TwitterPublishAgent < Agent
include TwitterConcern
include LiquidInterpolatable
cannot_be_scheduled!
@ -15,7 +16,7 @@ module Agents
To get oAuth credentials for Twitter, [follow these instructions](https://github.com/cantino/huginn/wiki/Getting-a-twitter-oauth-token).
You must also specify a `message_path` parameter: a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value to tweet.
You must also specify a `message` parameter, you can use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the message.
Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent.
MD
@ -31,7 +32,7 @@ module Agents
def default_options
{
'expected_update_period_in_days' => "10",
'message_path' => "text"
'message' => "{{text}}"
}
end
@ -41,7 +42,7 @@ module Agents
incoming_events = incoming_events.first(20)
end
incoming_events.each do |event|
tweet_text = Utils.value_at(event.payload, options['message_path'])
tweet_text = interpolate_string(options['message'], event.payload)
begin
tweet = publish_tweet tweet_text
create_event :payload => {

View file

@ -0,0 +1,41 @@
<ul class="dropdown-menu" role="menu">
<% if agent.can_be_scheduled? %>
<li>
<%= link_to '<span class="color-success glyphicon glyphicon-refresh"></span> Run'.html_safe, run_agent_path(agent, :return => returnTo), method: :post, :tabindex => "-1" %>
</li>
<% end %>
<li>
<%= link_to '<span class="glyphicon glyphicon-eye-open"></span> Show'.html_safe, agent_path(agent) %>
</li>
<li class="divider"></li>
<li>
<%= link_to '<span class="glyphicon glyphicon-pencil"></span> Edit agent'.html_safe, edit_agent_path(agent) %>
</li>
<li>
<%= link_to '<span class="glyphicon glyphicon-plus"></span> Clone agent'.html_safe, new_agent_path(id: agent), :tabindex => "-1" %>
</li>
<li>
<% if agent.disabled? %>
<%= link_to '<i class="glyphicon glyphicon-play"></i> Enable agent'.html_safe, agent_path(agent, :agent => { :disabled => false }, :return => returnTo), :method => :put %>
<% else %>
<%= link_to '<i class="glyphicon glyphicon-pause"></i> Disable agent'.html_safe, agent_path(agent, :agent => { :disabled => true }, :return => returnTo), :method => :put %>
<% end %>
</li>
<li class="divider"></li>
<% if agent.can_create_events? && agent.events.count > 0 %>
<li>
<%= link_to '<span class="color-danger glyphicon glyphicon-trash"></span> Delete all events'.html_safe, remove_events_agent_path(agent), method: :delete, data: {confirm: 'Are you sure you want to delete ALL events for this Agent?'}, :tabindex => "-1" %>
</li>
<% end %>
<li>
<%= link_to '<span class="color-danger glyphicon glyphicon-remove"></span> Delete agent'.html_safe, agent_path(agent), method: :delete, data: { confirm: 'Are you sure?' }, :tabindex => "-1" %>
</li>
</ul>

View file

@ -19,41 +19,41 @@
</tr>
<% @agents.each do |agent| %>
<tr class='<%= "agent-disabled" if agent.disabled? %>'>
<td>
<%= agent.name %>
<tr>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<%= link_to agent.name, agent_path(agent) %>
<br/>
<span class='text-muted'><%= agent.short_type.titleize %></span>
</td>
<td>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<% if agent.can_be_scheduled? %>
<%= agent.schedule.to_s.humanize.titleize %>
<% else %>
<span class='not-applicable'></span>
<% end %>
</td>
<td>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<% if agent.can_be_scheduled? %>
<%= agent.last_check_at ? time_ago_in_words(agent.last_check_at) + " ago" : "never" %>
<% else %>
<span class='not-applicable'></span>
<% end %>
</td>
<td>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<% if agent.can_create_events? %>
<%= agent.last_event_at ? time_ago_in_words(agent.last_event_at) + " ago" : "never" %>
<% else %>
<span class='not-applicable'></span>
<% end %>
</td>
<td>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<% if agent.can_receive_events? %>
<%= agent.last_receive_at ? time_ago_in_words(agent.last_receive_at) + " ago" : "never" %>
<% else %>
<span class='not-applicable'></span>
<% end %>
</td>
<td>
<td class='<%= "agent-disabled" if agent.disabled? %>'>
<% if agent.can_create_events? %>
<%= link_to(agent.events_count || 0, events_path(:agent => agent.to_param)) %>
<% else %>
@ -62,15 +62,11 @@
</td>
<td><%= working(agent) %></td>
<td>
<div class="btn-group btn-group-xs">
<%= link_to 'Show', agent_path(agent), class: "btn btn-default" %>
<%= link_to 'Edit', edit_agent_path(agent), class: "btn btn-default" %>
<%= link_to 'Delete', agent_path(agent), method: :delete, data: { confirm: 'Are you sure?' }, class: "btn btn-default" %>
<% if agent.can_be_scheduled? && !agent.disabled? %>
<%= link_to 'Run', run_agent_path(agent, :return => "index"), method: :post, class: "btn btn-default" %>
<% else %>
<%= link_to 'Run', "#", class: "btn btn-default disabled" %>
<% end %>
<div class="btn-group">
<button type="button" class="btn btn-default btn-sm dropdown-toggle" data-toggle="dropdown">
<span class="glyphicon glyphicon-th-list"></span> Actions <span class="caret"></span>
</button>
<%= render 'action_menu', :agent => agent, :returnTo => "index" %>
</div>
</td>
</tr>

View file

@ -2,6 +2,8 @@
<div class='row'>
<div class='col-md-2'>
<ul class="nav nav-pills nav-stacked" id="show-tabs">
<li><%= link_to '<span class="glyphicon glyphicon-chevron-left"></span> Back'.html_safe, agents_path %></li>
<% if agent_show_view(@agent).present? %>
<li class='active'><a href="#summary" data-toggle="tab"><span class='glyphicon glyphicon-picture'></span> Summary</a></li>
<li><a href="#details" data-toggle="tab"><span class='glyphicon glyphicon-indent-left'></span> Details</a></li>
@ -9,45 +11,18 @@
<li class='disabled'><a><span class='glyphicon glyphicon-picture'></span> Summary</a></li>
<li class='active'><a href="#details" data-toggle="tab"><span class='glyphicon glyphicon-indent-left'></span> Details</a></li>
<% end %>
<li><a href="#logs" data-toggle="tab" data-agent-id="<%= @agent.id %>" class='<%= @agent.recent_error_logs? ? 'recent-errors' : '' %>'><span class='glyphicon glyphicon-list-alt'></span> Logs</a></li>
<% if @agent.can_create_events? && @agent.events.count > 0 %>
<li><%= link_to '<span class="glyphicon glyphicon-random"></span> Events'.html_safe, events_path(:agent => @agent.to_param) %></li>
<% else %>
<li class='disabled'><a><span class='glyphicon glyphicon-random'></span> Events</a></li>
<% end %>
<li><%= link_to '<span class="glyphicon glyphicon-chevron-left"></span> Back'.html_safe, agents_path %></li>
<li><%= link_to '<span class="glyphicon glyphicon-pencil"></span> Edit'.html_safe, edit_agent_path(@agent) %></li>
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" href="#">Actions <b class="caret"></b></a>
<ul class="dropdown-menu" role="menu" aria-labelledby="dLabel">
<% if @agent.can_be_scheduled? %>
<li>
<%= link_to '<span class="glyphicon glyphicon-refresh"></span> Run'.html_safe, run_agent_path(@agent, :return => "show"), method: :post, :tabindex => "-1" %>
</li>
<% end %>
<% if @agent.can_create_events? && @agent.events.count > 0 %>
<li>
<%= link_to '<span class="glyphicon glyphicon-trash"></span> Delete all events'.html_safe, remove_events_agent_path(@agent), method: :delete, data: {confirm: 'Are you sure you want to delete ALL events for this Agent?'}, :tabindex => "-1" %>
</li>
<% end %>
<li>
<%= link_to '<span class="glyphicon glyphicon-plus"></span> Clone'.html_safe, new_agent_path(id: @agent), :tabindex => "-1" %>
</li>
<li>
<% if @agent.disabled? %>
<%= link_to '<i class="glyphicon glyphicon-play"></i> Enable agent'.html_safe, agent_path(@agent, :agent => { :disabled => false }, :return => "show"), :method => :put %>
<% else %>
<%= link_to '<i class="glyphicon glyphicon-pause"></i> Disable agent'.html_safe, agent_path(@agent, :agent => { :disabled => true }, :return => "show"), :method => :put %>
<% end %>
</li>
<li>
<%= link_to '<span class="glyphicon glyphicon-remove"></span> Delete'.html_safe, agent_path(@agent), method: :delete, data: { confirm: 'Are you sure?' }, :tabindex => "-1" %>
</li>
</ul>
<a class="dropdown-toggle" data-toggle="dropdown" href="#"><span class="glyphicon glyphicon-th-list"></span> Actions <span class="caret"></span></a>
<%= render 'action_menu', :agent => @agent, :returnTo => "show" %>
</li>
</ul>
</div>

View file

@ -1,7 +1,7 @@
<% if flash.keys.length > 0 %>
<div class="flash">
<% flash.each do |name, msg| %>
<div class="alert alert-<%= name.to_sym == :notice ? "success" : "error" %> alert-dismissable">
<div class="alert alert-<%= name.to_sym == :notice ? "success" : "danger" %> alert-dismissable">
<button type="button" class="close" data-dismiss="alert" aria-hidden="true">&times;</button>
<%= content_tag :div, msg, :id => "flash_#{name}" if msg.is_a?(String) %>
</div>

View file

@ -0,0 +1,45 @@
class MigrateAgentsToLiquidTemplating < ActiveRecord::Migration
def up
Agent.where(:type => 'Agents::HipchatAgent').each do |agent|
LiquidMigrator.convert_all_agent_options(agent)
end
Agent.where(:type => 'Agents::EventFormattingAgent').each do |agent|
agent.options['instructions'] = LiquidMigrator.convert_hash(agent.options['instructions'], {:merge_path_attributes => true, :leading_dollarsign_is_jsonpath => true})
agent.save
end
Agent.where(:type => 'Agents::PushbulletAgent').each do |agent|
LiquidMigrator.convert_all_agent_options(agent)
end
Agent.where(:type => 'Agents::JabberAgent').each do |agent|
LiquidMigrator.convert_all_agent_options(agent)
end
Agent.where(:type => 'Agents::DataOutputAgent').each do |agent|
LiquidMigrator.convert_all_agent_options(agent)
end
Agent.where(:type => 'Agents::TranslationAgent').each do |agent|
agent.options['content'] = LiquidMigrator.convert_hash(agent.options['content'], {:merge_path_attributes => true, :leading_dollarsign_is_jsonpath => true})
agent.save
end
Agent.where(:type => 'Agents::TwitterPublishAgent').each do |agent|
if (message = agent.options.delete('message_path')).present?
agent.options['message'] = "{{#{message}}}"
agent.save
end
end
Agent.where(:type => 'Agents::TriggerAgent').each do |agent|
agent.options['message'] = LiquidMigrator.convert_make_message(agent.options['message'])
agent.save
end
Agent.where(:type => 'Agents::PeakDetectorAgent').each do |agent|
agent.options['message'] = LiquidMigrator.convert_make_message(agent.options['message'])
agent.save
end
Agent.where(:type => 'Agents::HumanTaskAgent').each do |agent|
LiquidMigrator.convert_all_agent_options(agent)
end
end
def down
raise ActiveRecord::IrreversibleMigration, "Cannot revert migration to Liquid templating"
end
end

View file

@ -1,19 +1,19 @@
SITE
remote: http://community.opscode.com/api/v1
specs:
apt (2.3.8)
apt (2.4.0)
bluepill (2.3.1)
rsyslog (>= 0.0.0)
build-essential (2.0.0)
build-essential (2.0.2)
chef_handler (1.1.6)
dmg (2.2.0)
ohai (1.1.12)
ohai (2.0.0)
rsyslog (1.12.2)
runit (1.5.10)
build-essential (>= 0.0.0)
yum (~> 3.0)
yum-epel (>= 0.0.0)
windows (1.30.2)
windows (1.31.0)
chef_handler (>= 0.0.0)
yum (3.2.0)
yum-epel (0.3.6)
@ -32,9 +32,9 @@ GIT
GIT
remote: git://github.com/opscode-cookbooks/git.git
ref: master
sha: 76b0f9bb08fdd9e2e201fd70b72298097accdf96
sha: b1bca76aaf3a3a2131744f17f6e5087e22fa3c40
specs:
git (4.0.1)
git (4.0.3)
build-essential (>= 0.0.0)
dmg (>= 0.0.0)
runit (>= 1.0)
@ -45,20 +45,20 @@ GIT
GIT
remote: git://github.com/opscode-cookbooks/mysql.git
ref: master
sha: a2ff53f0ca6deca75aebf6da55ac381194ec7728
sha: 4b70e99730ab4a7ce6c1dd7a35654a764fb6e0fe
specs:
mysql (5.1.9)
mysql (5.2.13)
GIT
remote: git://github.com/opscode-cookbooks/nginx.git
ref: master
sha: 05b3a613f53a0b05c96f9206c5d67aa420f337fb
sha: 45588ee2a5c7144a0ef2a5992e7f273542236d27
specs:
nginx (2.6.3)
nginx (2.7.3)
apt (~> 2.2)
bluepill (~> 2.3)
build-essential (~> 2.0)
ohai (~> 1.1)
ohai (~> 2.0)
runit (~> 1.2)
yum-epel (~> 0.3)

View file

@ -16,16 +16,23 @@ group "huginn" do
action :create
end
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "rubygems").each do |pkg|
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "libffi-dev" "libssl-dev").each do |pkg|
package pkg do
action :install
end
end
bash "Setting default ruby version to 1.9" do
bash "Setting default ruby and gem versions to 1.9" do
code <<-EOH
update-alternatives --set ruby /usr/bin/ruby1.9.1
update-alternatives --set gem /usr/bin/gem1.9.1
if [ $(readlink /usr/bin/ruby) != "ruby1.9.1" ]
then
update-alternatives --set ruby /usr/bin/ruby1.9.1
fi
if [ $(readlink /usr/bin/gem) != "gem1.9.1" ]
then
update-alternatives --set gem /usr/bin/gem1.9.1
fi
EOH
end

View file

@ -14,14 +14,21 @@ group "huginn" do
members ["huginn"]
end
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8" "libmysqlclient-dev" "libffi-dev" "libssl-dev" "rubygems").each do |pkg|
%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "libffi-dev" "libssl-dev").each do |pkg|
package("#{pkg}")
end
bash "Setting default ruby version to 1.9" do
bash "Setting default ruby and gem versions to 1.9" do
code <<-EOH
update-alternatives --set ruby /usr/bin/ruby1.9.1
update-alternatives --set gem /usr/bin/gem1.9.1
if [ $(readlink /usr/bin/ruby) != "ruby1.9.1" ]
then
update-alternatives --set ruby /usr/bin/ruby1.9.1
fi
if [ $(readlink /usr/bin/gem) != "gem1.9.1" ]
then
update-alternatives --set gem /usr/bin/gem1.9.1
fi
EOH
end

78
lib/liquid_migrator.rb Normal file
View file

@ -0,0 +1,78 @@
module LiquidMigrator
def self.convert_all_agent_options(agent)
agent.options = self.convert_hash(agent.options, {:merge_path_attributes => true, :leading_dollarsign_is_jsonpath => true})
agent.save!
end
def self.convert_hash(hash, options={})
options = {:merge_path_attributes => false, :leading_dollarsign_is_jsonpath => false}.merge options
keys_to_remove = []
hash.tap do |hash|
hash.each_pair do |key, value|
case value.class.to_s
when 'String', 'FalseClass', 'TrueClass'
path_key = "#{key}_path"
if options[:merge_path_attributes] && !hash[path_key].nil?
# replace the value if the path is present
value = hash[path_key] if hash[path_key].present?
# in any case delete the path attibute
keys_to_remove << path_key
end
hash[key] = LiquidMigrator.convert_string value, options[:leading_dollarsign_is_jsonpath]
when 'ActiveSupport::HashWithIndifferentAccess'
hash[key] = convert_hash(hash[key], options)
when 'Array'
hash[key] = hash[key].collect { |k|
if k.class == String
convert_string(k, options[:leading_dollarsign_is_jsonpath])
else
convert_hash(k, options)
end
}
end
end
# remove the unneeded *_path attributes
end.select { |k, v| !keys_to_remove.include? k }
end
def self.convert_string(string, leading_dollarsign_is_jsonpath=false)
if string == true || string == false
# there might be empty *_path attributes for boolean defaults
string
elsif string[0] == '$' && leading_dollarsign_is_jsonpath
# in most cases a *_path attribute
convert_json_path string
else
# migrate the old interpolation syntax to the new liquid based
string.gsub(/<([^>]+)>/).each do
match = $1
if match =~ /\Aescape /
# convert the old escape syntax to a liquid filter
self.convert_json_path(match.gsub(/\Aescape /, '').strip, ' | uri_escape')
else
self.convert_json_path(match.strip)
end
end
end
end
def self.convert_make_message(string)
string.gsub(/<([^>]+)>/, "{{\\1}}")
end
def self.convert_json_path(string, filter = "")
check_path(string)
if string.start_with? '$.'
"{{#{string[2..-1]}#{filter}}}"
else
"{{#{string[1..-1]}#{filter}}}"
end
end
def self.check_path(string)
if string !~ /\A(\$\.?)?(\w+\.)*(\w+)\Z/
raise "JSONPath '#{string}' is too complex, please check your migration."
end
end
end

View file

@ -0,0 +1,22 @@
From: Nanashi <nanashi.gombeh@example.jp>
Date: Fri, 9 May 2014 16:00:00 +0900
Message-ID: <foo.123@mail.example.jp>
Subject: some subject
To: Jane <jane.doe@example.com>, John <john.doe@example.com>
MIME-Version: 1.0
Content-Type: multipart/alternative; boundary=d8c92622e09101e4bc833685557b
--d8c92622e09101e4bc833685557b
Content-Type: text/plain; charset=UTF-8
Some plain text
Some second line
--d8c92622e09101e4bc833685557b
Content-Type: text/html; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
<div dir=3D"ltr">Some HTML document<br>
Some second line of HTML<br></div>
--d8c92622e09101e4bc833685557b--

View file

@ -0,0 +1,20 @@
From: John <john.doe@example.com>
Date: Fri, 9 May 2014 17:00:00 +0900
Message-ID: <bar.456@mail.example.com>
Subject: Re: some subject
To: Jane <jane.doe@example.com>, Nanashi <nanashi.gombeh@example.jp>
MIME-Version: 1.0
Content-Type: multipart/alternative; boundary=d8c92622e09101e4bc833685557b
--d8c92622e09101e4bc833685557b
Content-Type: text/plain; charset=UTF-8
Some reply
--d8c92622e09101e4bc833685557b
Content-Type: text/html; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
<div dir=3D"ltr">Some HTML reply<br></div>
--d8c92622e09101e4bc833685557b--

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1,156 @@
require 'spec_helper'
describe LiquidMigrator do
describe "converting JSONPath strings" do
it "should work" do
LiquidMigrator.convert_string("$.data", true).should == "{{data}}"
LiquidMigrator.convert_string("$.data.test", true).should == "{{data.test}}"
LiquidMigrator.convert_string("$first_title", true).should == "{{first_title}}"
end
it "should ignore strings which just contain a JSONPath" do
LiquidMigrator.convert_string("$.data").should == "$.data"
LiquidMigrator.convert_string("$first_title").should == "$first_title"
LiquidMigrator.convert_string(" $.data", true).should == " $.data"
LiquidMigrator.convert_string("lorem $.data", true).should == "lorem $.data"
end
it "should raise an exception when encountering complex JSONPaths" do
expect { LiquidMigrator.convert_string("$.data.test.*", true) }.
to raise_error("JSONPath '$.data.test.*' is too complex, please check your migration.")
end
end
describe "converting escaped JSONPath strings" do
it "should work" do
LiquidMigrator.convert_string("Weather looks like <$.conditions> according to the forecast at <$.pretty_date.time>").should ==
"Weather looks like {{conditions}} according to the forecast at {{pretty_date.time}}"
end
it "should convert the 'escape' method correctly" do
LiquidMigrator.convert_string("Escaped: <escape $.content.name>\nNot escaped: <$.content.name>").should ==
"Escaped: {{content.name | uri_escape}}\nNot escaped: {{content.name}}"
end
it "should raise an exception when encountering complex JSONPaths" do
expect { LiquidMigrator.convert_string("Received <$.content.text.*> from <$.content.name> .") }.
to raise_error("JSONPath '$.content.text.*' is too complex, please check your migration.")
end
end
describe "migrating a hash" do
it "should convert every attribute" do
LiquidMigrator.convert_hash({'a' => "$.data", 'b' => "This is a <$.test>"}).should ==
{'a' => "$.data", 'b' => "This is a {{test}}"}
end
it "should work with leading_dollarsign_is_jsonpath" do
LiquidMigrator.convert_hash({'a' => "$.data", 'b' => "This is a <$.test>"}, leading_dollarsign_is_jsonpath: true).should ==
{'a' => "{{data}}", 'b' => "This is a {{test}}"}
end
it "should use the corresponding *_path attributes when using merge_path_attributes"do
LiquidMigrator.convert_hash({'a' => "default", 'a_path' => "$.data"}, {leading_dollarsign_is_jsonpath: true, merge_path_attributes: true}).should ==
{'a' => "{{data}}"}
end
it "should raise an exception when encountering complex JSONPaths" do
expect { LiquidMigrator.convert_hash({'b' => "This is <$.complex[2]>"}) }.
to raise_error("JSONPath '$.complex[2]' is too complex, please check your migration.")
end
end
describe "migrating the 'make_message' format" do
it "should work" do
LiquidMigrator.convert_make_message('<message>').should == '{{message}}'
LiquidMigrator.convert_make_message('<new.message>').should == '{{new.message}}'
LiquidMigrator.convert_make_message('Hello <world>. How is <nested.life>').should == 'Hello {{world}}. How is {{nested.life}}'
end
end
describe "migrating an actual agent" do
before do
valid_params = {
'auth_token' => 'token',
'room_name' => 'test',
'room_name_path' => '',
'username' => "Huginn",
'username_path' => '$.username',
'message' => "Hello from Huginn!",
'message_path' => '$.message',
'notify' => false,
'notify_path' => '',
'color' => 'yellow',
'color_path' => '',
}
@agent = Agents::HipchatAgent.new(:name => "somename", :options => valid_params)
@agent.user = users(:jane)
@agent.save!
end
it "should work" do
LiquidMigrator.convert_all_agent_options(@agent)
@agent.reload.options.should == {"auth_token" => 'token', 'color' => 'yellow', 'notify' => false, 'room_name' => 'test', 'username' => '{{username}}', 'message' => '{{message}}'}
end
it "should work with nested hashes" do
@agent.options['very'] = {'nested' => '$.value'}
LiquidMigrator.convert_all_agent_options(@agent)
@agent.reload.options.should == {"auth_token" => 'token', 'color' => 'yellow', 'very' => {'nested' => '{{value}}'}, 'notify' => false, 'room_name' => 'test', 'username' => '{{username}}', 'message' => '{{message}}'}
end
it "should work with nested arrays" do
@agent.options['array'] = ["one", "$.two"]
LiquidMigrator.convert_all_agent_options(@agent)
@agent.reload.options.should == {"auth_token" => 'token', 'color' => 'yellow', 'array' => ['one', '{{two}}'], 'notify' => false, 'room_name' => 'test', 'username' => '{{username}}', 'message' => '{{message}}'}
end
it "should raise an exception when encountering complex JSONPaths" do
@agent.options['username_path'] = "$.very.complex[*]"
expect { LiquidMigrator.convert_all_agent_options(@agent) }.
to raise_error("JSONPath '$.very.complex[*]' is too complex, please check your migration.")
end
it "should work with the human task agent" do
valid_params = {
'expected_receive_period_in_days' => 2,
'trigger_on' => "event",
'hit' =>
{
'assignments' => 1,
'title' => "Sentiment evaluation",
'description' => "Please rate the sentiment of this message: '<$.message>'",
'reward' => 0.05,
'lifetime_in_seconds' => 24 * 60 * 60,
'questions' =>
[
{
'type' => "selection",
'key' => "sentiment",
'name' => "Sentiment",
'required' => "true",
'question' => "Please select the best sentiment value:",
'selections' =>
[
{ 'key' => "happy", 'text' => "Happy" },
{ 'key' => "sad", 'text' => "Sad" },
{ 'key' => "neutral", 'text' => "Neutral" }
]
},
{
'type' => "free_text",
'key' => "feedback",
'name' => "Have any feedback for us?",
'required' => "false",
'question' => "Feedback",
'default' => "Type here...",
'min_length' => "2",
'max_length' => "2000"
}
]
}
}
@agent = Agents::HumanTaskAgent.new(:name => "somename", :options => valid_params)
@agent.user = users(:jane)
LiquidMigrator.convert_all_agent_options(@agent)
@agent.reload.options['hit']['description'].should == "Please rate the sentiment of this message: '{{message}}'"
end
end
end

View file

@ -1,8 +1,11 @@
# encoding: utf-8
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::DataOutputAgent do
it_behaves_like LiquidInterpolatable
let(:agent) do
_agent = Agents::DataOutputAgent.new(:name => 'My Data Output Agent')
_agent.options = _agent.default_options.merge('secrets' => ['secret1', 'secret2'], 'events_to_show' => 2)

View file

@ -6,13 +6,13 @@ describe Agents::EventFormattingAgent do
:name => "somename",
:options => {
:instructions => {
:message => "Received <$.content.text.*> from <$.content.name> .",
:subject => "Weather looks like <$.conditions> according to the forecast at <$.pretty_date.time>"
:message => "Received {{content.text}} from {{content.name}} .",
:subject => "Weather looks like {{conditions}} according to the forecast at {{pretty_date.time}}"
},
:mode => "clean",
:matchers => [
{
:path => "$.date.pretty",
:path => "{{date.pretty}}",
:regexp => "\\A(?<time>\\d\\d:\\d\\d [AP]M [A-Z]+)",
:to => "pretty_date",
},
@ -82,7 +82,7 @@ describe Agents::EventFormattingAgent do
it "should allow escaping" do
@event.payload[:content][:name] = "escape this!?"
@event.save!
@checker.options[:instructions][:message] = "Escaped: <escape $.content.name>\nNot escaped: <$.content.name>"
@checker.options[:instructions][:message] = "Escaped: {{content.name | uri_escape}}\nNot escaped: {{content.name}}"
@checker.save!
@checker.receive([@event])
Event.last.payload[:message].should == "Escaped: escape+this%21%3F\nNot escaped: escape this!?"

View file

@ -1,22 +1,17 @@
require 'spec_helper'
require 'models/concerns/json_path_options_overwritable'
require 'models/concerns/liquid_interpolatable'
describe Agents::HipchatAgent do
it_behaves_like JsonPathOptionsOverwritable
it_behaves_like LiquidInterpolatable
before(:each) do
@valid_params = {
'auth_token' => 'token',
'room_name' => 'test',
'room_name_path' => '',
'username' => "Huginn",
'username_path' => '$.username',
'message' => "Hello from Huginn!",
'message_path' => '$.message',
'username' => "{{username}}",
'message' => "{{message}}",
'notify' => false,
'notify_path' => '',
'color' => 'yellow',
'color_path' => '',
}
@checker = Agents::HipchatAgent.new(:name => "somename", :options => @valid_params)

View file

@ -1,6 +1,9 @@
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::HumanTaskAgent do
it_behaves_like LiquidInterpolatable
before do
@checker = Agents::HumanTaskAgent.new(:name => "my human task agent")
@checker.options = @checker.default_options
@ -116,19 +119,19 @@ describe Agents::HumanTaskAgent do
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
'instructions' => "Rank these by how funny they are",
'assignments' => 3,
'row_template' => "<$.joke>" }
'row_template' => "{{joke}}" }
@checker.should be_valid
@checker.options['poll_options'] = { 'instructions' => "Rank these by how funny they are",
'assignments' => 3,
'row_template' => "<$.joke>" }
'row_template' => "{{joke}}" }
@checker.should_not be_valid
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
'assignments' => 3,
'row_template' => "<$.joke>" }
'row_template' => "{{joke}}" }
@checker.should_not be_valid
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
'instructions' => "Rank these by how funny they are",
'row_template' => "<$.joke>" }
'row_template' => "{{joke}}" }
@checker.should_not be_valid
@checker.options['poll_options'] = { 'title' => "Take a poll about jokes",
'instructions' => "Rank these by how funny they are",
@ -207,9 +210,9 @@ describe Agents::HumanTaskAgent do
describe "creating hits" do
it "can create HITs based on events, interpolating their values" do
@checker.options['hit']['title'] = "Hi <.name>"
@checker.options['hit']['description'] = "Make something for <.name>"
@checker.options['hit']['questions'][0]['name'] = "<.name> Question 1"
@checker.options['hit']['title'] = "Hi {{name}}"
@checker.options['hit']['description'] = "Make something for {{name}}"
@checker.options['hit']['questions'][0]['name'] = "{{name}} Question 1"
question_form = nil
hitInterface = OpenStruct.new
@ -232,7 +235,7 @@ describe Agents::HumanTaskAgent do
end
it "works without an event too" do
@checker.options['hit']['title'] = "Hi <.name>"
@checker.options['hit']['title'] = "Hi {{name}}"
hitInterface = OpenStruct.new
hitInterface.id = 123
mock(hitInterface).question_form(instance_of Agents::HumanTaskAgent::AgentQuestionForm)
@ -483,7 +486,7 @@ describe Agents::HumanTaskAgent do
'title' => "Hi!",
'instructions' => "hello!",
'assignments' => 2,
'row_template' => "This is <.sentiment>"
'row_template' => "This is {{sentiment}}"
}
@event.save!
mock(RTurk::GetReviewableHITs).create { mock!.hit_ids { %w[JH3132836336DHG JH39AA63836DHG JH39AA63836DH12345] } }

View file

@ -0,0 +1,242 @@
require 'spec_helper'
require 'time'
describe Agents::ImapFolderAgent do
describe 'checking IMAP' do
before do
@site = {
'expected_update_period_in_days' => 1,
'host' => 'mail.example.net',
'ssl' => true,
'username' => 'foo',
'password' => 'bar',
'folders' => ['INBOX'],
'conditions' => {
}
}
@checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2)
@checker.user = users(:bob)
@checker.save!
message_mixin = Module.new {
def folder
'INBOX'
end
def uidvalidity
'100'
end
def has_attachment?
false
end
def body_parts(mime_types = %[text/plain text/enriched text/html])
mime_types.map { |type|
all_parts.find { |part|
part.mime_type == type
}
}.compact
end
}
@mails = [
Mail.read(Rails.root.join('spec/data_fixtures/imap1.eml')).tap { |mail|
mail.extend(message_mixin)
stub(mail).uid.returns(1)
},
Mail.read(Rails.root.join('spec/data_fixtures/imap2.eml')).tap { |mail|
mail.extend(message_mixin)
stub(mail).uid.returns(2)
stub(mail).has_attachment?.returns(true)
},
]
stub(@checker).each_unread_mail.returns { |yielder|
@mails.each(&yielder)
}
@payloads = [
{
'folder' => 'INBOX',
'from' => 'nanashi.gombeh@example.jp',
'to' => ['jane.doe@example.com', 'john.doe@example.com'],
'cc' => [],
'date' => '2014-05-09T16:00:00+09:00',
'subject' => 'some subject',
'body' => "Some plain text\nSome second line\n",
'has_attachment' => false,
'matches' => {},
'mime_type' => 'text/plain',
},
{
'folder' => 'INBOX',
'from' => 'john.doe@example.com',
'to' => ['jane.doe@example.com', 'nanashi.gombeh@example.jp'],
'cc' => [],
'subject' => 'Re: some subject',
'body' => "Some reply\n",
'date' => '2014-05-09T17:00:00+09:00',
'has_attachment' => true,
'matches' => {},
'mime_type' => 'text/plain',
}
]
end
describe 'validations' do
before do
@checker.should be_valid
end
it 'should validate the integer fields' do
@checker.options['expected_update_period_in_days'] = 'nonsense'
@checker.should_not be_valid
@checker.options['expected_update_period_in_days'] = '2'
@checker.should be_valid
@checker.options['port'] = -1
@checker.should_not be_valid
@checker.options['port'] = 'imap'
@checker.should_not be_valid
@checker.options['port'] = '143'
@checker.should be_valid
@checker.options['port'] = 993
@checker.should be_valid
end
it 'should validate the boolean fields' do
@checker.options['ssl'] = false
@checker.should be_valid
@checker.options['ssl'] = 'true'
@checker.should_not be_valid
end
it 'should validate regexp conditions' do
@checker.options['conditions'] = {
'subject' => '(foo'
}
@checker.should_not be_valid
@checker.options['conditions'] = {
'body' => '***'
}
@checker.should_not be_valid
@checker.options['conditions'] = {
'subject' => '\ARe:',
'body' => '(?<foo>http://\S+)'
}
@checker.should be_valid
end
end
describe '#check' do
it 'should check for mails and save memory' do
lambda { @checker.check }.should change { Event.count }.by(2)
@checker.memory['notified'].sort.should == @mails.map(&:message_id).sort
@checker.memory['seen'].should == @mails.each_with_object({}) { |mail, seen|
(seen[mail.uidvalidity] ||= []) << mail.uid
}
Event.last(2).map(&:payload) == @payloads
lambda { @checker.check }.should_not change { Event.count }
end
it 'should narrow mails by To' do
@checker.options['conditions']['to'] = 'John.Doe@*'
lambda { @checker.check }.should change { Event.count }.by(1)
@checker.memory['notified'].sort.should == [@mails.first.message_id]
@checker.memory['seen'].should == @mails.each_with_object({}) { |mail, seen|
(seen[mail.uidvalidity] ||= []) << mail.uid
}
Event.last.payload.should == @payloads.first
lambda { @checker.check }.should_not change { Event.count }
end
it 'should perform regexp matching and save named captures' do
@checker.options['conditions'].update(
'subject' => '\ARe: (?<a>.+)',
'body' => 'Some (?<b>.+) reply',
)
lambda { @checker.check }.should change { Event.count }.by(1)
@checker.memory['notified'].sort.should == [@mails.last.message_id]
@checker.memory['seen'].should == @mails.each_with_object({}) { |mail, seen|
(seen[mail.uidvalidity] ||= []) << mail.uid
}
Event.last.payload.should == @payloads.last.update(
'body' => "<div dir=\"ltr\">Some HTML reply<br></div>\n",
'matches' => { 'a' => 'some subject', 'b' => 'HTML' },
'mime_type' => 'text/html',
)
lambda { @checker.check }.should_not change { Event.count }
end
it 'should narrow mails by has_attachment (true)' do
@checker.options['conditions']['has_attachment'] = true
lambda { @checker.check }.should change { Event.count }.by(1)
Event.last.payload['subject'].should == 'Re: some subject'
end
it 'should narrow mails by has_attachment (false)' do
@checker.options['conditions']['has_attachment'] = false
lambda { @checker.check }.should change { Event.count }.by(1)
Event.last.payload['subject'].should == 'some subject'
end
it 'should narrow mail parts by MIME types' do
@checker.options['mime_types'] = %w[text/plain]
@checker.options['conditions'].update(
'subject' => '\ARe: (?<a>.+)',
'body' => 'Some (?<b>.+) reply',
)
lambda { @checker.check }.should_not change { Event.count }
@checker.memory['notified'].sort.should == []
@checker.memory['seen'].should == @mails.each_with_object({}) { |mail, seen|
(seen[mail.uidvalidity] ||= []) << mail.uid
}
end
it 'should never mark mails as read unless mark_as_read is true' do
@mails.each { |mail|
stub(mail).mark_as_read.never
}
lambda { @checker.check }.should change { Event.count }.by(2)
end
it 'should mark mails as read if mark_as_read is true' do
@checker.options['mark_as_read'] = true
@mails.each { |mail|
stub(mail).mark_as_read.once
}
lambda { @checker.check }.should change { Event.count }.by(2)
end
it 'should create just one event for multiple mails with the same Message-Id' do
@mails.first.message_id = @mails.last.message_id
@checker.options['mark_as_read'] = true
@mails.each { |mail|
stub(mail).mark_as_read.once
}
lambda { @checker.check }.should change { Event.count }.by(1)
end
end
end
end

View file

@ -1,6 +1,9 @@
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::JabberAgent do
it_behaves_like LiquidInterpolatable
let(:sent) { [] }
let(:config) {
{
@ -9,7 +12,7 @@ describe Agents::JabberAgent do
jabber_sender: 'foo@localhost',
jabber_receiver: 'bar@localhost',
jabber_password: 'password',
message: 'Warning! <$.title> - <$.url>',
message: 'Warning! {{title}} - {{url}}',
expected_receive_period_in_days: '2'
}
}

View file

@ -0,0 +1,110 @@
require 'spec_helper'
describe Agents::JiraAgent do
before(:each) do
stub_request(:get, /atlassian.com/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/jira.json")), :status => 200, :headers => {"Content-Type" => "text/json"})
@valid_params = {
:username => "user",
:password => "pass",
:jira_url => 'https://jira.atlassian.com',
:jql => 'resolution = unresolved',
:expected_update_period_in_days => '7',
:timeout => '1'
}
@checker = Agents::JiraAgent.new(:name => "jira-agent", :options => @valid_params)
@checker.user = users(:jane)
@checker.save!
end
describe "validating" do
before do
@checker.should be_valid
end
it "should work without username" do
@checker.options['username'] = nil
@checker.should be_valid
end
it "should require the jira password if username is specified" do
@checker.options['username'] = 'user'
@checker.options['password'] = nil
@checker.should_not be_valid
end
it "should require the jira url" do
@checker.options['jira_url'] = nil
@checker.should_not be_valid
end
it "should work without jql" do
@checker.options['jql'] = nil
@checker.should be_valid
end
it "should require the expected_update_period_in_days" do
@checker.options['expected_update_period_in_days'] = nil
@checker.should_not be_valid
end
it "should require timeout" do
@checker.options['timeout'] = nil
@checker.should_not be_valid
end
end
describe "helpers" do
it "should generate a correct request options hash" do
@checker.send(:request_options).should == {:basic_auth=>{:username=>"user", :password=>"pass"}, :headers => {"User-Agent" => "Huginn (https://github.com/cantino/huginn)"}}
end
it "should generate a correct request url" do
@checker.send(:request_url, 'foo=bar', 10).should == "https://jira.atlassian.com/rest/api/2/search?jql=foo%3Dbar&fields=*all&startAt=10"
end
it "should not set the 'since' time on the first run" do
expected_url = "https://jira.atlassian.com/rest/api/2/search?jql=resolution+%3D+unresolved&fields=*all&startAt=0"
expected_headers = {:headers=>{"User-Agent"=>"Huginn (https://github.com/cantino/huginn)"}, :basic_auth=>{:username=>"user", :password=>"pass"}}
reply = JSON.parse(File.read(Rails.root.join("spec/data_fixtures/jira.json")))
mock(@checker).get(expected_url, expected_headers).returns(reply)
@checker.check
end
it "should provide set the 'since' time after the first run" do
expected_url_1 = "https://jira.atlassian.com/rest/api/2/search?jql=resolution+%3D+unresolved&fields=*all&startAt=0"
expected_url_2 = "https://jira.atlassian.com/rest/api/2/search?jql=resolution+%3D+unresolved&fields=*all&startAt=0"
expected_headers = {:headers=>{"User-Agent"=>"Huginn (https://github.com/cantino/huginn)"}, :basic_auth=>{:username=>"user", :password=>"pass"}}
reply = JSON.parse(File.read(Rails.root.join("spec/data_fixtures/jira.json")))
mock(@checker) do
get(expected_url_1, expected_headers).returns(reply)
# time specification
get(/\d+-\d+-\d+\+\d+%3A\d+/, expected_headers).returns(reply)
end
@checker.check
@checker.check
end
end
describe "#check" do
it "should be able to retrieve issues" do
reply = JSON.parse(File.read(Rails.root.join("spec/data_fixtures/jira.json")))
mock(@checker).get(anything,anything).returns(reply)
expect { @checker.check }.to change { Event.count }.by(50)
end
end
describe "#working?" do
it "it is working when at least one event was emited" do
@checker.should_not be_working
@checker.check
@checker.reload.should be_working
end
end
end

View file

@ -1,6 +1,9 @@
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::PeakDetectorAgent do
it_behaves_like LiquidInterpolatable
before do
@valid_params = {
'name' => "my peak detector agent",

View file

@ -1,14 +1,14 @@
require 'spec_helper'
require 'models/concerns/json_path_options_overwritable'
require 'models/concerns/liquid_interpolatable'
describe Agents::PushbulletAgent do
it_behaves_like JsonPathOptionsOverwritable
it_behaves_like LiquidInterpolatable
before(:each) do
@valid_params = {
'api_key' => 'token',
'device_id' => '124',
'body_path' => '$.body',
'body' => '{{body}}',
'title' => 'hello from huginn'
}
@ -39,23 +39,18 @@ describe Agents::PushbulletAgent do
end
describe "helpers" do
it "it should return the correct basic_options" do
@checker.send(:basic_options).should == {:basic_auth => {:username =>@checker.options[:api_key], :password=>''},
:body => {:device_iden => @checker.options[:device_id], :type => 'note'}}
end
it "should return the query_options" do
@checker.send(:query_options, @event).should == @checker.send(:basic_options).deep_merge({
:body => {:title => 'hello from huginn', :body => 'One two test'}
})
@checker.send(:query_options, @event).should == {
:body => {:title => 'hello from huginn', :body => 'One two test', :device_iden => @checker.options[:device_id], :type => 'note'},
:basic_auth => {:username =>@checker.options[:api_key], :password=>''}
}
end
end
describe "#receive" do
it "send a message to the hipchat" do
stub_request(:post, "https://token:@api.pushbullet.com/api/pushes").
with(:body => "device_iden=124&type=note&title=hello%20from%20huginn&body=One%20two%20test").
with(:body => "device_iden=124&title=hello%20from%20huginn&body=One%20two%20test&type=note").
to_return(:status => 200, :body => "ok", :headers => {})
dont_allow(@checker).error
@checker.receive([@event])
@ -63,7 +58,7 @@ describe Agents::PushbulletAgent do
it "should log resquests which return an error" do
stub_request(:post, "https://token:@api.pushbullet.com/api/pushes").
with(:body => "device_iden=124&type=note&title=hello%20from%20huginn&body=One%20two%20test").
with(:body => "device_iden=124&title=hello%20from%20huginn&body=One%20two%20test&type=note").
to_return(:status => 200, :body => "error", :headers => {})
mock(@checker).error("error")
@checker.receive([@event])

View file

@ -1,6 +1,10 @@
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::TranslationAgent do
it_behaves_like LiquidInterpolatable
before do
@valid_params = {
:name => "somename",
@ -10,8 +14,8 @@ describe Agents::TranslationAgent do
:to => "fi",
:expected_receive_period_in_days => 1,
:content => {
:text => "$.message",
:content => "$.xyz"
:text => "{{message}}",
:content => "{{xyz}}"
}
}
}

View file

@ -1,6 +1,9 @@
require 'spec_helper'
require 'models/concerns/liquid_interpolatable'
describe Agents::TriggerAgent do
it_behaves_like LiquidInterpolatable
before do
@valid_params = {
'name' => "my trigger agent",
@ -11,7 +14,7 @@ describe Agents::TriggerAgent do
'value' => "a\\db",
'path' => "foo.bar.baz",
}],
'message' => "I saw '<foo.bar.baz>' from <name>"
'message' => "I saw '{{foo.bar.baz}}' from {{name}}"
}
}

View file

@ -9,7 +9,7 @@ describe Agents::TwitterPublishAgent do
:consumer_secret => "---",
:oauth_token => "---",
:oauth_token_secret => "---",
:message_path => "text"
:message => "{{text}}"
}
@checker = Agents::TwitterPublishAgent.new(:name => "HuginnBot", :options => @opts)

View file

@ -1,31 +0,0 @@
require 'spec_helper'
shared_examples_for JsonPathOptionsOverwritable do
before(:each) do
@valid_params = described_class.new.default_options
@checker = described_class.new(:name => "somename", :options => @valid_params)
@checker.user = users(:jane)
@event = Event.new
@event.agent = agents(:bob_weather_agent)
@event.payload = { :room_name => 'test room', :message => 'Looks like its going to rain', username: "Huggin user"}
@event.save!
end
describe "select_option" do
it "should use the room_name_path if specified" do
@checker.options['room_name_path'] = "$.room_name"
@checker.send(:select_option, @event, :room_name).should == "test room"
end
it "should use the normal option when the path option is blank" do
@checker.options['room_name'] = 'test'
@checker.send(:select_option, @event, :room_name).should == "test"
end
end
it "should merge all options" do
@checker.send(:merge_json_path_options, @event).symbolize_keys.keys.should == @checker.send(:options_with_path)
end
end

View file

@ -0,0 +1,67 @@
require 'spec_helper'
shared_examples_for LiquidInterpolatable do
before(:each) do
@valid_params = {
"normal" => "just some normal text",
"variable" => "{{variable}}",
"text" => "Some test with an embedded {{variable}}",
"escape" => "This should be {{hello_world | uri_escape}}"
}
@checker = described_class.new(:name => "somename", :options => @valid_params)
@checker.user = users(:jane)
@event = Event.new
@event.agent = agents(:bob_weather_agent)
@event.payload = { :variable => 'hello', :hello_world => "Hello world"}
@event.save!
end
describe "interpolating liquid templates" do
it "should work" do
@checker.interpolate_options(@checker.options, @event.payload).should == {
"normal" => "just some normal text",
"variable" => "hello",
"text" => "Some test with an embedded hello",
"escape" => "This should be Hello+world"
}
end
it "hsould work with arrays", focus: true do
@checker.options = {"value" => ["{{variable}}", "Much array", "Hey, {{hello_world}}"]}
@checker.interpolate_options(@checker.options, @event.payload).should == {
"value" => ["hello", "Much array", "Hey, Hello world"]
}
end
it "should work recursively" do
@checker.options['hash'] = {'recursive' => "{{variable}}"}
@checker.options['indifferent_hash'] = ActiveSupport::HashWithIndifferentAccess.new({'recursive' => "{{variable}}"})
@checker.interpolate_options(@checker.options, @event.payload).should == {
"normal" => "just some normal text",
"variable" => "hello",
"text" => "Some test with an embedded hello",
"escape" => "This should be Hello+world",
"hash" => {'recursive' => 'hello'},
"indifferent_hash" => {'recursive' => 'hello'},
}
end
it "should work for strings" do
@checker.interpolate_string("{{variable}}", @event.payload).should == "hello"
@checker.interpolate_string("{{variable}} you", @event.payload).should == "hello you"
end
end
describe "liquid tags" do
it "should work with existing credentials" do
@checker.interpolate_string("{% credential aws_key %}", {}).should == '2222222222-jane'
end
it "should raise an exception for undefined credentials" do
expect {
@checker.interpolate_string("{% credential unknown %}", {})
}.to raise_error
end
end
end