From 784142f056ad78975bc3fc978059177bc638b4f1 Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Tue, 26 May 2015 16:42:17 -0500 Subject: [PATCH 01/36] Updated agent picker to include agent description --- app/views/agents/_form.html.erb | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/app/views/agents/_form.html.erb b/app/views/agents/_form.html.erb index 6c0e60d7..8717dcbb 100644 --- a/app/views/agents/_form.html.erb +++ b/app/views/agents/_form.html.erb @@ -23,8 +23,28 @@ <% if @agent.new_record? %>
<%= f.label :type %> - <%= f.select :type, options_for_select([['Select an Agent Type', 'Agent']] + Agent.types.map(&:to_s).sort.map {|type| [type.gsub(/^.*::/, ''), type] }, @agent.type), {}, :class => 'select2 form-control' %> + <%= f.select :type, options_for_select([['Select an Agent Type', 'Agent', {title: ''}]] + Agent.types.map {|type| [type.name.gsub(/^.*::/, '').underscore.humanize.titleize, type, {title: type.description.to_s.strip.split(/\r?\n/)[0]}] }, @agent.type), {}, :class => 'form-control' %>
+ <% end %> From e0188d686e0fa516b3203b60907824159651620d Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Tue, 26 May 2015 17:07:56 -0500 Subject: [PATCH 02/36] Updated agent descriptions to ensure compatibility with new agent picker --- app/models/agents/data_output_agent.rb | 32 +++++++++------------ app/models/agents/dropbox_file_url_agent.rb | 4 +-- app/models/agents/dropbox_watch_agent.rb | 2 +- app/models/agents/rss_agent.rb | 28 +++++++++--------- app/models/agents/twitter_publish_agent.rb | 3 +- app/models/agents/twitter_stream_agent.rb | 5 ++-- app/models/agents/twitter_user_agent.rb | 3 +- app/models/agents/user_location_agent.rb | 16 +++++------ app/models/agents/webhook_agent.rb | 29 +++++++++---------- app/models/agents/witai_agent.rb | 5 ++-- app/models/agents/wunderlist_agent.rb | 7 ++--- 11 files changed, 63 insertions(+), 71 deletions(-) diff --git a/app/models/agents/data_output_agent.rb b/app/models/agents/data_output_agent.rb index a344d6b1..03f8fb31 100644 --- a/app/models/agents/data_output_agent.rb +++ b/app/models/agents/data_output_agent.rb @@ -2,28 +2,26 @@ module Agents class DataOutputAgent < Agent cannot_be_scheduled! - description do - <<-MD - The Agent outputs received events as either RSS or JSON. Use it to output a public or private stream of Huginn data. + description <<-MD + The Agent outputs received events as either RSS or JSON. Use it to output a public or private stream of Huginn data. - This Agent will output data at: + This Agent will output data at: - `https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret.xml` + `https://\#{ENV['DOMAIN']}/users/\#{user.id}/web_requests/\#{id || ''}/:secret.xml` - where `:secret` is one of the allowed secrets specified in your options and the extension can be `xml` or `json`. + where `:secret` is one of the allowed secrets specified in your options and the extension can be `xml` or `json`. - You can setup multiple secrets so that you can individually authorize external systems to - access your Huginn data. + You can setup multiple secrets so that you can individually authorize external systems to + access your Huginn data. - Options: + Options: - * `secrets` - An array of tokens that the requestor must provide for light-weight authentication. - * `expected_receive_period_in_days` - How often you expect data to be received by this Agent from other Agents. - * `template` - A JSON object representing a mapping between item output keys and incoming event values. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the values. The `item` key will be repeated for every Event. The `pubDate` key for each item will have the creation time of the Event unless given. - * `events_to_show` - The number of events to output in RSS or JSON. (default: `40`) - * `ttl` - A value for the element in RSS output. (default: `60`) + * `secrets` - An array of tokens that the requestor must provide for light-weight authentication. + * `expected_receive_period_in_days` - How often you expect data to be received by this Agent from other Agents. + * `template` - A JSON object representing a mapping between item output keys and incoming event values. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the values. The `item` key will be repeated for every Event. The `pubDate` key for each item will have the creation time of the Event unless given. + * `events_to_show` - The number of events to output in RSS or JSON. (default: `40`) + * `ttl` - A value for the element in RSS output. (default: `60`) MD - end def default_options { @@ -110,7 +108,6 @@ module Agents return [content, 200] else content = Utils.unindent(<<-XML) - #{feed_title.encode(:xml => :text)} @@ -119,10 +116,9 @@ module Agents #{Time.now.rfc2822.to_s.encode(:xml => :text)} #{Time.now.rfc2822.to_s.encode(:xml => :text)} #{feed_ttl} - XML - content += items.to_xml(:skip_types => true, :root => "items", :skip_instruct => true, :indent => 1).gsub(/^<\/?items>/, '').strip + content += items.to_xml(:skip_instruct => true, :skip_types => true, :root => "items", :indent => 1).gsub(/^<\/?items>/, '').strip content += Utils.unindent(<<-XML) diff --git a/app/models/agents/dropbox_file_url_agent.rb b/app/models/agents/dropbox_file_url_agent.rb index fa42fe22..f6109ae8 100644 --- a/app/models/agents/dropbox_file_url_agent.rb +++ b/app/models/agents/dropbox_file_url_agent.rb @@ -5,9 +5,9 @@ module Agents cannot_be_scheduled! description <<-MD + The _DropboxFileUrlAgent_ is used to work with Dropbox. It takes a file path (or multiple files paths) and emits events with [temporary links](https://www.dropbox.com/developers/core/docs#media). + #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?} - The _DropboxFileUrlAgent_ is used to work with Dropbox. It takes a file path (or multiple files paths) and emits - events with [temporary links](https://www.dropbox.com/developers/core/docs#media). The incoming event payload needs to have a `paths` key, with a comma-separated list of files you want the URL for. For example: diff --git a/app/models/agents/dropbox_watch_agent.rb b/app/models/agents/dropbox_watch_agent.rb index adaa2764..ef711a1e 100644 --- a/app/models/agents/dropbox_watch_agent.rb +++ b/app/models/agents/dropbox_watch_agent.rb @@ -6,8 +6,8 @@ module Agents default_schedule "every_1m" description <<-MD - #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?} The _DropboxWatchAgent_ watches the given `dir_to_watch` and emits events with the detected changes. + #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?} MD event_description <<-MD diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb index a2bd006f..5b0ef7ae 100644 --- a/app/models/agents/rss_agent.rb +++ b/app/models/agents/rss_agent.rb @@ -8,26 +8,24 @@ module Agents cannot_receive_events! default_schedule "every_1d" - description do - <<-MD - This Agent consumes RSS feeds and emits events when they change. + description <<-MD + This Agent consumes RSS feeds and emits events when they change. - This Agent is fairly simple, using [feed-normalizer](https://github.com/aasmith/feed-normalizer) as a base. For complex feeds - with additional field types, we recommend using a WebsiteAgent. See [this example](https://github.com/cantino/huginn/wiki/Agent-configuration-examples#itunes-trailers). + This Agent is fairly simple, using [feed-normalizer](https://github.com/aasmith/feed-normalizer) as a base. For complex feeds + with additional field types, we recommend using a WebsiteAgent. See [this example](https://github.com/cantino/huginn/wiki/Agent-configuration-examples#itunes-trailers). - If you want to *output* an RSS feed, use the DataOutputAgent. + If you want to *output* an RSS feed, use the DataOutputAgent. - Options: + Options: - * `url` - The URL of the RSS feed. - * `clean` - Attempt to use [feed-normalizer](https://github.com/aasmith/feed-normalizer)'s' `clean!` method to cleanup HTML in the feed. Set to `true` to use. - * `expected_update_period_in_days` - How often you expect this RSS feed to change. If more than this amount of time passes without an update, the Agent will mark itself as not working. - * `headers` - When present, it should be a hash of headers to send with the request. - * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. - * `disable_ssl_verification` - Set to `true` to disable ssl verification. - * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}"). + * `url` - The URL of the RSS feed. + * `clean` - Attempt to use [feed-normalizer](https://github.com/aasmith/feed-normalizer)'s' `clean!` method to cleanup HTML in the feed. Set to `true` to use. + * `expected_update_period_in_days` - How often you expect this RSS feed to change. If more than this amount of time passes without an update, the Agent will mark itself as not working. + * `headers` - When present, it should be a hash of headers to send with the request. + * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. + * `disable_ssl_verification` - Set to `true` to disable ssl verification. + * `user_agent` - A custom User-Agent name (default: "Faraday v\#{Faraday::VERSION}"). MD - end def default_options { diff --git a/app/models/agents/twitter_publish_agent.rb b/app/models/agents/twitter_publish_agent.rb index 2aae00f3..4dfdbc1d 100644 --- a/app/models/agents/twitter_publish_agent.rb +++ b/app/models/agents/twitter_publish_agent.rb @@ -5,9 +5,10 @@ module Agents cannot_be_scheduled! description <<-MD - #{twitter_dependencies_missing if dependencies_missing?} The TwitterPublishAgent publishes tweets from the events it receives. + #{twitter_dependencies_missing if dependencies_missing?} + To be able to use this Agent you need to authenticate with Twitter in the [Services](/services) section first. You must also specify a `message` parameter, you can use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to format the message. diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb index a054a956..8e398a84 100644 --- a/app/models/agents/twitter_stream_agent.rb +++ b/app/models/agents/twitter_stream_agent.rb @@ -5,9 +5,10 @@ module Agents cannot_receive_events! description <<-MD - #{twitter_dependencies_missing if dependencies_missing?} The TwitterStreamAgent follows the Twitter stream in real time, watching for certain keywords, or filters, that you provide. + #{twitter_dependencies_missing if dependencies_missing?} + To follow the Twitter stream, provide an array of `filters`. Multiple words in a filter must all show up in a tweet, but are independent of order. If you provide an array instead of a filter, the first entry will be considered primary and any additional values will be treated as aliases. @@ -122,4 +123,4 @@ module Agents end end end -end \ No newline at end of file +end diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 0907dd61..87999577 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -5,9 +5,10 @@ module Agents cannot_receive_events! description <<-MD - #{twitter_dependencies_missing if dependencies_missing?} The TwitterUserAgent follows the timeline of a specified Twitter user. + #{twitter_dependencies_missing if dependencies_missing?} + To be able to use this Agent you need to authenticate with Twitter in the [Services](/services) section first. You must also provide the `username` of the Twitter user to monitor. diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb index a545c549..c3232c08 100644 --- a/app/models/agents/user_location_agent.rb +++ b/app/models/agents/user_location_agent.rb @@ -6,19 +6,17 @@ module Agents gem_dependency_check { defined?(Haversine) } - description do - <<-MD - #{'## Include `haversine` in your Gemfile to use this Agent!' if dependencies_missing?} - The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location. + description <<-MD + The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location. + #{'## Include `haversine` in your Gemfile to use this Agent!' if dependencies_missing?} - Your POST path will be `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. + Your POST path will be `https://\#{ENV['DOMAIN']}/users/\#{user.id}/update_location/:secret` where `:secret` is specified in your options. - If you want to only keep more precise locations, set `max_accuracy` to the upper bound, in meters. The default name for this field is `accuracy`, but you can change this by setting a value for `accuracy_field`. + If you want to only keep more precise locations, set `max_accuracy` to the upper bound, in meters. The default name for this field is `accuracy`, but you can change this by setting a value for `accuracy_field`. - If you want to require a certain distance traveled, set `min_distance` to the minimum distance, in meters. Note that GPS readings and the measurement itself aren't exact, so don't rely on this for precision filtering. - MD - end + If you want to require a certain distance traveled, set `min_distance` to the minimum distance, in meters. Note that GPS readings and the measurement itself aren't exact, so don't rely on this for precision filtering. + MD event_description <<-MD Assuming you're using the iOS application, events look like this: diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index 92f38eba..c970deee 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -3,25 +3,24 @@ module Agents cannot_be_scheduled! cannot_receive_events! - description do - <<-MD - Use this Agent to create events by receiving webhooks from any source. + description <<-MD + Use this Agent to create events by receiving webhooks from any source. - In order to create events with this agent, make a POST request to: - ``` - https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret - ``` where `:secret` is specified in your options. + In order to create events with this agent, make a POST request to: + ``` + https://\#{ENV['DOMAIN']}/users/\#{user.id}/web_requests/\#{id || ''}/:secret + ``` where `:secret` is specified in your options. - Options: + Options: - * `secret` - A token that the host will provide for authentication. - * `expected_receive_period_in_days` - How often you expect to receive - events this way. Used to determine if the agent is working. - * `payload_path` - JSONPath of the attribute in the POST body to be - used as the Event payload. If `payload_path` points to an array, - Events will be created for each element. + * `secret` - A token that the host will provide for authentication. + * `expected_receive_period_in_days` - How often you expect to receive + events this way. Used to determine if the agent is working. + * `payload_path` - JSONPath of the attribute in the POST body to be + used as the Event payload. If `payload_path` points to an array, + Events will be created for each element. MD - end + event_description do <<-MD diff --git a/app/models/agents/witai_agent.rb b/app/models/agents/witai_agent.rb index a6afdf56..e7b66b6b 100644 --- a/app/models/agents/witai_agent.rb +++ b/app/models/agents/witai_agent.rb @@ -3,9 +3,8 @@ module Agents cannot_be_scheduled! description <<-MD - - `wit.ai` agent receives events, sends text query to your `wit.ai` instance and generates outcome events. Fill in `Server Access Token` of your `wit.ai` instance. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to fill query field. - `expected_receive_period_in_days` is the expected number of days by which agent should receive events. It helps in determining if the agent is working. + The `wit.ai` agent receives events, sends text query to your `wit.ai` instance and generates outcome events. Fill in `Server Access Token` of your `wit.ai` instance. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to fill query field. + `expected_receive_period_in_days` is the expected number of days by which agent should receive events. It helps in determining if the agent is working. MD event_description <<-MD diff --git a/app/models/agents/wunderlist_agent.rb b/app/models/agents/wunderlist_agent.rb index cc5d72b4..3aedf852 100644 --- a/app/models/agents/wunderlist_agent.rb +++ b/app/models/agents/wunderlist_agent.rb @@ -9,12 +9,11 @@ module Agents gem_dependency_check { Devise.omniauth_providers.include?(:wunderlist) } description <<-MD + The WunderlistAgent creates new Wunderlist tasks based on the incoming event. + #{'## Include the `omniauth-wunderlist` gem in your `Gemfile` and set `WUNDERLIST_OAUTH_KEY` and `WUNDERLIST_OAUTH_SECRET` in your environment to use this Agent' if dependencies_missing?} - The WunderlistAgent creates new new tasks based on the incoming event. - To be able to use this Agent you need to authenticate with Wunderlist in the [Services](/services) section first. - MD def default_options @@ -75,4 +74,4 @@ module Agents 'X-Client-ID' => ENV["WUNDERLIST_OAUTH_KEY"] }} end end -end \ No newline at end of file +end From d8f672a870a711a80ac238e3123d5c0773dac5e6 Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Tue, 26 May 2015 17:34:51 -0500 Subject: [PATCH 03/36] Whoops - keep my own changes out of this commit. --- app/models/agents/data_output_agent.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/agents/data_output_agent.rb b/app/models/agents/data_output_agent.rb index 03f8fb31..6b3b5049 100644 --- a/app/models/agents/data_output_agent.rb +++ b/app/models/agents/data_output_agent.rb @@ -108,6 +108,7 @@ module Agents return [content, 200] else content = Utils.unindent(<<-XML) + #{feed_title.encode(:xml => :text)} @@ -116,9 +117,10 @@ module Agents #{Time.now.rfc2822.to_s.encode(:xml => :text)} #{Time.now.rfc2822.to_s.encode(:xml => :text)} #{feed_ttl} + XML - content += items.to_xml(:skip_instruct => true, :skip_types => true, :root => "items", :indent => 1).gsub(/^<\/?items>/, '').strip + content += items.to_xml(:skip_types => true, :root => "items", :skip_instruct => true, :indent => 1).gsub(/^<\/?items>/, '').strip content += Utils.unindent(<<-XML) From fe8662244f8ba1e42d88f9defd17a6de7dbc0e0a Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Wed, 27 May 2015 17:05:38 -0500 Subject: [PATCH 04/36] Moved agent picker script to agent-edit-page.js.coffee, reverted heredoc blocks due to scope issue --- .../pages/agent-edit-page.js.coffee | 15 ++++++++++++++ app/models/agents/data_output_agent.rb | 5 +++-- app/models/agents/rss_agent.rb | 5 +++-- app/models/agents/user_location_agent.rb | 5 +++-- app/models/agents/webhook_agent.rb | 8 ++++---- app/views/agents/_form.html.erb | 20 ------------------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/app/assets/javascripts/pages/agent-edit-page.js.coffee b/app/assets/javascripts/pages/agent-edit-page.js.coffee index fe23a52a..ff0fda27 100644 --- a/app/assets/javascripts/pages/agent-edit-page.js.coffee +++ b/app/assets/javascripts/pages/agent-edit-page.js.coffee @@ -16,10 +16,25 @@ class @AgentEditPage if $("#agent_type").length $("#agent_type").on "change", => @handleTypeChange(false) @handleTypeChange(true) + $('#agent_type').select2 + width: 'resolve' + formatResult: formatAgentForSelect + escapeMarkup: (m) -> + m + matcher: (term, text, opt) -> + description = opt.attr('title') + text.toUpperCase().indexOf(term.toUpperCase()) >= 0 or description.toUpperCase().indexOf(term.toUpperCase()) >= 0 + else @enableDryRunButton() @buildAce() + formatAgentForSelect = (agent) -> + originalOption = agent.element + description = $(originalOption).attr('title') + description = if /^[a-zA-Z]/.test(description) then description else '' + '' + agent.text + '
' + description + handleTypeChange: (firstTime) -> $(".event-descriptions").html("").hide() type = $('#agent_type').val() diff --git a/app/models/agents/data_output_agent.rb b/app/models/agents/data_output_agent.rb index 6b3b5049..1eb7b71e 100644 --- a/app/models/agents/data_output_agent.rb +++ b/app/models/agents/data_output_agent.rb @@ -2,12 +2,12 @@ module Agents class DataOutputAgent < Agent cannot_be_scheduled! - description <<-MD + description do <<-MD The Agent outputs received events as either RSS or JSON. Use it to output a public or private stream of Huginn data. This Agent will output data at: - `https://\#{ENV['DOMAIN']}/users/\#{user.id}/web_requests/\#{id || ''}/:secret.xml` + `https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret.xml` where `:secret` is one of the allowed secrets specified in your options and the extension can be `xml` or `json`. @@ -22,6 +22,7 @@ module Agents * `events_to_show` - The number of events to output in RSS or JSON. (default: `40`) * `ttl` - A value for the element in RSS output. (default: `60`) MD + end def default_options { diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb index 5b0ef7ae..d1b085c1 100644 --- a/app/models/agents/rss_agent.rb +++ b/app/models/agents/rss_agent.rb @@ -8,7 +8,7 @@ module Agents cannot_receive_events! default_schedule "every_1d" - description <<-MD + description do <<-MD This Agent consumes RSS feeds and emits events when they change. This Agent is fairly simple, using [feed-normalizer](https://github.com/aasmith/feed-normalizer) as a base. For complex feeds @@ -24,8 +24,9 @@ module Agents * `headers` - When present, it should be a hash of headers to send with the request. * `basic_auth` - Specify HTTP basic auth parameters: `"username:password"`, or `["username", "password"]`. * `disable_ssl_verification` - Set to `true` to disable ssl verification. - * `user_agent` - A custom User-Agent name (default: "Faraday v\#{Faraday::VERSION}"). + * `user_agent` - A custom User-Agent name (default: "Faraday v#{Faraday::VERSION}"). MD + end def default_options { diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb index c3232c08..cb220995 100644 --- a/app/models/agents/user_location_agent.rb +++ b/app/models/agents/user_location_agent.rb @@ -6,17 +6,18 @@ module Agents gem_dependency_check { defined?(Haversine) } - description <<-MD + description do <<-MD The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location. #{'## Include `haversine` in your Gemfile to use this Agent!' if dependencies_missing?} - Your POST path will be `https://\#{ENV['DOMAIN']}/users/\#{user.id}/update_location/:secret` where `:secret` is specified in your options. + Your POST path will be `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. If you want to only keep more precise locations, set `max_accuracy` to the upper bound, in meters. The default name for this field is `accuracy`, but you can change this by setting a value for `accuracy_field`. If you want to require a certain distance traveled, set `min_distance` to the minimum distance, in meters. Note that GPS readings and the measurement itself aren't exact, so don't rely on this for precision filtering. MD + end event_description <<-MD Assuming you're using the iOS application, events look like this: diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index c970deee..52b0a693 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -3,12 +3,12 @@ module Agents cannot_be_scheduled! cannot_receive_events! - description <<-MD + description do <<-MD Use this Agent to create events by receiving webhooks from any source. - In order to create events with this agent, make a POST request to: + ``` - https://\#{ENV['DOMAIN']}/users/\#{user.id}/web_requests/\#{id || ''}/:secret + https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret ``` where `:secret` is specified in your options. Options: @@ -20,7 +20,7 @@ module Agents used as the Event payload. If `payload_path` points to an array, Events will be created for each element. MD - + end event_description do <<-MD diff --git a/app/views/agents/_form.html.erb b/app/views/agents/_form.html.erb index 8717dcbb..0121d182 100644 --- a/app/views/agents/_form.html.erb +++ b/app/views/agents/_form.html.erb @@ -25,26 +25,6 @@ <%= f.label :type %> <%= f.select :type, options_for_select([['Select an Agent Type', 'Agent', {title: ''}]] + Agent.types.map {|type| [type.name.gsub(/^.*::/, '').underscore.humanize.titleize, type, {title: type.description.to_s.strip.split(/\r?\n/)[0]}] }, @agent.type), {}, :class => 'form-control' %> - <% end %> From 71b2bfdd19ef6d87a72e99593345c7a24a50a308 Mon Sep 17 00:00:00 2001 From: Will Read Date: Sun, 9 Aug 2015 15:15:41 +0000 Subject: [PATCH 05/36] Intercept email in development by default - Actually "sends" out email without need to configure an SMTP host locally - Lets you use all kinds of email addresses, not just real ones - Prevents you from spamming yourself and others while developing --- .env.example | 5 +++-- Gemfile | 1 + Gemfile.lock | 7 +++++++ README.md | 4 +++- config/environments/development.rb | 7 +++++-- config/routes.rb | 4 ++++ .../huginn_production/files/default/env.example | 5 +++-- 7 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index f21c700e..26beee74 100644 --- a/.env.example +++ b/.env.example @@ -54,7 +54,7 @@ INVITATION_CODE=try-huginn # Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. # -# PLEASE NOTE: In order to enable emails locally (e.g., when not in the production Rails environment), +# PLEASE NOTE: In order to enable sending real emails via SMTP locally (e.g., when not in the production Rails environment), # you must also set SEND_EMAIL_IN_DEVELOPMENT to true below. # # If you have trouble with port 587 on Gmail, you can also try setting @@ -68,7 +68,8 @@ SMTP_PORT=587 SMTP_AUTHENTICATION=plain SMTP_ENABLE_STARTTLS_AUTO=true -# Send emails when running in the development Rails environment. +# Set to true to send real emails via SMTP when running in the development Rails environment. +# Set to false to have emails intercepted in development and displayed at http://localhost:3000/letter_opener SEND_EMAIL_IN_DEVELOPMENT=false # The address from which system emails will appear to be sent. diff --git a/Gemfile b/Gemfile index 39e22f23..706f94df 100644 --- a/Gemfile +++ b/Gemfile @@ -101,6 +101,7 @@ group :development do gem 'guard' gem 'guard-livereload' gem 'guard-rspec' + gem 'letter_opener_web' group :test do gem 'coveralls', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 1130f966..0b74a315 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -234,6 +234,12 @@ GEM kramdown (1.3.3) launchy (2.4.2) addressable (~> 2.3) + letter_opener (1.4.1) + launchy (~> 2.2) + letter_opener_web (1.3.0) + actionmailer (>= 3.2) + letter_opener (~> 1.0) + railties (>= 3.2) libv8 (3.16.14.7) liquid (3.0.6) listen (2.7.9) @@ -529,6 +535,7 @@ DEPENDENCIES jsonpath (~> 0.5.6) kaminari (~> 0.16.1) kramdown (~> 1.3.3) + letter_opener_web liquid (~> 3.0.3) mini_magick mqtt diff --git a/README.md b/README.md index 711af044..f9e0bec7 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,9 @@ If you just want to play around, you can simply fork this repository, then perfo * Read the [wiki][wiki] for usage examples and to get started making new Agents. * Periodically run `git fetch upstream` and then `git checkout master && git merge upstream/master` to merge in the newest version of Huginn. -Note: by default, emails are not sent in the `development` Rails environment, which is what you just setup. If you'd like to enable emails when playing with Huginn locally, set `SEND_EMAIL_IN_DEVELOPMENT` to `true` in your `.env` file. +Note: By default, emails are intercepted in the `development` Rails environment, which is what you just setup. You can view +them at [http://localhost:3000/letter_opener](http://localhost:3000/letter_opener). If you'd like to send real emails via SMTP when playing +with Huginn locally, set `SEND_EMAIL_IN_DEVELOPMENT` to `true` in your `.env` file. If you need more detailed instructions, see the [Novice setup guide][novice-setup-guide]. diff --git a/config/environments/development.rb b/config/environments/development.rb index 79459c2e..ce375e9f 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -39,8 +39,11 @@ Huginn::Application.configure do config.action_mailer.default_url_options = { :host => ENV['DOMAIN'] } config.action_mailer.asset_host = ENV['DOMAIN'] - config.action_mailer.perform_deliveries = ENV['SEND_EMAIL_IN_DEVELOPMENT'] == 'true' config.action_mailer.raise_delivery_errors = true - config.action_mailer.delivery_method = :smtp + if ENV['SEND_EMAIL_IN_DEVELOPMENT'] == 'true' + config.action_mailer.delivery_method = :smtp + else + config.action_mailer.delivery_method = :letter_opener + end # smtp_settings moved to config/initializers/action_mailer.rb end diff --git a/config/routes.rb b/config/routes.rb index 7726fc4f..06a8c91e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -74,6 +74,10 @@ Huginn::Application.routes.draw do devise_for :users, controllers: { omniauth_callbacks: 'omniauth_callbacks' }, sign_out_via: [:post, :delete] + + if Rails.env.development? + mount LetterOpenerWeb::Engine, at: "/letter_opener" + end get "/about" => "home#about" root :to => "home#index" diff --git a/deployment/site-cookbooks/huginn_production/files/default/env.example b/deployment/site-cookbooks/huginn_production/files/default/env.example index 17ca50af..506501a8 100644 --- a/deployment/site-cookbooks/huginn_production/files/default/env.example +++ b/deployment/site-cookbooks/huginn_production/files/default/env.example @@ -46,7 +46,7 @@ INVITATION_CODE=try-huginn # Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. # -# PLEASE NOTE: In order to enable emails locally (e.g., when not in the production Rails environment), +# PLEASE NOTE: In order to enable sending real emails via SMTP locally (e.g., when not in the production Rails environment), # you must also set SEND_EMAIL_IN_DEVELOPMENT to true below. SMTP_DOMAIN=your-domain-here.com @@ -57,7 +57,8 @@ SMTP_PORT=587 SMTP_AUTHENTICATION=plain SMTP_ENABLE_STARTTLS_AUTO=true -# Send emails when running in the development Rails environment. +# Set to true to send real emails via SMTP when running in the development Rails environment. +# Set to false to have emails intercepted in development and displayed at http://localhost:3000/letter_opener SEND_EMAIL_IN_DEVELOPMENT=false # The address from which system emails will appear to be sent. From ca2d2acaec7e79b742f420687b5a8fcea8b33ce0 Mon Sep 17 00:00:00 2001 From: Vlad Date: Fri, 14 Aug 2015 20:54:38 +0300 Subject: [PATCH 06/36] Make exclude_replies configurable --- app/models/agents/twitter_user_agent.rb | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 0907dd61..f435a3e7 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -13,6 +13,8 @@ module Agents You must also provide the `username` of the Twitter user to monitor. Set `include_retweets` to `false` to not include retweets (default: `true`) + + Set `exclude_replies` to `true` to exclude replies (default: `false`) Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent. @@ -53,6 +55,7 @@ module Agents { 'username' => 'tectonic', 'include_retweets' => 'true', + 'exclude_replies' => 'false', 'expected_update_period_in_days' => '2' } end @@ -64,6 +67,10 @@ module Agents if options[:include_retweets].present? && !%w[true false].include?(options[:include_retweets]) errors.add(:base, "include_retweets must be a boolean value string (true/false)") end + + if options[:exclude_replies].present? && !%w[true false].include?(options[:exclude_replies]) + errors.add(:base, "exclude_replies must be a boolean value string (true/false)") + end if options[:starting_at].present? Time.parse(options[:starting_at]) rescue errors.add(:base, "Error parsing starting_at") @@ -81,10 +88,14 @@ module Agents def include_retweets? interpolated[:include_retweets] != "false" end + + def exclude_replies? + interpolated[:exclude_replies] != "false" + end def check since_id = memory['since_id'] || nil - opts = {:count => 200, :include_rts => include_retweets?, :exclude_replies => false, :include_entities => true, :contributor_details => true} + opts = {:count => 200, :include_rts => include_retweets?, :exclude_replies => exclude_replies?, :include_entities => true, :contributor_details => true} opts.merge! :since_id => since_id unless since_id.nil? # http://rdoc.info/gems/twitter/Twitter/REST/Timelines#user_timeline-instance_method From ffe962647501a7aabbd6b1e785aaef1df7e2b76b Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Fri, 14 Aug 2015 19:59:20 -0700 Subject: [PATCH 07/36] remove spring because it frequently conflicts with foreman --- Gemfile | 2 -- Gemfile.lock | 7 +------ Guardfile | 2 +- bin/spring | 15 --------------- 4 files changed, 2 insertions(+), 24 deletions(-) delete mode 100755 bin/spring diff --git a/Gemfile b/Gemfile index 39e22f23..6f3b6a8a 100644 --- a/Gemfile +++ b/Gemfile @@ -112,8 +112,6 @@ group :development do gem 'rspec-rails', '~> 3.1' gem 'rspec-html-matchers', '~> 0.7' gem 'shoulda-matchers' - gem 'spring', '~> 1.3.0' - gem 'spring-commands-rspec' gem 'vcr' gem 'webmock', '~> 1.17.4', require: false end diff --git a/Gemfile.lock b/Gemfile.lock index 1130f966..485acc08 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -413,9 +413,6 @@ GEM slop (3.6.0) spectrum-rails (1.3.4) railties (>= 3.1) - spring (1.3.6) - spring-commands-rspec (1.0.4) - spring (>= 0.9.1) sprockets (3.2.0) rack (~> 1.0) sprockets-rails (2.3.1) @@ -562,8 +559,6 @@ DEPENDENCIES shoulda-matchers slack-notifier (~> 1.0.0) spectrum-rails - spring (~> 1.3.0) - spring-commands-rspec string-scrub therubyracer (~> 0.12.2) tumblr_client @@ -582,4 +577,4 @@ DEPENDENCIES xmpp4r (~> 0.5.6) BUNDLED WITH - 1.10.5 + 1.10.6 diff --git a/Guardfile b/Guardfile index 4e114775..8cd006f8 100644 --- a/Guardfile +++ b/Guardfile @@ -8,7 +8,7 @@ guard 'livereload' do watch(%r{(app|vendor)(/assets/\w+/(.+\.(css|js|html|png|jpg))).*}) { |m| "/assets/#{m[3]}" } end -guard :rspec, cmd: 'bundle exec spring rspec' do +guard :rspec, cmd: 'bundle exec rspec' do watch(%r{^spec/.+_spec\.rb$}) watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } watch('spec/spec_helper.rb') { "spec" } diff --git a/bin/spring b/bin/spring deleted file mode 100755 index e0918249..00000000 --- a/bin/spring +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env ruby - -# This file loads spring without using Bundler, in order to be fast. -# It gets overwritten when you run the `spring binstub` command. - -unless defined?(Spring) - require "rubygems" - require "bundler" - - if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m) - Gem.paths = { "GEM_PATH" => Bundler.bundle_path.to_s } - gem "spring", match[1] - require "spring/binstub" - end -end From ce84f159e0736a3742b997c51c725d03c386857e Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Fri, 14 Aug 2015 21:51:13 -0700 Subject: [PATCH 08/36] There is a button to add a new Agent directly in a Scenario; deleting an Agent from its show page no longer 404s; there is a button to destroy all delayed_jobs --- app/controllers/agents_controller.rb | 7 +++-- app/controllers/jobs_controller.rb | 9 ++++++ app/views/jobs/index.html.erb | 4 +++ app/views/scenarios/show.html.erb | 3 +- config/routes.rb | 1 + spec/controllers/agents_controller_spec.rb | 36 ++++++++++++++++------ spec/controllers/jobs_controller_spec.rb | 16 ++++++++-- 7 files changed, 60 insertions(+), 16 deletions(-) diff --git a/app/controllers/agents_controller.rb b/app/controllers/agents_controller.rb index 87845829..0c85a04a 100644 --- a/app/controllers/agents_controller.rb +++ b/app/controllers/agents_controller.rb @@ -148,6 +148,9 @@ class AgentsController < ApplicationController else @agent = agents.build end + + @agent.scenario_ids = [params[:scenario_id]] if params[:scenario_id] && current_user.scenarios.find_by(id: params[:scenario_id]) + initialize_presenter respond_to do |format| @@ -237,14 +240,14 @@ class AgentsController < ApplicationController if @agent && !@agent.destroyed? path = agent_path(@agent) end - when /\A#{Regexp::escape scenarios_path}\/\d+\Z/, agents_path + when /\A#{Regexp::escape scenarios_path}\/\d+\z/, agents_path path = ret end if path redirect_to path, notice: message else - super agents_path, notice: message + redirect_to agents_path, notice: message end end diff --git a/app/controllers/jobs_controller.rb b/app/controllers/jobs_controller.rb index 49befcbb..30ed39b6 100644 --- a/app/controllers/jobs_controller.rb +++ b/app/controllers/jobs_controller.rb @@ -48,6 +48,15 @@ class JobsController < ApplicationController end end + def destroy_all + Delayed::Job.delete_all + + respond_to do |format| + format.html { redirect_to jobs_path, notice: "All jobs removed." } + format.json { render json: '', status: :ok } + end + end + private def running? diff --git a/app/views/jobs/index.html.erb b/app/views/jobs/index.html.erb index 509a5e97..8ed88cc1 100644 --- a/app/views/jobs/index.html.erb +++ b/app/views/jobs/index.html.erb @@ -79,6 +79,10 @@ <%= link_to destroy_failed_jobs_path, class: "btn btn-default", method: :delete do %> Remove failed jobs <% end %> + + <%= link_to destroy_all_jobs_path, class: "btn btn-default", method: :delete, data: { confirm: "Are you sure you want to delete ALL pending jobs for all Huginn users?" } do %> + Remove all jobs + <% end %> diff --git a/app/views/scenarios/show.html.erb b/app/views/scenarios/show.html.erb index 6dded216..67461be9 100644 --- a/app/views/scenarios/show.html.erb +++ b/app/views/scenarios/show.html.erb @@ -16,10 +16,11 @@
<%= link_to icon_tag('glyphicon-chevron-left') + ' Back', scenarios_path, class: "btn btn-default" %> + <%= link_to icon_tag('glyphicon-plus') + ' New Agent', new_agent_path(scenario_id: @scenario.id), class: "btn btn-default" %> <%= link_to icon_tag('glyphicon-random') + ' View Diagram', scenario_diagram_path(@scenario), class: "btn btn-default" %> <%= link_to icon_tag('glyphicon-edit') + ' Edit', edit_scenario_path(@scenario), class: "btn btn-default" %> <% if @scenario.source_url.present? %> - <%= link_to icon_tag('glyphicon-plus') + ' Update', new_scenario_imports_path(url: @scenario.source_url), class: "btn btn-default" %> + <%= link_to icon_tag('glyphicon-refresh') + ' Update', new_scenario_imports_path(url: @scenario.source_url), class: "btn btn-default" %> <% end %> <%= link_to icon_tag('glyphicon-share-alt') + ' Share', share_scenario_path(@scenario), class: "btn btn-default" %> <%= link_to icon_tag('glyphicon-trash') + ' Delete', scenario_path(@scenario), method: :delete, data: { confirm: "This will remove the '#{@scenario.name}' Scenerio from all Agents and delete it. Are you sure?" }, class: "btn btn-default" %> diff --git a/config/routes.rb b/config/routes.rb index 7726fc4f..6a969c8a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -62,6 +62,7 @@ Huginn::Application.routes.draw do end collection do delete :destroy_failed + delete :destroy_all end end diff --git a/spec/controllers/agents_controller_spec.rb b/spec/controllers/agents_controller_spec.rb index ae3eb6fe..64c4c8ca 100644 --- a/spec/controllers/agents_controller_spec.rb +++ b/spec/controllers/agents_controller_spec.rb @@ -87,19 +87,35 @@ describe AgentsController do end end - describe "GET new with :id" do - it "opens a clone of a given Agent" do - sign_in users(:bob) - get :new, :id => agents(:bob_website_agent).to_param - expect(assigns(:agent).attributes).to eq(users(:bob).agents.build_clone(agents(:bob_website_agent)).attributes) + describe "GET new" do + describe "with :id" do + it "opens a clone of a given Agent" do + sign_in users(:bob) + get :new, :id => agents(:bob_website_agent).to_param + expect(assigns(:agent).attributes).to eq(users(:bob).agents.build_clone(agents(:bob_website_agent)).attributes) + end + + it "only allows the current user to clone his own Agent" do + sign_in users(:bob) + + expect { + get :new, :id => agents(:jane_website_agent).to_param + }.to raise_error(ActiveRecord::RecordNotFound) + end end - it "only allows the current user to clone his own Agent" do - sign_in users(:bob) + describe "with a scenario_id" do + it 'populates the assigned agent with the scenario' do + sign_in users(:bob) + get :new, :scenario_id => scenarios(:bob_weather).id + expect(assigns(:agent).scenario_ids).to eq([scenarios(:bob_weather).id]) + end - expect { - get :new, :id => agents(:jane_website_agent).to_param - }.to raise_error(ActiveRecord::RecordNotFound) + it "does not see other user's scenarios" do + sign_in users(:bob) + get :new, :scenario_id => scenarios(:jane_weather).id + expect(assigns(:agent).scenario_ids).to eq([]) + end end end diff --git a/spec/controllers/jobs_controller_spec.rb b/spec/controllers/jobs_controller_spec.rb index 5aa532be..a9f97ba1 100644 --- a/spec/controllers/jobs_controller_spec.rb +++ b/spec/controllers/jobs_controller_spec.rb @@ -1,7 +1,6 @@ require 'spec_helper' describe JobsController do - describe "GET index" do before do async_handler_yaml = @@ -75,8 +74,19 @@ describe JobsController do end it "just destroy failed jobs" do - expect { delete :destroy_failed, id: @failed.id }.to change(Delayed::Job, :count).by(-1) - expect { delete :destroy_failed, id: @running.id }.to change(Delayed::Job, :count).by(0) + expect { delete :destroy_failed }.to change(Delayed::Job, :count).by(-1) + end + end + + describe "DELETE destroy_all" do + before do + @failed = Delayed::Job.create(failed_at: Time.now - 1.minute) + @running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test') + sign_in users(:jane) + end + + it "destroys all jobs" do + expect { delete :destroy_all }.to change(Delayed::Job, :count).by(-2) end end end From 177009a5bbfd5d0d998f30dfd75d59f3182bb93b Mon Sep 17 00:00:00 2001 From: Vlad Date: Sat, 15 Aug 2015 15:11:36 +0300 Subject: [PATCH 09/36] fix exclude_replies? --- app/models/agents/twitter_user_agent.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index f435a3e7..902386b6 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -90,7 +90,7 @@ module Agents end def exclude_replies? - interpolated[:exclude_replies] != "false" + boolify(interpolated[:exclude_replies]) || false end def check From 266a10132b96ddd063688ef70aade7f7e636d7ec Mon Sep 17 00:00:00 2001 From: Vlad Date: Sat, 15 Aug 2015 15:21:08 +0300 Subject: [PATCH 10/36] Update twitter_user_agent_spec.rb --- spec/models/agents/twitter_user_agent_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/models/agents/twitter_user_agent_spec.rb b/spec/models/agents/twitter_user_agent_spec.rb index 6f7be07e..3a68f751 100644 --- a/spec/models/agents/twitter_user_agent_spec.rb +++ b/spec/models/agents/twitter_user_agent_spec.rb @@ -7,6 +7,8 @@ describe Agents::TwitterUserAgent do @opts = { :username => "tectonic", + :include_retweets => "true", + :exclude_replies => "false", :expected_update_period_in_days => "2", :starting_at => "Jan 01 00:00:01 +0000 2000", :consumer_key => "---", From 5f731d55bf7a091d6a04a951b0c1861083b20135 Mon Sep 17 00:00:00 2001 From: Will Read Date: Sat, 15 Aug 2015 13:56:32 -0700 Subject: [PATCH 11/36] Supress pop up emails - Emails will only be visible by visiting /letter_opener - Helps with those who leave Huginn running on their dev machine --- config/environments/development.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/development.rb b/config/environments/development.rb index ce375e9f..adc970d8 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -43,7 +43,7 @@ Huginn::Application.configure do if ENV['SEND_EMAIL_IN_DEVELOPMENT'] == 'true' config.action_mailer.delivery_method = :smtp else - config.action_mailer.delivery_method = :letter_opener + config.action_mailer.delivery_method = :letter_opener_web end # smtp_settings moved to config/initializers/action_mailer.rb end From 8713dce4d806225a6a4928d9711f10dabb7e7df5 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 16 Aug 2015 16:05:22 -0700 Subject: [PATCH 12/36] integrate @knu's suggestions --- app/controllers/agents_controller.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/controllers/agents_controller.rb b/app/controllers/agents_controller.rb index 0c85a04a..892a5dbc 100644 --- a/app/controllers/agents_controller.rb +++ b/app/controllers/agents_controller.rb @@ -239,6 +239,8 @@ class AgentsController < ApplicationController when "show" if @agent && !@agent.destroyed? path = agent_path(@agent) + else + path = agents_path end when /\A#{Regexp::escape scenarios_path}\/\d+\z/, agents_path path = ret @@ -247,7 +249,7 @@ class AgentsController < ApplicationController if path redirect_to path, notice: message else - redirect_to agents_path, notice: message + super agents_path, notice: message end end From 50ccc2ad34d61e7220ca39bcb6dafabc6f288109 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 16 Aug 2015 16:16:04 -0700 Subject: [PATCH 13/36] remove spring loads too --- bin/rails | 4 ---- bin/rake | 4 ---- bin/rspec | 4 ---- 3 files changed, 12 deletions(-) diff --git a/bin/rails b/bin/rails index 7feb6a30..728cd85a 100755 --- a/bin/rails +++ b/bin/rails @@ -1,8 +1,4 @@ #!/usr/bin/env ruby -begin - load File.expand_path("../spring", __FILE__) -rescue LoadError -end APP_PATH = File.expand_path('../../config/application', __FILE__) require_relative '../config/boot' require 'rails/commands' diff --git a/bin/rake b/bin/rake index 8017a027..17240489 100755 --- a/bin/rake +++ b/bin/rake @@ -1,8 +1,4 @@ #!/usr/bin/env ruby -begin - load File.expand_path("../spring", __FILE__) -rescue LoadError -end require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/rspec b/bin/rspec index 20060ebd..d72fadf3 100755 --- a/bin/rspec +++ b/bin/rspec @@ -1,7 +1,3 @@ #!/usr/bin/env ruby -begin - load File.expand_path("../spring", __FILE__) -rescue LoadError -end require 'bundler/setup' load Gem.bin_path('rspec-core', 'rspec') From dc06e3aed0a864e0012680f2adf4230c47ef033a Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 17 Aug 2015 00:01:04 -0700 Subject: [PATCH 14/36] do not delete running jobs --- app/controllers/jobs_controller.rb | 2 +- spec/controllers/jobs_controller_spec.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/jobs_controller.rb b/app/controllers/jobs_controller.rb index 30ed39b6..6f117ac1 100644 --- a/app/controllers/jobs_controller.rb +++ b/app/controllers/jobs_controller.rb @@ -49,7 +49,7 @@ class JobsController < ApplicationController end def destroy_all - Delayed::Job.delete_all + Delayed::Job.where(locked_at: nil).delete_all respond_to do |format| format.html { redirect_to jobs_path, notice: "All jobs removed." } diff --git a/spec/controllers/jobs_controller_spec.rb b/spec/controllers/jobs_controller_spec.rb index a9f97ba1..a1b675fc 100644 --- a/spec/controllers/jobs_controller_spec.rb +++ b/spec/controllers/jobs_controller_spec.rb @@ -70,6 +70,7 @@ describe JobsController do before do @failed = Delayed::Job.create(failed_at: Time.now - 1.minute) @running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test') + @pending = Delayed::Job.create sign_in users(:jane) end @@ -82,11 +83,13 @@ describe JobsController do before do @failed = Delayed::Job.create(failed_at: Time.now - 1.minute) @running = Delayed::Job.create(locked_at: Time.now, locked_by: 'test') + @pending = Delayed::Job.create sign_in users(:jane) end it "destroys all jobs" do expect { delete :destroy_all }.to change(Delayed::Job, :count).by(-2) + expect(Delayed::Job.find(@running.id)).to be end end end From ff050dd9be58eb98670bac86d190d35792f66231 Mon Sep 17 00:00:00 2001 From: Vlad Date: Mon, 17 Aug 2015 10:54:10 +0300 Subject: [PATCH 15/36] Skip exclude_replies validation --- app/models/agents/twitter_user_agent.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index 902386b6..1879129a 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -67,10 +67,6 @@ module Agents if options[:include_retweets].present? && !%w[true false].include?(options[:include_retweets]) errors.add(:base, "include_retweets must be a boolean value string (true/false)") end - - if options[:exclude_replies].present? && !%w[true false].include?(options[:exclude_replies]) - errors.add(:base, "exclude_replies must be a boolean value string (true/false)") - end if options[:starting_at].present? Time.parse(options[:starting_at]) rescue errors.add(:base, "Error parsing starting_at") From e9ac0295cca9e5504b93dbf8ee8b4aa91a6632d6 Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Mon, 17 Aug 2015 17:15:05 -0500 Subject: [PATCH 16/36] Updated agent picker to properly display/search descriptions that return procs --- .../javascripts/pages/agent-edit-page.js.coffee | 13 +++++++------ app/assets/stylesheets/application.css.scss.erb | 5 +++++ app/models/agents/user_location_agent.rb | 4 +--- app/views/agents/_form.html.erb | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/app/assets/javascripts/pages/agent-edit-page.js.coffee b/app/assets/javascripts/pages/agent-edit-page.js.coffee index ff0fda27..0ecb4bb7 100644 --- a/app/assets/javascripts/pages/agent-edit-page.js.coffee +++ b/app/assets/javascripts/pages/agent-edit-page.js.coffee @@ -16,6 +16,8 @@ class @AgentEditPage if $("#agent_type").length $("#agent_type").on "change", => @handleTypeChange(false) @handleTypeChange(true) + + # Update the dropdown to match agent description as well as agent name $('#agent_type').select2 width: 'resolve' formatResult: formatAgentForSelect @@ -29,12 +31,6 @@ class @AgentEditPage @enableDryRunButton() @buildAce() - formatAgentForSelect = (agent) -> - originalOption = agent.element - description = $(originalOption).attr('title') - description = if /^[a-zA-Z]/.test(description) then description else '' - '' + agent.text + '
' + description - handleTypeChange: (firstTime) -> $(".event-descriptions").html("").hide() type = $('#agent_type').val() @@ -192,5 +188,10 @@ class @AgentEditPage @updateFromEditors() Utils.handleDryRunButton(e.target) + formatAgentForSelect = (agent) -> + originalOption = agent.element + description = agent.element[0].title + '' + agent.text + '
' + description + $ -> Utils.registerPage(AgentEditPage, forPathsMatching: /^agents/) diff --git a/app/assets/stylesheets/application.css.scss.erb b/app/assets/stylesheets/application.css.scss.erb index 5c606999..fd4319e7 100644 --- a/app/assets/stylesheets/application.css.scss.erb +++ b/app/assets/stylesheets/application.css.scss.erb @@ -304,3 +304,8 @@ $service-colors: #55acee #8fc857 #444444 #2c4762 #007EE5 .label-service { @include services; } + +.select2-highlighted a { + color: yellow; + text-decoration: underline; +} diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb index cb220995..9add8bc5 100644 --- a/app/models/agents/user_location_agent.rb +++ b/app/models/agents/user_location_agent.rb @@ -7,12 +7,10 @@ module Agents gem_dependency_check { defined?(Haversine) } description do <<-MD - The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location. + The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location to `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. #{'## Include `haversine` in your Gemfile to use this Agent!' if dependencies_missing?} - Your POST path will be `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. - If you want to only keep more precise locations, set `max_accuracy` to the upper bound, in meters. The default name for this field is `accuracy`, but you can change this by setting a value for `accuracy_field`. If you want to require a certain distance traveled, set `min_distance` to the minimum distance, in meters. Note that GPS readings and the measurement itself aren't exact, so don't rely on this for precision filtering. diff --git a/app/views/agents/_form.html.erb b/app/views/agents/_form.html.erb index 0121d182..79755c95 100644 --- a/app/views/agents/_form.html.erb +++ b/app/views/agents/_form.html.erb @@ -23,7 +23,7 @@ <% if @agent.new_record? %>
<%= f.label :type %> - <%= f.select :type, options_for_select([['Select an Agent Type', 'Agent', {title: ''}]] + Agent.types.map {|type| [type.name.gsub(/^.*::/, '').underscore.humanize.titleize, type, {title: type.description.to_s.strip.split(/\r?\n/)[0]}] }, @agent.type), {}, :class => 'form-control' %> + <%= f.select :type, options_for_select([['Select an Agent Type', 'Agent', {title: ''}]] + Agent.types.map {|type| [type.name.gsub(/^.*::/, '').underscore.humanize.titleize, type, {title: h(Agent.build_for_type(type.name,current_user,{}).html_description.lines.first.strip)}] }, @agent.type), {}, :class => 'form-control' %>
<% end %>
From 7b85bad2675955760dc262d8b77fedbb14e9a194 Mon Sep 17 00:00:00 2001 From: Dustin Miller Date: Tue, 18 Aug 2015 10:41:32 -0500 Subject: [PATCH 17/36] Cleaned up Agent descriptions: consistent format for their beginning. --- app/models/agents/adioso_agent.rb | 1 + app/models/agents/basecamp_agent.rb | 3 +- app/models/agents/change_detector_agent.rb | 2 +- app/models/agents/commander_agent.rb | 2 +- app/models/agents/data_output_agent.rb | 2 +- app/models/agents/de_duplication_agent.rb | 2 +- app/models/agents/dropbox_file_url_agent.rb | 2 +- app/models/agents/dropbox_watch_agent.rb | 3 +- app/models/agents/email_agent.rb | 2 +- app/models/agents/email_digest_agent.rb | 2 +- app/models/agents/event_formatting_agent.rb | 2 +- app/models/agents/ftpsite_agent.rb | 4 +- .../agents/google_calendar_publish_agent.rb | 3 +- app/models/agents/growl_agent.rb | 3 +- app/models/agents/hipchat_agent.rb | 3 +- app/models/agents/human_task_agent.rb | 3 +- app/models/agents/imap_folder_agent.rb | 80 +++++-------------- app/models/agents/jabber_agent.rb | 3 +- app/models/agents/java_script_agent.rb | 2 +- app/models/agents/jira_agent.rb | 9 ++- app/models/agents/manual_event_agent.rb | 2 +- app/models/agents/mqtt_agent.rb | 3 +- app/models/agents/pdf_info_agent.rb | 2 + app/models/agents/peak_detector_agent.rb | 6 +- app/models/agents/post_agent.rb | 2 +- app/models/agents/public_transport_agent.rb | 6 +- app/models/agents/pushover_agent.rb | 2 +- app/models/agents/rss_agent.rb | 2 +- app/models/agents/scheduler_agent.rb | 2 +- app/models/agents/sentiment_agent.rb | 5 +- app/models/agents/shell_command_agent.rb | 2 +- app/models/agents/slack_agent.rb | 19 ++--- app/models/agents/stubhub_agent.rb | 4 +- app/models/agents/translation_agent.rb | 4 +- app/models/agents/trigger_agent.rb | 2 +- app/models/agents/tumblr_publish_agent.rb | 4 +- app/models/agents/twilio_agent.rb | 3 +- app/models/agents/twitter_publish_agent.rb | 2 +- app/models/agents/twitter_stream_agent.rb | 2 +- app/models/agents/twitter_user_agent.rb | 2 +- app/models/agents/user_location_agent.rb | 2 +- app/models/agents/weather_agent.rb | 3 +- app/models/agents/webhook_agent.rb | 9 ++- app/models/agents/website_agent.rb | 2 +- app/models/agents/weibo_publish_agent.rb | 7 +- app/models/agents/weibo_user_agent.rb | 3 +- app/models/agents/witai_agent.rb | 5 +- 47 files changed, 111 insertions(+), 129 deletions(-) diff --git a/app/models/agents/adioso_agent.rb b/app/models/agents/adioso_agent.rb index 58d01faa..a931de37 100644 --- a/app/models/agents/adioso_agent.rb +++ b/app/models/agents/adioso_agent.rb @@ -6,6 +6,7 @@ module Agents description <<-MD The Adioso Agent will tell you the minimum airline prices between a pair of cities, and within a certain period of time. + The currency is USD. Please make sure that the difference between `start_date` and `end_date` is less than 150 days. You will need to contact [Adioso](http://adioso.com/) for a `username` and `password`. MD diff --git a/app/models/agents/basecamp_agent.rb b/app/models/agents/basecamp_agent.rb index 14772298..cd505720 100644 --- a/app/models/agents/basecamp_agent.rb +++ b/app/models/agents/basecamp_agent.rb @@ -7,10 +7,9 @@ module Agents cannot_receive_events! description <<-MD - The BasecampAgent checks a Basecamp project for new Events + The Basecamp Agent checks a Basecamp project for new Events To be able to use this Agent you need to authenticate with 37signals in the [Services](/services) section first. - MD event_description <<-MD diff --git a/app/models/agents/change_detector_agent.rb b/app/models/agents/change_detector_agent.rb index 22ae469e..9a605ee1 100644 --- a/app/models/agents/change_detector_agent.rb +++ b/app/models/agents/change_detector_agent.rb @@ -3,7 +3,7 @@ module Agents cannot_be_scheduled! description <<-MD - The ChangeDetectorAgent receives a stream of events and emits a new event when a property of the received event changes. + The Change Detector Agent receives a stream of events and emits a new event when a property of the received event changes. `property` specifies the property to be watched. diff --git a/app/models/agents/commander_agent.rb b/app/models/agents/commander_agent.rb index 35c9837a..18aa7d56 100644 --- a/app/models/agents/commander_agent.rb +++ b/app/models/agents/commander_agent.rb @@ -5,7 +5,7 @@ module Agents cannot_create_events! description <<-MD - This agent is triggered by schedule or an incoming event and commands other agents ("targets") to run, disable, configure, or enable themselves. + The Commander Agent is triggered by schedule or an incoming event, and commands other agents ("targets") to run, disable, configure, or enable themselves. # Action types diff --git a/app/models/agents/data_output_agent.rb b/app/models/agents/data_output_agent.rb index 486cc3fb..07b8bd9b 100644 --- a/app/models/agents/data_output_agent.rb +++ b/app/models/agents/data_output_agent.rb @@ -4,7 +4,7 @@ module Agents description do <<-MD - The Agent outputs received events as either RSS or JSON. Use it to output a public or private stream of Huginn data. + The Data Output Agent outputs received events as either RSS or JSON. Use it to output a public or private stream of Huginn data. This Agent will output data at: diff --git a/app/models/agents/de_duplication_agent.rb b/app/models/agents/de_duplication_agent.rb index 3de2afe5..463bd548 100644 --- a/app/models/agents/de_duplication_agent.rb +++ b/app/models/agents/de_duplication_agent.rb @@ -4,7 +4,7 @@ module Agents cannot_be_scheduled! description <<-MD - The DeDuplicationAgent receives a stream of events and remits the event if it is not a duplicate. + The De-duplication Agent receives a stream of events and remits the event if it is not a duplicate. `property` the value that should be used to determine the uniqueness of the event (empty to use the whole payload) diff --git a/app/models/agents/dropbox_file_url_agent.rb b/app/models/agents/dropbox_file_url_agent.rb index f6109ae8..d74909f8 100644 --- a/app/models/agents/dropbox_file_url_agent.rb +++ b/app/models/agents/dropbox_file_url_agent.rb @@ -5,7 +5,7 @@ module Agents cannot_be_scheduled! description <<-MD - The _DropboxFileUrlAgent_ is used to work with Dropbox. It takes a file path (or multiple files paths) and emits events with [temporary links](https://www.dropbox.com/developers/core/docs#media). + The Dropbox File Url Agent is used to work with Dropbox. It takes a file path (or multiple file paths) and emits events with [temporary links](https://www.dropbox.com/developers/core/docs#media). #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?} diff --git a/app/models/agents/dropbox_watch_agent.rb b/app/models/agents/dropbox_watch_agent.rb index ef711a1e..8edecd3a 100644 --- a/app/models/agents/dropbox_watch_agent.rb +++ b/app/models/agents/dropbox_watch_agent.rb @@ -6,7 +6,8 @@ module Agents default_schedule "every_1m" description <<-MD - The _DropboxWatchAgent_ watches the given `dir_to_watch` and emits events with the detected changes. + The Dropbox Watch Agent watches the given `dir_to_watch` and emits events with the detected changes. + #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?} MD diff --git a/app/models/agents/email_agent.rb b/app/models/agents/email_agent.rb index 91667337..b3943bb1 100644 --- a/app/models/agents/email_agent.rb +++ b/app/models/agents/email_agent.rb @@ -6,7 +6,7 @@ module Agents cannot_create_events! description <<-MD - The EmailAgent sends any events it receives via email immediately. + The Email Agent sends any events it receives via email immediately. You can specify the email's subject line by providing a `subject` option, which can contain Liquid formatting. E.g., you could provide `"Huginn email"` to set a simple subject, or `{{subject}}` to use the `subject` key from the incoming Event. diff --git a/app/models/agents/email_digest_agent.rb b/app/models/agents/email_digest_agent.rb index 1e467b54..0472879d 100644 --- a/app/models/agents/email_digest_agent.rb +++ b/app/models/agents/email_digest_agent.rb @@ -7,7 +7,7 @@ module Agents cannot_create_events! description <<-MD - The EmailDigestAgent collects any Events sent to it and sends them all via email when scheduled. + The Email Digest Agent collects any Events sent to it and sends them all via email when scheduled. By default, the will have a `subject` and an optional `headline` before listing the Events. If the Events' payloads contain a `message`, that will be highlighted, otherwise everything in diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb index 5743e04a..9dd4a3c4 100644 --- a/app/models/agents/event_formatting_agent.rb +++ b/app/models/agents/event_formatting_agent.rb @@ -3,7 +3,7 @@ module Agents cannot_be_scheduled! description <<-MD - An Event Formatting Agent allows you to format incoming Events, adding new fields as needed. + The Event Formatting Agent allows you to format incoming Events, adding new fields as needed. For example, here is a possible Event: diff --git a/app/models/agents/ftpsite_agent.rb b/app/models/agents/ftpsite_agent.rb index 0dceb3c5..1a9ed6ff 100644 --- a/app/models/agents/ftpsite_agent.rb +++ b/app/models/agents/ftpsite_agent.rb @@ -9,8 +9,10 @@ module Agents gem_dependency_check { defined?(Net::FTP) && defined?(Net::FTP::List) } description <<-MD + The FTP Site Agent checks an FTP site and creates Events based on newly uploaded files in a directory. + #{'## Include `net-ftp-list` in your Gemfile to use this Agent!' if dependencies_missing?} - The FtpsiteAgent checks a FTP site and creates Events based on newly uploaded files in a directory. + Specify a `url` that represents a directory of an FTP site to watch, and a list of `patterns` to match against file names. diff --git a/app/models/agents/google_calendar_publish_agent.rb b/app/models/agents/google_calendar_publish_agent.rb index 5ad4e333..5a939bb7 100644 --- a/app/models/agents/google_calendar_publish_agent.rb +++ b/app/models/agents/google_calendar_publish_agent.rb @@ -7,8 +7,9 @@ module Agents gem_dependency_check { defined?(Google) && defined?(Google::APIClient) } description <<-MD + The Google Calendar Publish Agent creates events on your Google Calendar. + #{'## Include `google-api-client` in your Gemfile to use this Agent!' if dependencies_missing?} - The GoogleCalendarPublishAgent creates events on your google calendar. This agent relies on service accounts, rather than oauth. diff --git a/app/models/agents/growl_agent.rb b/app/models/agents/growl_agent.rb index 4fb7931a..b2274802 100644 --- a/app/models/agents/growl_agent.rb +++ b/app/models/agents/growl_agent.rb @@ -8,8 +8,9 @@ module Agents gem_dependency_check { defined?(Growl) } description <<-MD + The Growl Agent sends any events it receives to a Growl GNTP server immediately. + #{'## Include `ruby-growl` in your Gemfile to use this Agent!' if dependencies_missing?} - The GrowlAgent sends any events it receives to a Growl GNTP server immediately. It is assumed that events have a `message` or `text` key, which will hold the body of the growl notification, and a `subject` key, which will have the headline of the Growl notification. You can use Event Formatting Agent if your event does not provide these keys. diff --git a/app/models/agents/hipchat_agent.rb b/app/models/agents/hipchat_agent.rb index ee455a86..eac7437e 100644 --- a/app/models/agents/hipchat_agent.rb +++ b/app/models/agents/hipchat_agent.rb @@ -8,8 +8,9 @@ module Agents gem_dependency_check { defined?(HipChat) } description <<-MD + The Hipchat Agent sends messages to a Hipchat Room + #{'## Include `hipchat` in your Gemfile to use this Agent!' if dependencies_missing?} - The HipchatAgent sends messages to a Hipchat Room To authenticate you need to set the `auth_token`, you can get one at your Hipchat Group Admin page which you can find here: diff --git a/app/models/agents/human_task_agent.rb b/app/models/agents/human_task_agent.rb index 2a130b4d..467fcd54 100644 --- a/app/models/agents/human_task_agent.rb +++ b/app/models/agents/human_task_agent.rb @@ -5,8 +5,9 @@ module Agents gem_dependency_check { defined?(RTurk) } description <<-MD + The Human Task Agent is used to create Human Intelligence Tasks (HITs) on Mechanical Turk. + #{'## Include `rturk` in your Gemfile to use this Agent!' if dependencies_missing?} - You can use a HumanTaskAgent to create Human Intelligence Tasks (HITs) on Mechanical Turk. HITs can be created in response to events, or on a schedule. Set `trigger_on` to either `schedule` or `event`. diff --git a/app/models/agents/imap_folder_agent.rb b/app/models/agents/imap_folder_agent.rb index 1bc0c7dc..50a13c6f 100644 --- a/app/models/agents/imap_folder_agent.rb +++ b/app/models/agents/imap_folder_agent.rb @@ -11,90 +11,52 @@ module Agents default_schedule "every_30m" description <<-MD + The Imap Folder Agent checks an IMAP server in specified folders and creates Events based on new mails found since the last run. In the first visit to a folder, this agent only checks for the initial status and does not create events. - The ImapFolderAgent checks an IMAP server in specified folders - and creates Events based on new mails found since the last run. - In the first visit to a folder, this agent only checks for the - initial status and does not create events. - - Specify an IMAP server to connect with `host`, and set `ssl` to - true if the server supports IMAP over SSL. Specify `port` if - you need to connect to a port other than standard (143 or 993 - depending on the `ssl` value). + Specify an IMAP server to connect with `host`, and set `ssl` to true if the server supports IMAP over SSL. Specify `port` if you need to connect to a port other than standard (143 or 993 depending on the `ssl` value). Specify login credentials in `username` and `password`. List the names of folders to check in `folders`. - To narrow mails by conditions, build a `conditions` hash with - the following keys: + To narrow mails by conditions, build a `conditions` hash with the following keys: - - "subject" - - "body" + - `subject` + - `body` + Specify a regular expression to match against the decoded subject/body of each mail. - Specify a regular expression to match against the decoded - subject/body of each mail. + Use the `(?i)` directive for case-insensitive search. For example, a pattern `(?i)alert` will match "alert", "Alert"or "ALERT". You can also make only a part of a pattern to work case-insensitively: `Re: (?i:alert)` will match either "Re: Alert" or "Re: alert", but not "RE: alert". - Use the `(?i)` directive for case-insensitive search. For - example, a pattern `(?i)alert` will match "alert", "Alert" - or "ALERT". You can also make only a part of a pattern to - work case-insensitively: `Re: (?i:alert)` will match either - "Re: Alert" or "Re: alert", but not "RE: alert". + When a mail has multiple non-attachment text parts, they are prioritized according to the `mime_types` option (which see below) and the first part that matches a "body" pattern, if specified, will be chosen as the "body" value in a created event. - When a mail has multiple non-attachment text parts, they are - prioritized according to the `mime_types` option (which see - below) and the first part that matches a "body" pattern, if - specified, will be chosen as the "body" value in a created - event. + Named captures will appear in the "matches" hash in a created event. - Named captures will appear in the "matches" hash in a - created event. - - - "from", "to", "cc" - - Specify a shell glob pattern string that is matched against - mail addresses extracted from the corresponding header - values of each mail. + - `from`, `to`, `cc` + Specify a shell glob pattern string that is matched against mail addresses extracted from the corresponding header values of each mail. Patterns match addresses in case insensitive manner. - Multiple pattern strings can be specified in an array, in - which case a mail is selected if any of the patterns - matches. (i.e. patterns are OR'd) + Multiple pattern strings can be specified in an array, in which case a mail is selected if any of the patterns matches. (i.e. patterns are OR'd) - - "mime_types" + - `mime_types` + Specify an array of MIME types to tell which non-attachment part of a mail among its text/* parts should be used as mail body. The default value is `['text/plain', 'text/enriched', 'text/html']`. - Specify an array of MIME types to tell which non-attachment - part of a mail among its text/* parts should be used as mail - body. The default value is `['text/plain', 'text/enriched', - 'text/html']`. - - - "is_unread" - - Setting this to true or false means only mails that is - marked as unread or read respectively, are selected. + - `is_unread` + Setting this to true or false means only mails that is marked as unread or read respectively, are selected. If this key is unspecified or set to null, it is ignored. - - "has_attachment" - - Setting this to true or false means only mails that does or does - not have an attachment are selected. + - `has_attachment` + + Setting this to true or false means only mails that does or does not have an attachment are selected. If this key is unspecified or set to null, it is ignored. Set `mark_as_read` to true to mark found mails as read. - Each agent instance memorizes the highest UID of mails that are - found in the last run for each watched folder, so even if you - change a set of conditions so that it matches mails that are - missed previously, or if you alter the flag status of already - found mails, they will not show up as new events. + Each agent instance memorizes the highest UID of mails that are found in the last run for each watched folder, so even if you change a set of conditions so that it matches mails that are missed previously, or if you alter the flag status of already found mails, they will not show up as new events. - Also, in order to avoid duplicated notification it keeps a list - of Message-Id's of 100 most recent mails, so if multiple mails - of the same Message-Id are found, you will only see one event - out of them. + Also, in order to avoid duplicated notification it keeps a list of Message-Id's of 100 most recent mails, so if multiple mails of the same Message-Id are found, you will only see one event out of them. MD event_description <<-MD diff --git a/app/models/agents/jabber_agent.rb b/app/models/agents/jabber_agent.rb index c8f82952..73d0b613 100644 --- a/app/models/agents/jabber_agent.rb +++ b/app/models/agents/jabber_agent.rb @@ -6,8 +6,9 @@ module Agents gem_dependency_check { defined?(Jabber) } description <<-MD + The Jabber Agent will send any events it receives to your Jabber/XMPP IM account. + #{'## Include `xmpp4r` in your Gemfile to use this Agent!' if dependencies_missing?} - The JabberAgent will send any events it receives to your Jabber/XMPP IM account. Specify the `jabber_server` and `jabber_port` for your Jabber server. diff --git a/app/models/agents/java_script_agent.rb b/app/models/agents/java_script_agent.rb index 70dc082a..1b38c8d4 100644 --- a/app/models/agents/java_script_agent.rb +++ b/app/models/agents/java_script_agent.rb @@ -10,7 +10,7 @@ module Agents default_schedule "never" description <<-MD - This Agent allows you to write code in JavaScript that can create and receive events. If other Agents aren't meeting your needs, try this one! + The JavaScript Agent allows you to write code in JavaScript that can create and receive events. If other Agents aren't meeting your needs, try this one! You can put code in the `code` option, or put your code in a Credential and reference it from `code` with `credential:` (recommended). diff --git a/app/models/agents/jira_agent.rb b/app/models/agents/jira_agent.rb index 4ed981c4..a55a8eab 100644 --- a/app/models/agents/jira_agent.rb +++ b/app/models/agents/jira_agent.rb @@ -11,12 +11,13 @@ module Agents description <<-MD The Jira Agent subscribes to Jira issue updates. - `jira_url` specifies the full URL of the jira installation, including https:// - `jql` is an optional Jira Query Language-based filter to limit the flow of events. See [JQL Docs](https://confluence.atlassian.com/display/JIRA/Advanced+Searching) for details. - `username` and `password` are optional, and may need to be specified if your Jira instance is read-protected - `timeout` is an optional parameter that specifies how long the request processing may take in minutes. + - `jira_url` specifies the full URL of the jira installation, including https:// + - `jql` is an optional Jira Query Language-based filter to limit the flow of events. See [JQL Docs](https://confluence.atlassian.com/display/JIRA/Advanced+Searching) for details. + - `username` and `password` are optional, and may need to be specified if your Jira instance is read-protected + - `timeout` is an optional parameter that specifies how long the request processing may take in minutes. The agent does periodic queries and emits the events containing the updated issues in JSON format. + NOTE: upon the first execution, the agent will fetch everything available by the JQL query. So if it's not desirable, limit the `jql` query by date. MD diff --git a/app/models/agents/manual_event_agent.rb b/app/models/agents/manual_event_agent.rb index 7c045a97..92bf5ae2 100644 --- a/app/models/agents/manual_event_agent.rb +++ b/app/models/agents/manual_event_agent.rb @@ -4,7 +4,7 @@ module Agents cannot_receive_events! description <<-MD - Use this Agent to manually create Events for testing or other purposes. + The Manual Event Agent is used to manually create Events for testing or other purposes. MD event_description "User determined" diff --git a/app/models/agents/mqtt_agent.rb b/app/models/agents/mqtt_agent.rb index e60c575d..29967d95 100644 --- a/app/models/agents/mqtt_agent.rb +++ b/app/models/agents/mqtt_agent.rb @@ -6,8 +6,9 @@ module Agents gem_dependency_check { defined?(MQTT) } description <<-MD + The MQTT Agent allows both publication and subscription to an MQTT topic. + #{'## Include `mqtt` in your Gemfile to use this Agent!' if dependencies_missing?} - The MQTT agent allows both publication and subscription to an MQTT topic. MQTT is a generic transport protocol for machine to machine communication. diff --git a/app/models/agents/pdf_info_agent.rb b/app/models/agents/pdf_info_agent.rb index f3aad0bd..37d3420d 100644 --- a/app/models/agents/pdf_info_agent.rb +++ b/app/models/agents/pdf_info_agent.rb @@ -9,6 +9,8 @@ module Agents cannot_be_scheduled! description <<-MD + The PDF Info Agent returns the metadata contained within a given PDF file, using HyPDF. + #{'## Include the `hypdf` gem in your `Gemfile` to use PDFInfo Agents.' if dependencies_missing?} In order for this agent to work, you need to have [HyPDF](https://devcenter.heroku.com/articles/hypdf) running and configured. diff --git a/app/models/agents/peak_detector_agent.rb b/app/models/agents/peak_detector_agent.rb index 02e19c37..7c8f66a5 100644 --- a/app/models/agents/peak_detector_agent.rb +++ b/app/models/agents/peak_detector_agent.rb @@ -5,15 +5,13 @@ module Agents cannot_be_scheduled! description <<-MD - Use a PeakDetectorAgent to watch for peaks in an event stream. When a peak is detected, the resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: {{foo.bar}}`, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details. + The Peak Detector Agent will watch for peaks in an event stream. When a peak is detected, the resulting Event will have a payload message of `message`. You can include extractions in the message, for example: `I saw a bar of: {{foo.bar}}`, have a look at the [Wiki](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) for details. The `value_path` value is a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value of interest. `group_by_path` is a hash path that will be used to group values, if present. Set `expected_receive_period_in_days` to the maximum amount of time that you'd expect to pass between Events being received by this Agent. - You may set `window_duration_in_days` to change the default memory window length of `14` days, - `min_peak_spacing_in_days` to change the default minimum peak spacing of `2` days (peaks closer together will be ignored), and - `std_multiple` to change the default standard deviation threshold multiple of `3`. + You may set `window_duration_in_days` to change the default memory window length of `14` days, `min_peak_spacing_in_days` to change the default minimum peak spacing of `2` days (peaks closer together will be ignored), and `std_multiple` to change the default standard deviation threshold multiple of `3`. MD event_description <<-MD diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index 25e1a591..c984dbe3 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -7,7 +7,7 @@ module Agents default_schedule "never" description <<-MD - A PostAgent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`. + A Post Agent receives events from other agents (or runs periodically), merges those events with the [Liquid-interpolated](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) contents of `payload`, and sends the results as POST (or GET) requests to a specified url. To skip merging in the incoming event, but still send the interpolated payload, set `no_merge` to `true`. The `post_url` field must specify where you would like to send requests. Please include the URI scheme (`http` or `https`). diff --git a/app/models/agents/public_transport_agent.rb b/app/models/agents/public_transport_agent.rb index cac6bc58..e246a3e3 100644 --- a/app/models/agents/public_transport_agent.rb +++ b/app/models/agents/public_transport_agent.rb @@ -7,13 +7,15 @@ module Agents default_schedule "every_2m" description <<-MD + The Public Transport Request Agent generates Events based on NextBus GPS transit predictions. + Specify the following user settings: - * stops (array) * agency (string) + * stops (array) * alert_window_in_minutes (integer) - This Agent generates Events based on NextBus GPS transit predictions. First, select an agency by visiting [http://www.nextbus.com/predictor/agencySelector.jsp](http://www.nextbus.com/predictor/agencySelector.jsp) and finding your transit system. Once you find it, copy the part of the URL after `?a=`. For example, for the San Francisco MUNI system, you would end up on [http://www.nextbus.com/predictor/stopSelector.jsp?a=**sf-muni**](http://www.nextbus.com/predictor/stopSelector.jsp?a=sf-muni) and copy "sf-muni". Put that into this Agent's agency setting. + First, select an agency by visiting [http://www.nextbus.com/predictor/agencySelector.jsp](http://www.nextbus.com/predictor/agencySelector.jsp) and finding your transit system. Once you find it, copy the part of the URL after `?a=`. For example, for the San Francisco MUNI system, you would end up on [http://www.nextbus.com/predictor/stopSelector.jsp?a=**sf-muni**](http://www.nextbus.com/predictor/stopSelector.jsp?a=sf-muni) and copy "sf-muni". Put that into this Agent's agency setting. Next, find the stop tags that you care about. To find the tags for the sf-muni system, for the N route, visit this URL: [http://webservices.nextbus.com/service/publicXMLFeed?command=routeConfig&a=sf-muni&r=**N**](http://webservices.nextbus.com/service/publicXMLFeed?command=routeConfig&a=sf-muni&r=N) diff --git a/app/models/agents/pushover_agent.rb b/app/models/agents/pushover_agent.rb index ce3e431c..06cd268a 100644 --- a/app/models/agents/pushover_agent.rb +++ b/app/models/agents/pushover_agent.rb @@ -6,7 +6,7 @@ module Agents API_URL = 'https://api.pushover.net/1/messages.json' description <<-MD - The PushoverAgent receives and collects events and sends them via push notification to a user/group. + The Pushover Agent receives and collects events and sends them via push notification to a user/group. **You need a Pushover API Token:** [https://pushover.net/apps/build](https://pushover.net/apps/build) diff --git a/app/models/agents/rss_agent.rb b/app/models/agents/rss_agent.rb index 36dd798d..0530002f 100644 --- a/app/models/agents/rss_agent.rb +++ b/app/models/agents/rss_agent.rb @@ -13,7 +13,7 @@ module Agents description do <<-MD - This Agent consumes RSS feeds and emits events when they change. + The RSS Agent consumes RSS feeds and emits events when they change. This Agent is fairly simple, using [feed-normalizer](https://github.com/aasmith/feed-normalizer) as a base. For complex feeds with additional field types, we recommend using a WebsiteAgent. See [this example](https://github.com/cantino/huginn/wiki/Agent-configuration-examples#itunes-trailers). diff --git a/app/models/agents/scheduler_agent.rb b/app/models/agents/scheduler_agent.rb index 85b10499..e67d65f0 100644 --- a/app/models/agents/scheduler_agent.rb +++ b/app/models/agents/scheduler_agent.rb @@ -13,7 +13,7 @@ module Agents cattr_reader :second_precision_enabled description <<-MD - This agent periodically takes an action on target Agents according to a user-defined schedule. + The Scheduler Agent periodically takes an action on target Agents according to a user-defined schedule. # Action types diff --git a/app/models/agents/sentiment_agent.rb b/app/models/agents/sentiment_agent.rb index c48c183e..ad48dd0c 100644 --- a/app/models/agents/sentiment_agent.rb +++ b/app/models/agents/sentiment_agent.rb @@ -7,10 +7,9 @@ module Agents cannot_be_scheduled! description <<-MD - The SentimentAgent generates `good-bad` (psychological valence or happiness index), `active-passive` (arousal), - and `strong-weak` (dominance) score. It will output a value between 1 and 9. It will only work on English content. + The Sentiment Agent generates `good-bad` (psychological valence or happiness index), `active-passive` (arousal), and `strong-weak` (dominance) score. It will output a value between 1 and 9. It will only work on English content. - Make sure the content this agent is analyzing have sufficient length to get respectable results. + Make sure the content this agent is analyzing is of sufficient length to get respectable results. Provide a JSONPath in `content` field where content is residing and set `expected_receive_period_in_days` to the maximum number of days you would allow to be passed between events being received by this agent. MD diff --git a/app/models/agents/shell_command_agent.rb b/app/models/agents/shell_command_agent.rb index d73756ff..c8cf18c8 100644 --- a/app/models/agents/shell_command_agent.rb +++ b/app/models/agents/shell_command_agent.rb @@ -9,7 +9,7 @@ module Agents end description <<-MD - The ShellCommandAgent can execute commands on your local system, returning the output. + The Shell Command Agent will execute commands on your local system, returning the output. `command` specifies the command to be executed, and `path` will tell ShellCommandAgent in what directory to run this command. diff --git a/app/models/agents/slack_agent.rb b/app/models/agents/slack_agent.rb index b6db1879..33877a03 100644 --- a/app/models/agents/slack_agent.rb +++ b/app/models/agents/slack_agent.rb @@ -8,22 +8,19 @@ module Agents gem_dependency_check { defined?(Slack) } description <<-MD + The Slack Agent lets you receive events and send notifications to [Slack](https://slack.com/). + #{'## Include `slack-notifier` in your Gemfile to use this Agent!' if dependencies_missing?} - The SlackAgent lets you receive events and send notifications to [Slack](https://slack.com/). - To get started, you will first need to setup an incoming webhook. - Go to, https://your_team_name.slack.com/services/new/incoming-webhook, - choose a default channel and add the integration. + To get started, you will first need to configure an incoming webhook. + + - Go to `https://my.slack.com/services/new/incoming-webhook`, choose a default channel and add the integration. - Your webhook URL will look like: https://hooks.slack.com/services/random1/random2/token + Your webhook URL will look like: `https://hooks.slack.com/services/some/random/characters` - Once the webhook has been setup it can be used to post to other channels or ping team members. - To send a private message to team-mate, assign his username as `@username` to the channel option. - To communicate with a different webhook on slack, assign your custom webhook name to the webhook option. - Messages can also be formatted using [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid). + Once the webhook has been configured, it can be used to post to other channels or direct to team members. To send a private message to team member, use their @username as the channel. Messages can be formatted using [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid). - Finally, you can set a custom icon for this webhook in `icon`, either as [emoji](http://unicodey.com/emoji-data/table.htm) or an URL to an image. - Leaving this field blank will use the default icon for a webhook. + Finally, you can set a custom icon for this webhook in `icon`, either as [emoji](http://www.emoji-cheat-sheet.com) or an URL to an image. Leaving this field blank will use the default icon for a webhook. MD def default_options diff --git a/app/models/agents/stubhub_agent.rb b/app/models/agents/stubhub_agent.rb index 960ebec1..2c110388 100644 --- a/app/models/agents/stubhub_agent.rb +++ b/app/models/agents/stubhub_agent.rb @@ -3,7 +3,9 @@ module Agents cannot_receive_events! description <<-MD - This StubHubAgent creates an event for a given StubHub Event. It can be used to track how many tickets are available for the event and the minimum and maximum price. All that is required is that you paste in the url from the actual event, e.g. http://www.stubhub.com/outside-lands-music-festival-tickets/outside-lands-music-festival-3-day-pass-san-francisco-golden-gate-park-polo-fields-8-8-2014-9020701/ + The StubHub Agent creates an event for a given StubHub Event. + + It can be used to track how many tickets are available for the event and the minimum and maximum price. All that is required is that you paste in the url from the actual event, e.g. http://www.stubhub.com/outside-lands-music-festival-tickets/outside-lands-music-festival-3-day-pass-san-francisco-golden-gate-park-polo-fields-8-8-2014-9020701/ MD event_description <<-MD diff --git a/app/models/agents/translation_agent.rb b/app/models/agents/translation_agent.rb index 39b41b59..ec7ce1cc 100644 --- a/app/models/agents/translation_agent.rb +++ b/app/models/agents/translation_agent.rb @@ -3,8 +3,10 @@ module Agents cannot_be_scheduled! description <<-MD - You can use Translation Agent to translate text between natural languages. + The Translation Agent will attempt to translate text between natural languages. + Services are provided using Microsoft Translator. You can [sign up](https://datamarket.azure.com/dataset/bing/microsofttranslator) and [register your application](https://datamarket.azure.com/developer/applications/register) to get `client_id` and `client_secret` which are required to use this agent. + `to` must be filled with a [translator language code](http://msdn.microsoft.com/en-us/library/hh456380.aspx). Specify what you would like to translate in `content` field, you can use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) specify which part of the payload you want to translate. diff --git a/app/models/agents/trigger_agent.rb b/app/models/agents/trigger_agent.rb index 5e55393f..2060c161 100644 --- a/app/models/agents/trigger_agent.rb +++ b/app/models/agents/trigger_agent.rb @@ -5,7 +5,7 @@ module Agents VALID_COMPARISON_TYPES = %w[regex !regex field=value field>value] description <<-MD - Use a TriggerAgent to watch for a specific value in an Event payload. + The Trigger Agent will watch for a specific value in an Event payload. The `rules` array contains hashes of `path`, `value`, and `type`. The `path` value is a dotted path through a hash in [JSONPaths](http://goessner.net/articles/JsonPath/) syntax. diff --git a/app/models/agents/tumblr_publish_agent.rb b/app/models/agents/tumblr_publish_agent.rb index 6d323718..f395724a 100644 --- a/app/models/agents/tumblr_publish_agent.rb +++ b/app/models/agents/tumblr_publish_agent.rb @@ -7,9 +7,9 @@ module Agents cannot_be_scheduled! description <<-MD - #{'## Include `tumblr_client` and `omniauth-tumblr` in your Gemfile to use this Agent!' if dependencies_missing?} + The Tumblr Publish Agent publishes Tumblr posts from the events it receives. - The TumblrPublishAgent publishes Tumblr posts from the events it receives. + #{'## Include `tumblr_client` and `omniauth-tumblr` in your Gemfile to use this Agent!' if dependencies_missing?} To be able to use this Agent you need to authenticate with Tumblr in the [Services](/services) section first. diff --git a/app/models/agents/twilio_agent.rb b/app/models/agents/twilio_agent.rb index 91d1430a..75413b75 100644 --- a/app/models/agents/twilio_agent.rb +++ b/app/models/agents/twilio_agent.rb @@ -8,8 +8,9 @@ module Agents gem_dependency_check { defined?(Twilio) } description <<-MD + The Twilio Agent receives and collects events and sends them via text message (up to 160 characters) or gives you a call when scheduled. + #{'## Include `twilio-ruby` in your Gemfile to use this Agent!' if dependencies_missing?} - The TwilioAgent receives and collects events and sends them via text message (up to 160 characters) or gives you a call when scheduled. It is assumed that events have a `message`, `text`, or `sms` key, the value of which is sent as the content of the text message/call. You can use the EventFormattingAgent if your event does not provide these keys. diff --git a/app/models/agents/twitter_publish_agent.rb b/app/models/agents/twitter_publish_agent.rb index 4dfdbc1d..7ff5f3f9 100644 --- a/app/models/agents/twitter_publish_agent.rb +++ b/app/models/agents/twitter_publish_agent.rb @@ -5,7 +5,7 @@ module Agents cannot_be_scheduled! description <<-MD - The TwitterPublishAgent publishes tweets from the events it receives. + The Twitter Publish Agent publishes tweets from the events it receives. #{twitter_dependencies_missing if dependencies_missing?} diff --git a/app/models/agents/twitter_stream_agent.rb b/app/models/agents/twitter_stream_agent.rb index 8e398a84..3fbabe91 100644 --- a/app/models/agents/twitter_stream_agent.rb +++ b/app/models/agents/twitter_stream_agent.rb @@ -5,7 +5,7 @@ module Agents cannot_receive_events! description <<-MD - The TwitterStreamAgent follows the Twitter stream in real time, watching for certain keywords, or filters, that you provide. + The Twitter Stream Agent follows the Twitter stream in real time, watching for certain keywords, or filters, that you provide. #{twitter_dependencies_missing if dependencies_missing?} diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb index e7474f4c..3e1eab63 100644 --- a/app/models/agents/twitter_user_agent.rb +++ b/app/models/agents/twitter_user_agent.rb @@ -5,7 +5,7 @@ module Agents cannot_receive_events! description <<-MD - The TwitterUserAgent follows the timeline of a specified Twitter user. + The Twitter User Agent follows the timeline of a specified Twitter user. #{twitter_dependencies_missing if dependencies_missing?} diff --git a/app/models/agents/user_location_agent.rb b/app/models/agents/user_location_agent.rb index 9add8bc5..9297a3d0 100644 --- a/app/models/agents/user_location_agent.rb +++ b/app/models/agents/user_location_agent.rb @@ -7,7 +7,7 @@ module Agents gem_dependency_check { defined?(Haversine) } description do <<-MD - The UserLocationAgent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location to `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. + The User Location Agent creates events based on WebHook POSTS that contain a `latitude` and `longitude`. You can use the [POSTLocation](https://github.com/cantino/post_location) or [PostGPS](https://github.com/chriseidhof/PostGPS) iOS app to post your location to `https://#{ENV['DOMAIN']}/users/#{user.id}/update_location/:secret` where `:secret` is specified in your options. #{'## Include `haversine` in your Gemfile to use this Agent!' if dependencies_missing?} diff --git a/app/models/agents/weather_agent.rb b/app/models/agents/weather_agent.rb index 91cd1f17..c1d28313 100644 --- a/app/models/agents/weather_agent.rb +++ b/app/models/agents/weather_agent.rb @@ -8,8 +8,9 @@ module Agents gem_dependency_check { defined?(Wunderground) && defined?(ForecastIO) } description <<-MD + The Weather Agent creates an event for the day's weather at a given `location`. + #{'## Include `forecast_io` and `wunderground` in your Gemfile to use this Agent!' if dependencies_missing?} - The WeatherAgent creates an event for the day's weather at a given `location`. You also must select `which_day` you would like to get the weather for where the number 0 is for today and 1 is for tomorrow and so on. Weather is only returned for 1 week at a time. diff --git a/app/models/agents/webhook_agent.rb b/app/models/agents/webhook_agent.rb index 52b0a693..b21ed0b6 100644 --- a/app/models/agents/webhook_agent.rb +++ b/app/models/agents/webhook_agent.rb @@ -4,12 +4,13 @@ module Agents cannot_receive_events! description do <<-MD - Use this Agent to create events by receiving webhooks from any source. - In order to create events with this agent, make a POST request to: + The Webhook Agent will create events by receiving webhooks from any source. In order to create events with this agent, make a POST request to: ``` - https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ''}/:secret - ``` where `:secret` is specified in your options. + https://#{ENV['DOMAIN']}/users/#{user.id}/web_requests/#{id || ':id'}/#{options['secret'] || ':secret'} + ``` + + #{'The placeholder symbols above will be replaced by their values once the agent is saved.' unless id} Options: diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 2b4e2f64..d0104658 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -14,7 +14,7 @@ module Agents UNIQUENESS_FACTOR = 3 description <<-MD - The WebsiteAgent scrapes a website, XML document, or JSON feed and creates Events based on the results. + The Website Agent scrapes a website, XML document, or JSON feed and creates Events based on the results. Specify a `url` and select a `mode` for when to create Events based on the scraped data, either `all` or `on_change`. diff --git a/app/models/agents/weibo_publish_agent.rb b/app/models/agents/weibo_publish_agent.rb index 0dcae12c..1c733c50 100644 --- a/app/models/agents/weibo_publish_agent.rb +++ b/app/models/agents/weibo_publish_agent.rb @@ -7,12 +7,13 @@ module Agents cannot_be_scheduled! description <<-MD + The Weibo Publish Agent publishes tweets from the events it receives. + #{'## Include `weibo_2` in your Gemfile to use this Agent!' if dependencies_missing?} - The WeiboPublishAgent publishes tweets from the events it receives. - You must first set up a Weibo app and generate an `acess_token` for the user to send statuses as. + You must first set up a Weibo app and generate an `access_token` for the user that will be used for posting status updates. - Include that in options, along with the `app_key` and `app_secret` for your Weibo app. It's useful to also include the Weibo user id of the person to publish as. + You'll use that `access_token`, along with the `app_key` and `app_secret` for your Weibo app. You must also include the Weibo User ID (as `uid`) of the person to publish as. You must also specify a `message_path` parameter: a [JSONPaths](http://goessner.net/articles/JsonPath/) to the value to tweet. diff --git a/app/models/agents/weibo_user_agent.rb b/app/models/agents/weibo_user_agent.rb index 6c8373a8..295dd14a 100644 --- a/app/models/agents/weibo_user_agent.rb +++ b/app/models/agents/weibo_user_agent.rb @@ -7,8 +7,9 @@ module Agents cannot_receive_events! description <<-MD + The Weibo User Agent follows the timeline of a specified Weibo user. It uses this endpoint: http://open.weibo.com/wiki/2/statuses/user_timeline/en + #{'## Include `weibo_2` in your Gemfile to use this Agent!' if dependencies_missing?} - The WeiboUserAgent follows the timeline of a specified Weibo user. It uses this endpoint: http://open.weibo.com/wiki/2/statuses/user_timeline/en You must first set up a Weibo app and generate an `acess_token` to authenticate with. Provide that, along with the `app_key` and `app_secret` for your Weibo app in the options. diff --git a/app/models/agents/witai_agent.rb b/app/models/agents/witai_agent.rb index e7b66b6b..f5f23190 100644 --- a/app/models/agents/witai_agent.rb +++ b/app/models/agents/witai_agent.rb @@ -3,7 +3,10 @@ module Agents cannot_be_scheduled! description <<-MD - The `wit.ai` agent receives events, sends text query to your `wit.ai` instance and generates outcome events. Fill in `Server Access Token` of your `wit.ai` instance. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to fill query field. + The `wit.ai` agent receives events, sends a text query to your `wit.ai` instance and generates outcome events. + + Fill in `Server Access Token` of your `wit.ai` instance. Use [Liquid](https://github.com/cantino/huginn/wiki/Formatting-Events-using-Liquid) to fill query field. + `expected_receive_period_in_days` is the expected number of days by which agent should receive events. It helps in determining if the agent is working. MD From 0e4c9be723c716ee8a99d2504c26b02bfea3b4c2 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sat, 20 Jun 2015 21:21:21 +0200 Subject: [PATCH 18/36] First version of the manual deployment guide --- .gitignore | 1 + Gemfile | 2 +- Gemfile.lock | 6 +- Procfile | 24 ++- config/unicorn.rb.example | 35 ++++ doc/README.md | 11 ++ doc/install/README.md | 5 + doc/install/installation.md | 341 +++++++++++++++++++++++++++++++++++ doc/install/requirements.md | 68 +++++++ lib/support/logrotate/huginn | 20 ++ lib/support/nginx/huginn | 70 +++++++ lib/support/nginx/huginn-ssl | 119 ++++++++++++ 12 files changed, 693 insertions(+), 9 deletions(-) create mode 100644 config/unicorn.rb.example create mode 100644 doc/README.md create mode 100644 doc/install/README.md create mode 100644 doc/install/installation.md create mode 100644 doc/install/requirements.md create mode 100644 lib/support/logrotate/huginn create mode 100644 lib/support/nginx/huginn create mode 100644 lib/support/nginx/huginn-ssl diff --git a/.gitignore b/.gitignore index e85f3386..1a9b4ce8 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ deployment/cookbooks .ruby-gemset .ruby-version manifest.yml +config/unicorn.rb \ No newline at end of file diff --git a/Gemfile b/Gemfile index d6e9b343..74cee488 100644 --- a/Gemfile +++ b/Gemfile @@ -120,6 +120,7 @@ end group :production do gem 'rack', '> 1.5.0' + gem 'unicorn', '~> 4.9.0' end # Platform requirements. @@ -145,6 +146,5 @@ end on_heroku do gem 'pg' - gem 'unicorn' gem 'rails_12factor', group: :production end diff --git a/Gemfile.lock b/Gemfile.lock index 68362804..9ecfe29e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -230,7 +230,7 @@ GEM kaminari (0.16.1) actionpack (>= 3.0.0) activesupport (>= 3.0.0) - kgio (2.9.2) + kgio (2.9.3) kramdown (1.3.3) launchy (2.4.2) addressable (~> 2.3) @@ -472,7 +472,7 @@ GEM unf (0.1.4) unf_ext unf_ext (0.0.7.1) - unicorn (4.8.3) + unicorn (4.9.0) kgio (~> 2.6) rack raindrops (~> 0.7) @@ -576,7 +576,7 @@ DEPENDENCIES tzinfo (>= 1.2.0) tzinfo-data uglifier (>= 1.3.0) - unicorn + unicorn (~> 4.9.0) vcr webmock (~> 1.17.4) weibo_2! diff --git a/Procfile b/Procfile index feafae97..af004e85 100644 --- a/Procfile +++ b/Procfile @@ -1,13 +1,27 @@ +################# +# DEVELOPMENT # +################# + # Procfile for development using the new threaded worker (scheduler, twitter stream and delayed job) web: bundle exec rails server -b0.0.0.0 jobs: bundle exec rails runner bin/threaded.rb -# Possible Profile configuration for production: -# web: bundle exec unicorn -c config/unicorn/production.rb -# jobs: bundle exec rails runner bin/threaded.rb - # Old version with separate processes (use this if you have issues with the threaded version) -# web: bundle exec rails server -b0.0.0.0 +# web: bundle exec rails server +# schedule: bundle exec rails runner bin/schedule.rb +# twitter: bundle exec rails runner bin/twitter_stream.rb +# dj: bundle exec script/delayed_job run + +################# +# PRODUCTION # +################# + +# Using the threaded worker (consumes less RAM but can run slower) +# web: bundle exec unicorn -c config/unicorn.rb +# jobs: bundle exec rails runner bin/threaded.rb + +# Old version with separate processes (use this if you have issues with the threaded version) +# web: bundle exec unicorn -c config/unicorn.rb # schedule: bundle exec rails runner bin/schedule.rb # twitter: bundle exec rails runner bin/twitter_stream.rb # dj: bundle exec script/delayed_job run diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example new file mode 100644 index 00000000..465c7a8c --- /dev/null +++ b/config/unicorn.rb.example @@ -0,0 +1,35 @@ +wd = "/home/huginn/huginn" + +app_path = wd + +worker_processes 2 +preload_app true +timeout 180 +listen "#{wd}/tmp/sockets/unicorn.socket" + +working_directory app_path + +rails_env = ENV['RAILS_ENV'] || 'production' + +# Log everything to one file +stderr_path "log/unicorn.log" +stdout_path "log/unicorn.log" + +# Set master PID location +pid "#{wd}/tmp/pids/unicorn.pid" + +before_fork do |server, worker| + ActiveRecord::Base.connection.disconnect! + old_pid = "#{server.config[:pid]}.oldbin" + if File.exist?(old_pid) && server.pid != old_pid + begin + Process.kill("QUIT", File.read(old_pid).to_i) + rescue Errno::ENOENT, Errno::ESRCH + # someone else did our job for us + end + end +end + +after_fork do |server, worker| + ActiveRecord::Base.establish_connection +end diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000..dcaf5325 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,11 @@ +# Documentation + +## User documentation + +- [Check out Huginn with Docker](./tobewritten.md) Run a local Huginn installation using Docker + + +## Administrator documentation + +- [Install](install/README.md) Requirements, directory structures and installation from source. +- [Update](update/README.md) Update guides to upgrade your installation. diff --git a/doc/install/README.md b/doc/install/README.md new file mode 100644 index 00000000..0e83c948 --- /dev/null +++ b/doc/install/README.md @@ -0,0 +1,5 @@ +# Installation + +- [Requirements](requirements.md) Software and hardware requirements to run the Huginn installation +- [Install](installation.md) Installation guide for Ubundu/Debian +- [Update](update.md) Update an existing Huginn installation \ No newline at end of file diff --git a/doc/install/installation.md b/doc/install/installation.md new file mode 100644 index 00000000..b1af6601 --- /dev/null +++ b/doc/install/installation.md @@ -0,0 +1,341 @@ +# Installation from source + + +## Important Notes + +This guide is long because it covers many cases and includes all commands you need. + +This installation guide was created for and tested on **Debian/Ubuntu** operating systems. Please read [doc/install/requirements.md](./requirements.md) for hardware and operating system requirements. + +This is the official installation guide to set up a production server. To set up a **development installation** or for many other installation options please see [the getting started section of the readme](https://github.com/cantino/huginn#getting-started). + +The following steps have been known to work. Please **use caution when you deviate** from this guide. Make sure you don't violate any assumptions Huginn makes about its environment. For example many people run into permission problems because they change the location of directories or run services as the wrong user. + +If you find a bug/error in this guide please **submit a merge request**. + +## Overview + +The Huginn installation consists of setting up the following components: + +1. Packages / Dependencies +1. Ruby +1. System Users +1. Database +1. Huginn +1. Nginx + +## 1. Packages / Dependencies + +`sudo` is not installed on Debian by default. Make sure your system is +up-to-date and install it. + + # run as root! + apt-get update -y + apt-get upgrade -y + apt-get install sudo -y + +**Note:** During this installation some files will need to be edited manually. If you are familiar with vim set it as default editor with the commands below. If you are not familiar with vim please skip this and keep using the default editor. + + # Install vim and set as default editor + sudo apt-get install -y vim + sudo update-alternatives --set editor /usr/bin/vim.basic + +Import node.js repository (can be skipped on Ubuntu and Debian Jessie): + + curl -sL https://deb.nodesource.com/setup_0.12 | sudo bash - + +Install the required packages (needed to compile Ruby and native extensions to Ruby gems): + + sudo apt-get install -y build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz + + +## 2. Ruby + + +The use of Ruby version managers such as [RVM](http://rvm.io/), [rbenv](https://github.com/sstephenson/rbenv) or [chruby](https://github.com/postmodern/chruby) with Huginn in production frequently leads to hard-to-diagnose problems. Version managers are not supported and we strongly advise everyone to follow the instructions below to use a system Ruby. + +Remove the old Ruby versions if present: + + sudo apt-get remove -y ruby1.8 ruby1.9 + +Download Ruby and compile it: + + mkdir /tmp/ruby && cd /tmp/ruby + curl -L --progress http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.bz2 | tar xj + cd ruby-2.2.2 + ./configure --disable-install-rdoc + make -j`nproc` + sudo make install + +Install the bundler and foreman gems: + + sudo gem install bundler foreman --no-ri --no-rdoc + +## 3. System Users + +Create a user for Huginn: + + sudo adduser --disabled-login --gecos 'Huginn' huginn + +## 4. Database + +Install the database packages + + sudo apt-get install -y mysql-server mysql-client libmysqlclient-dev + + # Pick a MySQL root password (can be anything), type it and press enter, + # retype the MySQL root password and press enter + +Check the installed MySQL version (remeber if its >= 5.5.3 for the `.env` configuration done later): + + mysql --version + +Secure your installation + + sudo mysql_secure_installation + +Login to MySQL + + mysql -u root -p + + # Type the MySQL root password + +Create a user for Huginn do not type the `mysql>`, this is part of the prompt. Change `$password` in the command below to a real password you pick + + mysql> CREATE USER 'huginn'@'localhost' IDENTIFIED BY '$password'; + +Ensure you can use the InnoDB engine which is necessary to support long indexes + + mysql> SET storage_engine=INNODB; + + # If this fails, check your MySQL config files (e.g. `/etc/mysql/*.cnf`, `/etc/mysql/conf.d/*`) + # for the setting "innodb = off" + +Grant the Huginn user necessary permissions on the database + + mysql> GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, LOCK TABLES ON `huginn_production`.* TO 'huginn'@'localhost'; + +Quit the database session + + mysql> \q + +Try connecting to the new database with the new user + + sudo -u huginn -H mysql -u huginn -p -D huginn_production + + # Type the password you replaced $password with earlier + +You should now see `ERROR 1049 (42000): Unknown database 'huginn_production'` which is fine because we will create the database later. + +You are done installing the database and can go back to the rest of the installation. + + +## 6. Huginn + +### Clone the Source + + # We'll install Huginn into home directory of the user "huginn" + cd /home/huginn + + # Clone Huginn repository + #sudo -u huginn -H git clone https://github.com/cantino/huginn.git -b master huginn + # **FIXME** + sudo -u huginn -H git clone https://github.com/dsander/huginn.git -b deployment-guide huginn + + # Go to Huginn installation folder + cd /home/huginn/huginn + + # Copy the example Huginn config + sudo -u huginn -H cp .env.example .env + + # Create the log/, tmp/pids/ and tmp/sockets/ directories + sudo -u huginn mkdir -p log tmp/pids tmp/sockets + + # Make sure Huginn can write to the log/ and tmp/ directories + sudo chown -R huginn log/ tmp/ + sudo chmod -R u+rwX,go-w log/ tmp/ + + # Make sure permissions are set correctly + sudo chmod -R u+rwX,go-w log/ + sudo chmod -R u+rwX tmp/ + sudo -u huginn -H chmod o-rwx .env + + # Copy the example Unicorn config + sudo -u huginn -H cp config/unicorn.rb.example config/unicorn.rb + +### Install Gems + +**Note:** As of bundler 1.5.2, you can invoke `bundle install -jN` (where `N` the number of your processor cores) and enjoy parallel gem installation with measurable difference in completion time (~60% faster). Check the number of your cores with `nproc`. For more information check this [post](http://robots.thoughtbot.com/parallel-gem-installing-using-bundler). First make sure you have bundler >= 1.5.2 (run `bundle -v`) as it addresses some [issues](https://devcenter.heroku.com/changelog-items/411) that were [fixed](https://github.com/bundler/bundler/pull/2817) in 1.5.2. + + sudo -u huginn -H bundle install --deployment --without development test + + +### Configure it + + # Update Huginn config file and follow the instructions + sudo -u huginn -H editor .env + +If you are using a local MySQL server the database configuration should look like this (use the password of the huginn MySQL user you created earlier): + + DATABASE_ADAPTER=mysql2 + DATABASE_ENCODING=utf8 + DATABASE_RECONNECT=true + DATABASE_NAME=huginn_production + DATABASE_POOL=20 + DATABASE_USERNAME=huginn + DATABASE_PASSWORD="" + #DATABASE_HOST=your-domain-here.com + #DATABASE_PORT=3306 + #DATABASE_SOCKET=/tmp/mysql.sock + +**Important**: Uncomment the RAILS_ENV setting to run Huginn in the production rails environment + + RAILS_ENV=production + +Change the Unicorn config if needed, the [requirements.md](./requirements.md#unicorn-workers) has a section explaining the suggested amount of unicorn workers: + + # Increase the amount of workers if you expect to have a high load instance. + # 2 are enough for most use cases, if the server has less then 2GB of RAM + # decrease the worker amount to 1 + sudo -u huginn -H editor config/unicorn.rb + + +**Important Note:** Make sure to edit both `.env` and `unicorn.rb` to match your setup. + +**Note:** If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps. + + +### Initialize Database + + # Create the database + sudo -u huginn -H bundle exec rake db:create RAILS_ENV=production + + # Migrate to the latest version + sudo -u huginn -H bundle exec rake db:migrate RAILS_ENV=production + + # Create admin user and example agents + sudo -u huginn -H bundle exec rake db:seed RAILS_ENV=production + +When done you see `See the Huginn Wiki for more Agent examples! https://github.com/cantino/huginn/wiki` + +**Note:** This will create an initial user, you can set the username and password by supplying it in environmental variables `SEED_USERNAME` and`SEED_PASSWORD` as seen below. If you don't set the password (and it is set to the default one) please wait with exposing Huginn to the public internet until the installation is done and you've logged into the server and changed your password. + + sudo -u huginn -H bundle exec rake db:seed RAILS_ENV=production SEED_USERNAME=admin SEED_PASSWORD=yourpassword + +### Install Init Script + +Huginn uses foreman to generate the init scripts based on a `Procfile` + +Edit the `Procfile` and choose one of the suggested versions for production + + sudo -u huginn -H editor Procfile + +**Debian only** Install upstart and reboot the system (skip this step on Ubuntu): + + sudo apt-get install -y --force-yes upstart + sudo reboot + # After you you logged back in go to Huginn installation folder + cd /home/huginn/huginn + +Export the init scripts using foreman: + + sudo foreman export upstart -a huginn /etc/init + +**Note:** You have to re-export the init script every time you change the configuration in `.env`! + +### Setup Logrotate + + sudo cp lib/support/logrotate/huginn /etc/logrotate.d/huginn + +### Compile Assets + + sudo -u huginn -H bundle exec rake assets:precompile RAILS_ENV=production + +### Start Your Huginn Instance + + sudo start huginn + +## 7. Nginx + +**Note:** Nginx is the officially supported web server for Huginn. If you cannot or do not want to use Nginx as your web server, the wiki has a page on how to configure [apache](https://github.com/cantino/huginn/wiki/Apache-Huginn-configuration). + +### Installation + + sudo apt-get install -y nginx + +### Site Configuration + +Copy the example site config: + + sudo cp lib/support/nginx/huginn /etc/nginx/sites-available/huginn + sudo ln -s /etc/nginx/sites-available/huginn /etc/nginx/sites-enabled/huginn + +Make sure to edit the config file to match your setup, if you are running multiple nginx sites remove the `default_server` argument from the `listen` directives: + + # Change YOUR_SERVER_FQDN to the fully-qualified + # domain name of your host serving Huginn. + sudo editor /etc/nginx/sites-available/huginn + +Remove the default nginx site, **if huginn is the only enabled nginx site**: + + sudo rm /etc/nginx/sites-enabled/default + +**Note:** If you want to use HTTPS, replace the `huginn` Nginx config with `huginn-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. + +### Test Configuration + +Validate your `huginn` or `huginn-ssl` Nginx config file with the following command: + + sudo nginx -t + +You should receive `syntax is okay` and `test is successful` messages. If you receive errors check your `huginn` or `huginn-ssl` Nginx config file for typos, etc. as indicated in the error message given. + +### Restart + + sudo service nginx restart + +# Done! + +### Initial Login + +Visit YOUR_SERVER in your web browser for your first Huginn login. The setup has created a default admin account for you. You can use it to log in: + + admin + password + + +**Enjoy!** + +You can use `sudo start huginn` and `sudo stop huginn` to start and stop Huginn. + +## Advanced Setup Tips + +### Using HTTPS + +To use Huginn with HTTPS: + +1. In `.env`: + 1. Set the `FORCE_SSL` option to `true`. +1. Use the `huginn-ssl` Nginx example config instead of the `huginn` config: + 1. `sudo cp lib/support/nginx/huginn-ssl /etc/nginx/sites-available/huginn` + 1. Update `YOUR_SERVER_FQDN`. + 1. Update `ssl_certificate` and `ssl_certificate_key`. + 1. Review the configuration file and consider applying other security and performance enhancing features. + +Restart Nginx, export the init script and restart Huginn: + +``` +cd /home/huginn/huginn +sudo service nginx restart +sudo foreman export upstart -a huginn /etc/init +sudo restart huginn +``` + +Using a self-signed certificate is discouraged, but if you must use it follow the normal directions. Then generate the certificate: + +``` +sudo mkdir -p /etc/nginx/ssl/ +cd /etc/nginx/ssl/ +sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out huginn.crt -keyout huginn.key +sudo chmod o-r huginn.key +``` diff --git a/doc/install/requirements.md b/doc/install/requirements.md new file mode 100644 index 00000000..c4084938 --- /dev/null +++ b/doc/install/requirements.md @@ -0,0 +1,68 @@ +# Requirements + +## Operating Systems + +### Supported Unix distributions + +- Ubuntu (12.04 and 14.04) +- Debian (Jessie and Wheezy) + +### Unsupported Unix distributions + +- CentOS +- Red Hat Enterprise Linux +- OS X +- Arch Linux +- Fedora +- Gentoo +- FreeBSD + +On the above unsupported distributions is still possible to install Huginn. Follow the [installation guide](./installation.md) and substitute the `apt` commands with the corresponding package manager commands of your distribution. + +### Non-Unix operating systems such as Windows + +Huginn is developed for Unix operating systems. +Huginn does **not** run on Windows and we have no plans of supporting it in the near future. +Please consider using a virtual machine to run Huginn. + +## Ruby versions + +Huginn requires Ruby (MRI) 2.0, 2.1 or 2.2 +You will have to use the standard MRI implementation of Ruby. +We love [JRuby](http://jruby.org/) and [Rubinius](http://rubini.us/) but Huginn needs several Gems that have native extensions. + +## Hardware requirements + +### CPU + +- _single core_ setups will work but depending on the amount of Huginn Agents and users it will run a bit slower since the application server and background jobs can not run simultaneously +- _dual core_ setups are the **recommended** system/vps and will work well for a decent amount of Agents +- 3+ cores can be needed when running multiple DelayedJob workers + +### Memory + +You need at least 0.5GB of physical and 0.5GB of addressable memory (swap) to install and use Huginn with the default configuration! +With less memory you need to manually adjust the `Gemfile` and Huginn can respond with internal server errors when accessing the web interface. + +- 256MB RAM + 0.5GB of swap is the absolute minimum but we strongly **advise against** this amount of memory. See the Wiki page about running Huginn on [systems with low memory](https://github.com/cantino/huginn/wiki/Running-Huginn-on-minimal-systems-with-low-RAM-&-CPU-e.g.-Raspberry-Pi) +- 0.5GB RAM + 0.5GB swap will work relatively well with SSD drives, but can feel a bit slow due to swapping +- 1GB RAM + 1GB swap will work with two unicorn workers and the threaded background worker +- **2GB RAM** is the **recommended** memory size, it will support 2 unicorn workers and both the threaded and the old separate workers +- for each 300MB of additional RAM you can run one extra DelayedJob worker + +## Unicorn Workers + +It's possible to increase the amount of unicorn workers and this will usually help for to reduce the response time of the applications and increase the ability to handle parallel requests. + +For most instances we recommend using: CPU cores = unicorn workers. + +If you have a 512MB machine we recommend to configure only one Unicorn worker and use the threaded background worker to prevent excessive swapping. + + +## DelayedJob Workers + +A DelayedJob worker is a separate process which runs your Huginn Agents. It fetches Websites, polls external services for updates, etc. Depending on the amount of Agents and the check frequency of those you might need to run more than one worker (like it is done in the threaded setup). + +Estimating the amount of workers needed is easy. One worker can perform just one check at a time. +If you have 60 Agents checking websites every minute which take about 1 second to respond, one worker is fine. +If you need more Agents or are dealing with slow/unreliable websites/services, you should consider running additional workers. \ No newline at end of file diff --git a/lib/support/logrotate/huginn b/lib/support/logrotate/huginn new file mode 100644 index 00000000..3ccb42de --- /dev/null +++ b/lib/support/logrotate/huginn @@ -0,0 +1,20 @@ +/home/huginn/huginn/log/*.log { + daily + missingok + rotate 180 + # must use with delaycompress below + compress + dateext + + # this is important if using "compress" since we need to call + # the "lastaction" script below before compressing: + delaycompress + + # note the lack of the evil "copytruncate" option in this + # config. Unicorn supports the USR1 signal and we send it + # as our "lastaction" action: + lastaction + pid=/home/huginn/huginn/tmp/pids/unicorn.pid + test -s $pid && kill -USR1 "$(cat $pid)" + endscript +} \ No newline at end of file diff --git a/lib/support/nginx/huginn b/lib/support/nginx/huginn new file mode 100644 index 00000000..8e6b8413 --- /dev/null +++ b/lib/support/nginx/huginn @@ -0,0 +1,70 @@ +## Huginn +## +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### +## +## See installation.md#using-https for additional HTTPS configuration details. + +upstream huginn { + server unix:/home/huginn/huginn/tmp/sockets/unicorn.socket fail_timeout=0; +} + +## Normal HTTP host +server { + listen 0.0.0.0:80 default_server; + listen [::]:80 ipv6only=on default_server; + server_name YOUR_SERVER_FQDN; ## Replace this with something like huginn.example.com + server_tokens off; ## Don't show the nginx version number, a security best practice + root /home/huginn/huginn/public; + + ## Increase this if you want to upload large attachments + client_max_body_size 20m; + + ## Individual nginx logs for this Huginn vhost + access_log /var/log/nginx/huginn_access.log; + error_log /var/log/nginx/huginn_error.log; + + location / { + ## Serve static files from defined root folder. + ## @huginn is a named location for the upstream fallback, see below. + try_files $uri $uri/index.html $uri.html @huginn; + } + + ## If a file, which is not found in the root folder is requested, + ## then the proxy passes the request to the upsteam (huginn unicorn). + location @huginn { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + # gzip off; + + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://huginn; + } + + ## Enable gzip compression as per rails guide: + ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression + ## WARNING: If you are using relative urls remove the block below + ## See config/application.rb under "Relative url support" for the list of + ## other files that need to be changed for relative url support + location ~ ^/(assets)/ { + root /home/huginn/huginn/public; + gzip_static on; # to serve pre-gzipped version + expires max; + add_header Cache-Control public; + } + + error_page 502 /502.html; +} diff --git a/lib/support/nginx/huginn-ssl b/lib/support/nginx/huginn-ssl new file mode 100644 index 00000000..932fcf3c --- /dev/null +++ b/lib/support/nginx/huginn-ssl @@ -0,0 +1,119 @@ +## Huginn +## +## Modified from nginx http version +## Modified from http://blog.phusion.nl/2012/04/21/tutorial-setting-up-gitlab-on-debian-6/ +## Modified from https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html +## Modified from https://github.com/gitlabhq/gitlabhq/blob/master/lib/support/nginx/gitlab-ssl +## +## Lines starting with two hashes (##) are comments with information. +## Lines starting with one hash (#) are configuration parameters that can be uncommented. +## +################################### +## configuration ## +################################### +## +## See installation.md#using-https for additional HTTPS configuration details. + +upstream huginn { + server unix:/home/huginn/huginn/tmp/sockets/unicorn.socket fail_timeout=0; +} + +## Redirects all HTTP traffic to the HTTPS host +server { + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on default_server; + server_name YOUR_SERVER_FQDN; ## Replace this with something like huginn.example.com + server_tokens off; ## Don't show the nginx version number, a security best practice + return 301 https://$server_name$request_uri; + access_log /var/log/nginx/huginn_access.log; + error_log /var/log/nginx/huginn_error.log; +} + + +## HTTPS host +server { + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl default_server; + server_name YOUR_SERVER_FQDN; ## Replace this with something like huginn.example.com + server_tokens off; ## Don't show the nginx version number, a security best practice + root /home/git/huginn/public; + + ## Increase this if you want to upload large attachments + ## Or if you want to accept large git objects over http + client_max_body_size 20m; + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/nginx/ssl/huginn.crt; + ssl_certificate_key /etc/nginx/ssl/huginn.key; + + ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!CAMELLIA:!DES:!MD5:!PSK:!RC4'; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + + ## See app/controllers/application_controller.rb for headers set + + ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL. + ## Replace with your ssl_trusted_certificate. For more info see: + ## - https://medium.com/devops-programming/4445f4862461 + ## - https://www.ruby-forum.com/topic/4419319 + ## - https://www.digitalocean.com/community/tutorials/how-to-configure-ocsp-stapling-on-apache-and-nginx + # ssl_stapling on; + # ssl_stapling_verify on; + # ssl_trusted_certificate /etc/nginx/ssl/stapling.trusted.crt; + # resolver 208.67.222.222 208.67.222.220 valid=300s; # Can change to your DNS resolver if desired + # resolver_timeout 5s; + + ## [Optional] Generate a stronger DHE parameter: + ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096 + ## + # ssl_dhparam /etc/ssl/certs/dhparam.pem; + + ## Individual nginx logs for this huginn vhost + access_log /var/log/nginx/huginn_access.log; + error_log /var/log/nginx/huginn_error.log; + + location / { + ## Serve static files from defined root folder. + ## @huginn is a named location for the upstream fallback, see below. + try_files $uri $uri/index.html $uri.html @huginn; + } + + ## If a file, which is not found in the root folder is requested, + ## then the proxy passes the request to the upsteam (huginn unicorn). + location @huginn { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + gzip off; + + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Ssl on; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frame-Options SAMEORIGIN; + + proxy_pass http://huginn; + } + + ## Enable gzip compression as per rails guide: + ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression + ## WARNING: If you are using relative urls remove the block below + ## See config/application.rb under "Relative url support" for the list of + ## other files that need to be changed for relative url support + location ~ ^/(assets)/ { + root /home/huginn/huginn/public; + gzip_static on; # to serve pre-gzipped version + expires max; + add_header Cache-Control public; + } + + error_page 502 /502.html; +} From 3133e08e5d269aeebf7a8bb7a5981fd584342f64 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 21 Jun 2015 23:50:03 +0200 Subject: [PATCH 19/36] Add section about utf8mb4 database encoding --- doc/install/installation.md | 9 +++++++-- doc/install/requirements.md | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/install/installation.md b/doc/install/installation.md index b1af6601..0d6aff7c 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -178,16 +178,21 @@ You are done installing the database and can go back to the rest of the installa If you are using a local MySQL server the database configuration should look like this (use the password of the huginn MySQL user you created earlier): DATABASE_ADAPTER=mysql2 - DATABASE_ENCODING=utf8 DATABASE_RECONNECT=true DATABASE_NAME=huginn_production DATABASE_POOL=20 DATABASE_USERNAME=huginn - DATABASE_PASSWORD="" + DATABASE_PASSWORD='$password' #DATABASE_HOST=your-domain-here.com #DATABASE_PORT=3306 #DATABASE_SOCKET=/tmp/mysql.sock + DATABASE_ENCODING=utf8 + # MySQL only: If you are running a MySQL server >=5.5.3, you should + # set DATABASE_ENCODING to utf8mb4 instead of utf8 so that the + # database can hold 4-byte UTF-8 characters like emoji. + #DATABASE_ENCODING=utf8mb4 + **Important**: Uncomment the RAILS_ENV setting to run Huginn in the production rails environment RAILS_ENV=production diff --git a/doc/install/requirements.md b/doc/install/requirements.md index c4084938..e65f5f5d 100644 --- a/doc/install/requirements.md +++ b/doc/install/requirements.md @@ -2,7 +2,7 @@ ## Operating Systems -### Supported Unix distributions +### Supported Unix distributions by this guide - Ubuntu (12.04 and 14.04) - Debian (Jessie and Wheezy) @@ -17,7 +17,7 @@ - Gentoo - FreeBSD -On the above unsupported distributions is still possible to install Huginn. Follow the [installation guide](./installation.md) and substitute the `apt` commands with the corresponding package manager commands of your distribution. +On the above unsupported distributions is still possible to install Huginn, and many people do. Follow the [installation guide](./installation.md) and substitute the `apt` commands with the corresponding package manager commands of your distribution. ### Non-Unix operating systems such as Windows From 35f9bc419a61016610b878243bab588faff297d7 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 28 Jun 2015 23:38:25 +0200 Subject: [PATCH 20/36] Add update documentation --- doc/install/installation.md | 2 + doc/update/README.md | 87 +++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 doc/update/README.md diff --git a/doc/install/installation.md b/doc/install/installation.md index 0d6aff7c..c5ea53ee 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -244,6 +244,7 @@ Edit the `Procfile` and choose one of the suggested versions for production Export the init scripts using foreman: + sudo rm /etc/init/huginn* sudo foreman export upstart -a huginn /etc/init **Note:** You have to re-export the init script every time you change the configuration in `.env`! @@ -332,6 +333,7 @@ Restart Nginx, export the init script and restart Huginn: ``` cd /home/huginn/huginn sudo service nginx restart +sudo rm /etc/init/huginn* sudo foreman export upstart -a huginn /etc/init sudo restart huginn ``` diff --git a/doc/update/README.md b/doc/update/README.md new file mode 100644 index 00000000..9763bd9a --- /dev/null +++ b/doc/update/README.md @@ -0,0 +1,87 @@ +# Update + +### 0. Stop server + +``` +sudo stop huginn +``` + +### 1. Store the current version + +``` +cd /home/huginn/huginn +export OLD_VERSION=`git rev-parse HEAD` +``` + +### 2. Update the code + +Back up changed files + +``` +sudo -u huginn -H cp Procfile Procfile.bak +``` + +Get the new code +``` +sudo -u huginn -H git fetch --all +sudo -u huginn -H git checkout -- db/schema.rb Procfile +sudo -u huginn -H git checkout master +sudo -u huginn -H git pull +``` + +Restore backed up files + +``` +sudo -u huginn -H cp Procfile.bak Procfile +``` + +### 3. Install gems, migrate and precompile assets + +``` +cd /home/huginn/huginn + +sudo -u huginn -H bundle install --deployment --without development test + +# Run database migrations +sudo -u huginn -H bundle exec rake db:migrate RAILS_ENV=production + +# Clean up assets and cache +sudo -u huginn -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production + +``` + +### 4. Update the Procfile + +Check for changes made to the default `Procfile` +``` +sudo -u huginn -H git diff $OLD_VERSION..master Procfile +``` + +Update your `Procfile` if the default options of the version you are using changed +``` +sudo -u huginn -H editor Procfile +``` + +### 5. Update the .env file + +Check for changes made to the example `.env` +``` +sudo -u huginn -H git diff $OLD_VERSION..master .env.example +``` + +Update your `.env` with new options or changed defaults +``` +sudo -u huginn -H editor .env +``` + + +### 6. Export init script and start huginn + +``` +# Export the init script +sudo rm /etc/init/huginn* +sudo foreman export upstart -a huginn /etc/init +# Start huginn +sudo start huginn +``` + From 82566130b61a8706d46fd891be9e149378d19e6a Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Mon, 29 Jun 2015 00:05:30 +0200 Subject: [PATCH 21/36] Document how to run additional delayed job workers --- Procfile | 31 +++++++++++++++++++++++++------ doc/configuration/dotenv.md | 0 doc/configuration/procfile.md | 0 doc/install/README.md | 2 +- doc/install/installation.md | 2 ++ doc/update/README.md | 4 ++-- 6 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 doc/configuration/dotenv.md create mode 100644 doc/configuration/procfile.md diff --git a/Procfile b/Procfile index af004e85..75bbcf27 100644 --- a/Procfile +++ b/Procfile @@ -1,6 +1,6 @@ -################# -# DEVELOPMENT # -################# +############################### +# DEVELOPMENT # +############################### # Procfile for development using the new threaded worker (scheduler, twitter stream and delayed job) web: bundle exec rails server -b0.0.0.0 @@ -12,9 +12,9 @@ jobs: bundle exec rails runner bin/threaded.rb # twitter: bundle exec rails runner bin/twitter_stream.rb # dj: bundle exec script/delayed_job run -################# -# PRODUCTION # -################# +############################### +# PRODUCTION # +############################### # Using the threaded worker (consumes less RAM but can run slower) # web: bundle exec unicorn -c config/unicorn.rb @@ -25,3 +25,22 @@ jobs: bundle exec rails runner bin/threaded.rb # schedule: bundle exec rails runner bin/schedule.rb # twitter: bundle exec rails runner bin/twitter_stream.rb # dj: bundle exec script/delayed_job run + +############################### +# Multiple DelayedJob workers # +############################### +# Per default Huginn can just run one agent at a time. Using a lot of agents or calling slow +# external services frequently might require more DelayedJob workers (an indicator for this is +# a backlog in your 'Job Management' page). +# Every uncommented line starts an additional DelayedJob worker. This works for development, production +# and for the threaded and separate worker processes. Keep in mind one worker needs about 300MB of RAM. +# +#dj2: bundle exec script/delayed_job -i 2 run +#dj3: bundle exec script/delayed_job -i 3 run +#dj4: bundle exec script/delayed_job -i 4 run +#dj5: bundle exec script/delayed_job -i 5 run +#dj6: bundle exec script/delayed_job -i 6 run +#dj7: bundle exec script/delayed_job -i 7 run +#dj8: bundle exec script/delayed_job -i 8 run +#dj9: bundle exec script/delayed_job -i 9 run +#dj10: bundle exec script/delayed_job -i 10 run \ No newline at end of file diff --git a/doc/configuration/dotenv.md b/doc/configuration/dotenv.md new file mode 100644 index 00000000..e69de29b diff --git a/doc/configuration/procfile.md b/doc/configuration/procfile.md new file mode 100644 index 00000000..e69de29b diff --git a/doc/install/README.md b/doc/install/README.md index 0e83c948..e0a8078f 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -2,4 +2,4 @@ - [Requirements](requirements.md) Software and hardware requirements to run the Huginn installation - [Install](installation.md) Installation guide for Ubundu/Debian -- [Update](update.md) Update an existing Huginn installation \ No newline at end of file +- [Update](../update/README.md) Update an existing Huginn installation \ No newline at end of file diff --git a/doc/install/installation.md b/doc/install/installation.md index c5ea53ee..90b88af7 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -13,6 +13,8 @@ The following steps have been known to work. Please **use caution when you devia If you find a bug/error in this guide please **submit a merge request**. +If not stated otherwise all commands should be run as user with sudo permissions or as root. + ## Overview The Huginn installation consists of setting up the following components: diff --git a/doc/update/README.md b/doc/update/README.md index 9763bd9a..922fc741 100644 --- a/doc/update/README.md +++ b/doc/update/README.md @@ -75,13 +75,13 @@ sudo -u huginn -H editor .env ``` -### 6. Export init script and start huginn +### 6. Export init script and start Huginn ``` # Export the init script sudo rm /etc/init/huginn* sudo foreman export upstart -a huginn /etc/init -# Start huginn +# Start Huginn sudo start huginn ``` From 79300615f907108fef007c5e879d9459e785b05b Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Mon, 29 Jun 2015 23:09:31 +0200 Subject: [PATCH 22/36] Add docker documentation --- doc/README.md | 2 +- doc/install/docker.md | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 doc/install/docker.md diff --git a/doc/README.md b/doc/README.md index dcaf5325..02581423 100644 --- a/doc/README.md +++ b/doc/README.md @@ -2,7 +2,7 @@ ## User documentation -- [Check out Huginn with Docker](./tobewritten.md) Run a local Huginn installation using Docker +- [Check out Huginn with Docker](install/docker.md) Run a local Huginn installation using Docker ## Administrator documentation diff --git a/doc/install/docker.md b/doc/install/docker.md new file mode 100644 index 00000000..29c20de9 --- /dev/null +++ b/doc/install/docker.md @@ -0,0 +1,44 @@ +## Why run Huginn with docker + +You can play with or deploy Huginn inside of [docker](http://www.docker.io/). + +Getting Huginn up and running using docker is quick and painless once you have docker installed. The docker container is suitable for production and evaluation. Huginn uses environmental variables for configuration, so rather than having a .env file, the Docker container expects variables to be passed into the launch command. + +## Running the Container + +### Quick start to check out Huginn + +#### OSX GUI using Kitematic + +1. Download and install [Kitematic](https://www.docker.com/docker-kitematic) +* Start Kitematic and search for `cantino/huginn` +* Click `create` and wait for the container to be downloaded and booted +* Click on the link icon next to 'WEB PREVIEW' +* Log in to your Huginn instance using the username `admin` and password `password` + +#### OSX/Windows/Linux using docker machine + +1. Download [docker machine](https://docs.docker.com/machine/#installation) for your OS +* Follow the installation instructions untill you can successfully run `docker ps` +* Get the the IP of the VM running docker by running `docker-machine ls` +* Start your Huginn container using `docker run -it -p 5000:5000 cantino/huginn` +* Open Huginn in the browser [http://docker-machine ip:5000](http://:5000) +* Log in to your Huginn instance using the username `admin` and password `password` + +#### Linux + +1. Install docker using the [install instructions](https://docs.docker.com/installation/) +* Start your Huginn container using `docker run -it -p 5000:5000 cantino/huginn` +* Open Huginn in the browser [http://localhost:5000](http://localhost:5000) +* Log in to your Huginn instance using the username `admin` and password `password` + +## Configuration and linking to a database container + +Follow the [instructions on the docker hub registry](https://registry.hub.docker.com/u/cantino/huginn/) on how to configure Huginn using environment variables and linking the container to an external MySQL or PostgreSQL database. + +### Other options: + +Other Docker options: + +* If you don't want to use the official repo, see also: https://registry.hub.docker.com/u/andrewcurioso/huginn/ +* If you'd like to run Huginn's web process and job worker process in separate containers, another option is https://github.com/hackedu/huginn-docker. It also uses Unicorn as the web server and serves precompiled assets. From f815b1bdc7c299f81e65ed0ca3f979e4cc99b194 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sat, 4 Jul 2015 00:30:19 +0200 Subject: [PATCH 23/36] Moved heroku documentation and restructured directories --- doc/README.md | 18 ++- doc/{install/docker.md => docker/install.md} | 0 doc/heroku/install.md | 144 +++++++++++++++++++ doc/heroku/update.md | 9 ++ doc/{install => manual}/README.md | 4 +- doc/{install => manual}/installation.md | 0 doc/{install => manual}/requirements.md | 2 +- doc/{update/README.md => manual/update.md} | 0 8 files changed, 169 insertions(+), 8 deletions(-) rename doc/{install/docker.md => docker/install.md} (100%) create mode 100644 doc/heroku/install.md create mode 100644 doc/heroku/update.md rename doc/{install => manual}/README.md (66%) rename doc/{install => manual}/installation.md (100%) rename doc/{install => manual}/requirements.md (99%) rename doc/{update/README.md => manual/update.md} (100%) diff --git a/doc/README.md b/doc/README.md index 02581423..ea05ba3f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,11 +1,19 @@ # Documentation -## User documentation +## Installation and upgrading -- [Check out Huginn with Docker](install/docker.md) Run a local Huginn installation using Docker +### Docker (not recommended for production) +- [Check out Huginn with Docker](docker/install.md) Run a local Huginn installation using Docker -## Administrator documentation +### Manual installation -- [Install](install/README.md) Requirements, directory structures and installation from source. -- [Update](update/README.md) Update guides to upgrade your installation. +Manual installation which will guide through the steps to install Huginn on any Ubuntu 12.04/14.04 or Debian 6/7 server. + +- [Install](manual/README.md) Requirements, directory structures and installation from source. +- [Update](manual/update.md) Update your installation. + +### Heroku + +- [Deploy to Heroku](heroku/install.md) +- [Update](heroku/update.md) an existing Heroku deployment \ No newline at end of file diff --git a/doc/install/docker.md b/doc/docker/install.md similarity index 100% rename from doc/install/docker.md rename to doc/docker/install.md diff --git a/doc/heroku/install.md b/doc/heroku/install.md new file mode 100644 index 00000000..c2bdc3b2 --- /dev/null +++ b/doc/heroku/install.md @@ -0,0 +1,144 @@ +## Deploy to Heroku + +### Important things to keep in mind + +* Heroku's [Free plan](https://www.heroku.com/pricing) now requires your application to sleep for at least 6 hours a day, thus the `Hobby` plan is recommended. +* Heroku's free Postgres plan limits the number of database rows that you can have to 10,000, so you should be sure to set a low event retention schedule for your agents. +* The `setup_heroku` command points Heroku at a special Procfile (`deployment/heroku/Procfile.heroku`) that is designed to be run on only one Heroku web worker. If you want to run multiple workers, change the Heroku config variable `PROCFILE_PATH` with `heroku config:set PROCFILE_PATH=./Procfile` and switch back to the standard Huginn Procfile configuration. + +### Instructions + +* Install the [Heroku Toolbelt](https://toolbelt.heroku.com/) and then run `heroku login` +* Go into your huginn directory and run `bundle` +* Now, run the magic setup wizard: `bin/setup_heroku` +* That's it! +* If you make changes, you can re-run `bin/setup_heroku`, or just do `git push heroku master`. +* Signup for a free [Pingdom](https://www.pingdom.com/free/) account and have it ping your Huginn URL instance on Heroku. This should help keep your Heroku instance awake and watching the world for you. Keep in mind the Heroku's Free plan needs your application to sleep for 6 hours a day. + + +### Using your own mail server + +```bash +# Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com +# as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. +heroku config:set SMTP_DOMAIN=your-domain-here.com +heroku config:set SMTP_USER_NAME=you@gmail.com +heroku config:set SMTP_PASSWORD=somepassword +heroku config:set SMTP_SERVER=smtp.gmail.com + +# The address from which system emails will appear to be sent. +heroku config:set EMAIL_FROM_ADDRESS=you@gmail.com +``` + +### Backing up your data + +See: https://devcenter.heroku.com/articles/heroku-postgres-import-export + +### Example output from `bin/setup_heroku` + +``` +~/projects/oss/huginn (master)$ bin/setup_heroku + +Welcome andrew@example.com! It looks like you're logged into Heroku. + +It looks like you don't have a Heroku app set up yet for this repo. +You can either exit now and run 'heroku create', or I can do it for you. +Would you like me to create a Heroku app for you now in this repo? (y/n) y +Creating radiant-forest-1519... done, stack is cedar +http://radiant-forest-1519.herokuapp.com/ | git@heroku.com:radiant-forest-1519.git +Git remote heroku added +Your Heroku app name is radiant-forest-1519. Is this correct? (y/n) y +Setting up APP_SECRET_TOKEN... +Setting BUILDPACK_URL to https://github.com/ddollar/heroku-buildpack-multi.git +BUILDPACK_URL: https://github.com/ddollar/heroku-buildpack-multi.git +Setting PROCFILE_PATH to deployment/heroku/Procfile.heroku +PROCFILE_PATH: deployment/heroku/Procfile.heroku +Setting ON_HEROKU to true +Setting FORCE_SSL to true +Setting DOMAIN to radiant-forest-1519.herokuapp.com + +You need to set an invitation code for your Huginn instance. If you plan to share this instance, you will +tell this code to anyone who you'd like to invite. If you won't share it, then just set this to something +that people will not guess. +What code would you like to use? +What code would you like to use? something-secret +Setting INVITATION_CODE to something-secret + +Okay, let's setup outgoing email settings. The simplest solution is to use the free sendgrid Heroku addon. +If you'd like to use your own server, or your Gmail account, please see .env.example and set +SMTP_DOMAIN, SMTP_USER_NAME, SMTP_PASSWORD, and SMTP_SERVER with 'heroku config:set'. +Should I enable the free sendgrid addon? (y/n) y +Use `heroku addons:docs sendgrid` to view documentation. +SMTP_SERVER: smtp.sendgrid.net +SMTP_DOMAIN: heroku.com +SMTP_USER_NAME: app27830035@heroku.com +SMTP_PASSWORD: sflajgz0 +What email address would you like email to appear to be sent from? andrew@example.com +Setting EMAIL_FROM_ADDRESS to andrew@example.com +EMAIL_FROM_ADDRESS: andrew@example.com + +Should I push your current branch (master) to heroku? (y/n) y +This may take a moment... +Initializing repository, done. + +-----> Fetching custom git buildpack... done +-----> Multipack app detected +=====> Downloading Buildpack: https://github.com/cantino/heroku-selectable-procfile.git +=====> Detected Framework: Selectable Procfile +-----> Using deployment/heroku/Procfile.heroku as Procfile +=====> Downloading Buildpack: https://github.com/heroku/heroku-buildpack-ruby.git +=====> Detected Framework: Ruby +-----> Compiling Ruby/Rails +-----> Using Ruby version: ruby-2.0.0 +-----> Installing dependencies using 1.6.3 + Running: bundle install --without development:test --path vendor/bundle --binstubs vendor/bundle/bin -j4 --deployment + Fetching source index from https://rubygems.org/ + Fetching git://github.com/cantino/twitter-stream.git + Installing i18n 0.6.9 + Installing rake 10.3.2 + Installing minitest 5.3.5 + [...gems are installed...] + Your bundle is complete! + Gems in the groups development and test were not installed. + It was installed into ./vendor/bundle + Post-install message from httparty: + When you HTTParty, you must party hard! + Post-install message from rufus-scheduler: + Bundle completed (133.85s) + Cleaning up the bundler cache. +-----> Preparing app for Rails asset pipeline + Running: rake assets:precompile + I, [2014-07-26T20:36:06.069156 #5939] INFO -- : Writing /tmp/build_7b0d30bd-3c35-46dc-b73d-b5f05754d340/public/assets/select2x2-ec4bf2b76c97838b357413d72a2f69cf.png [...] + Asset precompilation completed (42.28s) + Cleaning assets + Running: rake assets:clean + +Using release configuration from last framework (Ruby). +-----> Discovering process types + Procfile declares types -> web + Default types for Multipack -> console, rake, worker + +-----> Compressing... done, 45.1MB +-----> Launching... done, v19 + http://radiant-forest-1519.herokuapp.com/ deployed to Heroku + +To git@heroku.com:radiant-forest-1519.git + * [new branch] master -> master +Running database migrations... +Running `rake db:migrate` attached to terminal... up, run.3341 + +[...migrations run...] + +I can make an admin user on your new Huginn instance and setup some example Agents. +Should I create a new admin user and some example Agents? (y/n) y + +Okay, what is your email address? andrew@example.com +And what username would you like to login as? andrew +Finally, what password would you like to use? +Just a moment... + + +Okay, you should be all set! Visit https://radiant-forest-1519.herokuapp.com and login as 'andrew' with your password. + +Done! +``` \ No newline at end of file diff --git a/doc/heroku/update.md b/doc/heroku/update.md new file mode 100644 index 00000000..8b377e86 --- /dev/null +++ b/doc/heroku/update.md @@ -0,0 +1,9 @@ +## Update an existing Heroku deployment + +Once you've run `bin/setup_heroku`, you should have 'cantino/huginn' as a remote in git. (Check with `git remote -v`.) Now, you can update your Heroku installation with the following commands: + +```sh +git fetch origin +git merge origin/master +git push -f heroku master # note: this will wipe out any code changes that only exist on Heroku! +``` diff --git a/doc/install/README.md b/doc/manual/README.md similarity index 66% rename from doc/install/README.md rename to doc/manual/README.md index e0a8078f..d8ddbae2 100644 --- a/doc/install/README.md +++ b/doc/manual/README.md @@ -1,5 +1,5 @@ -# Installation +# Manual Installation - [Requirements](requirements.md) Software and hardware requirements to run the Huginn installation - [Install](installation.md) Installation guide for Ubundu/Debian -- [Update](../update/README.md) Update an existing Huginn installation \ No newline at end of file +- [Update](update.md) Update an existing Huginn installation \ No newline at end of file diff --git a/doc/install/installation.md b/doc/manual/installation.md similarity index 100% rename from doc/install/installation.md rename to doc/manual/installation.md diff --git a/doc/install/requirements.md b/doc/manual/requirements.md similarity index 99% rename from doc/install/requirements.md rename to doc/manual/requirements.md index e65f5f5d..a9aa4c82 100644 --- a/doc/install/requirements.md +++ b/doc/manual/requirements.md @@ -4,7 +4,7 @@ ### Supported Unix distributions by this guide -- Ubuntu (12.04 and 14.04) +- Ubuntu (14.04 and 12.04) - Debian (Jessie and Wheezy) ### Unsupported Unix distributions diff --git a/doc/update/README.md b/doc/manual/update.md similarity index 100% rename from doc/update/README.md rename to doc/manual/update.md From cec02c1bd6d171246c426d2b660c8191fd69b3f0 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sat, 1 Aug 2015 16:27:25 +0200 Subject: [PATCH 24/36] Use runit instead of upstart as init system --- doc/manual/installation.md | 31 +++++-------- doc/manual/update.md | 37 ++++++++------- lib/tasks/production.rake | 94 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 37 deletions(-) create mode 100644 lib/tasks/production.rake diff --git a/doc/manual/installation.md b/doc/manual/installation.md index 90b88af7..3251e958 100644 --- a/doc/manual/installation.md +++ b/doc/manual/installation.md @@ -48,7 +48,7 @@ Import node.js repository (can be skipped on Ubuntu and Debian Jessie): Install the required packages (needed to compile Ruby and native extensions to Ruby gems): - sudo apt-get install -y build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz + sudo apt-get install -y runit build-essential git zlib1g-dev libyaml-dev libssl-dev libgdbm-dev libreadline-dev libncurses5-dev libffi-dev curl openssh-server checkinstall libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev logrotate python-docutils pkg-config cmake nodejs graphviz ## 2. Ruby @@ -229,6 +229,10 @@ When done you see `See the Huginn Wiki for more Agent examples! https://github. sudo -u huginn -H bundle exec rake db:seed RAILS_ENV=production SEED_USERNAME=admin SEED_PASSWORD=yourpassword +### Compile Assets + + sudo -u huginn -H bundle exec rake assets:precompile RAILS_ENV=production + ### Install Init Script Huginn uses foreman to generate the init scripts based on a `Procfile` @@ -237,31 +241,20 @@ Edit the `Procfile` and choose one of the suggested versions for production sudo -u huginn -H editor Procfile -**Debian only** Install upstart and reboot the system (skip this step on Ubuntu): +Export the init scripts: - sudo apt-get install -y --force-yes upstart - sudo reboot - # After you you logged back in go to Huginn installation folder - cd /home/huginn/huginn + sudo rake production:export -Export the init scripts using foreman: - - sudo rm /etc/init/huginn* - sudo foreman export upstart -a huginn /etc/init - -**Note:** You have to re-export the init script every time you change the configuration in `.env`! +**Note:** You have to re-export the init script every time you change the configuration in `.env` or your `Procfile`! ### Setup Logrotate sudo cp lib/support/logrotate/huginn /etc/logrotate.d/huginn -### Compile Assets - sudo -u huginn -H bundle exec rake assets:precompile RAILS_ENV=production +### Ensure Your Huginn Instance Is Running -### Start Your Huginn Instance - - sudo start huginn + sudo rake production:status ## 7. Nginx @@ -335,9 +328,7 @@ Restart Nginx, export the init script and restart Huginn: ``` cd /home/huginn/huginn sudo service nginx restart -sudo rm /etc/init/huginn* -sudo foreman export upstart -a huginn /etc/init -sudo restart huginn +sudo rake production:export ``` Using a self-signed certificate is discouraged, but if you must use it follow the normal directions. Then generate the certificate: diff --git a/doc/manual/update.md b/doc/manual/update.md index 922fc741..ca1de2dc 100644 --- a/doc/manual/update.md +++ b/doc/manual/update.md @@ -1,19 +1,25 @@ # Update -### 0. Stop server - -``` -sudo stop huginn -``` - -### 1. Store the current version +### 0. Ensure depencies are up to date ``` cd /home/huginn/huginn +sudo rake production:check +``` + +### 1. Stop server + +``` +sudo rake production:stop +``` + +### 2. Store the current version + +``` export OLD_VERSION=`git rev-parse HEAD` ``` -### 2. Update the code +### 3. Update the code Back up changed files @@ -35,7 +41,7 @@ Restore backed up files sudo -u huginn -H cp Procfile.bak Procfile ``` -### 3. Install gems, migrate and precompile assets +### 4. Install gems, migrate and precompile assets ``` cd /home/huginn/huginn @@ -46,11 +52,11 @@ sudo -u huginn -H bundle install --deployment --without development test sudo -u huginn -H bundle exec rake db:migrate RAILS_ENV=production # Clean up assets and cache -sudo -u huginn -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production +sudo -u huginn -H bundle exec rake assets:clean assets:precompile tmp:cache:clear RAILS_ENV=production ``` -### 4. Update the Procfile +### 5. Update the Procfile Check for changes made to the default `Procfile` ``` @@ -62,7 +68,7 @@ Update your `Procfile` if the default options of the version you are using chang sudo -u huginn -H editor Procfile ``` -### 5. Update the .env file +### 6. Update the .env file Check for changes made to the example `.env` ``` @@ -75,13 +81,10 @@ sudo -u huginn -H editor .env ``` -### 6. Export init script and start Huginn +### 7. Export init script and start Huginn ``` # Export the init script -sudo rm /etc/init/huginn* -sudo foreman export upstart -a huginn /etc/init -# Start Huginn -sudo start huginn +sudo rake production:export ``` diff --git a/lib/tasks/production.rake b/lib/tasks/production.rake new file mode 100644 index 00000000..d3ed2563 --- /dev/null +++ b/lib/tasks/production.rake @@ -0,0 +1,94 @@ +def failed; "[ \033[31mFAIL\033[0m ]"; end +def ok; "[ \033[32mOK\033[0m ]"; end + +def run_as_root + return true if ENV['USER'] == 'root' + puts "#{failed} Please run this command as root or with sudo\n\n" + exit -1 +end + +def runit_installed + return true unless `which sv` && $?.to_i != 0 + puts "#{failed} Please install runit: \n\nsudo apt-get install runit\n\n" + exit -1 +end + +def remove_upstart_config + return true unless File.exists?('/etc/init/huginn.conf') + puts "#{failed} Please stop huginn and remove the huginn upstart init scripts:\n\n" + puts "sudo stop huginn" + puts "sudo rm /etc/init/huginn*\n\n" + exit -1 +end + +namespace :production do + task :check do |t| + remove_upstart_config + runit_installed + puts "#{ok} Everything is fine" if t.application.top_level_tasks.include? 'production:check' + end + + task :stop => :check do + puts "Stopping huginn ..." + run_sv('stop') + end + + task :start => :check do + puts "Startig huginn ..." + run_sv('start') + end + + task :status => :check do + run_sv('status') + end + + task :restart => :check do + puts "Restarting huginn ..." + run_sv('restart') + end + + task :export => :check do + run_as_root + Rake::Task['production:stop'].execute + puts "Exporting new services ..." + run('rm -rf /etc/service/huginn*') + run('foreman export runit -a huginn -l /home/huginn/huginn/log /etc/service') + services = Dir.glob('/etc/service/huginn*') + while services.length > 0 + services.each do |p| + supervise = File.join(p, 'supervise') + next if !Dir.exists?(supervise) + run("chown -R huginn:huginn #{p}") + services.delete(p) + end + sleep 0.1 + end + end +end + +def run_sv(command) + Dir.glob('/etc/service/huginn*').each do |p| + with_retries do + run("sv #{command} #{File.basename(p)}") + end + end +end + +def run(cmd, verbose=false) + output = `#{cmd}` + if $?.to_i != 0 + raise "'#{cmd}' exited with a non-zero return value: #{output}" + end + puts output if verbose && output.strip != '' + output +end + +def with_retries(&block) + tries ||= 5 + output = block.call +rescue StandardError => e + retry unless (tries -= 1).zero? + raise e +else + puts output +end \ No newline at end of file From b35efc2eec94742031139409acb590eece319266 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sat, 1 Aug 2015 18:48:48 +0200 Subject: [PATCH 25/36] Update README with links to new documentation --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f9e0bec7..944e8271 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,11 @@ And now, some example screenshots. Below them are instructions to get you start ## Getting Started -### Quick Start +### Docker + +The quickest and easiest way to check out Huginn is to use the offical Docker image. Have a look at the [documentation](./doc/docker/install.md). + +### Local Installation If you just want to play around, you can simply fork this repository, then perform the following steps: @@ -82,12 +86,18 @@ All agents have specs! Test all specs with `bundle exec rspec`, or test a specif ## Deployment -Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup. Be sure to click 'View it' after launch!) +### Heroku + +Try Huginn on Heroku: [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy) (Takes a few minutes to setup. Read the [documentation](./doc/heroku/install.md) while you are waiting and be sure to click 'View it' after launch!) Huginn works on the free version of Heroku [with limitations](https://github.com/cantino/huginn/wiki/Run-Huginn-for-free-on-Heroku). For non-experimental use, we recommend Heroku's cheapest paid plan or our Docker container. Please see [the Huginn Wiki](https://github.com/cantino/huginn/wiki#deploying-huginn) for detailed deployment strategies for different providers. +### Manual installation on any server + +Have a look at the [installation guide](./doc/manual/README.md). + ### Optional Setup #### Setup for private development From a16fb09fa6476d1dae021a8d843801d8114f1734 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 2 Aug 2015 12:48:01 +0200 Subject: [PATCH 26/36] Add capistrano 3 and documentation --- .env.example | 8 ++++++ Capfile | 10 +++++-- Gemfile | 4 +++ Gemfile.lock | 21 +++++++++++++++ config/deploy.rb | 54 +++++++++++++++++++++++++++++++++++++ config/deploy/production.rb | 1 + doc/README.md | 1 + doc/manual/README.md | 3 ++- doc/manual/capistrano.md | 47 ++++++++++++++++++++++++++++++++ 9 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 config/deploy.rb create mode 100644 config/deploy/production.rb create mode 100644 doc/manual/capistrano.md diff --git a/.env.example b/.env.example index 32debfdb..0788a6ad 100644 --- a/.env.example +++ b/.env.example @@ -175,3 +175,11 @@ DELAYED_JOB_MAX_RUNTIME=2 # Amount of seconds for delayed_job to sleep before checking for new jobs DELAYED_JOB_SLEEP_DELAY=10 + +########################################################## +# Capistrano deployment (read the documentation FIXME) # +########################################################## + +#CAPISTRANO_DEPLOY_SERVER= +#CAPISTRANO_DEPLOY_USER= +#CAPISTRANO_DEPLOY_REPO_URL= diff --git a/Capfile b/Capfile index f3634e86..83b98d42 100644 --- a/Capfile +++ b/Capfile @@ -1,2 +1,8 @@ -load 'deploy' -load 'config/deploy' +# Load DSL and set up stages +require 'capistrano/setup' +# Include default deployment tasks +require 'capistrano/deploy' + +require 'capistrano/bundler' +require 'capistrano/rails/assets' +require 'capistrano/rails/migrations' diff --git a/Gemfile b/Gemfile index 74cee488..c305fc6e 100644 --- a/Gemfile +++ b/Gemfile @@ -103,6 +103,10 @@ group :development do gem 'guard-rspec' gem 'letter_opener_web' + gem 'capistrano', '~> 3.4.0' + gem 'capistrano-rails', '~> 1.1' + gem 'capistrano-bundler', '~> 1.1.4' + group :test do gem 'coveralls', require: false gem 'delorean' diff --git a/Gemfile.lock b/Gemfile.lock index 9ecfe29e..1a569e77 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,6 +84,16 @@ GEM rails (>= 3.1) buftok (0.2.0) builder (3.2.2) + capistrano (3.4.0) + i18n + rake (>= 10.0.0) + sshkit (~> 1.3) + capistrano-bundler (1.1.4) + capistrano (~> 3.1) + sshkit (~> 1.2) + capistrano-rails (1.1.3) + capistrano (~> 3.1) + capistrano-bundler (~> 1.1) celluloid (0.16.0) timers (~> 4.0.0) chronic (0.10.2) @@ -95,6 +105,7 @@ GEM coffee-script-source execjs coffee-script-source (1.9.1) + colorize (0.7.7) cookiejar (0.3.2) coveralls (0.7.1) multi_json (~> 1.3) @@ -267,6 +278,9 @@ GEM mysql2 (0.3.16) naught (1.0.0) net-ftp-list (3.2.8) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (2.9.2) netrc (0.10.3) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) @@ -425,6 +439,10 @@ GEM actionpack (>= 3.0) activesupport (>= 3.0) sprockets (>= 2.8, < 4.0) + sshkit (1.7.1) + colorize (>= 0.7.0) + net-scp (>= 1.1.2) + net-ssh (>= 2.8.0) string-scrub (0.0.5) systemu (2.6.4) term-ansicolor (1.3.0) @@ -500,6 +518,9 @@ DEPENDENCIES binding_of_caller bootstrap-kaminari-views (~> 0.0.3) bundler (>= 1.5.0) + capistrano (~> 3.4.0) + capistrano-bundler (~> 1.1.4) + capistrano-rails (~> 1.1) coffee-rails (~> 4.1.0) coveralls daemons (~> 1.1.9) diff --git a/config/deploy.rb b/config/deploy.rb new file mode 100644 index 00000000..125f3635 --- /dev/null +++ b/config/deploy.rb @@ -0,0 +1,54 @@ +require 'dotenv' +Dotenv.load + +# config valid only for current version of Capistrano +lock '3.4.0' + +set :application, 'huginn' +set :repo_url, ENV['CAPISTRANO_DEPLOY_REPO_URL'] || 'https://github.com/cantino/huginn.git' + +# Default branch is :master +set :branch, ENV['BRANCH'] || 'master' + +set :deploy_to, '/home/huginn' + +# Set to :debug for verbose ouput +set :log_level, :info + +# Default value for :linked_files is [] +set :linked_files, fetch(:linked_files, []).push('.env', 'Procfile', 'config/unicorn.rb') + +# Default value for linked_dirs is [] +set :linked_dirs, fetch(:linked_dirs, []).push('log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'vendor/bundle') + +# Default value for keep_releases is 5 +# set :keep_releases, 5 + +set :bundle_jobs, 4 + +set :conditionally_migrate, true # Defaults to false. If true, it's skip migration if files in db/migrate not modified + +task :deploy => [:production] + +namespace :deploy do + after 'check:make_linked_dirs', :migrate_to_cap do + on roles(:all) do + # Try to migrate from the manual installation to capistrano directory structure + next if test('[ -L ~/huginn ]') + fetch(:linked_files).each do |f| + if !test("[ -f ~/shared/#{f} ] ") && test("[ -f ~/huginn/#{f} ]") + execute("cp ~/huginn/#{f} ~/shared/#{f}") + end + end + execute('mv ~/huginn ~/huginn.manual') + execute('ln -s ~/current ~/huginn') + end + end + after :publishing, :restart do + on roles(:all) do + within release_path do + execute :rake, 'production:restart' + end + end + end +end diff --git a/config/deploy/production.rb b/config/deploy/production.rb new file mode 100644 index 00000000..f0196d2e --- /dev/null +++ b/config/deploy/production.rb @@ -0,0 +1 @@ +server ENV['CAPISTRANO_DEPLOY_SERVER'], user: ENV['CAPISTRANO_DEPLOY_USER'] || 'huginn', roles: %w{app db web} \ No newline at end of file diff --git a/doc/README.md b/doc/README.md index ea05ba3f..b9561173 100644 --- a/doc/README.md +++ b/doc/README.md @@ -12,6 +12,7 @@ Manual installation which will guide through the steps to install Huginn on any - [Install](manual/README.md) Requirements, directory structures and installation from source. - [Update](manual/update.md) Update your installation. +- Deploy updates via [Capistrano](manual/capistrano.md). ### Heroku diff --git a/doc/manual/README.md b/doc/manual/README.md index d8ddbae2..6293e77f 100644 --- a/doc/manual/README.md +++ b/doc/manual/README.md @@ -2,4 +2,5 @@ - [Requirements](requirements.md) Software and hardware requirements to run the Huginn installation - [Install](installation.md) Installation guide for Ubundu/Debian -- [Update](update.md) Update an existing Huginn installation \ No newline at end of file +- [Update](update.md) Update an existing Huginn installation +- Deploy updates via [Capistrano](capistrano.md) \ No newline at end of file diff --git a/doc/manual/capistrano.md b/doc/manual/capistrano.md new file mode 100644 index 00000000..8701340d --- /dev/null +++ b/doc/manual/capistrano.md @@ -0,0 +1,47 @@ +# Deploy updates via Capistrano + +After you followed the [manual installation guide](installation.md) it is simple to push updates to your huginn instance using capistrano. + +### 1. Ensure you have SSH access to your server via the huginn user + +Either set a password for the huginn user or add your public SSH key: + + # Set password + sudo passwd huginn + + # Or add a SSH key + sudo -u huginn -H mkdir -p /home/huginn/.ssh + sudo -u huginn -H editor /home/huginn/.ssh/authorized_keys + sudo -u huginn -H chmod -R 700 /home/huginn/.ssh + +### 2. Configure Capistrano on your local machine + +Add Capistrano configuration to you local `.env`: + + CAPISTRANO_DEPLOY_SERVER= + CAPISTRANO_DEPLOY_USER=huginn + CAPISTRANO_DEPLOY_REPO_URL=https://github.com/cantino/huginn.git + + +### 3. Run Capistrano + +You can now run Capistrano and update your server: + + cap production deploy + +If you want to deploy a different branch, pass it as environment variable: + + cap production deploy BRANCH=awesome-feature + +### Changes to remote .env and Procfile + +If you want to change the `.env`, `Procfile` or `config/unicorn.rb` of your installation you still need to do it on your server, do not forget to export the init scripts after your are done: + + cd /home/huginn/huginn + # Whichever you want to change + sudo -u huginn -H editor Procfile + sudo -u huginn -H editor .env + sudo -u huginn -H editor config/unicorn.rb + # Export init scripts and restart huginn + sudo rake production:export + From 9f260d6c217097ce16c1ffaf5f61e8f3f7aa8265 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 2 Aug 2015 16:36:55 +0200 Subject: [PATCH 27/36] Remove chef cookbooks and vagrant file Both are now maintained in a seperate repository: https://github.com/elijah/cookbook-huginn --- deployment/.chef/knife.rb | 3 - deployment/Cheffile | 11 -- deployment/Cheffile.lock | 71 ----------- deployment/Vagrantfile | 40 ------- deployment/nodes/.gitignore | 0 deployment/roles/huginn_development.json | 30 ----- deployment/roles/huginn_production.json | 30 ----- .../huginn_development/recipes/default.rb | 78 ------------ .../huginn_production/files/default/Procfile | 2 - .../files/default/env.example | 110 ----------------- .../files/default/nginx.conf | 36 ------ .../files/default/unicorn.rb | 35 ------ .../huginn_production/recipes/default.rb | 111 ------------------ 13 files changed, 557 deletions(-) delete mode 100644 deployment/.chef/knife.rb delete mode 100644 deployment/Cheffile delete mode 100644 deployment/Cheffile.lock delete mode 100644 deployment/Vagrantfile delete mode 100644 deployment/nodes/.gitignore delete mode 100644 deployment/roles/huginn_development.json delete mode 100644 deployment/roles/huginn_production.json delete mode 100644 deployment/site-cookbooks/huginn_development/recipes/default.rb delete mode 100644 deployment/site-cookbooks/huginn_production/files/default/Procfile delete mode 100644 deployment/site-cookbooks/huginn_production/files/default/env.example delete mode 100644 deployment/site-cookbooks/huginn_production/files/default/nginx.conf delete mode 100644 deployment/site-cookbooks/huginn_production/files/default/unicorn.rb delete mode 100644 deployment/site-cookbooks/huginn_production/recipes/default.rb diff --git a/deployment/.chef/knife.rb b/deployment/.chef/knife.rb deleted file mode 100644 index ed87ba43..00000000 --- a/deployment/.chef/knife.rb +++ /dev/null @@ -1,3 +0,0 @@ -cookbook_path ["cookbooks", "site-cookbooks"] -role_path "roles" -data_bag_path "data_bags" \ No newline at end of file diff --git a/deployment/Cheffile b/deployment/Cheffile deleted file mode 100644 index e8d16120..00000000 --- a/deployment/Cheffile +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env ruby -#^syntax detection - -site 'http://community.opscode.com/api/v1' - -cookbook 'ark' -cookbook 'runit' -cookbook 'git', :git => 'git://github.com/opscode-cookbooks/git.git' -cookbook 'nginx', :git => 'git://github.com/opscode-cookbooks/nginx.git' -cookbook 'mysql', :git => 'git://github.com/opscode-cookbooks/mysql.git' -cookbook 'nodejs', :git => 'git://github.com/redguide/nodejs.git' diff --git a/deployment/Cheffile.lock b/deployment/Cheffile.lock deleted file mode 100644 index a1d51694..00000000 --- a/deployment/Cheffile.lock +++ /dev/null @@ -1,71 +0,0 @@ -SITE - remote: http://community.opscode.com/api/v1 - specs: - apt (2.4.0) - bluepill (2.3.1) - rsyslog (>= 0.0.0) - build-essential (2.0.2) - chef_handler (1.1.6) - dmg (2.2.0) - ohai (2.0.0) - rsyslog (1.12.2) - runit (1.5.10) - build-essential (>= 0.0.0) - yum (~> 3.0) - yum-epel (>= 0.0.0) - windows (1.31.0) - chef_handler (>= 0.0.0) - yum (3.2.0) - yum-epel (0.3.6) - yum (~> 3.0) - -GIT - remote: git://github.com/redguide/nodejs.git - ref: master - sha: 91d8d11f189d13815d56af5700168e1cad88ae7f - specs: - nodejs (1.3.0) - apt (>= 0.0.0) - build-essential (>= 0.0.0) - yum-epel (>= 0.0.0) - -GIT - remote: git://github.com/opscode-cookbooks/git.git - ref: master - sha: b1bca76aaf3a3a2131744f17f6e5087e22fa3c40 - specs: - git (4.0.3) - build-essential (>= 0.0.0) - dmg (>= 0.0.0) - runit (>= 1.0) - windows (>= 0.0.0) - yum (~> 3.0) - yum-epel (>= 0.0.0) - -GIT - remote: git://github.com/opscode-cookbooks/mysql.git - ref: master - sha: 4b70e99730ab4a7ce6c1dd7a35654a764fb6e0fe - specs: - mysql (5.2.13) - -GIT - remote: git://github.com/opscode-cookbooks/nginx.git - ref: master - sha: 45588ee2a5c7144a0ef2a5992e7f273542236d27 - specs: - nginx (2.7.3) - apt (~> 2.2) - bluepill (~> 2.3) - build-essential (~> 2.0) - ohai (~> 2.0) - runit (~> 1.2) - yum-epel (~> 0.3) - -DEPENDENCIES - git (>= 0) - mysql (>= 0) - nginx (>= 0) - nodejs (>= 0) - runit (>= 0) - diff --git a/deployment/Vagrantfile b/deployment/Vagrantfile deleted file mode 100644 index dfaa8bac..00000000 --- a/deployment/Vagrantfile +++ /dev/null @@ -1,40 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - config.omnibus.chef_version = :latest - - config.vm.provision :chef_solo do |chef| - chef.roles_path = "roles" - chef.cookbooks_path = ["cookbooks", "site-cookbooks"] - chef.add_role("huginn_development") - # chef.add_role("huginn_production") - end - - config.vm.provider :virtualbox do |vb, override| - #vb.memory = 1024 - #vb.cpus = 4 - override.vm.box = "hashicorp/precise64" - override.vm.network :forwarded_port, host: 3000, guest: 3000 - end - - config.vm.provider :parallels do |prl, override| - override.vm.box = "parallels/ubuntu-12.04" - end - - config.vm.provider :aws do |aws, override| - aws.ami = ENV['AWS_AMI'] || "ami-828675f5" - aws.region = ENV['AWS_REGION'] || "eu-west-1" - aws.instance_type = "t1.micro" - - override.vm.box = "dummy" - override.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box" - override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"] - override.ssh.username = ENV['AWS_SSH_USER'] || "ubuntu" - - aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] - aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] - aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] - aws.security_groups = [ ENV["AWS_SECURITY_GROUP"] ] - end -end diff --git a/deployment/nodes/.gitignore b/deployment/nodes/.gitignore deleted file mode 100644 index e69de29b..00000000 diff --git a/deployment/roles/huginn_development.json b/deployment/roles/huginn_development.json deleted file mode 100644 index 22e48682..00000000 --- a/deployment/roles/huginn_development.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - -"name" : "huginn_development", - -"chef_type" : "role", - -"json_class" : "Chef::Role", - -"description" : "Huginn Development Environment", - -"default_attributes" : { - "mysql" : { - "server_root_password" : "", - "server_repl_password" : "", - "server_debian_password" : "" - }, - "nginx" : { - "init_style" : "upstart" - } -}, - -"run_list":[ - "recipe[git]", - "recipe[apt]", - "recipe[mysql::server]", - "recipe[mysql::client]", - "recipe[nodejs::nodejs_from_binary]", - "recipe[huginn_development]" - ] -} diff --git a/deployment/roles/huginn_production.json b/deployment/roles/huginn_production.json deleted file mode 100644 index 81ade074..00000000 --- a/deployment/roles/huginn_production.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - -"name" : "huginn_production", - -"chef_type" : "role", - -"json_class" : "Chef::Role", - -"description" : "Huginn Production Environment", - -"default_attributes" : { - "mysql": { - "server_root_password": "password", - "server_repl_password": "", - "server_debian_password": "" - }, - "nginx" : { - "init_style" : "upstart" - } -}, - -"run_list":[ - "recipe[git]", - "recipe[apt]", - "recipe[mysql::server]", - "recipe[nodejs::nodejs_from_binary]", - "recipe[nginx]", - "recipe[huginn_production]" - ] -} diff --git a/deployment/site-cookbooks/huginn_development/recipes/default.rb b/deployment/site-cookbooks/huginn_development/recipes/default.rb deleted file mode 100644 index 1909a272..00000000 --- a/deployment/site-cookbooks/huginn_development/recipes/default.rb +++ /dev/null @@ -1,78 +0,0 @@ -include_recipe 'apt' -include_recipe 'build-essential' - -user "huginn" do - action :create - system true - home "/home/huginn" - password "$6$ZwO6b.6tij$SMa8UIwtESGDxB37NwHsct.gJfXWmmflNbH.oypwJ9y0KkzMkCdw7D14iK7GX9C4CWSEcpGOFUow7p01rQFu5." - supports :manage_home => true - gid "sudo" - shell "/bin/bash" -end - -group "huginn" do - members ["huginn"] - action :create -end - -%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "libffi-dev" "libssl-dev").each do |pkg| - package pkg do - action :install - end -end - -bash "Setting default ruby and gem versions to 1.9" do - code <<-EOH - if [ $(readlink /usr/bin/ruby) != "ruby1.9.1" ] - then - update-alternatives --set ruby /usr/bin/ruby1.9.1 - fi - - if [ $(readlink /usr/bin/gem) != "gem1.9.1" ] - then - update-alternatives --set gem /usr/bin/gem1.9.1 - fi - EOH -end - -git "/home/huginn/huginn" do - repository 'git://github.com/cantino/huginn.git' - reference 'master' - action :sync - user "huginn" -end - -gem_package("rake") -gem_package("bundle") - -bash "Setting huginn user with NOPASSWD option" do - cwd "/etc/sudoers.d" - code <<-EOH - touch huginn - chmod 0440 huginn - echo "huginn ALL=(ALL) NOPASSWD:ALL" >> huginn - EOH -end - -bash "huginn dependencies" do - user "huginn" - cwd "/home/huginn/huginn" - code <<-EOH - export LANG="en_US.UTF-8" - export LC_ALL="en_US.UTF-8" - sudo bundle install - sed s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ .env.example > .env - sudo bundle exec rake db:create - sudo bundle exec rake db:migrate - sudo bundle exec rake db:seed - EOH -end - -bash "huginn has been installed and will start in a minute" do - user "huginn" - cwd "/home/huginn/huginn" - code <<-EOH - sudo nohup foreman start & - EOH -end diff --git a/deployment/site-cookbooks/huginn_production/files/default/Procfile b/deployment/site-cookbooks/huginn_production/files/default/Procfile deleted file mode 100644 index fcc42611..00000000 --- a/deployment/site-cookbooks/huginn_production/files/default/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: sudo bundle exec unicorn_rails -c config/unicorn.rb -E production -jobs: sudo RAILS_ENV=production bundle exec rails runner bin/threaded.rb \ No newline at end of file diff --git a/deployment/site-cookbooks/huginn_production/files/default/env.example b/deployment/site-cookbooks/huginn_production/files/default/env.example deleted file mode 100644 index aa0013b0..00000000 --- a/deployment/site-cookbooks/huginn_production/files/default/env.example +++ /dev/null @@ -1,110 +0,0 @@ -# ==== Required configuration settings for Huginn ==== - -# Replace the following with the output from "rake secret" -APP_SECRET_TOKEN=REPLACE_ME_NOW! - -# This is the domain where your Huginn instance will be running. The default should work -# for development, but it needs to be changed to your Huginn domain when you deploy to a -# production environment (e.g., yourdomain.com, possibly including a port). -#DOMAIN=localhost:3000 - -############################ -# Database Setup # -############################ - -DATABASE_ADAPTER=mysql2 -DATABASE_ENCODING=utf8 -DATABASE_RECONNECT=true -DATABASE_NAME=huginn_production -DATABASE_POOL=5 -DATABASE_USERNAME=root -DATABASE_PASSWORD=password -#DATABASE_HOST=your-domain-here.com -#DATABASE_PORT=3306 -#DATABASE_SOCKET=/tmp/mysql.sock - -# ==== Additional required production settings ==== - -# Configure Rails environment. This should only be needed in production and may cause errors in development. -RAILS_ENV=production - -# Should Rails force all requests to use SSL? -FORCE_SSL=false - -############################ -# Allowing Signups # -############################ - -# This invitation code will be required for users to signup with your Huginn installation. -# You can see its use in user.rb. PLEASE CHANGE THIS! -INVITATION_CODE=try-huginn - -# If you don't want to require users to have an invitation code, set this to true -SKIP_INVITATION_CODE=false - -############################# -# Email Configuration # -############################# - -# Outgoing email settings. To use Gmail or Google Apps, put your Google Apps domain or gmail.com -# as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD. -# -# PLEASE NOTE: In order to enable sending real emails via SMTP locally (e.g., when not in the production Rails environment), -# you must also set SEND_EMAIL_IN_DEVELOPMENT to true below. - -SMTP_DOMAIN=your-domain-here.com -SMTP_USER_NAME=you@gmail.com -SMTP_PASSWORD=somepassword -SMTP_SERVER=smtp.gmail.com -SMTP_PORT=587 -SMTP_AUTHENTICATION=plain -SMTP_ENABLE_STARTTLS_AUTO=true - -# Set to true to send real emails via SMTP when running in the development Rails environment. -# Set to false to have emails intercepted in development and displayed at http://localhost:3000/letter_opener -SEND_EMAIL_IN_DEVELOPMENT=false - -# The address from which system emails will appear to be sent. -EMAIL_FROM_ADDRESS=from_address@gmail.com - -########################### -# Agent Logging # -########################### - -# Number of lines of log messages to keep per Agent -AGENT_LOG_LENGTH=200 - -############################# -# AWS and Mechanical Turk # -############################# - -# AWS Credentials for MTurk -AWS_ACCESS_KEY_ID=YOUR-AWS-ACCESS-KEY-ID -AWS_ACCESS_KEY=YOUR-AWS-ACCESS-KEY - -# Set AWS_SANDBOX to true if you're developing Huginn code. -AWS_SANDBOX=false - -######################## -# Various Settings # -######################## - -# Specify the HTTP backend library for Faraday, used in WebsiteAgent. -# You can change this depending on the performance and stability you -# need for your service. Any choice other than "typhoeus", -# "net_http", or "em_http" should require you to bundle a corresponding -# gem via Gemfile. -FARADAY_HTTP_BACKEND=typhoeus - -# Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)] -# You should not allow this on a shared Huginn box because it is not secure. -ALLOW_JSONPATH_EVAL=false - -# Enable this setting to allow insecure Agents like the ShellCommandAgent. Only do this -# when you trust everyone using your Huginn installation. -ENABLE_INSECURE_AGENTS=false - -# Use Graphviz for generating diagrams instead of using Google Chart -# Tools. Specify a dot(1) command path built with SVG support -# enabled. -#USE_GRAPHVIZ_DOT=dot diff --git a/deployment/site-cookbooks/huginn_production/files/default/nginx.conf b/deployment/site-cookbooks/huginn_production/files/default/nginx.conf deleted file mode 100644 index 6729ccc0..00000000 --- a/deployment/site-cookbooks/huginn_production/files/default/nginx.conf +++ /dev/null @@ -1,36 +0,0 @@ -#worker_process 2; -user huginn huginn; - -events { - worker_connections 1024; - accept_mutex on; -} - -http { - types_hash_max_size 2048; - include mime.types; - - upstream huginn_server { - server unix:/home/huginn/shared/tmp/sockets/unicorn.sock; - } - - server { - listen 80; - server_name _; - keepalive_timeout 5; - root /home/huginn/current/public; - try_files $uri/index.html $uri.html $uri @app; - error_page 500 502 503 504 /500.html; - location = /500.html { - root /home/huginn/current/public; - } - location @app { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_pass http://huginn_server; - } -} -} - diff --git a/deployment/site-cookbooks/huginn_production/files/default/unicorn.rb b/deployment/site-cookbooks/huginn_production/files/default/unicorn.rb deleted file mode 100644 index 8a675426..00000000 --- a/deployment/site-cookbooks/huginn_production/files/default/unicorn.rb +++ /dev/null @@ -1,35 +0,0 @@ -app_path = "/home/huginn/current" - -worker_processes 2 -preload_app true -timeout 180 -listen '/home/huginn/shared/tmp/sockets/unicorn.sock' - -working_directory app_path - -rails_env = ENV['RAILS_ENV'] || 'production' - -# Log everything to one file -stderr_path "log/unicorn_out.log" -stdout_path "log/unicorn_err.log" - -# Set master PID location -pid '/home/huginn/shared/tmp/pids/unicorn.pid' - -before_fork do |server, worker| - defined?(ActiveRecord::Base) and - ActiveRecord::Base.connection.disconnect! - old_pid = "#{server.config[:pid]}.oldbin" - if File.exist?(old_pid) && server.pid != old_pid - begin - Process.kill("QUIT", File.read(old_pid).to_i) - rescue Errno::ENOENT, Errno::ESRCH - # someone else did our job for us - end - end -end - -after_fork do |server, worker| - defined?(ActiveRecord::Base) and - ActiveRecord::Base.establish_connection -end diff --git a/deployment/site-cookbooks/huginn_production/recipes/default.rb b/deployment/site-cookbooks/huginn_production/recipes/default.rb deleted file mode 100644 index 1bdd8ce2..00000000 --- a/deployment/site-cookbooks/huginn_production/recipes/default.rb +++ /dev/null @@ -1,111 +0,0 @@ -include_recipe 'apt' -include_recipe 'build-essential' - -user "huginn" do - system true - home "/home/huginn" - password "$6$ZwO6b.6tij$SMa8UIwtESGDxB37NwHsct.gJfXWmmflNbH.oypwJ9y0KkzMkCdw7D14iK7GX9C4CWSEcpGOFUow7p01rQFu5." - supports :manage_home => true - shell "/bin/bash" - gid "sudo" -end - -group "huginn" do - members ["huginn"] -end - -%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libmysqlclient-dev" "libffi-dev" "libssl-dev").each do |pkg| - package("#{pkg}") -end - -bash "Setting default ruby and gem versions to 1.9" do - code <<-EOH - if [ $(readlink /usr/bin/ruby) != "ruby1.9.1" ] - then - update-alternatives --set ruby /usr/bin/ruby1.9.1 - fi - - if [ $(readlink /usr/bin/gem) != "gem1.9.1" ] - then - update-alternatives --set gem /usr/bin/gem1.9.1 - fi - EOH -end - -gem_package("rake") -gem_package("bundle") - -service "nginx" do - supports :restart => true, :start => true, :stop => true, :reload => true - action :nothing -end - -bash "Setting huginn user with NOPASSWD option" do - cwd "/etc/sudoers.d" - code <<-EOH - touch huginn && chmod 0440 huginn - echo "huginn ALL=(ALL) NOPASSWD:ALL" >> huginn - EOH -end - -deploy "/home/huginn" do - repo "https://github.com/cantino/huginn.git" - branch "master" - user "huginn" - group "huginn" - environment "RAILS_ENV" => "production" - keep_releases 5 - create_dirs_before_symlink [] - symlinks "log" => "log" - symlink_before_migrate({}) - rollback_on_error true - notifies :enable, "service[nginx]" - notifies :start, "service[nginx]" - before_symlink do - %w(config log tmp).each do |dir| - directory "/home/huginn/shared/#{dir}" do - owner "huginn" - group "huginn" - recursive true - end - end - directory("/home/huginn/shared/tmp/pids") - directory("/home/huginn/shared/tmp/sockets") - %w(Procfile unicorn.rb nginx.conf).each do |file| - cookbook_file "/home/huginn/shared/config/#{file}" do - owner "huginn" - action :create_if_missing - end - end - cookbook_file "home/huginn/shared/config/.env" do - source "env.example" - mode "666" - owner "huginn" - action :create_if_missing - end - end - before_restart do - bash "huginn dependencies" do - cwd "/home/huginn/current" - user "huginn" - group "huginn" - code <<-EOH - export LANG="en_US.UTF-8" - export LC_ALL="en_US.UTF-8" - ln -nfs /home/huginn/shared/config/Procfile ./Procfile - ln -nfs /home/huginn/shared/config/.env ./.env - ln -nfs /home/huginn/shared/config/unicorn.rb ./config/unicorn.rb - sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/ - echo 'gem "unicorn", :group => :production' >> Gemfile - sudo bundle install --without=development --without=test - sed -i s/REPLACE_ME_NOW\!/$(sudo bundle exec rake secret)/ /home/huginn/shared/config/.env - sudo RAILS_ENV=production bundle exec rake db:create - sudo RAILS_ENV=production bundle exec rake db:migrate - sudo RAILS_ENV=production bundle exec rake db:seed - sudo RAILS_ENV=production bundle exec rake assets:precompile - sudo foreman export upstart /etc/init -a huginn -u huginn -l log - sudo start huginn - EOH - end - end -end From e5bc34568eb3709dec98301c51e3a4c99045ff9c Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 2 Aug 2015 16:41:32 +0200 Subject: [PATCH 28/36] Update heroku docs to reflect changes on the wiki --- doc/heroku/install.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/heroku/install.md b/doc/heroku/install.md index c2bdc3b2..7faa748c 100644 --- a/doc/heroku/install.md +++ b/doc/heroku/install.md @@ -1,8 +1,9 @@ ## Deploy to Heroku -### Important things to keep in mind +### Important things to keep in mindd on the free plan -* Heroku's [Free plan](https://www.heroku.com/pricing) now requires your application to sleep for at least 6 hours a day, thus the `Hobby` plan is recommended. +* Heroku's [free plan](https://www.heroku.com/pricing) limits total runtime per day to 18 hours. This means that Huginn must sleep some of the time, and so recurring tasks will only run if their recurrence frequency fits within the free plan's awake time, which is 30 minutes. Therefore, we recommend that you only use the every 1 minute, every 2 minute, and every 5 minute Agent scheduling options. +* When setting up Pingdom, set the `Check interval` to '60 minutes' so that your Huginn instance is up half the time. * Heroku's free Postgres plan limits the number of database rows that you can have to 10,000, so you should be sure to set a low event retention schedule for your agents. * The `setup_heroku` command points Heroku at a special Procfile (`deployment/heroku/Procfile.heroku`) that is designed to be run on only one Heroku web worker. If you want to run multiple workers, change the Heroku config variable `PROCFILE_PATH` with `heroku config:set PROCFILE_PATH=./Procfile` and switch back to the standard Huginn Procfile configuration. @@ -13,8 +14,7 @@ * Now, run the magic setup wizard: `bin/setup_heroku` * That's it! * If you make changes, you can re-run `bin/setup_heroku`, or just do `git push heroku master`. -* Signup for a free [Pingdom](https://www.pingdom.com/free/) account and have it ping your Huginn URL instance on Heroku. This should help keep your Heroku instance awake and watching the world for you. Keep in mind the Heroku's Free plan needs your application to sleep for 6 hours a day. - +* Signup for a free [Pingdom](https://www.pingdom.com/free/) account and have it ping your Huginn URL on Heroku once an hour. ### Using your own mail server From 2a5a8016d0b91f353ca1d4bc8d16e3aa3754d51b Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Tue, 4 Aug 2015 22:43:42 +0200 Subject: [PATCH 29/36] Incorporate feedback --- .env.example | 7 ++++--- Procfile | 4 ++++ config/unicorn.rb.example | 2 +- doc/README.md | 2 +- doc/docker/install.md | 8 ++++---- doc/heroku/install.md | 2 +- doc/manual/installation.md | 16 +++++++++------- doc/manual/requirements.md | 2 +- doc/manual/update.md | 2 ++ 9 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index 0788a6ad..8c25d3ce 100644 --- a/.env.example +++ b/.env.example @@ -176,9 +176,10 @@ DELAYED_JOB_MAX_RUNTIME=2 # Amount of seconds for delayed_job to sleep before checking for new jobs DELAYED_JOB_SLEEP_DELAY=10 -########################################################## -# Capistrano deployment (read the documentation FIXME) # -########################################################## +############################################################### +# Capistrano deployment, read the documentation: # +# https://github.com/cantino/huginn/doc/manual/capistrano.md # +############################################################### #CAPISTRANO_DEPLOY_SERVER= #CAPISTRANO_DEPLOY_USER= diff --git a/Procfile b/Procfile index 75bbcf27..aa8512c4 100644 --- a/Procfile +++ b/Procfile @@ -16,6 +16,10 @@ jobs: bundle exec rails runner bin/threaded.rb # PRODUCTION # ############################### +# You need to copy or link config/unicorn.rb.example to config/unicorn.rb for both production versions. +# Have a look at the deployment guides, if you want to set up huginn on your server: +# https://github.com/cantino/huginn/doc + # Using the threaded worker (consumes less RAM but can run slower) # web: bundle exec unicorn -c config/unicorn.rb # jobs: bundle exec rails runner bin/threaded.rb diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example index 465c7a8c..ee26cf2c 100644 --- a/config/unicorn.rb.example +++ b/config/unicorn.rb.example @@ -1,4 +1,4 @@ -wd = "/home/huginn/huginn" +wd = File.expand_path(File.join(File.dirname(__FILE__), '..')) app_path = wd diff --git a/doc/README.md b/doc/README.md index b9561173..fb5fa04d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -8,7 +8,7 @@ ### Manual installation -Manual installation which will guide through the steps to install Huginn on any Ubuntu 12.04/14.04 or Debian 6/7 server. +Manual installation instructions which will guide through the steps to install Huginn on any Ubuntu 12.04/14.04 or Debian 6/7 server. - [Install](manual/README.md) Requirements, directory structures and installation from source. - [Update](manual/update.md) Update your installation. diff --git a/doc/docker/install.md b/doc/docker/install.md index 29c20de9..fc683e6e 100644 --- a/doc/docker/install.md +++ b/doc/docker/install.md @@ -21,15 +21,15 @@ Getting Huginn up and running using docker is quick and painless once you have d 1. Download [docker machine](https://docs.docker.com/machine/#installation) for your OS * Follow the installation instructions untill you can successfully run `docker ps` * Get the the IP of the VM running docker by running `docker-machine ls` -* Start your Huginn container using `docker run -it -p 5000:5000 cantino/huginn` -* Open Huginn in the browser [http://docker-machine ip:5000](http://:5000) +* Start your Huginn container using `docker run -it -p 3000:3000 cantino/huginn` +* Open Huginn in the browser [http://docker-machine ip:3000](http://:3000) * Log in to your Huginn instance using the username `admin` and password `password` #### Linux 1. Install docker using the [install instructions](https://docs.docker.com/installation/) -* Start your Huginn container using `docker run -it -p 5000:5000 cantino/huginn` -* Open Huginn in the browser [http://localhost:5000](http://localhost:5000) +* Start your Huginn container using `docker run -it -p 3000:3000 cantino/huginn` +* Open Huginn in the browser [http://localhost:3000](http://localhost:3000) * Log in to your Huginn instance using the username `admin` and password `password` ## Configuration and linking to a database container diff --git a/doc/heroku/install.md b/doc/heroku/install.md index 7faa748c..31de0915 100644 --- a/doc/heroku/install.md +++ b/doc/heroku/install.md @@ -1,6 +1,6 @@ ## Deploy to Heroku -### Important things to keep in mindd on the free plan +### Important things to keep in mind on the free plan * Heroku's [free plan](https://www.heroku.com/pricing) limits total runtime per day to 18 hours. This means that Huginn must sleep some of the time, and so recurring tasks will only run if their recurrence frequency fits within the free plan's awake time, which is 30 minutes. Therefore, we recommend that you only use the every 1 minute, every 2 minute, and every 5 minute Agent scheduling options. * When setting up Pingdom, set the `Check interval` to '60 minutes' so that your Huginn instance is up half the time. diff --git a/doc/manual/installation.md b/doc/manual/installation.md index 3251e958..4030c862 100644 --- a/doc/manual/installation.md +++ b/doc/manual/installation.md @@ -11,7 +11,7 @@ This is the official installation guide to set up a production server. To set up The following steps have been known to work. Please **use caution when you deviate** from this guide. Make sure you don't violate any assumptions Huginn makes about its environment. For example many people run into permission problems because they change the location of directories or run services as the wrong user. -If you find a bug/error in this guide please **submit a merge request**. +If you find a bug/error in this guide please **submit a pull request**. If not stated otherwise all commands should be run as user with sudo permissions or as root. @@ -136,7 +136,7 @@ You are done installing the database and can go back to the rest of the installa ### Clone the Source - # We'll install Huginn into home directory of the user "huginn" + # We'll install Huginn into the home directory of the user "huginn" cd /home/huginn # Clone Huginn repository @@ -209,7 +209,7 @@ Change the Unicorn config if needed, the [requirements.md](./requirements.md#uni **Important Note:** Make sure to edit both `.env` and `unicorn.rb` to match your setup. -**Note:** If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps. +**Note:** If you want to use HTTPS, which is what we recommend, see [Using HTTPS](#using-https) for the additional steps. ### Initialize Database @@ -225,7 +225,7 @@ Change the Unicorn config if needed, the [requirements.md](./requirements.md#uni When done you see `See the Huginn Wiki for more Agent examples! https://github.com/cantino/huginn/wiki` -**Note:** This will create an initial user, you can set the username and password by supplying it in environmental variables `SEED_USERNAME` and`SEED_PASSWORD` as seen below. If you don't set the password (and it is set to the default one) please wait with exposing Huginn to the public internet until the installation is done and you've logged into the server and changed your password. +**Note:** This will create an initial user, you can set the username and password by supplying it in environmental variables `SEED_USERNAME` and `SEED_PASSWORD` as seen below. If you don't set the password (and it is set to the default one) please wait with exposing Huginn to the public internet until the installation is done and you've logged into the server and changed your password. sudo -u huginn -H bundle exec rake db:seed RAILS_ENV=production SEED_USERNAME=admin SEED_PASSWORD=yourpassword @@ -235,7 +235,7 @@ When done you see `See the Huginn Wiki for more Agent examples! https://github. ### Install Init Script -Huginn uses foreman to generate the init scripts based on a `Procfile` +Huginn uses [foreman](http://ddollar.github.io/foreman/) to generate the init scripts based on a `Procfile` Edit the `Procfile` and choose one of the suggested versions for production @@ -281,7 +281,7 @@ Remove the default nginx site, **if huginn is the only enabled nginx site**: sudo rm /etc/nginx/sites-enabled/default -**Note:** If you want to use HTTPS, replace the `huginn` Nginx config with `huginn-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. +**Note:** If you want to use HTTPS, which is what we recommend, replace the `huginn` Nginx config with `huginn-ssl`. See [Using HTTPS](#using-https) for HTTPS configuration details. ### Test Configuration @@ -307,7 +307,9 @@ Visit YOUR_SERVER in your web browser for your first Huginn login. The setup has **Enjoy!** -You can use `sudo start huginn` and `sudo stop huginn` to start and stop Huginn. +You can use `cd /home/huginn/huginn && sudo rake production:start` and `cd /home/huginn/huginn && sudo rake production:stop` to start and stop Huginn. + +Be sure to read the section about how to [update](./update.md) your Huginn installation as well! You can also use [Capistrano](./capistrano.md) to keep your installation up to date. ## Advanced Setup Tips diff --git a/doc/manual/requirements.md b/doc/manual/requirements.md index a9aa4c82..9879fedf 100644 --- a/doc/manual/requirements.md +++ b/doc/manual/requirements.md @@ -23,7 +23,7 @@ On the above unsupported distributions is still possible to install Huginn, and Huginn is developed for Unix operating systems. Huginn does **not** run on Windows and we have no plans of supporting it in the near future. -Please consider using a virtual machine to run Huginn. +Please consider using a virtual machine to run Huginn on Windows. ## Ruby versions diff --git a/doc/manual/update.md b/doc/manual/update.md index ca1de2dc..85b7424a 100644 --- a/doc/manual/update.md +++ b/doc/manual/update.md @@ -1,5 +1,7 @@ # Update +You can also use [Capistrano](./capistrano.md) to keep your installation up to date. + ### 0. Ensure depencies are up to date ``` From 97ceff62b3ce3e3d892c35c876a09a64bd77b1dd Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Wed, 19 Aug 2015 00:05:30 +0200 Subject: [PATCH 30/36] Add troubleshooting section and firewall recommendation --- doc/manual/installation.md | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/doc/manual/installation.md b/doc/manual/installation.md index 4030c862..446e26cd 100644 --- a/doc/manual/installation.md +++ b/doc/manual/installation.md @@ -15,6 +15,8 @@ If you find a bug/error in this guide please **submit a pull request**. If not stated otherwise all commands should be run as user with sudo permissions or as root. +When having problems during the installation please check the [troubleshooting](#troubleshooting) section. + ## Overview The Huginn installation consists of setting up the following components: @@ -305,12 +307,14 @@ Visit YOUR_SERVER in your web browser for your first Huginn login. The setup has password -**Enjoy!** +**Enjoy!** :sparkles: :star: :fireworks: You can use `cd /home/huginn/huginn && sudo rake production:start` and `cd /home/huginn/huginn && sudo rake production:stop` to start and stop Huginn. Be sure to read the section about how to [update](./update.md) your Huginn installation as well! You can also use [Capistrano](./capistrano.md) to keep your installation up to date. +**Note:** We also recommend applying standard security practices to your server, including installing a firewall ([ufw](https://wiki.ubuntu.com/UncomplicatedFirewall) is good on Ubuntu and also available for Debian). + ## Advanced Setup Tips ### Using HTTPS @@ -341,3 +345,59 @@ cd /etc/nginx/ssl/ sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out huginn.crt -keyout huginn.key sudo chmod o-r huginn.key ``` + +## Troubleshooting + +If something went wrong during the installation please make sure you followed the instructions and did not miss a step. + +When your Huginn instance still is not working first run the self check: + + cd /home/huginn/huginn + sudo rake production:check + +We are sorry when you are still having issues, now please check the various log files for error messages: + +#### Nginx error log `/var/log/nginx/huginn_error.log` + +This file should be empty, it is the first place to look because `nginx` is the first application handling the request your are sending to Huginn. + +Common problems: + +* `connect() to unix:/home/huginn/huginn/tmp/sockets/unicorn.socket failed`: The Unicorn application server is not running, ensure you uncommented one of the example configuration below the `PRODUCTION` label in your [Profile](#install-init-script) and the unicorn config file (`/home/huginn/huginn/config/unicorn.rb`) exists. +* `138 open() "/home/huginn/huginn/public/..." failed (13: Permission denied)`: The `/home/huginn/huginn/public` directory needs to be readable by the nginx user (which is per default `www-data`) + + +#### Unicorn log `/home/huginn/huginn/log/unicorn.log` + +Should only contain HTTP request log entries like: `10.0.2.2 - - [18/Aug/2015:21:15:12 +0000] "GET / HTTP/1.0" 200 - 0.0110` + +If you see ruby exception backtraces or other error messages the problem could be one of the following: + +* The configuration file `/home/huginn/huginn/config/unicorn.rb` does not exist +* Gem dependencies where not [installed](#install-gems) + +#### Rails Application log `/home/huginn/huginn/log/production.log` + +This file is pretty verbose, you want to look at it if you are getting the `We're sorry, but something went wrong.` error message when using Huginn. This is an example backtrace that can help you or other huginn developers locate the issue: + +``` +NoMethodError (undefined method `name' for nil:NilClass): + app/controllers/jobs_controller.rb:6:in `index' + config/initializers/silence_worker_status_logger.rb:5:in `call_with_silence_worker_status' +``` + +#### Runit/Background Worker logs `/home/huginn/huginn/log/*/current` + +Those files will contain error messages or backtraces if one of your agent is not performing as they should. The easiest way to debug an Agent is to watch all your log files for changes and trigger the agent to run via the Huginn web interface. + +The log file location depends your `Procfile` configuration, this command will give you a list of the available logs: + + ls -al /home/huginn/huginn/log/*/current + +When you want to monitor the background processes you can easily watch all the files for changes: + + tail -f /home/huginn/huginn/log/*/current + +### Still having problems? :crying_cat_face: + +You probably found an error message or exception backtrace you could not resolve. Please create a new [issue](https://github.com/cantino/huginn/issues) and include as much information as you could gather about the problem your are experiencing. \ No newline at end of file From 1361c5693b78a97ab6833fba069c3623e243bb1d Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Wed, 19 Aug 2015 21:54:17 +0200 Subject: [PATCH 31/36] Moved deployment configuration files and changed the repository Update ruby to 2.2.3 --- {lib/support => deployment}/logrotate/huginn | 0 {lib/support => deployment}/nginx/huginn | 0 {lib/support => deployment}/nginx/huginn-ssl | 0 doc/manual/installation.md | 14 ++++++-------- 4 files changed, 6 insertions(+), 8 deletions(-) rename {lib/support => deployment}/logrotate/huginn (100%) rename {lib/support => deployment}/nginx/huginn (100%) rename {lib/support => deployment}/nginx/huginn-ssl (100%) diff --git a/lib/support/logrotate/huginn b/deployment/logrotate/huginn similarity index 100% rename from lib/support/logrotate/huginn rename to deployment/logrotate/huginn diff --git a/lib/support/nginx/huginn b/deployment/nginx/huginn similarity index 100% rename from lib/support/nginx/huginn rename to deployment/nginx/huginn diff --git a/lib/support/nginx/huginn-ssl b/deployment/nginx/huginn-ssl similarity index 100% rename from lib/support/nginx/huginn-ssl rename to deployment/nginx/huginn-ssl diff --git a/doc/manual/installation.md b/doc/manual/installation.md index 446e26cd..f13dfd3b 100644 --- a/doc/manual/installation.md +++ b/doc/manual/installation.md @@ -65,8 +65,8 @@ Remove the old Ruby versions if present: Download Ruby and compile it: mkdir /tmp/ruby && cd /tmp/ruby - curl -L --progress http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.bz2 | tar xj - cd ruby-2.2.2 + curl -L --progress http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.3.tar.bz2 | tar xj + cd ruby-2.2.3 ./configure --disable-install-rdoc make -j`nproc` sudo make install @@ -142,9 +142,7 @@ You are done installing the database and can go back to the rest of the installa cd /home/huginn # Clone Huginn repository - #sudo -u huginn -H git clone https://github.com/cantino/huginn.git -b master huginn - # **FIXME** - sudo -u huginn -H git clone https://github.com/dsander/huginn.git -b deployment-guide huginn + sudo -u huginn -H git clone https://github.com/cantino/huginn.git -b master huginn # Go to Huginn installation folder cd /home/huginn/huginn @@ -251,7 +249,7 @@ Export the init scripts: ### Setup Logrotate - sudo cp lib/support/logrotate/huginn /etc/logrotate.d/huginn + sudo cp deployment/logrotate/huginn /etc/logrotate.d/huginn ### Ensure Your Huginn Instance Is Running @@ -270,7 +268,7 @@ Export the init scripts: Copy the example site config: - sudo cp lib/support/nginx/huginn /etc/nginx/sites-available/huginn + sudo cp deployment/nginx/huginn /etc/nginx/sites-available/huginn sudo ln -s /etc/nginx/sites-available/huginn /etc/nginx/sites-enabled/huginn Make sure to edit the config file to match your setup, if you are running multiple nginx sites remove the `default_server` argument from the `listen` directives: @@ -324,7 +322,7 @@ To use Huginn with HTTPS: 1. In `.env`: 1. Set the `FORCE_SSL` option to `true`. 1. Use the `huginn-ssl` Nginx example config instead of the `huginn` config: - 1. `sudo cp lib/support/nginx/huginn-ssl /etc/nginx/sites-available/huginn` + 1. `sudo cp deployment/nginx/huginn-ssl /etc/nginx/sites-available/huginn` 1. Update `YOUR_SERVER_FQDN`. 1. Update `ssl_certificate` and `ssl_certificate_key`. 1. Review the configuration file and consider applying other security and performance enhancing features. From 93c6b629b3b07e6afe892250c5ae0b57da63869a Mon Sep 17 00:00:00 2001 From: K1773R Date: Thu, 20 Aug 2015 17:36:45 +0200 Subject: [PATCH 32/36] fix guard-livereload min version --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index c305fc6e..c18b81be 100644 --- a/Gemfile +++ b/Gemfile @@ -99,7 +99,7 @@ group :development do gem 'binding_of_caller' gem 'quiet_assets' gem 'guard' - gem 'guard-livereload' + gem 'guard-livereload', '~> 2.2' gem 'guard-rspec' gem 'letter_opener_web' From 82bdd2caf82174c8066472d9e0c1aceccbd36e3c Mon Sep 17 00:00:00 2001 From: Stefan Siegl Date: Thu, 20 Aug 2015 22:16:00 +0200 Subject: [PATCH 33/36] Make post_agent handle content_type=xml --- app/models/agents/post_agent.rb | 5 ++++- spec/models/agents/post_agent_spec.rb | 26 +++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/app/models/agents/post_agent.rb b/app/models/agents/post_agent.rb index c984dbe3..65082840 100644 --- a/app/models/agents/post_agent.rb +++ b/app/models/agents/post_agent.rb @@ -13,7 +13,7 @@ module Agents The `method` used can be any of `get`, `post`, `put`, `patch`, and `delete`. - By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`). Change `content_type` to `json` to send JSON instead. + By default, non-GETs will be sent with form encoding (`application/x-www-form-urlencoded`). Change `content_type` to `json` to send JSON instead. Change `content_type` to `xml` to send XML, where the name of the root element may be specified using `xml_root`, defaulting to `post`. Other Options: @@ -102,6 +102,9 @@ module Agents when 'json' headers['Content-Type'] = 'application/json; charset=utf-8' body = data.to_json + when 'xml' + headers['Content-Type'] = 'text/xml; charset=utf-8' + body = data.to_xml(root: (interpolated(payload)[:xml_root] || 'post')) else body = data end diff --git a/spec/models/agents/post_agent_spec.rb b/spec/models/agents/post_agent_spec.rb index 168f2a32..4f12b6f2 100644 --- a/spec/models/agents/post_agent_spec.rb +++ b/spec/models/agents/post_agent_spec.rb @@ -38,11 +38,14 @@ describe Agents::PostAgent do when :get, :delete req.data = request.uri.query else - case request.headers['Content-Type'][/\A[^;\s]+/] + content_type = request.headers['Content-Type'][/\A[^;\s]+/] + case content_type when 'application/x-www-form-urlencoded' req.data = request.body when 'application/json' req.data = ActiveSupport::JSON.decode(request.body) + when 'text/xml' + req.data = Hash.from_xml(request.body) else raise "unexpected Content-Type: #{content_type}" end @@ -152,6 +155,27 @@ describe Agents::PostAgent do expect(@sent_requests[:post][0].data).to eq(@checker.options['payload']) end + it "sends options['payload'] as XML as a POST request" do + @checker.options['content_type'] = 'xml' + expect { + @checker.check + }.to change { @sent_requests[:post].length }.by(1) + + expect(@sent_requests[:post][0].data.keys).to eq([ 'post' ]) + expect(@sent_requests[:post][0].data['post']).to eq(@checker.options['payload']) + end + + it "sends options['payload'] as XML with custom root element name, as a POST request" do + @checker.options['content_type'] = 'xml' + @checker.options['xml_root'] = 'foobar' + expect { + @checker.check + }.to change { @sent_requests[:post].length }.by(1) + + expect(@sent_requests[:post][0].data.keys).to eq([ 'foobar' ]) + expect(@sent_requests[:post][0].data['foobar']).to eq(@checker.options['payload']) + end + it "sends options['payload'] as a GET request" do @checker.options['method'] = 'get' expect { From fde2b3521ebd91aa5d274f934cea9a29b5291211 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 20 Aug 2015 14:26:38 -0700 Subject: [PATCH 34/36] update Gemfile.lock for guard-livereload bump --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1a569e77..c7751b94 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -542,7 +542,7 @@ DEPENDENCIES geokit-rails (~> 2.0.1) google-api-client guard - guard-livereload + guard-livereload (~> 2.2) guard-rspec haversine hipchat (~> 1.2.0) From f0951da149bbb817438941139a618c2e2ae102b9 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 20 Aug 2015 14:32:26 -0700 Subject: [PATCH 35/36] remove puts statement --- app/concerns/markdown_class_attributes.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/concerns/markdown_class_attributes.rb b/app/concerns/markdown_class_attributes.rb index f418f3f1..361a7bf8 100644 --- a/app/concerns/markdown_class_attributes.rb +++ b/app/concerns/markdown_class_attributes.rb @@ -11,7 +11,7 @@ module MarkdownClassAttributes def #{attribute} if self.class.#{attribute}.is_a?(Proc) - Utils.unindent(self.instance_eval(&self.class.#{attribute}) || "No #{attribute} has been set.").tap {|i| p i} + Utils.unindent(self.instance_eval(&self.class.#{attribute}) || "No #{attribute} has been set.") else Utils.unindent(self.class.#{attribute} || "No #{attribute} has been set.") end @@ -29,4 +29,4 @@ module MarkdownClassAttributes end end end -end \ No newline at end of file +end From ce90e508056bdde5a18612f26e74d01c84f65aa1 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Fri, 21 Aug 2015 23:43:06 +0200 Subject: [PATCH 36/36] Include development group in docker build and switch default port --- docker/Dockerfile | 2 +- docker/README.md | 6 +++--- docker/scripts/setup | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fed77288..42e83a7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,7 +26,7 @@ RUN /scripts/setup VOLUME /var/lib/mysql -EXPOSE 5000 +EXPOSE 3000 CMD ["/scripts/init"] diff --git a/docker/README.md b/docker/README.md index de7ede59..ffd0ddfb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -47,7 +47,7 @@ The CMD launches Huginn via the scripts/init script. This may become the ENTRYPO Simple stand-alone usage: - docker run -it -p 5000:5000 cantino/huginn + docker run -it -p 3000:3000 cantino/huginn To link to another mysql container, for example: @@ -57,7 +57,7 @@ To link to another mysql container, for example: -e HUGINN_MYSQL_PASSWORD=somethingsecret \ -e HUGINN_MYSQL_ROOT_PASSWORD=somethingevenmoresecret \ cantino/huginn - docker run --rm --name huginn --link newcentury_mysql:MYSQL -p 5000:5000 \ + docker run --rm --name huginn --link newcentury_mysql:MYSQL -p 3000:3000 \ -e HUGINN_DATABASE_NAME=huginn \ -e HUGINN_DATABASE_USER=huginn \ -e HUGINN_DATABASE_PASSWORD=somethingsecret \ @@ -65,7 +65,7 @@ To link to another mysql container, for example: To link to another container named 'postgres': - docker run --rm --name huginn --link POSTGRES:mysql -p 5000:5000 -e "DATABASE_USER=huginn" -e "DATABASE_PASSWORD=pass@word" cantino/huginn + docker run --rm --name huginn --link POSTGRES:mysql -p 3000:3000 -e "DATABASE_USER=huginn" -e "DATABASE_PASSWORD=pass@word" cantino/huginn ## Environment Variables diff --git a/docker/scripts/setup b/docker/scripts/setup index c36c4d3c..9da60725 100755 --- a/docker/scripts/setup +++ b/docker/scripts/setup @@ -29,7 +29,7 @@ if [ -d "/scripts/cache" ]; then mv /scripts/cache vendor/ chown -R huginn:huginn vendor/cache fi -sudo -u huginn -H bundle install --deployment --without development test +sudo -u huginn -H bundle install --deployment --without test # silence setlocale message (THANKS DEBIAN!) cat > /etc/default/locale <