From 883a467fd94888531b1f6fc7fbe28bb91744b8b8 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 12 Aug 2014 15:11:02 +0900 Subject: [PATCH 01/19] FARADAY_HTTP_BACKEND now affects RssAgent and PostAgent too. --- .env.example | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 7a0c727a..8fed4f11 100644 --- a/.env.example +++ b/.env.example @@ -85,11 +85,11 @@ AWS_SANDBOX=false # Various Settings # ######################## -# Specify the HTTP backend library for Faraday, used in WebsiteAgent. -# You can change this depending on the performance and stability you -# need for your service. Any choice other than "typhoeus", -# "net_http", or "em_http" should require you to bundle a corresponding -# gem via Gemfile. +# Specify the HTTP backend library for Faraday, commonly used by +# WebsiteAgent, RssAgent and PostAgent. You can change this depending +# on the performance and stability you need for your service. Any +# choice other than "typhoeus", "net_http", or "em_http" should +# require you to bundle a corresponding gem via Gemfile. FARADAY_HTTP_BACKEND=typhoeus # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)] From 7ecbe78dffa42bc662fb4ed1e92d9bd322802abb Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 12 Aug 2014 15:31:26 +0900 Subject: [PATCH 02/19] Set a global default User-Agent value via an environment variable. Names starting with HTTP_ should be avoided in the context of Web app, hence the rather verbose name `DEFAULT_HTTP_USER_AGENT`. --- .env.example | 4 +++ app/concerns/web_request_concern.rb | 11 +++++--- .../shared_examples/web_request_concern.rb | 27 ++++++++++++++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 8fed4f11..4bc92dbd 100644 --- a/.env.example +++ b/.env.example @@ -92,6 +92,10 @@ AWS_SANDBOX=false # require you to bundle a corresponding gem via Gemfile. FARADAY_HTTP_BACKEND=typhoeus +# Specify the default User-Agent header value for HTTP requests made +# by Agents that allow overriding the User-Agent header value. +DEFAULT_HTTP_USER_AGENT="Huginn - https://github.com/cantino/huginn" + # Allow JSONPath eval expresions. i.e., $..price[?(@ < 20)] # You should not allow this on a shared Huginn box because it is not secure. ALLOW_JSONPATH_EVAL=false diff --git a/app/concerns/web_request_concern.rb b/app/concerns/web_request_concern.rb index b4f4616f..86d7bccb 100644 --- a/app/concerns/web_request_concern.rb +++ b/app/concerns/web_request_concern.rb @@ -21,9 +21,7 @@ module WebRequestConcern @faraday ||= Faraday.new { |builder| builder.headers = headers if headers.length > 0 - if (user_agent = interpolated['user_agent']).present? - builder.headers[:user_agent] = user_agent - end + builder.headers[:user_agent] = user_agent builder.use FaradayMiddleware::FollowRedirects builder.request :url_encoded @@ -58,4 +56,9 @@ module WebRequestConcern def faraday_backend ENV.fetch('FARADAY_HTTP_BACKEND', 'typhoeus').to_sym end -end \ No newline at end of file + + def user_agent + interpolated['user_agent'].presence || + ENV.fetch('DEFAULT_HTTP_USER_AGENT', Faraday.new.headers[:user_agent]) + end +end diff --git a/spec/support/shared_examples/web_request_concern.rb b/spec/support/shared_examples/web_request_concern.rb index 510c533d..db44229f 100644 --- a/spec/support/shared_examples/web_request_concern.rb +++ b/spec/support/shared_examples/web_request_concern.rb @@ -63,4 +63,29 @@ shared_examples_for WebRequestConcern do agent.should_not be_valid end end -end \ No newline at end of file + + describe "User-Agent" do + before do + @default_http_user_agent = ENV['DEFAULT_HTTP_USER_AGENT'] + ENV['DEFAULT_HTTP_USER_AGENT'] = nil + end + + after do + ENV['DEFAULT_HTTP_USER_AGENT'] = @default_http_user_agent + end + + it "should have the default value set by Faraday" do + agent.user_agent.should == Faraday.new.headers[:user_agent] + end + + it "should be overridden by the environment variable if present" do + ENV['DEFAULT_HTTP_USER_AGENT'] = 'Huginn - https://github.com/cantino/huginn' + agent.user_agent.should == 'Huginn - https://github.com/cantino/huginn' + end + + it "should be overriden by the value in options if present" do + agent.options['user_agent'] = 'Override' + agent.user_agent.should == 'Override' + end + end +end From a224a0b9089ac5fa5e0e00c0b8c4a49406628662 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 12 Aug 2014 18:58:35 +0900 Subject: [PATCH 03/19] Use content_tag. --- app/helpers/application_helper.rb | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index db1583ec..a76dea97 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,27 +1,19 @@ module ApplicationHelper def nav_link(name, path, options = {}) - (<<-HTML).html_safe -
  • - #{link_to name, path} -
    • - HTML + content_tag :li, link_to(name, path), class: current_page?(path) ? 'active' : '' end def yes_no(bool) - if bool - 'Yes'.html_safe - else - 'No'.html_safe - end + content_tag :span, bool ? 'Yes' : 'No', class: "label #{bool ? 'label-info' : 'label-default' }" end def working(agent) if agent.disabled? - link_to 'Disabled', agent_path(agent), :class => 'label label-warning' + link_to 'Disabled', agent_path(agent), class: 'label label-warning' elsif agent.working? - 'Yes'.html_safe + content_tag :span, 'Yes', class: 'label label-success' else - link_to 'No', agent_path(agent, :tab => (agent.recent_error_logs? ? 'logs' : 'details')), :class => 'label label-danger' + link_to 'No', agent_path(agent, tab: (agent.recent_error_logs? ? 'logs' : 'details')), class: 'label label-danger' end end end From 05e373614efea3858e4e01cfc67be8ab15630238 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Tue, 12 Aug 2014 19:33:12 +0900 Subject: [PATCH 04/19] Add a hover menu to the "Agents" nav link. --- .../stylesheets/application.css.scss.erb | 6 +++++ app/helpers/application_helper.rb | 26 +++++++++++++++++-- app/views/layouts/_navigation.html.erb | 8 +++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/app/assets/stylesheets/application.css.scss.erb b/app/assets/stylesheets/application.css.scss.erb index 866b0b17..2cbff3e3 100644 --- a/app/assets/stylesheets/application.css.scss.erb +++ b/app/assets/stylesheets/application.css.scss.erb @@ -98,6 +98,12 @@ span.not-applicable:after { right: 1px; } +.navbar { + .dropdown.dropdown-hover:hover .dropdown-menu { + display: block; + } +} + // Flash .flash { diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index a76dea97..87de4b8e 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,6 +1,28 @@ module ApplicationHelper - def nav_link(name, path, options = {}) - content_tag :li, link_to(name, path), class: current_page?(path) ? 'active' : '' + def nav_link(name, path, options = {}, &block) + if glyphicon = options[:glyphicon] + name = " ".html_safe + name + end + content = link_to(name, path) + active = current_page?(path) + if block + # Passing a block signifies that the link is a header of a hover + # menu which contains what's in the block. + begin + @nav_in_menu = true + @nav_link_active = active + content += capture(&block) + class_name = "dropdown dropdown-hover #{@nav_link_active ? 'active' : ''}" + ensure + @nav_in_menu = @nav_link_active = false + end + else + # Mark the menu header active if it contains the current page + @nav_link_active ||= active if @nav_in_menu + # An "active" menu item may be an eyesore, hence `!@nav_in_menu &&`. + class_name = !@nav_in_menu && active ? 'active' : '' + end + content_tag :li, content, class: class_name end def yes_no(bool) diff --git a/app/views/layouts/_navigation.html.erb b/app/views/layouts/_navigation.html.erb index 8a7d8915..f33d8981 100644 --- a/app/views/layouts/_navigation.html.erb +++ b/app/views/layouts/_navigation.html.erb @@ -12,7 +12,13 @@ <% if user_signed_in? %>
      - <%= nav_link "Agents", agents_path %> + <%= nav_link "Agents", agents_path do %> + + <% end %> <%= nav_link "Scenarios", scenarios_path %> <%= nav_link "Events", events_path %> <%= nav_link "Credentials", user_credentials_path %> From f292fc089c0a28a399afe840c23d947356a0806f Mon Sep 17 00:00:00 2001 From: Andrew Curioso Date: Tue, 12 Aug 2014 15:48:45 -0400 Subject: [PATCH 05/19] Sane docker file optimized for quick rebuilds --- Dockerfile | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6bc72689 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,56 @@ +FROM ubuntu +# MAINTAINER Someone + +# Update package list +RUN apt-get update + +# Set environmental variables +ENV HOME /root +ENV RBENV_ROOT $HOME/.rbenv +ENV RUBY_VERSION 1.9.3-p545 +ENV RUBYGEMS_VERSION 2.2.2 +ENV PATH $HOME/.rbenv/shims:$HOME/.rbenv/bin:$RBENV_ROOT/versions/$RUBY_VERSION/bin:$PATH + +# Install OS packages +RUN apt-get install -y build-essential curl zlib1g-dev libreadline-dev libssl-dev libcurl4-openssl-dev git libmysqlclient-dev + +RUN git clone https://github.com/sstephenson/rbenv.git $HOME/.rbenv +RUN git clone https://github.com/sstephenson/ruby-build.git $HOME/.rbenv/plugins/ruby-build + +# install & set global ruby version +RUN rbenv install $RUBY_VERSION +RUN rbenv global $RUBY_VERSION + +WORKDIR /usr/local/src + +RUN curl -O http://production.cf.rubygems.org/rubygems/rubygems-$RUBYGEMS_VERSION.tgz +RUN tar -xvf rubygems-$RUBYGEMS_VERSION.tgz +RUN cd rubygems-$RUBYGEMS_VERSION ; ruby setup.rb + +RUN gem install bundle + +RUN mkdir huginn +WORKDIR huginn + +# Add Gemfiles and run bundle ahead of time +# This way bundle does not have to rerun unless the Gemfile changes +# It drastically speeds up rebuilds +ADD Gemfile /usr/local/src/huginn/ +ADD Procfile /usr/local/src/huginn/ +ADD Gemfile.lock /usr/local/src/huginn/ +RUN bundle + +# Now add the rest of the source +ADD . /usr/local/src/huginn/ +RUN rm -rf /usr/local/src/huginn/.env + +# Add the environmental variables this way so that the -e option can override them +ENV DATABASE_HOST db +ENV DATABASE_NAME huginn +ENV DATABASE_USERNAME huginn + +# Expose the Rails port to the rest of the world +EXPOSE 3000 + +# Default command - optimized for upgradability +CMD ["foreman", "start"] From db847be31b3c058fda7ebd4568bf0acf0773e3f6 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Wed, 13 Aug 2014 17:51:21 +0900 Subject: [PATCH 06/19] Tweak gem dependencies to improve platform support for *BSD and Solaris. * ffi: pull in unreleased fixes from GitHub master on *BSD. * tzinfo: require >=1.2.0 to gain support for *BSD and Solaris, eliminating the need for the tzinfo-data gem. --- Gemfile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index acde98fd..7b9bbd0c 100644 --- a/Gemfile +++ b/Gemfile @@ -14,14 +14,22 @@ gem 'protected_attributes', '~>1.0.8' gem 'rails' , '4.1.4' case RUBY_PLATFORM -when /freebsd/ - # Seems FreeBSD's zoneinfo is not exactly what tzinfo expects - gem 'tzinfo-data' -else - # Windows does not include zoneinfo files, so bundle the tzinfo-data gem - gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw] +when /freebsd|netbsd|openbsd/ + # ffi (required by typhoeus via ethon) merged fixes for bugs fatal + # on these platforms after 1.9.3; no following release as yet. + gem 'ffi', github: 'ffi/ffi', branch: 'master' + + # tzinfo 1.2.0 has added support for reading zoneinfo on these + # platforms. + gem 'tzinfo', '>= 1.2.0' +when /solaris/ + # ditto + gem 'tzinfo', '>= 1.2.0' end +# Windows does not have zoneinfo files, so bundle the tzinfo-data gem. +gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw] + gem 'mysql2', '~> 0.3.16' gem 'devise', '~> 3.2.4' gem 'kaminari', '~> 0.16.1' From d529872e3717187b07241703f0a3085af3f95fa6 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 17 Aug 2014 15:22:49 +0200 Subject: [PATCH 07/19] TwitterStream bugfixes * do not run disabled `TwitterStreamAgents` * calling `sleep` within the EM loop blocks the EM thread and thereby blocks the reload callback and proper shutdown of the threaded worker * stopping the EM loop from outside of the `EM.run` blocks does not seem to work reliably, using a timer and checking if shutdown was requested works as expected * removed unnecessary sleep at the end of the EM loop which would delay the shutdown of the thread --- app/models/agent.rb | 2 ++ lib/twitter_stream.rb | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/models/agent.rb b/app/models/agent.rb index 5356b0ed..cb7b9181 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -55,6 +55,8 @@ class Agent < ActiveRecord::Base has_many :scenario_memberships, :dependent => :destroy, :inverse_of => :agent has_many :scenarios, :through => :scenario_memberships, :inverse_of => :agents + scope :active, -> { where(disabled: false) } + scope :of_type, lambda { |type| type = case type when String, Symbol, Class diff --git a/lib/twitter_stream.rb b/lib/twitter_stream.rb index 3aac876a..b6ad79bd 100644 --- a/lib/twitter_stream.rb +++ b/lib/twitter_stream.rb @@ -11,7 +11,6 @@ class TwitterStream def stop @running = false - EventMachine::stop_event_loop if EventMachine.reactor_running? end def stream!(filters, agent, &block) @@ -91,9 +90,13 @@ class TwitterStream def run while @running begin - agents = Agents::TwitterStreamAgent.all + agents = Agents::TwitterStreamAgent.active.all EventMachine::run do + EventMachine.add_periodic_timer(1) { + EventMachine::stop_event_loop if !@running + } + EventMachine.add_periodic_timer(RELOAD_TIMEOUT) { puts "Reloading EventMachine and all Agents..." EventMachine::stop_event_loop @@ -101,17 +104,14 @@ class TwitterStream if agents.length == 0 puts "No agents found. Will look again in a minute." - sleep 60 - EventMachine::stop_event_loop + EventMachine.add_timer(60) { + EventMachine::stop_event_loop + } else puts "Found #{agents.length} agent(s). Loading them now..." load_and_run agents end end - - print "Pausing..."; STDOUT.flush - sleep 1 - puts "done." rescue SignalException, SystemExit @running = false EventMachine::stop_event_loop if EventMachine.reactor_running? From 70f5495aca08baffe4e72d6fa0151e1a16e97f55 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Sun, 17 Aug 2014 09:51:19 -0700 Subject: [PATCH 08/19] method: post was required for nav link to work --- app/helpers/application_helper.rb | 4 ++-- app/views/layouts/_navigation.html.erb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 87de4b8e..2e2074c0 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -1,9 +1,9 @@ module ApplicationHelper def nav_link(name, path, options = {}, &block) - if glyphicon = options[:glyphicon] + if glyphicon = options.delete(:glyphicon) name = " ".html_safe + name end - content = link_to(name, path) + content = link_to(name, path, options) active = current_page?(path) if block # Passing a block signifies that the link is a header of a hover diff --git a/app/views/layouts/_navigation.html.erb b/app/views/layouts/_navigation.html.erb index f33d8981..0f26e416 100644 --- a/app/views/layouts/_navigation.html.erb +++ b/app/views/layouts/_navigation.html.erb @@ -15,7 +15,7 @@ <%= nav_link "Agents", agents_path do %> <% end %> From 9fd3ca7bf117750902bf5c27b40df637e3b4331b Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 17 Aug 2014 23:36:46 +0200 Subject: [PATCH 09/19] hipchat.rb expects the notify option to be a boolean --- app/models/agents/hipchat_agent.rb | 2 +- spec/models/agents/hipchat_agent_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/agents/hipchat_agent.rb b/app/models/agents/hipchat_agent.rb index 475ab275..26e74f3e 100644 --- a/app/models/agents/hipchat_agent.rb +++ b/app/models/agents/hipchat_agent.rb @@ -43,7 +43,7 @@ module Agents client = HipChat::Client.new(interpolated[:auth_token] || credential('hipchat_auth_token')) incoming_events.each do |event| mo = interpolated(event) - client[mo[:room_name]].send(mo[:username], mo[:message], :notify => boolify(mo[:notify]) ? 1 : 0, :color => mo[:color]) + client[mo[:room_name]].send(mo[:username], mo[:message], :notify => boolify(mo[:notify]), :color => mo[:color]) end end end diff --git a/spec/models/agents/hipchat_agent_spec.rb b/spec/models/agents/hipchat_agent_spec.rb index 8540bf4f..31f696c5 100644 --- a/spec/models/agents/hipchat_agent_spec.rb +++ b/spec/models/agents/hipchat_agent_spec.rb @@ -7,7 +7,7 @@ describe Agents::HipchatAgent do 'room_name' => 'test', 'username' => "{{username}}", 'message' => "{{message}}", - 'notify' => false, + 'notify' => 'false', 'color' => 'yellow', } @@ -53,7 +53,7 @@ describe Agents::HipchatAgent do describe "#receive" do it "send a message to the hipchat" do any_instance_of(HipChat::Room) do |obj| - mock(obj).send(@event.payload[:username], @event.payload[:message], {:notify => 0, :color => 'yellow'}) + mock(obj).send(@event.payload[:username], @event.payload[:message], {:notify => false, :color => 'yellow'}) end @checker.receive([@event]) end From d006fd662b4451c6e799b4bbe52fca5f5acf40f4 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 18 Aug 2014 16:25:39 +0900 Subject: [PATCH 10/19] Show user the correct default User-Agent value. --- app/concerns/web_request_concern.rb | 7 ++++++- app/models/agents/website_agent.rb | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/concerns/web_request_concern.rb b/app/concerns/web_request_concern.rb index 86d7bccb..ebb72d53 100644 --- a/app/concerns/web_request_concern.rb +++ b/app/concerns/web_request_concern.rb @@ -58,7 +58,12 @@ module WebRequestConcern end def user_agent - interpolated['user_agent'].presence || + interpolated['user_agent'].presence || self.class.default_user_agent + end + + module ClassMethods + def default_user_agent ENV.fetch('DEFAULT_HTTP_USER_AGENT', Faraday.new.headers[:user_agent]) + end end end diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index b2ce30af..d8b6c0a6 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -50,7 +50,7 @@ module Agents Set `force_encoding` to an encoding name if the website does not return a Content-Type header with a proper charset. - Set `user_agent` to a custom User-Agent name if the website does not like the default value ("Faraday v#{Faraday::VERSION}"). + Set `user_agent` to a custom User-Agent name if the website does not like the default value (`#{default_user_agent}`). The `headers` field is optional. When present, it should be a hash of headers to send with the request. From fca8051e81097ee4a4dd0ef6c8d6ee84a231b7d3 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Mon, 18 Aug 2014 19:10:46 +0900 Subject: [PATCH 11/19] Add a parser type `text` to WebsiteAgent. --- app/models/agents/website_agent.rb | 41 +++++++++++++++++-- spec/models/agents/website_agent_spec.rb | 52 ++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index d8b6c0a6..90c73cb0 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -19,7 +19,7 @@ module Agents `url` can be a single url, or an array of urls (for example, for multiple pages with the exact same structure but different content to scrape) - The `type` value can be `xml`, `html`, or `json`. + The `type` value can be `xml`, `html`, `json`, or `text`. To tell the Agent how to parse the content, specify `extract` as a hash with keys naming the extractions and values of hashes. @@ -40,6 +40,26 @@ module Agents "description": { "path": "results.data[*].description" } } + When parsing text, each sub-hash should contain a `regexp` and `index`. Output text is matched against the regular expression repeatedly from the beginning through to the end, collecting a captured group specified by `index` in each match. Each index should be either an integer or a string name which corresponds to `(?<_name_>...)`. For example, to parse lines of `_word_: _definition_`, the following should work: + + "extract": { + "word": { "regexp": "^(.+?): (.+)$", index: 1 }, + "definition": { "regexp": "^(.+?): (.+)$", index: 2 }, + } + + Or if you prefer names to numbers for index: + + "extract": { + "word": { "regexp": "^(?.+?): (?.+)$", index: 'word' }, + "definition": { "regexp": "^(?.+?): (?.+)$", index: 'definition' }, + } + + To extract the whole content as one event: + + "extract": { + "content": { "regexp": "\A(?:.|\n)*\z", index: 0 }, + } + Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful. Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`. @@ -140,7 +160,15 @@ module Agents else output = {} interpolated['extract'].each do |name, extraction_details| - if extraction_type == "json" + case extraction_type + when "text" + regexp = Regexp.new(extraction_details['regexp']) + result = [] + doc.scan(regexp) { + result << Regexp.last_match[extraction_details['index']] + } + log "Extracting #{extraction_type} at #{regexp}: #{result}" + when "json" result = Utils.values_at(doc, extraction_details['path']) log "Extracting #{extraction_type} at #{extraction_details['path']}: #{result}" else @@ -253,10 +281,13 @@ module Agents def extraction_type (interpolated['type'] || begin - if interpolated['url'] =~ /\.(rss|xml)$/i + case interpolated['url'] + when /\.(rss|xml)$/i "xml" - elsif interpolated['url'] =~ /\.json$/i + when /\.json$/i "json" + when /\.(txt|text)$/i + "text" else "html" end @@ -271,6 +302,8 @@ module Agents JSON.parse(data) when "html" Nokogiri::HTML(data) + when "text" + data else raise "Unknown extraction type #{extraction_type}" end diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index fdb4b03d..bda569d6 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -398,6 +398,58 @@ describe Agents::WebsiteAgent do event.payload['response']['title'].should == "hello!" end end + + describe "text parsing" do + before do + stub_request(:any, /text-site/).to_return(body: <<-EOF, status: 200) +water: wet +fire: hot + EOF + site = { + 'name' => 'Some Text Response', + 'expected_update_period_in_days' => '2', + 'type' => 'text', + 'url' => 'http://text-site.com', + 'mode' => 'on_change', + 'extract' => { + 'word' => { 'regexp' => '^(.+?): (.+)$', index: 1 }, + 'property' => { 'regexp' => '^(.+?): (.+)$', index: 2 }, + } + } + @checker = Agents::WebsiteAgent.new(name: 'Text Site', options: site) + @checker.user = users(:bob) + @checker.save! + end + + it "works with regexp" do + @checker.options = @checker.options.merge('extract' => { + 'word' => { 'regexp' => '^(?.+?): (?.+)$', index: 'word' }, + 'property' => { 'regexp' => '^(?.+?): (?.+)$', index: 'property' }, + }) + + lambda { + @checker.check + }.should change { Event.count }.by(2) + + event1, event2 = Event.last(2) + event1.payload['word'].should == 'water' + event1.payload['property'].should == 'wet' + event2.payload['word'].should == 'fire' + event2.payload['property'].should == 'hot' + end + + it "works with regexp with named capture" do + lambda { + @checker.check + }.should change { Event.count }.by(2) + + event1, event2 = Event.last(2) + event1.payload['word'].should == 'water' + event1.payload['property'].should == 'wet' + event2.payload['word'].should == 'fire' + event2.payload['property'].should == 'hot' + end + end end describe "#receive" do From 13a317804f976ea4171e31b44a72ff30a097f3a3 Mon Sep 17 00:00:00 2001 From: "Guilherme J. Tramontina" Date: Tue, 19 Aug 2014 21:16:23 -0300 Subject: [PATCH 12/19] Support for markdown on scenario description. --- Gemfile | 2 +- Gemfile.lock | 2 ++ app/helpers/markdown_helper.rb | 8 ++++++++ app/views/scenarios/show.html.erb | 2 +- spec/helpers/markdown_helper_spec.rb | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 app/helpers/markdown_helper.rb create mode 100644 spec/helpers/markdown_helper_spec.rb diff --git a/Gemfile b/Gemfile index 7b9bbd0c..27214007 100644 --- a/Gemfile +++ b/Gemfile @@ -85,6 +85,7 @@ gem 'feed-normalizer' gem 'slack-notifier', '~> 0.5.0' gem 'therubyracer', '~> 0.12.1' gem 'mqtt' +gem 'redcarpet', '~> 3.1.1' group :development do gem 'binding_of_caller' @@ -123,4 +124,3 @@ else gem 'unicorn', platform: :ruby_18 gem 'rails_12factor', platform: :ruby_18 end - diff --git a/Gemfile.lock b/Gemfile.lock index 6cfc8531..ae792baa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -233,6 +233,7 @@ GEM thor (>= 0.18.1, < 2.0) raindrops (0.13.0) rake (10.3.2) + redcarpet (3.1.2) ref (1.0.5) rest-client (1.6.7) mime-types (>= 1.16) @@ -404,6 +405,7 @@ DEPENDENCIES rack rails (= 4.1.4) rails_12factor + redcarpet (~> 3.1.1) rr rspec (~> 2.99) rspec-collection_matchers diff --git a/app/helpers/markdown_helper.rb b/app/helpers/markdown_helper.rb new file mode 100644 index 00000000..c06a81bb --- /dev/null +++ b/app/helpers/markdown_helper.rb @@ -0,0 +1,8 @@ +module MarkdownHelper + + def markdown(text) + markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML) + markdown.render(text).html_safe + end + +end diff --git a/app/views/scenarios/show.html.erb b/app/views/scenarios/show.html.erb index c76cbba2..3bccb6c2 100644 --- a/app/views/scenarios/show.html.erb +++ b/app/views/scenarios/show.html.erb @@ -6,7 +6,7 @@ <% if @scenario.description.present? %> -
      <%= @scenario.description %>
      +
      <%= markdown(@scenario.description) %>
      <% end %> <%= render 'agents/table', :returnTo => scenario_path(@scenario) %> diff --git a/spec/helpers/markdown_helper_spec.rb b/spec/helpers/markdown_helper_spec.rb new file mode 100644 index 00000000..acf29a28 --- /dev/null +++ b/spec/helpers/markdown_helper_spec.rb @@ -0,0 +1,14 @@ +require 'spec_helper' + +describe MarkdownHelper do + + describe '#markdown' do + + it 'renders HTML from a markdown text' do + markdown('# Header').should =~ /

      Header<\/h1>/ + markdown('## Header 2').should =~ /

      Header 2<\/h2>/ + end + + end + +end From dcaf62e85173e56650f5155e1a44ce02c44b1999 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Tue, 19 Aug 2014 20:30:23 -0700 Subject: [PATCH 13/19] update gems --- Gemfile | 2 +- Gemfile.lock | 95 +++++++++++++++++++++++++++------------------------- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/Gemfile b/Gemfile index 7b9bbd0c..33fb8f38 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ gem 'bundler', '>= 1.5.0' gem 'protected_attributes', '~>1.0.8' -gem 'rails' , '4.1.4' +gem 'rails' , '4.1.5' case RUBY_PLATFORM when /freebsd|netbsd|openbsd/ diff --git a/Gemfile.lock b/Gemfile.lock index 6cfc8531..e434eb55 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,27 +12,27 @@ GEM remote: https://rubygems.org/ specs: ace-rails-ap (2.0.1) - actionmailer (4.1.4) - actionpack (= 4.1.4) - actionview (= 4.1.4) + actionmailer (4.1.5) + actionpack (= 4.1.5) + actionview (= 4.1.5) mail (~> 2.5.4) - actionpack (4.1.4) - actionview (= 4.1.4) - activesupport (= 4.1.4) + actionpack (4.1.5) + actionview (= 4.1.5) + activesupport (= 4.1.5) rack (~> 1.5.2) rack-test (~> 0.6.2) - actionview (4.1.4) - activesupport (= 4.1.4) + actionview (4.1.5) + activesupport (= 4.1.5) builder (~> 3.1) erubis (~> 2.7.0) - activemodel (4.1.4) - activesupport (= 4.1.4) + activemodel (4.1.5) + activesupport (= 4.1.5) builder (~> 3.1) - activerecord (4.1.4) - activemodel (= 4.1.4) - activesupport (= 4.1.4) + activerecord (4.1.5) + activemodel (= 4.1.5) + activesupport (= 4.1.5) arel (~> 5.0.0) - activesupport (4.1.4) + activesupport (4.1.5) i18n (~> 0.6, >= 0.6.9) json (~> 1.7, >= 1.7.7) minitest (~> 5.1) @@ -60,12 +60,12 @@ GEM coffee-rails (4.0.1) coffee-script (>= 2.2.0) railties (>= 4.0.0, < 5.0) - coffee-script (2.2.0) + coffee-script (2.3.0) coffee-script-source execjs - coffee-script-source (1.7.0) + coffee-script-source (1.7.1) cookiejar (0.3.2) - coveralls (0.7.0) + coveralls (0.7.1) multi_json (~> 1.3) rest-client simplecov (>= 0.7) @@ -77,7 +77,7 @@ GEM debug_inspector (0.0.2) delayed_job (4.0.2) activesupport (>= 3.0, < 4.2) - delayed_job_active_record (4.0.1) + delayed_job_active_record (4.0.2) activerecord (>= 3.0, < 4.2) delayed_job (>= 3.0, < 4.1) delorean (2.1.0) @@ -154,7 +154,7 @@ GEM httparty (0.13.1) json (~> 1.8) multi_xml (>= 0.5.2) - i18n (0.6.9) + i18n (0.6.11) jquery-rails (3.1.1) railties (>= 3.0, < 5.0) thor (>= 0.14, < 2.0) @@ -181,7 +181,7 @@ GEM method_source (0.8.2) mime-types (1.25.1) mini_portile (0.6.0) - minitest (5.3.5) + minitest (5.4.0) mqtt (0.2.0) multi_json (1.10.1) multi_xml (0.5.5) @@ -189,7 +189,7 @@ GEM mysql2 (0.3.16) naught (1.0.0) net-ftp-list (3.2.8) - nokogiri (1.6.2.1) + nokogiri (1.6.3.1) mini_portile (= 0.6.0) oauth2 (0.9.4) faraday (>= 0.8, < 0.10) @@ -202,7 +202,7 @@ GEM polyglot (0.3.5) protected_attributes (1.0.8) activemodel (>= 4.0.1, < 5.0) - pry (0.10.0) + pry (0.10.1) coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) @@ -211,31 +211,34 @@ GEM rack (1.5.2) rack-test (0.6.2) rack (>= 1.0) - rails (4.1.4) - actionmailer (= 4.1.4) - actionpack (= 4.1.4) - actionview (= 4.1.4) - activemodel (= 4.1.4) - activerecord (= 4.1.4) - activesupport (= 4.1.4) + rails (4.1.5) + actionmailer (= 4.1.5) + actionpack (= 4.1.5) + actionview (= 4.1.5) + activemodel (= 4.1.5) + activerecord (= 4.1.5) + activesupport (= 4.1.5) bundler (>= 1.3.0, < 2.0) - railties (= 4.1.4) + railties (= 4.1.5) sprockets-rails (~> 2.0) rails_12factor (0.0.2) rails_serve_static_assets rails_stdout_logging rails_serve_static_assets (0.0.2) rails_stdout_logging (0.0.3) - railties (4.1.4) - actionpack (= 4.1.4) - activesupport (= 4.1.4) + railties (4.1.5) + actionpack (= 4.1.5) + activesupport (= 4.1.5) rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) raindrops (0.13.0) rake (10.3.2) + rdoc (4.1.1) + json (~> 1.4) ref (1.0.5) - rest-client (1.6.7) - mime-types (>= 1.16) + rest-client (1.6.8) + mime-types (~> 1.16) + rdoc (>= 2.4.2) retriable (1.4.1) rr (1.1.2) rspec (2.99.0) @@ -244,7 +247,7 @@ GEM rspec-mocks (~> 2.99.0) rspec-collection_matchers (1.0.0) rspec-expectations (>= 2.99.0.beta1) - rspec-core (2.99.1) + rspec-core (2.99.2) rspec-expectations (2.99.2) diff-lcs (>= 1.1.3, < 2.0) rspec-mocks (2.99.2) @@ -272,9 +275,9 @@ GEM sass (~> 3.2.0) sprockets (~> 2.8, <= 2.11.0) sprockets-rails (~> 2.0) - select2-rails (3.5.7) + select2-rails (3.5.9.1) thor (~> 0.14) - shoulda-matchers (2.6.1) + shoulda-matchers (2.6.2) activesupport (>= 3.0.0) signet (0.5.1) addressable (>= 2.2.3) @@ -283,13 +286,13 @@ GEM multi_json (>= 1.0.0) simple-rss (1.3.1) simple_oauth (0.2.0) - simplecov (0.8.2) + simplecov (0.9.0) docile (~> 1.1.0) multi_json simplecov-html (~> 0.8.0) simplecov-html (0.8.0) slack-notifier (0.5.0) - slop (3.5.0) + slop (3.6.0) sprockets (2.11.0) hike (~> 1.2) multi_json (~> 1.0) @@ -308,11 +311,11 @@ GEM thor (0.19.1) thread_safe (0.3.4) tilt (1.4.1) - tins (1.3.0) + tins (1.3.2) treetop (1.4.15) polyglot polyglot (>= 0.3.1) - twilio-ruby (3.11.5) + twilio-ruby (3.11.6) builder (>= 2.1.2) jwt (>= 0.1.2) multi_json (>= 1.3.0) @@ -329,9 +332,9 @@ GEM simple_oauth (~> 0.2.0) typhoeus (0.6.9) ethon (>= 0.7.1) - tzinfo (1.2.1) + tzinfo (1.2.2) thread_safe (~> 0.1) - uglifier (2.5.1) + uglifier (2.5.3) execjs (>= 0.3.0) json (>= 1.8.0) unicorn (4.8.3) @@ -340,7 +343,7 @@ GEM raindrops (~> 0.7) uuid (2.3.7) macaddr (~> 1.0) - uuidtools (2.1.4) + uuidtools (2.1.5) vcr (2.9.2) warden (1.2.3) rack (>= 1.0) @@ -402,7 +405,7 @@ DEPENDENCIES pry quiet_assets rack - rails (= 4.1.4) + rails (= 4.1.5) rails_12factor rr rspec (~> 2.99) From 74a882482a2503588b750f21038931c0ad02788b Mon Sep 17 00:00:00 2001 From: "Guilherme J. Tramontina" Date: Wed, 20 Aug 2014 01:20:55 -0300 Subject: [PATCH 14/19] Ditch Redcarpet in favor of the already in user Kramdown. --- Gemfile | 1 - Gemfile.lock | 2 -- app/helpers/markdown_helper.rb | 3 +-- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 27214007..f7cd5299 100644 --- a/Gemfile +++ b/Gemfile @@ -85,7 +85,6 @@ gem 'feed-normalizer' gem 'slack-notifier', '~> 0.5.0' gem 'therubyracer', '~> 0.12.1' gem 'mqtt' -gem 'redcarpet', '~> 3.1.1' group :development do gem 'binding_of_caller' diff --git a/Gemfile.lock b/Gemfile.lock index ae792baa..6cfc8531 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -233,7 +233,6 @@ GEM thor (>= 0.18.1, < 2.0) raindrops (0.13.0) rake (10.3.2) - redcarpet (3.1.2) ref (1.0.5) rest-client (1.6.7) mime-types (>= 1.16) @@ -405,7 +404,6 @@ DEPENDENCIES rack rails (= 4.1.4) rails_12factor - redcarpet (~> 3.1.1) rr rspec (~> 2.99) rspec-collection_matchers diff --git a/app/helpers/markdown_helper.rb b/app/helpers/markdown_helper.rb index c06a81bb..fcac45f3 100644 --- a/app/helpers/markdown_helper.rb +++ b/app/helpers/markdown_helper.rb @@ -1,8 +1,7 @@ module MarkdownHelper def markdown(text) - markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML) - markdown.render(text).html_safe + Kramdown::Document.new(text, :auto_ids => false).to_html.html_safe end end From f2ef00b683b89c565bdf8dfee05d032152955ea9 Mon Sep 17 00:00:00 2001 From: "Guilherme J. Tramontina" Date: Wed, 20 Aug 2014 03:08:34 -0300 Subject: [PATCH 15/19] Render the (about to be) imported scenario description as markdown. --- app/views/scenario_imports/_step_two.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/scenario_imports/_step_two.html.erb b/app/views/scenario_imports/_step_two.html.erb index f681b961..46116f41 100644 --- a/app/views/scenario_imports/_step_two.html.erb +++ b/app/views/scenario_imports/_step_two.html.erb @@ -30,7 +30,7 @@ <% if @scenario_import.parsed_data["description"].present? %> -
      <%= @scenario_import.parsed_data["description"] %>
      +
      <%= markdown(@scenario_import.parsed_data["description"]) %>
      <% end %> From 624d41ed26dd46d75fb13c6e90d72bc8cc2a17a8 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Thu, 21 Aug 2014 10:29:44 +0900 Subject: [PATCH 16/19] Properly show "None" for empty sources/receivers. --- app/views/agents/show.html.erb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/agents/show.html.erb b/app/views/agents/show.html.erb index af220661..86e2292c 100644 --- a/app/views/agents/show.html.erb +++ b/app/views/agents/show.html.erb @@ -110,8 +110,8 @@ <% if @agent.can_receive_events? %>

      Event sources: - <% if @agent.sources.length %> - <%= @agent.sources.map { |source_agent| link_to(source_agent.name, agent_path(source_agent)) }.to_sentence.html_safe %> + <% if (agents = @agent.sources).length > 0 %> + <%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %> <% else %> None <% end %> @@ -126,8 +126,8 @@ <% if @agent.can_create_events? %>

      Event receivers: - <% if @agent.receivers.length %> - <%= @agent.receivers.map { |receiver_agent| link_to(receiver_agent.name, agent_path(receiver_agent)) }.to_sentence.html_safe %> + <% if (agents = @agent.receivers).length > 0 %> + <%= agents.map { |agent| link_to(agent.name, agent_path(agent)) }.to_sentence.html_safe %> <% else %> None <% end %> From 39046da035d2cd62825914c7851de92af503de6c Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Thu, 21 Aug 2014 17:20:12 +0900 Subject: [PATCH 17/19] Explain the basics of Ruby's regexp and leave a pointer to ruby-doc.org. --- app/models/agents/website_agent.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 90c73cb0..9a05b3c2 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -57,9 +57,11 @@ module Agents To extract the whole content as one event: "extract": { - "content": { "regexp": "\A(?:.|\n)*\z", index: 0 }, + "content": { "regexp": "\A(?m:.)*\z", index: 0 }, } + Beware that `.` does not match the newline character (LF) unless the `m` flag is in effect, and `^`/`$` basically match every line beginning/end. See [this document](http://ruby-doc.org/core-#{RUBY_VERSION}/doc/regexp_rdoc.html) to learn the regular expression variant used in this service. + Note that for all of the formats, whatever you extract MUST have the same number of matches for each extractor. E.g., if you're extracting rows, all extractors must match all rows. For generating CSS selectors, something like [SelectorGadget](http://selectorgadget.com) may be helpful. Can be configured to use HTTP basic auth by including the `basic_auth` parameter with `"username:password"`, or `["username", "password"]`. From f5aeaa7008005031678dd09e88f28c201542f8f2 Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Thu, 21 Aug 2014 18:05:09 +0900 Subject: [PATCH 18/19] When a dropdown menu opens on hover, apply the style class "open". --- app/assets/javascripts/application.js.coffee.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/assets/javascripts/application.js.coffee.erb b/app/assets/javascripts/application.js.coffee.erb index 06b345a5..748815bf 100644 --- a/app/assets/javascripts/application.js.coffee.erb +++ b/app/assets/javascripts/application.js.coffee.erb @@ -60,6 +60,10 @@ showEventDescriptions = -> $(".event-descriptions").html("").hide() $(document).ready -> + $('.navbar .dropdown.dropdown-hover').hover \ + -> $(this).addClass('open'), + -> $(this).removeClass('open') + # JSON Editor window.jsonEditor = setupJsonEditor()[0] From 0f1b30bd9cf394076cb90f2b19740f2368aaaf8d Mon Sep 17 00:00:00 2001 From: Akinori MUSHA Date: Thu, 21 Aug 2014 22:34:25 +0900 Subject: [PATCH 19/19] Move require's for faraday to the right place. --- app/concerns/web_request_concern.rb | 3 +++ app/models/agents/website_agent.rb | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/concerns/web_request_concern.rb b/app/concerns/web_request_concern.rb index ebb72d53..80e96b02 100644 --- a/app/concerns/web_request_concern.rb +++ b/app/concerns/web_request_concern.rb @@ -1,3 +1,6 @@ +require 'faraday' +require 'faraday_middleware' + module WebRequestConcern extend ActiveSupport::Concern diff --git a/app/models/agents/website_agent.rb b/app/models/agents/website_agent.rb index 9a05b3c2..f1103588 100644 --- a/app/models/agents/website_agent.rb +++ b/app/models/agents/website_agent.rb @@ -1,6 +1,4 @@ require 'nokogiri' -require 'faraday' -require 'faraday_middleware' require 'date' module Agents