From d9bd6a991b971fb1011bc72764413655bf845e4f Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 27 Apr 2014 19:43:31 +0200 Subject: [PATCH 01/41] Added liquid templating and migrated first agents --- Gemfile | 1 + Gemfile.lock | 2 + app/concerns/liquid_interpolatable.rb | 27 +++++++ app/models/agents/event_formatting_agent.rb | 21 +++--- app/models/agents/hipchat_agent.rb | 16 +--- ..._migrate_hipchat_and_ef_agent_to_liquid.rb | 11 +++ lib/liquid_migrator.rb | 57 +++++++++++++++ spec/lib/liquid_migrator_spec.rb | 73 +++++++++++++++++++ .../agents/event_formatting_agent_spec.rb | 6 +- spec/models/agents/hipchat_agent_spec.rb | 13 +--- spec/models/concerns/liquid_interpolatable.rb | 31 ++++++++ 11 files changed, 223 insertions(+), 35 deletions(-) create mode 100644 app/concerns/liquid_interpolatable.rb create mode 100644 db/migrate/20140426202023_migrate_hipchat_and_ef_agent_to_liquid.rb create mode 100644 lib/liquid_migrator.rb create mode 100644 spec/lib/liquid_migrator_spec.rb create mode 100644 spec/models/concerns/liquid_interpolatable.rb diff --git a/Gemfile b/Gemfile index e3c91c4c..2e03cde6 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,7 @@ gem 'json', '~> 1.8.1' gem 'jsonpath', '~> 0.5.3' gem 'twilio-ruby', '~> 3.11.5' gem 'ruby-growl', '~> 4.1.0' +gem 'liquid', '~> 2.6.1' gem 'delayed_job', '~> 4.0.0' gem 'delayed_job_active_record', '~> 4.0.0' diff --git a/Gemfile.lock b/Gemfile.lock index 081c833b..1dbb4245 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -148,6 +148,7 @@ GEM activesupport (>= 3.0.0) kramdown (1.3.3) libv8 (3.16.14.3) + liquid (2.6.1) macaddr (1.7.1) systemu (~> 2.6.2) mail (2.5.4) @@ -337,6 +338,7 @@ DEPENDENCIES jsonpath (~> 0.5.3) kaminari (~> 0.15.1) kramdown (~> 1.3.3) + liquid (~> 2.6.1) mysql2 (~> 0.3.15) nokogiri (~> 1.6.1) protected_attributes (~> 1.0.7) diff --git a/app/concerns/liquid_interpolatable.rb b/app/concerns/liquid_interpolatable.rb new file mode 100644 index 00000000..f099204a --- /dev/null +++ b/app/concerns/liquid_interpolatable.rb @@ -0,0 +1,27 @@ +module LiquidInterpolatable + extend ActiveSupport::Concern + + def interpolate_options options, payload + duped_options = options.dup.tap do |duped_options| + duped_options.each_pair do |key, value| + if value.class == String + duped_options[key] = Liquid::Template.parse(value).render(payload) + else + duped_options[key] = value + end + end + end + duped_options + end + + require 'uri' + # Percent encoding for URI conforming to RFC 3986. + # Ref: http://tools.ietf.org/html/rfc3986#page-12 + module Huginn + def uri_escape(string) + CGI::escape string + end + end + + Liquid::Template.register_filter(LiquidInterpolatable::Huginn) +end diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb index 53765812..1f2bc74a 100644 --- a/app/models/agents/event_formatting_agent.rb +++ b/app/models/agents/event_formatting_agent.rb @@ -1,5 +1,6 @@ module Agents class EventFormattingAgent < Agent + include LiquidInterpolatable cannot_be_scheduled! description <<-MD @@ -24,11 +25,11 @@ module Agents You can use an Event Formatting Agent's `instructions` setting to do this in the following way: "instructions": { - "message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius.", - "subject": "$.data" + "message": "Today's conditions look like {{conditions}} with a high temperature of {{high.celsius}} degrees Celsius.", + "subject": "{{data}}" } - JSONPaths must be between < and > . Make sure that you don't use these symbols anywhere else. + FIXME Provide a link to a explanation on how to use liquid templating Events generated by this possible Event Formatting Agent will look like: @@ -60,18 +61,18 @@ module Agents So you can use it in `instructions` like this: "instructions": { - "message": "Today's conditions look like <$.conditions> with a high temperature of <$.high.celsius> degrees Celsius according to the forecast at <$.pretty_date.time>.", - "subject": "$.data" + "message": "Today's conditions look like <$.conditions> with a high temperature of {{high.celsius}} degrees Celsius according to the forecast at {{pretty_date.time}}.", + "subject": "{{data}}" } If you want to retain original contents of events and only add new keys, then set `mode` to `merge`, otherwise set it to `clean`. By default, the output event will have `agent` and `created_at` fields added as well, reflecting the original Agent type and Event creation time. You can skip these outputs by setting `skip_agent` and `skip_created_at` to `true`. - To CGI escape output (for example when creating a link), prefix with `escape`, like so: + To CGI escape output (for example when creating a link), use the Liquid `uri_escape` filter, like so: { - "message": "A peak was on Twitter in <$.group_by>. Search: https://twitter.com/search?q=" + "message": "A peak was on Twitter in {{group_by}}. Search: https://twitter.com/search?q={{group_by | uri_escape}}" } MD @@ -88,8 +89,8 @@ module Agents def default_options { 'instructions' => { - 'message' => "You received a text <$.text> from <$.fields.from>", - 'some_other_field' => "Looks like the weather is going to be <$.fields.weather>" + 'message' => "You received a text {{text}} from {{fields.from}}", + 'some_other_field' => "Looks like the weather is going to be {{fields.weather}}" }, 'matchers' => [], 'mode' => "clean", @@ -106,7 +107,7 @@ module Agents incoming_events.each do |event| formatted_event = options['mode'].to_s == "merge" ? event.payload.dup : {} payload = perform_matching(event.payload) - options['instructions'].each_pair {|key, value| formatted_event[key] = Utils.interpolate_jsonpaths(value, payload) } + formatted_event.merge! interpolate_options(options['instructions'], payload) formatted_event['agent'] = Agent.find(event.agent_id).type.slice!(8..-1) unless options['skip_agent'].to_s == "true" formatted_event['created_at'] = event.created_at unless options['skip_created_at'].to_s == "true" create_event :payload => formatted_event diff --git a/app/models/agents/hipchat_agent.rb b/app/models/agents/hipchat_agent.rb index d4a9b8c2..fc4015cf 100644 --- a/app/models/agents/hipchat_agent.rb +++ b/app/models/agents/hipchat_agent.rb @@ -1,6 +1,6 @@ module Agents class HipchatAgent < Agent - include JsonPathOptionsOverwritable + include LiquidInterpolatable cannot_be_scheduled! cannot_create_events! @@ -18,22 +18,17 @@ module Agents If you want your message to notify the room members change `notify` to "true". Modify the background color of your message via the `color` attribute (one of "yellow", "red", "green", "purple", "gray", or "random") - If you want to specify either of those attributes per event, you can provide a [JSONPath](http://goessner.net/articles/JsonPath/) for each of them (except the `auth_token`). + TODO: add a link to the wiki explaining how to use the Liquid templating MD def default_options { 'auth_token' => '', 'room_name' => '', - 'room_name_path' => '', 'username' => "Huginn", - 'username_path' => '', 'message' => "Hello from Huginn!", - 'message_path' => '', 'notify' => false, - 'notify_path' => '', 'color' => 'yellow', - 'color_path' => '', } end @@ -49,14 +44,9 @@ module Agents def receive(incoming_events) client = HipChat::Client.new(options[:auth_token]) incoming_events.each do |event| - mo = merge_json_path_options event + mo = interpolate_options options, event.payload client[mo[:room_name]].send(mo[:username], mo[:message], :notify => mo[:notify].to_s == 'true' ? 1 : 0, :color => mo[:color]) end end - - private - def options_with_path - [:room_name, :username, :message, :notify, :color] - end end end diff --git a/db/migrate/20140426202023_migrate_hipchat_and_ef_agent_to_liquid.rb b/db/migrate/20140426202023_migrate_hipchat_and_ef_agent_to_liquid.rb new file mode 100644 index 00000000..35d6f989 --- /dev/null +++ b/db/migrate/20140426202023_migrate_hipchat_and_ef_agent_to_liquid.rb @@ -0,0 +1,11 @@ +class MigrateHipchatAndEfAgentToLiquid < ActiveRecord::Migration + def change + Agent.where(:type => 'Agents::HipchatAgent').each do |agent| + LiquidMigrator.convert_all_agent_options(agent) + end + Agent.where(:type => 'Agents::EventFormattingAgent').each do |agent| + agent.options['instructions'] = LiquidMigrator.convert_hash(agent.options['instructions'], {:merge_path_attributes => true, :leading_dollarsign_is_jsonpath => true}) + agent.save + end + end +end diff --git a/lib/liquid_migrator.rb b/lib/liquid_migrator.rb new file mode 100644 index 00000000..b6675752 --- /dev/null +++ b/lib/liquid_migrator.rb @@ -0,0 +1,57 @@ +module LiquidMigrator + def self.convert_all_agent_options(agent) + agent.options = self.convert_hash(agent.options, {:merge_path_attributes => true, :leading_dollarsign_is_jsonpath => true}) + agent.save! + end + + def self.convert_hash(hash, options={}) + options = {:merge_path_attributes => false, :leading_dollarsign_is_jsonpath => false}.merge options + keys_to_remove = [] + hash.tap do |hash| + hash.each_pair do |key, value| + case value.class.to_s + when 'String', 'FalseClass', 'TrueClass' + path_key = "#{key}_path" + if options[:merge_path_attributes] && !hash[path_key].nil? + # replace the value if the path is present + value = hash[path_key] if hash[path_key].present? + # in any case delete the path attibute + keys_to_remove << path_key + end + hash[key] = LiquidMigrator.convert_string value, options[:leading_dollarsign_is_jsonpath] + when 'Hash' + # might want to make it recursive? + when 'Array' + # do we need it? + end + end + # remove the unneeded *_path attributes + end.select { |k, v| !keys_to_remove.include? k } + end + + def self.convert_string(string, leading_dollarsign_is_jsonpath=false) + if string == true || string == false + # there might be empty *_path attributes for boolean defaults + string + elsif string[0] == '$' && leading_dollarsign_is_jsonpath + # in most cases a *_path attribute + convert_json_path string + else + # migrate the old interpolation syntax to the new liquid based + string.gsub(/<([^>]+)>/).each do + match = $1 + if match =~ /\Aescape / + # convert the old escape syntax to a liquid filter + self.convert_json_path(match.gsub(/\Aescape /, '').strip, ' | uri_escape') + else + self.convert_json_path(match.strip) + end + end + end + end + + def self.convert_json_path(string, filter = "") + "{{#{string[2..-1].gsub(/\.\*\Z/, '')}#{filter}}}" + end +end + diff --git a/spec/lib/liquid_migrator_spec.rb b/spec/lib/liquid_migrator_spec.rb new file mode 100644 index 00000000..6664b7de --- /dev/null +++ b/spec/lib/liquid_migrator_spec.rb @@ -0,0 +1,73 @@ +require 'spec_helper' + +describe LiquidMigrator do + describe "converting JSONPath strings" do + it "should work" do + LiquidMigrator.convert_string("$.data", true).should == "{{data}}" + LiquidMigrator.convert_string("$.data.test", true).should == "{{data.test}}" + LiquidMigrator.convert_string("$.data.test.*", true).should == "{{data.test}}" + end + + it "should ignore strings which just contain a JSONPath" do + LiquidMigrator.convert_string("$.data").should == "$.data" + LiquidMigrator.convert_string(" $.data", true).should == " $.data" + LiquidMigrator.convert_string("lorem $.data", true).should == "lorem $.data" + end + end + + describe "converting escaped JSONPath strings" do + it "should work" do + LiquidMigrator.convert_string("Received <$.content.text.*> from <$.content.name> .").should == + "Received {{content.text}} from {{content.name}} ." + LiquidMigrator.convert_string("Weather looks like <$.conditions> according to the forecast at <$.pretty_date.time>").should == + "Weather looks like {{conditions}} according to the forecast at {{pretty_date.time}}" + end + + it "should convert the 'escape' method correctly" do + LiquidMigrator.convert_string("Escaped: \nNot escaped: <$.content.name>").should == + "Escaped: {{content.name | uri_escape}}\nNot escaped: {{content.name}}" + end + end + + describe "migrating a hash" do + it "should convert every attribute" do + LiquidMigrator.convert_hash({'a' => "$.data", 'b' => "This is a <$.test>"}).should == + {'a' => "$.data", 'b' => "This is a {{test}}"} + end + it "should work with leading_dollarsign_is_jsonpath" do + LiquidMigrator.convert_hash({'a' => "$.data", 'b' => "This is a <$.test>"}, leading_dollarsign_is_jsonpath: true).should == + {'a' => "{{data}}", 'b' => "This is a {{test}}"} + end + it "should use the corresponding *_path attributes when using merge_path_attributes"do + LiquidMigrator.convert_hash({'a' => "default", 'a_path' => "$.data"}, {leading_dollarsign_is_jsonpath: true, merge_path_attributes: true}).should == + {'a' => "{{data}}"} + end + end + + describe "migrating an actual agent" do + before do + valid_params = { + 'auth_token' => 'token', + 'room_name' => 'test', + 'room_name_path' => '', + 'username' => "Huginn", + 'username_path' => '$.username', + 'message' => "Hello from Huginn!", + 'message_path' => '$.message', + 'notify' => false, + 'notify_path' => '', + 'color' => 'yellow', + 'color_path' => '', + } + + @agent = Agents::HipchatAgent.new(:name => "somename", :options => valid_params) + @agent.user = users(:jane) + @agent.save! + end + + it "should work" do + LiquidMigrator.convert_all_agent_options(@agent) + @agent.reload.options.should == {"auth_token" => 'token', 'color' => 'yellow', 'notify' => false, 'room_name' => 'test', 'username' => '{{username}}', 'message' => '{{message}}'} + end + end +end \ No newline at end of file diff --git a/spec/models/agents/event_formatting_agent_spec.rb b/spec/models/agents/event_formatting_agent_spec.rb index f4b60b0d..d13b17c4 100644 --- a/spec/models/agents/event_formatting_agent_spec.rb +++ b/spec/models/agents/event_formatting_agent_spec.rb @@ -6,8 +6,8 @@ describe Agents::EventFormattingAgent do :name => "somename", :options => { :instructions => { - :message => "Received <$.content.text.*> from <$.content.name> .", - :subject => "Weather looks like <$.conditions> according to the forecast at <$.pretty_date.time>" + :message => "Received {{content.text}} from {{content.name}} .", + :subject => "Weather looks like {{conditions}} according to the forecast at {{pretty_date.time}}" }, :mode => "clean", :matchers => [ @@ -82,7 +82,7 @@ describe Agents::EventFormattingAgent do it "should allow escaping" do @event.payload[:content][:name] = "escape this!?" @event.save! - @checker.options[:instructions][:message] = "Escaped: \nNot escaped: <$.content.name>" + @checker.options[:instructions][:message] = "Escaped: {{content.name | uri_escape}}\nNot escaped: {{content.name}}" @checker.save! @checker.receive([@event]) Event.last.payload[:message].should == "Escaped: escape+this%21%3F\nNot escaped: escape this!?" diff --git a/spec/models/agents/hipchat_agent_spec.rb b/spec/models/agents/hipchat_agent_spec.rb index 62199a33..5f29b50e 100644 --- a/spec/models/agents/hipchat_agent_spec.rb +++ b/spec/models/agents/hipchat_agent_spec.rb @@ -1,22 +1,17 @@ require 'spec_helper' -require 'models/concerns/json_path_options_overwritable' +require 'models/concerns/liquid_interpolatable' describe Agents::HipchatAgent do - it_behaves_like JsonPathOptionsOverwritable + it_behaves_like LiquidInterpolatable before(:each) do @valid_params = { 'auth_token' => 'token', 'room_name' => 'test', - 'room_name_path' => '', - 'username' => "Huginn", - 'username_path' => '$.username', - 'message' => "Hello from Huginn!", - 'message_path' => '$.message', + 'username' => "{{username}}", + 'message' => "{{message}}", 'notify' => false, - 'notify_path' => '', 'color' => 'yellow', - 'color_path' => '', } @checker = Agents::HipchatAgent.new(:name => "somename", :options => @valid_params) diff --git a/spec/models/concerns/liquid_interpolatable.rb b/spec/models/concerns/liquid_interpolatable.rb new file mode 100644 index 00000000..1db0747e --- /dev/null +++ b/spec/models/concerns/liquid_interpolatable.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +shared_examples_for LiquidInterpolatable do + before(:each) do + @valid_params = { + "normal" => "just some normal text", + "variable" => "{{variable}}", + "text" => "Some test with an embedded {{variable}}", + "escape" => "This should be {{hello_world | uri_escape}}" + } + + @checker = described_class.new(:name => "somename", :options => @valid_params) + @checker.user = users(:jane) + + @event = Event.new + @event.agent = agents(:bob_weather_agent) + @event.payload = { :variable => 'hello', :hello_world => "Hello world"} + @event.save! + end + + describe "interpolating liquid templates" do + it "should work" do + @checker.send(:interpolate_options, @checker.options, @event.payload).should == { + "normal" => "just some normal text", + "variable" => "hello", + "text" => "Some test with an embedded hello", + "escape" => "This should be Hello+world" + } + end + end +end From e8ecbe25b88ff27bb77f19cb5a097c5cabbbee19 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Tue, 29 Apr 2014 19:58:02 +0200 Subject: [PATCH 02/41] Use liquid in the matches section of EventFormattingAgent --- app/concerns/liquid_interpolatable.rb | 4 ++++ app/models/agents/event_formatting_agent.rb | 4 ++-- spec/models/agents/event_formatting_agent_spec.rb | 2 +- spec/models/concerns/liquid_interpolatable.rb | 5 +++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/app/concerns/liquid_interpolatable.rb b/app/concerns/liquid_interpolatable.rb index f099204a..45039b75 100644 --- a/app/concerns/liquid_interpolatable.rb +++ b/app/concerns/liquid_interpolatable.rb @@ -14,6 +14,10 @@ module LiquidInterpolatable duped_options end + def interpolate_string string, payload + Liquid::Template.parse(string).render(payload) + end + require 'uri' # Percent encoding for URI conforming to RFC 3986. # Ref: http://tools.ietf.org/html/rfc3986#page-12 diff --git a/app/models/agents/event_formatting_agent.rb b/app/models/agents/event_formatting_agent.rb index 1f2bc74a..f0c5e4a6 100644 --- a/app/models/agents/event_formatting_agent.rb +++ b/app/models/agents/event_formatting_agent.rb @@ -43,7 +43,7 @@ module Agents { "matchers": [ { - "path": "$.date.pretty", + "path": "{{date.pretty}}", "regexp": "\\A(?