Merge pull request #163 from cantino/javascript-agent

Javascript agent

Gemfile

@@ -24,6 +24,7 @@ gem 'coffee-rails', '~> 3.2.1'
 gem 'uglifier', '>= 1.0.3'
 gem 'select2-rails'
 gem 'jquery-rails'
+gem 'ace-rails-ap'
 
 gem 'geokit-rails3'
 gem 'kramdown'
@@ -37,6 +38,8 @@ gem 'twitter-stream', '>=0.1.16'
 gem 'em-http-request'
 gem 'weibo_2'
 
+gem 'therubyracer'
+
 platforms :ruby_18 do
   gem 'system_timer'
   gem 'fastercsv'

Gemfile.lock

@@ -1,6 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    ace-rails-ap (2.0.1)
     actionmailer (3.2.13)
       actionpack (= 3.2.13)
       mail (~> 2.5.3)
@@ -124,6 +125,7 @@ GEM
       actionpack (>= 3.0.0)
       activesupport (>= 3.0.0)
     kramdown (1.1.0)
+    libv8 (3.16.14.3)
     mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
@@ -174,6 +176,7 @@ GEM
     rake (10.1.0)
     rdoc (3.12.2)
       json (~> 1.4)
+    ref (1.0.5)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     rr (1.1.2)
@@ -224,6 +227,9 @@ GEM
     system_timer (1.2.4)
     term-ansicolor (1.2.2)
       tins (~> 0.8)
+    therubyracer (0.12.0)
+      libv8 (~> 3.16.14.0)
+      ref
     thor (0.18.1)
     tilt (1.4.1)
     tins (0.13.1)
@@ -267,6 +273,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  ace-rails-ap
   better_errors
   binding_of_caller
   bootstrap-kaminari-views
@@ -300,6 +307,7 @@ DEPENDENCIES
   select2-rails
   shoulda-matchers
   system_timer
+  therubyracer
   twilio-ruby
   twitter
   twitter-stream (>= 0.1.16)

app/assets/javascripts/user_credentials.js.coffee

@@ -0,0 +1,27 @@
+#= require ace/ace
+#= require ace/mode-javascript.js
+#= require ace/mode-markdown.js
+#= require_self
+
+$ ->
+  editor = ace.edit("ace-credential-value")
+  editor.getSession().setTabSize(2)
+  editor.getSession().setUseSoftTabs(true)
+  editor.getSession().setUseWrapMode(false)
+  editor.setTheme("ace/theme/chrome")
+
+  setMode = ->
+    mode = $("#user_credential_mode").val()
+    if mode == 'java_script'
+      editor.getSession().setMode("ace/mode/javascript")
+    else
+      editor.getSession().setMode("ace/mode/text")
+
+  setMode()
+  $("#user_credential_mode").on 'change', setMode
+
+  $textarea = $('#user_credential_credential_value').hide()
+  editor.getSession().setValue($textarea.val())
+
+  $textarea.closest('form').on 'submit', ->
+    $textarea.val(editor.getSession().getValue())

app/assets/stylesheets/application.css.scss.erb

@@ -126,3 +126,11 @@ span.not-applicable:after {
 #show-tabs li a.recent-errors {
   font-weight: bold;
 }
+
+// Credentials
+
+#ace-credential-value {
+  position: relative;
+  width: 940px;
+  height: 400px;
+}

app/models/agent.rb

@@ -16,7 +16,7 @@ class Agent < ActiveRecord::Base
   load_types_in "Agents"
 
   SCHEDULES = %w[every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d
-                 midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm]
+                 midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never]
 
   EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })]
 
@@ -296,6 +296,7 @@ class Agent < ActiveRecord::Base
     # Given a schedule name, run `check` via `bulk_check` on all Agents with that schedule.
     # This is called by bin/schedule.rb for each schedule in `SCHEDULES`.
     def run_schedule(schedule)
+      return if schedule == 'never'
       types = where(:schedule => schedule).group(:type).pluck(:type)
       types.each do |type|
         type.constantize.bulk_check(schedule)

app/models/agents/java_script_agent.rb

@@ -0,0 +1,186 @@
+require 'date'
+require 'cgi'
+
+module Agents
+  class JavaScriptAgent < Agent
+    default_schedule "never"
+
+    description <<-MD
+      This Agent allows you to write code in JavaScript that can create and receive events.  If other Agents aren't meeting your needs, try this one!
+
+      You can put code in the `code` option, or put your code in a Credential and reference it from `code` with `credential:<name>` (recommended).
+
+      You can implement `Agent.check` and `Agent.receive` as you see fit.  The following methods will be available on Agent in the JavaScript environment:
+
+      * `this.createEvent(payload)`
+      * `this.incomingEvents()`
+      * `this.memory()`
+      * `this.memory(key)`
+      * `this.memory(keyToSet, valueToSet)`
+      * `this.options()`
+      * `this.options(key)`
+      * `this.log(message)`
+      * `this.error(message)`
+    MD
+
+    def validate_options
+      cred_name = credential_referenced_by_code
+      if cred_name
+        errors.add(:base, "The credential '#{cred_name}' referenced by code cannot be found") unless credential(cred_name).present?
+      else
+        errors.add(:base, "The 'code' option is required") unless options['code'].present?
+      end
+    end
+
+    def working?
+      return false if recent_error_logs?
+
+      if options['expected_update_period_in_days'].present?
+        return false unless event_created_within?(options['expected_update_period_in_days'])
+      end
+
+      if options['expected_receive_period_in_days'].present?
+        return false unless last_receive_at && last_receive_at > options['expected_receive_period_in_days'].to_i.days.ago
+      end
+
+      true
+    end
+
+    def check
+      log_errors do
+        execute_js("check")
+      end
+    end
+
+    def receive(incoming_events)
+      log_errors do
+        execute_js("receive", incoming_events)
+      end
+    end
+
+    def default_options
+      js_code = <<-JS
+        Agent.check = function() {
+          if (this.options('make_event')) {
+            this.createEvent({ 'message': 'I made an event!' });
+            var callCount = this.memory('callCount') || 0;
+            this.memory('callCount', callCount + 1);
+          }
+        };
+        
+        Agent.receive = function() {
+          var events = this.incomingEvents();
+          for(var i = 0; i < events.length; i++) {
+            this.createEvent({ 'message': 'I got an event!', 'event_was': events[i].payload });
+          }
+        }
+      JS
+
+      {
+        "code" => js_code.gsub(/[\n\r\t]/, '').strip,
+        'expected_receive_period_in_days' => "2",
+        'expected_update_period_in_days' => "2"
+      }
+    end
+
+    private
+
+    def execute_js(js_function, incoming_events = [])
+      js_function = js_function == "check" ? "check" : "receive"
+      context = V8::Context.new
+      context.eval(setup_javascript)
+
+      context["doCreateEvent"] = lambda { |a, y| create_event(payload: clean_nans(JSON.parse(y))).payload.to_json }
+      context["getIncomingEvents"] = lambda { |a| incoming_events.to_json }
+      context["getOptions"] = lambda { |a, x| options.to_json }
+      context["doLog"] = lambda { |a, x| log x }
+      context["doError"] = lambda { |a, x| error x }
+      context["getMemory"] = lambda do |a, x, y|
+        if x && y
+          memory[x] = clean_nans(y)
+        else
+          memory.to_json
+        end
+      end
+
+      context.eval(code)
+      context.eval("Agent.#{js_function}();")
+    end
+
+    def code
+      cred = credential_referenced_by_code
+      if cred
+        credential(cred) || 'Agent.check = function() { this.error("Unable to find credential"); };'
+      else
+        options['code']
+      end
+    end
+
+    def credential_referenced_by_code
+      options['code'] =~ /\Acredential:(.*)\Z/ && $1
+    end
+
+    def setup_javascript
+      <<-JS
+        function Agent() {};
+
+        Agent.createEvent = function(opts) {
+          return JSON.parse(doCreateEvent(JSON.stringify(opts)));
+        }
+
+        Agent.incomingEvents = function() {
+          return JSON.parse(getIncomingEvents());
+        }
+
+        Agent.memory = function(key, value) {
+          if (typeof(key) !== "undefined" && typeof(value) !== "undefined") {
+            getMemory(key, value);
+          } else if (typeof(key) !== "undefined") {
+            return JSON.parse(getMemory())[key];
+          } else {
+            return JSON.parse(getMemory());
+          }
+        }
+
+        Agent.options = function(key) {
+          if (typeof(key) !== "undefined") {
+            return JSON.parse(getOptions())[key];
+          } else {
+            return JSON.parse(getOptions());
+          }
+        }
+
+        Agent.log = function(message) {
+          doLog(message);
+        }
+
+        Agent.error = function(message) {
+          doError(message);
+        }
+
+        Agent.check = function(){};
+        Agent.receive = function(){};
+      JS
+    end
+
+    def log_errors
+      begin
+        yield
+      rescue V8::Error => e
+        error "JavaScript error: #{e.message}"
+      end
+    end
+
+    def clean_nans(input)
+      if input.is_a?(Array)
+        input.map {|v| clean_nans(v) }
+      elsif input.is_a?(Hash)
+        input.inject({}) { |m, (k, v)| m[k] = clean_nans(v); m }
+      elsif input.is_a?(Float) && input.nan?
+        'NaN'
+      else
+        input
+      end
+    end
+  end
+end

app/models/user_credential.rb

@@ -1,13 +1,17 @@
 class UserCredential < ActiveRecord::Base
-  attr_accessible :credential_name, :credential_value
+  MODES = %w[text java_script]
+
+  attr_accessible :credential_name, :credential_value, :mode
 
   belongs_to :user
 
   validates_presence_of :credential_name
   validates_presence_of :credential_value
+  validates_inclusion_of :mode, :in => MODES
   validates_presence_of :user_id
   validates_uniqueness_of :credential_name, :scope => :user_id
 
+  before_validation :default_mode_to_text
   before_save :trim_fields
 
   protected
@@ -16,4 +20,8 @@ class UserCredential < ActiveRecord::Base
     credential_name.strip!
     credential_value.strip!
   end
+
+  def default_mode_to_text
+    self.mode = 'text' unless mode.present?
+  end
 end

app/views/agents/_form.html.erb

@@ -44,7 +44,6 @@
     </div>
   </div>
 
-
   <div class='event-related-region' data-can-create-events="<%= @agent.can_create_events? %>">
     <div class="control-group">
       <%= f.label :keep_events_for, "Keep events", :class => 'control-label' %>

app/views/user_credentials/_form.html.erb

@@ -17,10 +17,18 @@
     </div>
   </div>
 
+  <div class="control-group">
+    <%= f.label :mode, :class => 'control-label' %>
+    <div class="controls">
+      <%= f.select :mode, options_for_select(UserCredential::MODES.map {|s| [s.classify, s] }, @user_credential.mode), {}, :class => 'span4' %>
+    </div>
+  </div>
+
   <div class="control-group">
     <%= f.label :credential_value, :class => 'control-label' %>
     <div class="controls">
       <%= f.text_area :credential_value, :class => 'span8', :rows => 10 %>
+      <div id="ace-credential-value"></div>
     </div>
   </div>
 
@@ -28,3 +36,5 @@
     <%= f.submit "Save Credential", :class => "btn btn-primary" %>
   </div>
 <% end %>
+
+<%= javascript_include_tag "user_credentials" %>

config/environments/production.rb

@@ -48,7 +48,7 @@ Huginn::Application.configure do
   end
 
   # Precompile additional assets (application.js.coffee.erb, application.css, and all non-JS/CSS are already added)
-  config.assets.precompile += %w( graphing.js )
+  config.assets.precompile += %w( graphing.js user_credentials.js )
 
   # Enable threaded mode
   # config.threadsafe!

db/migrate/20140210062747_add_mode_to_user_credentials.rb

@@ -0,0 +1,5 @@
+class AddModeToUserCredentials < ActiveRecord::Migration
+  def change
+    add_column :user_credentials, :mode, :string, :default => 'text', :null => false
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20140127164931) do
+ActiveRecord::Schema.define(:version => 20140210062747) do
 
   create_table "agent_logs", :force => true do |t|
     t.integer  "agent_id",                                             :null => false
@@ -96,11 +96,12 @@ ActiveRecord::Schema.define(:version => 20140127164931) do
   add_index "links", ["source_id", "receiver_id"], :name => "index_links_on_source_id_and_receiver_id"
 
   create_table "user_credentials", :force => true do |t|
-    t.integer  "user_id",          :null => false
-    t.string   "credential_name",  :null => false
-    t.text     "credential_value", :null => false
-    t.datetime "created_at",       :null => false
-    t.datetime "updated_at",       :null => false
+    t.integer  "user_id",                              :null => false
+    t.string   "credential_name",                      :null => false
+    t.text     "credential_value",                     :null => false
+    t.datetime "created_at",                           :null => false
+    t.datetime "updated_at",                           :null => false
+    t.string   "mode",             :default => "text", :null => false
   end
 
   add_index "user_credentials", ["user_id", "credential_name"], :name => "index_user_credentials_on_user_id_and_credential_name", :unique => true

spec/fixtures/events.yml

@@ -1,9 +1,9 @@
 bob_website_agent_event:
   user: bob
   agent: bob_website_agent
-  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>
+  payload: <%= { :title => "foo", :url => "http://foo.com" }.to_json.inspect %>
 
 jane_website_agent_event:
   user: jane
   agent: jane_website_agent
-  payload: <%= [{ :title => "foo", :url => "http://foo.com" }].to_json.inspect %>
+  payload: <%= { :title => "foo", :url => "http://foo.com" }.to_json.inspect %>

spec/fixtures/user_credentials.yml

@@ -2,15 +2,19 @@ bob_aws_key:
   user: bob
   credential_name: aws_key
   credential_value: 2222222222-bob
+  mode: text
 bob_aws_secret:
   user: bob
   credential_name: aws_secret
   credential_value: 1111111111-bob
+  mode: text
 jane_aws_key:
   user: jane
   credential_name: aws_key
   credential_value: 2222222222-jane
+  mode: text
 jane_aws_secret:
   user: jane
   credential_name: aws_secret
   credential_value: 1111111111-jabe
+  mode: text

spec/models/agent_spec.rb

@@ -25,6 +25,12 @@ describe Agent do
       do_not_allow(Agents::WebsiteAgent).async_check
       Agent.run_schedule("blah")
     end
+
+    it "will not run the 'never' schedule" do
+      agents(:bob_weather_agent).update_attribute 'schedule', 'never'
+      do_not_allow(Agents::WebsiteAgent).async_check
+      Agent.run_schedule("never")
+    end
   end
 
   describe "credential" do

spec/models/agents/java_script_agent_spec.rb

@@ -0,0 +1,228 @@
+require 'spec_helper'
+
+describe Agents::JavaScriptAgent do
+  before do
+    @valid_params = {
+      :name => "somename",
+      :options => {
+        :code => "Agent.check = function() { this.createEvent({ 'message': 'hi' }); };",
+      }
+    }
+
+    @agent = Agents::JavaScriptAgent.new(@valid_params)
+    @agent.user = users(:jane)
+    @agent.save!
+  end
+
+  describe "validations" do
+    it "requires 'code'" do
+      @agent.should be_valid
+      @agent.options['code'] = ''
+      @agent.should_not be_valid
+      @agent.options.delete('code')
+      @agent.should_not be_valid
+    end
+
+    it "accepts a credential, but it must exist" do
+      @agent.should be_valid
+      @agent.options['code'] = 'credential:foo'
+      @agent.should_not be_valid
+      users(:jane).user_credentials.create! :credential_name => "foo", :credential_value => "bar"
+      @agent.reload.should be_valid
+    end
+  end
+
+  describe "#working?" do
+    describe "when expected_update_period_in_days is set" do
+      it "returns false when more than expected_update_period_in_days have passed since the last event creation" do
+        @agent.options['expected_update_period_in_days'] = 1
+        @agent.save!
+        @agent.should_not be_working
+        @agent.check
+        @agent.reload.should be_working
+        three_days_from_now = 3.days.from_now
+        stub(Time).now { three_days_from_now }
+        @agent.should_not be_working
+      end
+    end
+
+    describe "when expected_receive_period_in_days is set" do
+      it "returns false when more than expected_receive_period_in_days have passed since the last event was received" do
+        @agent.options['expected_receive_period_in_days'] = 1
+        @agent.save!
+        @agent.should_not be_working
+        Agents::JavaScriptAgent.async_receive @agent.id, [events(:bob_website_agent_event).id]
+        @agent.reload.should be_working
+        two_days_from_now = 2.days.from_now
+        stub(Time).now { two_days_from_now }
+        @agent.reload.should_not be_working
+      end
+    end
+  end
+
+  describe "executing code" do
+    it "works by default" do
+      @agent.options = @agent.default_options
+      @agent.options['make_event'] = true
+      @agent.save!
+
+      lambda {
+        lambda {
+          @agent.receive([events(:bob_website_agent_event)])
+          @agent.check
+        }.should_not change { AgentLog.count }
+      }.should change { Event.count }.by(2)
+    end
+
+
+    describe "using credentials as code" do
+      before do
+        @agent.user.user_credentials.create :credential_name => 'code-foo', :credential_value => 'Agent.check = function() { this.log("ran it"); };'
+        @agent.options['code'] = 'credential:code-foo'
+        @agent.save!
+      end
+
+      it "accepts credentials" do
+        @agent.check
+        AgentLog.last.message.should == "ran it"
+      end
+
+      it "logs an error when the credential goes away" do
+        @agent.user.user_credentials.delete_all
+        @agent.reload.check
+        AgentLog.last.message.should == "Unable to find credential"
+      end
+    end
+
+    describe "error handling" do
+      it "should log an error when V8 has issues" do
+        @agent.options['code'] = 'syntax error!'
+        @agent.save!
+        lambda {
+          lambda {
+            @agent.check
+          }.should_not raise_error
+        }.should change { AgentLog.count }.by(1)
+        AgentLog.last.message.should =~ /Unexpected identifier/
+        AgentLog.last.level.should == 4
+      end
+
+      it "should log an error when JavaScript throws" do
+        @agent.options['code'] = 'Agent.check = function() { throw "oh no"; };'
+        @agent.save!
+        lambda {
+          lambda {
+            @agent.check
+          }.should_not raise_error
+        }.should change { AgentLog.count }.by(1)
+        AgentLog.last.message.should =~ /oh no/
+        AgentLog.last.level.should == 4
+      end
+
+      it "won't store NaNs" do
+        @agent.options['code'] = 'Agent.check = function() { this.memory("foo", NaN); };'
+        @agent.save!
+        @agent.check
+        @agent.memory['foo'].should == 'NaN' # string
+        @agent.save!
+        lambda { @agent.reload.memory }.should_not raise_error
+      end
+    end
+
+    describe "creating events" do
+      it "creates events with this.createEvent in the JavaScript environment" do
+        @agent.options['code'] = 'Agent.check = function() { this.createEvent({ message: "This is an event!", stuff: { foo: 5 } }); };'
+        @agent.save!
+        lambda {
+          lambda {
+            @agent.check
+          }.should_not change { AgentLog.count }
+        }.should change { Event.count }.by(1)
+        created_event = @agent.events.last
+        created_event.payload.should == { 'message' => "This is an event!", 'stuff' => { 'foo' => 5 } }
+      end
+    end
+
+    describe "logging" do
+      it "can output AgentLogs with this.log and this.error in the JavaScript environment" do
+        @agent.options['code'] = 'Agent.check = function() { this.log("woah"); this.error("WOAH!"); };'
+        @agent.save!
+        lambda {
+          lambda {
+            @agent.check
+          }.should_not raise_error
+        }.should change { AgentLog.count }.by(2)
+
+        log1, log2 = AgentLog.last(2)
+
+        log1.message.should == "woah"
+        log1.level.should == 3
+        log2.message.should == "WOAH!"
+        log2.level.should == 4
+      end
+    end
+
+    describe "getting incoming events" do
+      it "can access incoming events in the JavaScript enviroment via this.incomingEvents" do
+        event = Event.new
+        event.agent = agents(:bob_rain_notifier_agent)
+        event.payload = { :data => "Something you should know about" }
+        event.save!
+        event.reload
+
+        @agent.options['code'] = <<-JS
+          Agent.receive = function() {
+            var events = this.incomingEvents();
+            for(var i = 0; i < events.length; i++) {
+              this.createEvent({ 'message': 'I got an event!', 'event_was': events[i].payload });
+            }
+          }
+        JS
+
+        @agent.save!
+        lambda {
+          lambda {
+            @agent.receive([events(:bob_website_agent_event), event])
+          }.should_not change { AgentLog.count }
+        }.should change { Event.count }.by(2)
+        created_event = @agent.events.first
+        created_event.payload.should == { 'message' => "I got an event!", 'event_was' => { 'data' => "Something you should know about" } }
+      end
+    end
+
+    describe "getting and setting memory, getting options" do
+      it "can access options via this.options and work with memory via this.memory" do
+        @agent.options['code'] = <<-JS
+          Agent.check = function() {
+            if (this.options('make_event')) {
+              var callCount = this.memory('callCount') || 0;
+              this.memory('callCount', callCount + 1);
+            }
+          };
+        JS
+
+        @agent.save!
+
+        lambda {
+          lambda {
+
+            @agent.check
+            @agent.memory['callCount'].should_not be_present
+
+            @agent.options['make_event'] = true
+            @agent.check
+            @agent.memory['callCount'].should == 1
+
+            @agent.check
+            @agent.memory['callCount'].should == 2
+
+            @agent.memory['callCount'] = 20
+            @agent.check
+            @agent.memory['callCount'].should == 21
+
+          }.should_not change { AgentLog.count }
+        }.should_not change { Event.count }
+      end
+    end
+  end
+end