From 9a5e22a857e3205ce552e801a33887c214a74976 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Sat, 25 Apr 2015 23:15:03 -0400 Subject: [PATCH 01/13] env option to allow frequent event expiration, fixes #765 --- .env.example | 5 +++++ app/models/agent.rb | 11 ++++++++++- lib/huginn_scheduler.rb | 10 ++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 77790a99..5ee1273a 100644 --- a/.env.example +++ b/.env.example @@ -146,6 +146,11 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false # at the expense of time accuracy. #SCHEDULER_FREQUENCY=0.3 +# Specify whether events ages should be checked daily or frequently. +# A value of "normal" checks each day at midnight, while a value of +# "frequent" adds an option to keep events for 90 seconds and checks every 3 minutes +SCHEDULER_EXPIRATION_CHECK=normal + # Use Graphviz for generating diagrams instead of using Google Chart # Tools. Specify a dot(1) command path built with SVG support # enabled. diff --git a/app/models/agent.rb b/app/models/agent.rb index 5cc2496b..cda9e03f 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -22,6 +22,9 @@ class Agent < ActiveRecord::Base midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never] EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })] + if ENV['SCHEDULER_EXPIRATION_CHECK'] == 'frequent' + EVENT_RETENTION_SCHEDULES.push(["A minute", 60]) + end attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events @@ -129,12 +132,18 @@ class Agent < ActiveRecord::Base end def new_event_expiration_date - keep_events_for > 0 ? keep_events_for.days.from_now : nil + if keep_events_for == 60 + 90.seconds.from_now + else + keep_events_for > 0 ? keep_events_for.days.from_now : nil + end end def update_event_expirations! if keep_events_for == 0 events.update_all :expires_at => nil + elsif keep_events_for == 60 + events.update_all "expires_at = " + rdbms_date_add("created_at", "SECOND", 90) else events.update_all "expires_at = " + rdbms_date_add("created_at", "DAY", keep_events_for.to_i) end diff --git a/lib/huginn_scheduler.rb b/lib/huginn_scheduler.rb index bedd3528..7f96ed07 100644 --- a/lib/huginn_scheduler.rb +++ b/lib/huginn_scheduler.rb @@ -114,8 +114,14 @@ class HuginnScheduler end # Schedule event cleanup. - @rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do - cleanup_expired_events! + if ENV['SCHEDULER_EXPIRATION_CHECK'] == 'normal' + @rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do + cleanup_expired_events! + end + elsif ENV['SCHEDULER_EXPIRATION_CHECK'] == 'frequent' + @rufus_scheduler.every '3m' do + cleanup_expired_events! + end end # Schedule failed job cleanup. From 725bc245135799cab3ae1959236a3fc889761bd3 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Sat, 25 Apr 2015 23:30:24 -0400 Subject: [PATCH 02/13] move short event life option earlier in list --- app/models/agent.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/agent.rb b/app/models/agent.rb index cda9e03f..0c21b2ca 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -23,7 +23,7 @@ class Agent < ActiveRecord::Base EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })] if ENV['SCHEDULER_EXPIRATION_CHECK'] == 'frequent' - EVENT_RETENTION_SCHEDULES.push(["A minute", 60]) + EVENT_RETENTION_SCHEDULES.insert(1, ["A minute", 60]) end attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events From 855799d355259f01c85f10af5f6a7918844ce0c4 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 7 May 2015 08:14:16 -0400 Subject: [PATCH 03/13] make cleanup time fully configurable --- .env.example | 7 +++---- app/models/agent.rb | 3 --- lib/huginn_scheduler.rb | 10 ++-------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 5ee1273a..1da3a13a 100644 --- a/.env.example +++ b/.env.example @@ -146,10 +146,9 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false # at the expense of time accuracy. #SCHEDULER_FREQUENCY=0.3 -# Specify whether events ages should be checked daily or frequently. -# A value of "normal" checks each day at midnight, while a value of -# "frequent" adds an option to keep events for 90 seconds and checks every 3 minutes -SCHEDULER_EXPIRATION_CHECK=normal +# Specify the frequency with which the scheduler checks for event expiration. +# You can use `m` for minutes, `h` for hours, and `d` for days. +EVENT_EXPIRATION_CHECK="3h" # Use Graphviz for generating diagrams instead of using Google Chart # Tools. Specify a dot(1) command path built with SVG support diff --git a/app/models/agent.rb b/app/models/agent.rb index 0c21b2ca..827bc2de 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -22,9 +22,6 @@ class Agent < ActiveRecord::Base midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never] EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })] - if ENV['SCHEDULER_EXPIRATION_CHECK'] == 'frequent' - EVENT_RETENTION_SCHEDULES.insert(1, ["A minute", 60]) - end attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events diff --git a/lib/huginn_scheduler.rb b/lib/huginn_scheduler.rb index 7f96ed07..dc6861e6 100644 --- a/lib/huginn_scheduler.rb +++ b/lib/huginn_scheduler.rb @@ -114,14 +114,8 @@ class HuginnScheduler end # Schedule event cleanup. - if ENV['SCHEDULER_EXPIRATION_CHECK'] == 'normal' - @rufus_scheduler.cron "0 0 * * * " + tzinfo_friendly_timezone do - cleanup_expired_events! - end - elsif ENV['SCHEDULER_EXPIRATION_CHECK'] == 'frequent' - @rufus_scheduler.every '3m' do - cleanup_expired_events! - end + @rufus_scheduler.every ENV['EVENT_EXPIRATION_CHECK'] do + cleanup_expired_events! end # Schedule failed job cleanup. From 0d7069adbf65737fadbbb83515bab888aa96decd Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 11 May 2015 08:24:21 -0400 Subject: [PATCH 04/13] more work on cleanup time --- .env.example | 2 +- app/models/agent.rb | 12 +++--------- lib/huginn_scheduler.rb | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 1da3a13a..69f9c284 100644 --- a/.env.example +++ b/.env.example @@ -148,7 +148,7 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false # Specify the frequency with which the scheduler checks for event expiration. # You can use `m` for minutes, `h` for hours, and `d` for days. -EVENT_EXPIRATION_CHECK="3h" +EVENT_EXPIRATION_CHECK=3h # Use Graphviz for generating diagrams instead of using Google Chart # Tools. Specify a dot(1) command path built with SVG support diff --git a/app/models/agent.rb b/app/models/agent.rb index 827bc2de..186423ff 100644 --- a/app/models/agent.rb +++ b/app/models/agent.rb @@ -21,7 +21,7 @@ class Agent < ActiveRecord::Base SCHEDULES = %w[every_1m every_2m every_5m every_10m every_30m every_1h every_2h every_5h every_12h every_1d every_2d every_7d midnight 1am 2am 3am 4am 5am 6am 7am 8am 9am 10am 11am noon 1pm 2pm 3pm 4pm 5pm 6pm 7pm 8pm 9pm 10pm 11pm never] - EVENT_RETENTION_SCHEDULES = [["Forever", 0], ["1 day", 1], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n] })] + EVENT_RETENTION_SCHEDULES = [["Forever", 0], ['1 hour', 1.hour], ['6 hours', 6.hours], ["1 day", 1.day], *([2, 3, 4, 5, 7, 14, 21, 30, 45, 90, 180, 365].map {|n| ["#{n} days", n.days] })] attr_accessible :options, :memory, :name, :type, :schedule, :controller_ids, :control_target_ids, :disabled, :source_ids, :scenario_ids, :keep_events_for, :propagate_immediately, :drop_pending_events @@ -129,20 +129,14 @@ class Agent < ActiveRecord::Base end def new_event_expiration_date - if keep_events_for == 60 - 90.seconds.from_now - else - keep_events_for > 0 ? keep_events_for.days.from_now : nil - end + keep_events_for > 0 ? keep_events_for.seconds.from_now : nil end def update_event_expirations! if keep_events_for == 0 events.update_all :expires_at => nil - elsif keep_events_for == 60 - events.update_all "expires_at = " + rdbms_date_add("created_at", "SECOND", 90) else - events.update_all "expires_at = " + rdbms_date_add("created_at", "DAY", keep_events_for.to_i) + events.update_all "expires_at = " + rdbms_date_add("created_at", "SECOND", keep_events_for.to_i) end end diff --git a/lib/huginn_scheduler.rb b/lib/huginn_scheduler.rb index dc6861e6..9170d642 100644 --- a/lib/huginn_scheduler.rb +++ b/lib/huginn_scheduler.rb @@ -114,7 +114,7 @@ class HuginnScheduler end # Schedule event cleanup. - @rufus_scheduler.every ENV['EVENT_EXPIRATION_CHECK'] do + @rufus_scheduler.every ENV['EVENT_EXPIRATION_CHECK'].presence || '6h' do cleanup_expired_events! end From 38a5dbfd879d7ab9b9aaa204d96bb6f05e5c8339 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 11 May 2015 08:24:59 -0400 Subject: [PATCH 05/13] db migration for new cleanup time --- ...update_keep_events_for_to_be_in_seconds.rb | 13 ++ db/schema.rb | 154 +++++++++--------- 2 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb diff --git a/db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb b/db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb new file mode 100644 index 00000000..32018889 --- /dev/null +++ b/db/migrate/20150507153436_update_keep_events_for_to_be_in_seconds.rb @@ -0,0 +1,13 @@ +class UpdateKeepEventsForToBeInSeconds < ActiveRecord::Migration + class Agent < ActiveRecord::Base; end + + SECONDS_IN_DAY = 60 * 60 * 24 + + def up + Agent.update_all ['keep_events_for = keep_events_for * ?', SECONDS_IN_DAY] + end + + def down + Agent.update_all ['keep_events_for = keep_events_for / ?', SECONDS_IN_DAY] + end +end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index b95a492c..e52ae7f2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,39 +11,39 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20140906030139) do +ActiveRecord::Schema.define(version: 20150507153436) do - create_table "agent_logs", force: true do |t| - t.integer "agent_id", null: false - t.text "message", null: false, charset: "utf8mb4", collation: "utf8mb4_bin" - t.integer "level", default: 3, null: false - t.integer "inbound_event_id" - t.integer "outbound_event_id" + create_table "agent_logs", force: :cascade do |t| + t.integer "agent_id", limit: 4, null: false + t.text "message", limit: 65535, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" + t.integer "level", limit: 4, default: 3, null: false + t.integer "inbound_event_id", limit: 4 + t.integer "outbound_event_id", limit: 4 t.datetime "created_at" t.datetime "updated_at" end - create_table "agents", force: true do |t| - t.integer "user_id" - t.text "options", charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "type", collation: "utf8_bin" - t.string "name", charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "schedule", collation: "utf8_bin" - t.integer "events_count", default: 0, null: false + create_table "agents", force: :cascade do |t| + t.integer "user_id", limit: 4 + t.text "options", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "type", limit: 191, collation: "utf8_bin" + t.string "name", limit: 191, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "schedule", limit: 191, collation: "utf8_bin" + t.integer "events_count", limit: 4, default: 0, null: false t.datetime "last_check_at" t.datetime "last_receive_at" - t.integer "last_checked_event_id" + t.integer "last_checked_event_id", limit: 4 t.datetime "created_at" t.datetime "updated_at" - t.text "memory", limit: 2147483647, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "memory", limit: 4294967295, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "last_web_request_at" - t.integer "keep_events_for", default: 0, null: false + t.integer "keep_events_for", limit: 4, default: 0, null: false t.datetime "last_event_at" t.datetime "last_error_log_at" - t.boolean "propagate_immediately", default: false, null: false - t.boolean "disabled", default: false, null: false - t.integer "service_id" - t.string "guid", null: false + t.boolean "propagate_immediately", limit: 1, default: false, null: false + t.boolean "disabled", limit: 1, default: false, null: false + t.integer "service_id", limit: 4 + t.string "guid", limit: 191, null: false end add_index "agents", ["guid"], name: "index_agents_on_guid", using: :btree @@ -51,9 +51,9 @@ ActiveRecord::Schema.define(version: 20140906030139) do add_index "agents", ["type"], name: "index_agents_on_type", using: :btree add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree - create_table "control_links", force: true do |t| - t.integer "controller_id", null: false - t.integer "control_target_id", null: false + create_table "control_links", force: :cascade do |t| + t.integer "controller_id", limit: 4, null: false + t.integer "control_target_id", limit: 4, null: false t.datetime "created_at" t.datetime "updated_at" end @@ -61,25 +61,25 @@ ActiveRecord::Schema.define(version: 20140906030139) do add_index "control_links", ["control_target_id"], name: "index_control_links_on_control_target_id", using: :btree add_index "control_links", ["controller_id", "control_target_id"], name: "index_control_links_on_controller_id_and_control_target_id", unique: true, using: :btree - create_table "delayed_jobs", force: true do |t| - t.integer "priority", default: 0 - t.integer "attempts", default: 0 + create_table "delayed_jobs", force: :cascade do |t| + t.integer "priority", limit: 4, default: 0 + t.integer "attempts", limit: 4, default: 0 t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" - t.text "last_error", charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "last_error", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "run_at" t.datetime "locked_at" t.datetime "failed_at" - t.string "locked_by" - t.string "queue" + t.string "locked_by", limit: 191 + t.string "queue", limit: 191 t.datetime "created_at" t.datetime "updated_at" end add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree - create_table "events", force: true do |t| - t.integer "user_id" - t.integer "agent_id" + create_table "events", force: :cascade do |t| + t.integer "user_id", limit: 4 + t.integer "agent_id", limit: 4 t.decimal "lat", precision: 15, scale: 10 t.decimal "lng", precision: 15, scale: 10 t.text "payload", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" @@ -92,20 +92,20 @@ ActiveRecord::Schema.define(version: 20140906030139) do add_index "events", ["expires_at"], name: "index_events_on_expires_at", using: :btree add_index "events", ["user_id", "created_at"], name: "index_events_on_user_id_and_created_at", using: :btree - create_table "links", force: true do |t| - t.integer "source_id" - t.integer "receiver_id" + create_table "links", force: :cascade do |t| + t.integer "source_id", limit: 4 + t.integer "receiver_id", limit: 4 t.datetime "created_at" t.datetime "updated_at" - t.integer "event_id_at_creation", default: 0, null: false + t.integer "event_id_at_creation", limit: 4, default: 0, null: false end add_index "links", ["receiver_id", "source_id"], name: "index_links_on_receiver_id_and_source_id", using: :btree add_index "links", ["source_id", "receiver_id"], name: "index_links_on_source_id_and_receiver_id", using: :btree - create_table "scenario_memberships", force: true do |t| - t.integer "agent_id", null: false - t.integer "scenario_id", null: false + create_table "scenario_memberships", force: :cascade do |t| + t.integer "agent_id", limit: 4, null: false + t.integer "scenario_id", limit: 4, null: false t.datetime "created_at" t.datetime "updated_at" end @@ -113,71 +113,71 @@ ActiveRecord::Schema.define(version: 20140906030139) do add_index "scenario_memberships", ["agent_id"], name: "index_scenario_memberships_on_agent_id", using: :btree add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree - create_table "scenarios", force: true do |t| - t.string "name", null: false, charset: "utf8mb4", collation: "utf8mb4_bin" - t.integer "user_id", null: false + create_table "scenarios", force: :cascade do |t| + t.string "name", limit: 191, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" + t.integer "user_id", limit: 4, null: false t.datetime "created_at" t.datetime "updated_at" - t.text "description", charset: "utf8mb4", collation: "utf8mb4_bin" - t.boolean "public", default: false, null: false - t.string "guid", null: false, charset: "ascii", collation: "ascii_bin" - t.string "source_url" - t.string "tag_bg_color" - t.string "tag_fg_color" + t.text "description", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" + t.boolean "public", limit: 1, default: false, null: false + t.string "guid", limit: 191, null: false, charset: "ascii", collation: "ascii_bin" + t.string "source_url", limit: 191 + t.string "tag_bg_color", limit: 191 + t.string "tag_fg_color", limit: 191 end add_index "scenarios", ["user_id", "guid"], name: "index_scenarios_on_user_id_and_guid", unique: true, using: :btree - create_table "services", force: true do |t| - t.integer "user_id", null: false - t.string "provider", null: false - t.string "name", null: false - t.text "token", null: false - t.text "secret" - t.text "refresh_token" + create_table "services", force: :cascade do |t| + t.integer "user_id", limit: 4, null: false + t.string "provider", limit: 191, null: false + t.string "name", limit: 191, null: false + t.text "token", limit: 65535, null: false + t.text "secret", limit: 65535 + t.text "refresh_token", limit: 65535 t.datetime "expires_at" - t.boolean "global", default: false - t.text "options" + t.boolean "global", limit: 1, default: false + t.text "options", limit: 65535 t.datetime "created_at" t.datetime "updated_at" - t.string "uid" + t.string "uid", limit: 191 end add_index "services", ["provider"], name: "index_services_on_provider", using: :btree add_index "services", ["uid"], name: "index_services_on_uid", using: :btree add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree - create_table "user_credentials", force: true do |t| - t.integer "user_id", null: false - t.string "credential_name", null: false - t.text "credential_value", null: false + create_table "user_credentials", force: :cascade do |t| + t.integer "user_id", limit: 4, null: false + t.string "credential_name", limit: 191, null: false + t.text "credential_value", limit: 65535, null: false t.datetime "created_at" t.datetime "updated_at" - t.string "mode", default: "text", null: false, collation: "utf8_bin" + t.string "mode", limit: 191, default: "text", null: false, collation: "utf8_bin" end add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree - create_table "users", force: true do |t| - t.string "email", default: "", null: false, collation: "utf8_bin" - t.string "encrypted_password", default: "", null: false, charset: "ascii", collation: "ascii_bin" - t.string "reset_password_token", collation: "utf8_bin" + create_table "users", force: :cascade do |t| + t.string "email", limit: 191, default: "", null: false, collation: "utf8_bin" + t.string "encrypted_password", limit: 191, default: "", null: false, charset: "ascii", collation: "ascii_bin" + t.string "reset_password_token", limit: 191, collation: "utf8_bin" t.datetime "reset_password_sent_at" t.datetime "remember_created_at" - t.integer "sign_in_count", default: 0 + t.integer "sign_in_count", limit: 4, default: 0 t.datetime "current_sign_in_at" t.datetime "last_sign_in_at" - t.string "current_sign_in_ip" - t.string "last_sign_in_ip" + t.string "current_sign_in_ip", limit: 191 + t.string "last_sign_in_ip", limit: 191 t.datetime "created_at" t.datetime "updated_at" - t.boolean "admin", default: false, null: false - t.integer "failed_attempts", default: 0 - t.string "unlock_token" + t.boolean "admin", limit: 1, default: false, null: false + t.integer "failed_attempts", limit: 4, default: 0 + t.string "unlock_token", limit: 191 t.datetime "locked_at" - t.string "username", limit: 191, null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci" - t.string "invitation_code", null: false, collation: "utf8_bin" - t.integer "scenario_count", default: 0, null: false + t.string "username", limit: 191, charset: "utf8mb4", collation: "utf8mb4_unicode_ci" + t.string "invitation_code", limit: 191, null: false + t.integer "scenario_count", limit: 4, default: 0, null: false end add_index "users", ["email"], name: "index_users_on_email", unique: true, using: :btree From 5e1730351f50843fedfbf1aa73b0cff083d90053 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Sun, 24 May 2015 09:21:27 -0400 Subject: [PATCH 06/13] update some spec scripts to new cleanup time --- spec/models/agents/ftpsite_agent_spec.rb | 2 +- spec/models/agents/imap_folder_agent_spec.rb | 2 +- spec/models/agents/website_agent_spec.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/models/agents/ftpsite_agent_spec.rb b/spec/models/agents/ftpsite_agent_spec.rb index 3752f430..bb7495a0 100644 --- a/spec/models/agents/ftpsite_agent_spec.rb +++ b/spec/models/agents/ftpsite_agent_spec.rb @@ -9,7 +9,7 @@ describe Agents::FtpsiteAgent do 'url' => "ftp://ftp.example.org/pub/releases/", 'patterns' => ["example*.tar.gz"], } - @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2) + @checker = Agents::FtpsiteAgent.new(:name => "Example", :options => @site, :keep_events_for => 2.days) @checker.user = users(:bob) @checker.save! end diff --git a/spec/models/agents/imap_folder_agent_spec.rb b/spec/models/agents/imap_folder_agent_spec.rb index cd9210a3..f9a2c1c5 100644 --- a/spec/models/agents/imap_folder_agent_spec.rb +++ b/spec/models/agents/imap_folder_agent_spec.rb @@ -14,7 +14,7 @@ describe Agents::ImapFolderAgent do 'conditions' => { } } - @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2) + @checker = Agents::ImapFolderAgent.new(:name => 'Example', :options => @site, :keep_events_for => 2.days) @checker.user = users(:bob) @checker.save! diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 7f45cb90..9ce41f54 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -20,7 +20,7 @@ describe Agents::WebsiteAgent do 'hovertext' => { 'css' => "#comic img", 'value' => "@title" } } } - @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2) + @checker = Agents::WebsiteAgent.new(:name => "xkcd", :options => @valid_options, :keep_events_for => 2.days) @checker.user = users(:bob) @checker.save! end From b31a0ec3e7b9780a55e6bc895731e19a8588d20d Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 25 May 2015 11:34:33 -0700 Subject: [PATCH 07/13] fix specs and add support for translation in Scenario exports --- app/models/scenario_import.rb | 25 ++++- db/schema.rb | 142 ++++++++++++++++++---------- lib/agents_exporter.rb | 1 + spec/fixtures/agents.yml | 4 +- spec/lib/agents_exporter_spec.rb | 1 + spec/models/agent_spec.rb | 22 ++--- spec/models/scenario_import_spec.rb | 63 ++++++++++-- 7 files changed, 182 insertions(+), 76 deletions(-) diff --git a/app/models/scenario_import.rb b/app/models/scenario_import.rb index 75dd672e..441f755e 100644 --- a/app/models/scenario_import.rb +++ b/app/models/scenario_import.rb @@ -142,7 +142,7 @@ class ScenarioImport def generate_diff @agent_diffs = (parsed_data['agents'] || []).map.with_index do |agent_data, index| # AgentDiff is defined at the end of this file. - agent_diff = AgentDiff.new(agent_data) + agent_diff = AgentDiff.new(agent_data, parsed_data['schema_version']) if existing_scenario # If this Agent exists already, update the AgentDiff with the local version's information. agent_diff.diff_with! existing_scenario.agents.find_by(:guid => agent_data['guid']) @@ -184,14 +184,16 @@ class ScenarioImport end end - def initialize(agent_data) + def initialize(agent_data, schema_version) super() + @schema_version = schema_version @requires_merge = false self.agent = nil store! agent_data end BASE_FIELDS = %w[name schedule keep_events_for propagate_immediately disabled guid] + FIELDS_REQUIRING_TRANSLATION = %w[keep_events_for] def agent_exists? !!agent @@ -209,10 +211,27 @@ class ScenarioImport self.type = FieldDiff.new(agent_data["type"].split("::").pop) self.options = FieldDiff.new(agent_data['options'] || {}) BASE_FIELDS.each do |option| - self[option] = FieldDiff.new(agent_data[option]) if agent_data.has_key?(option) + if agent_data.has_key?(option) + value = agent_data[option] + value = send(:"translate_#{option}", value) if option.in?(FIELDS_REQUIRING_TRANSLATION) + self[option] = FieldDiff.new(value) + end end end + def translate_keep_events_for(old_value) + if schema_version < 1 + # Was stored in days, now is stored in seconds. + old_value.to_i.days + else + old_value + end + end + + def schema_version + (@schema_version || 0).to_i + end + def diff_with!(agent) return unless agent.present? diff --git a/db/schema.rb b/db/schema.rb index e52ae7f2..c4597c3f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -14,36 +14,36 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "agent_logs", force: :cascade do |t| - t.integer "agent_id", limit: 4, null: false - t.text "message", limit: 65535, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" - t.integer "level", limit: 4, default: 3, null: false + t.integer "agent_id", limit: 4, null: false + t.text "message", limit: 16777215, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" + t.integer "level", limit: 4, default: 3, null: false t.integer "inbound_event_id", limit: 4 t.integer "outbound_event_id", limit: 4 - t.datetime "created_at" - t.datetime "updated_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false end create_table "agents", force: :cascade do |t| t.integer "user_id", limit: 4 - t.text "options", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "type", limit: 191, collation: "utf8_bin" - t.string "name", limit: 191, charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "schedule", limit: 191, collation: "utf8_bin" + t.text "options", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "type", limit: 255, collation: "utf8_bin" + t.string "name", limit: 255, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "schedule", limit: 255, collation: "utf8_bin" t.integer "events_count", limit: 4, default: 0, null: false t.datetime "last_check_at" t.datetime "last_receive_at" t.integer "last_checked_event_id", limit: 4 - t.datetime "created_at" - t.datetime "updated_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.text "memory", limit: 4294967295, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "last_web_request_at" - t.integer "keep_events_for", limit: 4, default: 0, null: false t.datetime "last_event_at" t.datetime "last_error_log_at" + t.integer "keep_events_for", limit: 4, default: 0, null: false t.boolean "propagate_immediately", limit: 1, default: false, null: false t.boolean "disabled", limit: 1, default: false, null: false + t.string "guid", limit: 255, null: false, charset: "ascii", collation: "ascii_bin" t.integer "service_id", limit: 4 - t.string "guid", limit: 191, null: false end add_index "agents", ["guid"], name: "index_agents_on_guid", using: :btree @@ -51,6 +51,14 @@ ActiveRecord::Schema.define(version: 20150507153436) do add_index "agents", ["type"], name: "index_agents_on_type", using: :btree add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree + create_table "contacts", force: :cascade do |t| + t.text "message", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "name", limit: 255, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "email", limit: 255, collation: "utf8_bin" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "control_links", force: :cascade do |t| t.integer "controller_id", limit: 4, null: false t.integer "control_target_id", limit: 4, null: false @@ -64,15 +72,15 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "delayed_jobs", force: :cascade do |t| t.integer "priority", limit: 4, default: 0 t.integer "attempts", limit: 4, default: 0 - t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" - t.text "last_error", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "last_error", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "run_at" t.datetime "locked_at" t.datetime "failed_at" - t.string "locked_by", limit: 191 - t.string "queue", limit: 191 - t.datetime "created_at" - t.datetime "updated_at" + t.string "locked_by", limit: 255 + t.string "queue", limit: 255 + t.datetime "created_at", null: false + t.datetime "updated_at", null: false end add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree @@ -80,11 +88,11 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "events", force: :cascade do |t| t.integer "user_id", limit: 4 t.integer "agent_id", limit: 4 - t.decimal "lat", precision: 15, scale: 10 - t.decimal "lng", precision: 15, scale: 10 - t.text "payload", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" - t.datetime "created_at" - t.datetime "updated_at" + t.decimal "lat", precision: 15, scale: 10 + t.decimal "lng", precision: 15, scale: 10 + t.text "payload", limit: 4294967295, charset: "utf8mb4", collation: "utf8mb4_bin" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.datetime "expires_at" end @@ -95,14 +103,27 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "links", force: :cascade do |t| t.integer "source_id", limit: 4 t.integer "receiver_id", limit: 4 - t.datetime "created_at" - t.datetime "updated_at" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.integer "event_id_at_creation", limit: 4, default: 0, null: false end add_index "links", ["receiver_id", "source_id"], name: "index_links_on_receiver_id_and_source_id", using: :btree add_index "links", ["source_id", "receiver_id"], name: "index_links_on_source_id_and_receiver_id", using: :btree + create_table "rails_admin_histories", force: :cascade do |t| + t.text "message", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" + t.string "username", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" + t.integer "item", limit: 4 + t.string "table", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" + t.integer "month", limit: 2 + t.integer "year", limit: 8 + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + + add_index "rails_admin_histories", ["item", "table", "month", "year"], name: "index_rails_admin_histories", using: :btree + create_table "scenario_memberships", force: :cascade do |t| t.integer "agent_id", limit: 4, null: false t.integer "scenario_id", limit: 4, null: false @@ -114,69 +135,86 @@ ActiveRecord::Schema.define(version: 20150507153436) do add_index "scenario_memberships", ["scenario_id"], name: "index_scenario_memberships_on_scenario_id", using: :btree create_table "scenarios", force: :cascade do |t| - t.string "name", limit: 191, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" + t.string "name", limit: 255, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" t.integer "user_id", limit: 4, null: false t.datetime "created_at" t.datetime "updated_at" t.text "description", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" t.boolean "public", limit: 1, default: false, null: false - t.string "guid", limit: 191, null: false, charset: "ascii", collation: "ascii_bin" - t.string "source_url", limit: 191 - t.string "tag_bg_color", limit: 191 - t.string "tag_fg_color", limit: 191 + t.string "guid", limit: 255, null: false, charset: "ascii", collation: "ascii_bin" + t.string "source_url", limit: 255 + t.string "tag_bg_color", limit: 255 + t.string "tag_fg_color", limit: 255 end add_index "scenarios", ["user_id", "guid"], name: "index_scenarios_on_user_id_and_guid", unique: true, using: :btree create_table "services", force: :cascade do |t| t.integer "user_id", limit: 4, null: false - t.string "provider", limit: 191, null: false - t.string "name", limit: 191, null: false - t.text "token", limit: 65535, null: false - t.text "secret", limit: 65535 - t.text "refresh_token", limit: 65535 + t.string "provider", limit: 255, null: false, charset: "latin1", collation: "latin1_swedish_ci" + t.string "name", limit: 255, null: false, charset: "latin1", collation: "latin1_swedish_ci" + t.text "token", limit: 65535, null: false, charset: "latin1", collation: "latin1_swedish_ci" + t.text "secret", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" + t.text "refresh_token", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" t.datetime "expires_at" t.boolean "global", limit: 1, default: false - t.text "options", limit: 65535 + t.text "options", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" t.datetime "created_at" t.datetime "updated_at" - t.string "uid", limit: 191 + t.string "uid", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" end add_index "services", ["provider"], name: "index_services_on_provider", using: :btree add_index "services", ["uid"], name: "index_services_on_uid", using: :btree add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree + create_table "taggings", force: :cascade do |t| + t.integer "tag_id", limit: 4 + t.integer "taggable_id", limit: 4 + t.string "taggable_type", limit: 255, collation: "utf8_general_ci" + t.integer "tagger_id", limit: 4 + t.string "tagger_type", limit: 255, collation: "utf8_general_ci" + t.string "context", limit: 128, collation: "utf8_general_ci" + t.datetime "created_at" + end + + add_index "taggings", ["tag_id"], name: "index_taggings_on_tag_id", using: :btree + add_index "taggings", ["taggable_id", "taggable_type", "context"], name: "index_taggings_on_taggable_id_and_taggable_type_and_context", using: :btree + + create_table "tags", force: :cascade do |t| + t.string "name", limit: 255, collation: "utf8_general_ci" + end + create_table "user_credentials", force: :cascade do |t| t.integer "user_id", limit: 4, null: false - t.string "credential_name", limit: 191, null: false + t.string "credential_name", limit: 255, null: false t.text "credential_value", limit: 65535, null: false - t.datetime "created_at" - t.datetime "updated_at" - t.string "mode", limit: 191, default: "text", null: false, collation: "utf8_bin" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.string "mode", limit: 255, default: "text", null: false, collation: "utf8_bin" end add_index "user_credentials", ["user_id", "credential_name"], name: "index_user_credentials_on_user_id_and_credential_name", unique: true, using: :btree create_table "users", force: :cascade do |t| - t.string "email", limit: 191, default: "", null: false, collation: "utf8_bin" - t.string "encrypted_password", limit: 191, default: "", null: false, charset: "ascii", collation: "ascii_bin" - t.string "reset_password_token", limit: 191, collation: "utf8_bin" + t.string "email", limit: 255, default: "", null: false, collation: "utf8_bin" + t.string "encrypted_password", limit: 255, default: "", null: false, charset: "ascii", collation: "ascii_bin" + t.string "reset_password_token", limit: 255, collation: "utf8_bin" t.datetime "reset_password_sent_at" t.datetime "remember_created_at" t.integer "sign_in_count", limit: 4, default: 0 t.datetime "current_sign_in_at" t.datetime "last_sign_in_at" - t.string "current_sign_in_ip", limit: 191 - t.string "last_sign_in_ip", limit: 191 - t.datetime "created_at" - t.datetime "updated_at" + t.string "current_sign_in_ip", limit: 255 + t.string "last_sign_in_ip", limit: 255 + t.datetime "created_at", null: false + t.datetime "updated_at", null: false t.boolean "admin", limit: 1, default: false, null: false t.integer "failed_attempts", limit: 4, default: 0 - t.string "unlock_token", limit: 191 + t.string "unlock_token", limit: 255 t.datetime "locked_at" - t.string "username", limit: 191, charset: "utf8mb4", collation: "utf8mb4_unicode_ci" - t.string "invitation_code", limit: 191, null: false + t.string "username", limit: 191, null: false, charset: "utf8mb4", collation: "utf8mb4_unicode_ci" + t.string "invitation_code", limit: 255, null: false, collation: "utf8_bin" t.integer "scenario_count", limit: 4, default: 0, null: false end diff --git a/lib/agents_exporter.rb b/lib/agents_exporter.rb index 6c8ef458..08931ef2 100644 --- a/lib/agents_exporter.rb +++ b/lib/agents_exporter.rb @@ -12,6 +12,7 @@ class AgentsExporter def as_json(opts = {}) { + :schema_version => 1, :name => options[:name].presence || 'No name provided', :description => options[:description].presence || 'No description provided', :source_url => options[:source_url], diff --git a/spec/fixtures/agents.yml b/spec/fixtures/agents.yml index 9631a4ba..8febfe65 100644 --- a/spec/fixtures/agents.yml +++ b/spec/fixtures/agents.yml @@ -38,7 +38,7 @@ bob_weather_agent: schedule: "midnight" name: "SF Weather" guid: <%= SecureRandom.hex %> - keep_events_for: 45 + keep_events_for: <%= 45.days %> options: <%= { :location => 94102, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %> jane_weather_agent: @@ -47,7 +47,7 @@ jane_weather_agent: schedule: "midnight" name: "SF Weather" guid: <%= SecureRandom.hex %> - keep_events_for: 30 + keep_events_for: <%= 30.days %> options: <%= { :location => 94103, :lat => 37.779329, :lng => -122.41915, :api_key => 'test' }.to_json.inspect %> jane_rain_notifier_agent: diff --git a/spec/lib/agents_exporter_spec.rb b/spec/lib/agents_exporter_spec.rb index c79630fd..1d5b20dd 100644 --- a/spec/lib/agents_exporter_spec.rb +++ b/spec/lib/agents_exporter_spec.rb @@ -21,6 +21,7 @@ describe AgentsExporter do expect(data[:description]).to eq(description) expect(data[:source_url]).to eq(source_url) expect(data[:guid]).to eq(guid) + expect(data[:schema_version]).to eq(1) expect(data[:tag_fg_color]).to eq(tag_fg_color) expect(data[:tag_bg_color]).to eq(tag_bg_color) expect(Time.parse(data[:exported_at])).to be_within(2).of(Time.now.utc) diff --git a/spec/models/agent_spec.rb b/spec/models/agent_spec.rb index b9c3e940..de2212b1 100644 --- a/spec/models/agent_spec.rb +++ b/spec/models/agent_spec.rb @@ -546,11 +546,11 @@ describe Agent do expect(agent).to have(1).errors_on(:keep_events_for) agent.keep_events_for = "" expect(agent).to have(1).errors_on(:keep_events_for) - agent.keep_events_for = 5 + agent.keep_events_for = 5.days.to_i expect(agent).to be_valid agent.keep_events_for = 0 expect(agent).to be_valid - agent.keep_events_for = 365 + agent.keep_events_for = 365.days.to_i expect(agent).to be_valid # Rails seems to call to_i on the input. This guards against future changes to that behavior. @@ -564,7 +564,7 @@ describe Agent do @time = "2014-01-01 01:00:00 +00:00" time_travel_to @time do @agent = Agents::SomethingSource.new(:name => "something") - @agent.keep_events_for = 5 + @agent.keep_events_for = 5.days @agent.user = users(:bob) @agent.save! @event = @agent.create_event :payload => { "hello" => "world" } @@ -580,7 +580,7 @@ describe Agent do @agent.save! @agent.options[:foo] = "bar1" - @agent.keep_events_for = 5 + @agent.keep_events_for = 5.days @agent.save! end end @@ -590,7 +590,7 @@ describe Agent do time_travel_to @time do expect { @agent.options[:foo] = "bar1" - @agent.keep_events_for = 3 + @agent.keep_events_for = 3.days @agent.save! }.to change { @event.reload.expires_at } expect(@event.expires_at.to_i).to be_within(2).of(3.days.from_now.to_i) @@ -603,7 +603,7 @@ describe Agent do expect { @agent.options[:foo] = "bar2" - @agent.keep_events_for = 3 + @agent.keep_events_for = 3.days @agent.save! }.to change { @event.reload.expires_at } expect(@event.expires_at.to_i).to be_within(60 * 61).of(1.days.from_now.to_i) # The larger time is to deal with daylight savings @@ -635,7 +635,7 @@ describe Agent do @receiver = Agents::CannotBeScheduled.new( name: 'Agent', options: { foo: 'bar3' }, - keep_events_for: 3, + keep_events_for: 3.days, propagate_immediately: true) @receiver.user = users(:bob) @receiver.sources << @sender @@ -747,7 +747,7 @@ describe Agent do it "sets expires_at on created events" do event = agents(:jane_weather_agent).create_event :payload => { 'hi' => 'there' } - expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.days.from_now.to_i) + expect(event.expires_at.to_i).to be_within(5).of(agents(:jane_weather_agent).keep_events_for.seconds.from_now.to_i) end end @@ -836,7 +836,7 @@ describe AgentDrop do }, }, schedule: 'every_1h', - keep_events_for: 2) + keep_events_for: 2.days) @wsa1.user = users(:bob) @wsa1.save! @@ -853,7 +853,7 @@ describe AgentDrop do }, }, schedule: 'every_12h', - keep_events_for: 2) + keep_events_for: 2.days) @wsa2.user = users(:bob) @wsa2.save! @@ -868,7 +868,7 @@ describe AgentDrop do matchers: [], skip_created_at: 'false', }, - keep_events_for: 2, + keep_events_for: 2.days, propagate_immediately: true) @efa.user = users(:bob) @efa.sources << @wsa1 << @wsa2 diff --git a/spec/models/scenario_import_spec.rb b/spec/models/scenario_import_spec.rb index 4b92bf47..d8a13e38 100644 --- a/spec/models/scenario_import_spec.rb +++ b/spec/models/scenario_import_spec.rb @@ -30,7 +30,7 @@ describe ScenarioImport do :type => "Agents::WeatherAgent", :name => "a weather agent", :schedule => "5pm", - :keep_events_for => 14, + :keep_events_for => 14.days, :disabled => true, :guid => "a-weather-agent", :options => weather_agent_options @@ -61,6 +61,7 @@ describe ScenarioImport do end let(:valid_parsed_data) do { + :schema_version => 1, :name => name, :description => description, :guid => guid, @@ -203,7 +204,7 @@ describe ScenarioImport do expect(weather_agent.name).to eq("a weather agent") expect(weather_agent.schedule).to eq("5pm") - expect(weather_agent.keep_events_for).to eq(14) + expect(weather_agent.keep_events_for).to eq(14.days) expect(weather_agent.propagate_immediately).to be_falsey expect(weather_agent).to be_disabled expect(weather_agent.memory).to be_empty @@ -226,6 +227,23 @@ describe ScenarioImport do scenario_import.import }.to change { users(:bob).agents.count }.by(2) end + + context "when the schema_version is less than 1" do + before do + valid_parsed_weather_agent_data[:keep_events_for] = 2 + valid_parsed_data.delete(:schema_version) + end + + it "translates keep_events_for from days to seconds" do + scenario_import.import + expect(scenario_import.errors).to be_empty + weather_agent = scenario_import.scenario.agents.find_by(:guid => "a-weather-agent") + trigger_agent = scenario_import.scenario.agents.find_by(:guid => "a-trigger-agent") + + expect(weather_agent.keep_events_for).to eq(2.days) + expect(trigger_agent.keep_events_for).to eq(0) + end + end end describe "#generate_diff" do @@ -309,7 +327,7 @@ describe ScenarioImport do expect(weather_agent.name).to eq("a weather agent") expect(weather_agent.schedule).to eq("5pm") - expect(weather_agent.keep_events_for).to eq(14) + expect(weather_agent.keep_events_for).to eq(14.days) expect(weather_agent.propagate_immediately).to be_falsey expect(weather_agent).to be_disabled expect(weather_agent.memory).to be_empty @@ -330,7 +348,7 @@ describe ScenarioImport do "0" => { "name" => "updated name", "schedule" => "6pm", - "keep_events_for" => "2", + "keep_events_for" => 2.days.to_i, "disabled" => "false", "options" => weather_agent_options.merge("api_key" => "foo").to_json } @@ -343,7 +361,7 @@ describe ScenarioImport do weather_agent = existing_scenario.agents.find_by(:guid => "a-weather-agent") expect(weather_agent.name).to eq("updated name") expect(weather_agent.schedule).to eq("6pm") - expect(weather_agent.keep_events_for).to eq(2) + expect(weather_agent.keep_events_for).to eq(2.days) expect(weather_agent).not_to be_disabled expect(weather_agent.options).to eq(weather_agent_options.merge("api_key" => "foo")) end @@ -353,7 +371,7 @@ describe ScenarioImport do "0" => { "name" => "", "schedule" => "foo", - "keep_events_for" => "2", + "keep_events_for" => 2.days.to_i.to_s, "options" => weather_agent_options.merge("api_key" => "").to_json } } @@ -386,12 +404,40 @@ describe ScenarioImport do end end + context "when the schema_version is less than 1" do + it "translates keep_events_for from days to seconds" do + valid_parsed_data.delete(:schema_version) + valid_parsed_data[:agents] = [valid_parsed_weather_agent_data.merge(keep_events_for: 5)] + + scenario_import.merges = { + "0" => { + "name" => "a new name", + "schedule" => "6pm", + "keep_events_for" => 2.days.to_i.to_s, + "disabled" => "true", + "options" => weather_agent_options.merge("api_key" => "foo").to_json + } + } + + expect(scenario_import).to be_valid + + weather_agent_diff = scenario_import.agent_diffs[0] + + expect(weather_agent_diff.name.current).to eq(agents(:bob_weather_agent).name) + expect(weather_agent_diff.name.incoming).to eq('a weather agent') + expect(weather_agent_diff.name.updated).to eq('a new name') + expect(weather_agent_diff.keep_events_for.current).to eq(45.days.to_i) + expect(weather_agent_diff.keep_events_for.incoming).to eq(5.days.to_i) + expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_i.to_s) + end + end + it "sets the 'updated' FieldDiff values based on any feedback from the user" do scenario_import.merges = { "0" => { "name" => "a new name", "schedule" => "6pm", - "keep_events_for" => "2", + "keep_events_for" => 2.days.to_s, "disabled" => "true", "options" => weather_agent_options.merge("api_key" => "foo").to_json }, @@ -411,7 +457,8 @@ describe ScenarioImport do expect(weather_agent_diff.name.updated).to eq("a new name") expect(weather_agent_diff.schedule.updated).to eq("6pm") - expect(weather_agent_diff.keep_events_for.updated).to eq("2") + expect(weather_agent_diff.keep_events_for.current).to eq(45.days) + expect(weather_agent_diff.keep_events_for.updated).to eq(2.days.to_s) expect(weather_agent_diff.disabled.updated).to eq("true") expect(weather_agent_diff.options.updated).to eq(weather_agent_options.merge("api_key" => "foo")) end From 8d1a05fb7c0013d2f232d36d3fa537c7a3bcc403 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Thu, 16 Jul 2015 18:31:17 -0400 Subject: [PATCH 08/13] fix specs --- Gemfile.lock | 3 --- spec/models/agents/website_agent_spec.rb | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index af192e25..f49fea57 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -580,6 +580,3 @@ DEPENDENCIES weibo_2! wunderground (~> 1.2.0) xmpp4r (~> 0.5.6) - -BUNDLED WITH - 1.10.5 diff --git a/spec/models/agents/website_agent_spec.rb b/spec/models/agents/website_agent_spec.rb index 4edcbf45..a5af6798 100644 --- a/spec/models/agents/website_agent_spec.rb +++ b/spec/models/agents/website_agent_spec.rb @@ -386,7 +386,7 @@ describe Agents::WebsiteAgent do 'url' => { 'xpath' => '/feed/entry', 'value' => './link[1]/@href' }, 'thumbnail' => { 'xpath' => '/feed/entry', 'value' => './thumbnail/@url' }, } - }, keep_events_for: 2) + }, keep_events_for: 2.days) @checker.user = users(:bob) @checker.save! end From 82948977dad4ba73041f4fd4b929a2a4302f613c Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Fri, 17 Jul 2015 13:09:41 -0400 Subject: [PATCH 09/13] default is every 6h --- .env.example | 4 ++-- spec/models/scenario_import_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 7acb035f..8e404f6e 100644 --- a/.env.example +++ b/.env.example @@ -151,9 +151,9 @@ ENABLE_SECOND_PRECISION_SCHEDULE=false # at the expense of time accuracy. SCHEDULER_FREQUENCY=0.3 -# Specify the frequency with which the scheduler checks for event expiration. +# Specify the frequency with which the scheduler checks for and cleans up expired events. # You can use `m` for minutes, `h` for hours, and `d` for days. -EVENT_EXPIRATION_CHECK=3h +EVENT_EXPIRATION_CHECK=6h # Use Graphviz for generating diagrams instead of using Google Chart # Tools. Specify a dot(1) command path built with SVG support diff --git a/spec/models/scenario_import_spec.rb b/spec/models/scenario_import_spec.rb index d8a13e38..8c7ef395 100644 --- a/spec/models/scenario_import_spec.rb +++ b/spec/models/scenario_import_spec.rb @@ -348,7 +348,7 @@ describe ScenarioImport do "0" => { "name" => "updated name", "schedule" => "6pm", - "keep_events_for" => 2.days.to_i, + "keep_events_for" => 2.days.to_i.to_s, "disabled" => "false", "options" => weather_agent_options.merge("api_key" => "foo").to_json } @@ -361,7 +361,7 @@ describe ScenarioImport do weather_agent = existing_scenario.agents.find_by(:guid => "a-weather-agent") expect(weather_agent.name).to eq("updated name") expect(weather_agent.schedule).to eq("6pm") - expect(weather_agent.keep_events_for).to eq(2.days) + expect(weather_agent.keep_events_for).to eq(2.days.to_i) expect(weather_agent).not_to be_disabled expect(weather_agent.options).to eq(weather_agent_options.merge("api_key" => "foo")) end From b3e6275e4ab3057def2758786288297ce4e6d1bd Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Fri, 17 Jul 2015 13:41:16 -0400 Subject: [PATCH 10/13] grammer --- app/views/scenario_imports/_step_two.html.erb | 2 +- db/schema.rb | 100 ++++++------------ 2 files changed, 32 insertions(+), 70 deletions(-) diff --git a/app/views/scenario_imports/_step_two.html.erb b/app/views/scenario_imports/_step_two.html.erb index f38605b2..5867f407 100644 --- a/app/views/scenario_imports/_step_two.html.erb +++ b/app/views/scenario_imports/_step_two.html.erb @@ -14,7 +14,7 @@ This Scenario already exists in your system. The import will update your existing <%= scenario_label(@scenario_import.existing_scenario) %> Scenario's title, - description and tag colors. Below you can customize how the individual agents get updated. + description, and tag colors. Below you can customize how the individual agents get updated. <% end %> diff --git a/db/schema.rb b/db/schema.rb index c4597c3f..7957ce1e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -14,18 +14,18 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "agent_logs", force: :cascade do |t| - t.integer "agent_id", limit: 4, null: false - t.text "message", limit: 16777215, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" - t.integer "level", limit: 4, default: 3, null: false + t.integer "agent_id", limit: 4, null: false + t.text "message", limit: 65535, null: false, charset: "utf8mb4", collation: "utf8mb4_bin" + t.integer "level", limit: 4, default: 3, null: false t.integer "inbound_event_id", limit: 4 t.integer "outbound_event_id", limit: 4 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end create_table "agents", force: :cascade do |t| t.integer "user_id", limit: 4 - t.text "options", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "options", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" t.string "type", limit: 255, collation: "utf8_bin" t.string "name", limit: 255, charset: "utf8mb4", collation: "utf8mb4_bin" t.string "schedule", limit: 255, collation: "utf8_bin" @@ -33,13 +33,13 @@ ActiveRecord::Schema.define(version: 20150507153436) do t.datetime "last_check_at" t.datetime "last_receive_at" t.integer "last_checked_event_id", limit: 4 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.text "memory", limit: 4294967295, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "last_web_request_at" + t.integer "keep_events_for", limit: 4, default: 0, null: false t.datetime "last_event_at" t.datetime "last_error_log_at" - t.integer "keep_events_for", limit: 4, default: 0, null: false t.boolean "propagate_immediately", limit: 1, default: false, null: false t.boolean "disabled", limit: 1, default: false, null: false t.string "guid", limit: 255, null: false, charset: "ascii", collation: "ascii_bin" @@ -51,14 +51,6 @@ ActiveRecord::Schema.define(version: 20150507153436) do add_index "agents", ["type"], name: "index_agents_on_type", using: :btree add_index "agents", ["user_id", "created_at"], name: "index_agents_on_user_id_and_created_at", using: :btree - create_table "contacts", force: :cascade do |t| - t.text "message", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "name", limit: 255, charset: "utf8mb4", collation: "utf8mb4_bin" - t.string "email", limit: 255, collation: "utf8_bin" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - end - create_table "control_links", force: :cascade do |t| t.integer "controller_id", limit: 4, null: false t.integer "control_target_id", limit: 4, null: false @@ -72,15 +64,15 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "delayed_jobs", force: :cascade do |t| t.integer "priority", limit: 4, default: 0 t.integer "attempts", limit: 4, default: 0 - t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" - t.text "last_error", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "handler", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.text "last_error", limit: 65535, charset: "utf8mb4", collation: "utf8mb4_bin" t.datetime "run_at" t.datetime "locked_at" t.datetime "failed_at" t.string "locked_by", limit: 255 t.string "queue", limit: 255 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" end add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree @@ -88,11 +80,11 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "events", force: :cascade do |t| t.integer "user_id", limit: 4 t.integer "agent_id", limit: 4 - t.decimal "lat", precision: 15, scale: 10 - t.decimal "lng", precision: 15, scale: 10 - t.text "payload", limit: 4294967295, charset: "utf8mb4", collation: "utf8mb4_bin" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.decimal "lat", precision: 15, scale: 10 + t.decimal "lng", precision: 15, scale: 10 + t.text "payload", limit: 16777215, charset: "utf8mb4", collation: "utf8mb4_bin" + t.datetime "created_at" + t.datetime "updated_at" t.datetime "expires_at" end @@ -103,27 +95,14 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "links", force: :cascade do |t| t.integer "source_id", limit: 4 t.integer "receiver_id", limit: 4 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.integer "event_id_at_creation", limit: 4, default: 0, null: false end add_index "links", ["receiver_id", "source_id"], name: "index_links_on_receiver_id_and_source_id", using: :btree add_index "links", ["source_id", "receiver_id"], name: "index_links_on_source_id_and_receiver_id", using: :btree - create_table "rails_admin_histories", force: :cascade do |t| - t.text "message", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" - t.string "username", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" - t.integer "item", limit: 4 - t.string "table", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" - t.integer "month", limit: 2 - t.integer "year", limit: 8 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - end - - add_index "rails_admin_histories", ["item", "table", "month", "year"], name: "index_rails_admin_histories", using: :btree - create_table "scenario_memberships", force: :cascade do |t| t.integer "agent_id", limit: 4, null: false t.integer "scenario_id", limit: 4, null: false @@ -151,46 +130,29 @@ ActiveRecord::Schema.define(version: 20150507153436) do create_table "services", force: :cascade do |t| t.integer "user_id", limit: 4, null: false - t.string "provider", limit: 255, null: false, charset: "latin1", collation: "latin1_swedish_ci" - t.string "name", limit: 255, null: false, charset: "latin1", collation: "latin1_swedish_ci" - t.text "token", limit: 65535, null: false, charset: "latin1", collation: "latin1_swedish_ci" - t.text "secret", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" - t.text "refresh_token", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" + t.string "provider", limit: 255, null: false, collation: "utf8_general_ci" + t.string "name", limit: 255, null: false, collation: "utf8_general_ci" + t.text "token", limit: 65535, null: false, collation: "utf8_general_ci" + t.text "secret", limit: 65535, collation: "utf8_general_ci" + t.text "refresh_token", limit: 65535, collation: "utf8_general_ci" t.datetime "expires_at" t.boolean "global", limit: 1, default: false - t.text "options", limit: 65535, charset: "latin1", collation: "latin1_swedish_ci" + t.text "options", limit: 65535, collation: "utf8_general_ci" t.datetime "created_at" t.datetime "updated_at" - t.string "uid", limit: 255, charset: "latin1", collation: "latin1_swedish_ci" + t.string "uid", limit: 255, collation: "utf8_general_ci" end add_index "services", ["provider"], name: "index_services_on_provider", using: :btree add_index "services", ["uid"], name: "index_services_on_uid", using: :btree add_index "services", ["user_id", "global"], name: "index_services_on_user_id_and_global", using: :btree - create_table "taggings", force: :cascade do |t| - t.integer "tag_id", limit: 4 - t.integer "taggable_id", limit: 4 - t.string "taggable_type", limit: 255, collation: "utf8_general_ci" - t.integer "tagger_id", limit: 4 - t.string "tagger_type", limit: 255, collation: "utf8_general_ci" - t.string "context", limit: 128, collation: "utf8_general_ci" - t.datetime "created_at" - end - - add_index "taggings", ["tag_id"], name: "index_taggings_on_tag_id", using: :btree - add_index "taggings", ["taggable_id", "taggable_type", "context"], name: "index_taggings_on_taggable_id_and_taggable_type_and_context", using: :btree - - create_table "tags", force: :cascade do |t| - t.string "name", limit: 255, collation: "utf8_general_ci" - end - create_table "user_credentials", force: :cascade do |t| t.integer "user_id", limit: 4, null: false t.string "credential_name", limit: 255, null: false t.text "credential_value", limit: 65535, null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.string "mode", limit: 255, default: "text", null: false, collation: "utf8_bin" end @@ -207,8 +169,8 @@ ActiveRecord::Schema.define(version: 20150507153436) do t.datetime "last_sign_in_at" t.string "current_sign_in_ip", limit: 255 t.string "last_sign_in_ip", limit: 255 - t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.datetime "created_at" + t.datetime "updated_at" t.boolean "admin", limit: 1, default: false, null: false t.integer "failed_attempts", limit: 4, default: 0 t.string "unlock_token", limit: 255 From b074d390a532d47b24d2b692d111a8fe716aee42 Mon Sep 17 00:00:00 2001 From: Dominik Sander Date: Sun, 19 Jul 2015 12:31:53 +0200 Subject: [PATCH 11/13] Remove keep-awake ping on heroku Heroku's new Free tier requires applications to sleep for at least six hours a day, thus we can not keep huginn awake. Every paid tier does not but the application to sleep at all. --- deployment/heroku/unicorn.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deployment/heroku/unicorn.rb b/deployment/heroku/unicorn.rb index cc411fb2..b34cc809 100644 --- a/deployment/heroku/unicorn.rb +++ b/deployment/heroku/unicorn.rb @@ -17,11 +17,6 @@ Thread.new do sleep 45 - if ENV['DOMAIN'] - force_ssl = ENV['FORCE_SSL'] == 'true' - Net::HTTP.get_response(URI((force_ssl ? "https://" : "http://") + ENV['DOMAIN'])) - end - begin Process.getpgid worker_pid rescue Errno::ESRCH From fc8d347031c4be32f658e1e9051f0059b8e1bac3 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 20 Jul 2015 20:37:45 -0700 Subject: [PATCH 12/13] Update Gemfile.lock --- Gemfile.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Gemfile.lock b/Gemfile.lock index f49fea57..654febae 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -580,3 +580,6 @@ DEPENDENCIES weibo_2! wunderground (~> 1.2.0) xmpp4r (~> 0.5.6) + +BUNDLED WITH + 1.10.5 From debb769695bf20bdbdaf61d056be872b602d04d9 Mon Sep 17 00:00:00 2001 From: Andrew Cantino Date: Mon, 20 Jul 2015 20:37:59 -0700 Subject: [PATCH 13/13] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 654febae..af192e25 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -582,4 +582,4 @@ DEPENDENCIES xmpp4r (~> 0.5.6) BUNDLED WITH - 1.10.5 + 1.10.5