From d001d10334b35879b31e51e2a00e927130118cc1 Mon Sep 17 00:00:00 2001 From: Albert Sun Date: Fri, 5 Apr 2013 19:26:16 -0400 Subject: [PATCH] added a twitter user agent to fetch tweets from a specificied user --- app/models/agents/twitter_user_agent.rb | 177 ++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 app/models/agents/twitter_user_agent.rb diff --git a/app/models/agents/twitter_user_agent.rb b/app/models/agents/twitter_user_agent.rb new file mode 100644 index 00000000..8743e992 --- /dev/null +++ b/app/models/agents/twitter_user_agent.rb @@ -0,0 +1,177 @@ +require "twitter" + +module Agents + class TwitterUserAgent < Agent + cannot_receive_events! + + description <<-MD + The TwitterUserAgent follows the timeline of a specified Twitter user. + + You must set up a Twitter app and provide it's `consumer_key`, `consumer_secret`, `oauth_token` and `oauth_token_secret`, (Also shown as "Access token" on the Twitter developer's site.) along with the `username` of the Twitter user to monitor. + + Set `expected_update_period_in_days` to the maximum amount of time that you'd expect to pass between Events being created by this Agent. + MD + + event_description <<-MD + Events are the raw JSON provided by the Twitter API. Should look something like: + + { + :created_at=>"Thu Apr 04 13:27:48 +0000 2013", + :id=>319803490421596160, + :id_str=>"319803490421596160", + :text=> + "In which @jeresig goes to an art gallery and is \"the JavaScript programmer\". http://t.co/gt3PT1d3G1", + :source=> + "Twitter for Mac", + :truncated=>false, + :in_reply_to_status_id=>nil, + :in_reply_to_status_id_str=>nil, + :in_reply_to_user_id=>nil, + :in_reply_to_user_id_str=>nil, + :in_reply_to_screen_name=>nil, + :user=> + {:id=>2341001, + :id_str=>"2341001", + :name=>"Albert Sun", + :screen_name=>"albertsun", + :location=>"New York, NY", + :description=> + "News apps developer at NYT, formerly WSJ. graduated Penn 2010, geek, journalist, data-viz, nlp, gis, digital economics =)", + :url=>"http://albertsun.info", + :entities=> + {:url=> + {:urls=> + [{:url=>"http://albertsun.info", + :expanded_url=>nil, + :indices=>[0, 21]}]}, + :description=>{:urls=>[]}}, + :protected=>false, + :followers_count=>1857, + :friends_count=>798, + :listed_count=>115, + :created_at=>"Mon Mar 26 19:22:05 +0000 2007", + :favourites_count=>9, + :utc_offset=>-18000, + :time_zone=>"Eastern Time (US & Canada)", + :geo_enabled=>false, + :verified=>false, + :statuses_count=>2572, + :lang=>"en", + :contributors_enabled=>false, + :is_translator=>false, + :profile_background_color=>"1B2A2B", + :profile_background_image_url=> + "http://a0.twimg.com/profile_background_images/2802438/twitterbg.jpg", + :profile_background_image_url_https=> + "https://si0.twimg.com/profile_background_images/2802438/twitterbg.jpg", + :profile_background_tile=>false, + :profile_image_url=> + "http://a0.twimg.com/profile_images/110500205/profile-square_normal.jpg", + :profile_image_url_https=> + "https://si0.twimg.com/profile_images/110500205/profile-square_normal.jpg", + :profile_link_color=>"0000FF", + :profile_sidebar_border_color=>"87BC44", + :profile_sidebar_fill_color=>"E0FF92", + :profile_text_color=>"000000", + :profile_use_background_image=>true, + :default_profile=>false, + :default_profile_image=>false, + :following=>false, + :follow_request_sent=>false, + :notifications=>false}, + :geo=>nil, + :coordinates=>nil, + :place=>nil, + :contributors=>nil, + :retweet_count=>0, + :favorite_count=>0, + :entities=> + {:hashtags=>[], + :urls=> + [{:url=>"http://t.co/gt3PT1d3G1", + :expanded_url=> + "http://www.nytimes.com/2013/04/04/fashion/art-and-techology-a-clash-of-cultures.html?pagewanted=all", + :display_url=>"nytimes.com/2013/04/04/fas", + :indices=>[77, 99]}], + :user_mentions=> + [{:screen_name=>"jeresig", + :name=>"John Resig", + :id=>752673, + :id_str=>"752673", + :indices=>[9, 17]}]}, + :favorited=>false, + :retweeted=>false, + :possibly_sensitive=>false, + :lang=>"en" + } + MD + + default_schedule "every_1h" + + def validate_options + unless options[:username].present? && options[:expected_update_period_in_days].present? && options[:consumer_key].present? && options[:consumer_secret].present? && options[:oauth_token].present? && options[:oauth_token_secret].present? + errors.add(:base, "expected_update_period_in_days, username, consumer_key, consumer_secret, oauth_token and oauth_token_secret are required") + end + end + + def working? + (event = event_created_within(options[:expected_update_period_in_days].to_i.days)) && event.payload.present? + end + + def default_options + { + :username => "tectonic", + :expected_update_period_in_days => "2", + :consumer_key => "---", + :consumer_secret => "---", + :oauth_token => "---", + :oauth_token_secret => "---" + } + end + + # def process_tweet(filter, status) + # if options[:generate] == "counts" + # # Avoid memory pollution + # me = Agent.find(id) + # me.memory[:filter_counts] ||= {} + # me.memory[:filter_counts][filter.to_sym] ||= 0 + # me.memory[:filter_counts][filter.to_sym] += 1 + # me.save! + # else + # create_event :payload => status.merge(:filter => filter.to_s) + # end + # end + + def check + Twitter.configure do |config| + config.consumer_key = options[:consumer_key] + config.consumer_secret = options[:consumer_secret] + config.oauth_token = options[:oauth_token] + config.oauth_token_secret = options[:oauth_token_secret] + end + + since_id = memory[:since_id] || nil + max_id = memory[:max_id] || nil + opts = {:count => 200, :include_rts => true, :exclude_replies => false, :include_entities => true, :contributor_details => true} + opts.merge! :since_id => since_id unless since_id.nil? + opts.merge! :max_id => max_id unless max_id.nil? + + tweets = Twitter.user_timeline(options[:username], opts) + + tweets.each do |tweet| + memory[:since_id] = tweet.id if !memory[:since_id] || (tweet.id > memory[:since_id]) + + create_event :payload => tweet.attrs + end + + save! + # if memory[:filter_counts] && memory[:filter_counts].length > 0 + # memory[:filter_counts].each do |filter, count| + # create_event :payload => { :filter => filter.to_s, :count => count, :time => Time.now.to_i } + # end + # memory[:filter_counts] = {} + # save! + # end + end + end +end \ No newline at end of file