From abfbebb51931ce40d915d7ec11c9e26eebc9d16d Mon Sep 17 00:00:00 2001 From: alexesba Date: Mon, 12 Aug 2013 13:37:54 -0500 Subject: [PATCH 1/5] added sidekiq gem --- Gemfile | 8 +++++--- Gemfile.lock | 58 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/Gemfile b/Gemfile index a307998..3ecbca4 100644 --- a/Gemfile +++ b/Gemfile @@ -24,8 +24,6 @@ gem 'kaminari' # background jobs #======================================== -gem 'resque', :require => "resque/server" -gem 'resque-scheduler', :require => "resque_scheduler" gem 'rake' gem 'haml-rails' @@ -37,6 +35,11 @@ gem 'tire' gem 'crack' gem 'exception_notification', :require => 'exception_notifier' gem 'daemons' +gem 'sidekiq', require: 'sidekiq/web' +gem 'sinatra', '>= 1.3.0', :require => nil +gem 'sidekiq-status' +gem 'sidekiq-scheduler' +gem 'slim', '>= 1.1.0' gem 'daemons-rails' gem 'thin' gem 'newrelic_rpm' @@ -56,7 +59,6 @@ group :test do gem 'rspec-rails' gem 'fuubar' gem 'factory_girl_rails' - gem 'resque_spec' gem 'simplecov' gem 'spork-rails' end diff --git a/Gemfile.lock b/Gemfile.lock index 80f2904..ec61aa5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -46,6 +46,8 @@ GEM rack-test (>= 0.5.4) selenium-webdriver (~> 2.0) xpath (~> 0.1.4) + celluloid (0.14.1) + timers (>= 1.0.0) childprocess (0.3.1) ffi (~> 1.0.6) chunky_png (1.2.5) @@ -56,6 +58,7 @@ GEM sass (~> 3.1) configatron (2.9.0) yamler (>= 0.1.0) + connection_pool (1.1.0) crack (0.3.1) curb (0.7.18) daemons (1.1.8) @@ -168,7 +171,7 @@ GEM rack (1.4.1) rack-cache (1.1) rack (>= 0.4) - rack-protection (1.2.0) + rack-protection (1.5.0) rack rack-ssl (1.3.2) rack @@ -197,21 +200,9 @@ GEM ffi (>= 0.5.0) rdoc (3.12) json (~> 1.4) - redis (2.2.2) - redis-namespace (1.0.3) - redis (< 3.0.0) - resque (1.20.0) - multi_json (~> 1.0) - redis-namespace (~> 1.0.2) - sinatra (>= 0.9.2) - vegas (~> 0.1.2) - resque-scheduler (2.0.0) - redis (>= 2.0.1) - resque (>= 1.20.0) - rufus-scheduler - resque_spec (0.9.2) - resque (>= 1.19.0) - rspec (>= 2.5.0) + redis (3.0.4) + redis-namespace (1.3.1) + redis (~> 3.0.0) rest-client (1.6.7) mime-types (>= 1.16) rpx_now (0.6.24) @@ -233,7 +224,7 @@ GEM ruby-graphviz (1.0.5) ruby-progressbar (0.0.10) rubyzip (0.9.6.1) - rufus-scheduler (2.0.17) + rufus-scheduler (2.0.23) tzinfo (>= 0.3.23) sass (3.1.15) sass-rails (3.2.4) @@ -249,15 +240,30 @@ GEM ffi (~> 1.0) multi_json (~> 1.0) rubyzip + sidekiq (2.13.1) + celluloid (>= 0.14.1) + connection_pool (>= 1.0.0) + json + redis (>= 3.0) + redis-namespace + sidekiq-scheduler (0.4.1) + redis (>= 2.0.1) + rufus-scheduler (~> 2.0) + sidekiq (~> 2.0) + sidekiq-status (0.3.1) + sidekiq (~> 2.7) simple_oauth (0.1.8) simplecov (0.6.1) multi_json (~> 1.0) simplecov-html (~> 0.5.3) simplecov-html (0.5.3) - sinatra (1.3.2) - rack (~> 1.3, >= 1.3.6) - rack-protection (~> 1.2) + sinatra (1.3.6) + rack (~> 1.4) + rack-protection (~> 1.3) tilt (~> 1.3, >= 1.3.3) + slim (2.0.1) + temple (~> 0.6.6) + tilt (>= 1.3.3, < 2.1) slop (2.4.4) spork (1.0.0rc3) spork-rails (3.2.0) @@ -268,6 +274,7 @@ GEM rack (~> 1.0) tilt (~> 1.1, != 1.3.0) state_machine (1.1.2) + temple (0.6.6) thin (1.4.1) daemons (>= 1.0.9) eventmachine (>= 0.12.6) @@ -276,6 +283,7 @@ GEM tidy_ffi (0.1.4) ffi (>= 0.3.5) tilt (1.3.3) + timers (1.1.0) tire (0.3.12) activemodel (~> 3.0) multi_json (~> 1.0) @@ -293,8 +301,6 @@ GEM http_parser.rb (~> 0.5.1) simple_oauth (~> 0.1.4) tzinfo (0.3.31) - vegas (0.1.11) - rack (>= 1.0.0) warden (1.1.1) rack (>= 1.0) xpath (0.1.4) @@ -333,14 +339,16 @@ DEPENDENCIES pry rails (= 3.2.2) rake - resque - resque-scheduler - resque_spec rspec-rails ruby-graphviz sass-rails scrapi + sidekiq + sidekiq-scheduler + sidekiq-status simplecov + sinatra (>= 1.3.0) + slim (>= 1.1.0) spork-rails state_machine thin From a43f0c4338a9c3778dd68fcf95f84ff4fb69e9fe Mon Sep 17 00:00:00 2001 From: alexesba Date: Mon, 12 Aug 2013 13:38:31 -0500 Subject: [PATCH 2/5] use the module sidekiq for all the workers --- app/workers/calculate_ranking.rb | 6 +++--- app/workers/clean_indexes.rb | 6 +++--- app/workers/entry_content_fetcher.rb | 6 +++--- app/workers/entry_indexer.rb | 6 +++--- app/workers/entry_localizer.rb | 6 +++--- app/workers/entry_monitor.rb | 6 +++--- app/workers/entry_tagger.rb | 7 ++++--- app/workers/facebook_counter.rb | 7 ++++--- app/workers/feed_loader.rb | 6 +++--- app/workers/feed_updater.rb | 6 +++--- app/workers/reindex_feed.rb | 6 +++--- 11 files changed, 35 insertions(+), 33 deletions(-) diff --git a/app/workers/calculate_ranking.rb b/app/workers/calculate_ranking.rb index 0459924..96da6e7 100644 --- a/app/workers/calculate_ranking.rb +++ b/app/workers/calculate_ranking.rb @@ -1,8 +1,8 @@ class CalculateRanking - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :feed_entry + include Sidekiq::Worker + sidekiq_options :queue => :feed_entry, :retry => 1, :backtrace => true - def self.perform(entry_id) + def perform(entry_id) entry = FeedEntry.find(entry_id) entry.update_facebook_stats entry.calculate_social_rank diff --git a/app/workers/clean_indexes.rb b/app/workers/clean_indexes.rb index eadf81e..2ad4a11 100644 --- a/app/workers/clean_indexes.rb +++ b/app/workers/clean_indexes.rb @@ -1,8 +1,8 @@ class CleanIndexes - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :clean_index + include Sidekiq::Worker + sidekiq_options :queue => :clean_index, :retry => 1, :backtrace => true - def self.perform + def perform FeedEntry.remove_this_entries end diff --git a/app/workers/entry_content_fetcher.rb b/app/workers/entry_content_fetcher.rb index 30e8f1b..d70b171 100644 --- a/app/workers/entry_content_fetcher.rb +++ b/app/workers/entry_content_fetcher.rb @@ -1,8 +1,8 @@ class EntryContentFetcher - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :entries + include Sidekiq::Worker + sidekiq_options :queue => :entries, :retry => 1, :backtrace => true - def self.perform(entry_id) + def perform(entry_id) entry = FeedEntry.find_by_id(entry_id) entry.fetch end diff --git a/app/workers/entry_indexer.rb b/app/workers/entry_indexer.rb index 4e854e4..57e6b5a 100644 --- a/app/workers/entry_indexer.rb +++ b/app/workers/entry_indexer.rb @@ -1,8 +1,8 @@ class EntryIndexer - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :entries + include Sidekiq::Worker + sidekiq_options :queue => :entries, :retry => 1, :backtrace => true - def self.perform(entry_id) + def perform(entry_id) entry = FeedEntry.find(entry_id) entry.index_in_searchify end diff --git a/app/workers/entry_localizer.rb b/app/workers/entry_localizer.rb index 16b8cb8..0b489c7 100644 --- a/app/workers/entry_localizer.rb +++ b/app/workers/entry_localizer.rb @@ -1,8 +1,8 @@ class EntryLocalizer - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :entries + include Sidekiq::Worker + sidekiq_options :queue => :entries, :retry => 1, :backtrace => true - def self.perform(entry_id) + def perform(entry_id) entry = FeedEntry.find(entry_id) FeedEntry.localize(entry) end diff --git a/app/workers/entry_monitor.rb b/app/workers/entry_monitor.rb index d682b82..f3a168c 100644 --- a/app/workers/entry_monitor.rb +++ b/app/workers/entry_monitor.rb @@ -1,8 +1,8 @@ class EntryMonitor - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :reindex_entries + include Sidekiq::Worker + sidekiq_options :queue => :reindex_entries, :retry => 1, :backtrace => true - def self.perform + def perform NewsFeed.find_each do |feed| feed.bg_reindex_feed end diff --git a/app/workers/entry_tagger.rb b/app/workers/entry_tagger.rb index bd1af47..62861fe 100644 --- a/app/workers/entry_tagger.rb +++ b/app/workers/entry_tagger.rb @@ -1,8 +1,9 @@ class EntryTagger - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :entries + include Sidekiq::Worker + sidekiq_options :queue => :entries, :retry => 1, :backtrace => true - def self.perform(entry_id) + + def perform(entry_id) entry = FeedEntry.find(entry_id) FeedEntry.tag(entry) end diff --git a/app/workers/facebook_counter.rb b/app/workers/facebook_counter.rb index e88997d..017bb8f 100644 --- a/app/workers/facebook_counter.rb +++ b/app/workers/facebook_counter.rb @@ -1,7 +1,8 @@ class FacebookCounter - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :feed_entry - def self.perform + include Sidekiq::Worker + sidekiq_options :queue => :feed_entry, :retry => 1, :backtrace => true + + def perform FeedEntry.to_recalculate.each do |entry| entry.bg_calculate_social_rank end diff --git a/app/workers/feed_loader.rb b/app/workers/feed_loader.rb index dfaeba8..07d1884 100644 --- a/app/workers/feed_loader.rb +++ b/app/workers/feed_loader.rb @@ -1,8 +1,8 @@ class FeedLoader - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :feeds + include Sidekiq::Worker + sidekiq_options :queue => :feeds, :retry => 1, :backtrace => true - def self.perform(news_feed_id) + def perform(news_feed_id) news_feed = NewsFeed.find(news_feed_id) news_feed.load_entries end diff --git a/app/workers/feed_updater.rb b/app/workers/feed_updater.rb index 8c20208..0aacae8 100644 --- a/app/workers/feed_updater.rb +++ b/app/workers/feed_updater.rb @@ -1,8 +1,8 @@ class FeedUpdater - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :feeds + include Sidekiq::Worker + sidekiq_options :queue => :feeds, :retry => 1, :backtrace => true - def self.perform + def perform NewsFeed.find_each do |feed| feed.update_entries end diff --git a/app/workers/reindex_feed.rb b/app/workers/reindex_feed.rb index 0ac93c5..d2031c8 100644 --- a/app/workers/reindex_feed.rb +++ b/app/workers/reindex_feed.rb @@ -1,8 +1,8 @@ class ReindexFeed - include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation - @queue = :reindex_entries + include Sidekiq::Worker + sidekiq_options :queue => :reindex_entries, :retry => 1, :backtrace => true - def self.perform(feed_id) + def perform(feed_id) NewsFeed.find(feed_id).reindex_feed end end From 7334379cd40572e8f46898040d9c538df16a9e1d Mon Sep 17 00:00:00 2001 From: alexesba Date: Mon, 12 Aug 2013 13:39:38 -0500 Subject: [PATCH 3/5] removed resque server and mounted sidekiq --- config/routes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/routes.rb b/config/routes.rb index 971a404..5729b8b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -9,7 +9,7 @@ namespace :admin do - mount SecureResqueServer.new, :at => "/resque" + mount Sidekiq::Web => '/sidekiq' resources :news_feeds do get :process_entries, :on => :member From 62a53410319d03d583760b43c29203b642fb0b3d Mon Sep 17 00:00:00 2001 From: alexesba Date: Mon, 12 Aug 2013 13:40:08 -0500 Subject: [PATCH 4/5] removed old version rvm config file and deleted reque config --- .rvmrc | 1 - config/initializers/resque.rb | 11 ----------- config/resque_scheduler.yml | 9 --------- 3 files changed, 21 deletions(-) delete mode 100644 .rvmrc delete mode 100644 config/initializers/resque.rb delete mode 100644 config/resque_scheduler.yml diff --git a/.rvmrc b/.rvmrc deleted file mode 100644 index 9765ede..0000000 --- a/.rvmrc +++ /dev/null @@ -1 +0,0 @@ -rvm use ruby-1.9.2-head@dimensions --create diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index 90e7c33..0000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'resque' -require 'resque_scheduler' -require 'resque_scheduler/server' - -rails_root = Rails.root || File.dirname(__FILE__) + '/../..' -rails_env = Rails.env || 'development' - -resque_config = YAML.load_file(rails_root.to_s + '/config/resque.yml') -Resque.redis = resque_config[rails_env] - -Resque.schedule = YAML.load_file(File.join(Rails.root, 'config/resque_scheduler.yml')) diff --git a/config/resque_scheduler.yml b/config/resque_scheduler.yml deleted file mode 100644 index 019cbb7..0000000 --- a/config/resque_scheduler.yml +++ /dev/null @@ -1,9 +0,0 @@ -CountFacebookLikes: - cron: "0 */1 * * *" - class: FacebookCounter - description: "Updates the facebook likes count" - -UpdateFeeds: - cron: "45 */1 * * *" - class: FeedUpdater - description: "Update the feeds from the sources" From 225eae98c3c824356932bb5d5b8f766b60aea5f7 Mon Sep 17 00:00:00 2001 From: alexesba Date: Mon, 12 Aug 2013 13:40:54 -0500 Subject: [PATCH 5/5] added new ruby-verion files and config files for sidekiq, include the scheduler jobs --- .ruby-gemset | 1 + .ruby-version | 1 + config/initializers/sidekiq.rb | 26 ++++++++++++++++++++++++++ config/sidekiq.yml | 8 ++++++++ config/sidekiq_scheduler.yml | 9 +++++++++ 5 files changed, 45 insertions(+) create mode 100644 .ruby-gemset create mode 100644 .ruby-version create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/sidekiq.yml create mode 100644 config/sidekiq_scheduler.yml diff --git a/.ruby-gemset b/.ruby-gemset new file mode 100644 index 0000000..3329d5e --- /dev/null +++ b/.ruby-gemset @@ -0,0 +1 @@ +dimensions diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..12011a7 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +ruby-1.9.3-p392-perf diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..d58ba1a --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,26 @@ +require 'sidekiq' +require 'sidekiq-scheduler/client' +require 'sidekiq-scheduler/schedule' +require 'sidekiq-scheduler/worker' +require 'sidekiq-scheduler/version' +require 'sidekiq/scheduler' + +rails_root = Rails.root || File.dirname(__FILE__) + '/../..' +rails_env = Rails.env || 'development' +resque_config = YAML.load_file(rails_root.to_s + '/config/resque.yml') + +Sidekiq.configure_server do |config| + config.redis = { :url => "redis://#{resque_config['development']}", :namespace => 'dimensions' } + config.server_middleware do |chain| + chain.add Sidekiq::Status::ServerMiddleware, expiration: 30.minutes # default + end +end + +Sidekiq.configure_client do |config| + config.redis = { :url => "redis://#{resque_config['development']}", :namespace => 'dimensions' } + config.client_middleware do |chain| + chain.add Sidekiq::Status::ClientMiddleware + end +end + +Sidekiq.schedule = YAML.load_file(rails_root.to_s + '/config/sidekiq_scheduler.yml') diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..c98e867 --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,8 @@ +--- +:verbose: false +:pidfile: ./tmp/pids/sidekiq.pid +:logfile: ./log/sidekiq.log +:concurrency: 20 +:queues: + - feed_entry + - feeds diff --git a/config/sidekiq_scheduler.yml b/config/sidekiq_scheduler.yml new file mode 100644 index 0000000..019cbb7 --- /dev/null +++ b/config/sidekiq_scheduler.yml @@ -0,0 +1,9 @@ +CountFacebookLikes: + cron: "0 */1 * * *" + class: FacebookCounter + description: "Updates the facebook likes count" + +UpdateFeeds: + cron: "45 */1 * * *" + class: FeedUpdater + description: "Update the feeds from the sources"