Refactored index manipulation methods, added zero-downtime index rese…

…tting

.travis.yml

@@ -4,6 +4,6 @@ rvm:
   - 2.0.0
   # - rbx
 before_install:
-  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.9.tar.gz | tar xz -C /tmp
+  - curl -# https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.10.tar.gz | tar xz -C /tmp
 before_script:
-  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-0.90.9/bin/elasticsearch" rake elasticsearch:start
+  - TEST_CLUSTER_COMMAND="/tmp/elasticsearch-0.90.10/bin/elasticsearch" rake elasticsearch:start

CHANGELOG.md

@@ -1,5 +1,13 @@
 # master
 
+  * `chewy:reset` and `chewy:reset:all` rake tasks are now trying to reset index with zero downtime if it is possible.
+
+  * Added `chewy:update:all` rake task.
+
+  * Methods `.create`, `.create!`, `.delete`, `.delete`, `reset!` are now supports index name suffix passing as the first argument. See [actions.rb](lib/chewy/index/actions.rb) for more details.
+
+  * Method `reset` renamed to `reset!`.
+
   * Added common loading scope for AR adapter. Also removed scope proc argument, now it executes just in main load scope context.
 
     `CitiesIndex.all.load(scope: {city: City.include(:country)})`
@@ -8,9 +16,9 @@
 
 # Version 0.1.0
 
-  * Added filters simplified DSL. See [filters.rb](lib/chewy/query/filters.rb) for more info.
+  * Added filters simplified DSL. See [filters.rb](lib/chewy/query/filters.rb) for more details.
 
-  * Queries and filters join system reworked. See [query.rb](lib/chewy/query.rb) for more info.
+  * Queries and filters join system reworked. See [query.rb](lib/chewy/query.rb) for more details.
 
   * Added query `merge` method
 

README.md

@@ -162,9 +162,11 @@ UsersIndex::User.import User.where('rating > 100').to_a # or import specified us
 UsersIndex::User.import [1, 2, 42] # pass even ids for import, it will be handled in the most effective way
 
 UsersIndex.import # import every defined type
-UsersIndex.reset # purges index and imports default data for all types
+UsersIndex.reset! # purges index and imports default data for all types
 ```
 
+See [actions.rb](lib/chewy/index/actions.rb) for more details.
+
 Also if passed user is #destroyed? or specified id is not existing in the database, import will perform `delete` index for this it
 
 ### Observing strategies
@@ -253,7 +255,7 @@ UsersIndex::User.filter{ name == 'Fred' }.filter{ age < 42 }.filter_mode(:should
 UsersIndex::User.filter{ name == 'Fred' }.filter{ age < 42 }.filter_mode('75%') # will be wrapped with bool `should` filter with `minimum_should_match: '75%'`
 ```
 
-See [query.rb](lib/chewy/query.rb) for more info.
+See [query.rb](lib/chewy/query.rb) for more details.
 
 ### Filters query DSL.
 
@@ -534,7 +536,7 @@ Compliance cheatsheet for filters and DSL expressions:
   UsersIndex.filter{ has_parent(:blog).filter{ text == 'bonsai three' } }
   ```
 
-See [filters.rb](lib/chewy/query/filters.rb) for more info.
+See [filters.rb](lib/chewy/query/filters.rb) for more details.
 
 ### Objects loading
 

filters

@@ -62,6 +62,8 @@ nested
   name.nested()
 
 query q()
+type
+  .types()
 
 geo bounding box
 geo distance
@@ -71,7 +73,6 @@ geoshape
 geohash cell
 
 indices
-type
 ids
 
 limit

lib/chewy/index.rb

@@ -1,9 +1,11 @@
 require 'chewy/index/actions'
+require 'chewy/index/aliases'
 require 'chewy/index/search'
 
 module Chewy
   class Index
     include Actions
+    include Aliases
     include Search
 
     singleton_class.delegate :client, to: 'Chewy'
@@ -50,6 +52,10 @@ def self.index_name(suggest = nil)
       @index_name or raise UndefinedIndex
     end
 
+    def self.build_index_name options = {}
+      [index_name, options[:suffix]].reject(&:blank?).join(?_)
+    end
+
     def self.settings_hash
       _settings.present? ? {settings: _settings} : {}
     end
@@ -71,13 +77,8 @@ def self.search_type
       type_names
     end
 
-    def self.import
-      types.all? { |t| t.import }
-    end
-
-    def self.reset
-      purge!
-      import
+    def self.import options = {}
+      types.all? { |t| t.import options }
     end
   end
 end

lib/chewy/index/actions.rb

@@ -1,41 +1,143 @@
 module Chewy
   class Index
+    # Module provides per-index actions, such as deletion,
+    # creation and existance check.
+    #
     module Actions
       extend ActiveSupport::Concern
 
       module ClassMethods
+        # Checks index existance. Returns true or false
+        #
+        #   UsersIndex.exist? #=> true
+        #
         def exists?
           client.indices.exists(index: index_name)
         end
 
-        def create
-          create!
+        # Creates index and applies mappings and settings.
+        # Returns false in case of unsuccessful creation.
+        #
+        #   UsersIndex.create # creates index named `users`
+        #
+        # Index name suffix might be passed optionally. In this case,
+        # method creates index with suffix and makes unsuffixed alias
+        # for it.
+        #
+        #   UsersIndex.create '01-2013' # creates index `uses_01-2013` and alias `users` for it
+        #   UsersIndex.create '01-2013', alias: false # creates index `uses_01-2013` only and no alias
+        #
+        # Suffixed index names might be used for zero-downtime mapping change, for example.
+        # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
+        #
+        def create *args
+          create! *args
         rescue Elasticsearch::Transport::Transport::Errors::BadRequest
           false
         end
 
-        def create!
-          client.indices.create(index: index_name, body: index_params)
+        # Creates index and applies mappings and settings.
+        # Raises elasticsearch-ruby transport error in case of
+        # unsuccessfull creation.
+        #
+        #   UsersIndex.create! # creates index named `users`
+        #
+        # Index name suffix might be passed optionally. In this case,
+        # method creates index with suffix and makes unsuffixed alias
+        # for it.
+        #
+        #   UsersIndex.create! '01-2014' # creates index `users_01-2014` and alias `users` for it
+        #   UsersIndex.create! '01-2014', alias: false # creates index `users_01-2014` only and no alias
+        #
+        # Suffixed index names might be used for zero-downtime mapping change, for example.
+        # Description: (http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
+        #
+        def create! *args
+          options = args.extract_options!.reverse_merge!(alias: true)
+          name = build_index_name(suffix: args.first)
+          result = client.indices.create(index: name, body: index_params)
+          result &&= client.indices.put_alias(index: name, name: index_name) if options[:alias] && name != index_name
+          result
         end
 
-        def delete
-          delete!
+        # Deletes ES index. Returns false in case of error.
+        #
+        #   UsersIndex.delete # deletes `users` index
+        #
+        # Supports index suffix passed as the first argument
+        #
+        #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
+        #
+        def delete suffix = nil
+          delete! suffix
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           false
         end
 
-        def delete!
-          client.indices.delete(index: index_name)
+        # Deletes ES index. Raises elasticsearch-ruby transport error
+        # in case of error.
+        #
+        #   UsersIndex.delete # deletes `users` index
+        #
+        # Supports index suffix passed as the first argument
+        #
+        #   UsersIndex.delete '01-2014' # deletes `users_01-2014` index
+        #
+        def delete! suffix = nil
+          client.indices.delete index: build_index_name(suffix: suffix)
         end
 
-        def purge
-          delete
-          create
+        # Deletes and recreates index. Supports suffixes.
+        # Returns result of index creation.
+        #
+        #   UsersIndex.purge # deletes and creates `users` index
+        #   UsersIndex.purge '01-2014' # deletes `users` and `users_01-2014` indexes, creates `users_01-2014`
+        #
+        def purge suffix = nil
+          delete if suffix.present?
+          delete suffix
+          create suffix
         end
 
-        def purge!
-          delete
-          create!
+        # Deletes and recreates index. Supports suffixes.
+        # Returns result of index creation. Raises error in case
+        # of unsuccessfull creation
+        #
+        #   UsersIndex.purge! # deletes and creates `users` index
+        #   UsersIndex.purge! '01-2014' # deletes `users` and `users_01-2014` indexes, creates `users_01-2014`
+        #
+        def purge! suffix = nil
+          delete if suffix.present?
+          delete suffix
+          create! suffix
+        end
+
+        # Deletes, creates and imports data to the index.
+        # Returns import result
+        #
+        #   UsersIndex.reset!
+        #
+        # If index name suffix passed as the first argument - performs
+        # zero-downtime index resetting (described here:
+        # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/).
+        #
+        #   UsersIndex.reset! Time.now.to_i
+        #
+        def reset! suffix = nil
+          if suffix.present? && (indexes = self.indexes).any?
+            create suffix, alias: false
+            result = import suffix: suffix
+            client.indices.update_aliases body: {actions: [
+              *indexes.map do |index|
+                {remove: {index: index, alias: index_name}}
+              end,
+              {add: {index: build_index_name(suffix: suffix), alias: index_name}}
+            ]}
+            result
+          else
+            purge! suffix
+            import
+          end
         end
       end
     end

lib/chewy/index/aliases.rb

@@ -0,0 +1,21 @@
+module Chewy
+  class Index
+    module Aliases
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def indexes
+          client.indices.get_alias(name: index_name).keys
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          []
+        end
+
+        def aliases
+          client.indices.get_alias(index: index_name, name: '*')[index_name].try(:[], 'aliases').try(:keys) || []
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          []
+        end
+      end
+    end
+  end
+end

lib/chewy/type/import.rb

@@ -5,13 +5,13 @@ module Import
 
       module ClassMethods
         def bulk(options = {})
-          client.bulk options.merge(index: index.index_name, type: type_name)
+          suffix = options.delete(:suffix)
+          client.bulk options.merge(index: index.build_index_name(suffix: suffix), type: type_name)
         end
 
         def import(*args)
           import_options = args.extract_options!
-          bulk_options = import_options.extract!(:refresh).reverse_merge!(refresh: true)
-          identify = {_index: index.index_name, _type: type_name}
+          bulk_options = import_options.extract!(:refresh, :suffix).reverse_merge!(refresh: true)
 
           adapter.import(*args, import_options) do |action_objects|
             payload = {type: self}
@@ -20,9 +20,9 @@ def import(*args)
             ActiveSupport::Notifications.instrument 'import_objects.chewy', payload do
               body = action_objects.each.with_object([]) do |(action, objects), result|
                 result.concat(if action == :delete
-                  objects.map { |object| { action => identify.merge(_id: object.respond_to?(:id) ? object.id : object) } }
+                  objects.map { |object| { action => {_id: object.respond_to?(:id) ? object.id : object} } }
                 else
-                  objects.map { |object| { action => identify.merge(_id: object.id, data: object_data(object)) } }
+                  objects.map { |object| { action => {_id: object.id, data: object_data(object)} } }
                 end)
               end
               body.any? ? !!bulk(bulk_options.merge(body: body)) : true

lib/tasks/chewy.rake

@@ -1,4 +1,9 @@
 namespace :chewy do
+  desc 'Destroy, recreate and import data to specified index'
+  task :reset, [:index] => :environment do |task, args|
+    "#{args[:index].camelize}Index".constantize.reset! (Time.now.to_f * 1000).round
+  end
+
   namespace :reset do
     desc 'Destroy, recreate and import data for all found indexes'
     task all: :environment do
@@ -8,20 +13,30 @@ namespace :chewy do
 
       Chewy::Index.descendants.each do |index|
         puts "Resetting #{index}"
-        index.reset
+        index.reset! (Time.now.to_f * 1000).round
       end
     end
   end
 
-  desc 'Destroy, recreate and import data to specified index'
-  task :reset, [:index] => :environment do |task, args|
-    "#{args[:index].camelize}Index".constantize.reset
-  end
-
-  desc 'Updates specified index'
+  desc 'Updates data specified index'
   task :update, [:index] => :environment do |task, args|
     index = "#{args[:index].camelize}Index".constantize
-    raise "Index `#{index.index_name}` does not exists. Use rake chewy:reset[index] to create and update it." unless index.exists?
+    raise "Index `#{index.index_name}` does not exists. Use rake chewy:reset[#{index.index_name}] to create and update it." unless index.exists?
     index.import
   end
+
+  namespace :update do
+    desc 'Updates data for all found indexes'
+    task all: :environment do
+      Rails.application.config.paths['app/chewy'].existent.each do |dir|
+        Dir.glob(File.join(dir, '**/*.rb')).each { |file| require file }
+      end
+
+      Chewy::Index.descendants.each do |index|
+        puts "Updating #{index}"
+        puts "Index `#{index.index_name}` does not exists. Use rake chewy:reset[#{index.index_name}] to create and update it." unless index.exists?
+        index.import
+      end
+    end
+  end
 end