Merge pull request #143 from marcoscarceles/elasticsearch-2.1.2

Elasticsearch 2.1.2

Author: puneetbehl

.gitignore

@@ -8,4 +8,7 @@ web-app
 *.log
 *.zip
 plugin.xml
-/docs
+/docs
+
+# Elasticsearch plugins installation
+test/resources/plugins

ElasticsearchGrailsPlugin.groovy

@@ -36,7 +36,7 @@ class ElasticsearchGrailsPlugin {
 
     private static final Logger LOG = LoggerFactory.getLogger(this)
 
-    def version = '0.0.4.x-SNAPSHOT'
+    def version = '0.1.0'
     def grailsVersion = '2.2.0 > *'
 
     def loadAfter = ['services', 'mongodb']

README.asciidoc

@@ -16,7 +16,7 @@ Edit your project's +BuildConfig.groovy+ file, by adding the plugin's dependency
 ----
 grails.project.dependency.resolution = {
     plugins {
-        runtime ':elasticsearch:0.0.4.6'
+        runtime ':elasticsearch:0.1.0'
     }
 }
 ----

application.properties

@@ -1,4 +1,4 @@
 #Grails Metadata file
-#Tue Apr 21 20:14:27 CST 2015
-app.grails.version=2.4.4
+#Wed Feb 17 12:24:47 CET 2016
+app.grails.version=2.4.5
 app.name=elasticsearch

grails-app/conf/BuildConfig.groovy

@@ -40,15 +40,25 @@ grails.project.dependency.resolution = {
                 "org.grails:grails-datastore-core:$datastoreVersion",
                 "org.grails:grails-datastore-web:$datastoreVersion", excludes)
 
-        runtime 'org.elasticsearch:elasticsearch-groovy:1.6.0', {
+        runtime 'org.elasticsearch:elasticsearch-groovy:2.1.2', {
             excludes 'groovy-all'
         }
-        runtime "org.elasticsearch:elasticsearch-mapper-attachments:2.6.0"
+        runtime "org.elasticsearch:elasticsearch-mapper-attachments:3.1.2"
         runtime 'com.spatial4j:spatial4j:0.4.1'
 
         compile 'com.vividsolutions:jts:1.13'
 
         test 'com.googlecode.json-simple:json-simple:1.1.1'
+
+        //BUG: Elasticsearch 2.1.2 does not explictly sets a version of jackson libraries (but needs >2.6), so grails will pick up the latest from any other plugin/dependency
+        def jacksonVersion = '2.6.2'
+        compile "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+        compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jacksonVersion}"
+        compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jacksonVersion}"
+        compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jacksonVersion}"
+
+        def nettyVersion = '3.10.5.Final'
+        compile "io.netty:netty:${nettyVersion}"
     }
 
     plugins {

grails-app/conf/Config.groovy

@@ -51,7 +51,7 @@ environments {
             datastoreImpl = 'hibernateDatastore'
 
             index {
-                store.type = 'memory'
+                store.type = 'simplefs'
                 analysis {
                     filter {
                         replace_synonyms {
@@ -78,6 +78,8 @@ environments {
                     }
                 }
             }
+
+            path.plugins = 'test/resources/plugins'
         }
     }
 

grails-app/conf/DefaultElasticSearch.groovy

@@ -111,7 +111,7 @@ environments {
     test {
         elasticSearch {
             client.mode = 'local'
-            index.store.type = 'memory' // store local node in memory and not on disk
+            index.store.type = 'simplefs' // store local node in memory and not on disk
         }
     }
     production {

grails-app/domain/test/upperCase/UpperCase.groovy

@@ -1,7 +1,7 @@
 package test.upperCase
 
 class UpperCase {
-    def name;
+    String name;
     static mapping = {
 
         autoImport(false)

grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchAdminService.groovy

@@ -2,12 +2,12 @@ package org.grails.plugins.elasticsearch
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest
-import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest
-import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse
+import org.elasticsearch.action.support.broadcast.BroadcastResponse
 import org.elasticsearch.client.Client
 import org.elasticsearch.client.Requests
 import org.grails.plugins.elasticsearch.mapping.SearchableClassMapping
@@ -43,7 +43,7 @@ class ElasticSearchAdminService {
 
         // Refresh ES
         elasticSearchHelper.withElasticSearch { Client client ->
-            BroadcastOperationResponse response
+            BroadcastResponse response
             if (!indices) {
                 response = client.admin().indices().refresh(Requests.refreshRequest()).actionGet()
             } else {
@@ -134,21 +134,6 @@ class ElasticSearchAdminService {
         }
     }
 
-    /**
-     * Deletes a mapping on an index
-     * @param index The index the mapping will be deleted on
-     * @param type The type which mapping will be deleted
-     */
-    void deleteMapping(String index, String type) {
-        log.info("Deleting Elasticsearch mapping for ${index} and type ${type} ...")
-        elasticSearchHelper.withElasticSearch { Client client ->
-            client.admin().indices().deleteMapping(
-                    new DeleteMappingRequest(index).
-                            types(type)
-            ).actionGet()
-        }
-    }
-
     /**
      * Creates mappings on a type
      * @param index The index where the mapping is being created
@@ -167,7 +152,7 @@ class ElasticSearchAdminService {
     }
 
     /**
-     * Check whether a mpping exists
+     * Check whether a mapping exists
      * @param index The name of the index to check on
      * @param type The type which mapping is being checked
      * @return true if the mapping exists
@@ -252,7 +237,7 @@ class ElasticSearchAdminService {
     String indexPointedBy(String alias) {
         elasticSearchHelper.withElasticSearch { Client client ->
             def index = client.admin().indices().getAliases(new GetAliasesRequest().aliases([alias] as String[])).actionGet().getAliases()?.find {
-                it.value.element.alias() == alias
+                alias in it.value*.alias()
             }?.key
             if (!index) {
                 LOG.debug("Alias ${alias} does not exist")
@@ -362,31 +347,12 @@ class ElasticSearchAdminService {
     }
 
     /**
-     * Waits for an index to be on Yellow status
-     * @param index
-     * @return
-     */
-    def waitForIndex(index) {
-        elasticSearchHelper.withElasticSearch { Client client ->
-            try {
-                LOG.debug("Waiting at least yellow status on ${index}")
-                client.admin().cluster().prepareHealth(index)
-                        .setWaitForYellowStatus()
-                        .execute().actionGet()
-            } catch (Exception e) {
-                // ignore any exceptions due to non-existing index.
-                LOG.debug('Index health', e)
-            }
-        }
-    }
-
-    /**
-     * Waits for the cluster to be on Yellow status
+     * Waits for the cluster to be on (Yellow) status
      */
-    void waitForClusterYellowStatus() {
+    void waitForClusterStatus(ClusterHealthStatus status=ClusterHealthStatus.YELLOW) {
         elasticSearchHelper.withElasticSearch { Client client ->
             ClusterHealthResponse response = client.admin().cluster().health(
-                    new ClusterHealthRequest([] as String[]).waitForYellowStatus()).actionGet()
+                    new ClusterHealthRequest([] as String[]).waitForStatus(status)).actionGet()
             LOG.debug("Cluster status: ${response.status}")
         }
     }

grails-app/services/org/grails/plugins/elasticsearch/ElasticSearchService.groovy

@@ -15,6 +15,7 @@
  */
 package org.grails.plugins.elasticsearch
 
+import groovy.transform.CompileStatic
 import org.codehaus.groovy.grails.commons.GrailsApplication
 import org.codehaus.groovy.grails.plugins.support.aware.GrailsApplicationAware
 import org.elasticsearch.action.count.CountRequest
@@ -23,8 +24,8 @@ import org.elasticsearch.action.search.SearchType
 import org.elasticsearch.action.support.QuerySourceBuilder
 import org.elasticsearch.client.Client
 import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.index.query.QueryStringQueryBuilder
-import org.elasticsearch.index.query.FilterBuilder
 import org.elasticsearch.search.SearchHit
 import org.elasticsearch.search.builder.SearchSourceBuilder
 import org.elasticsearch.search.highlight.HighlightBuilder
@@ -35,7 +36,7 @@ import org.grails.plugins.elasticsearch.util.GXContentBuilder
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
-import static org.elasticsearch.index.query.QueryBuilders.queryString
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery
 import static org.elasticsearch.index.query.QueryStringQueryBuilder.Operator
 
 class ElasticSearchService implements GrailsApplicationAware {
@@ -336,7 +337,7 @@ class ElasticSearchService implements GrailsApplicationAware {
             request.source(new GXContentBuilder().buildAsBytes(query))
         } else {
             Operator defaultOperator = params['default_operator'] ?: Operator.AND
-            QueryStringQueryBuilder builder = queryString(query).defaultOperator(defaultOperator)
+            QueryStringQueryBuilder builder = queryStringQuery(query).defaultOperator(defaultOperator)
             if (params.analyzer) {
                 builder.analyzer(params.analyzer)
             }
@@ -351,7 +352,7 @@ class ElasticSearchService implements GrailsApplicationAware {
      *
      * @param params The query parameters
      * @param query The search query, whether a String, a Closure or a QueryBuilder
-     * @param filter The search filter, whether a Closure or a FilterBuilder
+     * @param filter The search filter, whether a Closure or a QueryBuilder
      * @return The SearchRequest instance
      */
     private SearchRequest buildSearchRequest(query, filter, Map params) {
@@ -404,7 +405,7 @@ class ElasticSearchService implements GrailsApplicationAware {
 
     SearchSourceBuilder setQueryInSource(SearchSourceBuilder source, String query, Map params = [:]) {
         Operator defaultOperator = params['default_operator'] ?: Operator.AND
-        QueryStringQueryBuilder builder = queryString(query).defaultOperator(defaultOperator)
+        QueryStringQueryBuilder builder = queryStringQuery(query).defaultOperator(defaultOperator)
         if (params.analyzer) {
             builder.analyzer(params.analyzer)
         }
@@ -423,7 +424,7 @@ class ElasticSearchService implements GrailsApplicationAware {
 		source.postFilter(new GXContentBuilder().buildAsBytes(filter))
 	}
 
-	SearchSourceBuilder setFilterInSource(SearchSourceBuilder source, FilterBuilder filter, Map params = [:]){
+	SearchSourceBuilder setFilterInSource(SearchSourceBuilder source, QueryBuilder filter, Map params = [:]){
 		source.postFilter(filter)
 	}
 

scripts/_Events.groovy

@@ -1,5 +1,54 @@
 import org.apache.commons.io.FileUtils
 
+eventAllTestsStart = { msg ->
+    File pluginsDir = new File('test/resources/plugins')
+
+    Map<String, String> elasticsearchPlugins = [
+            'elasticsearch-mapper-attachments' : '3.1.2'
+    ]
+
+    if (!pluginsDir.exists()) {
+
+        println "Installing Elasticsearch Plugins ..."
+
+        pluginsDir.mkdirs()
+
+        elasticsearchPlugins.each { pluginName, pluginVersion ->
+
+            String versionedPluginName = "${pluginName}-${pluginVersion}"
+            String artifactURL = "http://search.maven.org/remotecontent?filepath=org/elasticsearch/${pluginName}/${pluginVersion}/${versionedPluginName}.zip"
+
+            //Download handling redirects
+            File tmpZip = new File(pluginsDir, "${versionedPluginName}.zip")
+            while( artifactURL ) {
+                artifactURL.toURL().openConnection().with { conn ->
+                    conn.instanceFollowRedirects = false
+                    artifactURL = conn.getHeaderField( "Location" )
+                    if( !artifactURL ) {
+                        tmpZip.withOutputStream { out ->
+                            conn.inputStream.with { inp ->
+                                out << inp
+                                inp.close()
+                            }
+                        }
+                    }
+                }
+            }
+
+            //Explode the plugin
+            File explodedPlugin = new File(pluginsDir, versionedPluginName)
+            explodedPlugin.mkdirs()
+            new AntBuilder().unzip(src: tmpZip.path, dest: explodedPlugin.path, overwrite: false)
+
+            //Delete downloaded file
+            tmpZip.delete()
+            println "Elasticsearch Plugin : ${versionedPluginName} INSTALLED!"
+        }
+
+
+    }
+}
+
 eventTestCaseStart = { name ->
     println '-' * 60
     println "| $name : started"

src/docs/guide/introduction/history.gdoc

@@ -1,5 +1,12 @@
 h4. History
 
+* February 24, 2016
+** 0.1.0
+*** Upgrade to ElasticSearch 2.1.2 - This is a breaking upgrade, please refer to:
+**** [Breaking changes in 2.1|https://www.elastic.co/guide/en/elasticsearch/reference/2.1/breaking-changes-2.1.html]
+**** [Breaking changes in 2.0|https://www.elastic.co/guide/en/elasticsearch/reference/2.1/breaking-changes-2.0.html]
+**** New @deleteIndex@ migration strategy (although @alias@ is still the recommended one)
+**** The @delete@ migration strategy has been deprecated and it not behaves as the @none@ migration strategy
 * June 30, 2015
 ** 0.0.4.5
 *** Upgrade to ElasticSearch 1.6.0

src/docs/guide/mapping/mappingMigrations.gdoc

@@ -6,7 +6,8 @@ h3. Migration Strategies
 The migration strategy is defined by the @elasticSearch.migration.strategy@ configuration property and it accepts three values:
 
 - @'none'@
-- @'delete'@
+- @'deleteIndex'@
+- --@'delete'@-- _is no longer supported since the upgrade to Elasticsearch > 2.0_
 - @'alias'@
 
 The default strategy is @'alias'@ as it is the only strategy that can achieve zero-downtime migrations and thus [recommended by Elasticsearch|http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/]
@@ -19,11 +20,11 @@ This option keeps the original behaviour the plugin used before the Migration St
 It will be responsability for the application administrator to manually fix the problem.
 This configuration was left as a backwards compatibility and it will prevent the application from booting successfully, therefore we *discourage teams from using this option*.
 
-h4. Migration Strategy 'delete'
+h4. Migration Strategy 'deleteIndex'
 
-When choosing this option, when a conflict occurs installing  mapping, the application will delete the existing mapping for the type, alongside with all content indexed on that index and type and recreated the mapping. There are a couple of important details on this information:
-- Only documents indexed on the conflicting mapping will be deleted, any other document on a different mapping on the same (or other) index will remain untouched.
+When choosing this option, when a conflict occurs installing  mapping, the application will delete the existing index for the type, *alongside with all content indexed on that index (including content from other mappings)* and it will recreate the index and all its mappings. There are a couple of important details on this information:
 - Deleted documents can be automatically reindexed on startup by using the @elasticSearch.bulkIndexOnStartup@ configuration property (See below)
+- New indices will be created with a version number and the rightaliases, to make them compatible with potential future 'alias' migrations (without requiring additional index deletions)
 - Using this configuration there will always be a time window (between deletion and reindexation) where documents can't be found by search, therefore this option cannot achieve a *zero-downtime* deployment
 
 See [Dealing with deleted content] below for more details on automatic indexing.