Update elasticsearch

.gitignore

@@ -16,3 +16,4 @@
 .noseids
 Dockerfile
 /env_api
+/venv

c2corg_api/scripts/es/fill_index.py

@@ -52,7 +52,7 @@ def main(argv=sys.argv):
 
 def fill_index(session, batch_size=1000):
     client = elasticsearch_config['client']
-    index_name = elasticsearch_config['index']
+    index_prefix = elasticsearch_config['index_prefix']
 
     status = {
         'start_time': datetime.now(),
@@ -80,7 +80,7 @@ def progress(count, total_count):
 
             for doc in sync.get_documents(session, doc_type, batch_size,
                                           ignore_redirects=True):
-                batch.add(to_search_document(doc, index_name))
+                batch.add(to_search_document(doc, index_prefix))
 
                 count += 1
                 progress(count, total)

c2corg_api/scripts/es/sync.py

@@ -279,14 +279,14 @@ def sync_documents(session, changed_documents, batch_size):
 def sync_deleted_documents(session, deleted_documents, batch_size):
     client = elasticsearch_config['client']
     batch = ElasticBatch(client, batch_size)
-    index = elasticsearch_config['index']
+    index_prefix = elasticsearch_config['index_prefix']
     n = 0
     with batch:
         for document_id, doc_type in deleted_documents:
             batch.add({
-                '_index': index,
+                '_index': f"{index_prefix}_{doc_type}",
                 '_id': document_id,
-                '_type': doc_type,
+                # '_type': doc_type,
                 'id': document_id,
                 '_op_type': 'delete'
             })
@@ -363,10 +363,10 @@ def get_documents(session, doc_type, batch_size, document_ids=None,
 
 def create_search_documents(doc_type, documents, batch):
     to_search_document = search_documents[doc_type].to_search_document
-    index = elasticsearch_config['index']
+    index_prefix = elasticsearch_config['index_prefix']
     n = 0
     for doc in documents:
-        batch.add(to_search_document(doc, index))
+        batch.add(to_search_document(doc, index_prefix))
         n += 1
     log.info('Sent {} document(s) of type {}'.format(n, doc_type))
 

c2corg_api/scripts/initializees.py

@@ -1,16 +1,16 @@
 import os
 import sys
 
-from c2corg_api.search.mappings.area_mapping import SearchArea
-from c2corg_api.search.mappings.article_mapping import SearchArticle
-from c2corg_api.search.mappings.book_mapping import SearchBook
-from c2corg_api.search.mappings.image_mapping import SearchImage
-from c2corg_api.search.mappings.outing_mapping import SearchOuting
-from c2corg_api.search.mappings.xreport_mapping import SearchXreport
-from c2corg_api.search.mappings.route_mapping import SearchRoute
-from c2corg_api.search.mappings.topo_map_mapping import SearchTopoMap
-from c2corg_api.search.mappings.user_mapping import SearchUser
-from c2corg_api.search.mappings.waypoint_mapping import SearchWaypoint
+from c2corg_api.search.mappings.area_mapping import SearchArea, AREA_TYPE
+from c2corg_api.search.mappings.article_mapping import SearchArticle, ARTICLE_TYPE
+from c2corg_api.search.mappings.book_mapping import SearchBook, BOOK_TYPE
+from c2corg_api.search.mappings.image_mapping import SearchImage, IMAGE_TYPE
+from c2corg_api.search.mappings.outing_mapping import SearchOuting, OUTING_TYPE
+from c2corg_api.search.mappings.xreport_mapping import SearchXreport, XREPORT_TYPE
+from c2corg_api.search.mappings.route_mapping import SearchRoute, ROUTE_TYPE
+from c2corg_api.search.mappings.topo_map_mapping import SearchTopoMap, MAP_TYPE
+from c2corg_api.search.mappings.user_mapping import SearchUser, USERPROFILE_TYPE
+from c2corg_api.search.mappings.waypoint_mapping import SearchWaypoint, WAYPOINT_TYPE
 from elasticsearch_dsl import Index
 
 from pyramid.paster import (
@@ -20,9 +20,23 @@
 
 from pyramid.scripts.common import parse_vars
 
-from c2corg_api.search.mapping import analysis_settings
+from c2corg_api.search.mapping import es_index_settings
 from c2corg_api.search import configure_es_from_config, elasticsearch_config
 
+# TODO : use from c2corg_api.search import search_documents
+
+_types = [
+    (SearchArea, AREA_TYPE),
+    (SearchArticle, ARTICLE_TYPE),
+    (SearchBook, BOOK_TYPE),
+    (SearchImage, IMAGE_TYPE),
+    (SearchOuting, OUTING_TYPE),
+    (SearchXreport, XREPORT_TYPE),
+    (SearchRoute, ROUTE_TYPE),
+    (SearchTopoMap, MAP_TYPE),
+    (SearchUser, USERPROFILE_TYPE),
+    (SearchWaypoint, WAYPOINT_TYPE),
+]
 
 def usage(argv):
     cmd = os.path.basename(argv[0])
@@ -46,45 +60,60 @@ def setup_es():
     """Create the ElasticSearch index and configure the mapping.
     """
     client = elasticsearch_config['client']
-    index_name = elasticsearch_config['index']
+    index_prefix = elasticsearch_config['index_prefix']
 
     info = client.info()
     print('ElasticSearch version: {0}'.format(info['version']['number']))
 
-    if client.indices.exists(index_name):
-        print('Index "{0}" already exists. To re-create the index, manually '
-              'delete the index and run this script again.'.format(index_name))
-        print('To delete the index run:')
-        print('curl -XDELETE \'http://{0}:{1}/{2}/\''.format(
-            elasticsearch_config['host'], elasticsearch_config['port'],
-            index_name))
-        sys.exit(0)
+    for klass, letter in _types:
+        index_name = f"{index_prefix}_{letter}"
 
-    index = Index(index_name)
-    index.settings(analysis=analysis_settings)
+        if client.indices.exists(index_name):
+            print('Index "{0}" already exists. To re-create the index, manually '
+                'delete the index and run this script again.'.format(index_name))
+            print('To delete the index run:')
+            print('curl -XDELETE \'http://{0}:{1}/{2}/\''.format(
+                elasticsearch_config['host'], elasticsearch_config['port'],
+                index_name))
+            sys.exit(0)
 
-    index.doc_type(SearchArea)
-    index.doc_type(SearchBook)
-    index.doc_type(SearchImage)
-    index.doc_type(SearchOuting)
-    index.doc_type(SearchXreport)
-    index.doc_type(SearchRoute)
-    index.doc_type(SearchTopoMap)
-    index.doc_type(SearchUser)
-    index.doc_type(SearchWaypoint)
-    index.doc_type(SearchArticle)
+        index = Index(index_name)
+        index.settings(**es_index_settings)
 
-    index.create()
+        index.document(klass)
+        index.create()
+        print('Index "{0}" created'.format(index_name))
 
-    print('Index "{0}" created'.format(index_name))
+    # index = Index(index_name)
+    # index.settings(**es_index_settings)
+
+    # index.document(SearchArea)
+    # index.document(SearchBook)
+    # index.document(SearchImage)
+    # index.document(SearchOuting)
+    # index.document(SearchXreport)
+    # index.document(SearchRoute)
+    # index.document(SearchTopoMap)
+    # index.document(SearchUser)
+    # index.document(SearchWaypoint)
+    # index.document(SearchArticle)
+
+    # index.create()
+
+    # print('Index "{0}" created'.format(index_name))
 
 
 def drop_index(silent=True):
     """Remove the ElasticSearch index.
     """
-    index = Index(elasticsearch_config['index'])
-    try:
-        index.delete()
-    except Exception as exc:
-        if not silent:
-            raise exc
+
+    index_prefix = elasticsearch_config['index_prefix']
+
+    for _, letter in _types:
+        index = Index(f"{index_prefix}_{letter}")
+
+        try:
+            index.delete()
+        except Exception as exc:
+            if not silent:
+                raise exc

c2corg_api/search/__init__.py

@@ -35,7 +35,7 @@
 
 elasticsearch_config = {
     'client': None,
-    'index': None,
+    'index_prefix': None,
     'host': None,
     'port': None
 }
@@ -53,7 +53,7 @@ def configure_es_from_config(settings):
     client = client_from_config(settings)
     connections.add_connection('default', client)
     elasticsearch_config['client'] = client
-    elasticsearch_config['index'] = settings['elasticsearch.index']
+    elasticsearch_config['index_prefix'] = settings['elasticsearch.index']
     elasticsearch_config['host'] = settings['elasticsearch.host']
     elasticsearch_config['port'] = int(settings['elasticsearch.port'])
 
@@ -77,8 +77,7 @@ def __init__(self, settings):
 def create_search(document_type):
     return Search(
         using=elasticsearch_config['client'],
-        index=elasticsearch_config['index'],
-        doc_type=search_documents[document_type])
+        index=elasticsearch_config['index_prefix'] + "_" + document_type)
 
 
 def get_text_query(search_term, lang=None):

c2corg_api/search/advanced_search.py

@@ -22,7 +22,7 @@ def search(url_params, meta_params, doc_type):
     # only request the document ids from ES
     response = query.execute()
     document_ids = [int(doc.meta.id) for doc in response]
-    total = response.hits.total
+    total = response.hits.total.value
 
     return document_ids, total