From 487eb8320ccefda8c579d667094e4d398e906866 Mon Sep 17 00:00:00 2001 From: Tomas Barton Date: Mon, 10 Aug 2020 09:55:42 +0200 Subject: [PATCH 1/2] Remove default type as Elastic 7 deprecates document types See: https://www.elastic.co/guide/en/elasticsearch/reference/7.x/removal-of-types.html `_doc` is a dummy document type that will work as well with older Elastic versions --- elasticsearch/CMakeLists.txt | 2 +- elasticsearch/io_modules/encoders/elasticsearch/common.lua | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elasticsearch/CMakeLists.txt b/elasticsearch/CMakeLists.txt index b175db3cb..f8ab4096e 100644 --- a/elasticsearch/CMakeLists.txt +++ b/elasticsearch/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0) -project(elasticsearch VERSION 1.0.8 LANGUAGES C) +project(elasticsearch VERSION 1.0.9 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Elasticsearch encoders and outputs") set(CPACK_DEBIAN_PACKAGE_DEPENDS "${PACKAGE_PREFIX}-heka (>= 1.0), ${PACKAGE_PREFIX}-rjson (>= 1.0), ${PACKAGE_PREFIX}-cjson (>= 2.1), ${PACKAGE_PREFIX}-socket (>= 3.0)") string(REGEX REPLACE "[()]" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_DEBIAN_PACKAGE_DEPENDS}) diff --git a/elasticsearch/io_modules/encoders/elasticsearch/common.lua b/elasticsearch/io_modules/encoders/elasticsearch/common.lua index 815c26c61..895e0e6cd 100644 --- a/elasticsearch/io_modules/encoders/elasticsearch/common.lua +++ b/elasticsearch/io_modules/encoders/elasticsearch/common.lua @@ -19,7 +19,7 @@ encoders_elasticsearch_common = { -- String to use as the `_type` key's value in the generated JSON. -- Supports field interpolation as described below. - type_name = "message" -- optional, default shown + type_name = "_doc" -- optional, default shown -- String to use as the `_id` key's value in the generated JSON. -- Supports field interpolation as described below. @@ -126,7 +126,7 @@ function load_encoder_cfg() end if cfg.type_name == nil then - cfg.type_name = "message" + cfg.type_name = "_doc" else assert(type(cfg.type_name) == "string", "type_name must be nil or a string") end From 0387ec3c4d2c5468215e99aa6b4e816a18729420 Mon Sep 17 00:00:00 2001 From: Tomas Barton Date: Mon, 10 Aug 2020 21:52:41 +0200 Subject: [PATCH 2/2] Add es_version parameter, print error when incorrect type is being used --- .../io_modules/encoders/elasticsearch/common.lua | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/elasticsearch/io_modules/encoders/elasticsearch/common.lua b/elasticsearch/io_modules/encoders/elasticsearch/common.lua index 895e0e6cd..899f1977a 100644 --- a/elasticsearch/io_modules/encoders/elasticsearch/common.lua +++ b/elasticsearch/io_modules/encoders/elasticsearch/common.lua @@ -13,13 +13,17 @@ encoders_elasticsearch_common = { -- than the system time. es_index_from_timestamp = false -- optional, default shown + -- Elastic 7 and newer are moving to type-less documents (use `type_name = nil` or `_doc`) + -- https://www.elastic.co/guide/en/elasticsearch/reference/7.x/removal-of-types.html + es_version = 5 -- optional, default shown + -- String to use as the `_index` key's value in the generated JSON. -- Supports field interpolation as described below. index = "heka-%{%Y.%m.%d}" -- optional, default shown -- String to use as the `_type` key's value in the generated JSON. -- Supports field interpolation as described below. - type_name = "_doc" -- optional, default shown + type_name = nil -- optional, default shown -- String to use as the `_id` key's value in the generated JSON. -- Supports field interpolation as described below. @@ -126,7 +130,11 @@ function load_encoder_cfg() end if cfg.type_name == nil then - cfg.type_name = "_doc" + if cfg.es_version < 7 then + cfg.type_name = "message" + else + assert(cfg.type_name == "_doc", "type_name must be nil or _doc, types are deprecated since Elasic 7.0") + end else assert(type(cfg.type_name) == "string", "type_name must be nil or a string") end